From a791e47a5433db93018c218b7e254221ccd10afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A1=BE=E7=9B=BC?= Date: Wed, 21 Oct 2026 14:03:31 +0800 Subject: [PATCH] feat(cua): streamline computer use and port native reliability fixes (#14846) * feat(cua): preload the JavaScript computer use runtime * refactor(cua): remove the native Wayland backend * feat(cua): move app computer use into the driver * feat(cua): preserve background delivery and port platform fixes * feat(cua): add explicit desktop input and browser runtime facade * feat(cua): package the unified 0.26.0 computer use runtime * fix(cua): finalize cross-platform native reliability * fix(cua): isolate runtime releases from CLI versioning * fix(cua): keep Electron out of root workspace installs * fix(cua): synchronize consumers and async cleanup tests * fix(browser): migrate smoke to preloaded cua runtime * docs(cua): document MCP setup for Qwen Code Claude Code and Codex --- .github/workflows/.size-baseline | 2 +- .github/workflows/cd-cua-driver.yml | 259 +- .pnpmfile.mjs | 1 + AGENTS.md | 2 +- README.md | 1 + docs/design/cua-upgrade-2026-10.md | 332 ++ docs/design/cua-upgrade-2026-10.zh-CN.md | 347 ++ docs/design/cua-upgrade-acceptance-2026-10.md | 229 ++ .../cua-upgrade-acceptance-2026-10.zh-CN.md | 223 ++ docs/design/cua-upgrade-platform-ports.md | 564 +++ .../cua-upgrade-platform-ports.zh-CN.md | 247 ++ docs/users/features/computer-use.md | 146 +- package.json | 1 + packages/browser-use/LICENSE | 203 + packages/browser-use/build.mjs | 4 +- packages/browser-use/package.json | 12 +- .../browser-use/scripts/smoke-checks.test.ts | 222 +- .../scripts/smoke-qwen-saucedemo.ts | 133 +- .../scripts/smoke-transcript.test.ts | 14 +- .../browser-use/scripts/smoke-transcript.ts | 4 +- .../browser-use/scripts/verify-package.mjs | 61 + ...-kernel.test.ts => runtime-kernel.test.ts} | 39 +- .../skills/bundled-skills.integration.test.ts | 17 +- .../src/skills/bundled/browser-use/SKILL.md | 159 - .../skills/bundled/browser-use/skill.test.ts | 138 - .../src/skills/bundled/computer-use/SKILL.md | 295 -- .../skills/bundled/computer-use/SKILL.test.ts | 143 - .../bundled/computer-use/agents/openai.yaml | 4 - packages/cua-driver/.vendored-patches.md | 24 + packages/cua-driver/README.md | 138 +- packages/cua-driver/contract/manifest.json | 749 ++-- .../0.13.2-overnight-stabilization-plan.md | 5 + packages/cua-driver/docs/action-support.md | 16 +- ...rowser-cross-platform-hardening-journal.md | 5 + ...rowser-existing-profile-attachment-plan.md | 5 + .../docs/browser-semantic-state-plan.md | 5 + .../browser-tool-implementation-journal.md | 5 + .../docs/browser-tool-implementation-plan.md | 5 + ...ursor-modifier-badge-restructuring-plan.md | 5 + packages/cua-driver/docs/cursor-themes.md | 10 +- ...river-permission-modes-and-consent-plan.md | 5 + ...permission-modes-implementation-journal.md | 5 + .../docs/e2e-convergence-journal.md | 5 + .../docs/linux-desktop-validation.md | 9 +- .../docs/linux-support-completion-plan.md | 5 + ...mission-adapters-and-session-modes-plan.md | 5 + ...ission-simplification-execution-journal.md | 5 + .../cua-driver/docs/pr-2161-landing-plan.md | 5 + ...rce-of-truth-and-uniffi-evaluation-plan.md | 5 + .../docs/test-harness-convergence-plan.md | 5 + .../cua-driver/docs/test-harnesses-guide.md | 30 +- packages/cua-driver/docs/test-matrix.md | 17 +- .../docs/wayland-compositor-identity-plan.md | 102 - packages/cua-driver/rust/CHANGELOG.md | 12 + packages/cua-driver/rust/Cargo.lock | 598 +-- packages/cua-driver/rust/Cargo.toml | 2 +- packages/cua-driver/rust/README.md | 2 +- .../rust/Skills/cua-driver/BROWSER.md | 1 - .../rust/Skills/cua-driver/LINUX.md | 67 +- .../rust/Skills/cua-driver/SKILL.md | 18 +- packages/cua-driver/rust/VERSION | 2 +- .../src/bin/cua-contract-gen.rs | 32 +- .../cua-driver-contract/src/computer_use.rs | 146 + .../crates/cua-driver-contract/src/inputs.rs | 159 +- .../crates/cua-driver-contract/src/lib.rs | 114 +- .../crates/cua-driver-contract/src/outputs.rs | 15 +- .../cua-driver-contract/src/verification.rs | 2 +- .../rust/crates/cua-driver-core/Cargo.toml | 1 + .../cua-driver-core/src/action_record.rs | 111 +- .../cua-driver-core/src/authorization.rs | 24 + .../cua-driver-core/src/browser/prepare.rs | 22 +- .../cua-driver-core/src/capture_scope.rs | 2 + .../cua-driver-core/src/computer_use.rs | 1418 +++++++ .../cua-driver-core/src/desktop_keyboard.rs | 274 ++ .../cua-driver-core/src/element_cache.rs | 45 +- .../cua-driver-core/src/element_token.rs | 264 +- .../rust/crates/cua-driver-core/src/lib.rs | 2 + .../crates/cua-driver-core/src/protocol.rs | 78 +- .../crates/cua-driver-core/src/recording.rs | 355 +- .../cua-driver-core/src/recording_tools.rs | 21 +- .../crates/cua-driver-core/src/session.rs | 303 +- .../cua-driver-core/src/session_manifest.rs | 34 +- .../rust/crates/cua-driver-core/src/tool.rs | 631 ++- .../cua-driver-core/src/video_ffmpeg.rs | 121 +- .../tests/recording_capture_budget.rs | 221 ++ .../rust/crates/cua-driver-sdk/Cargo.toml | 3 - .../crates/cua-driver-sdk/src/embedded.rs | 125 +- .../rust/crates/cua-driver-sdk/src/lib.rs | 27 +- .../rust/crates/cua-driver-sdk/src/runtime.rs | 19 +- .../rust/crates/cua-driver-testkit/src/e2e.rs | 129 +- .../crates/cua-driver-testkit/src/observer.rs | 555 +-- .../crates/cua-driver-testkit/src/sentinel.rs | 457 +-- .../rust/crates/cua-driver/Cargo.toml | 8 +- .../rust/crates/cua-driver/src/autostart.rs | 84 +- .../rust/crates/cua-driver/src/doctor.rs | 38 +- .../rust/crates/cua-driver/src/main.rs | 9 + .../rust/crates/cua-driver/src/proxy.rs | 13 + .../rust/crates/cua-driver/src/sdk_adapter.rs | 52 +- .../rust/crates/cua-driver/src/serve.rs | 77 +- .../rust/crates/cua-driver/tests/README.md | 3 +- .../tests/agent_cursor_showcase_test.rs | 8 +- .../tests/cross_platform_behavior_test.rs | 51 +- .../tests/desktop_scope_linux_test.rs | 8 +- .../cua-driver/tests/harness_gtk3_test.rs | 101 +- .../cua-driver/tests/private_worker_test.rs | 36 +- .../tests/standalone_browser_behavior_test.rs | 150 +- .../tests/wayland_overlay_idle_test.rs | 178 - .../rust/crates/platform-linux/Cargo.toml | 114 +- .../platform-linux/examples/libei_input.rs | 57 - .../examples/screenshot_cascade.rs | 52 - .../protocol/virtual-keyboard-unstable-v1.xml | 68 - .../platform-linux/src/at_point_policy.rs | 104 + .../crates/platform-linux/src/atspi/cache.rs | 228 +- .../crates/platform-linux/src/atspi/mod.rs | 55 +- .../crates/platform-linux/src/atspi/native.rs | 795 ++-- .../platform-linux/src/browser_consent_ui.rs | 18 +- .../platform-linux/src/browser_platform.rs | 262 +- .../platform-linux/src/browser_setup_ui.rs | 46 +- .../rust/crates/platform-linux/src/capture.rs | 325 +- .../platform-linux/src/health_report.rs | 508 +-- .../platform-linux/src/input/delivery.rs | 83 +- .../platform-linux/src/input/focus_guard.rs | 1063 +++++ .../src/input/keymap_barrier.rs | 150 + .../crates/platform-linux/src/input/mod.rs | 3067 ++++++++++++--- .../platform-linux/src/input/mpx_keyboard.rs | 1313 +++++++ .../platform-linux/src/input/mpx_owner.rs | 204 + .../platform-linux/src/input/user_input.rs | 140 + .../rust/crates/platform-linux/src/lib.rs | 74 +- .../rust/crates/platform-linux/src/overlay.rs | 109 +- .../rust/crates/platform-linux/src/pip/mod.rs | 11 - .../rust/crates/platform-linux/src/proc_fs.rs | 9 + .../platform-linux/src/recording_hooks.rs | 61 +- .../crates/platform-linux/src/terminal.rs | 8 +- .../crates/platform-linux/src/tools/impl_.rs | 2434 ++++-------- .../crates/platform-linux/src/tools/stubs.rs | 28 +- .../platform-linux/src/video_wayland.rs | 287 -- .../src/wayland/ext_screencopy.rs | 422 -- .../src/wayland/ext_toplevel.rs | 431 -- .../platform-linux/src/wayland/libei.rs | 1421 ------- .../crates/platform-linux/src/wayland/mod.rs | 3496 ----------------- .../platform-linux/src/wayland/overlay.rs | 868 ---- .../src/wayland/persistent_vptr.rs | 336 -- .../platform-linux/src/wayland/portal.rs | 18 - .../src/wayland/portal_screencast.rs | 465 --- .../src/wayland/portal_screenshot.rs | 142 - .../src/wayland/shell_helper.rs | 695 ---- .../platform-linux/src/wayland/sway_ipc.rs | 329 -- .../src/wayland/virtual_keyboard.rs | 271 -- .../rust/crates/platform-linux/src/x11/mod.rs | 179 +- .../platform-linux/src/x11_client_pid.rs | 291 ++ .../platform-linux/tests/wayland_app.rs | 71 - .../crates/platform-macos/src/apple_events.rs | 577 +++ .../crates/platform-macos/src/apps/mod.rs | 116 +- .../crates/platform-macos/src/ax/app_text.rs | 2 +- .../crates/platform-macos/src/ax/bindings.rs | 172 +- .../crates/platform-macos/src/ax/cache.rs | 234 +- .../platform-macos/src/ax/enablement.rs | 301 +- .../platform-macos/src/ax/exact_target.rs | 149 +- .../crates/platform-macos/src/ax/launch.rs | 52 + .../rust/crates/platform-macos/src/ax/mod.rs | 2 + .../platform-macos/src/ax/reveal_scroll.rs | 451 +++ .../rust/crates/platform-macos/src/ax/tree.rs | 531 ++- .../rust/crates/platform-macos/src/capture.rs | 270 +- .../platform-macos/src/cursor/overlay.rs | 8 +- .../platform-macos/src/document_edit.rs | 276 ++ .../crates/platform-macos/src/focus_guard.rs | 23 +- .../crates/platform-macos/src/focus_steal.rs | 4 + .../platform-macos/src/input/app_focus.rs | 26 +- .../platform-macos/src/input/app_pointer.rs | 50 +- .../platform-macos/src/input/keyboard.rs | 96 +- .../crates/platform-macos/src/input/mouse.rs | 499 ++- .../platform-macos/src/input/skylight.rs | 288 +- .../platform-macos/src/key_focus_guard.rs | 964 +++++ .../rust/crates/platform-macos/src/lib.rs | 6 + .../platform-macos/src/recording_hooks.rs | 144 +- .../crates/platform-macos/src/tools/click.rs | 347 +- .../platform-macos/src/tools/double_click.rs | 55 +- .../crates/platform-macos/src/tools/drag.rs | 10 +- .../src/tools/get_screen_size.rs | 68 +- .../src/tools/get_window_state.rs | 77 +- .../crates/platform-macos/src/tools/hotkey.rs | 44 +- .../platform-macos/src/tools/invoke_menu.rs | 2 +- .../platform-macos/src/tools/list_apps.rs | 13 +- .../platform-macos/src/tools/list_windows.rs | 40 +- .../src/tools/menu_equivalent.rs | 316 ++ .../crates/platform-macos/src/tools/mod.rs | 72 +- .../crates/platform-macos/src/tools/paste.rs | 2 +- .../src/tools/perform_secondary_action.rs | 28 +- .../platform-macos/src/tools/press_key.rs | 4 +- .../platform-macos/src/tools/right_click.rs | 27 +- .../crates/platform-macos/src/tools/scroll.rs | 153 +- .../platform-macos/src/tools/select_text.rs | 24 +- .../platform-macos/src/tools/set_value.rs | 469 ++- .../platform-macos/src/tools/type_text.rs | 99 +- .../src/tools/type_text_chars.rs | 8 +- .../crates/platform-macos/src/video_sckit.rs | 227 +- .../src/window_change_detector.rs | 11 +- .../rust/crates/platform-macos/src/windows.rs | 65 +- .../rust/crates/platform-windows/Cargo.toml | 5 + .../src/browser_launch_token.rs | 513 +++ .../src/browser_standard_user.rs | 349 ++ .../crates/platform-windows/src/capture.rs | 107 +- .../platform-windows/src/input/delivery.rs | 26 +- .../platform-windows/src/input/keyboard.rs | 292 +- .../crates/platform-windows/src/input/mod.rs | 1 + .../platform-windows/src/input/mouse.rs | 312 +- .../platform-windows/src/input/user_input.rs | 126 + .../rust/crates/platform-windows/src/lib.rs | 7 + .../platform-windows/src/recording_hooks.rs | 247 +- .../src/standard_user_launch.rs | 360 ++ .../platform-windows/src/tools/impl_.rs | 620 +-- .../crates/platform-windows/src/uia/cache.rs | 246 +- .../src/uia/cache_uaf_repro.rs | 174 +- .../crates/platform-windows/src/uia/mod.rs | 32 +- .../platform-windows/src/uia/windows_enum.rs | 150 +- .../crates/platform-windows/src/uia_worker.rs | 18 +- .../platform-windows/src/win32/combo.rs | 361 ++ .../src/win32/installed_apps.rs | 67 +- .../crates/platform-windows/src/win32/mod.rs | 2 + .../cua-driver/scripts/_install-common.psm1 | 65 +- .../cua-driver/scripts/_install-local-rust.sh | 21 - packages/cua-driver/scripts/_install-rust.sh | 178 +- .../cua-driver/scripts/build-node-runtime.mjs | 16 +- .../scripts/generate-uniffi-bindings.mjs | 11 +- packages/cua-driver/scripts/install-local.ps1 | 11 +- packages/cua-driver/scripts/install.ps1 | 121 +- packages/cua-driver/scripts/install.sh | 4 +- packages/cua-driver/scripts/resolve-ubrn.mjs | 21 + .../tests/install-windows-regression.ps1 | 97 +- .../scripts/tests/test_install_local.py | 14 - .../tests/test_install_macos_recovery.py | 130 + .../tests/test_uninstall_claude_ownership.py | 198 + .../tests/uninstall-windows-regression.ps1 | 84 + packages/cua-driver/scripts/uninstall.ps1 | 199 +- packages/cua-driver/scripts/uninstall.sh | 401 +- .../scripts/verify-cua-sdk-package.mjs | 31 +- .../apps/cross-platform/electron/main.js | 48 +- .../cua-driver/tests/fixtures/build/linux.sh | 3 +- packages/cua-driver/typescript/README.md | 5 +- .../typescript/computer-use/README.md | 261 +- .../cua-driver/typescript/computer-use/app.js | 298 +- .../typescript/computer-use/browser.js | 56 + .../typescript/computer-use/documentation.js | 29 + .../typescript/computer-use/index.d.ts | 382 +- .../typescript/computer-use/index.js | 879 +---- .../typescript/computer-use/interface.json | 721 ++++ .../typescript/computer-use/runtime.d.ts | 57 + .../typescript/computer-use/runtime.js | 117 + .../computer-use/test/app-lifecycle.test.mjs | 174 - .../test/app-oriented-baseline.test.mjs | 102 - .../typescript/computer-use/test/app.test.mjs | 628 --- .../computer-use/test/computer-use.test.mjs | 2351 +---------- .../test/daemon-integration.test.mjs | 60 +- .../computer-use/test/platform.test.mjs | 2 +- .../computer-use/test/runtime.test.mjs | 92 + .../test/text-operations.test.mjs | 224 -- .../computer-use/test/types.test.ts | 152 +- .../cua-driver/typescript/package-lock.json | 301 -- packages/cua-driver/typescript/package.json | 22 +- .../typescript/scripts/install-native.mjs | 8 +- .../scripts/stage-computer-use-skill.mjs | 38 - .../typescript/src/native-assets.ts | 2 +- .../typescript/test/native-assets.test.mjs | 19 +- .../typescript/test/skill-package.test.mjs | 68 - packages/cua-driver/wayland-helper/README.md | 68 - packages/cua-driver/wayland-helper/install.sh | 25 - .../wayland-helper/winrects@cua/extension.js | 1053 ----- .../wayland-helper/winrects@cua/metadata.json | 7 - packages/node-repl/README.md | 163 +- packages/node-repl/package.json | 7 +- .../node-repl/scripts/lifecycle-smoke.mjs | 2 +- packages/node-repl/scripts/mcp-smoke.mjs | 13 +- packages/node-repl/scripts/verify-package.mjs | 28 +- .../node-repl/src/desktop-relay/constants.ts | 14 +- .../src/desktop-relay/mcp-child-relay.test.ts | 23 +- .../src/desktop-relay/mcp-child-relay.ts | 25 +- packages/node-repl/src/index.ts | 8 +- packages/node-repl/src/kernel-manager.test.ts | 35 +- packages/node-repl/src/kernel-manager.ts | 26 +- packages/node-repl/src/mcp-server.test.ts | 531 +-- packages/node-repl/src/mcp-server.ts | 409 +- .../node-repl/src/node-repl.lines.test.ts | 4 +- packages/node-repl/src/node-repl.napi.test.ts | 4 +- .../node-repl/src/node-repl.reload.test.ts | 4 +- .../node-repl/src/node-repl.scale.test.ts | 10 +- .../node-repl/src/node-repl.semantics.test.ts | 6 +- packages/node-repl/src/protocol.ts | 1 + packages/node-repl/src/runtime/kernel.mjs | 30 +- .../desktop-relay/desktop-relay-client.ts | 2 +- pnpm-lock.yaml | 38 +- pnpm-workspace.yaml | 1 + scripts/copy-browser-use-assets.js | 24 +- scripts/copy_bundle_assets.js | 7 +- scripts/copy_files.js | 23 +- scripts/dev.js | 26 - scripts/release-packages.mjs | 2 + scripts/tests/browser-use-assets.test.js | 171 +- .../tests/cua-driver-release-workflow.test.js | 6 +- scripts/tests/dev.test.js | 4 +- scripts/tests/package-assets.test.js | 8 +- scripts/tests/package-scripts.test.js | 5 + scripts/tests/release-versioning.test.js | 9 + scripts/tests/release-workflow.test.js | 64 +- 303 files changed, 28408 insertions(+), 27909 deletions(-) create mode 100644 docs/design/cua-upgrade-2026-10.md create mode 100644 docs/design/cua-upgrade-2026-10.zh-CN.md create mode 100644 docs/design/cua-upgrade-acceptance-2026-10.md create mode 100644 docs/design/cua-upgrade-acceptance-2026-10.zh-CN.md create mode 100644 docs/design/cua-upgrade-platform-ports.md create mode 100644 docs/design/cua-upgrade-platform-ports.zh-CN.md create mode 100644 packages/browser-use/LICENSE create mode 100644 packages/browser-use/scripts/verify-package.mjs rename packages/browser-use/src/{skill-runtime-kernel.test.ts => runtime-kernel.test.ts} (80%) delete mode 100644 packages/core/src/skills/bundled/browser-use/SKILL.md delete mode 100644 packages/core/src/skills/bundled/browser-use/skill.test.ts delete mode 100644 packages/core/src/skills/bundled/computer-use/SKILL.md delete mode 100644 packages/core/src/skills/bundled/computer-use/SKILL.test.ts delete mode 100644 packages/core/src/skills/bundled/computer-use/agents/openai.yaml delete mode 100644 packages/cua-driver/docs/wayland-compositor-identity-plan.md create mode 100644 packages/cua-driver/rust/crates/cua-driver-contract/src/computer_use.rs create mode 100644 packages/cua-driver/rust/crates/cua-driver-core/src/computer_use.rs create mode 100644 packages/cua-driver/rust/crates/cua-driver-core/src/desktop_keyboard.rs create mode 100644 packages/cua-driver/rust/crates/cua-driver-core/tests/recording_capture_budget.rs delete mode 100644 packages/cua-driver/rust/crates/cua-driver/tests/wayland_overlay_idle_test.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/examples/libei_input.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/protocol/virtual-keyboard-unstable-v1.xml create mode 100644 packages/cua-driver/rust/crates/platform-linux/src/at_point_policy.rs create mode 100644 packages/cua-driver/rust/crates/platform-linux/src/input/focus_guard.rs create mode 100644 packages/cua-driver/rust/crates/platform-linux/src/input/keymap_barrier.rs create mode 100644 packages/cua-driver/rust/crates/platform-linux/src/input/mpx_keyboard.rs create mode 100644 packages/cua-driver/rust/crates/platform-linux/src/input/mpx_owner.rs create mode 100644 packages/cua-driver/rust/crates/platform-linux/src/input/user_input.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/video_wayland.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/portal.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/src/wayland/virtual_keyboard.rs create mode 100644 packages/cua-driver/rust/crates/platform-linux/src/x11_client_pid.rs delete mode 100644 packages/cua-driver/rust/crates/platform-linux/tests/wayland_app.rs create mode 100644 packages/cua-driver/rust/crates/platform-macos/src/apple_events.rs create mode 100644 packages/cua-driver/rust/crates/platform-macos/src/ax/launch.rs create mode 100644 packages/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs create mode 100644 packages/cua-driver/rust/crates/platform-macos/src/document_edit.rs create mode 100644 packages/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs create mode 100644 packages/cua-driver/rust/crates/platform-macos/src/tools/menu_equivalent.rs create mode 100644 packages/cua-driver/rust/crates/platform-windows/src/browser_launch_token.rs create mode 100644 packages/cua-driver/rust/crates/platform-windows/src/browser_standard_user.rs create mode 100644 packages/cua-driver/rust/crates/platform-windows/src/input/user_input.rs create mode 100644 packages/cua-driver/rust/crates/platform-windows/src/standard_user_launch.rs create mode 100644 packages/cua-driver/rust/crates/platform-windows/src/win32/combo.rs create mode 100644 packages/cua-driver/scripts/resolve-ubrn.mjs create mode 100644 packages/cua-driver/scripts/tests/test_install_macos_recovery.py create mode 100644 packages/cua-driver/scripts/tests/test_uninstall_claude_ownership.py create mode 100644 packages/cua-driver/scripts/tests/uninstall-windows-regression.ps1 create mode 100644 packages/cua-driver/typescript/computer-use/browser.js create mode 100644 packages/cua-driver/typescript/computer-use/documentation.js create mode 100644 packages/cua-driver/typescript/computer-use/interface.json create mode 100644 packages/cua-driver/typescript/computer-use/runtime.d.ts create mode 100644 packages/cua-driver/typescript/computer-use/runtime.js delete mode 100644 packages/cua-driver/typescript/computer-use/test/app-lifecycle.test.mjs delete mode 100644 packages/cua-driver/typescript/computer-use/test/app-oriented-baseline.test.mjs delete mode 100644 packages/cua-driver/typescript/computer-use/test/app.test.mjs create mode 100644 packages/cua-driver/typescript/computer-use/test/runtime.test.mjs delete mode 100644 packages/cua-driver/typescript/computer-use/test/text-operations.test.mjs delete mode 100644 packages/cua-driver/typescript/package-lock.json delete mode 100644 packages/cua-driver/typescript/scripts/stage-computer-use-skill.mjs delete mode 100644 packages/cua-driver/typescript/test/skill-package.test.mjs delete mode 100644 packages/cua-driver/wayland-helper/README.md delete mode 100755 packages/cua-driver/wayland-helper/install.sh delete mode 100644 packages/cua-driver/wayland-helper/winrects@cua/extension.js delete mode 100644 packages/cua-driver/wayland-helper/winrects@cua/metadata.json diff --git a/.github/workflows/.size-baseline b/.github/workflows/.size-baseline index 7916f4650e6..33f73f81b7e 100644 --- a/.github/workflows/.size-baseline +++ b/.github/workflows/.size-baseline @@ -17,7 +17,7 @@ 3480 audio-capture-prebuilds.yml 9023 auto-minimize-spam.yml 9256 build-and-publish-image.yml -50773 cd-cua-driver.yml +55276 cd-cua-driver.yml 2222 cd-mobile-mcp.yml 147136 ci.yml 4193 codeql.yml diff --git a/.github/workflows/cd-cua-driver.yml b/.github/workflows/cd-cua-driver.yml index 1a600efb55a..cba34139e08 100644 --- a/.github/workflows/cd-cua-driver.yml +++ b/.github/workflows/cd-cua-driver.yml @@ -20,8 +20,8 @@ name: 'CD: Qwen CUA Driver' # Linux : cua-driver-rs--linux-{x86_64,arm64}.tar.gz + *-binary.tar.gz # Windows: cua-driver-rs--windows-{x86_64,arm64}.zip + *-binary.zip # npm : @qwen-code/cua-sdk downloads and verifies the matching *-binary -# archive above. @qwen-code/node-repl-mcp is independently -# versioned and ships from the same verified workflow. +# archive above. SDK, browser SDK and Node REPL share one version +# and ship from the same verified workflow. on: push: tags: ['cua-driver-rs-v*'] @@ -30,7 +30,7 @@ on: version: description: 'Version to release (without v prefix)' required: true - default: '0.20.11' + default: '0.26.0' notarize: description: 'Codesign + notarize the macOS artifacts (false to skip during @@ -40,18 +40,11 @@ on: default: true dry_run: description: - 'Build release assets and clean-install both npm packages without + 'Build release assets and clean-install all three npm packages without creating a GitHub Release, npm version, or tag.' required: false type: 'boolean' default: true - node_repl_only: - description: - 'Only verify/publish @qwen-code/node-repl-mcp; do not build or release - cua-driver assets.' - required: false - type: 'boolean' - default: false permissions: contents: 'write' jobs: @@ -109,8 +102,6 @@ jobs: # shellcheck disable=SC2016 README_PS1_VERSION=$(sed -nE 's/^\$env:CUA_DRIVER_RS_VERSION = "([^"]+)"$/\1/p' packages/cua-driver/README.md) README_EXPECTED_VERSION=$(sed -nE 's/^Expected: qwen-cua-driver ([0-9A-Za-z.-]+)\.$/\1/p' packages/cua-driver/README.md) - SKILL_SDK_VERSION=$(sed -nE 's/^npm install --no-save --package-lock=false @qwen-code\/cua-sdk@([^ ]+)$/\1/p' packages/core/src/skills/bundled/computer-use/SKILL.md | sort -u) - USER_GUIDE_SDK_VERSION=$(sed -nE 's/^npm install --no-save --package-lock=false @qwen-code\/cua-sdk@([^ ]+)$/\1/p' docs/users/features/computer-use.md) for PAIR in \ "shell installer:$INSTALL_SH_VERSION" \ "PowerShell installer:$INSTALL_PS1_VERSION" \ @@ -119,9 +110,7 @@ jobs: "driver skill:$DRIVER_SKILL_VERSION" \ "README shell command:$README_SH_VERSION" \ "README PowerShell command:$README_PS1_VERSION" \ - "README expected output:$README_EXPECTED_VERSION" \ - "Computer Use skill SDK:$SKILL_SDK_VERSION" \ - "Computer Use user guide SDK:$USER_GUIDE_SDK_VERSION"; do + "README expected output:$README_EXPECTED_VERSION"; do LABEL=${PAIR%%:*} VALUE=${PAIR#*:} if [[ "$VALUE" != "$VERSION" ]]; then @@ -140,26 +129,26 @@ jobs: echo "::error::Invalid Node REPL package version $NODE_REPL_VERSION" exit 1 fi - SKILL_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' packages/core/src/skills/bundled/computer-use/SKILL.md) - USER_GUIDE_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' docs/users/features/computer-use.md) - BROWSER_USE_SKILL_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' packages/core/src/skills/bundled/browser-use/SKILL.md) - for PAIR in \ - "Computer Use skill Node REPL:$SKILL_NODE_REPL_VERSION" \ - "Computer Use user guide Node REPL:$USER_GUIDE_NODE_REPL_VERSION" \ - "Browser Use skill Node REPL:$BROWSER_USE_SKILL_NODE_REPL_VERSION"; do - LABEL=${PAIR%%:*} - VALUE=${PAIR#*:} - if [[ "$VALUE" != "$NODE_REPL_VERSION" ]]; then - echo "::error::$LABEL version $VALUE does not match Node REPL version $NODE_REPL_VERSION" + BROWSER_VERSION=$(node -p "require('./packages/browser-use/package.json').version") + SDK_BROWSER_VERSION=$(node -p "require('./packages/cua-driver/typescript/package.json').dependencies['@qwen-code/browser-use']") + NODE_SDK_VERSION=$(node -p "require('./packages/node-repl/package.json').dependencies['@qwen-code/cua-sdk']") + for VALUE in "$NODE_REPL_VERSION" "$BROWSER_VERSION" "$SDK_BROWSER_VERSION" "$NODE_SDK_VERSION"; do + if [[ "$VALUE" != "$VERSION" ]]; then + echo "::error::SDK, browser SDK, Node REPL and native release must use the same version $VERSION; found $VALUE" exit 1 fi done + USER_GUIDE_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' docs/users/features/computer-use.md) + if [[ "$USER_GUIDE_NODE_REPL_VERSION" != "$NODE_REPL_VERSION" ]]; then + echo "::error::Computer Use user guide Node REPL version $USER_GUIDE_NODE_REPL_VERSION does not match Node REPL version $NODE_REPL_VERSION" + exit 1 + fi if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "$GITHUB_REF" != "refs/heads/main" ]]; then echo "::error::A production dispatch must run from protected main" exit 1 fi - if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "${{ inputs.node_repl_only }}" != "true" && "${{ inputs.notarize }}" != "true" ]]; then + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "${{ inputs.notarize }}" != "true" ]]; then echo "::error::A production dispatch must enable macOS codesigning and notarization" exit 1 fi @@ -173,25 +162,17 @@ jobs: echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "node_repl_version=$NODE_REPL_VERSION" >> "$GITHUB_OUTPUT" - uses: 'dtolnay/rust-toolchain@1.97.1' - if: - "github.event_name != 'workflow_dispatch' || inputs.node_repl_only == - false" - name: 'Test model payload filtering' - if: - "github.event_name != 'workflow_dispatch' || inputs.node_repl_only == - false" working-directory: 'packages/cua-driver/rust' run: 'cargo test -p cua-driver-core --locked model_payload' # ── Linux (x86_64 + arm64) ─────────────────────────────────────────────── # Built inside debian:11 (glibc 2.31) so binaries run on Debian 11+, # Ubuntu 20.04+, RHEL/Rocky 8+. arm64 builds natively on a hosted ARM - # runner to avoid hand-wiring a cross linker for the X11/Wayland C deps. + # runner to avoid hand-wiring a cross linker for the X11 C deps. build-linux: name: 'linux-${{ matrix.arch }}' needs: ['validate-version'] - if: "github.event_name != 'workflow_dispatch' || inputs.node_repl_only == - false" runs-on: '${{ matrix.runner }}' strategy: fail-fast: false @@ -219,14 +200,11 @@ jobs: apt-get -o Acquire::Retries=3 update apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ git ca-certificates curl build-essential pkg-config \ - libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev \ - libxkbcommon-dev + libx11-dev libxi-dev libxtst-dev libxext-dev - uses: 'actions/checkout@v4' - uses: 'actions/setup-node@v4' with: node-version: '22' - cache: 'npm' - cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json' - name: 'Determine version' id: 'version' shell: 'bash' @@ -253,11 +231,11 @@ jobs: cua-driver-linux-${{ matrix.arch }}- - name: 'Build (release)' working-directory: 'packages/cua-driver/rust' - run: 'cargo build -p cua-driver -p cursor-theme-cli -p cua-driver-sdk --release --locked --features cua-driver/portal-input --target ${{ matrix.target }}' + run: 'cargo build -p cua-driver -p cursor-theme-cli -p cua-driver-sdk --release --locked --target ${{ matrix.target }}' - name: 'Build Electron-compatible Node runtime' working-directory: 'packages/cua-driver' run: | - npm --prefix typescript ci --ignore-scripts + corepack pnpm -C ../.. install --frozen-lockfile --ignore-scripts --prefer-offline node scripts/build-node-runtime.mjs \ --target "${{ matrix.target }}" \ --output "rust/target/${{ matrix.target }}/release/cua_driver_node_runtime.node" @@ -273,12 +251,11 @@ jobs: cp "target/${TARGET}/release/libcua_driver_sdk.so" "release/${STAGE}/" cp "target/${TARGET}/release/cua_driver_node_runtime.node" "release/${STAGE}/" cp include/cua_driver_abi.h "release/${STAGE}/" - cp -R ../wayland-helper "release/${STAGE}/wayland-helper" cp ../../../LICENSE "release/${STAGE}/LICENSE" (cd release && tar -czf "${STAGE}.tar.gz" "${STAGE}") tar -czf "release/${STAGE}-binary.tar.gz" -C "release/${STAGE}" \ qwen-cua-driver cua-cursor-theme libcua_driver_sdk.so \ - cua_driver_node_runtime.node cua_driver_abi.h wayland-helper + cua_driver_node_runtime.node cua_driver_abi.h ls -lh "release/${STAGE}.tar.gz" "release/${STAGE}-binary.tar.gz" - uses: 'actions/upload-artifact@v4' with: @@ -290,8 +267,6 @@ jobs: build-windows: name: 'windows-${{ matrix.arch }}' needs: ['validate-version'] - if: "github.event_name != 'workflow_dispatch' || inputs.node_repl_only == - false" runs-on: 'windows-latest' strategy: fail-fast: false @@ -306,8 +281,6 @@ jobs: - uses: 'actions/setup-node@v4' with: node-version: '22' - cache: 'npm' - cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json' - name: 'Determine version' id: 'version' shell: 'bash' @@ -344,7 +317,7 @@ jobs: working-directory: 'packages/cua-driver' shell: 'bash' run: | - npm --prefix typescript ci --ignore-scripts + corepack pnpm -C ../.. install --frozen-lockfile --ignore-scripts --prefer-offline node scripts/build-node-runtime.mjs \ --target "${{ matrix.target }}" \ --output "rust/target/${{ matrix.target }}/release/cua_driver_node_runtime.node" @@ -387,8 +360,6 @@ jobs: build-macos: name: 'macos-universal' needs: ['validate-version'] - if: "github.event_name != 'workflow_dispatch' || inputs.node_repl_only == - false" runs-on: 'macos-26' env: # Sign+notarize on tag push (real release); on manual dispatch honor the @@ -399,8 +370,6 @@ jobs: - uses: 'actions/setup-node@v4' with: node-version: '22' - cache: 'npm' - cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json' - name: 'Determine version' id: 'version' shell: 'bash' @@ -436,7 +405,7 @@ jobs: - name: 'Build Electron-compatible Node runtimes' working-directory: 'packages/cua-driver' run: | - npm --prefix typescript ci --ignore-scripts + corepack pnpm -C ../.. install --frozen-lockfile --ignore-scripts --prefer-offline node scripts/build-node-runtime.mjs \ --target aarch64-apple-darwin \ --output rust/target/aarch64-apple-darwin/release/cua_driver_node_runtime.node @@ -624,7 +593,13 @@ jobs: verify-sdk-package: name: '@qwen-code/cua-sdk package dry-run' - needs: ['validate-version', 'build-linux', 'verify-release-artifacts'] + needs: + [ + 'validate-version', + 'build-linux', + 'verify-release-artifacts', + 'verify-browser-package', + ] runs-on: 'ubuntu-latest' permissions: contents: 'read' @@ -633,8 +608,6 @@ jobs: - uses: 'actions/setup-node@v4' with: node-version: '22' - cache: 'npm' - cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json' registry-url: 'https://registry.npmjs.org' scope: '@qwen-code' - name: 'Install npm 11' @@ -664,9 +637,14 @@ jobs: (cd "$RELEASE_DIR" && sha256sum "$(basename "$ARCHIVE")" > checksums.txt) echo "directory=$NATIVE_DIR" >> "$GITHUB_OUTPUT" echo "release_directory=$RELEASE_DIR" >> "$GITHUB_OUTPUT" - - name: 'Install SDK dependencies' - working-directory: 'packages/cua-driver/typescript' - run: 'npm ci --ignore-scripts --no-audit --no-fund' + - name: 'Install SDK workspace dependencies' + run: | + corepack pnpm install --frozen-lockfile --ignore-scripts --prefer-offline + npm run build --workspace @qwen-code/browser-use + - uses: 'actions/download-artifact@v4' + with: + name: 'browser-use-npm-${{ needs.validate-version.outputs.version }}' + path: '${{ runner.temp }}/browser-use-package' - name: 'Test SDK and native loader' working-directory: 'packages/cua-driver/typescript' env: @@ -678,6 +656,7 @@ jobs: QWEN_CUA_SDK_NATIVE_DIR: '${{ steps.native.outputs.directory }}' run: | node packages/cua-driver/scripts/verify-cua-sdk-package.mjs \ + --browser-tarball "${RUNNER_TEMP}/browser-use-package/qwen-code-browser-use-${{ needs.validate-version.outputs.version }}.tgz" \ --native-dir "${QWEN_CUA_SDK_NATIVE_DIR}" \ --release-dir "${{ steps.native.outputs.release_directory }}" \ --output-dir "${RUNNER_TEMP}/cua-sdk-package" @@ -688,9 +667,39 @@ jobs: if-no-files-found: 'error' retention-days: 7 + verify-browser-package: + name: '@qwen-code/browser-use package dry-run' + needs: ['validate-version'] + runs-on: 'ubuntu-latest' + permissions: + contents: 'read' + steps: + - uses: 'actions/checkout@v4' + - uses: 'actions/setup-node@v4' + with: + node-version: '22' + - name: 'Install workspace dependencies' + run: 'corepack pnpm install --frozen-lockfile --ignore-scripts --prefer-offline' + - name: 'Test and pack existing browser SDK' + run: | + npm run test --workspace @qwen-code/browser-use + node packages/browser-use/scripts/verify-package.mjs --output-dir "${RUNNER_TEMP}/browser-use-package" + - uses: 'actions/upload-artifact@v4' + with: + name: 'browser-use-npm-${{ needs.validate-version.outputs.version }}' + path: '${{ runner.temp }}/browser-use-package/*.tgz' + if-no-files-found: 'error' + retention-days: 7 + verify-node-repl-package: name: '@qwen-code/node-repl-mcp package dry-run' - needs: ['validate-version'] + needs: + [ + 'validate-version', + 'verify-sdk-package', + 'verify-browser-package', + 'build-linux', + ] runs-on: 'ubuntu-latest' permissions: contents: 'read' @@ -705,6 +714,24 @@ jobs: run: 'npm install --global npm@11.19.0' - name: 'Install workspace dependencies' run: 'corepack pnpm install --frozen-lockfile --ignore-scripts --prefer-offline --reporter=append-only' + - uses: 'actions/download-artifact@v4' + with: + name: 'cua-sdk-npm-${{ needs.validate-version.outputs.version }}' + path: '${{ runner.temp }}/cua-sdk-package' + - uses: 'actions/download-artifact@v4' + with: + name: 'browser-use-npm-${{ needs.validate-version.outputs.version }}' + path: '${{ runner.temp }}/browser-use-package' + - uses: 'actions/download-artifact@v4' + with: + name: 'cua-driver-rs-linux-x86_64' + path: '${{ runner.temp }}/cua-driver-linux' + - name: 'Prepare matching native payload and SDK declarations' + run: | + mkdir -p "${RUNNER_TEMP}/native" + tar -xzf "${RUNNER_TEMP}/cua-driver-linux/cua-driver-rs-${{ needs.validate-version.outputs.version }}-linux-x86_64-binary.tar.gz" -C "${RUNNER_TEMP}/native" + npm run build --workspace @qwen-code/browser-use + npm run build --workspace @qwen-code/cua-sdk - name: 'Test standalone Node REPL package' working-directory: 'packages/node-repl' run: | @@ -715,8 +742,12 @@ jobs: npm run smoke:mcp npm run smoke:lifecycle - name: 'Pack, publish-dry-run, and clean-install Node REPL tarball' + env: + QWEN_CUA_SDK_NATIVE_DIR: '${{ runner.temp }}/native' run: | node packages/node-repl/scripts/verify-package.mjs \ + --sdk-tarball "${RUNNER_TEMP}/cua-sdk-package/qwen-code-cua-sdk-${{ needs.validate-version.outputs.version }}.tgz" \ + --browser-tarball "${RUNNER_TEMP}/browser-use-package/qwen-code-browser-use-${{ needs.validate-version.outputs.version }}.tgz" \ --output-dir "${RUNNER_TEMP}/node-repl-package" - uses: 'actions/upload-artifact@v4' with: @@ -736,6 +767,7 @@ jobs: 'build-macos', 'verify-release-artifacts', 'verify-sdk-package', + 'verify-browser-package', 'verify-node-repl-package', ] runs-on: 'ubuntu-latest' @@ -743,8 +775,7 @@ jobs: version: '${{ steps.version.outputs.version }}' if: "startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name - == 'workflow_dispatch' && inputs.dry_run == false && - inputs.node_repl_only == false)" + == 'workflow_dispatch' && inputs.dry_run == false)" steps: - uses: 'actions/checkout@v4' - uses: 'actions/download-artifact@v4' @@ -845,7 +876,7 @@ jobs: on the target machine before UIAccess activation - **Node.js**: the same workflow publishes `@qwen-code/cua-sdk` after a clean install against these release assets and publishes - the independently versioned `@qwen-code/node-repl-mcp` package + the same-version browser SDK and `@qwen-code/node-repl-mcp` package after its MCP protocol smoke tests Enable relative coordinates: `CUA_DRIVER_RS_COORDINATE_SPACE=1` @@ -856,11 +887,11 @@ jobs: publish-sdk: name: 'Publish @qwen-code/cua-sdk' - needs: ['validate-version', 'verify-sdk-package', 'release'] + needs: + ['validate-version', 'verify-sdk-package', 'release', 'publish-browser'] if: "startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name - == 'workflow_dispatch' && inputs.dry_run == false && - inputs.node_repl_only == false)" + == 'workflow_dispatch' && inputs.dry_run == false)" runs-on: 'ubuntu-latest' environment: name: 'production-release' @@ -917,13 +948,17 @@ jobs: if (!Array.isArray(listing.tools) || listing.tools.length === 0) { throw new Error("native SDK returned no tools"); } - const revisionSupported = await computer.supportsObservationRevision(); - if (typeof revisionSupported !== "boolean") { - throw new Error("ComputerUse capability probe returned an invalid result"); + let safeRefusal = false; + try { + await computer.appCall("click", { scope: "desktop", x: 1, y: 1 }); + } catch (error) { + if (error.code !== "screenshot_required" || error.sent !== false) throw error; + safeRefusal = true; } + if (!safeRefusal || listing.tools.length !== 19) throw new Error("Computer Use session/manifest contract mismatch"); console.log(JSON.stringify({ toolCount: listing.tools.length, - revisionSupported, + safeRefusal, })); } finally { await computer.close(); @@ -972,15 +1007,81 @@ jobs: echo '::error::Published npm version did not become visible' exit 1 + publish-browser: + name: 'Publish @qwen-code/browser-use' + needs: ['validate-version', 'verify-browser-package', 'release'] + if: "startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false)" + runs-on: 'ubuntu-latest' + environment: + name: 'production-release' + url: 'https://www.npmjs.com/package/@qwen-code/browser-use/v/${{ + needs.validate-version.outputs.version }}' + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: 'actions/setup-node@v4' + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + scope: '@qwen-code' + - name: 'Install npm 11' + run: 'npm install --global npm@11.19.0' + - uses: 'actions/download-artifact@v4' + with: + name: 'browser-use-npm-${{ + needs.validate-version.outputs.version }}' + path: '${{ runner.temp }}/browser-use-package' + - name: 'Publish immutable Browser SDK tarball' + env: + VERSION: '${{ needs.validate-version.outputs.version }}' + run: | + set -euo pipefail + TARBALL=$(find "${RUNNER_TEMP}/browser-use-package" -type f -name '*.tgz' -print -quit) + if [[ -z "$TARBALL" ]]; then + echo '::error::Packed @qwen-code/browser-use tarball is missing' + exit 1 + fi + LOCAL_INTEGRITY="sha512-$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A)" + REMOTE_INTEGRITY=$(npm view "@qwen-code/browser-use@${VERSION}" dist.integrity 2>/dev/null || true) + if [[ -n "$REMOTE_INTEGRITY" ]]; then + if [[ "$REMOTE_INTEGRITY" != "$LOCAL_INTEGRITY" ]]; then + echo "::error::@qwen-code/browser-use@${VERSION} already exists with different integrity" + exit 1 + fi + echo "@qwen-code/browser-use@${VERSION} already contains this exact tarball" + exit 0 + fi + NPM_TAG=latest + [[ "$VERSION" == *-* ]] && NPM_TAG=next + npm publish "$TARBALL" --provenance --access public --tag "$NPM_TAG" + - name: 'Verify npm registry integrity' + env: + VERSION: '${{ needs.validate-version.outputs.version }}' + run: | + set -euo pipefail + TARBALL=$(find "${RUNNER_TEMP}/browser-use-package" -type f -name '*.tgz' -print -quit) + LOCAL_INTEGRITY="sha512-$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A)" + for attempt in {1..20}; do + REMOTE_INTEGRITY=$(npm view "@qwen-code/browser-use@${VERSION}" dist.integrity 2>/dev/null || true) + if [[ "$REMOTE_INTEGRITY" == "$LOCAL_INTEGRITY" ]]; then + echo "Verified npm registry integrity on attempt $attempt" + exit 0 + fi + if [[ -n "$REMOTE_INTEGRITY" ]]; then + echo '::error::Published npm artifact integrity does not match the verified tarball' + exit 1 + fi + sleep 10 + done + echo '::error::Published npm version did not become visible' + exit 1 + publish-node-repl: name: 'Publish @qwen-code/node-repl-mcp' - needs: ['validate-version', 'verify-node-repl-package', 'release'] - if: "always() && needs.validate-version.result == 'success' && - needs.verify-node-repl-package.result == 'success' && - (needs.release.result == 'success' || (github.event_name == - 'workflow_dispatch' && inputs.node_repl_only == true)) && - (startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || - (github.event_name == 'workflow_dispatch' && inputs.dry_run == false))" + needs: + ['validate-version', 'verify-node-repl-package', 'release', 'publish-sdk'] + if: "startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false)" runs-on: 'ubuntu-latest' environment: name: 'production-release' diff --git a/.pnpmfile.mjs b/.pnpmfile.mjs index b5536fda2c2..8ad60abde10 100644 --- a/.pnpmfile.mjs +++ b/.pnpmfile.mjs @@ -26,6 +26,7 @@ export const workspacePackageNames = new Set([ '@qwen-code/channel-wecom', '@qwen-code/channel-weixin', '@qwen-code/chrome-bridge', + '@qwen-code/cua-sdk', '@qwen-code/external-context', '@qwen-code/external-context-mem0', '@qwen-code/test-failure-explorer', diff --git a/AGENTS.md b/AGENTS.md index 6201d9899b7..597db3baad4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,7 +76,7 @@ Runs the CLI via `tsx` with `DEV=true`. Changes to `packages/core` or `packages/cli` are reflected immediately without rebuilding. Browser Use reuses its built runtime, like other compiled workspace dependencies. After editing Browser Use, run `npm run build --workspace=@qwen-code/browser-use` to rebuild -and stage its runtime for dev. Normal `npm install` prepares it automatically. +its runtime for dev. Normal `npm install` prepares it automatically. ### Unit Testing diff --git a/README.md b/README.md index de3e0db8121..8d067e82e8b 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ For automation and custom integrations: - **Headless** — Run `qwen -p "..."` in scripts, CI/CD, or batch jobs. - **SDKs** — Build with [TypeScript](./packages/sdk-typescript/README.md), [Python](./packages/sdk-python/README.md), or [Java](./packages/sdk-java/qwencode/README.md). - **Daemon** — Run `qwen serve` to connect clients over HTTP + SSE (ACP). _(experimental)_ [Daemon guide](https://qwenlm.github.io/qwen-code-docs/en/users/qwen-serve/). +- **Computer Use** — Set up [native MCP tools](./packages/cua-driver/README.md#agent-integration) or [Node REPL + SDK](./packages/node-repl/README.md#usage) in Qwen Code, Claude Code, or Codex.
SDK example (Python) diff --git a/docs/design/cua-upgrade-2026-10.md b/docs/design/cua-upgrade-2026-10.md new file mode 100644 index 00000000000..5bb51daf6b5 --- /dev/null +++ b/docs/design/cua-upgrade-2026-10.md @@ -0,0 +1,332 @@ +# qwen-code Computer Use upgrade design and decisions (2026-10-15–16) + +[English](cua-upgrade-2026-10.md) | [简体中文](cua-upgrade-2026-10.zh-CN.md) + +This document records the background, analysis and decisions from these two days for implementation and later reference. + +- Decisions are dated. Detailed evidence is listed in the document index. +- **[Measured]** means supported by tests or data; **[Inference]** means inferred from code or observations; **[Pending]** means verification is required. + +--- + +## 0. Background + +**Scope** + +- `packages/cua-driver`: the Rust driver forked from trycua/cua `cua-driver-rs-v0.20.0`, distributed as qwen-cua-driver / QwenCuaDriver.app. +- `@qwen-code/cua-sdk` (`packages/cua-driver/typescript`, then 0.20.11). +- `@qwen-code/node-repl-mcp` (`packages/node-repl`, then 0.1.7). +- The bundled `computer-use` skill. +- The browser SDK `@qwen-code/browser-use` (0.25.2), included under topic 5. + +**Why upgrade** + +1. Other implementations have progressed: + - Codex `cua_repl` exposes a `js` entry, supplies documentation at runtime, automatically displays state from `getApp`, and returns void from actions. + - Upstream cua-driver reached 0.35.4, with 499 commits between the two versions. +2. RecreationBench and OSWorld-V2 exposed these problems: + - **Comparable quality, lower efficiency**: across the final 12 RecreationBench groups and 450 results, our quality and token use were comparable to official Codex and QoderCU, but elapsed time was about 1.5 times higher. Input per round was 110–130k, close to the 128k limit, making cost roughly proportional to round count. **[Measured]** + - **Too many rounds**: in the first macOS stable20 round, our generation count was 2.47 times the official implementation's, and input per generation was 2.05 times higher. On 41 tasks passed by all three implementations, calls were official 4, Qoder 12 and ours 28. The OSWorld-V2 candidate made about twice as many requests as direct mode. **[Measured]** + - **Cost of always foregrounding macOS**: commit `0ceac09` on 09-13 changed macOS input to activate B before every input, based on a misattributed Codex Windows document. Total time rose 37%; one five-action cell took 8.61 s versus official 1.61 s. **[Measured]** + +**Overall goal, decided 10-15**: complete computer-use tasks with fewer rounds, fewer tokens and clear, concise steps. + +--- + +## 1. Overall direction (10-15) + +1. Supply runtime documentation in place of skills and simplify node-repl by removing wait / cancel, following Codex. +2. Maintain one underlying service. The driver remains directly callable; the node-repl path wraps that same driver through the SDK. Upstream synchronization means porting or referencing specific fixes and capabilities rather than merging its entire codebase. +3. Exclude upstream `run_script` / `run_steps` and its upper-level interfaces. Synchronize native operation coverage, reliability and efficiency. +4. Decide foreground/background behavior separately for each platform, as described in section 2. +5. Evaluate rounds, tokens and clarity of steps. + +**Addition, 10-16**: direct driver MCP use, without node-repl, must remain fully usable and is equally important. + +--- + +## 2. Foreground / background (topic 4) + +The shared scenario is a user typing in A while the agent operates B behind A. + +### 2.1 macOS (10-15) + +**Analysis** + +- There are three ways to deliver input to B: + 1. AX accessibility actions such as press, value writes and focus changes, without foregrounding B. + 2. Process-directed events, optionally telling B that it has been activated through “synthetic focus,” while the actual system foreground remains unchanged. + 3. Global events, requiring B to become the actual foreground and affecting the user's keys and pointer. +- Codex and upstream primarily use 1 and 2; ours primarily uses 3 after `0ceac09`. +- Our implementation waits a **fixed 1 s after each action** to check for new windows (`window_change_detector.rs:152`). + +**Decisions** + +- Combine AX and process-directed events, drawing on Codex and upstream. +- Initially omit an automatic foreground/global-event route; Codex and upstream do not use it for this path. +- Verify post-action waits: if Codex completes correctly without waiting, that wait may explain the efficiency gap. +- Initially test Codex's directed drag and align paste with Codex. +- Design tests on mini using official Codex and upstream 0.35.4 as comparison targets. + +**mini smoke observations, 10-15/16**: one run per case; **[Measured]**, but preliminary. The formal batch finished and still required analysis at the time. + +- **Codex returns after dispatch without a fixed wait**: control actions took about 100–150 ms. Opening a sheet and typing without another observation succeeded 3/3. The roughly 2 s smoke delay came from an unanswered system authorization dialog and disappeared after it was handled. +- **Codex synthetic focus works**: B considers itself active and keyboard-focused while system foreground stays A. It can paste with `cmd+v` and press Return in a background sheet; upstream cannot do those two actions but explicitly refuses them. +- **Codex gaps**: + - `typeText` lost Chinese and emoji, 0/3, while reporting success. + - Coordinate right-click reached neither A nor B. + - Drag/drop and list reorder reported success without effect; two of three runs swallowed one user key. Slider drag alone worked. +- **Upstream problems**: + - Popup `setValue` worked, but text-field `setValue` did not trigger the application's change callback. + - AXPress on a text field returned -25206 without fallback. + - Each action took about 1 s. +- **Ours, forced foreground** passed the most cases but took about 1.1 s per action, foregrounded B and moved the pointer. Restoring A closed menus just opened in B. + +**Implementation details decided from this evidence** + +- Remove the fixed post-action wait. Return after dispatch and move waiting into observation, as described in topic 3. +- Explicitly refuse background drag/drop and list reorder, suggesting alternatives. Use `setValue` for sliders rather than Codex's instantaneous drag. +- Align paste with Codex: temporarily use and then restore the clipboard. +- Use directed keys for ASCII; use AX insertion at the caret plus readback for non-ASCII to avoid Codex's lost text. **[Pending]** +- Implement popup `setValue` by opening the menu and selecting the item, porting upstream behavior. +- Use AXPress for controls that support press; otherwise use a directed click with synthetic focus. +- Port upstream background protection: + - Do not send A a synthetic deactivation record (#4943). + - `key_focus_guard` (#4943 #4955 #5091 #5129). + - Restore A after B activates itself, except when the user moved the mouse or modifiers within 1 s (#4976). + - Explicitly stamp mouse modifier state (#5051). + - Save and rename with Apple Events and readback (#5057). + - Refuse typing without a focused text field using `no_focused_text_input`. + - Return `write_not_applied` when readback shows no write. +- Merge these protections with existing `app_focus.rs` synthetic focus into one observer. +- Do not implement Codex's visible cursor at this stage; no decision to include it was made. + +### 2.2 Windows (10-15/16) + +**Analysis** + +- Four routes deliver input to B: + 1. UIA semantics without foreground activation. + 2. PostMessage window messages without activation, often ignored by Chromium, WPF and Tk. + 3. Synthetic pointer injection. + 4. SendInput, which requires the actual foreground. +- Current implementations: + - **Codex** activates B, with up to 20 attempts and an Alt technique, then uses SendInput. It does not guarantee restoring A and monitors user input. + - **Upstream** prefers background and returns `background_unavailable` for known ignored combinations. + - **Our App layer** always activates B and uses SendInput, bypassing UIA Invoke even for element clicks, then synchronously restores A. It depends on a signed UIAccess worker. +- The fork already contains upstream background routes and ignored-delivery classifications; App currently forces foreground. +- Known defects: + - Foreground typing waits only 40 ms before restoring A; trailing characters can enter A. + - Text is sent as one batch without checking foreground between characters. + - **Default Windows npm installation is broken**: the distributed worker is unsigned, while the installer requires a valid signature (`install-native.mjs:157`). + +**Decision: hybrid mode** + +1. Try UIA semantics, window messages and synthetic pointer input first. **Activate B and use SendInput only after pre-dispatch `background_unavailable`, with no side effect.** Dispatched but unconfirmed input reports uncertainty and is never retried. +2. Keep verified restoration of A and preserve a user-selected C. Port #4500, waiting for B to drain input before restoring, and #4901, checking foreground per character. +3. Stop and report an error if the user acts during foreground delivery, following Codex. Do not intercept physical input. +4. Initially use upstream's elevated daemon with ordinary-user tokens for applications it launches. Retain UIAccess code; signing is being requested. +5. `typeText` always inserts at the caret; `setValue` always replaces the whole value. + +Other fixes: #3671 avoids fallback after timeout and duplicate clicks; #4774 implements native combo selection; refuse Tk clicks; set VK_LWIN flags; bound worker pipe reads. + +### 2.3 Linux (10-16) + +**Analysis** + +- X11 and Wayland are different display protocols. + - X11 has a central server and no application isolation, allowing cross-window input, screenshots and global simulation. + - Wayland delegates to the compositor, isolates applications, and requires compositor-specific automation access. +- Codex supports only X11; its 26.1007 documentation and code contain no Wayland implementation. +- Upstream has full X11 support; Wayland is disabled by default and remains limited when enabled. +- X11 has four input routes: + 1. AT-SPI semantics. + 2. Directed SendEvent plus FocusIn, used by Codex and often ignored by GTK/Chromium. + 3. A virtual second pointer through MPX + uinput, used by upstream and requiring real Xorg and uinput. + 4. Foreground XTest, currently used by ours. + +**Decisions** + +- **Remove Wayland entirely and support only X11**: + - Remove 21 fork files: `platform-linux/src/wayland/*`, `video_wayland.rs`, `wayland-helper/`, libei examples, Wayland tests and documentation. + - Pure Wayland sessions report “X11 only.” + - X11 applications under XWayland remain supported. + - Do not synchronize upstream Wayland changes. +- **Design for real shared desktops**; a test machine is only a test environment. Foreground transactions retain “restore A, preserve C.” +- **Use the Windows structure**: + 1. AT-SPI. + 2. Virtual second pointer where supported. + 3. Foreground XTest only after `background_unavailable`. + 4. Stop on user input during foreground delivery. + 5. Never retry dispatched but unconfirmed input. +- Use directed events only for Qt / Tk / Xt applications known to accept them. +- **Audit and port X11 / AT-SPI fixes after upstream 0.20.0**, checking whether the fork already contains each: + - Input: #3882, #3761, #3762, #3864, #3618, #4248, #4188 and Chinese text loss #5139 / #5158. + - MPX: #3601, #2888, #5070. + - AT-SPI: #4903, #4984, #5074, #4375, GTK combo `set_value`, #4292. + - Other: #3758, #4281, #3814, #4529, #4715, #3658, #3778. +- Repair fork defects as well: + - Send FocusIn only to Qt5, and follow Codex's FocusOut conditions. + - Restore focus after X11 activation fails. + - Fix per-window scaling identified 09-17, with 164 / 76 px offsets; double scaling in `mouse_button_up`; X11 Page_Up / Page_Down; and on-demand Chromium classification. + +--- + +## 3. Topics 1–7 (10-16) + +### Topic 1: runtime documentation replaces skills; simplify node-repl, following Codex + +- **Tools**: expose only `js` (`code`, `timeout_ms` default 30 s, `title`) and `js_reset`. `add_node_module_dir` is host configuration. Remove wait / cancel and follow Codex timeout behavior. +- **Guidance**: + - MCP instructions contain one sentence. + - The `js` description explains only the entry point. + - First execution imports the SDK, creates global `cua` and prints core documentation for the current platform only. + - After compaction, the model calls `cua.rewriteDocumentation()`. +- **Entry**: `getApp` displays full state; observations display automatically, with `emit:false` to disable display. +- Remove the bundled computer-use and browser-use `SKILL.md` files afterward. Deleting these content files is the sole exception to leaving qwen-code itself untouched. + +**Evidence**: historical wait requests used only to retrieve results occurred 1008 times and consumed 130 million tokens; Codex's five-layer guidance was measured through the MCP handshake. + +### Topic 2: driver as the only backend; SDK as a thin wrapper + +- **2a Move into the driver**: + - Application/window resolution: accept `app` directly and resolve in one round trip, replacing at least three SDK round trips per action. + - Short IDs. + - Observation rendering, character limits and diff. + - Foreground/background policy. + - Structured errors. +- The SDK retains global `cua`, typed methods, automatic display, runtime documentation and serialization per application. Generate MCP descriptions and runtime documentation from one interface definition. +- **2b Simplify tools**: expose a Codex-like set of core operations by default. Upstream recording, cursor themes, zoom, replay, administration and `browser_*` stay disabled by default and are opt-in. +- **2c** MCP tool names and parameters correspond one-to-one to JS methods. + +### Topic 3: observation + +- Split into `getAXState / getScreenshot / getAXStateAndScreenshot`, following Codex. Return a string, remove `.elements` and display automatically. +- Do not capture screenshots for text-only observations. Remove the SDK's fixed `includeScreenshot:true`; Linux may save 1.6–2 s per observation. **[Inference from the 09-18 diagnosis]** +- Return actions after dispatch. Wait for stability during observation, initially 300–500 ms, with the final limit based on tests. +- First `getApp` and window changes return full AX without a screenshot; later observations default to diff. **All three platforms use diff**, intentionally differing from Codex's full Linux/Windows observations. +- Also repair per-window screenshot mapping, avoid recapture after incomplete AX (4.4–5.3 s in Chrome), match image MIME to bytes, and wake Chromium AT-SPI before its first tree read (#5074). + +### Topic 4: action results and errors + +- Success returns `void`, without `effect`. Confirm the result through observation within the same code invocation. +- Failures throw `{code, reason, sent: true/false, next-step advice}` from the driver, preserved by SDK and direct MCP. Initial codes: + +| Code | Meaning | +| -------------------------------------- | -------------------------------------------- | +| `element_stale` | ID from an older snapshot; no input sent | +| `not_settable` | Control cannot be written directly | +| `write_not_applied` | Readback did not change | +| `no_focused_text_input` | Focus is outside a text field | +| `target_occluded` | Target point is covered | +| `unsupported_on_platform` | Operation unsupported on this platform | +| `user_input_detected` | Foreground delivery stopped on user activity | +| `foreground_unavailable` | Foreground activation failed; no input sent | +| `app_not_running` / `window_not_found` | Application not running / no window found | + +- Do not expose `background_unavailable` to the model; the driver handles its permitted foreground fallback. +- IDs are valid only for the selected window's latest snapshot. Refuse old IDs before dispatch. +- Sent but unconfirmed input does not throw: return normally with “Sent but not confirmed; observe before retrying.” + +**Evidence**: in macOS smoke tests, three distinct failures were rewritten by the SDK into the same generic message, forcing another observation to determine the cause. + +### Topic 5: capability gaps + +- **5a Browser**: reuse and migrate existing `@qwen-code/browser-use`. It already runs in node-repl, using Playwright over CDP through the Qwen Chrome extension to the user's Chrome, with void actions and coded errors. Expose `getBrowser / getTab / createBrowserTab` under `cua`, **without functional changes or upstream browser code**. Do not add Windows or diff support yet. +- **5b** Add explicit `cua.desktop`: full-display screenshots and screen-coordinate input. This naturally uses foreground and is selected only for explicit desktop operations. +- **5c** Add click modifiers, `click(p, {key})`, and `key_down / key_up`. +- **5d** Do not add per-application authorization. + +### Topic 6: distribution and versions + +- **6a** Leave qwen-code cli/core unchanged; node-repl remains separately registered MCP. +- **6b** Give SDK, native packages and browser SDK one version. Stop following upstream version numbers; record “ported from upstream #xxxx” in the changelog. +- **6c** Download native payloads on demand and verify SHA256 using the existing mechanism. + +### Topic 7: evaluation + +- Record tool and model evaluation separately; the user moved OSWorld evaluation after release on 2026-10-20: + - **Layer 1, tools without models**: compare the candidate, Codex and upstream on the same cases for correctness, action latency, interference and success-without-effect. The driver merge gate is at least the better comparator. The user clarified on 2026-10-20 that the observed transient macOS target/menu cover is acceptable and expected; record it and recovery separately rather than applying a strict zero-cover blocker. This does not accept wrong-target input, changed click semantics or replay after uncertain delivery. + - **Layer 2, small model/OSWorld evaluation after release**: primary metrics are **rounds, tokens and wall-clock time**, provided success rate does not regress. This is post-release follow-up rather than a pre-release gate; no improvement or nonregression result is claimed before execution. +- **Tasks**: + - macOS and Windows use 20 OSWorld-V2 tasks selected in two AP groups (`sh-prod-1`, Qoder + Node REPL scaffold, `azure.gpt-6-astra`): + - `group-5cff67aa55934a09a45160e3194b5801-s1`: macOS, mean 0.26, one full-score task. + - `group-bc7a3539903b4735bc3508e04e5791d5-s1`: Windows, mean 0.12, zero full-score tasks. + - Both use task_v2_001, 010, 020, 022, 027, 038, 044, 046, 051, 052, 055, 059, 060, 067, 076, 080, 088, 096, 101, 107. + - Linux initially uses the same 20 IDs, replacing missing/inapplicable tasks from the Linux set. **[Pending verification]** +- **Rules**: change one variable at a time; record infrastructure failures separately and exclude/rerun them; missing usage yields lower bounds only; reuse existing comparator trajectories. + +--- + +## 4. Implementation checklist by package + +**cua-driver, Rust** + +- Platform dispatch: macOS section 2.1, Windows hybrid section 2.2 and Linux X11 hybrid section 2.3. The driver decides; SDK no longer forces foreground. +- Remove Wayland and reject pure Wayland sessions. +- Move app/window resolution, short IDs, rendering/diff on all three platforms, waiting and structured errors into the driver. +- Simplify default MCP tools; align tools and JS methods; generate descriptions and documentation from one interface. +- Add desktop operations, click modifiers and `key_down / key_up`. +- Port macOS protection, Windows #4500 #4901 #3671 #4774 and the Linux X11 / AT-SPI list. +- Use the elevated Windows daemon topology; installation no longer depends on signing. + +**cua-sdk, TS** + +- Thin wrapper: global `cua`, typed methods, automatic display with `emit:false`, platform documentation, `rewriteDocumentation()` and per-app serialization. +- Remove App business logic: forced foreground, generic error rewriting and fixed screenshots. +- Wrap browser-use through `cua.getBrowser / getTab / createBrowserTab`. + +**node-repl-mcp** + +- `js` + `js_reset`, default 30 s; remove wait / cancel; host-only `add_node_module_dir`. +- Preload `cua` and documentation in the startup banner. + +**browser-use**: reference and migrate only, without functional changes. + +**qwen-code itself**: delete only the two bundled computer-use/browser-use `SKILL.md` files; no code changes. + +**Versions**: one shared version and changelog port references. + +--- + +## 5. Test environment snapshot (10-16) + +| Platform | Machine | Status at that time | +| -------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| macOS | mini, 30.221.187.223, `/Users/tutu/cua-fgbg-20261015` | Five harness problems repaired. Formal three-implementation × three-run batch, covering fixture cases and TextEdit / Excel / Finder / Chrome, finished pending analysis. Upstream T13 / T14 / T16 awaited Automation approval; T15 lacked LibreOffice. | +| Windows | windows-ecs-new, 8.217.1.105, `C:\cua-fgbg-20261016` | Upstream 0.35.4 and ours 0.20.11 prepared; new test certificate `83923F67…AB98`, valid through 2027-01-14, with signed/timestamped worker. SentinelA / UserTyper / Sampler compiled. Test processes still needed interactive desktop session 2; the borrowed-token launcher failed. An explicit choice remained between a one-off `schtasks /it` task and a runner started by the user in remote desktop. | +| Linux | linux-ecs, 47.76.93.28 | Dedicated `cua-fgbg-osworld-20261016` container using OSWorld-V2, QEMU software emulation and ports 127.0.0.1:50916 / 58916; existing containers preserved. Harness pending. | + +AP at that time: mini ap-client upgraded from ArtLab to 0.5.1 and passed its version gate. The two group exports were reported at `/Users/tutu/cua-fgbg-20261015/ap-groups/`. + +--- + +## 6. Corrections made during discussion + +- “Codex waits conditionally for about 2 s after actions” was incorrect. An unanswered authorization dialog delayed screenshot capture until timeout; the formal batch had no such wait. +- The “official: input methods activate the target window” quotation used for `0ceac09` came from Codex's **Windows** SDK, not macOS. +- The launcher borrowing an explorer token still produced session-0 children. Fixture results from that session were invalidated. + +--- + +## 7. Document index + +| Document | Contents | +| -------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| `BRIEF.md` | Research brief | +| `findings/code-architecture.md` | Existing architecture | +| `findings/codex-latest-delta.md` | Codex 26.924 → 26.1007 changes | +| `findings/upstream-delta.md` | Upstream 0.20 → 0.35.4 delta and synchronization simulation | +| `findings/design-discussions.md` | Historical decision ledger, including conflicting 09-17 / 09-21 decisions | +| `findings/fgbg-mechanics.md` | Platform foreground/background mechanisms at code level | +| `findings/fgbg-evidence.md` | Experiments and decision evidence | +| `findings/rb-final-comparison.md`, `rb-sdk-rounds.md` | RecreationBench comparison and round analysis | +| `findings/osworld-stable20.md` | OSWorld-V2 and macOS stable20 evidence | +| `topic4-fgbg/codex-fgbg-2026-09-29-reconstructed.md` | Reconstructed 09-29 Codex platform analysis | +| `topic4-fgbg/macos.md`, `macos-test-plan.md`, `macos-shakeout-20261015.md` | macOS analysis, plan and smoke observations | +| `topic4-fgbg/windows.md`, `windows-test-plan.md` | Windows analysis, decisions and plan | +| `topic4-fgbg/linux.md` | Linux analysis and decisions | + +## 8. Current execution scope (2026-10-19) + +The user prioritizes the first four verification items: macOS, Windows and Linux X11 native acceptance, followed by runtime, installation and package checks. OSWorld evaluation is scheduled after release by the user’s 2026-10-20 decision; future model runs use `gpt-6-sol`. On 2026-10-20 the user requested a fresh same-mini Codex tool comparison and clarified acceptance of observed transient macOS cover. All remaining native GUI tests run on mini. The 10-16 environment table above is historical. Current evidence and candidate identities are recorded in the [acceptance ledger](cua-upgrade-acceptance-2026-10.md#acceptance-clarification-2026-10-20). The user now authorizes pushing this feature branch and submitting its PR to main. OSWorld results are not a release prerequisite; production release execution is a separate action and no benchmark result is marked passed. diff --git a/docs/design/cua-upgrade-2026-10.zh-CN.md b/docs/design/cua-upgrade-2026-10.zh-CN.md new file mode 100644 index 00000000000..1fb3c6e7534 --- /dev/null +++ b/docs/design/cua-upgrade-2026-10.zh-CN.md @@ -0,0 +1,347 @@ +# qwen-code Computer Use 升级方案与讨论记录（2026-10-15 ~ 10-16） + +[English](cua-upgrade-2026-10.md) | [简体中文](cua-upgrade-2026-10.zh-CN.md) + +本文汇总这两天讨论的背景、分析过程和全部决定，供后续实现和查阅。 + +- 每条决定都标了日期。细节和证据见文末"文档索引"里的分题文档。 +- 标注约定：**【实测】** 有测试或数据支撑；**【推断】** 是根据代码或现象做的判断；**【待测】** 需要后续测试确认。 + +--- + +## 0. 背景 + +**范围** + +- `packages/cua-driver`：Rust driver。从 trycua/cua 的 `cua-driver-rs-v0.20.0` fork 而来，以 qwen-cua-driver / QwenCuaDriver.app 发布。 +- `@qwen-code/cua-sdk`（`packages/cua-driver/typescript`，当前 0.20.11）。 +- `@qwen-code/node-repl-mcp`（`packages/node-repl`，当前 0.1.7）。 +- bundled `computer-use` skill。 +- 浏览器 SDK `@qwen-code/browser-use`（0.25.2），在议题 5 里纳入。 + +**为什么升级** + +1. 市面上的方案有了新进展： + - Codex 的 `cua_repl`：模型只看到 `js` 工具，文档在运行时下发，`getApp` 自动输出状态，动作返回 void； + - 上游 cua-driver 到了 0.35.4，两个版本之间有 499 个提交。 +2. RecreationBench 和 OSWorld-V2 实验暴露的问题： + - **质量接近，效率差**：RecreationBench 最终 12 组、450 个结果里，我们和 Codex 官方、QoderCU 质量相当，token 接近，但用时约 1.5 倍；单轮输入在 110–130k，接近 128k 上限，所以成本几乎和轮次成正比【实测】。 + - **轮次多**：macOS stable20 第一轮里，我们的生成次数是官方的 2.47 倍，单次输入是 2.05 倍【实测】。三方都通过的 41 题，调用数分别是官方 4、Qoder 12、我们 28【实测】。OSWorld-V2 的候选方案请求数约为 direct 的 2 倍【实测】。 + - **macOS 一律前台的代价**：09-13 的提交 `0ceac09` 把 macOS 输入改成"一律先激活 B 再输入"，依据是一句被误认的 Codex Windows 文档。改后总时间 +37%；一个 cell 里 5 个动作，我们 8.61 s，官方 1.61 s【实测】。 + +**总目标（10-15 定）**：用尽可能少的轮次、少的 token、清晰简洁的步骤完成 computer-use 任务。 + +--- + +## 1. 总体方向（10-15 定） + +1. 和 Codex 一样：运行时下发文档，替代 skill；简化 node-repl 工具（去掉 wait / cancel）。 +2. 只维护一套底层服务：driver 始终可以作为底层工具直接调用；node-repl 路径就是 SDK 包装同一个 driver。和上游的同步从"代码合并"改为"移植或引用具体的修复和功能"。 +3. 不跟进上游的 `run_script` / `run_steps` 和它的上层调用接口。同步只看底层操作的丰富度、稳定性和效率。 +4. 前台/后台按平台单独讨论（见第 2 节）。 +5. 评测目标：少轮次、少 token、步骤清晰。 + +**补充（10-16）**：driver 作为不经过 node-repl、直接调用的 MCP 工具，也必须完整可用。这条路径同样重要。 + +--- + +## 2. 前台 / 后台（议题 4） + +场景统一为：用户在 A 里打字，Agent 要操作被 A 挡住的 B。 + +### 2.1 macOS（10-15 定） + +**分析** + +- 把输入送到 B 只有三条路： + 1. AX 辅助功能动作：按下、写值、设焦点，不需要 B 在前台； + 2. 定向事件：事件指定投给 B 的进程，必要时先单独告诉 B"你已被激活"（下称"合成焦点"），系统真实前台不变； + 3. 全局事件：B 必须先成为真前台，用户的按键和指针都会被影响。 +- Codex 和上游都以 1、2 为主，我们反而以 3 为主（`0ceac09` 之后）。 +- 我们每个动作后**固定等 1 s**，看有没有新窗口出现（`window_change_detector.rs:152`）。 + +**决定** + +- 设计"AX + 定向事件"相结合的方案，借鉴 Codex 和上游。 +- **前台（全局事件）路径先不出现**，Codex 和上游都不用它。 +- 核实动作后的等待："如果 Codex 没有任何等待也能正确完成任务，这个等待可能就是影响效率的点"。 +- 拖动先试 Codex 的定向拖动；paste 对齐 Codex。 +- 设计测试任务，用 Codex 官方和上游 0.35.4 的结果作为预期，在 mini 上验证。 + +**mini 冒烟的观察**（10-15/16，每个用例 1 次，【实测】但只是预备数据；正式批次已跑完，结果待整理） + +- **Codex 动作发出就返回，没有固定等待**：控件动作约 100–150 ms；点开 sheet 后不观察、直接在 sheet 里输入，3/3 成功。冒烟时看到的约 2 s 等待，是 Codex 服务的系统授权弹窗没人处理造成的，弹窗点掉后就消失了。 +- **Codex 的合成焦点有效**：B 自认为激活、成为键盘焦点，系统前台仍是 A。靠这一步，它能在后台 `cmd+v` 粘贴、在 sheet 里按回车；上游做不到这两件事，但会明确拒绝，不假装成功。 +- **Codex 的缺口**： + - `typeText` 丢中文和 emoji（0/3），接口照样返回成功； + - 坐标右键 A 和 B 都收不到； + - 拖放和列表重排返回成功但没生效，而且 3 次里 2 次吞掉了用户的一个按键；只有拖滑块成功。 +- **上游的问题**： + - 弹出菜单的 setValue 能用，但文本框 setValue 不触发应用自己的变更回调； + - AXPress 对文本框报错 -25206，而且不回退； + - 每个动作约 1 s。 +- **我们（强制前台）**：通过的用例最多，但每个动作约 1.1 s，B 被切到前台、指针被移动；还 A 时会把 B 刚打开的菜单关掉。 + +**据此确定的实现细则**（属于已定方向内的细化） + +- 去掉动作后的固定等待，动作发出即返回；等待放进观察（见议题 3）。 +- 拖放和列表重排在后台**明确拒绝**，并提示替代路径；滑块改用 setValue。不用 Codex 的瞬间拖动。 +- paste 对齐 Codex：临时使用剪贴板，事后恢复。 +- 输入：ASCII 走定向按键；非 ASCII 走 AX 在光标处插入并读回（避开 Codex 丢字）【待测】。 +- 弹出菜单的 setValue 改为"打开菜单并选中该项"（移植上游）。 +- 点击：支持"按下"动作的控件用 AXPress；其余用定向点击加合成焦点。 +- 移植上游的后台保护： + - 不给 A 发失焦记录（#4943）； + - key_focus_guard（#4943 #4955 #5091 #5129）； + - B 自激活后切回 A，用户 1 s 内动过鼠标或修饰键就不切（#4976）； + - 鼠标事件显式写修饰键状态（#5051）； + - 保存和改名用 Apple Events 并读回验证（#5057）； + - 焦点不在文本框时拒绝打字 `no_focused_text_input`； + - 写入未生效报 `write_not_applied`。 + + 这些保护要和现有的 `app_focus.rs` 合成焦点合并成一个观察者。 + +- Codex 的可见光标：先不做（未表态即按不做处理）。 + +### 2.2 Windows（10-15/16 定） + +**分析** + +- 把输入送到 B 有四条路： + 1. UIA 语义操作，不需要前台； + 2. 窗口消息 PostMessage，不需要前台，但 Chromium、WPF、Tk 等经常不认； + 3. 合成指针注入； + 4. SendInput，必须是真前台。 +- 三方现状： + - **Codex**：先激活 B（最多 20 轮，带 Alt 技巧），再用 SendInput；不保证还 A；有用户输入监测。 + - **上游**：后台优先，已知会被丢弃的组合直接返回 `background_unavailable`。 + - **我们 App**：一律激活 B + SendInput（连元素点击都跳过 UIA Invoke），再同步还 A，依赖签名的 UIAccess worker。 +- 我们 fork 其实已经带着上游的后台实现（丢弃判定表等），只是 App 层强制成了前台。 +- 已知缺陷： + - 前台打字只等 40 ms 就还 A，末尾的字可能进 A； + - 文本一次发完，中途不检查前台； + - **Windows 默认 npm 安装是坏的**：发布的 worker 未签名，安装器却要求签名有效（`install-native.mjs:157`）。 + +**决定：混合模式** + +1. 先走上游式后台：UIA 语义 → 窗口消息 → 合成指针。**只有返回 `background_unavailable`（派发前拒绝，还没有副作用）时，才激活 B 并用 SendInput**。已派发但效果未确认的，报"未确认"，不重试。 +2. 还 A：保留"读回确认，用户已切到 C 就不拉回"，补上 #4500（等 B 读完输入再还）和 #4901（逐字检查前台）。 +3. 前台段期间用户在操作：与 Codex 一致，检测到就停下报错，不拦截物理输入。 +4. 权限：先改用上游的"高权限 daemon + 它启动的应用用普通用户 token"。UIAccess 代码保留，签名在申请中。 +5. 语义：`typeText` 一律在光标处插入；`setValue` 一律整体替换。 + +其余小修：#3671（超时不回退，避免双击）、#4774（combo box 真实选择）、Tk 点击拒绝、VK_LWIN 标志、worker 管道读取加超时。 + +### 2.3 Linux（10-16 定） + +**分析** + +- X11 和 Wayland 是两种显示协议。 + - X11 有中心的 X server，应用之间不隔离，可以给别的窗口发事件、截屏、模拟全局输入。 + - Wayland 由合成器统一管理，应用相互隔离，自动化只能靠各家合成器开的口子。 +- Codex：只做 X11（26.1007 文档和代码里没有 Wayland）。 +- 上游：X11 完整支持；Wayland 默认关闭，打开后也很受限。 +- X11 上把输入送到 B 有四条路： + 1. AT-SPI 语义操作； + 2. 定向事件 SendEvent 加 FocusIn（Codex 的做法；GTK、Chromium 常常不认）； + 3. 虚拟第二指针 MPX + uinput（上游的做法，需要真实 Xorg 和 uinput）； + 4. 前台 XTest（我们现在的做法）。 + +**决定** + +- **完全放弃 Wayland，只支持 X11**： + - 删除 fork 的 Wayland 后端，共 21 个文件（`platform-linux/src/wayland/*`、`video_wayland.rs`、`wayland-helper/`、libei 示例、Wayland 测试和文档）； + - 纯 Wayland 会话直接报"只支持 X11"； + - XWayland 下的 X11 应用照常支持； + - 不再同步上游的 Wayland 提交。 +- **按真实使用场景设计**：要考虑共享桌面，测试机只是测试环境。前台段保留"还 A + 检查 C"。 +- **与 Windows 同一结构**： + 1. AT-SPI； + 2. 虚拟第二指针（机器支持时）； + 3. 只有返回 `background_unavailable` 时才转前台 XTest； + 4. 前台段期间检测到用户输入就停下； + 5. 已派发未确认的不重试。 + + 定向事件只用于已知能认的 Qt / Tk / Xt 应用。 + +- **同步上游 0.20.0 之后的 X11 / AT-SPI 修复**（逐个核对 fork 是否已有）： + - 输入：#3882、#3761、#3762、#3864、#3618、#4248、#4188、中文丢字 #5139 / #5158； + - MPX：#3601、#2888、#5070； + - AT-SPI：#4903、#4984、#5074、#4375、GTK combo set_value、#4292； + - 其他：#3758、#4281、#3814、#4529、#4715、#3658、#3778。 +- fork 自身的缺陷一起修： + - FocusIn 只对 Qt5 发，FocusOut 按 Codex 的条件发； + - X11 激活失败后还原焦点； + - 09-17 发现的坐标缩放按窗口区分（164 / 76 px 偏移）、`mouse_button_up` 重复缩放、X11 的 Page_Up / Page_Down、Chromium 进程分类按需执行。 + +--- + +## 3. 总议题 1–7（10-16 定） + +### 议题 1：运行时文档替代 skill + 简化 node-repl —— 全部与 Codex 一致 + +- **工具**：模型只看到 `js`（`code`、`timeout_ms` 默认 30 s、`title`）和 `js_reset`。`add_node_module_dir` 只由宿主配置。删除 wait / cancel，超时行为照 Codex。 +- **引导**： + - MCP instructions 只有一句话； + - `js` 工具描述只讲入口； + - 第一次执行时自动 import SDK、建好全局 `cua`，输出核心文档（只输出当前平台的部分）； + - 上下文压缩后由模型调 `cua.rewriteDocumentation()`。 +- **入口**：`getApp` 自动输出状态；观察方法默认自动显示，`emit:false` 关闭。 +- 改完之后，从 bundled skill 里删掉 computer-use 和 browser-use 两个 `SKILL.md`。这是"不动 qwen-code 本体"的唯一例外，只删内容文件。 + +**依据**：历史上"只为拿结果"的 wait 请求有 1008 次、1.3 亿 token；Codex 的五层引导经 MCP 握手实测。 + +### 议题 2：driver 是唯一底层，SDK 只做包装 + +- **2a 下沉到 driver**： + - 应用和窗口的解析：动作直接接受 `app`，一次往返完成；现在 SDK 每个动作至少 3 次往返； + - 短 ID； + - 观察的渲染、字符上限和 diff； + - 前后台策略； + - 结构化错误。 + + SDK 只保留全局 `cua`、类型化方法、自动显示、运行时文档和同一应用的串行。一份接口说明同时生成 MCP 工具描述和运行时文档。 + +- **2b 精简工具面**：默认只暴露类似 Codex 的核心十几个工具。上游的大量工具（录屏、光标主题、zoom、轨迹回放、管理类、`browser_*`）默认关闭，按需打开。 +- **2c** MCP 工具名和参数与 JS 方法一一对应。 + +### 议题 3：观察 + +- 照 Codex 拆成 `getAXState / getScreenshot / getAXStateAndScreenshot`，返回字符串，去掉 `.elements`，默认自动显示。 +- 只要文本时不截图：去掉 SDK 固定传的 `includeScreenshot:true`。Linux 上每次观察能省 1.6–2 s【推断，依据 09-18 定位】。 +- 动作发出即返回；观察时等界面稳定再采集，上限按测试数据定，先从 300–500 ms 起步。 +- 首次 `getApp` 和切换窗口时给全量，只输出 AX、不带截图；之后默认 diff。**三平台都用 diff**（有意不跟 Codex：Codex 在 Linux / Windows 上始终全量）。 +- 一并修复： + - 截图坐标映射按窗口区分； + - AX 不完整时不自动重截（Chrome 上 4.4–5.3 s）； + - 图片 MIME 声明与实际一致； + - Chromium 的 AT-SPI 首次读树前先唤醒（#5074）。 + +### 议题 4：动作返回值和错误 + +- 成功返回 `void`，去掉 `effect`。结果靠同一次调用里的观察确认。 +- 失败抛出 `{code, reason, sent: true/false, 下一步建议}`。driver 给出，SDK 原样透传，MCP 路径字段相同。初始错误码： + + | 错误码 | 含义 | + | -------------------------------------- | ------------------------------ | + | `element_stale` | 编号来自旧快照，没发送 | + | `not_settable` | 该控件不能直接写值 | + | `write_not_applied` | 写入后读回没变 | + | `no_focused_text_input` | 焦点不在文本框 | + | `target_occluded` | 点击点被挡住 | + | `unsupported_on_platform` | 本平台不支持该操作 | + | `user_input_detected` | 前台段检测到用户在操作，已停下 | + | `foreground_unavailable` | 转前台失败，没发送 | + | `app_not_running` / `window_not_found` | 应用没运行 / 找不到窗口 | + + `background_unavailable` 不暴露给模型（driver 会自动转前台）。 + +- 元素编号只在该窗口最新一次快照内有效，用旧编号在发送前报错。 +- "已发送但确认不了"不抛异常：动作照常返回，输出里加一行"已发送但未确认生效，重试前先看状态"。 + +**依据**：mac 冒烟里三种不同的失败被 SDK 改写成同一句通用文案，模型只能再观察一轮去猜。 + +### 议题 5：能力缺口 + +- **5a 浏览器**：直接引用、迁移仓库里的 `@qwen-code/browser-use`。它已经跑在 node-repl 里：Playwright 经 CDP 连到 Qwen Chrome 扩展，再到用户的 Chrome；动作返回 void，错误带错误码。在 `cua` 下以 `getBrowser / getTab / createBrowserTab` 接入，**不做功能改动**，不用上游 cua-driver 的浏览器内容。暂不补 Windows，暂不加 diff。 +- **5b** 加显式的 `cua.desktop`：全屏截图和按屏幕坐标的输入。天然走前台，只在明确操作桌面时使用。 +- **5c** 补"点击时按住修饰键"（`click(p, {key})`）和 `key_down / key_up`。 +- **5d** 不做按应用授权。 + +### 议题 6：分发与版本 + +- **6a** 这次不动 qwen-code 本体（cli/core）。node-repl 仍作为单独注册的 MCP。 +- **6b** SDK、原生包、浏览器 SDK 统一一个版本号。driver 不再跟上游版本号，changelog 里写"移植自上游 #xxxx"。 +- **6c** 原生包按需下载，校验 sha256（沿用现有机制）。 + +### 议题 7：评测 + +- 工具层与模型层分别记录；用户于 2026-10-20 将 OSWorld 评测安排到发布后： + - **第 1 层：工具层，不用模型**。同一套用例对比新方案、Codex 和上游，看正确性、每个动作耗时、打扰项、返回成功但没生效的次数。作为 driver 改动的合入门槛：至少达到 Codex 和上游中较好的那个。用户于 2026-10-20 澄清，已观察到的 macOS 目标窗口/菜单短暂遮挡可接受且符合预期；逐项记录遮挡与恢复，不再以严格零遮挡阻塞。错误目标输入、改变点击语义或不确定后重放仍不被接受。 + - **第 2 层：发布后的小规模模型/OSWorld 评测**。主指标是**轮次、token、墙钟时间**，前提是成功率不降。此项为发布后跟进，不再作为发布前门槛；实际执行前不声称性能提升或成功率不回退。 +- **任务集**： + - mac 和 Windows 用 AP 上两个 group 筛过的 OSWorld-V2 20 题（`sh-prod-1`，scaffold 为 Qoder + Node REPL，模型 `azure.gpt-6-astra`）： + - `group-5cff67aa55934a09a45160e3194b5801-s1`：macOS，平均 0.26，满分 1 题； + - `group-bc7a3539903b4735bc3508e04e5791d5-s1`：Windows，平均 0.12，满分 0 题。 + - 两组是同一组题号：task_v2_001、010、020、022、027、038、044、046、051、052、055、059、060、067、076、080、088、096、101、107。 + - Linux 计划用同样这 20 个题号，缺的或不适用的从 Linux 全集补齐【待核实】。 +- **规则**： + - 每次只改一个变量； + - 基础设施失败单独记录、剔除重跑； + - usage 缺失的轮次只算下限； + - 对照组复用已有轨迹。 + +--- + +## 4. 实施清单（按包） + +**cua-driver（Rust）** + +- 前后台：macOS（2.1）、Windows 混合（2.2）、Linux X11 混合（2.3）。driver 自动决策，不再接受 SDK 强制前台。 +- 删除 Wayland 后端；纯 Wayland 会话直接报错。 +- 下沉：应用 / 窗口解析、短 ID、观察渲染与 diff（三平台）、等待逻辑（动作即返回，观察里等）、结构化错误码。 +- 精简默认 MCP 工具面；工具和 JS 方法一一对应；一份接口说明生成工具描述和运行时文档。 +- 新增：`desktop` 操作、点击按住修饰键、`key_down / key_up`。 +- 移植上游修复：mac 保护、Windows 修复（#4500 #4901 #3671 #4774 等）、Linux X11 / AT-SPI 修复清单。 +- Windows 改用高权限 daemon 形态；安装不再依赖签名。 + +**cua-sdk（TS）** + +- 变成薄包装：全局 `cua`、类型化方法、自动显示（`emit:false`）、运行时文档（按平台）、`rewriteDocumentation()`、同一应用串行。 +- 删除 App 层的业务逻辑：强制前台、通用错误改写、固定截图。 +- `cua.getBrowser / getTab / createBrowserTab` 包装 browser-use。 + +**node-repl-mcp** + +- 工具改为 `js` + `js_reset`，默认 30 s，删 wait / cancel；`add_node_module_dir` 只由宿主配置。 +- 启动 banner 预载 `cua` 并输出文档。 + +**browser-use**：只做引用和迁移，不改功能。 + +**qwen-code 本体**：只删两个 bundled `SKILL.md`（computer-use、browser-use），代码不动。 + +**版本**：统一版本号；changelog 记录移植来源。 + +--- + +## 5. 测试环境现状（截至 10-16） + +| 平台 | 机器 | 状态 | +| ------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| macOS | mini（30.221.187.223），`/Users/tutu/cua-fgbg-20261015` | harness 已修好 5 个问题。正式批次（三方 × 3 次，fixture 用例加 TextEdit / Excel / Finder / Chrome）**已跑完，结果待整理**。上游的 T13 / T14 / T16 需要人工点 Automation 授权，未跑；T15 因 mini 没装 LibreOffice 未跑 | +| Windows | windows-ecs-new（8.217.1.105），`C:\cua-fgbg-20261016` | 上游 0.35.4 和我们 0.20.11 已就绪：换了新的测试签名证书 `83923F67…AB98`，有效到 2027-01-14，worker 已签名并带时间戳。SentinelA / UserTyper / Sampler 已编译。**卡点：测试进程要在交互桌面 session 2 里运行**，借令牌的启动器没跑通，需要选一种方式（一次性计划任务 `schtasks /it`，或用户在远程桌面启动常驻执行器） | +| Linux | linux-ecs（47.76.93.28） | 新起了容器 `cua-fgbg-osworld-20261016`（OSWorld-V2 镜像，QEMU 软件模拟，端口 127.0.0.1:50916 / 58916），不影响已有容器。harness 待搭 | + +AP：mini 上的 ap-client 已从 ArtLab 升级到 0.5.1，版本检查通过；两个 group 的导出在 mini 的 `/Users/tutu/cua-fgbg-20261015/ap-groups/`。 + +--- + +## 6. 讨论过程中的更正 + +- "Codex 动作后有约 2 s 的条件等待"是错的：冒烟时 Codex 服务的授权弹窗没人处理，截图卡到超时。正式批次里没有这个等待。 +- `0ceac09` 改成一律前台时引用的"官方：输入方法自行激活目标窗口"，出自 Codex 的 **Windows** SDK，不是 macOS。 +- Windows 上借 explorer 令牌的启动器，实测子进程仍在 session 0，之前在那上面跑出的 fixture 结果作废。 + +--- + +## 7. 文档索引 + +| 文档 | 内容 | +| -------------------------------------------------------------------------- | -------------------------------------------------- | +| `BRIEF.md` | 调研任务说明 | +| `findings/code-architecture.md` | 现有代码结构 | +| `findings/codex-latest-delta.md` | Codex 26.924 → 26.1007 的变化 | +| `findings/upstream-delta.md` | 上游 0.20 → 0.35.4 的差异与同步模拟 | +| `findings/design-discussions.md` | 历史设计讨论账本（09-17 / 09-21 两次决定的冲突等） | +| `findings/fgbg-mechanics.md` | 三平台前后台机制（代码级） | +| `findings/fgbg-evidence.md` | 前后台相关的实验与决策证据 | +| `findings/rb-final-comparison.md`、`rb-sdk-rounds.md` | RecreationBench 对比与轮次分析 | +| `findings/osworld-stable20.md` | OSWorld-V2 与 macOS stable20 证据 | +| `topic4-fgbg/codex-fgbg-2026-09-29-reconstructed.md` | 09-29 Codex 三平台前后台逆向（重建版） | +| `topic4-fgbg/macos.md`、`macos-test-plan.md`、`macos-shakeout-20261015.md` | macOS 分析、测试方案、冒烟记录 | +| `topic4-fgbg/windows.md`、`windows-test-plan.md` | Windows 分析与决定、测试方案 | +| `topic4-fgbg/linux.md` | Linux 分析与决定 | + +## 8. 当前执行范围（2026-10-19） + +用户现阶段优先前四项验证：macOS、Windows、Linux X11 真机验收，以及运行时、安装与打包检查。用户于 2026-10-20 明确将 OSWorld 评测放在发布后，未来模型运行改用 `gpt-6-sol`。用户于 2026-10-20 要求重新进行同机 Codex 工具对照，并澄清已观察的 macOS 短暂遮挡可接受；剩余真机 GUI 测试均在 mini 执行。上方 10-16 环境表为历史快照。当前证据与候选身份见[验收账本](cua-upgrade-acceptance-2026-10.zh-CN.md#验收口径澄清2026-10-20)。用户现已授权推送此功能分支并向 main 提交本次 PR。OSWorld 结果不再是发布前置条件；正式发布执行为独立动作，未将任何评测结果标为通过。 diff --git a/docs/design/cua-upgrade-acceptance-2026-10.md b/docs/design/cua-upgrade-acceptance-2026-10.md new file mode 100644 index 00000000000..c5e56367b79 --- /dev/null +++ b/docs/design/cua-upgrade-acceptance-2026-10.md @@ -0,0 +1,229 @@ +# Computer Use upgrade acceptance record (2026-10-16) + +[English](cua-upgrade-acceptance-2026-10.md) | [简体中文](cua-upgrade-acceptance-2026-10.zh-CN.md) + +Implementation follows the [approved decisions](cua-upgrade-2026-10.md) and [platform port ledger](cua-upgrade-platform-ports.md). Work is on `lazzy/cua-upgrade-2026-10`, based on `a8eb74fdf5b60851e1fc4a8e258e043273df4411`. Version `0.26.0` is the local development target and has not been released. At the F31 native checkpoint, no push, PR creation, shared-branch merge or release workflow had occurred; isolated local merge-validation commits are identified below. The subsequent PR submission decision is recorded next. + +## PR submission and evaluation schedule (2026-10-20) + +The user authorized this feature PR and explicitly moved OSWorld evaluation after release. The native/runtime acceptance facts below retain their candidate identities and limitations; absent benchmark results are not marked passed. The release audit also separates the three Computer Use npm packages from CLI version bumps and publication, while preserving their shared 0.26.0 version and dependency publication order. Production release execution has not been requested in this submission step. + +## Acceptance clarification (2026-10-20) + +The user confirmed mini unlocked, clarified that the previously observed transient macOS target/menu cover is acceptable and expected, and requested a fresh Codex comparison. Strict zero-cover is no longer a blocker. Historical strict-oracle failures below remain factual measurements of cover; they are not evidence that cover disappeared or that macOS makes prevention impossible. Functional correctness, exact target ownership, requested modifier semantics, uncertainty/no replay and recovery remain separate checks. + +All remaining native GUI tests and the new comparison run on mini. The earlier local Writer preflight sent zero document actions, and the subsequently abandoned local comparator preparation sent zero Codex actions and was cleaned. Local work covers source, compilation, packages and merge validation. OSWorld/model evaluation remains deferred. The immutable clarification and new run reports are kept in the worktree's ignored evidence directories. + +The adopted F31 repair passes two fresh mini Writer background workflows: the original observer reads the exact seed body unfocused before one public App click and focused afterward; one input changes the live text without changing the file; one toolbar Save persists exactly that text in valid FODT XML. The prior v27 negative and rejected F28/F30 experiments remain separate. F31 uses existing native focus preparation and a read-only caret query before the existing AXFocused setter; the public acknowledgement stays conservative and no uncertain action is replayed. Its native tree is `6abc23df3e6066a73f5110bfe77c041def4cc90852b5c5d6bde9ae537033eec3` (453 native plus 12 support inputs). Each successful Writer case has one document window; existing ambiguous-keyboard-destination protection remains and these cases do not prove arbitrary multi-window focus transitions. + +Fresh v27 mini canvas, popup and context-menu effects pass 3/3 with receiver modifiers and recovery recorded; the F31 change is restricted to official LibreOffice text areas. The requested official Codex comparison is inconclusive: even the installed matching bundled CLI and existing mini consumer profile fail at application binding with app-server exit `-10005`, before any native action. A separate ordinary stdio initialization succeeds, so the cause is unresolved. No Codex functional failure, performance comparison or zero-cover conclusion follows. Comparator-owned services, sockets and fixtures were removed while the shared remote service was preserved. + +## Local implementation + +P1–P6 implementation and packaging changes are in an isolated worktree. Node REPL exposes only `js/js_reset`, defaults to 30 seconds, and preloads global `cua` and platform documentation. The driver owns app/window resolution, short IDs, observation diffs on all three platforms, screenshot coordinates and dispatch errors. Direct MCP exposes 19 default operations. macOS retains background AX and directed events; Windows/Linux may use foreground fallback only after explicit pre-dispatch refusal. Explicit desktop input and held keys belong to a session; normal end/reset awaits release. Browser integration returns the existing browser-use SDK objects without changing browser behavior. + +Native Wayland implementation is removed; X11 applications, including X11 apps under XWayland, remain supported. SDK, native packages, browser-use and Node REPL share version `0.26.0`. Standalone npm installations pin dependencies and verify native download checksums. Default Windows installation does not require a signed UIAccess worker; explicit opt-in still checks signing and a safe installation path. Qwen CLI/core product changes consist solely of removing two bundled `SKILL.md` files. + +Local checkpoints: P1 `cf89555254`, P2 `d2995a50c5`, P3 `b7c4d9c86c`, P4 `56a3ec2ba5`, P5 `fcc473d4a6`; P6 is carried by the commit containing this record. + +## Evidence obtained + +| Scope | Result and limits | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Repository build/types | Root build and typecheck pass, including final browser-use type checking. | +| Node REPL | 68 relevant tests pass, including asynchronous reset/dispose cleanup in real child processes. An actual standalone tarball installed into a consumer directory without repository dependencies passes documentation preload, variable persistence and reset checks. | +| SDK / browser-use | SDK: 51 pass, 6 skip. Browser: 594 pass. Wrapper tests use the original SDK; they do not establish Chrome GUI success. All three npm packages were actually packed; publishing was dry-run only. | +| Linux native | Locked all-target checking and SDK/driver builds pass in an isolated ARM64 container. Coordinate tests: 38; Computer Use: 5. Real Xvfb desktop chain: 96/96; App coordinate chain: 29/29. Actual display resize was skipped. | +| Linux first installation | Real archive/checksum HTTP download, cache hit, incomplete-cache repair, bad-SHA refusal, missing-checksum refusal and postinstall: 14/14. The actual downloaded SDK/bridge loads from cache and passes GTK input, screenshot and key-release checks: 14/14. This does not use native overrides or development symlinks. | +| macOS native | All-target checking and ARM64/Intel SDK/bridge builds pass. Both combined universal files contain arm64/x86_64. Loading was executed only on ARM64; Intel runtime and notarization were not verified. | +| Windows | GNU cross all-target checking passes. P6 did not execute desktop/UAC tests. Pure-state regressions establish that post-dispatch UIA errors and worker disconnection no longer cause a second click. | +| Release configuration | Shared-version constraints, dependency publication order and production gates pass. The workflow was not triggered. | + +Closing review repaired three consumer defects: an old normalization environment variable transformed canonical pixel coordinates twice; a post-dispatch Windows UIA error was considered safe to click again; direct MCP initialize still described obsolete tools and normalized coordinates. Failed reproductions and post-fix evidence are retained separately. + +The final P6 native source manifest is v5: 688 files, tree SHA-256 `c8a662cdc5c53eaa4cbbd14b146b8e844a86fc060d1b8f16a8f7bb4014c7dbee`. It again passes Linux/Windows GNU all-target checks, actual MCP initialize/list 11/11, exact desktop chain 96/96, actual download 14/14 and cached SDK GUI chain 14/14. SDK/driver were rebuilt for v5; the bridge explicitly reuses the fresh v2 build because its source and ABI did not change. Earlier v2/v3/v4 results retain their original identities. + +Raw logs and independent reports are in `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/` in this worktree. Main report: `.qwen/e2e-tests/cua-upgrade-2026-10.md`; P6 report: `p6-validation.md`; self-audit: `.qwen/investigations/cua-upgrade-2026-10/parent-self-audit.md`. + +After P6 fixes, two complete diff/consumer audit passes found no further issues, and the independent P6 boundary review was CLEAN. Codex `/review` is unavailable in this environment; Qwen Review was not used as a substitute. + +## Historical P7 checkpoints (2026-10-19) + +mini is unlocked and executes actual AppKit input, screenshot and key-lifecycle tests through the existing GUI Node host. Windows has built the real MSVC driver, SDK and pinned Node bridge. Linux executes GTK/AT-SPI tests in an Ubuntu Xorg guest on ECS. Earlier lock-screen and cross-compilation-only limits no longer describe the whole current state; historical P6 results retain their artifact identities. + +Review of upstream `0.20.0 → 0.35.4` found applicable fixes beyond the P4 ledger. Current changes and evidence include: + +| Fix | Evidence and remaining limits | +| ------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Linux top-left window coordinate offset (#4967) | Same GTK scene hits an adjacent row before the fix and the exact target afterward. | +| Linux missing `_NET_WM_PID` (#4214) | Removing the property loses discovery/observation before the fix. XRes preserves the correct PID, window, AX and screenshot afterward. | +| Linux temporary keyboard mapping lifetime | Candidate and upstream both lose remapped Unicode while the GTK main loop is busy for 3 seconds. The queue-barrier candidate preserves input in busy GTK/Chrome scenes. Final v5b timeout returns normal sent/unconfirmed without replay; missing protocol support refuses before input. This does not substitute for physical MPX or all toolkit/IME acceptance. | +| macOS non-finite window dimensions (#5125) | Actual CFNumber tests fail before and pass after, retaining normal and negative coordinates. | +| macOS screenshot PATH (#3755) | Narrow PATH reproduces ENOENT on the real machine; the candidate returns an actual 1920×1080 PNG under narrow and normal PATH. | +| macOS raw clicks (#2907/#4690/#5137) | Same-PID parent/sheet reproduces lost middle-click targeting and duplicate events. Two real-machine reruns restore exact-window targeting and the correct event-pair counts. Default App middle-click was already correct and is a separate route. | +| macOS Electron background scrolling (#4861/#5079) | Normal v4 SDK passes seven native cases: raw page/inner × AX/pixel and default App inner retain page +404px or inner +114px, with the other region unchanged. Exhausted inner raw/App refuse with actual sent:false and no offset changes. v2/v3's reversed -96px down movement remains the before result. Read-only native diagnostics identify ROW35 above the viewport as a false bottom sliver; v4 excludes the wrong document-order candidate. Each case makes one public request, without replay, foreground fallback or browser APIs. Main routes retain uncertain ACKs. Six geometry unit tests pass. v2's controlled 43 cases and the later controlled-program startup timeout remain separate; 44 passes are not claimed. Normal v7 independently repeats and passes all seven GUI cases. This does not establish all directions, AX layouts or human concurrency. | +| Element token/native-object binding (#3616/#3873) | Actual core/token/cache controlled interleavings reproduce old tokens reading new objects. Capture/resolve/reverse cases pass afterward and all three consumers compile. Windows native-v2 MSVC cache: 5 pass, 2 explicit skips; tokens: 30 pass. No GUI click was performed by this reproduction. | +| MCP schema (#4888) | Actual default 19 and EXTRA 70 wire declarations pass offline compatibility checks; runtime still refuses invalid target combinations. No real model provider was requested. | +| Windows GUI-host console (#5136) | MSVC tests verify no console appears for a console-less host and inheritance remains for a console host. | +| Windows screenshot failure cleanup (#4333) | Actual GDI allocation/fault tests pass without handle growth over repeated failures. Interactive screenshot acceptance remains separate. | +| Windows shortcut environment variables (#3975) | Real IShellLink returns an unusable literal path before, and resolves the target while retaining arguments after. | +| Implicit-session idle recovery (#4283) | The same real MCP connection resumes after about 34 seconds. Explicitly ended/named sessions remain terminated; old element tokens are invalid. | +| Recording ownership/stop/time (#3632/#3714/#5144/#5145) | Public entrypoints verify named ownership, encoder errors and timing. Independent retest finds an omitted implicit connection; a second fix passes same/cross-connection and lifecycle regressions. | +| macOS recording completion (#4238) | Real-machine before results include false empty recordings and files growing after success. Five candidate 120ms recordings decode fully immediately; file SHA remains unchanged one second later. | +| Recording AX budgets/session lock (#4215) | Actual ToolRegistry with a controlled provider establishes that a hung after-capture no longer blocks state/stop. Six cases cover the 2-second return ceiling, same-PID busy refusal, late-result discard and stop→start isolation. Two native MSVC tests cover recording-thread COM initialization and release order, with all-target checking. This is not a GUI AX-hang test. | +| macOS Retina metadata (#3328) | At the same actual display mode, 3600px/1800pt, the public interface returns 1 before and 2 after. This does not imply default click coordinates were wrong. | +| macOS accessory-app identification (#3456) | On mini, the same installed accessory app loses running status in both default/running_only before; native-v3 returns actual PID 9402 through both public entrypoints. Exact installation paths and every running instance are retained; another copy sharing a bundle ID is not substituted. | +| Symlinked output directories (#4210) | Manifest and public ToolRegistry tests refuse a new file below actual `/tmp`-style directory aliases before and resolve its authorized path with one dispatch after. Outside-root, broken-link and file-parent cases still refuse. This is not a GUI screenshot test. | +| Windows named-pipe cleanup (#4279) | A private Win32 pipe reproduces creation failure and recovery after release. Source adds bounded retry and shared PID/SDK cleanup. Native-v4 MSVC driver/SDK build, 6 cache tests and 30 token tests pass; production daemon retry/cleanup still needs the interactive desktop entrypoint. | +| Uninstall MCP configuration ownership (upstream `ff770f5d5`, etc.) | Isolated fixtures verify five foreign configuration classes are retained and removal uses the actual executable of the selected payload. No complete uninstall ran. Configuration may already be committed if later deletion fails; original-byte backups permit recovery. | +| Typed-schema numeric formats (#3462) | The upstream regression fails before and passes after. Inputs/outputs retain valid formats and literals; 37 contract tests and 1 generator test pass. | +| Windows installer ancestor protection (#4398) | Twelve independent controlled cleanup/legacy-delegation checks pass. PID cleanup protects ancestors and skips task End when a driver ancestor exists. Sealed UnitOnly passes without changing PATH or performing actual task/full installation. | +| Windows autostart/isolated installation (#3180/#4284) | 32 controlled tests, 11 native MSVC unit tests and all-target checks with restored compile fixtures pass. Cancellation/null no longer reports success; replacement does not first delete the task; isolated release installation preserves the shared task. Real UAC, scheduler atomicity, login startup and full installation remain separate. | +| macOS release-install recovery (#3297) | Actual shell branches reproduce copy failure losing the old app before. Eighteen isolated scenarios and 350 independent checks pass after, covering staged/post-copy verification, copy/rename/link failure, retained backup on restore failure, first install, TERM and repeated cleanup. Permanent regression tests include original-byte restoration. These are controlled filesystem transactions, not actual install/signing/permission operations. No automatic TCC reset or SIGKILL/power-loss recovery is claimed. | +| Chromium AX readiness (#3783) | Sixteen controlled actual-function cases, 2 old/new publication races and 2 actual core budget-propagation tests pass. A shared 500ms readiness budget adds no polling for actions/screenshots/recording. Real Chrome refuses opt-in; the accepted path remains unverified. | +| AppKit launch-window recovery (#4309) | The 1-second active-search budget shares the original observation start. Actual-function controlled tests: 27; G17 outer-consumer regressions: 49. A fix removes active retry after sleep exhausts the budget. One Mac unit test and Mac/SDK all-target checks pass. Final App body/native IPC are outside this ceiling; real AppKit launch timing remains unverified. | +| Ordinary macOS AX traversal budget (#3882) | Controlled actual traversal reproduces raw/App reading after 1 second before. After: 27 traversal scenarios and 3 node-admission checks; preparation is separate, an exactly-at-deadline complete tree is not falsely truncated, and recording absolute budgets/legacy node limits remain intact. Actual core ToolRegistry/default App/Mac version/cache/token consumers pass 6 controlled snapshots and 22 independent checks: old-ID refusal, stable resolved bindings, partial-result warning and cleanup. Six tree, 4 cache and 27 core version unit tests plus Mac/SDK all-target checks pass. Full platform/SDK/GUI slow-AX behavior remains unverified; no whole-call 1-second hard deadline is claimed. | +| macOS backing-store capture (#4996) | Thirty-two controlled after checks pass; current-Space public/private raw/App routes return correct PNGs. A later normal candidate captures B on Space68 while A is foreground on Space1: raw/App images and five ROIs pass and are byte-identical. All 120 capture samples retain Space1 and foreground/topmost A; synthetic input reaches A 6/6. Normal binaries do not record the private route, so causality is not inferred. The full batch remains FAIL because its fullscreen-exit-button precondition was false; cleanup passes after owned processes exit. | +| macOS omitted AXWindow recovery (#4068/#4085/#4509) | Four native files add exact PID/SPI identity, restricted remote-token recovery and default App/tree/facts consumers. Three Mac unit tests, Mac/SDK all-target checks, 24 independent boundary checks and 49 actual outer-consumer checks pass. Actual omitted-window acceptance remains incomplete. | +| macOS special-key flags | After real Space preparation fails, a no-delivery native reproduction identifies missing structural flags shared by candidate/upstream. Four named-key builders pass native 128-keycode comparisons, explicit modifier/release and ambient-isolation checks; 11 unit tests and Mac/SDK all-target checks pass. One normal candidate Ctrl+Left is independently observed to return to Space1 about 638ms from call start. There is no resend; B does not receive the system-consumed Left, and its actual event flags are not inferred. | + +The stable Linux nested-Xorg focus/end-session batch passes 31 checks; one insufficient TCG-timing case remains inconclusive. Unicode and mapping restoration pass under normal response. A later 3-second GTK-main-loop pause makes 13 remapped Unicode keypresses become NoSymbol in both candidate/upstream, independently identifying premature map restoration. The original frozen primary-GNOME batch remains an infrastructure confound. Client ping passes in GTK/Chrome, but Chrome responds before DOM input completes, so it cannot establish semantic success. Native-v5a/v5b separately validate mechanism and final response: timeout can still lose text and requires observation before another action. + +MPX now confirms the complete ordered batch from the exact virtual keyboard. Unconfirmed/failed devices retain pending state; their next request refuses before mapping, focus or input. Initial warm-up also needs complete confirmation before temporary-key planning. Independent controlled lifecycle tests improve from 7/10 violations before to 10/10 passes after; 13 event-matcher cases, 12 Linux native tests and all-target checking pass. Real kernel inter-batch delay, hot-unplug and physical MPX were not established by those controlled results. Current v7 native regression adds 16/16 MPX and 7/7 key-alias passes; dedicated real uinput/evdev/Xorg acceptance is ongoing. + +macOS T04 raises the target over the user's window throughout three native-v3 runs, about 3.1–3.3 seconds, while the user window remains front/key and receives all synthetic input. Dual-query diagnostics distinguish the complete window list from actual visible order. Fix 1 switches to visible order, but reactivation succeeds without restoring order and the rerun fails. Fix 2 raises only the current foreground AXFocusedWindow: three runs remove sustained cover but retain 99.4, 200.5 and 192.8ms transients. **The zero-interference gate has not passed.** One preparation-only diagnostic observes no cover, without excluding short transients or guard masking. A separate requested-pair-only switch to public CGEventPostToPid confirms actual click effect via source-PID mouseDown, native field editor and final forced AX focus, yet A observes 194.110ms cover and independent WindowServer ordering 196.695ms. Public publishing does not demonstrate a remedy and is not adopted in production. Synthetic keys reach A 10/10; only one key occurs within the click and none during observed cover, so this does not prove human-input safety during cover. T06 menu/T10 target self-activation regressions pass, as does exact Unicode readback through explicit Desktop foreground input. Later exact-window SPI tightening belongs to a new candidate. The independent two-top-level-window fixture has not started successfully; parent/sheet tests do not replace it. + +Electron native scrolling has the main-route port and seven-case real-machine evidence, including the reproduced exhausted-inner reversal. Without visible anchors the original geometry heuristic remains; reordered/multi-column AX graphs remain outside the proof. Ordinary AX partial-result and actual-consumer controlled checks are complete; full SDK/native slow-AX boundaries remain. Cross-Space capture has the effect evidence above. Exact AX-window recovery, accepted Chromium readiness and AppKit launch recovery still require their own native cases. Cross-Space AX returns 85 elements with explicit incompleteness and does not prove remote-token recovery. Default App selection, pre-action checking and observation forwarding remain in scope; changing only the raw walker cannot establish completion. + +Full-diff review also finds extra native references not owned by the macOS recording hook. Independent actual walker/hook/cache tests combine controlled AX data with real CoreFoundation retain counts: three before calls leak 1, 2 and 3 references; repeated complete/incomplete after calls return to zero, existing tokens remain valid and seven JSON payloads differ only in elapsed time. The fix reuses the unpublished snapshot owner without registering/replacing action tokens. Three native regression tests pass; all 39 evidence entries are verified byte-for-byte. This is not native GUI/AX acceptance and does not cover serialization failure or actual budget truncation. + +In the same review, independent complete-tool-module tests with controlled providers reproduce raw double/right-click replay through pixels after semantic effect followed by AX error. Once AXOpen/AXShowMenu is attempted, the candidate returns normal sent/unconfirmed with a warning and does not replay. Successful semantic and unadvertised-semantic pointer routes remain. Six double-click and 4 right-click controlled cases pass; three uncertain responses retain unverifiable/accessibility and warnings through actual core public projection and typed validation. A mistaken actual-foreground report for a foreground request is also fixed: AX without a foreground transaction reports background. The added native projection regression and existing route test pass. Default App's early forwarding is a control, not evidence it shares the bug. An actual AX server effect-then-error GUI proof is still pending. + +G34 explicitly rejects #5124/#4856's blanket native omnibox paste/newline refusal: the approved selected App/window contract allows native input at current focus. ASCII Return and non-ASCII AX insertion are tested separately; insertion readback does not prove navigation. Hotkey, pressKey and paste are reviewed individually. #4835 cold foreground field/selection differences remain a narrow raw/EXTRA reproduction task, unreachable through default App; no AllWindows or fixed stabilization sleep is introduced. + +G07's latest production-entrypoint test exits during SDK creation in SSH Session0 for lack of an interactive user session, in 23ms, before reaching named-pipe acceptance. No private pipe exists and the shared driver is unchanged. Pipes themselves can work in Session0; that does not make this entrypoint reachable. G23's second standalone startup diagnostic compiles but has no readiness/stage logs after its single 8-second launch. Another diagnostic of the same binary and ordinary Node launch remains unready after that window. One 1-second sample exits 0 with 89 `_dyld_start +0` main-thread samples and unavailable images; it does not locate the wait mechanism. Neither case dispatches AXPress; the later diagnostic creates no SDK/AX session. No TCC/EDR cause is inferred or security settings changed on that basis. An existing Electron host with a task-owned native addon is being prepared as a new fixture. + +All 499 upstream changes are classified; the former insufficient-evidence and unreviewed-P4-reference categories are empty. Applicable differences still need reproduction, porting or explicit disposition; classification is not acceptance. The latest 2026-10-19 whole-JavaScript build/typecheck pass; SDK: 51 pass, 6 explicit skips. The initial missing-corepack failure remains, followed by success using a task-local entrypoint and existing pinned local caches, without global installation. Normal v7 has 453 native inputs plus 12 exact compile-support files, tree `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693`. Mac/SDK, Linux/driver/SDK and Windows GNU/driver/SDK all-target checks and the default SDK build pass. Linux x64 and Windows MSVC driver/SDK releases now also build from that exact manifest. Windows adds 2 COM, 11 autostart and 1 bounded-capture pass. Linux's current normal direct-MCP GUI tests pass GTK 7/7 and Chrome 3/3. The unchanged core retains its exact-source eight-check checkpoint. Earlier omitted-fixture failures are retained. + +Normal v7 independently passes G05's seven GUI cases using SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe`. Independent fixture IPC confirms inner +114px/page +404px and unchanged exhausted offsets. All 143 ordering/pointer/clipboard-count samples retain A; owned processes/LaunchAgents are cleaned and shared processes retain their lifetimes. The root verifies all 634 referenced evidence entries. Sampling is 8.444–12.213Hz, not proof of zero transient interference or human concurrency. + +P7's first full audit covered 89 changed paths and found the recording/replay issues above. A later complete clean pass was followed by a second pass finding missing bilingual design documents; translation repair resets the consecutive-clean count. Two complete clean passes over the final updated scope are still required. Full three-platform regression remains ongoing. Detailed before/after results, invalid batches and artifact identities remain in `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/p7-*`; P6 results do not establish acceptance of subsequent P7 changes. + +## Historical external acceptance checkpoint + +Windows ordinary SSH is restored and current console/runner/shared-driver lifetimes are rechecked. The user explicitly approves the task-owned `\QwenCuaP7Acceptance` task: no triggers, existing Administrator InteractiveToken, Limited, no stored password, at most 2 hours, verified deletion afterward. Current v7 interactive acceptance is being prepared. No other process token is borrowed. The user also approves normal Automation dialogs for the existing mini Node host controlling Finder/TextEdit, and test-machine permissions needed for this work; no TCC database edits are required. Linux's dedicated real uinput/evdev/Xorg chain is a separate topology from the nested guest. Nested Xorg alone does not establish physical user input, MPX or XWayland acceptance. + +Three-way tool comparison and OSWorld-20 model evaluation remain incomplete. Earlier provider 400s remain historical failures. Current gpt-6.1-sol passes global qwen 0.25.2 text preflight and actual CLI→MCP→Node REPL execution. The third effective synthetic-image preflight returns a real PNG and correctly identifies all four color/shape/position facts, using 2 model requests and 1 tool call. The first two timeout/transcription refusals produced no image and remain separate. No native desktop session was created, so this is not GUI acceptance. + +All 20 local task definitions are located and frozen; they default to Linux and only two contain explicit Windows branches. A bounded mini search of 1,003 non-credential text files did not locate the historical adaptation. The existing stable20 set uses different tasks/scaffolding. The user now explicitly authorizes read/export of historical groups `group-5cff67aa55934a09a45160e3194b5801-s1` (macOS) and `group-bc7a3539903b4735bc3508e04e5791d5-s1` (Windows). Each actual AP request passes the exact official mini AP Client 0.5.1 version gate. Group metadata, jobs and evaluations are recovered; artifact export is ongoing. Raw exports stay private on mini; credentials are not copied into this repository or local artifacts. Both groups contain 20 terminal Succeeded jobs, which does not imply task success: macOS mean score 0.2613, 1/20 passed; Windows 0.1206, 0/20 passed. Their template commits differ (`d1b8341d17659205b9aa735e0390f54301be0dc7` / `b70f9f90b3df26f177d78279bc942490f7cc37e7`) and both use azure.gpt-6-astra, unlike the current gpt-6.1-sol preflight. No new AP job, retry or cancellation is authorized or performed. Current candidate OSWorld execution count remains 0, and no matched success-rate, token, turn, wall-time or Codex/upstream improvement claim is made. These pre-release gates remain active. + +## Historical v13 P7 acceptance snapshot (2026-10-19) + +This historical snapshot superseded the earlier checkpoints at v13. Its source identities and open items describe that checkpoint; the current scope and later verification are recorded below. + +### Current candidate and distribution + +The current native freeze is v13: 453 production/build inputs plus 12 exact compile-support files, tree SHA-256 `c43e1b300706eff833a9999f889b7f7c418a414b6863d3b3c102ca339770f83c`. Linux uses its fresh v13 CLI and SDK. macOS tests use the normal default-feature v11 SDK (`4131a02af8d64a88572e981dbd9dfa5e7d0eaa9ec5ac5509be341c159d59ce9d`); Windows uses the native MSVC v10 executable (`c5ca0054810eb4512986aef0f6231e81497ada15d1809bcb9bd5521016ef3fca`) and SDK (`91f871c885e13e856ee8284af0efed149232abbf58d9c7fc4584dee3d1f02283`). The intervening Linux-only changes require a target-specific source-equivalence audit before those other artifacts establish current acceptance. + +All three publishable packages pass local npm pack dry runs at 0.26.0: CUA SDK 34 files, Browser Use 9 files and Node REPL MCP 46 files. The whole JavaScript build and typecheck pass, with SDK 51 pass and 6 explicit skips. The complete current working-tree snapshot merges without conflict with main `a250fd033941cab10c88d6eb8beb5756405193b9`; no branch/ref, push, PR or release was created by that object-only check. Final formatting and two complete clean source-audit passes remain required. + +### Native fixes and actual effects + +- macOS G23 now has an actual AppKit AX server proof: four v10 cases cover raw/default App with a handler returning NO or raising NSException after toggling. Each performs one request and one effect. The exception response is normal sent/unconfirmed with no pointer replay. Returning NO can still produce AX success and is kept separate. A v11 regression is pending. +- macOS default App T04 now uses exact editable AX focus for a single unmodified left click on a supported native field. Three fresh v11 click-only cases independently confirm the field editor and fresh AX focus without value changes or observed cover. All observed canary characters reach A, but none of those click request windows contains a new keyDown; they do not prove input concurrency. The full click-and-type regression passes; a higher-frequency overlap run is ongoing. Pointer-only diagnostic variants that still raised B are not adopted or counted as fixed. +- Linux MPX successful delivery now carries an explicit background/LinuxUinput execution record, avoiding a public action_outcome_mismatch after text had actually arrived. Ordered raw-key confirmation, pending-device refusal and session cleanup prevent replay after timeout. +- Openbox 3.6.1 promotes a virtual master's core FocusIn to global activation. Fresh zero-key controls attribute this to the WM/Xorg interaction. v12 refuses MPX keyboard before map/device/input work on that identified WM. Six native requests return background_unavailable/sent:false with zero input. The public hybrid route may then use foreground once. It restores the original foreground, core focus, pointer, map and stacking top; a lower-window order change remains observable and is not counted as background zero interference. Mutter 42.9 preserves the stronger complete order in the corresponding background tests. +- Linux AT-SPI InsertText requires a UTF-8 byte length while caret offsets remain character positions. The earlier implementation and upstream 0.35.4 truncated real mixed text. v13 fixes the shared length conversion before dispatch. Ten native/public/SDK/facade cases now read back exact CJK, emoji, combining characters and nonzero-caret insertion with zero raw key events. Readback does not change the runtime's conservative sent/unconfirmed projection. + +### Platform verification and remaining native work + +Windows v10 completes 14 native regressions, 68 direct-MCP/SDK/native/Edge checks, 10 production Node REPL kernel checks and 12 ordinary private production-serve checks. The last two cover actual kernel lifecycle and pipe ownership respectively, not a Node REPL MCP-wire GUI session or injected later-instance pipe failure. An ordinary serve supports low-level CuaDriver.connect; the retained ComputerUse.connect factory still requires authenticated session ownership and cannot bind that ordinary serve. Documentation now states that boundary. WinForms background samples preserve A and pointer; Edge typing deliberately uses the one permitted foreground fallback. Upstream 0.35.4 meets the expected results on 12 native and 8 Edge checks. No verified Windows Codex entrypoint was available. + +Linux v13 passes 53 focused native tests, with one pre-existing ignored test, all-target checks and release builds. Ten semantic Unicode/caret cases and a real-kernel MPX smoke pass. v12 separately covers responsive/blocked/timeout/pending-refusal/forget-and-fresh lifecycle through native, SDK and Node facade consumers on private real Xorg displays. The external-input interruption proof uses a separate owned uinput device, not a human physical device. Neither a software-rendered Mutter container nor nested Xorg is presented as shared GNOME, native Wayland, physical-human concurrency or VM performance evidence. + +The macOS v11 full fixture matrix has run, with its detailed independent readback still in progress. Remaining observations are retained as failures or limitations: sheet typing refuses with same_pid_keyboard_ambiguity; WKWebView typing refuses with no_focused_text_input; two short layer-order overlap samples need attribution. A non-document native field's AXValue is replaced while the fixture delegate callback stays absent; that stronger callback oracle remains false, and no second input is added to manufacture it. Normal TextEdit save and Finder rename still need their approved Automation regression. Clipboard generation alone cannot establish restored clipboard contents. + +The temporary Windows task has been deleted and read back absent, its runner has exited, and all owned helpers are gone. Limited task configuration is recorded separately from the built-in Administrator's actual High integrity. Shared driver/process lifetimes are preserved. Linux input processes and temporary devices are removed, while the private display fixtures remain for final cleanup. macOS owns a desktop mutex and cleans exact run-owned process/LaunchAgent identities. Synthetic canaries and sampled focus/order observations are always distinguished from physical-user proof. + +### Matched model evaluation + +The approved read-only AP export is complete using the official mini AP Client 0.5.1 with the version gate before each actual request. Raw credential-bearing exports stay private on mini. Both historical groups use qodercli 1.1.52 and azure.gpt-6-astra. Their image/template versions differ. macOS scores average 0.26128 with 1/20 passed; Windows 0.12058 with 0/20 passed. Terminal Succeeded is not task success. Historical token counters are unavailable zero-filled fields, not zero consumption; only 13 Windows traces have complete final turn/time metrics. + +The exact baked macOS pilot cases/plugins have been recovered, with all 20 instructions matching the AP exports. Windows matches 19/20: its historical 001 instruction names Calendar.app, while the recovered Windows override correctly uses Thunderbird Calendar. Base-layer seeds, genuine runner/generic scoring, websites and dependencies are being restored with complete layer hashes and platform-specific overlays. Setup must preserve existing files/profiles, use run-owned listeners/process cleanup, and fail preflight when Calendar persistence, application, codec or website requirements are missing. Scorer/gold files are never supplied to the autonomous model. + +A fresh matched comparison must keep the harness, task seed bytes, instruction/rubric, model/provider and budgets constant across the intended old/new tool stacks. Historical Qoder results remain a reference, not that controlled baseline. The current gpt-6.1-sol global Qwen CLI passes text, real Node REPL and synthetic-image transport checks. A separately deployed production Node REPL MCP process on mini also passes initialize, exact js/js_reset listing, real kernel/preload, reset and EOF cleanup over the existing SSH connection without model credentials or a native GUI session. Actual candidate OSWorld task execution remains zero. Success-rate nonregression and round/token/wall-time gates remain open; no performance or three-way parity claim is made. + +## Historical first-four scope (2026-10-19) + +The user now prioritizes native macOS, Windows and Linux X11 acceptance plus runtime, installation and package checks. Codex/upstream comparisons and OSWorld/model evaluation are deferred. Future model runs are configured for `gpt-6-sol`; no new model/AP task is part of this scope. Completing these four items will not establish the deferred model gates or authorize publication. + +### Candidate identity and completed checks + +The validated v21 native source freeze has 453 production/build inputs and 12 support files, tree SHA-256 `11b0fafab2419448a16542a73cde08cea58ee67b3a7b922fdfe12fee456f9dad`. Relative to v20, only the CLI entry changes: when effective telemetry consent is false, a finite command runs directly instead of starting an unnecessary observer child. Consent precedence and the enabled path are preserved. The macOS backend remains the v20 SDK: pre-signing SDK binary SHA-256 `886bf05463a6c8273d0125b28482ee4f5ba8e9c1cffe5b4d5230e5fef2e77eee`, or `134b334ea8b7e912e51ac0c27383d71b6c245e17b8f99f69cbee6e08c07fb064` after the candidate Developer ID signature. Code-section equivalence is independently checked. Windows and Linux backend inputs remain target-equivalent to their v19 acceptance artifacts; new CLI checks keep their own version labels. + +- macOS v20 passes fresh sheet typing and single-Return confirmation, WebKit click/type/submit, exact paste with clipboard type/byte restoration, and safe negative/positive two-window input. Production Node REPL MCP passes eight real wire/lifecycle checks with exact deployed source/dependency identities. TextEdit Unicode save and Finder semantic rename have independent C9 disk readback. Finder inline keyboard rename safely refuses when native parent/window ownership cannot be proved. The internal key-window preparation contains NaN-location mouse records: the old Swift JSON logger omitted them, so earlier claims of no internal mouse events are withdrawn. +- Windows v19 passes actual MSVC checking/building and 37 focused tests, retaining the independently identified native GUI regressions. A production MCP-wire action times out after its one real WinForms effect; no replay occurs, and reset/EOF cleanup passes. Complete CI-shaped packages pass owned fresh, official-old-byte layout upgrade, repeat, uninstall and purge checks. Unsigned explicit UIAccess deployment is rejected; neither signed UIAccess nor ARM64 native execution is claimed. The current isolated uninstaller passes 14 actual Windows PowerShell regressions, including a task mixing owned Exec and foreign ComHandler actions. Shared processes and the deleted temporary task remain protected. +- Linux v19 passes 53 focused tests, one explicit ignored test, actual release builds and native MCP initialization/listing of 19 tools. Real Mutter/Xorg/uinput input preserves the complete sampled focus/order and exact Unicode; production Node MCP passes 15 checks with real AX readback. CLI installation and SDK cold/warm cache tests use complete packages. Openbox fallback restores the original foreground/top window but changes lower stacking order; it is not counted as background zero interference. Synthetic owned input is distinct from physical-human concurrency. +- All three npm package dry runs pass at 0.26.0; SDK tests pass 51 with six explicit skips. The repaired Node MCP preload test holds SDK evaluation behind an observed barrier before timing out, then checks the next real request; the full MCP file passes eight tests, with 113 Node tests covered across the retained runs. Independent mutation evidence detects a late continuation. Installer Python checks pass 57 with five explicit skips and four subtests. + +A subsequent v22 freeze (`de5d5030e7dbf5b3f64c7946dc8d335ee2d988db14e8fe0994226d2f6f980245`) adds an Excel-only Apple Events compatibility fix and corrects the window-order comment. Excel uses full-name1773 and saved1842 rather than the missing Standard Suite path/modified properties; a case-sensitive full-path guard precedes its one save. Exact and mismatched-path read-only argv probes pass. Six existing Apple Events unit tests, Mac/SDK/CLI all-target checks and the ARM release build pass; the SDK binary is `04d996ff2b51337bf07a72da408dee0ea0b987081b209bb680e28e8eb3f92499`. ARM and Intel release builds pass. Fresh v22 native Excel save also passes: one save returns in 316.586 ms, the modified flag changes true to false, and an independent XLSX oracle confirms the exact A1 marker with B1 unchanged. Both earlier dirty documents remain unchanged. Its click still produces 48.937083 ms of observed cover, so this is a save-function pass, not zero-interference acceptance. The updated v22 Mac package checks below also pass within their disclosed scope. Windows/Linux target inputs are unchanged. + +### Runtime, installer and package verification + +The v21 Windows/Linux CLI appendix now passes actual builds, existing consent/disabled tests, default-off and explicit-off finite commands, native MCP initialization/listing/EOF and current owned install/repeat/purge. Its 287 files and all four complete package archives have an independent parent hash/member readback. + +On mochi, the coherent v22 macOS candidate uses the existing product Developer ID team `NF4574S59H` for CLI, theme, SDK and bundle, without timestamp or notarization. The signed SDK hash is `5a0dd6d011e633fee9bd6b751444ee82430cc0c8632a5ab43e5c4143cfa93946`; packaged code sections match the fresh ARM and Intel inputs. ARM and Rosetta Intel finite/native MCP checks, installed ARM checks, SDK cold/warm installation and twelve production Node MCP checks pass. Fresh install, upgrade from a private complete signed 0.17 bundle, and repeat install pass fourteen assertions; the replacement satisfies the old signing requirement. The original global bundle and v21 artifacts remain unchanged. The parent independently verifies all 636 regular evidence files (738 tar entries including 102 directories), all four package archives and their members, native identities and all-architecture signatures for CLI, theme, SDK and app. The unchanged CI NAPI artifact retains its documented unsigned Intel-slice boundary. + +These checks complete the fourth item for v22 within the disclosed owned/local candidate scope: macOS ARM plus Rosetta Intel, and Windows/Linux x64 with verified unchanged target inputs. Installer tests use documented destination/writeability/download/stop-helper seams and real filesystem, signing and replacement operations. The two actual rollback fault cases are retained from v21 under the unchanged installer source, not newly executed v22 faults. Shared services are preserved; this does not establish stock global replacement, Windows signed UIAccess, ARM hardware coverage or official notarization. Earlier installed ad-hoc ARM failures remain retained; the coherent Developer ID candidates pass that installed runtime path without a security-policy change. The baseline absolute nested-local SDK import defect is retained with identical base/current loader bytes; installed bare-package and global `cua` consumers pass separately. + +### Remaining gates and evidence limits + +macOS canvas pointer input still raises the target over A for roughly 100–200 ms despite preserving foreground identity and canary delivery. Menu coverage also remains open. Exact effect plus eventual restoration does not satisfy zero cover. Authentication, public-post, key-preparation, default mouse-subtype and PrivateSource diagnostic variants retain coverage and are rejected; private variants never replace the normal artifact. PrivateSource produces one exact native effect but still about 199 ms of coverage. A read-only causal review does not establish system impossibility; none of these pointer factors addresses AX-only popup coverage. A fresh whole-preparation-omission counterfactual still covers A for about 206 ms and loses the native canvas effect, so that candidate is rejected too. The subsequent read-only review finds no evidence-backed next external-driver repair hypothesis; it does not prove system impossibility. + +A fresh normal-v20 slider alternative passes: one `setValue('80')` produces both value80 and the independent native callback80, with no observed disturbance. Excel's original selected-grid state and LibreOffice's newly opened document state return `no_focused_text_input` before dispatch; neither is a saved-document pass. Excel Edit subsequently confirms the typed marker through its native formula-editor description and Return through A2. An initial save remains unknown after an Automation prompt and caller deadline; after normal authorization, a fresh document’s one save returns `automation_failed (-50)`, `sent:true`, with no disk marker. The v22 native save fix is verified above; its pointer coverage remains a separate failure. LibreOffice’s public body click leaves a group focused; a separate read-only body probe reports settable AXValue/AXFocused but actual focus false. No replacement input or saved-document pass is claimed. The native still-launching public chain now passes: an independently identified installed accessory fixture is still launching when its one `getApp` starts, with over 900 ms of native launch delay remaining; it returns the exact AX marker in 1.056 seconds, followed by independent finished-launching proof. Twenty action-plus-settle order samples show no owned coverage and six synthetic canaries arrive at A. This does not prove which retry ran or that the old 400 ms implementation failed: the first AX IPC may itself block until readiness. Earlier no-request and scratch-helper-exclusion attempts are retained separately. One pre-overlap MCP sample has an external System Settings window above A; without per-RPC absolute timing it is not attributed to the candidate or reported as uninterrupted A-on-top. + +Immutable reports, raw failures and manifests remain under `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/`, including `p7-normal-v20-native-regression-independent-report.md`, `p7-windows-first4-final-v19-20261019.md`, `p7-linux-final19-first4-acceptance.md`, `p7-mac-zero-cover-causal-boundary-review.md` and the versioned package appendices. Evidence-only clean passes do not count as the required two final full-source audit passes. The earlier main merge check is historical; final source validation and mergeability must cover the completed tree. No push, PR, merge or release has occurred. + +### Historical v25 checkpoint (2026-10-20) + +The current 453 native plus12 support inputs have tree SHA-256 `6faac284650aeffaa64c5fa6b220ab1799ae25f69f0e68845e21159bf7d50e5a`. Linux v24 repairs partial-send classification at prefix/first-character/key-release/flush boundaries, admits MPX pending input only after focus validation, and cleans up failed press/probe release. Actual Linux focused tests, all-target checking, CLI/SDK release, private install, finite MCP and SDK cold/warm checks pass. Four fresh Xorg/Mutter cases exercise public CLI Unicode and named keys, the current SDK C ABI trusted-session seam, and one paused-target uncertain request. The last case delivers only `a` after resume, correctly remains sent/unconfirmed, and has no extra physical events at return, after1.7seconds or session end; it is not a complete-text success. The parent verifies38 evidence hashes, original callbacks, full distinct old/new Xorg logs and owned-container cleanup. Earlier tail-log count overclaims retain their correction and originals. + +v25 changes only macOS exact-target resolution relative to v24. A real LibreOffice failure showed a same-window app-focused `AXGroup` explicitly reporting false while its text body actually reported true. This precise condition now invokes the existing bounded, complete, unique text-focus proof; PID/window binding, cross-window rejection and final rechecks remain. ARM and Intel CLI/SDK release builds, six existing exact-target units and all-target checks pass; the new ARM SDK has a verified Developer ID signature. The new coherent universal Developer ID package passes ARM/Rosetta finite commands and direct MCP, packed SDK cold/warm installation, production Node MCP, and14 owned fresh/official0.17-upgrade/repeat assertions. The parent independently verifies four archives and all38 regular member occurrences,11 result reports,465 source/support pins and five all-architecture signatures. Signing remains offline without timestamp/notarization, and installation uses the disclosed private-prefix/local-endpoint seams. Complete current-source audits and final merge preparation remain separate checks. Windows/Linux/shared inputs are unchanged by this Mac repair. + +Fresh original T15 confirms one public ASCII insertion through independent body AXValue and selection readback. Its one Save times out waiting for OS Automation permission, reports sent:true, and has no independent disk marker; no Save is replayed. LO reports AXEdited=false even with the changed body, so that attribute is not a save oracle. The local canary process has no posting permission:121 posting attempts and zero receiving Sentinel keydowns do not prove canary delivery or uninterrupted user typing. Numeric foreground/top/pointer samples retain their bounded observation scope. Earlier LO focus observations above remain historical; positive insertion now passes, while saved-document acceptance stays open. + +Strict pointer/menu zero-cover still fails. A fresh signed own-window bit21 control returns Set0 but reads the requested bit unchanged, then Clear0 and exact baseline restoration. It does not validate a foreign-window remedy. Earlier unsigned-helper SIGKILL and its empty logs remain stage-unknown, and no unsupported OS-wide impossibility claim follows. All-platform acceptance and PR readiness remain incomplete until these native gates, final checks and current-tree merge preparation pass. No push, PR or release is authorized. + +### v26 LibreOffice pointer safety checkpoint (2026-10-20) + +The v25 checkpoint above is historical. Current v26 has native tree SHA-256 `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3`; its 465-input manifest is `f16523c3823554f6d0e74d257ee232d768b7b8cebe17a80bf6b787d2591b4042`. Only the three macOS click tools changed relative to v25. Windows, Linux, shared native inputs and the JavaScript SDK are unchanged. + +A fresh default public Calc request targeted C5, but three independent exact-window Name Box reads returned B28. The original sent/unconfirmed request was not replayed. Exact LibreOffice 26.2.6.3 source reads the global `NSEvent.mouseLocation` for VCL mouse callbacks; the observed screenshot-coordinate inversion was coherent. This establishes an incompatible pointer-reading mechanism, without proving the precise native callback responsible for B28. + +The repair adapts upstream #4864's refusal principle using the fork's existing NSRunningApplication identity for `org.libreoffice.script`. It prevents PID-routed pixel click, right-click, double-click and element-pointer alternatives from acting at an unrelated hardware-pointer position. It preserves advertised AX actions and safe editable-focus/AX-selection paths. The old raw foreground pointer routes are also refused because they do not move the hardware pointer. This is narrower than upstream's mapped-library detector: rebranded or unbundled VCL processes are not identified by this rule. Canonical public Mac actions remain background-only and do not expose a new JavaScript foreground option. + +An independent new-document v26 request returns public `background_unavailable`, `sent:false` and request `committed:false`; three bounded same-window Name Box reads remain A1. This passes safe-refusal verification, not C5 background-selection success or raw AXPress runtime equivalence. The original stricter harness outcome remains failed because it expected internal `effect`/`toolkit` fields that the existing public error projection deliberately reduces to code/reason/sent/next. Its raw output and the separate contract-based adjudication are both retained. SDK `operation.dispatched:true` means entry into the FFI request lifecycle, not physical input delivery. + +Current ARM click14, AX-dispatch1 and double-click1 focused tests, all-target checks and release CLI/SDK builds pass; the ARM SDK has a verified offline Developer ID signature. Current-source packaging and further native regressions are recorded separately. The temporary local validation commit `a9fe798b6dac938392f450f2edf55456a6dfc6e0` retains the clean merge of fixed main `5dcf7c7cb162e825d95d531081f67ee60da3c0e1`. Its complete 8 GiB build script passes in201.876seconds and normal root typecheck in72.948seconds, including integration and without NODE_OPTIONS for typecheck. These are verification timings, not a performance comparison. Earlier default4GiB merge/main failures remain failures; that command was not rerun or marked green for v26. + +The independent v25 public-toolbar Save passed actual FODT XML verification; the original Cmd-S/AppleEvents Save remains UNKNOWN and was never replayed. A later own-window bit21 positive control passed, but the trusted normal-caller foreign-window experiment did not change the bit and was rejected after exact restoration. Historical Codex canvas samples received Command despite an unmodified client request, with unknown origin, so they cannot establish an ordinary unmodified-click baseline; its composite popup/context samples also retain actual cover. Strict macOS pointer/menu zero-cover remains FAIL. Current-source audits, native acceptance and PR readiness must be assessed separately; no shipping P7 commit, push, PR or release has occurred. + +### Historical v27 Writer focus candidate — not native accepted (2026-10-20) + +The v26 package checkpoint is complete: ARM/Intel builds, focused checks, coherent universal archives and all11 non-GUI runtime/install report groups pass. A fresh independent readback verifies92 evidence files,38 regular archive members,465 source inputs and5 all-architecture signatures. These package/protocol results do not close Writer or zero-cover acceptance. + +A source-slice reproduction executes the actual v26 predicates and shows that an official LibreOffice AXTextArea with no advertised action is refused even when its value/focus attributes are writable. A separate target-only, read-only query of the preserved v25 Writer process confirms exactly one same-window body, empty action list, and successful true AXValue/AXFocused settable flags. Its application-focused element is AXGroup while the body reports AXFocused=true. This is real old-instance property evidence, not a current action pass. + +The intermediate v27 candidate changes only click.rs from v26: the default App single-unmodified-left-click branch may focus a known official LibreOffice AXTextArea through the existing editable-focus path, with both settable gates and current PID/window ownership checks intact. Other applications' text areas remain excluded. The raw click implementation, generic role helper and original tests are byte-identical to v26. Focus/value confirmation is unchanged: the AXGroup mismatch can still produce an unconfirmed result, followed by observation; there is no pixel fallback, replay, caret relocation or stronger success claim. Native tree is `c52f7cb6ac11aa65a9d7a226c5d4e38f55ee52820bcc4085ed4314b006b40538`, manifest SHA-256 `a3aed6e4f6b77b532395729b611f9817de06bac65c6d64c4811e0ed17c870ad1` (465 inputs). Focused builds/review are separate from the still-open fresh Writer click/type GUI test. + +The latest Writer Stage1 stopped before LibreOffice or document actions because the owned sentinel could not obtain the foreground. System-window content remains unknown after Computer Use rejected access to that system app; no alternate channel bypass was used. Mini still reports locked in a fresh authenticated read-only check. Strict pointer/menu zero-cover remains FAIL. The successful full8GiB build/typecheck belongs to local merged checkpoint a9fe798b; it is not a full-build claim for the development checkout. Only the four later Markdown updates and this Mac Rust delta are intended for a refreshed local validation snapshot; any inherited build evidence must state that boundary. A new validation commit also changes the generated Git commit constant, so strict byte-identical compiler-input equivalence is not claimed. No final P7 shipping commit, push, PR or release. + +## F31 native and distribution checkpoint (2026-10-20) + +The accepted Writer source adds nine guarded lines to v27 click.rs and exactly matches the mini-tested SDK `091df63289f3542c8d1518f0be0db458863b7f87d4002de281579fca377509ab`. Both fresh documents independently transition from body AXFocused=false to true after one App click. Their one input and one toolbar Save persist exact XML, with saved hashes `83d5fd0c80a0e654d18f55b6d181bfb6be6ca1463a09b1885cad45393071cdb2` and `c4d557fa50ab5d7f449891d8fd139f0da24334880a44b31268bdd911f7ab01d1`. The parent verifies all 46 sealed evidence files and the first native read in each case. Pointer and text-clipboard state stay at each case's starting values; sampling gaps of up to 122.5/118.1 ms do not prove uninterrupted zero cover. The earlier v27 desktop click/input/save positive is a separate explicit foreground workflow. The original Cmd-S/AppleEvents request remains UNKNOWN and was not replayed. + +F31 ARM/Intel CLI and SDK release builds and relevant native/all-target checks pass. The coherent universal packages pass all 11 non-GUI report groups: ARM/Rosetta finite commands and direct MCP, packed SDK cold/warm installation, production Node MCP, private owned installation/replacement and installed ARM/Rosetta commands/MCP. The final manifests retain exact slice/archive/signature identities and the disclosed offline Developer ID signing and private installer seams. A task-specific caffeinate identity preflight failed before these checks launched; its correction and the subsequent actual full run are preserved separately. Windows v21 and Linux v24 retain their original native identities; the intervening Mac changes do not alter their target inputs. + +Formal Codex review of the new delta produced one multi-window concern, recorded and triaged rather than described as zero findings. The existing preparation sends process-scoped focus records, so its keyboard ambiguity gate remains; two single-window positives do not establish arbitrary multi-window support. Earlier complete source audits keep their pinned scope; final updated-delta audits, latest-main default build/typecheck, source equivalence and PR draft are recorded in the ignored final readiness evidence. No local source/build result substitutes for a native effect. + +The first-four scope is platform-native effects plus runtime/installation/distribution verification, with the documented safe refusals and uncertainty limits. G17/G18/G24 accepted-opt-in native coverage, the later-instance pipe-fault injection and physical-human concurrency retain their individual unverified limits. The requested new Codex behavior comparison is inconclusive before action; matched model/OSWorld success, tokens, turns and timing remain deferred. None is silently scored as a pass, and no publication is authorized. diff --git a/docs/design/cua-upgrade-acceptance-2026-10.zh-CN.md b/docs/design/cua-upgrade-acceptance-2026-10.zh-CN.md new file mode 100644 index 00000000000..7dcc5369adb --- /dev/null +++ b/docs/design/cua-upgrade-acceptance-2026-10.zh-CN.md @@ -0,0 +1,223 @@ +# Computer Use 升级验收记录（2026-10-16） + +[English](cua-upgrade-acceptance-2026-10.md) | [简体中文](cua-upgrade-acceptance-2026-10.zh-CN.md) + +实现以 [讨论结论](cua-upgrade-2026-10.md) 和 [平台移植清单](cua-upgrade-platform-ports.md) 为准。工作位于 `lazzy/cua-upgrade-2026-10`，基于 `a8eb74fdf5b60851e1fc4a8e258e043273df4411`。版本 `0.26.0` 是开发目标，尚未发布；F31 原生检查点完成时尚未推送、创建 PR、合入共享分支或运行发布工作流，隔离的本地合并验证提交分别记录于下文。此后的 PR 提交决定见下一节。 + +## PR 提交与评测安排（2026-10-20） + +用户已授权提交本次功能 PR，并明确将 OSWorld 评测放到发布后。下文原生/运行时验收事实保留原候选身份和边界，未执行的评测不记为通过。发布审计还将三个 Computer Use npm 包从 CLI 版本更新及发布中排除，保留其统一 0.26.0 及依赖发布顺序。本次提交步骤没有要求执行正式发布。 + +## 验收口径澄清（2026-10-20） + +用户确认 mini 已解锁，澄清此前观察到的 macOS 目标窗口/菜单短暂遮挡可接受且符合预期，并要求重新对比 Codex。严格零遮挡不再是阻塞项。下文历史严格判据的失败仍作为真实遮挡测量保留，不表示遮挡已消失，也不证明 macOS 无法避免它。功能正确性、精确目标归属、请求修饰键语义、不确定后不重放与恢复分别核验。 + +剩余真机 GUI 测试和新对照均在 mini 上执行。此前本机 Writer 准备阶段未发送文档动作；随后放弃的本机对照准备也未发送 Codex 动作，并已清理。本地工作限于源码、编译、包及合并验证。OSWorld/模型评测仍延期。不可变澄清记录与新用例报告保留在工作树的忽略证据目录。 + +已采纳的 F31 修复通过 mini 两个新实例的 Writer 后台流程：原 observer 在一次公开 App 点击前读取精确种子正文未聚焦，点击后读取已聚焦；一次输入改变正文且文件不变；一次工具栏 Save 将完全一致的正文保存为有效 FODT XML。此前 v27 负例及已否决的 F28/F30 试验分别保留。F31 在既有 AXFocused 写入前使用现有原生焦点准备和只读 caret 查询；公开回执仍保守，未确认动作不重放。原生 tree 为 `6abc23df3e6066a73f5110bfe77c041def4cc90852b5c5d6bde9ae537033eec3`（453 项原生加 12 项支持输入）。两个成功 Writer 用例各有一个文档窗口；既有键盘目标歧义保护保留，不据此宣称任意多窗口焦点切换成功。 + +mini 新 v27 画布、下拉和右键菜单原生效果 3/3 通过，接收修饰键及恢复均有记录；F31 改动仅限官方 LibreOffice 文本区。新要求的官方 Codex 对比结论为无法比较：即使使用当前安装包匹配的 bundled CLI 和 mini 既有消费者 profile，仍在应用绑定阶段以 app-server exit `-10005` 退出，未派发任何原生动作。另行普通 stdio 初始化成功，原因尚未确定。不据此推断 Codex 功能失败、性能差异或零遮挡。对照自有服务、socket 和夹具已清理，共享远程服务保留。 + +## 本地实现 + +P1–P6 的代码和打包改动已落在独立 worktree。Node REPL 只暴露 `js/js_reset`，默认 30 秒，预载全局 `cua` 和当前平台文档。Driver 管理应用/窗口解析、短 ID、三平台观察 diff、截图坐标和派发错误；直接 MCP 默认 19 个操作。macOS 保留后台 AX/定向事件；Windows/Linux 仅在明确派发前拒绝时转前台。显式桌面输入和 held keys 绑定会话，正常结束/reset 等待释放。浏览器接口直接返回仓库 browser-use SDK 的对象，未修改浏览器功能。 + +Native Wayland 实现已删除，X11 应用（包括 XWayland 中的 X11 应用）保留。SDK、原生、browser-use 和 Node REPL 统一为 `0.26.0`；独立 npm 安装使用固定版本依赖，原生包按校验和下载。Windows 默认安装无需签名 UIAccess worker；显式 opt-in 仍校验签名和安全安装路径。Qwen CLI/core 的产品差异只有删除两个 bundled `SKILL.md`。 + +本地检查点：P1 `cf89555254`、P2 `d2995a50c5`、P3 `b7c4d9c86c`、P4 `56a3ec2ba5`、P5 `fcc473d4a6`；P6 由本记录所在提交承载。 + +## 已取得的证据 + +| 范围 | 结果与边界 | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 仓库构建/类型 | 根目录 build、typecheck 通过；最终 browser-use 类型检查通过。 | +| Node REPL | 68 个相关测试通过，含真实子进程 reset/dispose 异步清理。独立 tarball 在无仓库依赖的消费者目录安装，通过文档预载、变量持久化和 reset 验证。 | +| SDK / browser-use | SDK 51 通过、6 跳过；浏览器 594 通过。浏览器包装测试使用原 SDK，无 Chrome GUI 成功声明。三个 npm 包均实际打包，仅执行 publish dry-run。 | +| Linux 原生 | 独立 ARM64 容器中 locked all-target check、SDK/driver 构建通过。规范坐标测试 38、Computer Use 测试 5 通过。Xvfb 真实桌面链 96/96、App 坐标链 29/29；实际 display resize 跳过。 | +| Linux 首次安装 | 实际 archive/checksums HTTP 下载、缓存命中、残缺修复、坏 SHA 拒绝、缺 checksum 拒绝及 postinstall 14/14；从下载缓存加载真实 SDK/桥并完成 GTK 输入/截图/按键释放链 14/14。不是 native override 或开发 symlink 验收。 | +| macOS 原生 | all-target check、ARM64 和 Intel SDK/桥构建通过，组合的两个 universal 文件均包含 arm64/x86_64。仅在 ARM64 主机执行加载；未声称 Intel 运行或 notarization 验证。 | +| Windows | GNU 交叉 all-target check 通过；未执行 Windows 桌面/UAC 测试。派发后 UIA 错误和 worker 断连的纯状态回归确认不再发送第二次点击。 | +| 发布配置 | 同版本约束、依赖发布顺序和 production gate 测试通过。工作流没有被触发。 | + +收尾审查修复了三个消费者问题：旧归一化环境变量重复变换规范像素坐标；Windows UIA 派发后错误被当成可再次点击；直接 MCP initialize 仍给出旧工具和归一化说明。对应失败复现与修复后证据分别保留。 + +最终原生源清单为 P6 v5：688 文件，树 SHA-256 `c8a662cdc5c53eaa4cbbd14b146b8e844a86fc060d1b8f16a8f7bb4014c7dbee`。v5 再次通过 Linux/Windows GNU all-target、真实 MCP initialize/list 11/11、精确桌面链 96/96、实际下载 14/14 和缓存加载 SDK GUI 链 14/14。v5 SDK/driver 重新构建；桥因源和 ABI 未变，明确复用 v2 的 fresh build。早期 v2/v3/v4 证据仍保留各自版本。 + +详细原始日志与独立测试报告保存在当前 worktree 的 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/`，主报告为 `.qwen/e2e-tests/cua-upgrade-2026-10.md`，P6 报告为 `p6-validation.md`；自审记录位于 `.qwen/investigations/cua-upgrade-2026-10/parent-self-audit.md`。 + +完整差异和消费者自审在修复后连续两轮无新发现，独立测试工程师的 P6 边界复核为 CLEAN。当前环境未提供 Codex `/review` 工作流，未使用 Qwen Review 代替。 + +## P7 历史检查点（2026-10-19） + +mini 已解锁，并通过既有 GUI Node 宿主执行真实 AppKit 输入、截图和按键生命周期测试。Windows 已完成真实 MSVC driver、SDK 和固定版本 Node bridge 构建；Linux 在 ECS 内的 Ubuntu Xorg guest 执行真实 GTK/AT-SPI 测试。早期“锁屏”与“仅交叉编译”的限制不再描述全部当前状态，但 P6 的历史结果仍按原产物保留。 + +上游 `0.20.0 → 0.35.4` 的核对发现 P4 清单之外仍有适用修复。当前源改动及证据包括： + +| 修复 | 证据及剩余边界 | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Linux 左上角窗口坐标偏移（#4967） | GTK 同场景修复前点到相邻行，修复后精确命中目标行。 | +| Linux 缺少 `_NET_WM_PID`（#4214） | 同窗口移除属性后，修复前丢失发现与观察，修复后通过 XRes 保持正确 PID、窗口、AX 和截图。 | +| Linux 临时键盘映射生命周期 | 候选与上游共有：目标 GTK 主循环忙碌 3 秒时丢失 remapped Unicode。队列屏障候选在 GTK/Chrome 忙碌场景完整输入；最终 v5b 对超时正常返回 sent/unconfirmed，无回放；无协议支持在输入前拒绝。未代替物理 MPX 或所有 toolkit/IME 验收。 | +| macOS 非有限窗口尺寸（#5125） | 真实 CFNumber 测试修复前失败、修复后通过，保留正常和负坐标。 | +| macOS 截图 PATH（#3755） | 真机窄 PATH 复现 ENOENT；新候选在窄 PATH 和正常 PATH 均返回真实 1920×1080 PNG。 | +| macOS raw 点击（#2907/#4690/#5137） | 同 PID 父窗/sheet 复现中键丢窗口和双份事件；新候选两轮真机复测恢复精确窗口与正确目标事件对数。默认 App 中键原本正常，未混为同一路径。 | +| macOS Electron 后台滚动（#4861/#5079） | 正常 v4 SDK 七格真机通过：raw page/inner × AX/pixel 和默认 App inner 保持页面 +404px、内层 +114px，另一滚动区不动；内层到底时 raw/App 均真实拒绝、sent:false、两偏移不变。此前 v2/v3 的 down 逆向 -96px 保留为 before；只读原生诊断确认上方 ROW35 被误报为底边片段，v4 以可见文档顺序排除错误候选。每格一次请求，无重放、前台回退或浏览器接口，主路径仍为未确认 ACK。六项几何单测通过；v2 受控 43 项与后续受控程序启动超时独立保留，未称 44 项通过。普通 v7 独立重跑七格并全部通过；不是全部方向、AX 布局或真人并发验收。 | +| 元素 token 与原生对象绑定（#3616/#3873） | 真实 core/token/cache 受控交错复现旧 token 读取新对象；新候选通过 capture/resolve/reverse 三种交错，三端消费者编译通过；Windows native-v2 MSVC 缓存回归 5 通过、2 明确跳过，token 30 项通过。该复现未执行 GUI 点击。 | +| MCP schema（#4888） | 默认 19 与 EXTRA 70 个真实 wire 声明通过离线兼容检查；运行时仍拒绝非法目标组合。未请求真实模型 provider。 | +| Windows GUI 宿主控制台（#5136） | MSVC 测试确认无控制台宿主不弹控制台，有控制台宿主保留继承。 | +| Windows 截图失败资源清理（#4333） | 实际 GDI 分配及失败注入测试通过，连续失败无 handle 增长；交互桌面截图尚未验收。 | +| Windows 快捷方式环境变量（#3975） | 真实 IShellLink 修复前返回不可用字面路径，修复后解析目标且保留参数。 | +| 隐式会话空闲恢复（#4283） | 同一真实 MCP 连接等待约 34 秒后恢复调用；显式结束/命名会话保持终止，旧元素 token 失效。 | +| 录制归属/停止/时间（#3632/#3714/#5144/#5145） | 公共入口验证命名归属、encoder 错误和时间修复；独立复测发现隐式连接遗漏，二次修复通过同连接/跨连接/lifecycle 回归。 | +| macOS 录屏完成时机（#4238） | 真机修复前出现 false empty 和成功返回后文件增长；新候选 5 次 120ms 短录制均立即可完整解码，一秒后文件 SHA 不变。 | +| 录制 AX 预算与会话锁（#4215） | 真实 ToolRegistry 加受控 provider 的独立 before/after 证明：挂起的 after 捕获不再阻塞 state/stop；2 秒返回上限、同 PID 忙碌拒绝、迟到结果丢弃和 stop→start 隔离共 6 项通过。Windows 录制线程 COM 初始化/节点释放顺序两项原生 MSVC 测试及全目标检查通过；不是 GUI AX 挂起验收。 | +| macOS Retina 缩放元数据（#3328） | 同一真实显示模式 3600px/1800pt，公共接口修复前返回 1，修复后返回 2；未据此推断默认点击坐标错误。 | +| macOS 辅助应用识别（#3456） | mini 同一已安装 accessory 应用，修复前 default/running_only 均丢失运行状态；native-v3 两个公共入口均返回实际 PID 9402。保留精确安装路径与所有运行实例，不以同 bundle ID 误认其他副本。 | +| 输出路径含目录符号链接（#4210） | manifest 与公共 ToolRegistry 分别验证：修复前拒绝实际 `/tmp` 等目录别名下的新文件，修复后规范化授权路径并仅派发一次；目录外、断链和文件父节点仍拒绝。不是 GUI 截图验收。 | +| Windows named pipe 错误清理（#4279） | 私有 Win32 pipe 复现创建失败与释放后恢复；源码加入有界重试及统一 PID/SDK 清理。native-v4 实际 MSVC driver/SDK 构建通过，缓存 6 项、token 30 项通过；生产 daemon 重试/清理运行验收仍待交互桌面入口。 | +| 卸载 MCP 配置归属（上游 `ff770f5d5` 等） | 独立临时 fixture 验证五类 foreign 配置不再误删，按本次选定待删除 payload 的真实可执行路径清理。没有执行完整卸载；后续删除失败时配置可能已先提交，原字节备份可恢复。 | +| typed schema 数值格式（#3462） | 上游回归用例修复前失败、修复后通过；输入/输出保留合法格式与字面值，contract 37 项及生成器 1 项通过。 | +| Windows 安装器祖先进程保护（#4398） | 独立 12 项受控清理/legacy 委托验证通过；按 PID 清理并保护祖先，存在 driver 祖先时跳过任务级 End。密封 UnitOnly 通过，未改变用户 PATH 或执行真实任务/完整安装。 | +| Windows 自动启动与隔离安装（#3180/#4284） | 32 项独立受控测试、MSVC 11 项原生单测、补齐编译夹具后的全目标检查通过。取消/null 不再报成功、替换前不先删除任务、隔离 release 安装保留共享任务。真实 UAC、Task Scheduler 原子性、登录启动和完整安装仍未验。 | +| macOS release 安装失败恢复（#3297） | 两项实际 shell 分支 before 复现复制失败丢旧应用；适配后 18 个隔离场景及 350 项独立核验通过，涵盖 staged/复制后校验、复制/rename/link 失败、恢复失败保留备份、首次安装、TERM 与重复清理。已加入旧字节恢复等长期回归测试。只验证受控文件事务，没有执行真实安装/签名/权限操作；不移植自动 TCC reset，不声称 SIGKILL/断电自动恢复。 | +| Chromium AX readiness（#3783） | 16 项实际函数受控测试、2 项旧/新发布竞态对照、2 项实际 core 预算传播测试通过；共享 500ms readiness 预算，不为动作/截图/录制新增轮询。真实 Chrome 拒绝 opt-in，accepted 路径尚未验收。 | +| AppKit 启动窗口恢复（#4309） | 1 秒主动查找额度共享原观察起点；27 项实际函数受控验证和 G17 外层消费回归 49 项通过。修复睡眠后预算已耗尽仍主动重试的问题；Mac 单测 1 项及 Mac/SDK 全目标检查通过。最终 App body 和原生 IPC 不受此硬限时约束，真实 AppKit 启动时序尚未验收。 | +| 普通 macOS AX 遍历预算（#3882） | 真实遍历函数受控 before 确认普通 raw/App 在 1 秒后继续读取；修复后 27 个遍历场景和 3 个节点准入控制通过，准备时间独立、完整树恰好到期不误报截断，录制绝对预算和 legacy 节点上限行为保留。真实 core ToolRegistry、默认 App、Mac 版本/cache/token 消费链完成 6 次受控快照与 22 项独立核验，旧 ID 拒绝、已解析绑定不漂移、部分结果警告及清理均通过。Mac 树 6 项、缓存 4 项、core 版本比较 27 项单测及 Mac/SDK 全目标检查通过。完整平台/SDK/GUI 路径仍待验收；没有整体调用 1 秒硬截止或真机慢 AX 延迟结论。 | +| macOS backing-store 截图（#4996） | 独立受控 after 32 项通过；当前 Space 下 public/private raw/App 四链返回正确 PNG。后续正常候选在 B 位于 Space68、前台 A 位于 Space1 时，raw/App 截图内容与五处 ROI 通过，二者字节一致；capture 段 120 个采样均保持 Space1 和 A 前台/顶层，模拟输入 6/6 到 A。正常库未记录实际私有路由，不能据此归因；全屏退出按钮预置不成立使通用整轮结果仍为 FAIL，自有进程终止后清理通过。 | +| macOS 遗漏 AXWindow 恢复（#4068/#4085/#4509） | 四个原生文件已补精确 PID/SPI ID、受限 remote-token 和默认 App/树/facts 消费者；Mac 单测 3 项与 Mac/SDK 全目标检查通过，独立探测边界 24 项与外层实际消费函数 49 项通过。真实遗漏 AXWindow 验收尚未完成。 | +| macOS 特殊键标志 | 真机 Space 准备失败后，无投递原生复现确认候选和上游共有结构位丢失；四条 named-key builder 修复后通过原生 128 键码对照、显式修饰/释放与环境隔离验证，原生单测 11 项及 Mac/SDK 全目标检查通过。正常新候选单次 Ctrl+Left 后，独立 Space oracle 在调用起点后约 638ms 确认返回 Space1；没有补发，B 未收到被系统消费的 Left，不推定其真实事件 flags。 | + +Linux 稳定 nested Xorg 的焦点/结束会话批次 31 项通过，一项 TCG 时序不足保留为 inconclusive；正常响应时 Unicode 与映射恢复通过。后续受控暂停 GTK 主循环 3 秒时，候选和上游都出现 13 个 remapped Unicode 按键变为 NoSymbol，已独立定位临时映射恢复过早；原主 GNOME 冻结批次仍单独保留为环境混杂。客户端 ping 实验在 GTK/Chrome 通过，但 Chrome 回应早于 DOM 输入完成，不能作为语义成功证明；native-v5a/v5b 已分别验证机制与最终返回契约，超时仍可能丢字，必须先观察再决定下一步。 + +MPX 已改为确认准确虚拟键盘的完整有序批次；未确认/发送失败的设备保留 pending，下一请求在改映射、焦点或输入前拒绝。首次设备预热也须完整确认，并先于临时键码规划。独立受控生命周期测试由 before 的 7/10 违反变为 after 10/10 通过，13 项事件匹配回归、12 项 Linux 原生单测及全目标检查通过。真实内核跨批延迟、设备热拔插和物理 MPX 仍未验收。 + +macOS T04 在 native-v3 的三轮独立真机测试中均发生目标窗口升层，持续覆盖用户窗口至测试结束（约 3.1–3.3 秒）；用户窗口保持 front/key，模拟输入全部到达。双查询诊断确认全窗口列表与实际可见层序不同；第一版改为可见列表后，重新激活应用仍返回成功却未恢复层序，首轮失败。第二版只提升当前前台的 AXFocusedWindow，三轮已消除持续覆盖，但仍有 99.4、200.5、192.8 ms 瞬时覆盖，**零干扰门槛未通过**。后续一次只发准备事件的独立诊断未观察到覆盖，不能排除采样及现有 guard 掩盖短暂升层。另一次仅把请求 pair 改用公共 CGEventPostToPid：精确源 PID 的 mouseDown、原生 field editor 与强制最终 AX 焦点均确认点击生效，但 A 自身仍观测到 194.110 ms 覆盖，独立 WindowServer 层序口径为 196.695 ms；公共 publisher 未显示补救效果，未改生产代码。该格 10/10 模拟按键到 A，实际点击调用内仅一键、已观测覆盖期间零键，不证明覆盖时真人输入安全。T06 菜单、T10 目标自激活回归通过；显式 Desktop 前台 Unicode 输入精确读回。后续精确窗口 SPI 校验收紧仍须归入新候选验证，不混用前述产物证据。独立双顶层窗口夹具尚未成功启动，父窗/sheet 测试不替代该场景。 + +Electron 后台原生滚动已完成上述主路径移植及七格真机验证，包括已复现的内层逆向边界。无可见锚点时保留原几何策略，重排或多列 AX 图谱仍不在此次证明范围。普通 macOS AX 树的有界部分结果及实际消费者受控验证已完成，完整平台/SDK 真机慢 AX 边界待验。跨 Space 截图已有上述真机效果证据；准确 AX 窗口恢复、Chromium accepted readiness 和 AppKit 启动恢复仍须完成各自真机边界。此次跨 Space 公共 AX 返回 85 个元素但明确不完整，未证明 remote-token 实际恢复。默认 App 选窗、动作前校验和观察结果转发均属于核验范围，不能只改 raw walker 就报完成。其余可选原生/安装路径的取舍见逐提交覆盖台账。 + +全差异自审还发现 macOS 录制钩子未接管可操作节点的额外原生引用。独立测试复用实际 walker、hook 与缓存，受控 AX 数据加真实 CoreFoundation 引用计数证明：修复前三次调用累计泄漏 1、2、3 份，修复后完整及不完整树的重复调用均归零，既有 token 保持有效，七份 JSON 除耗时外一致。修复复用现有未发布快照的所有权对象，不注册或替换动作 token；三项原生回归测试通过。39 项证据已逐字核验；不将受控引用测试计为真实 GUI/AX 验收，也未验证序列化失败或真实预算截断。 + +同轮独立测试用完整实际工具模块配合受控 provider，复现 raw 双击和右键在语义效果发生后返回 AX 错误时继续投递像素动作。已尝试的 AXOpen/AXShowMenu 现在直接返回 sent/unconfirmed 正常响应和警告，不回放；成功语义路径及未提供语义动作时的像素路径保留。完整原生工具模块的双击六格、右键四格受控复验通过，三项不确定响应经真实 core 公开投影和 typed 校验后仍为 unverifiable/accessibility，警告保留。复核发现的 foreground 请求被误记为实际前台方式也已修复：未执行前台事务的 AX 路径明确报告 background。新增原生投影回归与既有路由单测通过。默认 App 提前转发是对照，不推定默认 App 受影响；尚无真实 AX 服务器“生效后报错”的 GUI 证明。 + +G34 的 #5124/#4856 一律禁止 native omnibox 粘贴/换行策略明确不采纳：已定合同允许 selected App/window 当前焦点的合法原生输入，ASCII Return 与非 ASCII AX 插入分开验证，插入读回不证明导航。hotkey、pressKey、paste 三个入口分别核对。#4835 的冷前台字段/选区差异仅属 raw/EXTRA，默认 App 不可达，保留窄范围待复现，未引入 AllWindows 或固定稳定等待。 + +G07 最新生产入口实测表明 SSH Session0 在 SDK 创建时就因缺少交互用户会话而退出（23ms），尚未进入 named-pipe 接受循环；私有 pipe 不存在，共享驱动未变。管道本身可运行于 Session0，但这不等于当前生产入口可达。G23 第二版独立启动诊断正常编译，唯一启动 8 秒无 readiness 或阶段日志。后续同一 binary、相同普通 Node 启动路径的一格诊断，原 8 秒窗口结束后仍未就绪；一次 1 秒进程采样 exit0，仅得到 89 条 `_dyld_start +0` 主线程采样，映像信息不可用，不能定位具体等待机制。独立 Electron 宿主加任务自有 native addon 夹具正在准备；此前两格仍未执行，两格均未派发 AXPress，后续诊断完全不创建 SDK/AX 会话，没有据此推断 TCC/EDR 原因或修改安全设置。 + +499 条上游变更已完成逐项分类，原“证据不足”和“仅引用 P4 未复核”项已归零；仍有适用差异待复现、移植或明确取舍，分类完成不等于验收完成。2026-10-19 最新 JavaScript 整库 build、typecheck 通过，SDK 测试 51 通过、6 明确跳过；缺少 corepack 的首次构建失败保留，后续使用任务内入口和本机已有工具/固定缓存版本完成，未全局安装。当前普通 v7 的 453 文件统一原生清单通过 Mac/SDK、Linux/driver/SDK 和 Windows GNU/driver/SDK all-target 检查，默认 SDK 构建通过；12 个精确编译支持文件另列，树为 `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693`。Linux x64 和 Windows MSVC 的 driver/SDK release 均从此清单构建通过；Windows 本轮 2 项 COM、11 项 autostart、1 项有界捕获回归通过。Linux 当前正常 direct-MCP GUI 批次 GTK 7/7、Chrome 3/3 通过。相对 G05 v4 新增上述录制引用及 raw 语义错误不回放修复；v7 独立七格滚动真机全部通过，使用 SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe`。独立 fixture IPC 确认 inner +114px/page +404px、耗尽边界偏移不变；143 个层序/鼠标/剪贴板计数采样均保持 A，自有进程和 LaunchAgent 已清理，共享进程生命周期不变；主代理逐字核验 634 项引用。8.444–12.213Hz 采样不证明零瞬时干扰或真人并发。未变更 core 沿用八项通过的精确源检查点。12 个精确编译支持文件另附清单，早期遗漏夹具的失败保留。P7 第一轮已覆盖 89 个当前变更路径并发现上述问题；后续完整一轮干净审查之后，第二轮发现设计文档缺少双语版本，补齐翻译后连续干净计数重置。最终更新范围的完整两轮干净自审尚未完成。三端完整真机回归仍在进行。详细 before/after、无效批次与产物身份保存在 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/p7-*`；不得把历史 P6 检查结果用于证明后续 P7 改动已全部通过。 + +## 外部验收历史检查点 + +Windows 普通 SSH 已恢复，当前 console/runner/共享驱动生命周期已核验。用户明确批准独占临时任务 `\QwenCuaP7Acceptance`：无触发器、现有 Administrator InteractiveToken、Limited、不保存密码、最长两小时，完成后核验并删除；当前 v7 交互验收正在准备。未借用其他进程令牌。用户也批准 mini 现有 Node 控制 Finder/TextEdit 的正常 Automation 对话框，以及本任务所需的测试机权限；无需修改 TCC 数据库。Linux 独占的真实 uinput/evdev/Xorg 链与原 nested guest 分开记录，nested Xorg 本身不证明物理用户输入、MPX 或 XWayland 验收。 + +工具层三方对照与 OSWorld 20 题模型评测尚未完成。此前两个模型的 provider 400 保留为历史失败记录；2026-10-19 当前配置的 gpt-6.1-sol 已通过全局 qwen 0.25.2 文本预检和真实 CLI→MCP→Node REPL 单次代码执行。合成图像预检第三次有效调用返回实际 PNG，模型正确识别四项颜色、形状和位置（2 次模型请求、1 次工具调用）；前两次超时/代码转录拒绝均未输出图像，独立保留。未创建原生桌面会话，不能视为 GUI 验收。20/20 个本地题目定义已定位并冻结，均默认 Linux，仅两题有明确 Windows 分支；历史 macOS/Windows 适配正在恢复。mini 两处既定线索的有界只读核查覆盖 1003 个非凭据文本文件，未命中目标组；已有 stable20 是另一版题目与 scaffold，不能替代。执行数为 0，当前模型与历史基线的 gpt-6-astra 不同。用户已明确批准只读导出历史 macOS 组 `group-5cff67aa55934a09a45160e3194b5801-s1` 与 Windows 组 `group-bc7a3539903b4735bc3508e04e5791d5-s1`。每次真实 AP 请求均通过 mini 官方 AP Client 0.5.1 的准确入口门禁，已取回 group/jobs/eval，artifact 导出进行中。原始导出仅留 mini 私有目录，不向本仓库或本地证据复制凭据。两组各有 20 个 terminal Succeeded job，但不等于题目成功：Mac 均分 0.2613、1/20 通过，Windows 0.1206、0/20 通过。模板提交分别为 `d1b8341d17659205b9aa735e0390f54301be0dc7` 与 `b70f9f90b3df26f177d78279bc942490f7cc37e7`，二者均用 azure.gpt-6-astra，与当前预检模型不同。未授权或执行任何新 AP 提交、重试或取消，没有匹配条件下成功率、token、轮次、耗时或 Codex/上游改进结论。这些发布前验收条件仍然有效。 + +## P7 v13 历史验收快照（2026-10-19） + +本节是 v13 当时取代早期检查点的历史快照。其中身份和开放项仅描述该检查点，当前范围及后续验证见下方。 + +### 当前候选与分发 + +当前原生冻结版为 v13：453 个生产/构建输入和 12 个精确编译支持文件，tree SHA-256 为 `c43e1b300706eff833a9999f889b7f7c418a414b6863d3b3c102ca339770f83c`。Linux 使用新编译的 v13 CLI/SDK。macOS 测试使用正常默认特性的 v11 SDK（`4131a02af8d64a88572e981dbd9dfa5e7d0eaa9ec5ac5509be341c159d59ce9d`）；Windows 使用原生 MSVC v10 可执行文件（`c5ca0054810eb4512986aef0f6231e81497ada15d1809bcb9bd5521016ef3fca`）与 SDK（`91f871c885e13e856ee8284af0efed149232abbf58d9c7fc4584dee3d1f02283`）。中间仅 Linux 的改动仍需逐目标编译输入等同性审计，才能让其他平台的产物支持当前验收。 + +三个待发布包的 npm pack 本地 dry run 均通过，版本统一 0.26.0：CUA SDK 34 文件、Browser Use 9 文件、Node REPL MCP 46 文件。全库 JavaScript build/typecheck 通过，SDK 51 项通过、6 项明确跳过。完整当前工作树快照与 main `a250fd033941cab10c88d6eb8beb5756405193b9` 的合并无冲突；该检查仅创建 Git 对象，没有更新分支/ref，没有 push、PR 或发布。最终格式检查和两轮完整连续干净的源码自审仍待完成。 + +### 原生修复与实际效果 + +- macOS G23 已取得真实 AppKit AX server 证据：v10 四格覆盖 raw/default App，handler 切换状态后返回 NO 或抛 NSException。每格仅一次请求、一次效果。异常返回正常 sent/unconfirmed，不重放指针。返回 NO 仍可能得到 AX 成功，单独保留。v11 回归待执行。 +- macOS 默认 App T04 对支持的原生文本框，单次无修饰左击走精确可编辑 AX 焦点。v11 三次 fresh 点击用例独立确认 field editor 与 fresh AX 焦点、值不变、未观察到遮挡。已观察的 canary 字符均到 A，但三次 click 请求窗口都没有新 keyDown，不能证明输入并发。完整 click 后 type 回归通过，正在补高频重叠测试。仍抬高 B 的纯指针诊断变体未采用，也不计修复通过。 +- Linux MPX 成功派发现在携带明确 background/LinuxUinput 执行记录，避免实际文字已到达后公共接口返回 action_outcome_mismatch。完整有序 raw-key 确认、pending 设备拒绝与会话清理阻止超时后的重放。 +- Openbox 3.6.1 将虚拟 master 的 core FocusIn 提升为全局激活。fresh 零按键对照归因到 WM/Xorg 行为。v12 对该已识别 WM 在映射、创建设备和输入前拒绝 MPX 键盘；六项原生请求返回 background_unavailable/sent:false，零输入。公共 hybrid 路径随后可仅一次走 foreground。原前台、core focus、指针、映射和层序最上窗口恢复；低层窗口顺序仍发生可观察变化，不计后台零打扰。Mutter 42.9 对应后台测试保持更强的完整层序。 +- Linux AT-SPI InsertText 的长度单位为 UTF-8 字节，而 caret offset 仍是字符位置。旧实现与上游 0.35.4 都会截断真实混合文本。v13 在派发前修复共享长度换算。十项 native/public/SDK/facade 用例精确回读中文、emoji、组合字符与非零 caret 插入，零 raw-key 事件。读回不改变运行时保守的 sent/unconfirmed 投影。 + +### 平台验证与剩余原生工作 + +Windows v10 完成 14 项原生回归、68 项 direct MCP/SDK/native/Edge 检查、10 项生产 Node REPL kernel 检查和 12 项普通私有 production serve 检查。后两者分别验证真实 kernel 生命周期及管道归属，并非 Node REPL MCP wire GUI 会话或注入后续 pipe instance 创建失败。普通 serve 支持低层 CuaDriver.connect；保留的 ComputerUse.connect 仍要求认证会话归属，不能绑定该普通 serve，文档已写明。WinForms 后台样本保持 A 与指针；Edge 输入按合同走唯一允许的 foreground fallback。上游 0.35.4 的 12 项 native 与 8 项 Edge 检查达到各自预期。没有可核验的 Windows Codex 入口。 + +Linux v13 的 53 项 focused native 测试、all-target 检查和 release 构建通过，一个既有 ignored 测试保留。十项语义 Unicode/caret 用例及 real-kernel MPX smoke 通过。v12 在私有真实 Xorg 显示上另行覆盖 native/SDK/Node facade 的响应、阻塞、超时、pending 拒绝、forget 后 fresh 生命周期。外部输入中止使用独立自有 uinput 设备，并非真人物理设备。软件渲染 Mutter 容器与 nested Xorg 均不冒充共享 GNOME、原生 Wayland、真人并发或 VM 性能证据。 + +macOS v11 完整 fixture 矩阵已执行，详细独立复核仍在进行。剩余观察保留为失败或限制：sheet 输入被 same_pid_keyboard_ambiguity 拒绝；WKWebView 输入被 no_focused_text_input 拒绝；两项短层序重叠样本待归因。非文档原生文本框 AXValue 已替换，但 fixture delegate 回调缺失；该更强回调 oracle 仍为 false，不增加第二次输入制造回调。正常 TextEdit 保存与 Finder 重命名仍需完成已获批的 Automation 回归。仅凭剪贴板 generation 不能证明内容已恢复。 + +Windows 临时任务已删除并读回 absent，runner 已退出，自有辅助进程均清理。Limited 任务配置与内置 Administrator 实际 High integrity 分开记录。共享 driver/进程寿命保持。Linux 输入进程与临时设备已移除，私有显示 fixture 保留供最终清理。macOS 持桌面互斥锁，仅清理精确归属本轮的进程与 LaunchAgent 身份。合成 canary 与采样的焦点/层序观察始终与物理真人证明区分。 + +### 匹配模型评测 + +已获批的只读 AP 导出完成，每次实际请求前都对 mini 上官方 AP Client 0.5.1 执行版本门禁。含凭据的原始导出仅在 mini 私有位置保留。两组历史评测使用 qodercli 1.1.52 和 azure.gpt-6-astra，镜像/template 版本不同。macOS 平均分 0.26128、1/20 passed；Windows 0.12058、0/20 passed。任务状态 Succeeded 不等于任务成功。历史 token 是不可用的零填字段，不是零消耗；Windows 仅 13 条 trace 有完整最终轮次/时间指标。 + +已找回精确 baked macOS pilot cases/plugins，20 项指令均与 AP 导出相同。Windows 19/20 相同：历史 001 指令写 Calendar.app，而恢复的 Windows override 正确指定 Thunderbird Calendar。基础镜像层的 seed、真正 runner/generic scorer、网站与依赖正在按完整 layer hash 和各平台 overlay 恢复。setup 必须保护现有文件/profile，仅清理本轮 listener/进程；Calendar 持久化、应用、codec 或网站依赖缺失必须让 preflight 失败。scorer/gold 不提供给自主模型。 + +fresh 匹配比较必须固定 harness、任务 seed 字节、指令/rubric、模型/provider 与预算，只改变预定的新旧工具栈。历史 Qoder 结果仅为参考，不能充当该控制基线。当前 gpt-6.1-sol 的全局 Qwen CLI 文字、真实 Node REPL 和合成图片传输检查通过。mini 独立部署的生产 Node REPL MCP 也已通过现有 SSH 连接验证 initialize、精确 js/js_reset 列表、真实 kernel/preload、reset 与 EOF 清理，未转移模型凭据、未创建原生 GUI 会话。当前候选 OSWorld 实际执行数仍为零。成功率不回退与轮次/token/耗时门槛继续开放；未宣称性能提升或三方持平。 + +## 历史前四项范围（2026-10-19） + +用户现阶段优先要求完成 macOS、Windows、Linux X11 真机验收，以及运行时、安装与打包检查。Codex/上游比较及 OSWorld/模型评测延后。未来模型调用已配置为 `gpt-6-sol`，本范围不新增模型/AP 任务。前四项通过不等于延后的模型门槛通过，也不授权发布。 + +### 候选身份与已完成检查 + +已验证的 v21 原生源码冻结版本包含 453 个生产/构建输入与 12 个支持文件，tree SHA-256 为 `11b0fafab2419448a16542a73cde08cea58ee67b3a7b922fdfe12fee456f9dad`。相对 v20 仅 CLI 入口改变：有效遥测偏好为 false 时直接执行有限命令，省去无用的观察子进程。偏好优先级与启用路径保持。macOS 后端仍为 v20 SDK：签名前 SDK 二进制 SHA-256 为 `886bf05463a6c8273d0125b28482ee4f5ba8e9c1cffe5b4d5230e5fef2e77eee`，候选 Developer ID 签名后为 `134b334ea8b7e912e51ac0c27383d71b6c245e17b8f99f69cbee6e08c07fb064`，代码段等同性已独立核验。Windows/Linux 后端输入与各自 v19 验收产物逐目标等同；新 CLI 检查保留自己的版本标签。 + +- macOS v20 fresh 通过 sheet 输入与单次 Return 确认、WebKit 点击/输入/提交、精确粘贴与剪贴板类型/字节恢复，以及双窗安全拒绝/正向输入。生产 Node REPL MCP 的八项真实 wire/生命周期检查通过，部署源码/依赖身份精确。C9 TextEdit Unicode 保存与 Finder 语义重命名另有独立磁盘读回。Finder inline 键盘重命名无法证明原生 parent/window 归属时安全拒绝。内部 key-window 准备包含 NaN 坐标 mouse records；旧 Swift JSON logger 会漏记，先前“无内部鼠标事件”的说法已撤回。 +- Windows v19 实际 MSVC 检查/构建、37 项 focused 测试通过，保留各自明确身份的原生 GUI 回归。生产 MCP wire 动作在 WinForms 一次真实生效后超时，没有重放，reset/EOF 清理通过。完整 CI 形态包通过自有 fresh、正式旧字节布局升级、repeat、uninstall/purge。显式部署 unsigned UIAccess 被拒绝，不声称真实 signed UIAccess 或 ARM64 原生执行。当前隔离卸载器通过 14 项实际 Windows PowerShell 回归，包括 owned Exec 与 foreign ComHandler 混合 task。共享进程受保护，临时任务已删除。 +- Linux v19 的 53 项 focused 测试、一个明确 ignored 测试、实际 release 构建，以及 native MCP 初始化并列出 19 项工具通过。真实 Mutter/Xorg/uinput 输入保持完整采样焦点/层序并精确输入 Unicode；生产 Node MCP 的 15 项真实 AX 读回检查通过。CLI 安装与 SDK 冷/热缓存测试使用完整包。Openbox fallback 恢复原前台/最上窗口，但低层层序改变，不计后台零打扰。自有合成输入与物理真人并发分开。 +- 三个 npm 包在 0.26.0 的 dry run 通过；SDK 51 项通过、六项明确跳过。修复后的 Node MCP preload 测试先将 SDK 求值阻塞在已观察的 barrier 后再触发超时，并验证下一次真实请求；整个 MCP 文件八项通过，保留批次共覆盖 113 项 Node 测试。独立 mutation 证据能检出迟到 continuation。安装器 Python 检查 57 项通过、五项明确跳过、四项 subtest。 + +后续 v22 冻结版本（`de5d5030e7dbf5b3f64c7946dc8d335ee2d988db14e8fe0994226d2f6f980245`）新增仅 Excel 的 Apple Events 兼容修复，并修正窗口层序注释。Excel 使用 full-name1773 和 saved1842，替代缺失的 Standard Suite 路径/modified 属性；唯一保存前严格匹配完整路径及大小写。精确路径和不匹配路径的只读 argv 检查通过。六项既有 Apple Events 单测、Mac/SDK/CLI all-target 检查与 ARM release 构建通过；SDK 二进制哈希为 `04d996ff2b51337bf07a72da408dee0ea0b987081b209bb680e28e8eb3f92499`。ARM 与 Intel release 构建通过。fresh v22 原生 Excel 保存也已通过：唯一保存 316.586 毫秒返回，modified 从 true 变为 false，独立 XLSX oracle 确认 A1 精确标记且 B1 不变；两个旧 dirty 文档未改。该格点击仍观察到 48.937083 毫秒覆盖，因此只计保存功能通过，不计零打扰。以下更新后的 v22 Mac 包检查也已在披露范围内通过。Windows/Linux 目标输入未变。 + +### 运行时、安装器与包验证 + +v21 Windows/Linux CLI 附录现已通过实际构建、既有遥测偏好/关闭测试、默认关闭与显式关闭有限命令、native MCP 初始化/列表/EOF 和当前自有安装/repeat/purge。287 份文件与四个完整包已由主控独立核验哈希及成员。 + +mochi 上一致的 v22 macOS 候选对 CLI、theme、SDK 与 bundle 使用现有产品 Developer ID 团队 `NF4574S59H`，无时间戳、未公证。签名后 SDK 哈希为 `5a0dd6d011e633fee9bd6b751444ee82430cc0c8632a5ab43e5c4143cfa93946`，打包代码段与 fresh ARM/Intel 输入相同。ARM 与 Rosetta Intel 的有限命令/native MCP、安装后 ARM、SDK 冷热安装和生产 Node MCP 十二项均通过。新装、实际完整签名 0.17 bundle 私有副本升级、重复安装共十四项断言通过，替换产物满足旧签名要求；原全局 bundle 与 v21 产物不变。主控独立核验了全部 636 个普通证据文件（tar 总共 738 个条目，含 102 个目录）、四个包及其成员、原生身份，以及 CLI/theme/SDK/app 的全架构签名。未改动的 CI NAPI 产物保留其 Intel slice 未签名的已披露边界。 + +这些检查在披露的自有/本地候选范围内完成 v22 第四项：macOS ARM 加 Rosetta Intel，Windows/Linux x64 通过目标输入未变核验。安装测试采用已记录的目标路径/可写性/下载/stop-helper seams，文件系统、签名与替换操作真实执行。两项真实 rollback 故障用例保留自 v21，安装器源码未变，不冒称本轮 fresh v22 故障测试。共享服务受保护，证据不延伸为 stock 全局替换、Windows signed UIAccess、ARM 硬件覆盖或正式公证。此前 installed ad-hoc ARM 启动失败仍保留，一致 Developer ID 候选无需改变安全策略即可通过该安装后运行路径。原有绝对路径 nested-local SDK 导入缺陷保留，base/current loader 字节相同；installed bare-package 与全局 `cua` consumer 另行通过。 + +### 剩余门槛与证据边界 + +macOS 画布指针输入仍使目标覆盖 A 约 100–200 毫秒，虽然前台身份与 canary 到达保持。菜单覆盖也仍开放。实际效果正确、最终恢复不满足零覆盖。authentication、public-post、key-preparation、默认 mouse subtype 与 PrivateSource 诊断变体仍有覆盖，均已否决；私有变体不替换正常产物。PrivateSource 虽然原生效果精确一次，仍覆盖约 199 毫秒。只读因果复核不能证明系统不可实现；这些指针因子都没有处理 AX-only popup 覆盖。后续 fresh 完整省略 preparation 的反事实仍覆盖约 206 毫秒，并丢失原生画布效果，因此也否决该候选。之后的只读复核没有找到有证据支撑的下一项外部 driver 修复假设；这不证明系统不可实现。 + +正常 v20 fresh 滑块替代路径通过：唯一 `setValue('80')` 同时得到 value80 与独立原生 callback80，未观察到打扰。Excel 原有选中网格状态与 LibreOffice 刚打开文档的状态均在派发前返回 `no_focused_text_input`，不计文档保存成功；Excel Edit 后续通过原生公式编辑栏 description 确认输入标记，通过 A2 确认 Return。首次保存遇到 Automation 弹窗及 caller deadline，结果未知；正常授权后，新文档唯一保存明确返回 `automation_failed (-50)`、`sent:true`，磁盘没有标记；以上 v22 原生保存修复已验证，指针覆盖仍为独立失败。LibreOffice 公共正文 click 后仍聚焦 group；另一次只读正文检查确认 AXValue/AXFocused 可写，但实际焦点为 false。不以替代写入或保存成功计分。原生 still-launching 公共链现已通过：独立确认的已安装 accessory 夹具在唯一 `getApp` 开始时仍未完成启动，真实启动延迟尚余 900 毫秒以上；1.056 秒后返回精确 AX 标记，独立观察随后确认启动完成。动作加 settle 的 20 份层序采样未见自有窗口遮挡，六次合成 canary 均到 A。这不能证明具体执行了哪次 retry，也不能证明旧 400 毫秒实现失败：首个 AX IPC 本身可能阻塞至 ready。先前无请求与 scratch helper 被排除的尝试单独保留。MCP 一项 overlap 前采样有外部 System Settings 窗口在 A 上方，缺少逐 RPC 绝对时间，不能归因给候选，也不能称全程 A 在最上。 + +不可变报告、失败原始文件和 manifest 保留在 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/`，包括 `p7-normal-v20-native-regression-independent-report.md`、`p7-windows-first4-final-v19-20261019.md`、`p7-linux-final19-first4-acceptance.md`、`p7-mac-zero-cover-causal-boundary-review.md` 及各版本打包附录。仅证据的 clean pass 不计入最终完整源码连续两轮自审。先前 main 合并检查属于历史记录，最终源码验证和可合并性仍需覆盖完成后的整树。没有 push、PR、merge 或 release。 + +### 历史 v25 检查点（2026-10-20） + +当前453个原生输入加12个支持文件的 tree SHA-256 为 `6faac284650aeffaa64c5fa6b220ab1799ae25f69f0e68845e21159bf7d50e5a`。Linux v24 修复前缀/首字符/按键释放/flush 的部分派发分类、MPX 焦点准入后的 pending，以及失败按下/探测键释放清理。实际 Linux 定向测试、all-target、CLI/SDK release、私有安装、有限 MCP 及 SDK 冷热检查通过。fresh Xorg/Mutter 四例覆盖公开 CLI 中文/命名键、当前 SDK C ABI trusted-session 路径，以及暂停自有目标后的单次不确定请求。最后一例恢复后仅得到 `a`，回执正确保持已发送未确认；返回、等待1.7秒及结束会话三点没有新增物理事件，不算完整文字送达。主控核验38份证据、原回调、不同旧/新 Xorg 完整日志及自有容器清理；早期尾部片段计数夸大保留原件和勘误。 + +v25 相对 v24 只改 Mac 精确焦点解析。真实 LibreOffice 失败显示：同窗应用焦点指向明确 AXFocused=false 的 AXGroup，而正文实际 AXFocused=true。仅此精确条件转入现有有界、完整且唯一的文本焦点证明；PID/窗口绑定、跨窗拒绝和末尾复核均保留。ARM 与 Intel CLI/SDK release、六项既有精确目标测试及 all-target 检查通过；新 ARM SDK 的 Developer ID 签名已核验。一致的通用 Developer ID 新包已通过 ARM/Rosetta 有限命令与直接 MCP、打包 SDK 冷热安装、生产 Node MCP，以及14项自有新装/正式0.17升级/重复安装断言。主控独立核验四份归档及全部38个普通文件成员实例、11份结果报告、465个源码/支持 pin 和五项全架构签名。签名保持离线、无时间戳/公证，安装使用已披露私有 prefix/本地下载 seam。完整当前源码自审与最终合并准备仍独立进行；该 Mac 修复未改 Windows/Linux/共享输入。 + +fresh 原始 T15 的一次公开 ASCII 插入，经独立正文 AXValue 和选区读回确认。唯一 Save 因等待系统 Automation 权限超时，报告 sent:true，独立磁盘没有标记；未重放 Save。LO 即使正文已改变仍报告 AXEdited=false，不能把该属性当保存 oracle。本机 canary 进程没有投递权限：121次 posting 尝试与接收端零 keydown 不能证明 canary 送达或用户输入不中断。数值前台/顶层/指针采样只保留其有界观察范围。上方 LO 焦点结果保留历史身份；正向插入现已通过，文稿落盘验收仍开放。 + +严格指针/菜单零遮挡仍失败。fresh 签名自有窗口 bit21 正控中，Set 返回0但该位读数未变，随后一次 Clear 返回0且基线精确恢复；不能采用为外部窗口修复。早期无签名 helper 的 SIGKILL/空日志仍阶段未知，不据此断言整个系统不可能。真机门槛、最终检查及当前树合并准备未全部通过前，整体验收和 PR 准备均未完成；没有授权 push、PR 或发布。 + +### v26 LibreOffice 指针安全检查点（2026-10-20） + +上面的 v25 检查点属于历史记录。当前 v26 原生 tree SHA-256 为 `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3`，465项输入 manifest 为 `f16523c3823554f6d0e74d257ee232d768b7b8cebe17a80bf6b787d2591b4042`。相对 v25 仅改三个 Mac 点击工具，Windows、Linux、共享原生输入与 JavaScript SDK 均未改变。 + +新建 Calc 文档的一次默认公开点击请求以 C5 为目标，但三次同窗独立 Name Box 读数均为 B28。原请求已发送未确认，未重放。精确 LibreOffice 26.2.6.3 源码通过全局 `NSEvent.mouseLocation` 计算 VCL 鼠标回调位置；本次截图坐标反算一致。这确立了全局指针读取与定向事件不兼容的机制，尚不能唯一证明产生 B28 的具体原生回调。 + +本修复借鉴上游 #4864 的拒绝原则，使用本 fork 现有 NSRunningApplication 身份识别 `org.libreoffice.script`。其 PID 定向像素点击、右键、双击及元素像素回退不再作用于无关的硬件指针位置；已广告 AX 动作与安全的可编辑焦点/AX选区路径保留。旧 raw foreground 指针后端没有移动硬件指针，因此也拒绝。该识别比上游映射库检测更窄，不覆盖改名或无 bundle 的 VCL 进程。规范公开 Mac 动作仍固定后台，不新增 JavaScript foreground 选项。 + +另一个新文档的 v26 请求明确返回公开 `background_unavailable`、`sent:false` 和请求级 `committed:false`，三次有界同窗 Name Box 读数均保持 A1。通过的是安全拒绝验证，不是后台 C5 选区成功，也不是 raw AXPress 真机等价。原严格 harness 因错误期待公开错误投影不会保留的内部 effect/toolkit 字段而失败，原输出与独立按现有 code/reason/sent/next 契约作出的判定分别保留。SDK 的 `operation.dispatched:true` 仅表示进入 FFI 请求生命周期，不是物理输入证据。 + +当前 ARM click14、AX-dispatch1、double-click1 定向测试、all-target 检查及 CLI/SDK release 构建通过，ARM SDK 离线 Developer ID 签名已核验。当前源码的包验证与其他真机回归另行记录。临时本地验证提交 `a9fe798b6dac938392f450f2edf55456a6dfc6e0` 保留固定主线 `5dcf7c7cb162e825d95d531081f67ee60da3c0e1` 的实际干净合并；完整8GiB构建脚本201.876秒、正常root typecheck72.948秒均通过，后者含integration且不带NODE_OPTIONS。这些是验证耗时，不是性能对比。此前默认4GiB合并候选/主线失败均保留，v26未重跑或宣称该命令通过。 + +独立 v25 公开工具栏保存已通过实际 FODT XML 核验，原 Cmd-S/AppleEvents 保存仍 UNKNOWN、从未重放。后续自有窗口 bit21 正控通过，但已信任宿主以正常 caller 修改外窗的试验未改变该位，精确恢复后否决。历史 Codex 画布样本在调用未指定修饰键时收到 Command，来源未知，不能作为普通无修饰点击基线；其 popup/context 组合样本也保留实际遮挡失败。严格 Mac 指针/菜单零遮挡仍 FAIL。源码自审、真机验收和 PR 就绪分别判定；未形成最终 P7 发布提交，没有 push、PR 或 release。 + +### 历史 v27 Writer 焦点候选——未经真机验收（2026-10-20） + +v26包检查点已完成：ARM/Intel构建、定向检查、一致通用归档与全部11组非GUI运行时/安装报告通过。新独立读回核验92份证据、38个归档普通成员、465项源码输入和5项全架构签名。这些包/协议结果不关闭Writer或零遮挡验收。 + +受控源码切片执行实际v26谓词，确认官方LibreOffice的无广告action AXTextArea即使value/focus属性可写仍会被拒绝。另一次针对保留v25 Writer进程的目标限定只读查询，确认唯一同窗正文、空action及AXValue/AXFocused settable成功且为true。应用级focused element为AXGroup，正文却报告AXFocused=true。这是旧实例真实属性，不是当前动作通过。 + +中间v27候选相对v26只改click.rs：默认App的单次无修饰左键分支可通过既有可编辑焦点路径聚焦已识别的官方LibreOffice AXTextArea，保留两项settable及当前PID/窗口归属检查，其他应用的textarea仍排除；raw点击实现、通用角色helper及原测试与v26逐字节相同。焦点/值确认规则未放宽：AXGroup不匹配仍可返回未确认，之后须观察；没有像素fallback、重放、光标位置改写或更强成功声明。原生tree为`c52f7cb6ac11aa65a9d7a226c5d4e38f55ee52820bcc4085ed4314b006b40538`，manifest SHA-256为`a3aed6e4f6b77b532395729b611f9817de06bac65c6d64c4811e0ed17c870ad1`，共465输入。定向构建/review与尚未完成的fresh Writer点击/输入GUI验收分开。 + +最新Writer Stage1因自有sentinel无法取得前台，在LibreOffice及文档动作启动前停止。Computer Use拒绝访问该系统app，系统窗口内容仍未知，没有换通道绕过；mini经新认证只读检查仍锁屏。指针/菜单严格零遮挡仍FAIL。完整8GiB构建/typecheck成功属于本地合并检查点a9fe798b，不是开发checkout全库构建通过。仅计划将之后四份Markdown更新与该Mac Rust增量同步至本地验证快照；继承构建证据时须明确此边界；新验证提交还会改变构建生成的Git commit常量，不能声称编译输入严格逐字节等同。没有最终P7发布提交、push、PR或release。 + +## F31 原生与分发检查点（2026-10-20） + +已采纳 Writer 源码相对 v27 的 click.rs 增加九行受门禁保护的逻辑，与 mini 实测 SDK `091df63289f3542c8d1518f0be0db458863b7f87d4002de281579fca377509ab` 完全对应。两个新文档均在一次 App 点击后，独立读到正文 AXFocused=false→true；各自一次输入和一次工具栏 Save 保存精确 XML，文件 hash 分别为 `83d5fd0c80a0e654d18f55b6d181bfb6be6ca1463a09b1885cad45393071cdb2` 和 `c4d557fa50ab5d7f449891d8fd139f0da24334880a44b31268bdd911f7ab01d1`。主控核验全部 46 份封存证据及每例首次原生读回。指针与文本剪贴板保留各例起始值；最大 122.5/118.1ms 采样间隙不证明全程零遮挡。此前 v27 桌面点击/输入/保存属于另一显式前台流程；原 Cmd-S/AppleEvents 请求仍 UNKNOWN，从未重放。 + +F31 ARM/Intel CLI 与 SDK release 构建、相关原生及 all-target 检查通过。一致通用包通过全部 11 组非 GUI 报告：ARM/Rosetta 有限命令及直接 MCP、packed SDK 冷热安装、生产 Node MCP、私有自有安装/替换，以及安装后 ARM/Rosetta 命令/MCP。最终 manifest 保留精确 slice/归档/签名身份，并保留离线 Developer ID 签名和私有安装器 seam 边界。一次任务自加的 caffeinate 身份预检在这些检查启动前失败，勘误与随后实际完整运行分别保留。Windows v21 与 Linux v24 保留原真机身份，此后 Mac 改动未改变其目标输入。 + +新差异的正式 Codex review 提出一项多窗口疑问，已记录并 triage，不宣称零反馈。现有准备会发送进程级焦点 records，因此保留键盘歧义门禁；两个单窗口正例不证明任意多窗口支持。此前完整源码自审保留固定范围；最终更新差异自审、最新 main 默认构建/typecheck、源码等价与 PR 草稿另存忽略的最终就绪证据。本地源码/构建结果不代替原生效果。 + +前四项范围为三平台原生效果及运行时/安装/分发验证，保留已记录的安全拒绝和不确定性边界。G17/G18/G24 accepted-opt-in 原生覆盖、later-instance 管道故障注入及物理真人并发仍分别未经验证。新要求的 Codex 行为对照在动作前受阻，无法比较；同条件模型/OSWorld 成功率、token、轮次与耗时仍延期。均不默记通过，也未授权发布。 diff --git a/docs/design/cua-upgrade-platform-ports.md b/docs/design/cua-upgrade-platform-ports.md new file mode 100644 index 00000000000..21cd025edb2 --- /dev/null +++ b/docs/design/cua-upgrade-platform-ports.md @@ -0,0 +1,564 @@ +# Computer Use platform ports (2026-10) + +[English](cua-upgrade-platform-ports.md) | [简体中文](cua-upgrade-platform-ports.zh-CN.md) + +This is the implementation ledger for the approved [upgrade design](./cua-upgrade-2026-10.md). Code adapted from trycua/cua through `cua-driver-rs-v0.35.4` is MIT licensed, Copyright (c) 2025 Cua AI. Only the native driver library is a port source. The upstream bench package and its license are not included. + +## Dispatch contract + +App input resolves an exact application/window inside the driver. macOS uses accessibility and process-directed events; there is no automatic global foreground fallback. Windows and Linux may try the foreground rung once, exclusively after a native `background_unavailable` error with `sent:false`. An actuator error after a possible dispatch never replays input. Explicit desktop input is a separate foreground surface. + +Foreground transactions observe physical input without consuming it. They stop remaining input on detection, preserve an external foreground C, and restore prior A only while the transaction still owns B. Windows drains the target input queue before restoring; text checks the target between UTF-16 units. Linux confirms both EWMH activation and core input focus before XTest. + +Success acknowledges dispatch, not application effects. Input calls do not wait for screenshot changes. The next observation supplies effect evidence. A held press and a multi-click gesture retain only the timing required to deliver the gesture. + +## macOS + +| Upstream reference | Adaptation | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| #4943 | Do not send A a synthetic deactivation notice when directing focus to B. | +| #4943, #4955, #5091, #5129 | Key-focus protection shares the existing per-process `app_focus` observer; bounded tail/order correction and intentional-focus epochs prevent competing observers. | +| #4976 | Physical mouse/modifier activity grants the user's switch priority. The existing activation guard checks the same physical-input state. | +| #5051 | Directed mouse events explicitly set modifier flags, including the empty set. | +| #5057 | Existing-file saves and Finder item renames use bounded Apple Events and readback. Arguments are passed as script arguments, not concatenated into AppleScript. Automation consent follows normal macOS TCC. | +| Popup selection | Open a popup, select a unique matching option through AX, close a menu opened by this call on refusal. Safari's existing DOM fallback remains restricted to the actual foreground document. | +| Text insertion | ASCII uses directed events. Non-ASCII requires a focused editable element and AX selected-text support; expected insertion is read back. No keyboard replay after an uncertain AX write. | +| Document model | A verified AX write to a document-backed AppKit field registers an edit through a directed, net-zero editing shim; refusal is reported if registration cannot be verified. | + +Background drag refuses before dispatch. Sliders use `setValue`. Paste keeps the clipboard restoration transaction. The old fixed one-second post-action wait is removed. Neither AX acknowledgement nor event delivery alone is reported as a confirmed effect. + +P7 controlled-provider tests of the complete native tool modules reproduced +raw double-click and right-click replay after an advertised AXOpen/AXShowMenu +applied an effect but returned an error. Once +either AX action is attempted, its error now returns a normal sent/unconfirmed +acknowledgement and warning; no pointer route follows. Successful semantic +actions and a pointer route selected because no action is advertised retain +their behavior. These AX acknowledgements explicitly identify background +delivery, even when the raw request selected foreground, because this semantic +branch performs no foreground transaction. Independent controlled tests execute +both complete native tool modules and the actual core public projection: all +six double-click and four right-click cases pass, with three uncertain cases +remaining unverifiable/accessibility and preserving their warning. Default App +early forwarding is a control; these tests do not demonstrate an App defect or +an actual macOS AX server returning an error after an effect. + +## Windows + +| Upstream reference | Adaptation | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #4500 | Attach to the actual target input queue and use a sentinel read to bound queue draining before restoration. | +| #4901 | Check foreground ownership and physical input between UTF-16 units; stop after a stalled unit without replay. | +| #3671 | A timed-out UIA point action is distinct from a preflight absence of a suitable pattern, so an uncertain action cannot become a second coordinate click. | +| #4774 | Native combo selection uses the control's selection and notification messages and verifies the selected value. | +| Tk | Refuse the unsupported background pointer rung before input; the shared App dispatcher may then use foreground once. | +| Extended keys | Left/right Windows keys carry the extended-key flag. | +| Worker IPC | Bound worker response reading; a timeout says input may have been sent. UIAccess routing is explicitly opt-in. | +| Standard-user launch | The elevated daemon derives and verifies a Medium token from its own token with administrator membership deny-only. No other process token is opened or borrowed. Direct executable and shell-helper launches use that token; failures refuse rather than launching elevated. | + +The normal installer/UAC flow is retained. Cross-compilation is build evidence only; it does not prove a normal interactive Windows desktop, UAC behavior, token facts at runtime, or shared-desktop interference. + +## Linux + +Native Wayland support, helpers, dependencies and installer payloads are removed. X11 and XWayland X11 applications share the same X11 implementation. A compositor-native Wayland surface is unsupported. + +| Upstream reference | Adaptation / existing equivalent | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #3882 | MPX pointer/keyboard dispatch, toolkit-aware refusal, occlusion and focus guards, no input shields. Existing fork exact-window checks remain. | +| #3761 | Flush/check key press before hold timing; unconditional final server round-trip before closing a short-lived connection. A later P7 reproduction shows that server delivery alone is insufficient for temporary keymaps; the client-queue fix below remains under validation. | +| #3762 | Check pointer presses/releases, keep gesture timing, release a queued button/modifiers after an error. | +| #3864 | Preserve the fork's exact observed bus/object click identities and per-window snapshot validation. Session-owned MPX device persistence comes from the MPX umbrella rather than this identity patch. | +| #3618 | Modified foreground drag brackets the gesture with real modifier transitions and cleanup. Unsupported modified background drag refuses before dispatch. | +| #4248 | The new App contract has no post-action effect classifier: closing B's dialog cannot be mislabeled a no-op because the WM parks focus. Existing exact-target foreground restoration preserves C and does not infer confirmation from unattributed popups. The upstream screenshot/window-change wait is intentionally not copied. | +| #4188 | Exclude application/frame/window shells from coordinate AX activation; recognize both NUL-separated and space-joined Chromium helper command lines. | +| #5139, #5158 | Unicode keysyms and full-character preflight; reserve/remap spare keycodes before the first event; serialize temporary keymap changes. No silently skipped characters. | +| #3601 | Master ownership includes boot, PID namespace, UID and process start identity. Reap only stale devices proven to belong to this driver family. | +| #2888 | Conservatively refuse MPX on KDE where device hotplug may reconfigure physical input. No automatic permission changes to `/dev/uinput`. | +| #5070 | Background keyboard refusal names a missing AT-SPI bus and its normal setup path; the diagnostic bus probe is bounded. | +| #4903 | Remove AT-SPI P2P feature/peer calls; use one registered accessibility connection. | +| #4984 | Rebase tree bounds only against a valid complete screen frame; reject incomplete/zero-size frame geometry. | +| #5074 | Wake an AT-SPI application with attributes before its first child walk. | +| #4375 | First semantic walk budget is two seconds; later walks one second. No generic full-tree retry for an incomplete observation. | +| GTK combo | `Selection.SelectChild` picks a unique option and its selected-child name is read back. No focus/foreground change for selection. | +| #4292 | Preserve text content separately from labels; prefer numeric Value for numeric controls and do not read password Text. | +| #3758 | Decode DirectColor screenshots with the X colormap rather than treating them as TrueColor masks. | +| #4281 | A successful kill waits for actual process exit; zombies count as exited. | +| #3814 | A delivered exact-window frame explicitly reports `screenshot_frame_valid:true`. | +| #4529 | Prepare overlays unmapped with empty input/bounding shapes; map only while painting visible content and unmap idle frames. | +| #4715 | Non-Linux stub registry no longer advertises the removed screenshot tool; portable menu schemas do not pass through cursor-only schema assertions. | +| #3658 | Enter/Return/KP/Numpad aliases share the terminal key gate and key mapping. | +| #3778 | Existing opt-in native browser tools share the session cursor registry; inactive tabs do not move the visible cursor. This does not add upstream browser tools to the default surface or change the repository Browser SDK. | + +Fork repairs also preserve per-window screenshot scaling, do not scale an omitted mouse-up position twice, restrict Qt synthetic FocusIn to Qt5, condition FocusOut on actual core focus ownership, and classify Chromium only when a route needs the information. + +### P7 Linux temporary keymap lifetime + +Both the candidate and upstream lose remapped Unicode characters when an owned +GTK application's event loop is paused for three seconds: the server delivers +the keycodes, the driver restores the map, and the client later translates them +as NoSymbol. This is independently reproduced in nested Xorg, separate from the +unstable main GNOME session. + +The candidate now prepares an X11 client-queue acknowledgement before changing +any mapping or sending input. Only operations needing a spare keycode pay this +cost. The target or a same-X-client ancestor must advertise `_NET_WM_PING`; +another connection in the same process does not qualify. XTest also checks the +focused client before each character and before/after the acknowledgement. +After dispatch, the exact nonce echo on the root window releases the temporary +maps. A five-second timeout restores the maps and returns a normal +sent-but-unconfirmed acknowledgement, with no replay. A detected focus change +during partial typing still stops remaining input with an error. Missing protocol +support refuses before input. The existing ASCII path retains its server +round-trip, with no added ping. Sparse pointer modifier maps refuse before input +because a borrowed keycode does not establish a real modifier mapping. + +MPX additionally requires raw-key server-delivery confirmation before sending +the client ping. A controlled reproduction found that the last-key predicate +accepted an earlier repeated release after only two of six planned events. The +revised predicate requires every transition in order from the actual virtual +slave, including repeated keys and Shift. A legacy zero source ID is accepted +only if the event's device ID is that exact slave, never the shared master. +Unexpected same-source transitions leave delivery unconfirmed. A retained +virtual keyboard with unconfirmed input refuses the next request before +mapping, focus or dispatch; normal session cleanup removes that state with the +device. Device warm-up must also confirm its full batch, before the planner +can borrow a keycode. The existing keymap-to-MPX lock order is preserved. +Independent controlled lifecycle cases pass 10/10, matcher cases 13/13 and +actual Linux unit tests 16/16 after the warm-up preflight below. Kernel backlog +and physical-device validation remain pending; the controlled tests replace OS +boundaries with stubs. +Before warm-up, XKB must identify the actual slave and prove X248 has no symbol +in any group or level. An occupied, invalid or unreadable mapping refuses with +zero emissions and retains the pending state. The fixed keycode is never +temporarily remapped to make this check pass. Independent checks cover 20 +mapping/query cases and three retained-pending entrypoint cases; C header and +Rust layout measurements agree. The real read-only Xephyr slave's empty-group +mapping is a shape control, not evidence for a new uinput device. +No focus grab, input shield, clipboard change, permanent keymap or new helper +process is introduced. GTK and Chrome probes establish the queue mechanism; +Chrome's echo precedes DOM input completion, so it is explicitly not semantic +success. Native-v5b GTK responsive, three-second blocked, timeout, wrong-nonce +and unsupported-protocol cases pass their respective effect/refusal/uncertainty +checks. Chrome's responsive/blocked/timeout mechanism checks use v5a; v5b changes +only uncertainty reporting, which is tested separately. Physical MPX remains +unverified. + +## P7: upstream correctness closure + +### Ordinary macOS AX walk budget (G31 / upstream #3882) + +The ordinary raw and default App observation currently limit tree depth and +node count, but only the recorder has a time budget. Adapt the upstream +one-second node-walk budget to the ordinary observation entry. Start it at the +first admitted, nonduplicate node, after window selection and readiness work. +Keep the separate Chromium readiness and AppKit launch budgets, the existing +native messaging timeout, and the outer observation timeout. This cooperative +limit cannot cancel a blocked native read or guarantee a one-second total call. + +A deliberate time cutoff must preserve the completed prefix and mark the +observation truncated and incomplete. Successful attempted reads retain +`read_complete`, matching the existing node/depth-cap semantics; an actual AX +read failure still clears it. Bounded revisions return a full replacement when +the prefix changes, so missing rows are not deletion evidence. Raw tokens must +remain bound to their own immutable captured objects across later captures. +Verify these consumers with actual walker results before accepting the fix. + +The legacy node-only walker and recording's absolute deadline keep their +existing behavior. No public timeout parameter, new output field, worker or +action retry is introduced. A finite slow-node reproduction and setup-delay +control precede implementation; native latency remains a separate acceptance +boundary. + +### Windows installer parent protection (G32 / upstream #4398) + +The release installer's shared daemon cleanup can kill the driver process that +launched `update --apply`, including its installer child, and can misclassify +that protected parent as a stale daemon. The independent reproduction executes +the exact PowerShell functions with all process, task and pipe boundaries +stubbed; no real process or scheduled task is stopped. + +Adapt the upstream live ancestor walk, rejecting a reused parent PID whose +creation time is later than its child's. Cleanup targets only non-ancestor +driver PIDs and excludes ancestors from the survivor/repair list. If a driver +ancestor is present, skip the task-wide `/End` operation as well: a PID filter +cannot protect an ancestor from Task Scheduler termination. An unavailable +ancestor query stops cleanup before any termination. Legacy migration uses +the same helper instead of a second image-name tree-kill implementation. +The separate `-local` installer identity is unchanged. + +Validate ancestor and unrelated-process cases, stale reporting, PID reuse, +cycles, query errors and legacy delegation through isolated command stubs. +The regression script gains an explicit unit-only entry that exits before +builds and real Task Scheduler tests. This source change neither registers +nor changes a scheduled task or Windows permissions. + +### Windows autostart failure reporting (G33 / upstream #3180 and #4284) + +Replace a task with `Register-ScheduledTask -Force` instead of deleting it +first. Elevation must return a failure when UAC is cancelled, no process is +returned or the child has no exit code; otherwise forward the child's actual +exit code. Quote the executable as a literal PowerShell argument. Reuse the +existing task-status query to verify successful registration; an unavailable +query reports that registration could not be verified, rather than printing +an unconditional success. Cancelled or unverified elevation is not retried +automatically. + +Both Windows installers should print registration success only after their +registration call succeeds. An isolated release install selected by +`CUA_DRIVER_RS_HOME` or `CUA_DRIVER_RS_INSTALL_DIR` leaves the shared daemon and +autostart task alone unless +`-AutoStart` was explicitly requested. This guard also surrounds the shared +daemon cleanup before registration; guarding only the closing autostart block +would still terminate the ordinary installation's daemon. Default installation +and the separate local product identity retain their existing routing. + +Independent before/after tests execute the actual PowerShell payloads and +installer blocks against task/UAC/cleanup stubs. They must cover cancellation, +null/failed children, preservation of an existing task on registration failure, +accurate summaries and isolated/default/explicit parameter combinations. These +tests do not perform UAC, task registration or a live installation. Separately authorized native task and owned installation lifecycle tests have since run; their identities and limits are recorded in the current acceptance appendix. + +### macOS background window order (T04) + +The on-screen CGWindow query is authoritative for visible stacking order; +the all-window query can disagree even for windows it marks on-screen. +Real desktop diagnostics also found that reactivating the already-frontmost +application can return success after a second without restoring its order. +The guard will instead request AXRaise only on that application's current +AXFocusedWindow, after checking its PID and exact visible layer-0 window ID. +WindowServer must still name the same front application, and recent hardware +mouse/modifier activity prevents restoration. AX calls use a 100 ms messaging +timeout, references remain owned for the call, and the existing bounded watch +limits remain. This changes neither key focus nor the background action's +dispatch path. API acceptance alone is not proof of restored order. + +A single owned-window primitive probe restored visible order about 23 ms +after AXRaise while preserving all owned synthetic typing. Three production fix-2 rounds +eliminated sustained covering but retained 99.4, 200.5 and 192.8 ms of transient +covering, so the zero-interference gate remains open. All owned synthetic typing stayed +in A; popup/context, intentional activation and explicit foreground regressions +passed. A subsequent strict-ID diagnostic observed the rise after the requested +mouse-down, before mouse-up; the activation/warm-up phase showed no rise in that +run. This supports further investigation of requested pointer delivery, not +removing the warm-up on speculation. The earlier activation-based fix failed +and remains recorded. + +One later preparation-only cell skipped both requested events and observed no +covering; unchanged guards and sampling still limit that negative result. A +separate task-only cell retained preparation and changed only the requested +pair to one public CGEventPostToPid publisher per event. Exact-source receipt, +native field-editor state and forced final AX focus verified the intended click, +but SentinelA still observed 194.110 ms of covering; the independent sampled +WindowServer-order interval was 196.695 ms. This transport change is not a +demonstrated remedy and was not applied to production. All ten synthetic +canaries reached A, but only one was sent within the click call and none during +the observed covering interval. This does not establish human-input safety +during covering or general zero interference. + +### Off-Space window observation (G17 / upstream #4996) + +Add the upstream single-window SkyLight backing-store capture to the shared +capture layer. A known off-Space target tries it before the public SCK/shell +chain; current-Space and unknown targets retain public-first ordering. Resolve +the private symbol dynamically, retain the returned image independently of its +owning CFArray, reject empty or fully transparent images, and preserve the +existing image dimension limits and PNG conversion. This adds no process, +permission, activation or Space switch. + +The private route must validate the existing WindowCaptureIdentity before and +after capture, and refuse attached-window or unresolved attachment geometry. +It shares the existing one-worker gate and the three-second native capture +budget with SCK; an unreturned native call retains the permit, including after +the caller times out. Public shell compatibility retains its current behavior. +All consumers continue through the same capture function. A backing-store image +can be stale, particularly for off-Space Chromium, so raw and default App +observations must carry that limitation rather than imply a new frame proves +an action's effect. + +Verification first executes the current entry point with controlled native +boundaries, then checks routing, identity changes, attachment refusal, CF +ownership, transparent pixels and worker limits. Those checks do not replace +real off-Space capture or the separate exact AX remote-token recovery work. + +### Exact omitted AXWindow recovery (G17 / upstream #4068, #4085, #4509) + +After public window, focus/main and attached-sheet discovery fail, a bounded +remote-token probe may recover the requested AXWindow. It requires a current +same-PID layer-0 WindowServer window known to be off-Space, or known to be on +the current Space but off-screen. Unknown membership does not qualify. Each +candidate must have the AXWindow role, exact PID and exact direct SPI window +ID; frame/title approximation cannot establish this identity. The optional +private symbol is resolved dynamically. The 300 ms probe budget is clipped +to any earlier recording deadline, with at most 50 ms per native message and +2,000 candidate tokens. Native calls remain cooperatively bounded. + +The tree and fresh action facts retain their existing completeness, ownership, +minimized/hidden and keyboard-ambiguity gates. Default App selection tries the +probe only once after public selection/reopen has finished, and only with one +candidate among all same-PID layer-0 windows. A recovered root still resolves +its attached sheet; an incomplete, ambiguous or foreign sheet refuses selection. +An exactly proven App surface can be selected despite an incomplete global AX +array, as with the existing focused-window path; its subsequent tree/facts must +still pass their stronger completeness gates. No input is sent, window activated, +Space switched, or AX object cached across requests by this recovery. + +### Chromium observation readiness (G24 / upstream #3783) + +An accepted modern accessibility opt-in remains pending until AXWebArea is +observed. Default App AX observations share one 500 ms readiness deadline +across window selection and tree walking. Recording, input facts and pure +screenshots request opt-in without polling or consuming a wait attempt. +Positive evidence returns immediately; pending results have a 30-second +cooldown and at most three wait attempts per process lifetime. The cache is +keyed by PID and kernel start stamp, with lifetime validation serialized with +publication so a stale completion cannot overwrite a newer process's readiness. + +Independent controlled verification covers 16 readiness cases, the old/new +publication race, and two actual core propagation tests. Real Chrome rejected +the opt-in in the available fixture, so accepted-path native readiness remains +unverified. The 500 ms budget is not a whole-observation latency bound. The +separate AppKit still-launching recovery is described below. + +### Native named-key flags found during G17 acceptance + +The global chord, desktop held-key transition and both PID named-key builders +overwrite CoreGraphics flags to isolate explicitly requested modifiers. This +also strips the intrinsic Function/NumericPad bits of special keys. Independent +native event construction reproduced the loss without posting any input; the +upstream global helper shares it. Preserve the keycode-specific structural bits +and add only explicit or session-owned modifiers. Do not copy factory flags, +which may contain unrelated physical modifiers. Fn itself remains a held +modifier, so its release has no intrinsic Fn bit. Unicode and bare forwarding +keep their existing event semantics. Routing, timing and single-dispatch +behavior are unchanged. Real Space navigation needs a separate effect oracle. + +### AppKit launching-window lookup (G24 / upstream #4309) + +The separate AppKit recovery uses a one-second native lookup allowance, the +exact upstream default. Its absolute deadline shares the existing observation +start: subtract the named Chromium readiness budget from that deadline, then +add the launch allowance. Chromium still receives only 500 ms. No readiness +deadline means no launch wait, including recording and action facts. This +internal coupling is covered by a shared-anchor regression test. + +Only unresolved windows in a process known to be still launching receive a +retry. Default App selection waits for natural resolution instead of issuing +another reopen request; exact tree discovery retains its full scope/ownership +checks. Retry pauses are at most 50 ms and leave room for the previous attempt's +duration. Each discarded attempt releases its AX objects and incomplete state; +the final attempt determines the result. A tree walk tries the G17 remote-token +fallback at most once across retries. This is a cooperative retry allowance, +not a hard bound on native calls or the complete observation. Existing finished +application reopening, node-walk budgets and outer timeouts remain separate. + +Independent verification passed 27 controlled actual-function cases and 49 +G17 consumer regressions. It caught a sleep-overrun defect; the repaired loop +checks its deadline again before another active AX retry. The default App +wrapper still calls its ordinary final body, which may itself resolve AX. +A later native G24 fixture establishes still-launching state at the single public request, followed by the correct AX marker in 1.056 seconds and independent finished-launching readback. This validates that cold public chain, but does not identify the retry responsible, prove the old 400 ms path fails, or impose a hard one-second bound: the first AX IPC may itself wait for readiness. + +### Recording AX capture budget (G30 / upstream #4215) + +Recording AX callbacks run outside the recording-session lock with a fixed +two-second caller backstop. macOS also gives the recording tree walker a +one-second cooperative node deadline, retaining a completed prefix. Ordinary +observations keep their existing budgets. At most four native capture workers +may run, with one per PID; a timeout holds its slot until the provider actually +returns. Late results are discarded. The worker inherits the requesting +runtime, authorization and trusted session evidence, with no primary-runtime +fallback. The callback registration signature remains unchanged. + +Before and after capture, the recording verifies that the generation and owner +still match. Only the second check and artifact commit hold the session lock, +so stopping or replacing a recording cannot admit a late capture into either +session. Screenshot privacy rules and video-stop behavior retain their existing +boundaries. This adds native threads, not a process or public budget option. + +The new Windows recording thread owns its COM apartment: successful COM +initialization must be paired with uninitialization after all recording-only +node bindings are released. It must not release an apartment when initialization +fails, including a conflicting apartment mode, or publish recording nodes into +the action cache. Other UIA callers retain their existing lifecycle. + +The macOS recording hook also owns the walk's extra actionable-element retains. +It uses the existing unpublished snapshot owner to release them after JSON +serialization, without replacing action tokens. Independent actual-walker/hook +tests with controlled AX data and real CoreFoundation reference counts reproduce +one leaked retain per call before the fix and zero after repeated complete and +incomplete captures. Existing tokens remain valid and seven payloads match +apart from elapsed time. Three native regression tests and macOS/SDK all-target +checks pass. These are ownership tests; native GUI capture, serialization failure +and an actual elapsed-budget truncation are not established by this fixture. + +### Snapshot identity (G18 / upstream #3616 and #3873) + +Upstream #3616 / #3873 require the snapshot identity and its native element +bindings to agree. A controlled concurrent observation reproduced an old App +short ID resolving to a replacement native object: token resolution and the +per-window cache lookup were separate operations. + +The existing token registry will publish the window, index membership, token +generation and an immutable owned native payload in one entry. Resolution keeps +an `Arc` to that payload. Platform actions read a bound cache view, including +Windows kind/role/geometry and Linux identity/actions, instead of the latest +mutable window cache. This covers default App calls, raw native calls and their +interleaving without holding a GUI lock across an observation or action. Token +format, runtime isolation, same-window invalidation and the existing LRU stay +unchanged; no upstream SnapshotStore or new public API is introduced. + +Publication changes affect the three `get_window_state` implementations and +native element caches. Consumers include click/double/right click, keyboard +input, setValue, scrolling, secondary actions and Mac selectText. Both Mac +`ClickTool` and the separate `invoke_app_click` path use the bound view. Recording +markers also retain the exact binding rather than resolving a bare index again. +Pending captures own their native references and release them on failure or +cancellation. Runtime/target cleanup releases registry ownership; a previously +resolved action keeps its own payload alive until it finishes. + +A degraded Mac AX-window observation revokes the previous target snapshot in +the calling runtime; it does not publish an unresolved tree or revoke another +runtime's observation. Each platform cache's `clear_target` also revokes this +runtime's tokens. Existing watchers separately clear all runtime lanes only +after proving that the native window or process has closed. Session dispatch +guards defer end cleanup until the observation finishes; cancellation releases +unpublished pending payloads rather than publishing after cleanup. + +Verification pauses publication and lookup around a competing snapshot, checks +old-object identity or a stale refusal, and covers cross-session/raw/App calls, +sparse Linux indices, Windows UIA/MSAA metadata, retain/release accounting and +screenshot-only observations. Native desktop acceptance remains separate. + +### Implicit session recovery after idle cleanup (G08 / upstream #4283) + +An idle sweep currently leaves the still-connected unnamed MCP/SDK lifecycle +permanently ended. Record whether a tombstone came from idle reclamation of its +owner's implicit lifecycle. Only that same owner may recreate it, at the existing +session admission boundary, after all cleanup hooks complete. Recheck under the +tombstone lock before removing it and inserting the next lifecycle record. +Named sessions, explicit end, transport exit and revocation remain terminal. +The daemon adapter's early mirror must admit this one recoverable state and +follow core revival notifications. Direct SDK and MCP share core admission. + +Recreation must not restore prior resources. Snapshot entries retain the private +publishing session identity so end cleanup can retire only its native bindings; +App short IDs, capture scope, grants and recording already have owner cleanup. +Test unfinished cleanup, wrong owner, named/explicit/transport termination, +concurrent admission and an old raw snapshot token after recreation, then repeat +the actual direct MCP idle-TTL reproduction. + +### Native Electron background scrolling (G05) + +Upstream #4861/#5079 reveal content through native accessibility when Electron +ignores background wheel input. This is a native application operation; it does +not use the upstream browser API. Reproduce the candidate's current refusal and +compare the same owned scroll fixture with upstream before implementation. +The native before now confirms five candidate refusals with unchanged offsets; +stock 0.35.4 moves the inner pane by 114 pixels and the page by 404 pixels in +four corresponding single-request cases. + +Adapt the geometry and clamped-row selection while preserving the +approved dispatch contract: prove the exact window and native element ancestry, +choose the innermost relevant scroll container, and send at most one +`AXScrollToVisible` action. Do not copy the upstream fixed 150 ms effect wait or +its retry on another container after an unconfirmed dispatch. Return dispatch +acknowledgement and use the next observation for effect evidence. A preflight +absence may refuse; an uncertain action must not be replayed. Existing Browser +SDK behavior remains outside this change. + +Selection is read-only and bounded by 4,000 descendants, depth 80 and a private +one-second cooperative search budget. Failure to select safely refuses before +dispatch. The selected native proxy and process mutation lease live in the +blocking worker through the final exact-target check and one AX action, including +when its async caller is cancelled. A native AX error after dispatch is an +uncertain acknowledgement with a warning; it never starts another route. These +bounds do not impose a hard deadline on an in-flight native read. + +The normal v3 SDK passes five native Electron fixture cases: raw inner/page +scrolling through AX and pixel targets, plus default App inner scrolling. Each +case uses one request; independent fixture IPC observes inner +114 pixels or +page +404 pixels while the other region remains unchanged. The original +unconfirmed acknowledgement is preserved. A separate exhausted-inner fixture +reveals the before defect: both v2 and v3 move the inner pane up 96 pixels +after a single down request. Normal v4 repeats the five main cases and passes +raw/App exhausted-inner refusals with actual sent:false and unchanged offsets. +These seven cases do not establish all-direction behavior, +human concurrency or absence of shorter unsampled interference. The v2 +controlled 43-case result and v3 four geometry unit tests remain distinct from +the v3 controlled executable's startup timeout. + +The exhausted-inner diagnostic identifies an earlier row reported at the +bottom edge as a one-pixel sliver. The native container is correct; revealing +that row reverses the intended movement. The v4 selector uses its existing +visible-leaf set to reject clamped targets on the wrong side in document +order: down/right after the last visible leaf, up/left before the first. +It adds no native reads or actions and leaves real-frame selection unchanged. +When no visible anchor exists, the previous heuristic remains; reordered or +multicolumn AX graphs still need separate evidence. This is a scoped repair +for the reproduced case, not a universal direction guarantee. + +Normal v7 independently repeats all seven native GUI cases with source tree `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693` and SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe`: seven pass, without replay or foreground fallback. Independent fixture IPC confirms the same +114px/+404px offsets and unchanged exhausted boundaries. The 143 samples retain foreground/topmost A, pointer position and clipboard count. All 634 referenced entries are verified. Sampling at 8.444–12.213Hz does not establish absence of shorter interference or human concurrency. + +### macOS release replacement recovery (G36) + +The actual release-install branch loses the previous bundle if copying fails: +two isolated before cases confirm that failure before copying or after a partial +copy leaves a broken executable symlink. Adapt #3297's staged validation and +backup/rollback, using the existing bundle path and signing identity. Verify the +downloaded bundle before touching the installed application, keep its previous +bytes in a sibling backup until the copied bundle and executable are validated, +and restore them on copy/verification failure or a handled interruption. Preserve +the backup and report its path if restoration fails. A process kill or power loss +cannot execute the shell trap and is not claimed to recover automatically. + +Compare the replacement against the previous designated signing requirement to +report permission continuity accurately. Do not import upstream automatic TCC +resets: installation never changes authorization records. Defer old local-build +cleanup on macOS until replacement succeeds. This keeps the existing installer, +daemon, bundle location and permission owner; no new service or helper process +architecture is introduced. Validation uses isolated directories and controlled +system-command boundaries, not a real application or permission installation. + +### Additional verified upstream ports + +| Reference | Adaptation and evidence | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #4967 / #4214 | Linux top-level frame rebasing and local XRes PID fallback. Real GTK before/after fixes wrong-row input and missing windows without `_NET_WM_PID`; scoped to the tested Xorg environment. | +| #2907 / #4690 / #5137 | macOS window clicks select one native delivery route; middle clicks retain the exact window. Two fresh mini parent/sheet rounds remove duplicate target events. | +| #3755 / #5125 | macOS screenshot uses the absolute system executable; window bounds reject non-finite native numbers. Restricted-PATH real capture and native CFNumber regression pass. | +| #5136 / #4333 / #3975 | Windows console inheritance, GDI failure cleanup and resolved shortcut environment paths. Actual MSVC tests pass; interactive desktop behavior is a separate pending gate. | +| #3277 / #4985 | Embedded driver preserves explicit telemetry and update-check preferences through its environment filter. | +| #3462 / #4888 | Shared typed schema generation removes Schemars-only numeric formats; input declarations use the supported dialect while runtime validation remains authoritative. Declaration tests and offline MCP wire checks are distinct from provider acceptance. | +| #3632 | Recording reserves turns only for its owner session or implicit trusted connection; lifecycle calls are omitted. Independent testing caught an implicit-owner omission; the second fix passes public registry checks for named owners, implicit connection owners, foreign connections and lifecycle exclusion. | +| #4238 | ScreenCaptureKit stop waits for the recording-output finish callback. Real mini baseline reproduces false empty-file failures and post-return growth; five new-candidate 120ms recordings decode immediately and retain the same file hash after one second. | +| #3328 | Display scaling reads the current mode's backing pixels divided by points. The public tool reports 2 instead of 1 on the same real Retina display; coordinate dispatch is a separate contract. | +| #3714 / #5144 / #5145 | Encoder exit/timeout becomes an explicit error, and recording timestamps preserve one start baseline with ISO-8601 action timestamps. Public API probes verify the failure details and output metadata. | + +This table records implemented groups, not a claim that every upstream commit has +been reconciled. The full inventory and remaining native gaps are tracked in the +acceptance evidence ledger. A successful cross-platform build does not replace +real desktop effect and interference checks. + +## Validation boundaries + +See the ignored `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/` reports for immutable source manifests, build logs, protocol calls and fixture readback. The current [acceptance snapshot](cua-upgrade-acceptance-2026-10.md#acceptance-clarification-2026-10-20) separates earlier controlled/GNU/Xvfb checks from the later actual Windows MSVC and interactive-desktop tests, Linux Xorg/uinput/Mutter tests and macOS native fixtures. None establishes physical-human concurrency. The user clarified that observed transient macOS target/menu cover is acceptable; retain measurements and recovery separately from functional correctness. Model task success, token counts and timing parity require actual second-layer evidence. The user scheduled OSWorld after release on 2026-10-20; it is not a pre-release gate. + +Later acceptance repairs supplement the upstream ports. Exact AppKit key-window records fix the reproduced sheet Return failure; they contain internal NaN-location mouse events and do not solve pointer/window coverage. Isolated Windows uninstall ownership now covers every task action and only the selected process/install roots, verified on actual PowerShell. Finite CLI commands skip their observation self-wrapper when effective telemetry consent is false; existing consent precedence and enabled behavior remain. This CLI-only v21 change is verified separately from unchanged SDK backend artifacts. Package/runtime appendices retain local developer signing and installer seam limits; they do not establish official signing/notarization or a production release. + +### Acceptance-driven input boundary repairs + +Linux v24 distinguishes pre-dispatch refusal from uncertain delivery throughout text prefixes, first/named-key presses, releases and flushes. Event-emission errors carry sent uncertainty even when the first requested press fails; preinput focus checks still refuse before input. MPX pending requested input is retained only after target-focus admission. A failed press and the focus-probe key each get bounded best-effort release cleanup, retaining uncertainty instead of claiming completed text. Real focused tests and fresh Xorg/Mutter cases separately verify this behavior; an uncertain paused-target result is never automatically replayed or scored as complete text. + +Mac v25 handles an app-focused same-window AXGroup that explicitly reports AXFocused=false while a genuine text descendant holds focus. It delegates only this shape to the existing complete, unique, bounded actual-focus search. Unknown focus, cross-window focus and other roles keep their existing behavior; PID/window checks and final focus/window revalidation remain. Actual LibreOffice ASCII insertion passes. The original Cmd-S/AppleEvents Save remains UNKNOWN; an independent v25 toolbar Save has actual FODT evidence. Accepted transient target/menu cover remains measured in the acceptance record. AX settable/edited flags alone do not establish an applied edit or saved file. + +### v26 pointer-reading target refusal + +A real Calc request selected B28 instead of requested C5. Exact LibreOffice VCL source computes mouse callback coordinates from the global hardware pointer, which PID-routed clicks do not move. Following upstream #4864's safety principle, the fork now identifies the official `org.libreoffice.script` app through its existing NSRunningApplication boundary and refuses routed pixel click/right-click/double-click alternatives before input. Raw foreground PID routes also refuse; merely activating a window does not fix this mismatch. Advertised AX actions and safe AX editable-focus/selection remain available. This scoped identity rule is not upstream's general mapped-library detector, so it makes no claim for rebranded/unbundled VCL targets. Canonical public Mac delivery stays background-only. + +The v26 tree `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3` changes only these three Mac tools from v25. A fresh public-contract refusal returns background_unavailable/sent:false and leaves the same exact-window Name Box at A1 in three bounded reads. This is a safety refusal, not successful C5 background selection; raw AXPress runtime equivalence remains separate. The historical strict zero-cover predicate was not met, but the user subsequently clarified that observed transient target/menu cover is acceptable. See the v26 acceptance checkpoint for native/source/build evidence and the retained original harness-assertion failure. + +### v27 scoped Writer editable focus + +The official LibreOffice Writer body exposes AXTextArea with no advertised action and writable AXValue/AXFocused. The intermediate v27 candidate admits only this bundle's text area into the default App branch's editable-focus path, retaining both attribute gates, exact ownership, single unmodified left-click conditions and value-preserving focus readback. Generic text areas do not gain this shortcut; the raw path is unchanged from v26. Fresh mini native reads show that this admission alone does not establish focus, despite a successful AX setter status. + +F31 adds existing exact-window background keyboard preparation and an existing read-only selection-range query before the single AXFocused setter. The same-version LibreOffice caret getter initializes its cursor cache/context, which explains the purpose of the otherwise non-obvious read. There is no range write, caret relocation, text replacement or pixel fallback. The original observer, which makes no caret query, reads the same seed body false before and true after one fresh mini App click. One input and one toolbar Save then produce exact live and saved XML text. Conservative confirmation and sent/unconfirmed results remain: observe before the next action, never replay. A preparation failure can follow native focus records and must not be treated as proof that nothing was sent. Rejected prep-only and range-write experiments retain their own source and native evidence. + +Preparation retains the existing keyboard-target ambiguity gate because it can send a PID-scoped ReturnKeyFocus notification before the exact-window records. An AX element binding alone does not prove that notification safe with competing keyboard destinations. The two fresh Writer positives each have one document window; arbitrary multi-window focus switching is not established or enabled by deleting the safety gate. diff --git a/docs/design/cua-upgrade-platform-ports.zh-CN.md b/docs/design/cua-upgrade-platform-ports.zh-CN.md new file mode 100644 index 00000000000..f48260a6dea --- /dev/null +++ b/docs/design/cua-upgrade-platform-ports.zh-CN.md @@ -0,0 +1,247 @@ +# Computer Use 平台移植记录（2026-10） + +[English](cua-upgrade-platform-ports.md) | [简体中文](cua-upgrade-platform-ports.zh-CN.md) + +本文是已批准[升级方案](./cua-upgrade-2026-10.zh-CN.md)的实施台账。移植来源为 trycua/cua 截至 `cua-driver-rs-v0.35.4` 的 MIT 代码，Copyright (c) 2025 Cua AI。仅原生 driver 库作为来源，不纳入上游 bench 包及其许可证。 + +## 派发合同 + +App 输入在 driver 内解析精确应用和窗口。macOS 使用辅助功能与进程定向事件，不自动回退全局前台。Windows/Linux 仅在原生层返回派发前 `background_unavailable` 且 `sent:false` 时，允许一次前台路径。可能已经派发后的执行器错误不回放。显式桌面输入是独立前台接口。 + +前台事务观察物理输入而不消耗它；检测到后停止剩余输入，保留用户选择的前台 C，仅在仍拥有 B 时恢复原 A。Windows 恢复前等待目标输入队列排空，并在 UTF-16 单元间检查目标。Linux 在 XTest 前同时确认 EWMH 激活和核心输入焦点。 + +成功表示派发确认，不表示应用效果已确认。输入调用不等待截图变化，下一次观察提供效果证据。持续按压和多击仅保留完成手势所需的时序。 + +## macOS + +| 上游来源 | 适配 | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| #4943 | 定向焦点到 B 时，不给 A 发送合成失焦通知。 | +| #4943, #4955, #5091, #5129 | 键盘焦点保护复用进程级 `app_focus` 观察者；有界尾部/层序恢复和主动焦点 epoch 避免观察者竞争。 | +| #4976 | 物理鼠标/修饰键活动优先保留用户切换；现有激活保护检查同一物理输入状态。 | +| #5051 | 定向鼠标事件显式设置修饰键，包括空集合。 | +| #5057 | 保存已有文件和 Finder 改名使用有界 Apple Events 并读回；参数作为脚本参数传入，不拼接 AppleScript；Automation 走正常 TCC 授权。 | +| 弹出选择 | AX 打开菜单并选择唯一匹配项；拒绝时关闭本次打开的菜单。Safari 原有 DOM 回退仍仅限实际前台文档。 | +| 文本插入 | ASCII 使用定向事件；非 ASCII 要求聚焦可编辑元素和 AX selected-text 支持，并读回预期插入；不确定 AX 写入后不重放键盘。 | +| 文档模型 | 已验证的 AppKit 文档字段 AX 写入，通过定向且净变化为零的编辑 shim 注册修改；无法验证注册时拒绝。 | + +后台拖动在派发前拒绝；滑块使用 `setValue`；粘贴保留剪贴板恢复事务。删除旧的动作后固定一秒等待。AX 确认或事件投递本身都不算效果确认。 + +P7 对完整原生工具模块的受控 provider 测试复现：raw 双击/右键的 AXOpen/AXShowMenu 先产生效果再报错，旧实现继续发送鼠标动作。现在一旦尝试该 AX 动作，错误直接返回正常 sent/unconfirmed 响应与警告，不进入鼠标路径。成功语义路径及未提供语义动作时选择的鼠标路径保持原行为。该语义分支未执行前台事务，因此即使请求 foreground，也明确报告实际 background。独立测试执行完整模块及真实 core 公开投影，双击六格、右键四格均通过，三项不确定结果保持 unverifiable/accessibility 并保留警告。默认 App 的提前转发仅作对照，不证明 App 缺陷或真实 AX 服务器生效后报错。 + +## Windows + +| 上游来源 | 适配 | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #4500 | 连接实际目标输入队列，以 sentinel 读有界等待排空再恢复。 | +| #4901 | UTF-16 单元之间检查前台所有权和物理输入；单元停滞后停止，不回放。 | +| #3671 | UIA 点动作超时与派发前不存在合适 pattern 分开，不确定动作不能变成第二次坐标点击。 | +| #4774 | 原生 combo 使用控件选择/通知消息，并验证选中值。 | +| Tk | 输入前拒绝不支持的后台鼠标路径，随后共享 App 派发器可前台执行一次。 | +| 扩展键 | 左右 Windows 键保留 extended-key 标志。 | +| Worker IPC | 有界读取响应；超时说明可能已发送。UIAccess 路由明确 opt-in。 | +| 普通用户启动 | 高权限 daemon 从自身 token 派生并验证 Medium token，管理员成员为 deny-only；不打开/借用其他进程 token。直接可执行文件和 shell helper 均使用它，失败则拒绝，不高权限启动应用。 | + +保留正常安装/UAC 流程。交叉编译仅证明构建，不证明交互桌面、UAC、运行时 token 或共享桌面打扰。 + +## Linux + +删除原生 Wayland、helper、依赖和安装载荷。X11 和 XWayland 中的 X11 应用共用 X11 实现；不支持 compositor 原生 Wayland 表面。 + +| 上游来源 | 适配或现有等效实现 | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| #3882 | MPX 鼠标/键盘、toolkit 拒绝规则、遮挡和焦点保护，不使用输入屏障；保留 fork 精确窗口检查。 | +| #3761 | hold 计时前 flush/check 按下；短连接关闭前无条件 server round-trip。P7 后续复现表明临时映射仅靠 server 送达不够，下面的客户端队列修复仍需验收。 | +| #3762 | 检查按下/释放，保留时序，出错后释放已排队按钮和修饰键。 | +| #3864 | 保留观察到的精确 bus/object 点击身份和窗口快照校验；session-owned MPX 持久化来自 MPX 总体适配，而非此身份补丁。 | +| #3618 | 修饰键前台拖动用真实键状态包围并清理；不支持的后台组合在派发前拒绝。 | +| #4248 | 新 App 合同无动作后效果分类器，不会因 WM 暂存焦点把关闭 B 对话框误判无效果。精确前台恢复保留 C，不从未知 popup 推断确认；不移植上游截图/窗口变化等待。 | +| #4188 | 坐标 AX 激活排除 application/frame/window 外壳；识别 NUL 分隔和空格拼接的 Chromium helper 命令行。 | +| #5139, #5158 | Unicode keysym、全文字符预检；首个事件前预留/重映射空键码，序列化临时映射，不静默漏字。 | +| #3601 | master 所有权包含 boot、PID namespace、UID、进程开始身份；仅回收已证明属于本 driver 家族的过期设备。 | +| #2888 | KDE 设备热插拔可能重配物理输入，因此保守拒绝 MPX；不自动改变 `/dev/uinput` 权限。 | +| #5070 | 后台键盘拒绝明确指出缺 AT-SPI bus 和正常配置路径；诊断探测有界。 | +| #4903 | 删除 AT-SPI P2P feature/peer 调用，使用一个注册连接。 | +| #4984 | 仅基于有效完整屏幕 frame 重定位；拒绝不完整或零尺寸几何。 | +| #5074 | 首次子树遍历前通过 attributes 唤醒应用。 | +| #4375 | 首次语义遍历两秒，之后一秒；不对不完整观察做通用整树重试。 | +| GTK combo | `Selection.SelectChild` 选择唯一选项，读回 selected-child 名称，不改焦点/前台。 | +| #4292 | 文本与标签分开；数字控件优先 numeric Value，不读取密码 Text。 | +| #3758 | DirectColor 截图用 X colormap 解码，不当作 TrueColor mask。 | +| #4281 | kill 成功等待实际退出；zombie 算已退出。 | +| #3814 | 精确窗口截图明确 `screenshot_frame_valid:true`。 | +| #4529 | overlay 未映射时准备空 input/bounding shape，仅绘制可见内容时映射，空闲帧取消映射。 | +| #4715 | 非 Linux stub 不再声明已删除截图工具；portable menu schema 不走 cursor-only 断言。 | +| #3658 | Enter/Return/KP/Numpad 别名共用终端 gate 和映射。 | +| #3778 | 既有 opt-in 原生浏览器工具共用 session cursor；非活动 tab 不移动可见光标，不把上游浏览器工具加入默认接口，也不改仓库 Browser SDK。 | + +fork 修复还包括按窗口保留截图缩放，不重复缩放省略位置的 mouse-up，Qt 合成 FocusIn 限 Qt5，按实际核心焦点决定 FocusOut，仅在路由需要时分类 Chromium。 + +### P7 Linux 临时键盘映射生命周期 + +在自有 GTK 主循环暂停三秒时，候选和上游都丢失重映射 Unicode：server 已送键码，driver 恢复映射，客户端之后把它译为 NoSymbol。此复现使用独立 nested Xorg，与不稳定主 GNOME 分开。 + +候选在修改映射或输入前准备 X11 客户端队列确认，仅需要空键码的操作付出此成本。目标或同 X client 的祖先必须声明 `_NET_WM_PING`；同进程另一连接不算。XTest 在每字及确认前后检查焦点 client。派发后收到 root 上精确 nonce 回声再释放临时映射。五秒超时恢复映射并正常返回已发送但未确认，不回放；部分输入期间发现焦点变化仍停止后续输入并报错；缺协议支持则输入前拒绝。ASCII 保留 server round-trip，不增加 ping。鼠标修饰键不在现有 map 时提前拒绝，因为借键码不能建立真实 modifier mapping。 + +MPX 在 ping 前还必须确认 raw-key server delivery。受控复现发现旧 last-key 谓词在六个计划事件只到两个时，把较早的重复 release 当成完成。新实现要求准确虚拟 slave 的每次有序转换，包括重复键和 Shift。旧版零 source ID 仅在 device ID 等于该 slave 时允许，不能用共享 master。相同 source 的意外转换使结果未确认；保留设备的 pending 状态，下一请求在映射、焦点或输入前拒绝，正常 session 清理随设备删除。预热也确认完整批次，且先于借键码；保留 keymap→MPX 锁顺序。独立生命周期 10/10、匹配器 13/13、下述预热校验后的 Linux 单测 16/16 通过。内核积压和物理设备仍待验；受控测试替换了 OS 边界。 + +预热前 XKB 必须识别准确 slave，证明 X248 所有 group/level 都无符号。占用、无效或不可读时零输入拒绝并保持 pending；不临时重映射固定键使检查通过。独立验证覆盖 20 个 map/query 和三个 pending 入口；C header 与 Rust layout 一致。真实只读 Xephyr slave 的 empty-group map 仅为形状对照，不代表新 uinput 设备。 + +不引入焦点 grab、输入屏障、剪贴板变更、永久映射或新 helper 进程。GTK/Chrome 证明队列机制；Chrome 回声早于 DOM 输入结束，明确不是语义成功。native-v5b 的 GTK 响应、暂停三秒、超时、错误 nonce、缺协议分别通过效果/拒绝/不确定检查。Chrome 机制检查使用 v5a；v5b 仅改不确定结果报告，另有验证。物理 MPX 未验证。 + +## P7：上游正确性收口 + +### 普通 macOS AX 遍历预算（G31 / 上游 #3882） + +原 raw 和默认 App 观察仅限制深度/节点数，时间预算仅用于录制。适配上游一秒节点遍历预算，从窗口选择和 readiness 之后第一个非重复、准入节点开始。Chromium readiness、AppKit launch、原生 messaging timeout 和外层观察 timeout 分开；此协作限制不能取消挂起的原生读，也不保证整体一秒返回。 + +主动截断保留完整前缀，标记 truncated/incomplete。成功尝试的读保留 `read_complete`，与既有节点/深度上限一致；真实 AX 错误清除它。前缀变化时 bounded revision 返回完整替换，缺行不算删除。raw token 跨后续观察仍绑定自己的不可变对象；接受前用真实 walker 结果验证消费者。 + +legacy 节点预算与录制绝对期限保留。不新增公开 timeout 参数、输出字段、worker 或动作重试。先做有限慢节点复现与准备延迟对照，再实现；原生延迟另行验收。 + +### Windows 安装器祖先保护（G32 / 上游 #4398） + +共享清理可能结束发起 `update --apply` 的 driver 和 installer 子进程，并把受保护父进程误判为 stale。独立复现执行真实 PowerShell 函数，进程/task/pipe 全部 stub，不停止真实进程或任务。 + +适配 live ancestor walk，父进程创建时间晚于子进程则视为 PID 重用而拒绝继续。仅清理非祖先 driver PID，survivor/repair 列表排除祖先。存在 driver 祖先时跳过 task-wide `/End`，PID filter 不能保护其免受 Task Scheduler 终止。祖先查询不可用时任何结束动作前停止。legacy migration 复用同一 helper，移除另一份 image-name tree kill；独立 `-local` 产品身份不变。 + +隔离 stub 验证祖先、无关进程、stale、PID 重用、cycle、查询错误及 legacy 委托。回归脚本新增 unit-only，在构建/真实任务测试前退出。此源修改本身不注册/改变任务或 Windows 权限。 + +### Windows 自动启动失败报告（G33 / 上游 #3180 和 #4284） + +以 `Register-ScheduledTask -Force` 替换任务，避免先删。UAC 取消、无进程或无 exit code 均失败，否则转发子进程实际码；可执行文件作为 PowerShell 字面参数引用。复用状态查询验证注册；查询不可用时报告无法确认，不无条件成功。不自动重试取消或未确认的提升。 + +两个安装器仅在注册调用成功后提示成功。用 `CUA_DRIVER_RS_HOME` 或 `CUA_DRIVER_RS_INSTALL_DIR` 选择隔离 release 安装时，除非显式 `-AutoStart`，保留共享 daemon/task。此 guard 也包围注册前共享清理，仅保护结尾会仍杀普通安装的 daemon。默认安装与独立 local 身份保留原路由。 + +独立 before/after 执行真实 PowerShell payload 和 installer block，task/UAC/cleanup 为 stub；覆盖取消、null/失败子进程、注册失败保留旧任务、摘要及隔离/默认/显式参数组合。这些不是 UAC、真实注册或完整安装；后续已在独立授权下执行真实任务与自有安装生命周期测试，身份及边界见当前验收附录。 + +### macOS 后台窗口层序（T04） + +可见层序以 on-screen CGWindow 查询为准；all-window 查询对标为 on-screen 的行仍可能排序不同。真机诊断还发现重新激活已在前台的应用，一秒后返回成功却不恢复层序。guard 改为仅对其当前 AXFocusedWindow 请求 AXRaise，核验 PID 与精确可见 layer-0 ID。WindowServer 必须仍指向同一前台应用，近期物理鼠标/修饰键活动阻止恢复；AX messaging timeout 为 100ms，引用全程持有，保留现有有界观察。不改键盘焦点或后台派发路径；API 接受不证明层序恢复。 + +单个自有窗口原语探测在 AXRaise 约 23ms 后恢复可见层序，保留全部模拟输入。生产 fix-2 三轮消除持续覆盖，但仍有 99.4、200.5、192.8ms 瞬态，零干扰门槛未关闭。全部模拟输入到 A；popup/context、主动激活、显式前台回归通过。后续 strict-ID 诊断在请求 mouse-down 后、mouse-up 前观察到升层，激活/预热阶段未出现；支持继续调查请求的鼠标投递，不能据此猜测移除预热。早期激活式修复失败并保留。 + +后续 preparation-only 一格跳过两个请求事件，未观测覆盖，但现有 guard 和采样限制该阴性结果。另一 task-only 格保留准备，仅把请求 pair 改为每事件一次公共 CGEventPostToPid；精确源 PID 事件、原生 field-editor 和强制最终 AX focus 确认点击，却仍被 SentinelA 观测到 194.110ms 覆盖，独立 WindowServer 采样口径 196.695ms。未证明此传输变化能修复，因此未入生产。十个模拟 canary 全到 A，但调用内仅一个，覆盖期间零个；不证明覆盖时真人输入安全或普遍零干扰。 + +### 跨 Space 窗口观察（G17 / 上游 #4996） + +共享 capture 层新增上游单窗口 SkyLight backing-store 捕获。已知 off-Space 时先试私有路径，再 public SCK/shell；当前或未知 Space 保留 public-first。动态解析私有符号，返回图像从其 CFArray 独立 retain，拒绝空或全透明图，保留尺寸限制和 PNG 转换。不新增进程、权限、激活或 Space 切换。 + +私有路径 capture 前后校验原有 WindowCaptureIdentity，拒绝 attached-window 或不明确的 attachment 几何。与 SCK 共用一 worker gate 和三秒原生预算；未返回调用在外层超时后仍持 permit。public shell 兼容行为不变。所有消费者共用 capture 函数。backing-store 可能过时，尤其 off-Space Chromium；raw 和默认 App 观察必须说明，不能把新图当作动作效果证明。 + +先以受控原生边界运行真实入口，验证顺序、身份变化、attachment 拒绝、CF 所有权、透明像素和 worker 上限。这不代替真实 off-Space capture，也不代替独立精确 AX remote-token 恢复。 + +### 精确遗漏 AXWindow 恢复（G17 / 上游 #4068、#4085、#4509） + +公共窗口、focus/main、attached-sheet 发现失败后，可用有界 remote-token 恢复指定 AXWindow。要求实时同 PID、layer-0 WindowServer 窗口，已知 off-Space 或已知当前 Space 但 off-screen；未知 membership 不允许。候选必须 AXWindow role、精确 PID、直接 SPI window ID，frame/title 近似不能证明身份。私有符号动态解析；300ms 预算被更早的 recording deadline 截短，每原生消息至多 50ms、至多 2,000 tokens，仍是协作边界。 + +tree 和 fresh action facts 保留完整性、owner、minimized/hidden、键盘歧义 gate。默认 App 在 public selection/reopen 后仅一次探测，且所有同 PID layer-0 窗口中仅一个候选。恢复 root 仍需解析 attached sheet；不完整、歧义或外来 sheet 拒绝。与现有 focused-window 路径一样，精确已证明的 App 表面可以在全局 AX array 不完整时被选中，但后续 tree/facts 必须满足更严格完整性检查。此恢复不发输入、不激活/切 Space、不跨请求缓存 AX 对象。 + +### Chromium 观察 readiness（G24 / 上游 #3783） + +现代 opt-in 接受后仍为 pending，直到观测 AXWebArea。默认 App AX 观察在选窗和遍历间共用 500ms deadline。录制、动作 facts 和纯截图仅 opt-in，不轮询或消耗等待次数。正证据立即返回；pending cooldown 30 秒，每进程生命周期至多三次等待。cache 以 PID+kernel start stamp 为键，lifetime 检查与发布序列化，旧完成不能覆盖新进程 readiness。 + +独立受控验证覆盖 16 个 readiness、旧/新发布竞态及两个真实 core 传播测试。现有 Chrome fixture 拒绝 opt-in，accepted-path 真机未验证。500ms 不是整个观察延迟界限；另有下面的 AppKit still-launching 恢复。 + +### G17 验收发现的原生命名键标志 + +global chord、desktop held-key 与两个 PID named-key builder 覆写 flags 以隔离显式修饰键，也误删特殊键内在 Function/NumericPad 位。独立原生 event construction 无投递复现，上游 global helper 共有。保留按键码结构位，仅加显式或 session-owned modifier；不复制可能混入物理状态的 factory flags。Fn 本身仍是持有修饰键，释放不自带 Fn。Unicode 与裸转发语义不变，路由、时序、单次派发不变；真实 Space 导航另需效果 oracle。 + +### AppKit 启动窗口查找（G24 / 上游 #4309） + +独立 AppKit 恢复使用上游默认一秒原生 lookup allowance。absolute deadline 共享现有观察起点：从 readiness deadline 减 Chromium budget，再加 launch allowance；Chromium 仍仅 500ms。没有 readiness deadline 就不等待 launch，包括录制和动作 facts；共享起点有回归测试。 + +仅已知 still-launching 进程的 unresolved window 重试。默认 App 等自然解析，不再 reopen；精确 tree discovery 保留 scope/owner 校验。pause 至多 50ms，给上次 attempt 耗时留余量；丢弃 attempt 释放 AX 和不完整状态，最终 attempt 决定结果。tree 跨重试至多一次 G17 remote-token。它是协作 retry allowance，不硬约束原生调用/整体观察；已完成应用 reopen、node-walk 与外层 timeout 分开。 + +独立 27 个 actual-function 受控场景、49 个 G17 consumer 回归通过，发现并修复 sleep-overrun：睡眠后、主动 AX retry 前再查 deadline。默认 App wrapper 仍调用普通最终 body，可能再次解析 AX。后续原生 G24 夹具确认唯一公共请求开始时仍在启动，1.056 秒后获得正确 AX 标记，并独立确认启动完成。这验证该冷启动公共链，不能确定具体哪次 retry 生效、证明旧 400ms 路径失败，或给出硬一秒上界；首个 AX IPC 本身可能等待 ready。 + +### 录制 AX 捕获预算（G30 / 上游 #4215） + +录制 callback 在 session lock 外执行，调用方固定两秒 backstop。macOS tree 另有一秒协作 node deadline，保留已完成前缀；普通观察保留各自预算。至多四个 worker、每 PID 一个；超时后直到 provider 真返回才释放 slot，迟到结果丢弃。worker 继承请求 runtime、authorization 和 trusted session evidence，不回退 primary。callback 注册签名不变。 + +捕获前后检查 generation/owner；仅第二次检查和提交 artifacts 持 session lock，stop/替换不能让旧捕获写入任一新会话。截图隐私与 video stop 边界不变。增加原生线程，不增加进程或公开 budget。 + +新 Windows recording thread 拥有 COM apartment：成功初始化后，录制专用 raw node binding 全部释放，再 uninitialize；初始化失败或 mode 冲突不释放 apartment，不发布录制节点到 action cache。其他 UIA 调用生命周期不变。 + +macOS hook 同时接管 walker 对 actionable 节点的额外 retain，复用未发布 snapshot owner 在 JSON 序列化后释放，不替换 action token。实际 walker/hook 加受控 AX、真实 CF count 独立测试复现旧版每次漏一份，修复后重复完整/不完整捕获零泄漏，旧 token 有效，七份 payload 除耗时一致。三项原生回归、Mac/SDK all-target 通过。这是所有权测试，不证明 GUI、序列化失败或实际 elapsed-budget 截断。 + +### 快照身份（G18 / 上游 #3616 和 #3873） + +上游要求快照身份与原生 binding 一致。受控并发观察复现旧 App 短 ID 命中替换对象：token 解析和窗口 cache 查找原为两步。 + +现有 token registry 一次发布 window、index membership、token generation 和不可变自有 native payload；解析持有其 `Arc`。平台 action 使用 bound cache，包括 Windows kind/role/geometry 和 Linux identity/actions，不读最新可变窗口 cache。覆盖默认 App、raw 及交错，无需跨观察/动作持 GUI lock。token 格式、runtime 隔离、同窗口失效和 LRU 不变，不引入上游 SnapshotStore 或新公开 API。 + +发布影响三端 `get_window_state` 与元素 cache；消费者包括 click/double/right、键盘、setValue、scroll、secondary action、Mac selectText。Mac `ClickTool` 和独立 `invoke_app_click` 都用 bound view。录制 marker 保留准确 binding，不重新解析裸 index。pending capture 拥有原生引用，在失败/取消时释放；runtime/target 清理 registry 所有权，已解析 action 自有 payload 到结束。 + +退化 Mac AX-window 观察仅撤销调用 runtime 的旧 target，不发布 unresolved tree，不撤销别的 runtime。三端 `clear_target` 同时撤销当前 runtime token。watcher 只有证明窗口/进程已关闭才跨 runtime 清理。session dispatch guard 延后 end cleanup 至观察结束；取消释放未发布 payload，不在 cleanup 后发布。 + +验证在竞争快照前后暂停 publication/lookup，要求旧对象身份或 stale 拒绝，覆盖跨 session/raw/App、稀疏 Linux index、Windows UIA/MSAA metadata、引用计数和 screenshot-only。原生桌面验收另计。 + +### 空闲清理后的隐式会话恢复（G08 / 上游 #4283） + +原 idle sweep 会让仍连接的 unnamed MCP/SDK 生命周期永久结束。tombstone 记录是否来自 owner implicit lifecycle 的 idle 回收；仅同 owner 在 cleanup 完成后、原 admission 边界重建，在 tombstone lock 下复核后删旧并插入下一 episode。命名、显式 end、transport exit、revoke 保持终止。daemon adapter 早期 mirror 允许此唯一 recoverable 状态并跟随 core revive。SDK/MCP 共用 admission。 + +重建不恢复旧资源。snapshot 带私有 publishing session 身份，仅清理该 owner 的 binding；App 短 ID、capture scope、grant、recording 已有 owner cleanup。测试未完成 cleanup、错误 owner、命名/显式/transport 终止、并发 admission、重建后旧 raw token，再重复真实 direct MCP idle-TTL 复现。 + +### 原生 Electron 后台滚动（G05） + +Electron 忽略后台 wheel 时，上游 #4861/#5079 通过原生辅助功能 reveal。它是应用原生动作，不用上游 browser API。先复现候选拒绝并与上游比较；native before 确认候选五次拒绝、偏移不变，stock 0.35.4 四次单请求分别使 inner+114/page+404。 + +适配几何与 clamped-row 选择，保留合同：证明精确窗口/原生祖先，选最内层相关容器，至多一次 `AXScrollToVisible`。不复制上游固定 150ms effect wait 或不确定派发后换容器重试。返回派发 ACK，下一观察取证；preflight 不存在可拒绝，不确定不能回放。Browser SDK 不在变更内。 + +只读选择限 4,000 descendants、depth80、私有一秒协作搜索；安全选择失败则派发前拒绝。native proxy 与进程 mutation lease 在 blocking worker 中保留到最终 exact gate 和唯一 AX action，包括 async caller 取消。AX 错误后返回不确定 ACK/警告，不换路由；限制不硬截止挂起的原生读。 + +普通 v3 通过五格真实 Electron fixture：raw inner/page×AX/pixel，加默认 App inner；每格一次请求，独立 fixture IPC 读 inner+114 或 page+404，另一层不变，保持未确认 ACK。独立 exhausted-inner 复现 v2/v3 单次 down 却向上96px。普通 v4 重跑五主格，并通过 raw/App exhausted 拒绝，真实 `sent:false`、偏移不变。七格不证明全部方向、真人并发或更短采样间隔的干扰。v2 受控43项、v3四个几何单测与 v3 受控程序启动超时分开保留。 + +exhausted 诊断确认容器正确，但上方旧 row 被报为底边1px sliver，reveal 使方向反转。v4 用既有 visible-leaf 集合排除文档顺序错误侧候选：down/right 在最后可见 leaf 之后，up/left 在第一 leaf 之前。不新增原生读/动作，不改真实 frame 选择。没有可见锚点时保留旧 heuristic，重排/多列 AX 图仍需另验；这是限定修复，不是普遍方向保证。 + +普通 v7 以源树 `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693` 和 SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe` 独立重跑七格真实 GUI，全部通过，无回放或前台回退。独立 fixture IPC 确认同样 +114px/+404px 和耗尽边界偏移不变；143 个采样保持 A 前台/顶层、鼠标位置与剪贴板计数。634 项引用已逐字核验。8.444–12.213Hz 采样不证明不存在更短干扰或真人并发。 + +### macOS release 替换恢复（G36） + +真实 release 分支在复制失败时丢旧 bundle；两个隔离 before 场景确认复制前或部分复制失败都留下坏 executable symlink。适配 #3297 的 staged validation/backup/rollback，保留现有 bundle 路径和签名身份。动旧应用前验证下载 bundle，copied bundle 和 executable 通过前保留 sibling backup；复制/验证失败或可处理信号恢复旧字节。恢复失败保留 backup 并报路径。process kill/断电不能运行 trap，不声称自动恢复。 + +以旧 designated signing requirement 验证 replacement，准确报告权限连续性。不移植自动 TCC reset，安装不改授权记录；Mac local-build cleanup 延后到 replacement 成功。保留 installer、daemon、bundle location、permission owner，不引入新 service/helper 架构。验证使用隔离目录和受控系统命令，不是真实应用/权限安装。 + +### 其他已验证上游移植 + +| 来源 | 适配和证据 | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #4967 / #4214 | Linux 顶层 frame 重定位和本地 XRes PID fallback；真实 GTK before/after 修正错行输入及无 `_NET_WM_PID` 的窗口发现，限实测 Xorg 环境。 | +| #2907 / #4690 / #5137 | Mac 窗口点击选单一 native 路由，中键保留精确 window；mini 两轮 parent/sheet 去除重复事件。 | +| #3755 / #5125 | Mac 截图用绝对系统路径，窗口几何拒绝非有限数；窄 PATH 真截图和 CFNumber 回归通过。 | +| #5136 / #4333 / #3975 | Windows console 继承、GDI 失败清理、shortcut 环境路径解析；实际 MSVC 测试通过，交互桌面另待验。 | +| #3277 / #4985 | embedded 环境过滤保留显式 telemetry/update-check 偏好。 | +| #3462 / #4888 | typed schema 去 Schemars 专用 numeric format，声明用兼容 dialect，runtime 校验仍权威；声明测试/离线 MCP wire 与真实 provider 分开。 | +| #3632 | 录制仅 owner session/隐式 trusted connection 的动作，忽略 lifecycle；独立测试发现隐式 owner 漏洞，二次修复通过命名/隐式/外来连接/lifecycle 公共 registry 验证。 | +| #4238 | SCK stop 等 recording-output 完成 callback；mini baseline 复现误报空文件及返回后增长，新候选五次120ms录制均立即解码，一秒后 hash 不变。 | +| #3328 | display scale 用当前 mode backing pixels/points；同 Retina 显示器公开值由1改2，坐标派发另有合同。 | +| #3714 / #5144 / #5145 | encoder exit/timeout 明确报错；recording 时间用同一开始基线、action ISO-8601；公开 API 探测验证细节和 metadata。 | + +此表记录已实现组，不声称每个上游提交均已收口。完整清单与原生缺口见 acceptance ledger；跨平台构建不代替真实效果/干扰。 + +## 验证边界 + +忽略的 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/` 保存不可变 source manifest、构建日志、协议调用和 fixture 读回。[当前验收快照](cua-upgrade-acceptance-2026-10.zh-CN.md#验收口径澄清2026-10-20) 将早期受控/GNU/Xvfb 检查与后来的真实 Windows MSVC/交互桌面、Linux Xorg/uinput/Mutter、macOS 原生 fixture 分开。均不证明物理真人并发。用户澄清已观察到的 macOS 目标窗口/菜单短暂遮挡可接受；遮挡测量、恢复与功能正确性分别保留。模型成功率、token 与时长等效性仍须实际第二层证据。用户于 2026-10-20 将 OSWorld 安排在发布后，不再作为发布前门槛。 + +后续验收修复补充上游移植：精确 AppKit key-window records 修复已复现的 sheet Return 失败，内部含 NaN 坐标 mouse 事件，并未解决指针/窗口覆盖。Windows 隔离卸载归属检查覆盖全部 task action，进程/安装范围限选定根目录，已在真实 PowerShell 验证。有限 CLI 命令在有效遥测偏好为 false 时跳过观察 self-wrapper，保留既有偏好优先级和启用行为；该 v21 CLI 改动与未变的 SDK 后端分开验证。运行时/打包附录保留本地开发签名与 installer seam 边界，不证明官方签名/notarization 或正式发布。 + +### 验收驱动的输入边界修复 + +Linux v24 在文字前缀、首字符/命名键按下、释放及 flush 全链区分派发前拒绝与不确定送达。事件发送错误即使发生在首次请求按下也保留 sent 不确定性，输入前焦点检查仍可在派发前拒绝。MPX 仅在目标焦点准入后保留请求 pending。失败按下及焦点探测键各作有界 best-effort 释放清理，不把不确定性宣称为完整文字完成。真实定向测试和 fresh Xorg/Mutter 用例分开验证；暂停目标的不确定结果不自动重放，也不计完整送达。 + +Mac v25 处理同窗应用焦点 AXGroup 明确 AXFocused=false、实际文本后代持有焦点的形态，仅将此条件转入现有完整、唯一、有界的实际焦点搜索。未知焦点、跨窗焦点及其他角色保留原行为；PID/窗口检查和末尾焦点/窗口复核均不放宽。LibreOffice ASCII 插入已实际通过。原 Cmd-S/AppleEvents 保存仍 UNKNOWN；独立 v25 工具栏 Save 有真实 FODT 证据。可接受的目标窗口/菜单短暂遮挡仍保留测量。单独 AX settable/edited 标志不证明编辑生效或文件已保存。 + +### v26 全局指针读取目标的拒绝 + +真实 Calc 请求以 C5 为目标，却选中 B28。精确 LibreOffice VCL 源码用全局硬件指针计算鼠标回调坐标，而 PID 定向点击不移动硬件指针。借鉴上游 #4864 的安全原则，本 fork 通过现有 NSRunningApplication 边界识别正式 `org.libreoffice.script`，在输入前拒绝其定向像素点击、右键、双击及元素像素回退。旧 raw foreground PID 路径也拒绝，仅激活窗口不能解决坐标不兼容。已广告 AX 动作与安全的 AX 可编辑焦点/选区路径保留。这是限定应用身份的规则，不是上游通用映射库检测，不声明覆盖改名或无 bundle 的 VCL 目标。规范公开 Mac 投递仍固定后台。 + +v26 tree `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3` 相对 v25 仅改这三个 Mac 工具。独立新文档的公开契约回归返回 background_unavailable/sent:false，三次有界同窗 Name Box 读数均保持 A1。通过的是安全拒绝，不是后台 C5 选区成功；raw AXPress 真机等价仍另行判定。历史严格零遮挡判据未满足，但用户随后澄清已观察到的目标窗口/菜单短暂遮挡可接受。原生/源码/构建证据与保留的原 harness 断言失败见 v26 验收检查点。 + +### v27限定Writer可编辑焦点 + +官方 LibreOffice Writer 正文暴露无广告 action 的 AXTextArea，且 AXValue/AXFocused 可写。中间 v27 候选只允许此 bundle 的 textarea 进入默认 App 分支的可编辑焦点路径，保留两项属性门禁、精确归属、单次无修饰左键条件及保持原值的焦点读回。通用 textarea 不获得该捷径，raw 路径保持 v26 原样。mini 新实例的原生读取证明，仅增加准入仍无法建立焦点，尽管 AX setter 返回成功。 + +F31 在单次 AXFocused setter 前增加既有精确窗口的后台键盘焦点准备和既有只读选区查询。同版本 LibreOffice 的 caret getter 会初始化 cursor 缓存/context，这说明了该读取的目的。不写选区、不移动 caret、不替换正文、不回退像素。未查询 caret 的原 observer 在 mini 新实例的一次 App 点击前后读到同一种子正文 false→true，随后一次输入和一次工具栏 Save 得到完全一致的实时正文与已保存 XML。保守确认及 sent/unconfirmed 回执保留：先观察后续状态，不重放。准备失败前可能已发送原生焦点 records，不能据此宣称未派发。已否决的仅准备、写选区试验分别保留源码和真机证据。 + +准备过程保留既有键盘目标歧义门禁，因为精确窗口 records 前可能发送 PID 级 ReturnKeyFocus 通知。单独 AX 元素绑定不能证明在多个键盘目标间发送该通知安全。两个新 Writer 正例各有一个文档窗口，不通过删除安全门禁来启用或宣称任意多窗口焦点切换。 diff --git a/docs/users/features/computer-use.md b/docs/users/features/computer-use.md index 8f37baf2643..23acbec1eb1 100644 --- a/docs/users/features/computer-use.md +++ b/docs/users/features/computer-use.md @@ -1,129 +1,81 @@ # Computer Use -Qwen Code includes a `computer-use` skill that teaches the model how to -operate desktop applications through two separately installed packages: +Computer Use runs through the standalone Node REPL MCP server. Its first `js` call loads global `cua` and displays documentation for the current platform. Computer Use and Browser Use no longer require bundled skills. ```text -bundled computer-use skill - -> @qwen-code/node-repl-mcp - -> @qwen-code/cua-sdk/computer-use - -> native cua-driver accessibility backend +Qwen Code / any MCP client + -> @qwen-code/node-repl-mcp (js, js_reset) + -> @qwen-code/cua-sdk/runtime + -> native Qwen CUA Driver (desktop applications) + -> @qwen-code/browser-use (Qwen browser extension) ``` -Qwen Code does not bundle the MCP server, SDK, or native driver. The skill -installs the external packages automatically when they are missing. +## Setup -> [!warning] -> -> Computer Use can read application UI and control mouse and keyboard input. -> Use it only in trusted environments and review MCP approvals carefully. - -## Automatic setup - -Node.js 22 or later and npm are required. - -When first used, the skill runs these commands itself: +Node.js 22 or later and npm are required. Register the MCP server, then restart Qwen Code: ```bash -qwen mcp add --scope user node-repl npx -y @qwen-code/node-repl-mcp@0.1.7 -npm install --no-save --package-lock=false @qwen-code/cua-sdk@0.20.11 +qwen mcp add --scope user node-repl npx -y @qwen-code/node-repl-mcp@0.26.0 ``` -Restart Qwen Code after the MCP server is first added. The skill then resumes -the desktop task through `node_repl`. +The server installs the matching SDK and browser SDK as dependencies. The native SDK payload is downloaded for the current platform and verified against the matching release's SHA-256 checksums. It is cached outside the npm package; no workspace SDK bootstrap command is needed. Version 0.26.0 is a local development target until its release is published. -The SDK installation leaves `package.json` and the lockfile unchanged, but it -does write to the workspace's `node_modules`. Its postinstall downloads and -verifies the native payload for the current platform. - -Removing the MCP configuration or workspace SDK installation disables the -execution path; there is no legacy fallback. - -## Use - -Ask Qwen Code to use `$computer-use` for the desktop task. After bootstrap, it -uses the app workflow on macOS: - -1. binds the application with `computer.getApp(nameOrIdentifierOrPath)`; -2. reads `app.getState()` for compact accessibility text, followed by automatic - incremental updates; -3. performs one or more actions using the short element IDs in that text; -4. fetches the latest state before deciding what to do next; and -5. closes the SDK client and resets the REPL only when no other persistent - state is needed. - -The driver is the only component that computes observation diffs. Model code -uses the typed SDK methods and does not dispatch arbitrary driver tool names. -The app handle tracks the current window and dialogs, keeps native element -identity internally, and delegates input to the native driver. Model code does -not choose foreground/background modes. Unconfirmed actions are not replayed. -`getState()` can open a discovered stopped app; actions never restart it. -Existing exact-window APIs remain available on Windows and Linux. +## Application tasks ```js -const app = await computer.getApp('Microsoft Excel'); -nodeRepl.write((await app.getState()).text); -// Use an element ID from the returned state. +var app = await cua.getApp('Microsoft Excel'); +// Use a short ID from the automatically displayed AX state. await app.click(37); await app.typeText('hello'); -nodeRepl.write((await app.getState()).text); +await app.getAXState(); ``` -Refresh state after opening or closing a dialog before reusing element IDs. -Each App state refresh captures the current screenshot internally. The default -return keeps it hidden; request it explicitly with -`app.getState({ includeScreenshot: true })` when the model needs the image. +`getApp` displays full AX text without an image. `getAXState` displays subsequent diffs. `getScreenshot` captures only the current window; `getAXStateAndScreenshot` captures both. Observations return strings and display themselves; `{emit:false}` suppresses output. Request a screenshot before using `{x,y}` coordinates, which are pixels in that window's current screenshot. -## Permissions +The driver resolves applications/windows, owns current short IDs and coordinates, and chooses input delivery. macOS uses accessibility and directed events. Windows and Linux try background delivery first and use foreground input only after a refusal that proves nothing was sent. Foreground input stops when physical user activity is detected; it restores the prior application only if the user has not switched elsewhere. A dispatch acknowledgement does not prove the UI changed: observe before retrying. -The Node REPL is an MCP server that executes model-authored JavaScript with -ordinary Node.js authority. Its calls follow Qwen Code's normal -[MCP approval flow](./approval-mode.md). The SDK also enforces native -authorization. +Linux supports X11, including X11 applications under XWayland. Native Wayland applications are unsupported. Pass DISPLAY, XAUTHORITY and the desktop accessibility session bus to the server. -On macOS, accessibility observation and input require Accessibility permission. -Screenshots additionally require Screen Recording permission. macOS may -attribute the grant to the terminal or IDE that launched Qwen Code. Windows and -Linux use their platform accessibility and input facilities. +## Desktop and browser tasks -## Use the computer in front of you from a remote session +Use the explicit desktop interface for full-screen tasks: -When Qwen Code runs on a headless machine (a dev box, a server), the skill can -still drive the desktop you are sitting at: that computer lends its own -`node_repl` to one remote session. It works on macOS today. +```js +await cua.desktop.getScreenshot(); +await cua.desktop.click({ x: 300, y: 200 }, { key: 'shift' }); +``` -Set it up once on your computer: +Desktop coordinates are pixels in the latest primary-display screenshot. Desktop input controls the visible foreground desktop and can move the real pointer. For held keys, always use `try/finally` with `cua.desktop.keyDown(key)` and `keyUp(key)`. Held keys belong to the calling session and are released on graceful reset/end. While holding keys, use `keyDown/keyUp` for chords and desktop pointer actions without explicit modifiers; release them before text, ordinary shortcuts or application input. Abrupt OS process termination cannot guarantee cleanup. + +```js +var browser = await cua.getBrowser('Chrome'); +// Browser documentation and tabs are displayed on first selection. +var tab = await cua.getTab('TAB_ID', { browser: 'Chrome' }); +await tab.playwright.domSnapshot(); +``` + +`getBrowser`, `getTab`, and `createBrowserTab` return the existing repository browser-use SDK handles. Follow the selected browser's runtime documentation and connect the Qwen browser extension as before. The integration adds no browser windows API, DOM diff or new Windows support. + +## Permissions and lifecycle + +The server executes JavaScript with ordinary Node.js permissions and follows the client's MCP approval flow. On macOS, Accessibility is required for input/AX observation and Screen Recording for screenshots; Automation may be requested for document save/Finder rename. Grants belong to the host process's OS responsibility chain. On Windows, the normal installed daemon uses the existing UAC/logon-task path; applications it launches use a token derived from its own token at normal-user integrity. UIAccess is optional: explicitly setting `CUA_DRIVER_USE_UIACCESS=1` requires a trusted signed worker under Program Files and fails if unavailable. Default npm installation does not stage that worker or require its signature. + +`js` completes in one call and defaults to a 30-second timeout. A timeout can leave external effects; observe before retrying. `js_reset` discards bindings and gracefully closes the owning native/browser session. After context compaction call `cua.rewriteDocumentation()` before continuing. + +## Remote desktop relay + +For an existing macOS desktop relay setup, install its matching server: ```bash -npx -y @qwen-code/node-repl-mcp@0.1.7 desktop-relay install +npx -y @qwen-code/node-repl-mcp@0.26.0 desktop-relay install ``` -This installs `node_repl` and the SDK under `~/.qwen/desktop-relay` and registers -a launchd socket on `127.0.0.1:47821`. Nothing runs in the background; launchd -starts a short-lived process only when something connects. +This uses the existing local launchd socket at `127.0.0.1:47821`. In a secure Web Shell, choose **Use this computer**, then **Connect this computer**. The desktop prompts for each connection. The remote daemon must enable `QWEN_SERVE_CLIENT_MCP_OVER_WS=1`. -- **From the Web Shell.** The remote daemon must run with - `QWEN_SERVE_CLIENT_MCP_OVER_WS=1`, and the Web Shell must be a secure page - (https, or `http://localhost` through an SSH tunnel). In a session, choose - **Use this computer** in the sidebar footer, then **Connect this computer**. +## Direct MCP and troubleshooting -A dialog on your computer asks you to allow every connection. An allowed session -can run code on your computer with your permissions and see and control its -screen, just like local Computer Use, until you disconnect it or it ends. macOS -asks to allow `node` under Accessibility and Screen Recording the first time. +`qwen-cua-driver mcp` exposes the same 19 native Computer Use operations directly. App calls use `app`; desktop-capable calls use `scope:"desktop"` instead. `key_down/key_up` are desktop-only. Advanced native tools remain available through the host opt-in `CUA_DRIVER_EXTRA_TOOLS=1`. -## Troubleshooting +If the server is unavailable, check `qwen mcp list` and restart the client after adding it. If capture fails, check the current desktop session and OS permissions. If native download fails, check the exact release version/checksum asset or supply a locally built matching payload through `QWEN_CUA_SDK_NATIVE_DIR`. A locked desktop cannot provide GUI acceptance evidence. -- If `node_repl` is still unavailable after automatic setup, restart Qwen Code - and verify the server with `qwen mcp list`. -- If the SDK import still fails after automatic setup, confirm Qwen Code is - running from the workspace where the package was installed. -- After a timeout, cancellation, reset, or kernel crash, bootstrap the SDK - client again and request fresh state. - -## See also - -- [Skills](./skills.md) -- [MCP servers](./mcp.md) -- [Approval Mode](./approval-mode.md) -- [Sandboxing](./sandbox.md) +See [MCP servers](./mcp.md), [Approval Mode](./approval-mode.md) and [Sandboxing](./sandbox.md). diff --git a/package.json b/package.json index 422ce05e34f..84c5c29d22e 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "type": "module", "workspaces": [ "packages/*", + "packages/cua-driver/typescript", "packages/channels/base", "packages/channels/telegram", "packages/channels/weixin", diff --git a/packages/browser-use/LICENSE b/packages/browser-use/LICENSE new file mode 100644 index 00000000000..2d4f1fee822 --- /dev/null +++ b/packages/browser-use/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Google LLC + Copyright 2025 Qwen + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/packages/browser-use/build.mjs b/packages/browser-use/build.mjs index c1313197797..943c16f829c 100644 --- a/packages/browser-use/build.mjs +++ b/packages/browser-use/build.mjs @@ -31,8 +31,8 @@ const nodeBanner = // reads process.env/process.platform, so bind `process` through createRequire, // the one route the kernel leaves open. The load check at the bottom of this // file runs in Node's main realm, where `process` is always a global, so it -// cannot detect this binding going missing; src/skill-runtime-kernel.test.ts -// runs the skill's first cell through the real kernel for that. +// cannot detect this binding going missing; src/runtime-kernel.test.ts +// imports the SDK through the real kernel for that. const kernelBanner = "import { createRequire as __qwenCreateRequire } from 'node:module'; const process = __qwenCreateRequire(import.meta.url)('node:process');"; diff --git a/packages/browser-use/package.json b/packages/browser-use/package.json index 9f5a3bf13eb..0c379fa1db4 100644 --- a/packages/browser-use/package.json +++ b/packages/browser-use/package.json @@ -1,7 +1,6 @@ { "name": "@qwen-code/browser-use", - "private": true, - "version": "0.25.3", + "version": "0.26.0", "description": "Browser Use SDK for the Qwen Code Node REPL", "type": "module", "license": "Apache-2.0", @@ -26,7 +25,8 @@ "dist/sdk/types.d.ts", "dist/native-host.js", "dist/scripts/native-host-setup.js", - "NOTICE" + "NOTICE", + "LICENSE" ], "engines": { "node": ">=22" @@ -40,7 +40,8 @@ "smoke:qwen-saucedemo": "npm run build --workspace=@qwen-code/node-repl-mcp && npm run build --workspace=@qwen-code/chrome-bridge && npm run build && npm --prefix ../.. run bundle && node dist/scripts/smoke-qwen-saucedemo.js", "test:ci": "vitest run", "typecheck": "tsc --noEmit", - "test": "vitest run" + "test": "vitest run", + "prepack": "npm run build" }, "devDependencies": { "@types/node": "^22.0.0", @@ -51,5 +52,8 @@ }, "dependencies": { "playwright-core": "1.62.1" + }, + "publishConfig": { + "access": "public" } } diff --git a/packages/browser-use/scripts/smoke-checks.test.ts b/packages/browser-use/scripts/smoke-checks.test.ts index 98246a1a433..34dc9c81086 100644 --- a/packages/browser-use/scripts/smoke-checks.test.ts +++ b/packages/browser-use/scripts/smoke-checks.test.ts @@ -5,7 +5,17 @@ */ import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { runInNewContext } from 'node:vm'; import { describe, expect, it, vi } from 'vitest'; import { transformSync } from 'esbuild'; @@ -59,6 +69,87 @@ describe('preflight screenshot validation', () => { }); }); +describe('smoke artifact preflight', () => { + it.each([false, true])( + 'resolves import-only CUA exports before checking Browser SDK artifacts (missing host: %s)', + (missingHost) => { + const root = mkdtempSync(join(tmpdir(), 'qwen-smoke-preflight-')); + const write = (name: string, text = '') => { + const target = join(root, name); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, text); + }; + try { + write('dist/cli.js'); + write('packages/node-repl/dist/index.js'); + const sdk = 'packages/node-repl/node_modules/@qwen-code/cua-sdk'; + write( + `${sdk}/package.json`, + JSON.stringify({ + type: 'module', + exports: { './runtime': { import: './lib/runtime-entry.js' } }, + }), + ); + write(`${sdk}/lib/runtime-entry.js`); + const browser = `${sdk}/node_modules/@qwen-code/browser-use`; + write( + `${browser}/package.json`, + JSON.stringify({ exports: { './package.json': './package.json' } }), + ); + write(`${browser}/dist/index.js`); + if (!missingHost) write(`${browser}/dist/native-host.js`); + write(`${browser}/node_modules/playwright-core/package.json`, '{}'); + const smoke = readFileSync( + new URL('./smoke-qwen-saucedemo.ts', import.meta.url), + 'utf8', + ); + const start = smoke.indexOf('const packageRoot ='); + const end = smoke.indexOf("await withManagedChrome('sauce'"); + assert.ok(start >= 0 && end > start); + const builtUrl = pathToFileURL( + join( + root, + 'packages/browser-use/dist/scripts/smoke-qwen-saucedemo.js', + ), + ).href; + const code = transformSync( + smoke + .slice(start, end) + .replaceAll('import.meta.url', JSON.stringify(builtUrl)), + { loader: 'ts', target: 'node22' }, + ).code; + const result = spawnSync( + process.execPath, + [ + '--input-type=module', + '--eval', + ` + import assert from 'node:assert/strict'; + import { access } from 'node:fs/promises'; + import { createRequire } from 'node:module'; + import { dirname, join, resolve } from 'node:path'; + import { fileURLToPath } from 'node:url'; + import { spawnSync } from 'node:child_process'; + ${code} + `, + ], + { cwd: tmpdir(), encoding: 'utf8', timeout: 20_000 }, + ); + expect(result.error).toBeUndefined(); + if (missingHost) { + expect(result.status).not.toBe(0); + expect(result.stderr).toContain('ENOENT'); + expect(result.stderr).toContain('native-host.js'); + } else { + expect(result.status, result.stderr).toBe(0); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); + describe('smoke observation claims', () => { const smoke = readFileSync( new URL('./smoke-qwen-saucedemo.ts', import.meta.url), @@ -66,33 +157,61 @@ describe('smoke observation claims', () => { ); const checks = smoke.match(/const checks = \{[\s\S]*?\n {4}\};/)?.[0]; assert.ok(checks, 'Smoke checks must be exercised'); - const helperStart = smoke.indexOf('function stagedRuntimeImported('); + const helperStart = smoke.indexOf('function preloadedBrowserUsed('); const helperEnd = smoke.indexOf('\n}\n', helperStart) + 3; - assert.ok(helperStart >= 0, 'stagedRuntimeImported must be exercised'); - const stagedRuntimeImported = runInNewContext( + assert.ok(helperStart >= 0, 'preloadedBrowserUsed must be exercised'); + const preloadedBrowserUsed = runInNewContext( transformSync(smoke.slice(helperStart, helperEnd), { loader: 'ts', target: 'node22', - }).code + '\nstagedRuntimeImported;', + }).code + '\npreloadedBrowserUsed;', {}, - ) as ( - calls: Array<{ code: string; output: string }>, - browserRuntimeRoot: string, - builtinSkillRoot: string, - ) => boolean; + ) as (calls: Array<{ code: string; output: string }>) => boolean; + + const bootstrap = { + code: 'var browser = await cua.getBrowser();', + output: '# Qwen Browser Use\nBrowser documentation', + }; it.each([ - '// .type( .click( .finalize(', - 'nodeRepl.write(".type( .click( .finalize(");', - ])('does not report SDK setup from unrelated action text: %s', (code) => { + ['the preloaded browser bootstrap', [bootstrap], true], + ['no calls', [], false], + ['a failed or unobserved bootstrap', [{ ...bootstrap, output: '' }], false], + [ + 'a comment', + [{ ...bootstrap, code: '// await cua.getBrowser();' }], + false, + ], + [ + 'printed code', + [{ ...bootstrap, code: 'nodeRepl.write("cua.getBrowser()");' }], + false, + ], + [ + 'a manually imported SDK', + [ + { + ...bootstrap, + code: "await (await import('/elsewhere/index.js')).setupBrowserRuntime();", + }, + ], + false, + ], + [ + 'setup preceding the bootstrap', + [{ code: 'var cua = replacement;', output: '' }, bootstrap], + false, + ], + [ + 'a fallback import after bootstrap', + [bootstrap, { code: "await import('/elsewhere/index.js');", output: '' }], + false, + ], + ])('reports preloaded Browser Use for %s', (_label, calls, expected) => { const result = runInNewContext(checks + '\nchecks;', { - calls: [{ code, output: '' }], - joinedCode: code, + calls, joinedOutput: 'Total: $36.69', - browserRuntimeEntry: '/skill/runtime/index.js', - browserRuntimeRoot: '/skill/runtime', - builtinSkillRoot: '/skill', - stagedRuntimeImported, + preloadedBrowserUsed, events: [], moduleDirectoryRegistrationRequested: () => false, completion: { @@ -102,73 +221,10 @@ describe('smoke observation claims', () => { cartMatchesLowestThree: true, finalAnswer: '$36.69', }) as Record; - - expect(result.browserSdkImported).toBe(false); + expect(result.preloadedBrowserUsed).toBe(expected); expect(result.checkoutCompletionObserved).toBe(true); expect(result.reportedTotalIsExpected).toBe(true); }); - - it.each([ - [ - 'a literal entry path', - "await (await import('/skill/runtime/index.js')).setupBrowserRuntime();", - true, - ], - [ - 'an entry built from the staged runtime directory', - "const SKILL_BASE = '/skill/runtime';\nawait (await import(SKILL_BASE + '/index.js')).setupBrowserRuntime();", - true, - ], - [ - 'an entry built from the skill root', - "const base = '/skill';\nglobalThis.SKILL_BASE = base;\nawait (await import(SKILL_BASE + '/runtime/index.js')).setupBrowserRuntime();", - true, - ], - [ - 'a runtime loaded from somewhere else', - "await (await import('/elsewhere/runtime/index.js')).setupBrowserRuntime();", - false, - ], - [ - 'the skill root without setup', - "import('/skill/runtime/index.js');", - false, - ], - [ - 'a staged import that threw and fell back to a foreign entry', - "let sdk; try { sdk = await import('/skill/runtime/index.js'); } catch { sdk = await import('/repo/packages/browser-use/dist/index.js'); } await sdk.setupBrowserRuntime();", - false, - ], - [ - 'the setup export reached through an alias', - "const { setupBrowserRuntime: setup } = await import('/skill/runtime/index.js');\nawait setup();", - true, - ], - [ - 'the skill root echoed in prose next to a foreign import', - "// staged at /skill/runtime/index.js\nawait (await import('/elsewhere/index.js')).setupBrowserRuntime();", - false, - ], - ])('reports SDK setup for %s', (_label, code, expected) => { - const result = runInNewContext(checks + '\nchecks;', { - calls: [{ code, output: '' }], - joinedCode: code, - joinedOutput: 'Total: $36.69', - browserRuntimeEntry: '/skill/runtime/index.js', - browserRuntimeRoot: '/skill/runtime', - builtinSkillRoot: '/skill', - stagedRuntimeImported, - events: [], - moduleDirectoryRegistrationRequested: () => false, - completion: { - url: 'https://www.saucedemo.com/checkout-complete.html', - snapshot: 'Thank you for your order!', - }, - cartMatchesLowestThree: true, - finalAnswer: '$36.69', - }) as Record; - expect(result.browserSdkImported).toBe(expected); - }); }); describe('smoke price reads', () => { diff --git a/packages/browser-use/scripts/smoke-qwen-saucedemo.ts b/packages/browser-use/scripts/smoke-qwen-saucedemo.ts index 82ed40dced3..a43fe567913 100644 --- a/packages/browser-use/scripts/smoke-qwen-saucedemo.ts +++ b/packages/browser-use/scripts/smoke-qwen-saucedemo.ts @@ -5,7 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { spawn } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { access, chmod, @@ -16,6 +16,7 @@ import { rm, writeFile, } from 'node:fs/promises'; +import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -32,7 +33,8 @@ import { const PROMPT = [ '这是 Qwen Browser Use 的隔离 SauceDemo 冒烟测试。', - '使用 browser-use skill 和标准 node_repl，', + '使用预加载的 cua 和 Node REPL 的 js 工具。', + '首次 js 调用只执行 var browser = await cua.getBrowser();，阅读返回文档后再继续；不要手动导入 SDK 或注册模块目录。', '用 standard_user / secret_sauce 登录 https://www.saucedemo.com/，', '动态识别价格最低的三件商品并加入购物车，完成整个结账流程，', '提交后验证完成页，最后告诉我含税总价。', @@ -52,33 +54,48 @@ interface CommandResult { durationMs: number; } +const apiKey = process.env['DASHSCOPE_API_KEY']; +assert(apiKey, 'DASHSCOPE_API_KEY is required'); +const baseUrl = process.env['DASHSCOPE_BASE_URL']; +assert(baseUrl, 'DASHSCOPE_BASE_URL is required'); +const model = process.env['QWEN_BROWSER_USE_MODEL']; +assert(model, 'QWEN_BROWSER_USE_MODEL is required'); +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const repositoryRoot = resolve(packageRoot, '../..'); +const qwenEntry = join(repositoryRoot, 'dist/cli.js'); +const nodeReplEntry = join(repositoryRoot, 'packages/node-repl/dist/index.js'); +// Resolve import-only exports from the MCP host, independent of the smoke cwd. +const cuaResolution = spawnSync( + process.execPath, + [ + '--input-type=module', + '--eval', + "process.stdout.write(import.meta.resolve('@qwen-code/cua-sdk/runtime'))", + ], + { cwd: dirname(nodeReplEntry), encoding: 'utf8', timeout: 10_000 }, +); +assert( + cuaResolution.status === 0, + 'Could not resolve CUA runtime: ' + + (cuaResolution.error?.message ?? cuaResolution.stderr), +); +const cuaRuntimeEntry = fileURLToPath(cuaResolution.stdout); +const browserManifest = createRequire(cuaRuntimeEntry).resolve( + '@qwen-code/browser-use/package.json', +); +const browserPackageRoot = dirname(browserManifest); +await Promise.all( + [ + qwenEntry, + nodeReplEntry, + cuaRuntimeEntry, + join(browserPackageRoot, 'dist/index.js'), + join(browserPackageRoot, 'dist/native-host.js'), + createRequire(browserManifest).resolve('playwright-core/package.json'), + ].map((path) => access(path)), +); + await withManagedChrome('sauce', async (chrome) => { - const apiKey = process.env['DASHSCOPE_API_KEY']; - assert(apiKey, 'DASHSCOPE_API_KEY is required'); - const baseUrl = process.env['DASHSCOPE_BASE_URL']; - assert(baseUrl, 'DASHSCOPE_BASE_URL is required'); - const model = process.env['QWEN_BROWSER_USE_MODEL']; - assert(model, 'QWEN_BROWSER_USE_MODEL is required'); - const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); - const repositoryRoot = resolve(packageRoot, '../..'); - const qwenEntry = join(repositoryRoot, 'dist/cli.js'); - const nodeReplEntry = join( - repositoryRoot, - 'packages/node-repl/dist/index.js', - ); - const builtinSkillRoot = join(repositoryRoot, 'dist/bundled/browser-use'); - const browserRuntimeRoot = join(builtinSkillRoot, 'runtime'); - const browserRuntimeEntry = join(browserRuntimeRoot, 'index.js'); - await Promise.all( - [ - qwenEntry, - nodeReplEntry, - join(builtinSkillRoot, 'SKILL.md'), - browserRuntimeEntry, - join(browserRuntimeRoot, 'native-host.js'), - join(browserRuntimeRoot, 'node_modules/playwright-core/package.json'), - ].map((path) => access(path)), - ); const runId = new Date() .toISOString() .replaceAll(':', '') @@ -158,7 +175,7 @@ await withManagedChrome('sauce', async (chrome) => { '--allowed-mcp-server-names', 'node-repl', '--core-tools', - 'skill,tool_search', + 'tool_search', '--approval-mode', 'yolo', '--max-wall-time', @@ -230,11 +247,7 @@ await withManagedChrome('sauce', async (chrome) => { const joinedOutput = calls.map((call) => call.output).join('\n'); const checks = { existingNodeReplUsed: calls.length > 0, - browserSdkImported: stagedRuntimeImported( - calls, - browserRuntimeRoot, - builtinSkillRoot, - ), + preloadedBrowserUsed: preloadedBrowserUsed(calls), noModuleDirectoryRegistered: !moduleDirectoryRegistrationRequested(events), checkoutCompletionObserved: @@ -364,46 +377,18 @@ function assertNoSecret(secret: string, ...values: string[]): void { ); } -// A run proves it loaded the staged built-in runtime only through an import -// expression that names that runtime: a literal path under the staged -// runtime directory, or a path the same or an earlier cell assembled from the -// skill root the CLI substituted into SKILL.md. The cell must also reach the -// setup export (a destructuring alias still names it). A literal entry loaded -// from anywhere else — the fallback a model reaches for when the staged -// import throws — disqualifies the run, because it is not the shipped -// runtime that was exercised. -function stagedRuntimeImported( - calls: NodeReplCall[], - browserRuntimeRoot: string, - builtinSkillRoot: string, -): boolean { - const entryFile = /\/(?:index|native-host)\.js/; - let rootBound = false; - let staged = false; - for (const call of calls) { - for (const root of [builtinSkillRoot, browserRuntimeRoot]) { - if ( - call.code.includes(`'${root}'`) || - call.code.includes(`"${root}"`) || - call.code.includes(`\`${root}\``) - ) - rootBound = true; - } - let importsStaged = false; - for (const match of call.code.matchAll(/import\(([^()]*)\)/g)) { - const expression = match[1] ?? ''; - const literal = /^\s*(['"`])([^'"`$]+)\1\s*$/.exec(expression)?.[2]; - if (literal !== undefined) { - if (literal.startsWith(`${browserRuntimeRoot}/`)) importsStaged = true; - else if (entryFile.test(literal)) return false; - continue; - } - if (/index\.js/.test(expression) && rootBound) importsStaged = true; - } - if (importsStaged && call.code.includes('setupBrowserRuntime')) - staged = true; - } - return staged; +// The first successful cell must acquire the preloaded browser and receive +// its documentation. A manual import fallback does not exercise that entry. +function preloadedBrowserUsed(calls: NodeReplCall[]): boolean { + const first = calls[0]; + return ( + first !== undefined && + /^\s*(?:var|let|const)\s+\w+\s*=\s*await\s+cua\.getBrowser\(\)\s*;?\s*$/.test( + first.code, + ) && + first.output.includes('# Qwen Browser Use') && + calls.every((call) => !/\bimport\s*\(/.test(call.code)) + ); } // The prompt makes the model confirm cart.html with a separate tab.url() call diff --git a/packages/browser-use/scripts/smoke-transcript.test.ts b/packages/browser-use/scripts/smoke-transcript.test.ts index 67e964d0722..113575a007a 100644 --- a/packages/browser-use/scripts/smoke-transcript.test.ts +++ b/packages/browser-use/scripts/smoke-transcript.test.ts @@ -86,23 +86,25 @@ describe('smoke transcript tool uses', () => { it('collects REPL cells invoked directly and through the deferred-tool bridge', () => { expect( collectSuccessfulNodeReplCalls([ - toolUse('a', 'mcp__node-repl__node_repl', { code: 'direct' }), + toolUse('a', 'mcp__node-repl__js', { code: 'direct' }), toolUse('b', 'tool_call', { - name: 'mcp__node-repl__node_repl', + name: 'mcp__node-repl__js', arguments: { code: 'bridged' }, }), toolUse('c', 'tool_call', { - name: 'mcp__node-repl__node_repl', + name: 'mcp__node-repl__js', arguments: { code: 'failed' }, }), toolUse('d', 'tool_call', { - name: 'mcp__node-repl__node_repl_reset', + name: 'mcp__node-repl__js_reset', arguments: { code: 'other tool' }, }), toolResult('a', 'out-a'), toolResult('b', 'out-b'), toolResult('c', 'out-c', true), toolResult('d', 'out-d'), + toolUse('legacy', 'mcp__node-repl__node_repl', { code: 'retired' }), + toolResult('legacy', 'old-output'), ]), ).toEqual([ { code: 'direct', output: 'out-a' }, @@ -128,12 +130,12 @@ describe('smoke transcript tool uses', () => { ], [ 'an ordinary REPL cell', - toolUse('t', 'mcp__node-repl__node_repl', { code: '1' }), + toolUse('t', 'mcp__node-repl__js', { code: '1' }), false, ], [ 'a bridged call to another tool', - toolUse('t', 'tool_call', { name: 'mcp__node-repl__node_repl_reset' }), + toolUse('t', 'tool_call', { name: 'mcp__node-repl__js_reset' }), false, ], ])( diff --git a/packages/browser-use/scripts/smoke-transcript.ts b/packages/browser-use/scripts/smoke-transcript.ts index 04ae32ad21c..49a0f663f4c 100644 --- a/packages/browser-use/scripts/smoke-transcript.ts +++ b/packages/browser-use/scripts/smoke-transcript.ts @@ -62,7 +62,7 @@ export function collectSuccessfulNodeReplCalls( for (const { id, name, input } of toolUses(events)) { if ( typeof id === 'string' && - name === 'mcp__node-repl__node_repl' && + name === 'mcp__node-repl__js' && isRecord(input) && typeof input.code === 'string' ) { @@ -91,7 +91,7 @@ export function collectSuccessfulNodeReplCalls( return successful; } -// The bundled SDK loads its own dependencies. A run that registered a module +// The preloaded SDK loads its own dependencies. A run that registered a module // directory would pass even if that stopped working, so it does not count. export function moduleDirectoryRegistrationRequested( events: unknown[], diff --git a/packages/browser-use/scripts/verify-package.mjs b/packages/browser-use/scripts/verify-package.mjs new file mode 100644 index 00000000000..21ffcdd0784 --- /dev/null +++ b/packages/browser-use/scripts/verify-package.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import { mkdirSync, readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const index = process.argv.indexOf('--output-dir'); +if (index < 0 || !process.argv[index + 1]) + throw new Error('missing --output-dir'); +const output = resolve(process.argv[index + 1]); +mkdirSync(output, { recursive: true }); +function run(args, capture = false) { + const result = spawnSync('npm', args, { + cwd: root, + encoding: 'utf8', + stdio: capture ? 'pipe' : 'inherit', + }); + if (result.error) throw result.error; + if (result.status !== 0) + throw new Error(result.stderr || `npm exited ${result.status}`); + return result.stdout; +} +const manifest = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); +const [packed] = JSON.parse( + run(['pack', '.', '--pack-destination', output, '--json'], true), +); +if ( + packed.name !== '@qwen-code/browser-use' || + packed.version !== manifest.version +) + throw new Error('browser package identity mismatch'); +const files = new Set(packed.files.map((file) => file.path)); +for (const name of [ + 'LICENSE', + 'NOTICE', + 'dist/index.js', + 'dist/index.d.ts', + 'dist/core/primitives.d.ts', + 'dist/sdk/types.d.ts', + 'dist/native-host.js', + 'dist/scripts/native-host-setup.js', +]) { + if (!files.has(name)) throw new Error(`browser package missing ${name}`); +} +run([ + 'publish', + join(output, packed.filename), + '--dry-run', + '--access', + 'public', + '--registry=https://registry.npmjs.org', +]); +process.stdout.write( + JSON.stringify({ + name: packed.name, + version: packed.version, + tarball: join(output, packed.filename), + integrity: packed.integrity, + }) + '\n', +); diff --git a/packages/browser-use/src/skill-runtime-kernel.test.ts b/packages/browser-use/src/runtime-kernel.test.ts similarity index 80% rename from packages/browser-use/src/skill-runtime-kernel.test.ts rename to packages/browser-use/src/runtime-kernel.test.ts index 95fc3b5d6db..32dd24ab27b 100644 --- a/packages/browser-use/src/skill-runtime-kernel.test.ts +++ b/packages/browser-use/src/runtime-kernel.test.ts @@ -11,18 +11,9 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import type { TestContext } from 'vitest'; import { afterEach, describe, expect, it, vi } from 'vitest'; -// The bundled skill has the model import the staged runtime/index.js from -// inside the node_repl kernel's untrusted vm context, which has no `process` -// global. build.mjs's own load check imports dist/index.js in Node's main -// realm and therefore cannot see that difference, so this test runs the -// skill's first cell, verbatim from SKILL.md, through the real kernel built in -// packages/node-repl/dist against a runtime it stages itself, the way the -// bundle step and `npm run dev` do, into a temporary skill directory. +// Import the packaged browser runtime inside the real Node REPL kernel, +// including its pinned Playwright dependency and its VM process binding. const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); -const skillFile = path.join( - repoRoot, - 'packages/core/src/skills/bundled/browser-use/SKILL.md', -); const browserUseDist = path.join(repoRoot, 'packages/browser-use/dist'); const nodeReplDist = path.join(repoRoot, 'packages/node-repl/dist'); const requiredArtifacts = [ @@ -40,7 +31,7 @@ interface KernelOutcome { interface Kernel { exec(request: { code: string; timeoutMs: number }): Promise; - dispose(): void; + dispose(): Promise; } interface KernelManagerModule { @@ -61,23 +52,15 @@ interface CopyAssetsModule { copyBrowserUseAssets(root: string, skillDir: string): void; } -function firstSkillCell(skillBase: string): string { - const skill = fs.readFileSync(skillFile, 'utf8'); - const cell = /```js\n([\s\S]*?)```/.exec(skill)?.[1]; - if (cell === undefined) throw new Error('SKILL.md has no ```js cell'); - expect(cell).toContain("import('/absolute/skill/base/runtime/index.js')"); - return cell.replaceAll('/absolute/skill/base', skillBase); -} - describe( - 'bundled skill runtime inside the node_repl kernel', + 'browser runtime inside the node_repl kernel', { timeout: 90_000 }, () => { afterEach(() => { vi.unstubAllEnvs(); }); - async function runFirstSkillCell( + async function runBrowserRuntime( context: TestContext, prepareRuntime: (runtimeDir: string) => void = () => {}, ): Promise<{ outcome: KernelOutcome; runtimeDir: string }> { @@ -129,12 +112,12 @@ describe( }); try { const outcome = await manager.exec({ - code: firstSkillCell(skillBase), + code: `var browserRuntime = await import(${JSON.stringify(pathToFileURL(path.join(runtimeDir, 'index.js')).href)}); var browserRuntimeAgent = await browserRuntime.setupBrowserRuntime(); await browserRuntimeAgent.browsers.list();`, timeoutMs: 60_000, }); return { outcome, runtimeDir }; } finally { - manager.dispose(); + await manager.dispose(); fs.rmSync(tmpRootDir, { recursive: true, force: true }); } } @@ -152,8 +135,8 @@ describe( }); } - it("runs SKILL.md's first cell up to the Chrome bridge without a ReferenceError", async (context) => { - const { outcome } = await runFirstSkillCell(context); + it('loads the SDK up to the Chrome bridge without a ReferenceError', async (context) => { + const { outcome } = await runBrowserRuntime(context); // A `ReferenceError: process is not defined` here means the staged // runtime lost the kernel-realm `process` binding that build.mjs adds @@ -167,7 +150,7 @@ describe( }); it('reports an incomplete runtime when its bundled playwright-core is missing', async (context) => { - const { outcome, runtimeDir } = await runFirstSkillCell( + const { outcome, runtimeDir } = await runBrowserRuntime( context, (runtime) => { fs.rmSync(path.join(runtime, 'node_modules/playwright-core'), { @@ -187,7 +170,7 @@ describe( it('refuses a playwright-core other than the pinned version', async (context) => { // Stands in for a missing bundled copy that Node's lookup replaced with // an unrelated install further up the directory tree. - const { outcome, runtimeDir } = await runFirstSkillCell( + const { outcome, runtimeDir } = await runBrowserRuntime( context, (runtime) => { const manifestPath = path.join( diff --git a/packages/core/src/skills/bundled-skills.integration.test.ts b/packages/core/src/skills/bundled-skills.integration.test.ts index 2794d1cf5e9..111791346b4 100644 --- a/packages/core/src/skills/bundled-skills.integration.test.ts +++ b/packages/core/src/skills/bundled-skills.integration.test.ts @@ -14,6 +14,8 @@ import { type AvailableSkillEntry, } from '../tools/skill-utils.js'; import { parseSkillContent } from './skill-load.js'; +import { SkillManager } from './skill-manager.js'; +import { makeFakeConfig } from '../test-utils/config.js'; // Bundled skills are loaded from disk at runtime by SkillManager. A typo in // frontmatter (missing `description`, malformed YAML, broken `---` delimiter, @@ -30,7 +32,11 @@ const bundledDir = path.join( const skillNames = fs .readdirSync(bundledDir, { withFileTypes: true }) - .filter((d) => d.isDirectory()) + .filter( + (d) => + d.isDirectory() && + fs.existsSync(path.join(bundledDir, d.name, 'SKILL.md')), + ) .map((d) => d.name); describe('bundled SKILL.md files', () => { @@ -38,6 +44,15 @@ describe('bundled SKILL.md files', () => { expect(skillNames.length).toBeGreaterThan(0); }); + it.each(['browser-use', 'computer-use'])( + 'keeps retired %s documentation out of the bundled skill catalog', + async (name) => { + expect(skillNames).not.toContain(name); + const manager = new SkillManager(makeFakeConfig()); + await expect(manager.loadSkill(name, 'bundled')).resolves.toBeNull(); + }, + ); + it.each(skillNames)('%s/SKILL.md parses with required fields', (name) => { const file = path.join(bundledDir, name, 'SKILL.md'); const content = fs.readFileSync(file, 'utf8'); diff --git a/packages/core/src/skills/bundled/browser-use/SKILL.md b/packages/core/src/skills/bundled/browser-use/SKILL.md deleted file mode 100644 index 923d26c0b24..00000000000 --- a/packages/core/src/skills/bundled/browser-use/SKILL.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -name: browser-use -description: Control the user's Chrome through the existing persistent Node REPL and the Qwen Browser SDK. ---- - -# Browser Use - -Browser Use is bundled with Qwen Code and uses the generic Node REPL MCP tools -and the Qwen Chrome extension. Do not install a separate Qwen extension or -start a separate Browser Use MCP server. - -## Setup - -Install and enable the Qwen Code Chrome extension in the profile the user wants -to use. On macOS and Linux, first use of this SDK automatically registers the -shared Native Messaging host in the user's installation directory. Windows is -not supported: no Native Messaging host is ever registered there, so the -extension has nothing to connect back to. Calls that start a browser session -fail with `BROWSER_DISCONNECTED`, and `browsers.list()` waits out the connect -budget and returns an empty list: on Windows an empty list means the platform -is unsupported, not that no profile exists. If the user is on Windows, tell them -Browser Use needs macOS or Linux and stop. Do not retry, and do not send them to -the Chrome Web Store or to `chrome://extensions`; neither can fix this. A browser -task opts into this local setup, which can finish before the extension connects. -The SDK checks the actual connection and protocol instead of reading Chrome's -extension preferences. - -If the extension does not connect on macOS or Linux, ask the user to open Chrome -and check the extension in the intended profile. It installs from the Chrome Web -Store: -https://chromewebstore.google.com/detail/qwen-code/hdhmmjclhibojdddmancfgbkleahfaph - -If the store reports it is not available in the user's region, they can build -it from `packages/chrome-extension` in the Qwen Code repository (its -README) and load `dist/extension` through `chrome://extensions` (Developer mode -→ Load unpacked). After installation or enabling it, retry the connection. - -A usable Host of the same protocol is reused. A Host installed by a newer Qwen -Code is never downgraded: if setup reports one, tell the user to update Qwen -Code and stop. The Host files persist after Qwen exits. The user can inspect or -remove them with `node /runtime/scripts/native-host-setup.js status` -or `uninstall`; `install` explicitly switches the Host to this Qwen Code's copy -from the next time Chrome starts it. A later Browser Use initialization can -register the Host again. If required Host files cannot be read or written, -report the failing action and path so the user can resolve access and retry. - -Multiple Qwen sessions can use the same profile, each controlling its own tabs. -A tab held by another session reports `TAB_OWNERSHIP_CONFLICT`. Use another tab -or wait for that session to release it. The setup below binds the runtime to -the default profile through `browserAgent.browsers.get('chrome')`. Only when the -user names a specific Chrome profile, call `browserAgent.browsers.list()` before -that line and pass the returned ID to `browserAgent.browsers.get(id)` instead. -The runtime stays bound to the first profile it connects to; to switch profiles, -call `node_repl_reset` and run the setup again. - -If `node_repl` is unavailable, configure it with: - -```bash -qwen mcp add --scope user node-repl npx -y @qwen-code/node-repl-mcp@0.1.7 -``` - -Then tell the user to restart Qwen Code and stop. Do not start a separate -Browser Use MCP server. Screenshot metadata requires `@qwen-code/node-repl-mcp` -0.1.7 or later, so keep this exact pin. - -Qwen reports the absolute `Base directory for this skill` when loading this -file. Use that directory as ``. The bundled SDK ships with its -dependencies: import it directly, without installing packages or registering -module directories, replacing the example skill base below with that absolute -path. If the import reports a missing file or an incomplete runtime, stop and -report an incomplete Browser Use runtime instead of installing dependencies -into the workspace: - -```js -globalThis.browserAgent ??= await ( - await import('/absolute/skill/base/runtime/index.js') -).setupBrowserRuntime(); -globalThis.browser ??= await browserAgent.browsers.get('chrome'); -nodeRepl.write(await browser.documentation()); -``` - -Create a tab or claim an exact result returned by `openTabs()`: - -```js -globalThis.tab = await browser.tabs.new(); -await tab.goto('https://example.com/'); -nodeRepl.write(await tab.playwright.domSnapshot()); -``` - -```js -const candidates = await browser.user.openTabs(); -nodeRepl.write(candidates); -globalThis.tab = await browser.user.claimTab( - candidates.find((candidate) => candidate.url === 'https://example.com/'), -); -``` - -## Interaction - -Observe before acting and observe again after acting. Prefer semantic -Playwright locators. Use `tab.dom_cua.get_visible_dom()` and its `node_id` -values when snapshot refs are clearer, and use `tab.cua` for visual coordinate -targets. - -```js -await tab.playwright.getByRole('button', { name: 'Continue' }).click(); -await tab.playwright.getByLabel('Email').type('user@example.com'); -await tab.dom_cua.click({ node_id: 'e4' }); -await tab.dom_cua.type({ text: 'hello' }); -``` - -When an action should navigate, arm the Playwright watcher around it: - -```js -await tab.playwright.expectNavigation( - () => tab.playwright.getByRole('link', { name: 'Next' }).click(), - { url: '**/next.html' }, -); -``` - -Render screenshots with their metadata: - -```js -await nodeRepl.emitImage(await tab.screenshot()); -``` - -The metadata reaches the model only through `@qwen-code/node-repl-mcp` 0.1.6 -or later; an older server accepts the call and silently drops it. If -`node_repl` was registered before this skill existed, check its command with -`qwen mcp list`, and when it pins an older version, re-register it with the -`qwen mcp add` command from Setup above and ask the user to restart Qwen Code. - -Use `tab.dev.logs()` for bounded console diagnostics. Arm -`tab.playwright.waitForEvent('download' | 'filechooser')` before the action that -triggers it. Handle JavaScript dialogs through `tab.getJsDialog()`. Use a -focused `tab.playwright.evaluate()` only when locators cannot obtain the needed -data. - -Use `browser.user.history()` only when the task needs browser history, with a -focused query and bounded result count. - -## Finish - -Treat `browser.tabs.finalize({ keep })` as the final browser action of the turn. -Omitted agent-created tabs close; omitted claimed tabs are released without -closing. `keep` is the complete set for this call. Keep a user-facing result as -`deliverable`, and keep a live page under control as `handoff`. Repeat a -handoff in each later turn that still needs it; omitting it from the next -finalize call or closing the runtime closes an agent-created tab. - -```js -await browser.tabs.finalize({ - keep: [{ tab, status: 'deliverable' }], -}); -``` - -The Node kernel preserves `globalThis` and top-level bindings across cells. -Keep cells short, await every Browser SDK promise, and use `nodeRepl.write()` to -return text or structured data. diff --git a/packages/core/src/skills/bundled/browser-use/skill.test.ts b/packages/core/src/skills/bundled/browser-use/skill.test.ts deleted file mode 100644 index e1c1bbbdae1..00000000000 --- a/packages/core/src/skills/bundled/browser-use/skill.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -/** - * @license - * Copyright 2026 Qwen Team - * SPDX-License-Identifier: Apache-2.0 - */ - -import fs from 'node:fs'; -import { fileURLToPath } from 'node:url'; -import { runInNewContext } from 'node:vm'; -import { describe, expect, it, vi } from 'vitest'; -import { makeFakeConfig } from '../../../test-utils/config.js'; -import { SkillManager } from '../../skill-manager.js'; - -const skillUrl = new URL('./SKILL.md', import.meta.url); -const skill = fs.readFileSync(skillUrl, 'utf8'); -const nodeReplPackage = JSON.parse( - fs.readFileSync( - new URL('../../../../../node-repl/package.json', import.meta.url), - 'utf8', - ), -) as { version: string }; - -describe('bundled browser-use skill', () => { - it('is discovered without a Qwen extension', async () => { - const manager = new SkillManager(makeFakeConfig()); - - await expect( - manager.loadSkill('browser-use', 'bundled'), - ).resolves.toMatchObject({ - name: 'browser-use', - level: 'bundled', - filePath: fileURLToPath(skillUrl), - }); - }); - - it('loads its bundled runtime through the generic Node REPL', () => { - expect(skill).toContain('If `node_repl` is unavailable'); - expect(skill).toContain('qwen mcp add --scope user node-repl'); - expect(skill).toContain( - `@qwen-code/node-repl-mcp@${nodeReplPackage.version}`, - ); - expect(skill).not.toContain('@qwen-code/node-repl-mcp@latest'); - // The SDK loads its own bundled dependencies, so setup must not send the - // model through a module-directory registration first. - expect(skill).not.toContain('node_repl_add_node_module_dir'); - expect(skill).not.toContain('node_modules'); - expect(skill).toContain("import('/absolute/skill/base/runtime/index.js')"); - expect(skill).not.toContain(''); - expect(skill).not.toContain('qwen extensions install'); - expect(skill).not.toContain('npm install --no-save'); - }); - - it('pins the Node REPL version that carries screenshot metadata', () => { - const pin = skill.match( - /^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@(\S+)$/m, - )?.[1]; - expect(pin).toBe(nodeReplPackage.version); - expect(skill.replace(/\s+/g, ' ')).toContain( - `Screenshot metadata requires \`@qwen-code/node-repl-mcp\` ${nodeReplPackage.version} or later`, - ); - }); - - it('names where the Qwen Chrome extension comes from', () => { - expect(skill).toContain( - 'https://chromewebstore.google.com/detail/qwen-code/hdhmmjclhibojdddmancfgbkleahfaph', - ); - expect(skill).toContain('`packages/chrome-extension`'); - expect(skill).toContain('`dist/extension`'); - expect(skill).toContain('Load unpacked'); - }); - - it('tells the agent that Windows is unsupported', () => { - const prose = skill.replace(/\s+/g, ' '); - expect(prose).toContain( - 'Windows is not supported: no Native Messaging host is ever registered there', - ); - expect(prose).toContain( - 'Calls that start a browser session fail with `BROWSER_DISCONNECTED`, and `browsers.list()` waits out the connect budget and returns an empty list', - ); - expect(prose).toContain( - 'tell them Browser Use needs macOS or Linux and stop', - ); - expect(prose).toContain( - 'Do not retry, and do not send them to the Chrome Web Store', - ); - expect(prose).toContain( - 'If the extension does not connect on macOS or Linux', - ); - }); - - it('uses the current Browser SDK contract', () => { - expect(skill).toContain('setupBrowserRuntime()'); - expect(skill).toContain('nodeRepl.write'); - expect(skill).toContain('browser.tabs.finalize'); - expect(skill).toContain('complete set'); - expect(skill).toContain('handoff in each later turn'); - expect(skill).toContain('node_id'); - expect(skill).not.toContain('setupBrowserRuntime(nodeRepl)'); - expect(skill).not.toContain('dev.network'); - expect(skill).not.toContain('markDeliverable'); - expect(skill).not.toContain('markHandoff'); - expect(skill).not.toContain('domSnapshot({ filter:'); - }); - - it('passes the complete screenshot and metadata to emitImage', async () => { - const example = [...skill.matchAll(/```js\n([\s\S]*?)```/g)].find( - ([, code]) => code.includes('tab.screenshot()'), - )?.[1]; - expect(example).toBeDefined(); - - const shot = { - bytes: new Uint8Array([1, 2, 3]), - mimeType: 'image/jpeg', - metadata: { - width: 1280, - height: 720, - viewport: { width: 1280, height: 720 }, - devicePixelRatio: 2, - coordinateSpace: 'css-pixels', - }, - }; - const screenshot = vi.fn().mockResolvedValue(shot); - const output: unknown[] = []; - - await runInNewContext(`(async () => {\n${example}\n})()`, { - tab: { screenshot }, - nodeRepl: { - write: (text: string) => output.push({ text }), - emitImage: async (image: unknown) => { - output.push({ image }); - }, - }, - }); - - expect(screenshot).toHaveBeenCalledTimes(1); - expect(output).toEqual([{ image: shot }]); - }); -}); diff --git a/packages/core/src/skills/bundled/computer-use/SKILL.md b/packages/core/src/skills/bundled/computer-use/SKILL.md deleted file mode 100644 index eafc040fe63..00000000000 --- a/packages/core/src/skills/bundled/computer-use/SKILL.md +++ /dev/null @@ -1,295 +0,0 @@ ---- -name: computer-use -description: Control local desktop applications through Computer Use for tasks that require reading or operating app UI. Prefer purpose-built connectors, APIs, or CLIs when available. ---- - -## node_repl + @qwen-code/cua-sdk (Computer Use) - -- Use `node_repl` (JavaScript) for all Computer Use actions. -- If both servers are available, use `node_repl` from the - `desktop-node-repl` MCP server; the regular server runs on the Qwen Code host. -- Do not use other technologies besides `node_repl` for computer interactions, unless specifically requested by the user (e.g. AppleScript, `osascript`, JXA, System Events, synthesized input). -- Prefer a dedicated plugin or skill when it can complete the task; use Computer Use for app interactions that are not exposed through a more specific interface. -- `node_repl` state is persistent across calls. -- For text output, use `nodeRepl.write(...)`. It takes a string; use `JSON.stringify(...)` only for textual metadata. For an observation, write its `.text` and emit its screenshots as images; do not stringify an observation or driver result containing image bytes. -- Omit `yield_time_ms` for ordinary UI calls to use the default 10-second wait. A shorter yield does not speed up the action and can add a `node_repl_wait` round. Use a shorter yield only when you need control back before completion; if a cell is still running, collect its result with `node_repl_wait` before issuing dependent actions. - -## Forwarding results in Codex code mode - -When calling `node_repl` through `tools.*` inside Codex's outer `functions.exec`, -forward each returned `content` block by its type. `nodeRepl.emitImage(...)` -produces an MCP image block; the outer script must pass that block to `image()` -for the model to receive an image. Prefer the desktop relay tool when it is -present; otherwise use the regular `node-repl` server: - -```js -// A code-mode `tools` object throws on an unknown key, so probe with `in`: -// reading an unbound tool would abort the script before any fallback ran. -const DESKTOP_NODE_REPL = 'mcp__desktop_node_repl__node_repl'; -const nodeReplTool = - DESKTOP_NODE_REPL in tools - ? tools[DESKTOP_NODE_REPL] - : tools.mcp__node_repl__node_repl; -const result = await nodeReplTool({ code }); -for (const block of result.content ?? []) { - if (block.type === 'text') { - text(block.text); - } else if (block.type === 'image') { - image(block); - } -} -``` - -Here `code` is the JavaScript to run in the persistent Node REPL. Keep the -forwarding loop in the outer code-mode script, outside that `code` string. -Apply the same loop to `node_repl_wait` results: images may arrive only when a -running cell completes. Forward text blocks too, including running-cell IDs and -errors. Direct MCP tool calls do not need this outer forwarding loop. - -Do not use `text(result)`, `text(block)`, or `JSON.stringify(result)` to forward -an MCP result containing images: this turns image base64 into text, consuming -context without showing the image. `image()` accepts one image block, not the -whole result, so do not use `image(result)` either. - -## Bootstrap - -If `desktop-node-repl` is connected, skip the installation commands below: it -already provides `node_repl` and the SDK on the connected computer. Continue -with the `computer` initialization below. Otherwise, if `node_repl` is -unavailable, run: - -```bash -qwen mcp add --scope user node-repl npx -y @qwen-code/node-repl-mcp@0.1.7 -npm install --no-save --package-lock=false @qwen-code/cua-sdk@0.20.11 -``` - -Tell the user to restart Qwen Code, then stop. If only the SDK import is missing, -run the second command and retry. - -Reuse an existing `computer` connected to the intended desktop. Otherwise import -once per fresh `node_repl` session. The same App workflow below applies to macOS, -Linux and Windows; no additional platform resource is needed: - -```js -globalThis.computer = await ( - await import('@qwen-code/cua-sdk/computer-use') -).ComputerUse.create(); -var platform = await computer.getPlatform(); -nodeRepl.write(`Connected platform: ${platform}`); -``` - -Use the connected platform for shortcuts, not the CLI or Node host operating system. -When the task identifies an app, combine initialization with `computer.getApp()` -and its first `getState()` in the same call. Read that state before editing or input. - -## API surface - -```ts -type Point = number | { x: number; y: number }; -type ComputerUse = { - getPlatform: () => Promise<'macos' | 'linux' | 'windows'>; - getApp: (nameOrIdentifierOrPath: string) => Promise; - listApps: () => Promise< - Array<{ id: string; displayName: string; isRunning: boolean }> - >; - close: () => Promise; -}; -type App = { - getState: (options?: { - disableDiff?: boolean; - includeScreenshot?: boolean; - maxTextChars?: number; - }) => Promise; - click: ( - point: Point, - options?: { button?: 'left' | 'right' | 'middle'; count?: number }, - ) => Promise; - doubleClick: (point: Point) => Promise; - rightClick: ( - point: Point, - options?: { modifier?: string[] }, - ) => Promise; - setValue: (element: number, value: string) => Promise; - performSecondaryAction: (element: number, action: string) => Promise; - typeText: (text: string) => Promise; - paste: ( - text: string, - options?: { format?: 'text' | 'md' | 'html'; signal?: AbortSignal }, - ) => Promise; - selectText: ( - element: number, - text: string, - options?: { - prefix?: string; - suffix?: string; - selection?: 'text' | 'cursor_before' | 'cursor_after'; - signal?: AbortSignal; - }, - ) => Promise; - pressKey: ( - key: string, - options?: { modifiers?: string[] }, - ) => Promise; - hotkey: (keys: string[]) => Promise; - scroll: ( - point: Point, - options: { direction: 'up' | 'down' | 'left' | 'right'; amount?: number }, - ) => Promise; - drag: (options: { - fromX: number; - fromY: number; - toX: number; - toY: number; - }) => Promise; -}; -type State = { - app: string; - window: string; - mode: 'full' | 'diff' | 'no_change'; - text: string; - screenshot?: { images: Array<{ mimeType: string; dataBase64: string }> }; -}; -``` - -## Workflow - -`paste` and `selectText` currently require macOS. On Linux/Windows, use the shared -`typeText`, `setValue`, `pressKey` and observed actions. Unsupported text methods -fail with `unsupported_platform`; do not retry them as window failures. - -### 1. Initialize - -If initialization already bound the task's app and returned its state, reuse -that `app` and observation. Otherwise, bind the app named by the task, then read its state. -`getApp()` binds identity; `getState()` can open a discovered stopped app. Combine these -steps in one Node REPL call: - -```js -var app = await computer.getApp('Microsoft Excel'); -nodeRepl.write((await app.getState()).text); -``` - -The app handle tracks its current window and dialog. Read the returned window -title to confirm the intended document. If the app is unknown or its name is -ambiguous, discover applications with `computer.listApps()` and use a matching -application `id` when it distinguishes the app. Two running instances may have -the same ID; retrying that ID cannot resolve the ambiguity. Ask the user to keep -only the intended instance open rather than guessing a target or retrying it. - -AX text uses short numeric IDs, such as `[37] TextField "Name"`. Use IDs from -the current observation for element actions. IDs can change when the app's -window or session changes. Disabled and static-text rows are observation-only. - -For token efficiency, the accessibility tree will be returned -as a diff when appropriate. Prefer this default diff output. A full state -replaces the previous state; a diff updates it; no-change preserves it. If you -need a full replacement, use `disableDiff: true` only when the previous state -is unavailable or no longer useful. Do not disregard the text and then assume -that a subsequent diff will reproduce the information you skipped. - -Returned text defaults to at most 12,000 characters. Set `maxTextChars` (minimum 512) to adjust the limit. A truncation notice means some captured rows were -omitted; request `app.getState({ disableDiff: true, maxTextChars: 24000 })` when -you need more full text. An omitted row does not prove an element is absent. -Traversal-limited captures cover only the captured nodes: identical captures -can return no-change, while changes return full captured state. Use current -captured IDs; after a read failure, use only IDs from the latest observation. - -### 2. Actions using app - -After performing one or more UI actions, call `app.getState()` before deciding -what to do next. Batch actions whose target remains the same, then print only -the state needed for the next decision: - -```js -await app.click(37); -await app.hotkey([platform === 'macos' ? 'super' : 'ctrl', 'a']); -await app.typeText('hello'); -await app.pressKey('Return'); -nodeRepl.write((await app.getState()).text); -``` - -Use the actual ID from your observation; `37` is only an example. - -An observation is a decision boundary. When the current state already identifies -the controls and the next actions are known, combine those actions and saving -in the same call. Read state after the batch. End the batch at a new dialog, -menu, changed target or uncertain result; use that state before choosing the -next action. Do not split a known sequence merely to put each action in its own -call. - -- Prefer element IDs to coordinates. `setValue(id, value)` changes a writable control, and `performSecondaryAction(id, action)` invokes a secondary action listed for that element. Use an observed action name rather than guessing. -- When an action opens or closes a dialog, sheet or menu, end the batch and call `app.getState()` to read the new window and IDs before continuing. -- `No open application window.` means the app is still running without a document window. If closing it completed the task, finish instead of retrying actions; otherwise open the intended file or window first. -- An action error can occur after the UI already changed. Read state before deciding whether to retry. Partial, unconfirmed or cancelled actions must not be blindly repeated. -- Coordinate actions use pixels in this app's current screenshot, with `(0, 0)` at its top-left. Every App observation refreshes that frame internally. Request `includeScreenshot: true` when you need to inspect the image, especially after a window change. Do not infer coordinates from another window or desktop screenshot. -- `pressKey` sends one key, optionally with modifiers. `hotkey` sends a combination such as `['super', 's']`. Use the connected platform's appropriate shortcut: macOS generally uses `super`; Linux/Windows generally use `ctrl`. -- App input manages any required activation internally and restores the previous focus, unless the user has moved it elsewhere. If the platform cannot confirm the target or restore focus, the action reports an error. There is no delivery-mode choice and no automatic replay after an uncertain result. -- On Linux, some compositor sessions cannot confirm an exact App target. An `app_window_unavailable` result can mean the session does not support this workflow; ask the user to use a supported desktop session instead of retrying input. -- Literal `\n` or `\r` in `typeText` sends Return. In a composer or form this may submit rather than insert a newline. -- If AX is incomplete or does not explain the interface, request a screenshot and inspect it. Only currently captured actionable IDs can be used for element actions. - -## Paste and select text - -These App methods are available on macOS only. Read current state first and use -an observed element ID for selection. - -```js -await app.selectText(37, 'draft', { prefix: 'Status: ' }); -nodeRepl.write((await app.getState()).text); -``` - -Use a real ID and text from your observation; the example assumes that the -selected element contains exactly one matching `draft` immediately after -`Status: `. - -After confirming the selection, replace it with plain text: - -```js -await app.paste('ready'); -nodeRepl.write((await app.getState()).text); -``` - -- `selectText(element, text)` selects one exact, case-sensitive match. Optional - `prefix` and `suffix` must be immediately adjacent to that match. Missing or - ambiguous matches fail; provide enough observed context to identify one. -- `selection` defaults to `'text'`. Use `'cursor_before'` or `'cursor_after'` to - place the insertion point at a match boundary instead of selecting the text. - The element must support a writable text selection; static labels may not. -- `paste(text)` defaults to plain text. Use `{ format: 'md' }` for Markdown or - `{ format: 'html' }` for HTML; the receiving app determines which supplied - format it accepts. Use plain text when markup should remain literal. -- Paste uses the clipboard temporarily. It restores the previous clipboard only - while it still owns that transaction; a newer external clipboard change is - preserved. -- Both methods accept an optional `signal`. A completed dispatch, error or - cancellation does not by itself prove what changed. Observe state before - deciding whether to retry; do not blindly repeat unconfirmed text operations. - -## Reading screenshots - -Every App observation captures the current screenshot internally, independent -of whether AX returns full state, a diff or no-change. The default return omits -the image. `includeScreenshot: true` exposes it when visual inspection is needed. -Prefer the text-only default when it identifies the controls and confirms the -requested change. Request an image to resolve missing or ambiguous information, -choose coordinates or verify an appearance that AX does not describe. - -```js -var state = await app.getState({ includeScreenshot: true }); -nodeRepl.write(state.text); -for (const image of state.screenshot?.images ?? []) { - await nodeRepl.emitImage(`data:${image.mimeType};base64,${image.dataBase64}`); -} -``` - -Include connection cleanup at the end of the call that emits the final -verification. Inspect that result before reporting success; reconnect and -continue if it reveals unfinished work. A separate cleanup-only model turn is -unnecessary: - -```js -await computer.close(); -globalThis.computer = undefined; -``` - -Reset the Node REPL only when no other persistent state is needed. diff --git a/packages/core/src/skills/bundled/computer-use/SKILL.test.ts b/packages/core/src/skills/bundled/computer-use/SKILL.test.ts deleted file mode 100644 index 7d4caffe39e..00000000000 --- a/packages/core/src/skills/bundled/computer-use/SKILL.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -/** - * @license - * Copyright 2026 Qwen - * SPDX-License-Identifier: Apache-2.0 - */ - -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { runInNewContext } from 'node:vm'; -import { describe, expect, it, vi } from 'vitest'; -import { parseSkillContent } from '../../skill-load.js'; - -function loadComputerUseSkill() { - const skillPath = path.join( - path.dirname(fileURLToPath(import.meta.url)), - 'SKILL.md', - ); - const config = parseSkillContent( - fs.readFileSync(skillPath, 'utf-8'), - skillPath, - ); - return { config, body: config.body }; -} - -describe('bundled computer-use skill', () => { - it.each([true, false])( - 'runs the forwarding example with desktop relay available=%s', - async (desktopAvailable) => { - const { body } = loadComputerUseSkill(); - const example = body.match(/```js\n([\s\S]*?)\n```/)?.[1]; - expect(example).toBeDefined(); - const screenshot = { - type: 'image', - data: 'screenshot', - mimeType: 'image/png', - }; - const result = { - content: [{ type: 'text', text: 'macos' }, screenshot], - }; - const desktop = vi.fn().mockResolvedValue(result); - const regular = vi.fn().mockResolvedValue(result); - const text = vi.fn(); - const image = vi.fn(); - // Mirror the code-mode host (host.ts): a Proxy that throws on an - // unknown key, so an unbound desktop tool must be probed with `in`. - const toolTarget = Object.assign(Object.create(null), { - mcp__node_repl__node_repl: regular, - ...(desktopAvailable - ? { mcp__desktop_node_repl__node_repl: desktop } - : {}), - }) as Record; - const tools = new Proxy(toolTarget, { - get(target, property) { - if (typeof property === 'string' && !(property in target)) { - throw new Error( - `Unknown or unavailable code mode tool: ${property}`, - ); - } - return Reflect.get(target, property) as unknown; - }, - }); - await runInNewContext(`(async () => {${example}})()`, { - tools, - ALL_TOOLS: desktopAvailable - ? [{ name: 'mcp__desktop_node_repl__node_repl' }] - : [], - code: 'return platform', - text, - image, - }); - expect(desktopAvailable ? desktop : regular).toHaveBeenCalledWith({ - code: 'return platform', - }); - expect(desktopAvailable ? regular : desktop).not.toHaveBeenCalled(); - expect(text).toHaveBeenCalledWith('macos'); - expect(image).toHaveBeenCalledWith(screenshot); - }, - ); - - it('loads a self-contained App workflow for every connected platform', () => { - const { config, body } = loadComputerUseSkill(); - expect(config.name).toBe('computer-use'); - expect(config.allowedTools).toBeUndefined(); - expect(body).toContain('`desktop-node-repl` MCP server'); - expect(body).toContain('mcp__desktop_node_repl__node_repl'); - expect(body).toContain('skip the installation commands below'); - expect(body).toContain('ComputerUse.create()'); - expect(body).toContain('await computer.getPlatform()'); - expect(body).toContain('computer.getApp('); - expect(body).toContain('app.getState('); - expect(body).toContain('app.click(37)'); - expect(body).not.toMatch( - /references\/|computer\.observeWindow\(|computer\.listWindows\(|elementToken|windowId/, - ); - }); - - it('preserves batching, incremental observation and safe refresh guidance', () => { - const { body } = loadComputerUseSkill(); - expect(body).toMatch(/After performing one or more UI actions/); - expect(body).toMatch(/Batch actions whose target remains the same/); - expect(body).toContain('Prefer this default diff output'); - expect(body).toContain('disableDiff: true'); - expect(body).toMatch(/window or session changes/); - expect(body).toContain('maxTextChars?: number'); - expect(body).toContain('12,000 characters'); - expect(body).toContain('Only currently captured actionable IDs'); - expect(body).toMatch( - /Partial, unconfirmed or cancelled actions must not be blindly repeated/, - ); - expect(body).not.toMatch( - /RecreationBench|benchmark|evaluator|score|failure count/i, - ); - }); - - it('requests screenshots separately and keeps the persistent REPL lifecycle', () => { - const { body } = loadComputerUseSkill(); - const screenshotSection = body.split('## Reading screenshots')[1]; - expect(screenshotSection).toContain('includeScreenshot: true'); - expect(screenshotSection).not.toContain('disableDiff'); - expect(screenshotSection).toContain('image.dataBase64'); - expect(screenshotSection).toContain('nodeRepl.write(state.text)'); - expect(body).toContain('await computer.close()'); - expect(body).toContain( - 'Reset the Node REPL only when no other persistent state is needed.', - ); - }); - - it('documents the macOS text methods and their uncertainty boundaries', () => { - const { body } = loadComputerUseSkill(); - expect(body).toContain('app.selectText(37,'); - expect(body).toContain("await app.paste('ready')"); - expect(body).toContain("format?: 'text' | 'md' | 'html'"); - expect(body).toContain( - "selection?: 'text' | 'cursor_before' | 'cursor_after'", - ); - expect(body).toContain('immediately adjacent'); - expect(body).toContain('Missing or'); - expect(body).toContain('ambiguous matches fail'); - expect(body).toContain('Observe state before'); - expect(body).toContain('newer external clipboard change'); - }); -}); diff --git a/packages/core/src/skills/bundled/computer-use/agents/openai.yaml b/packages/core/src/skills/bundled/computer-use/agents/openai.yaml deleted file mode 100644 index 71804ab3555..00000000000 --- a/packages/core/src/skills/bundled/computer-use/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: 'Computer Use' - short_description: 'Control desktop apps through the Qwen CUA SDK' - default_prompt: 'Use $computer-use to inspect and operate the requested desktop application safely.' diff --git a/packages/cua-driver/.vendored-patches.md b/packages/cua-driver/.vendored-patches.md index 021496b9798..3e8ba035444 100644 --- a/packages/cua-driver/.vendored-patches.md +++ b/packages/cua-driver/.vendored-patches.md @@ -34,3 +34,27 @@ already in the upstream delta, so: If `` is still *before* a PR merges, that row stays; the cherry-pick keeps carrying the fix. + + +## Qwen 0.26.0 platform ports + +The base remains `cua-driver-rs-v0.20.0`; this is a Qwen release version, not a new wholesale upstream snapshot. Selected MIT driver changes from upstream 0.35.4 were adapted to this fork. Native Wayland was removed. No upstream bench code is included. + +The complete per-PR port ledger, implementation differences and verification limits are in [the platform port record](../../docs/design/cua-upgrade-platform-ports.md). macOS ports cover #4943, #4955, #4976, #5051, #5057, #5091 and #5129. Windows covers #4500, #4901, #3671 and #4774. Linux covers #3882, #3761, #3762, #3864, #3618, #4248, #4188, #5139, #5158, #3601, #2888, #5070, #4903, #4984, #5074, #4375, #4292, #3758, #4281, #3814, #4529, #4715, #3658 and #3778. + +The default 19-operation Computer Use contract, driver-owned observations/dispatch, Node REPL runtime documentation, desktop key ownership and browser-use wrapper are Qwen integration changes. The legacy native tool surface remains a host opt-in. + +### P7 follow-up ports + +The native audit also added scoped adaptations for omitted-window capture and AX identity (#4996, #4068, #4085, #4509), Chromium AX readiness (#3783), AppKit launch-window recovery (#4309), ordinary macOS AX walk budgets (#3882), native Electron background scrolling (#4861, #5079), recording capture budgets (#4215), immutable snapshot identity (#3616, #3873), implicit idle-session recovery (#4283), macOS release replacement recovery (#3297), Windows installer ancestor protection (#4398), and Windows autostart error handling and isolated installation (#3180, #4284). These are adapted implementations, not a claim that each entire upstream commit was copied or that all native acceptance has passed. The macOS installer preserves authorization records; upstream automatic TCC resets are not included. + +The [platform port record](../../docs/design/cua-upgrade-platform-ports.md) describes the retained fork contracts and additional ports. The [acceptance record](../../docs/design/cua-upgrade-acceptance-2026-10.md) separates controlled tests, native execution and outstanding validation. The upstream base tag stays unchanged until a complete snapshot synchronization occurs. + +### Additional Qwen acceptance repairs + +These are fork repairs found while testing the adapted implementation, rather than additional whole upstream ports: + +- Preserve a normal uncertain acknowledgement after an attempted AX semantic action; never replay an AX error through a pointer actuator. Apple Events save and Finder rename carry their actual background transport and independent readback evidence. +- Resolve exact native macOS window identity before keyboard delivery, establish its AppKit key-window state, and omit the separate titlebar activation click. The native focus records themselves produce internal NaN-location mouse events; they are distinct from the requested pointer action. This does not claim that pointer or menu window covering has been eliminated. +- Use byte lengths for AT-SPI text insertion and preserve character-based caret offsets. Keep MPX mappings and input ownership until native completion, including the uncertain timeout path; reject unsupported WM delivery before dispatch. +- Limit an isolated Windows uninstall to its selected runtime and owned task actions. Unknown or mixed non-Exec task actions preserve the shared task; legacy default-install cleanup retains its existing scope. diff --git a/packages/cua-driver/README.md b/packages/cua-driver/README.md index 99c5af1f17d..79ce3eaea9c 100644 --- a/packages/cua-driver/README.md +++ b/packages/cua-driver/README.md @@ -11,21 +11,41 @@ documented in [`.vendored-patches.md`](.vendored-patches.md) and ## Install +The commands below target release **0.26.0** and require its GitHub Release +(`cua-driver-rs-v0.26.0`) to be published. The native driver does not require +Node.js or npm. Install the MCP client you want to use separately. + macOS and Linux: ```bash -CUA_DRIVER_RS_VERSION=0.20.11 \ +CUA_DRIVER_RS_VERSION=0.26.0 \ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.sh)" ``` Windows PowerShell: ```powershell -$env:CUA_DRIVER_RS_VERSION = "0.20.11" +$env:CUA_DRIVER_RS_VERSION = "0.26.0" irm https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.ps1 | iex ``` -Expected: qwen-cua-driver 0.20.11. +The default executable is `~/.local/bin/qwen-cua-driver` on macOS/Linux and +`%LOCALAPPDATA%\Programs\Qwen\qwen-cua-driver\bin\qwen-cua-driver.exe` on +Windows. Check the installation: + +```bash +# macOS / Linux +"$HOME/.local/bin/qwen-cua-driver" --version +``` + +```powershell +# Windows PowerShell +$driver = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver\bin\qwen-cua-driver.exe" +& $driver --version +``` + +Expected: `qwen-cua-driver 0.26.0`. If you chose a custom install directory, +use its absolute executable path in the client commands below. The released product uses Qwen-owned identities throughout: @@ -44,20 +64,99 @@ that state directory. ## Agent integration -Run the MCP server directly: +Choose an interface for your agent: + +| Interface | Install and configure | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Direct native MCP tools | Install this driver, then follow the client instructions below. | +| Persistent JavaScript with preloaded `cua` (Node REPL + SDK) | Follow the [Node REPL README](../node-repl/README.md#usage). It installs the matching SDK dependencies automatically and does not require this standalone driver installation. | + +Both interfaces work with Qwen Code, Claude Code, and Codex over stdio MCP. +Claude Code and Codex can use them independently of Qwen Code. The version +`0.26.0` here belongs to the CUA components; it does not require a Qwen Code +CLI or Web Shell release with that version. + +### Qwen Code + +On macOS/Linux, register the installed executable at user scope: ```bash -qwen-cua-driver mcp +qwen mcp add --scope user cua-driver "$HOME/.local/bin/qwen-cua-driver" mcp ``` -Register it with Qwen Code: +This writes `mcpServers.cua-driver` in `~/.qwen/settings.json`. Use +`--scope project` for the current project's `.qwen/settings.json` instead. + +### Claude Code + +On macOS/Linux: ```bash -qwen mcp add cua-driver qwen-cua-driver mcp +claude mcp add --scope user --transport stdio cua-driver -- "$HOME/.local/bin/qwen-cua-driver" mcp ``` -Other MCP clients can use the same executable and arguments. Shell-oriented -automation can call tools through `qwen-cua-driver call`. +User scope is stored in `~/.claude.json`. Use `--scope project` to write the +current project's `.mcp.json` instead. See the +[Claude Code MCP documentation](https://code.claude.com/docs/en/mcp). + +### Codex + +On macOS/Linux: + +```bash +codex mcp add cua-driver -- "$HOME/.local/bin/qwen-cua-driver" mcp +``` + +Alternatively, add this table to `~/.codex/config.toml`, replacing the path +with your actual installed executable. TOML command strings are literal paths; +do not use `$HOME` or `~` in place of the absolute path. + +```toml +[mcp_servers.cua-driver] +command = "/absolute/path/to/.local/bin/qwen-cua-driver" +args = ["mcp"] +``` + +See the [official OpenAI MCP documentation](https://learn.chatgpt.com/docs/extend/mcp). + +### Windows client commands + +In PowerShell, use the installed executable's absolute path. Run the command +for your chosen client: + +```powershell +$driver = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver\bin\qwen-cua-driver.exe" + +# Qwen Code +qwen mcp add --scope user cua-driver $driver mcp + +# Claude Code +claude mcp add --scope user --transport stdio cua-driver -- $driver mcp + +# Codex +codex mcp add cua-driver -- $driver mcp +``` + +### Permissions and connection check + +Restart the client after registration. Check its server list with +`qwen mcp list`, `claude mcp list`, or `codex mcp list`, and use `/mcp` inside +the session to inspect the connection and available tools. The default native +MCP interface exposes 19 Computer Use tools; advanced tools require the +explicit `CUA_DRIVER_EXTRA_TOOLS=1` opt-in. + +On macOS, grant Accessibility and Screen Recording to +`/Applications/QwenCuaDriver.app`; normal `mcp` uses its app-owned daemon. +On Windows, run the installer in the logged-in desktop user's session and +allow its UAC prompt. On Linux, use an X11 desktop (including X11 apps under +XWayland); native Wayland apps are unsupported. Launch the client with that +desktop session's environment. Codex needs the matching environment allowlist +shown in [Linux desktop sessions](../node-repl/README.md#linux-desktop-sessions), +applied to its `mcp_servers.cua-driver` table. + +Clients start `qwen-cua-driver mcp` themselves; a separately started MCP +process is unnecessary. Shell-oriented automation can instead call tools +through `qwen-cua-driver call`. ## 0.20 runtime and SDK surface @@ -156,15 +255,15 @@ release workflow. ## Repository layout -| Path | Purpose | -| --- | --- | -| `rust/` | Cargo workspace for the CLI, daemon, SDK, platform crates, and tests | -| `python/` | Python SDK and bundled-runtime wrapper | -| `typescript/` | TypeScript SDK and native runtime loader | -| `contract/` | Generated portable contract and fixtures | -| `tests/fixtures/` | Cross-platform GUI harness applications | -| `scripts/` | Release/local install, uninstall, generation, and sync helpers | -| `docs/` | Package-local architecture and behavior notes | +| Path | Purpose | +| ----------------- | -------------------------------------------------------------------- | +| `rust/` | Cargo workspace for the CLI, daemon, SDK, platform crates, and tests | +| `python/` | Python SDK and bundled-runtime wrapper | +| `typescript/` | TypeScript SDK and native runtime loader | +| `contract/` | Generated portable contract and fixtures | +| `tests/fixtures/` | Cross-platform GUI harness applications | +| `scripts/` | Release/local install, uninstall, generation, and sync helpers | +| `docs/` | Package-local architecture and behavior notes | Start with [`rust/README.md`](rust/README.md), [`docs/test-matrix.md`](docs/test-matrix.md), and @@ -181,5 +280,4 @@ cargo test -p cua-driver-core ``` Generated contract and language bindings must be checked with the package-local -scripts before release. Signed/notarized macOS, Windows UIAccess, Linux X11 and -Wayland, and real MCP/model verification remain platform release gates. +scripts before release. Signed/notarized macOS, Windows UIAccess, Linux X11, and real MCP/model verification remain platform release gates. diff --git a/packages/cua-driver/contract/manifest.json b/packages/cua-driver/contract/manifest.json index ad2bea6cf2d..b09ceef5d28 100644 --- a/packages/cua-driver/contract/manifest.json +++ b/packages/cua-driver/contract/manifest.json @@ -47,66 +47,44 @@ "type": "integer" }, "scope": { - "const": "desktop", - "description": "Deprecated flat desktop target retained for wire compatibility." + "description": "Deprecated flat desktop target retained for wire compatibility.", + "enum": [ + "desktop" + ], + "type": "string" }, "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", "type": "string" }, "target": { - "anyOf": [ - { - "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.", - "oneOf": [ - { - "additionalProperties": true, - "properties": { - "kind": { - "const": "window", - "type": "string" - }, - "pid": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "format": "uint64", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind", - "pid", - "window_id" - ], - "type": "object" - }, - { - "additionalProperties": true, - "properties": { - "display_id": { - "type": "string" - }, - "kind": { - "const": "desktop", - "type": "string" - } - }, - "required": [ - "kind", - "display_id" - ], - "type": "object" - } - ] + "additionalProperties": true, + "description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.", + "properties": { + "display_id": { + "description": "Use primary for the primary display.", + "type": "string" }, - { - "type": "null" + "kind": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" } - ] + }, + "required": [ + "kind" + ], + "type": "object" }, "x": { "type": "number" @@ -128,7 +106,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -443,8 +420,11 @@ "type": "array" }, "scope": { - "const": "desktop", - "description": "Deprecated flat desktop target retained for wire compatibility." + "description": "Deprecated flat desktop target retained for wire compatibility.", + "enum": [ + "desktop" + ], + "type": "string" }, "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", @@ -456,58 +436,33 @@ "type": "integer" }, "target": { - "anyOf": [ - { - "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.", - "oneOf": [ - { - "additionalProperties": true, - "properties": { - "kind": { - "const": "window", - "type": "string" - }, - "pid": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "format": "uint64", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind", - "pid", - "window_id" - ], - "type": "object" - }, - { - "additionalProperties": true, - "properties": { - "display_id": { - "type": "string" - }, - "kind": { - "const": "desktop", - "type": "string" - } - }, - "required": [ - "kind", - "display_id" - ], - "type": "object" - } - ] + "additionalProperties": true, + "description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.", + "properties": { + "display_id": { + "description": "Use primary for the primary display.", + "type": "string" }, - { - "type": "null" + "kind": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" } - ] + }, + "required": [ + "kind" + ], + "type": "object" }, "to_x": { "type": "number" @@ -531,7 +486,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -839,39 +793,30 @@ "motion": { "properties": { "arc_flow": { - "format": "double", "type": "number" }, "arc_size": { - "format": "double", "type": "number" }, "dwell_after_click_ms": { - "format": "double", "type": "number" }, "end_handle": { - "format": "double", "type": "number" }, "glide_duration_ms": { - "format": "double", "type": "number" }, "idle_hide_ms": { - "format": "double", "type": "number" }, "spring": { - "format": "double", "type": "number" }, "start_handle": { - "format": "double", "type": "number" }, "turn_radius": { - "format": "double", "type": "number" } }, @@ -891,11 +836,9 @@ "position": { "properties": { "x": { - "format": "double", "type": "number" }, "y": { - "format": "double", "type": "number" } }, @@ -950,7 +893,6 @@ "visual_state": { "properties": { "frame": { - "format": "uint64", "minimum": 0, "type": "integer" }, @@ -964,7 +906,6 @@ "type": "string" }, "preempted_count": { - "format": "uint64", "minimum": 0, "type": "integer" }, @@ -1264,12 +1205,10 @@ "type": "boolean" }, "expires_in_seconds": { - "format": "uint64", "minimum": 0, "type": "integer" }, "idle_seconds": { - "format": "uint64", "minimum": 0, "type": "integer" }, @@ -1448,10 +1387,7 @@ "properties": { "app_context": { "description": "Use the compact macOS application observation with contextual menus.", - "type": [ - "boolean", - "null" - ] + "type": "boolean" }, "include_screenshot": { "description": "Default true. Set false to skip the screenshot and return the tree only.", @@ -1491,7 +1427,8 @@ "type": "string" }, "version": { - "const": 1, + "maximum": 1, + "minimum": 1, "type": "integer" } }, @@ -1631,66 +1568,44 @@ "type": "array" }, "scope": { - "const": "desktop", - "description": "Deprecated flat desktop target retained for wire compatibility." + "description": "Deprecated flat desktop target retained for wire compatibility.", + "enum": [ + "desktop" + ], + "type": "string" }, "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", "type": "string" }, "target": { - "anyOf": [ - { - "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.", - "oneOf": [ - { - "additionalProperties": true, - "properties": { - "kind": { - "const": "window", - "type": "string" - }, - "pid": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "format": "uint64", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind", - "pid", - "window_id" - ], - "type": "object" - }, - { - "additionalProperties": true, - "properties": { - "display_id": { - "type": "string" - }, - "kind": { - "const": "desktop", - "type": "string" - } - }, - "required": [ - "kind", - "display_id" - ], - "type": "object" - } - ] + "additionalProperties": true, + "description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.", + "properties": { + "display_id": { + "description": "Use primary for the primary display.", + "type": "string" }, - { - "type": "null" + "kind": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" } - ] + }, + "required": [ + "kind" + ], + "type": "object" } }, "required": [ @@ -1705,7 +1620,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -1876,7 +1790,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -2014,12 +1927,8 @@ }, "limit": { "description": "Maximum number of content-free summaries to return (default 50, max\n100). Ordinary agent transports are scoped to their own lease.", - "format": "uint32", "minimum": 0, - "type": [ - "integer", - "null" - ] + "type": "integer" } }, "required": [], @@ -2055,12 +1964,10 @@ "type": "boolean" }, "expires_in_seconds": { - "format": "uint64", "minimum": 0, "type": "integer" }, "idle_seconds": { - "format": "uint64", "minimum": 0, "type": "integer" }, @@ -2146,67 +2053,44 @@ "additionalProperties": false, "properties": { "scope": { - "const": "desktop", - "description": "Deprecated flat desktop target retained for wire compatibility." + "description": "Deprecated flat desktop target retained for wire compatibility.", + "enum": [ + "desktop" + ], + "type": "string" }, "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", "type": "string" }, "target": { - "anyOf": [ - { - "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.", - "oneOf": [ - { - "additionalProperties": true, - "properties": { - "kind": { - "const": "window", - "type": "string" - }, - "pid": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "format": "uint64", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind", - "pid", - "window_id" - ], - "type": "object" - }, - { - "additionalProperties": true, - "properties": { - "display_id": { - "type": "string" - }, - "kind": { - "const": "desktop", - "type": "string" - } - }, - "required": [ - "kind", - "display_id" - ], - "type": "object" - } - ] + "additionalProperties": true, + "description": "Preferred per-call target. New callers should set this field.", + "properties": { + "display_id": { + "description": "Use primary for the primary display.", + "type": "string" }, - { - "type": "null" + "kind": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" } + }, + "required": [ + "kind" ], - "description": "Preferred per-call target. New callers should set this field." + "type": "object" }, "x": { "type": "number" @@ -2228,7 +2112,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -2361,11 +2244,7 @@ "additionalProperties": false, "properties": { "app_context": { - "default": null, - "type": [ - "boolean", - "null" - ] + "type": "boolean" }, "format": { "default": "text", @@ -2402,7 +2281,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -2566,7 +2444,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -2709,66 +2586,44 @@ "type": "array" }, "scope": { - "const": "desktop", - "description": "Deprecated flat desktop target retained for wire compatibility." + "description": "Deprecated flat desktop target retained for wire compatibility.", + "enum": [ + "desktop" + ], + "type": "string" }, "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", "type": "string" }, "target": { - "anyOf": [ - { - "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.", - "oneOf": [ - { - "additionalProperties": true, - "properties": { - "kind": { - "const": "window", - "type": "string" - }, - "pid": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "format": "uint64", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind", - "pid", - "window_id" - ], - "type": "object" - }, - { - "additionalProperties": true, - "properties": { - "display_id": { - "type": "string" - }, - "kind": { - "const": "desktop", - "type": "string" - } - }, - "required": [ - "kind", - "display_id" - ], - "type": "object" - } - ] + "additionalProperties": true, + "description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.", + "properties": { + "display_id": { + "description": "Use primary for the primary display.", + "type": "string" }, - { - "type": "null" + "kind": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" } - ] + }, + "required": [ + "kind" + ], + "type": "object" } }, "required": [ @@ -2783,7 +2638,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -2938,66 +2792,44 @@ "type": "string" }, "scope": { - "const": "desktop", - "description": "Deprecated flat desktop target retained for wire compatibility." + "description": "Deprecated flat desktop target retained for wire compatibility.", + "enum": [ + "desktop" + ], + "type": "string" }, "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", "type": "string" }, "target": { - "anyOf": [ - { - "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.", - "oneOf": [ - { - "additionalProperties": true, - "properties": { - "kind": { - "const": "window", - "type": "string" - }, - "pid": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "format": "uint64", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind", - "pid", - "window_id" - ], - "type": "object" - }, - { - "additionalProperties": true, - "properties": { - "display_id": { - "type": "string" - }, - "kind": { - "const": "desktop", - "type": "string" - } - }, - "required": [ - "kind", - "display_id" - ], - "type": "object" - } - ] + "additionalProperties": true, + "description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.", + "properties": { + "display_id": { + "description": "Use primary for the primary display.", + "type": "string" }, - { - "type": "null" + "kind": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" } - ] + }, + "required": [ + "kind" + ], + "type": "object" }, "x": { "type": "number" @@ -3020,7 +2852,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -3161,10 +2992,7 @@ "type": "integer" }, "prefix": { - "type": [ - "string", - "null" - ] + "type": "string" }, "selection": { "default": "text", @@ -3176,10 +3004,7 @@ "type": "string" }, "suffix": { - "type": [ - "string", - "null" - ] + "type": "string" }, "text": { "type": "string" @@ -3204,7 +3029,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -3385,70 +3209,34 @@ "additionalProperties": false, "properties": { "arc_flow": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "arc_size": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "dwell_after_click_ms": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "end_handle": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "glide_duration_ms": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "idle_hide_ms": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "session": { "type": "string" }, "spring": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "start_handle": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" }, "turn_radius": { - "format": "double", - "type": [ - "number", - "null" - ] + "type": "number" } }, "required": [ @@ -3462,39 +3250,30 @@ "motion": { "properties": { "arc_flow": { - "format": "double", "type": "number" }, "arc_size": { - "format": "double", "type": "number" }, "dwell_after_click_ms": { - "format": "double", "type": "number" }, "end_handle": { - "format": "double", "type": "number" }, "glide_duration_ms": { - "format": "double", "type": "number" }, "idle_hide_ms": { - "format": "double", "type": "number" }, "spring": { - "format": "double", "type": "number" }, "start_handle": { - "format": "double", "type": "number" }, "turn_radius": { - "format": "double", "type": "number" } }, @@ -3676,7 +3455,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -3852,7 +3630,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -3995,6 +3772,7 @@ "type": "string" }, "cursor_theme": { + "additionalProperties": true, "description": "Optional initial cursor theme. The host applies it before the cursor is\nfirst made visible, avoiding a flash of the default theme.", "properties": { "reduced_motion": { @@ -4013,10 +3791,7 @@ "required": [ "theme_id" ], - "type": [ - "object", - "null" - ] + "type": "object" }, "session": { "description": "Optional stable public label for this run (e.g. \"research-run-1\").\nWhen omitted, the authenticated transport lease's implicit session is\ncreated or returned.", @@ -4124,66 +3899,44 @@ "additionalProperties": false, "properties": { "scope": { - "const": "desktop", - "description": "Deprecated flat desktop target retained for wire compatibility." + "description": "Deprecated flat desktop target retained for wire compatibility.", + "enum": [ + "desktop" + ], + "type": "string" }, "session": { "description": "For multi-call work, prefer a short public session label and repeat it on every call that\naccepts it. Omit it to use the authenticated transport's implicit lifecycle session.", "type": "string" }, "target": { - "anyOf": [ - { - "description": "Exact capture/input target selected independently for each action.\n\n`display_id=\"primary\"` is the portable desktop target in this release.\nPlatforms that cannot address another display reject it explicitly rather\nthan silently changing coordinate spaces.", - "oneOf": [ - { - "additionalProperties": true, - "properties": { - "kind": { - "const": "window", - "type": "string" - }, - "pid": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "window_id": { - "format": "uint64", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "kind", - "pid", - "window_id" - ], - "type": "object" - }, - { - "additionalProperties": true, - "properties": { - "display_id": { - "type": "string" - }, - "kind": { - "const": "desktop", - "type": "string" - } - }, - "required": [ - "kind", - "display_id" - ], - "type": "object" - } - ] + "additionalProperties": true, + "description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.", + "properties": { + "display_id": { + "description": "Use primary for the primary display.", + "type": "string" }, - { - "type": "null" + "kind": { + "enum": [ + "window", + "desktop" + ], + "type": "string" + }, + "pid": { + "minimum": 0, + "type": "integer" + }, + "window_id": { + "minimum": 0, + "type": "integer" } - ] + }, + "required": [ + "kind" + ], + "type": "object" }, "text": { "type": "string" @@ -4201,7 +3954,6 @@ "additionalProperties": false, "properties": { "delivered_count": { - "format": "uint32", "minimum": 0, "type": [ "integer", @@ -4342,23 +4094,14 @@ "additionalProperties": false, "properties": { "enabled": { - "type": [ - "boolean", - "null" - ] + "type": "boolean" }, "exists": { "description": "Assert that at least one trusted element matches the selector.\n\nElement walks are not yet exhaustive on every platform, so absence\ncannot be proven. `false` is rejected instead of returning an\nindefinitely-unknown predicate.", - "enum": [ - true - ], "type": "boolean" }, "selected": { - "type": [ - "boolean", - "null" - ] + "type": "boolean" }, "selector": { "additionalProperties": false, @@ -4376,19 +4119,13 @@ "type": "object" }, "value_equals": { - "type": [ - "string", - "null" - ] + "type": "string" } }, "required": [ "selector" ], - "type": [ - "object", - "null" - ] + "type": "object" }, "window": { "additionalProperties": false, @@ -4420,22 +4157,14 @@ "width", "height" ], - "type": [ - "object", - "null" - ] + "type": "object" }, "exists": { - "type": [ - "boolean", - "null" - ] + "type": "boolean" } }, - "type": [ - "object", - "null" - ] + "required": [], + "type": "object" } }, "required": [], @@ -4447,10 +4176,7 @@ }, "include_screenshot": { "description": "Return the final window screenshot as image content for a multimodal\ncaller. The driver does not interpret that image.", - "type": [ - "boolean", - "null" - ] + "type": "boolean" }, "pid": { "description": "Exact process whose window may be observed.", @@ -4491,7 +4217,6 @@ "additionalProperties": true, "properties": { "elapsed_ms": { - "format": "uint64", "minimum": 0, "type": "integer" }, @@ -4499,7 +4224,6 @@ "items": { "properties": { "index": { - "format": "uint64", "minimum": 0, "type": "integer" }, @@ -4552,7 +4276,6 @@ "type": "array" }, "samples": { - "format": "uint64", "minimum": 0, "type": "integer" }, diff --git a/packages/cua-driver/docs/0.13.2-overnight-stabilization-plan.md b/packages/cua-driver/docs/0.13.2-overnight-stabilization-plan.md index 6b2a655ea20..9407f09c1c1 100644 --- a/packages/cua-driver/docs/0.13.2-overnight-stabilization-plan.md +++ b/packages/cua-driver/docs/0.13.2-overnight-stabilization-plan.md @@ -1,5 +1,10 @@ # Cua Driver 0.13.2 Overnight Stabilization Plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Status:** Draft for review **Baseline:** Cua Driver 0.13.1 is already released **Source baseline:** `origin/main` at `1e77ab5536eb88ca4cb7d3b6cc1fb51eea03f0e4` diff --git a/packages/cua-driver/docs/action-support.md b/packages/cua-driver/docs/action-support.md index 526ea5401c6..985b5192d6b 100644 --- a/packages/cua-driver/docs/action-support.md +++ b/packages/cua-driver/docs/action-support.md @@ -21,7 +21,6 @@ when that is the background-safe route. | Windows/Win32 | `64e82449` | Run `29257963004`: 122/122 rows, 99 delivered and 23 exact refusals | | macOS/Quartz | `0b9b5d94` and `64e82449` | Logged-in local matrix: 138 deliveries and 6 exact refusals passed; one cursor-interrupted row passed alone on the final head, for effective 145/145 coverage | | Linux/X11 | `268a6ab1` | Run `29254936043`: 116/116 rows, 75 delivered and 41 exact refusals | -| Linux/Sway | `268a6ab1` and `64e82449` | Run `29255208927` passed native and capture 36/36; run `29257961614` passed shared 80/80, for effective 116/116 coverage | ## Shared web harnesses @@ -36,14 +35,10 @@ Foreground rows listed here are delivered for both AX and PX. | macOS | WKWebView | left/right/double click AX/PX, type text AX/PX, press key AX/PX, hotkey AX/PX, scroll AX/PX, child window AX/PX, editor save AX | drag PX: `background_unavailable` | | Linux X11 | Electron | Background AX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Background PX left click, AX/PX right and double click, PX child window and drag, AX/PX keyboard and scroll, and AX editor save return exact `background_unavailable` | | Linux X11 | Tauri | Background AX/PX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Other background pointer, keyboard, scroll, drag, and editor-save shapes return exact `background_unavailable` | -| Linux Sway | Electron | The full 40-cell catalog has accepted fixture-owned evidence; safe semantic and compositor routes deliver | Focus-bound and raw background shapes without a target-addressed route return their declared exact refusal | -| Linux Sway | Tauri | The full 40-cell WebKitGTK catalog has accepted fixture-owned evidence, including live compositor geometry and roleless WebProcess targeting | Focus-bound and raw background shapes without a target-addressed route return their declared exact refusal | All shared background rows require fixture state, focus, z-order, and no-leaked-input evidence. Windows, macOS, and X11 also require cursor -preservation. Wayland reports the cursor oracle as unsupported until the -environment can prove it; [issue #2194](https://github.com/trycua/cua/issues/2194) -tracks possible observer routes. +preservation. Foreground rows require fixture state and retain a video plus before/after turn evidence. @@ -69,16 +64,13 @@ integrity-level harness can exercise it. ## Native Linux -Linux is recorded per display server and compositor because Wayland protocols -are capabilities, not one uniform API. +Linux supports X11, including X11 applications under XWayland. Native Wayland +backends and their validation lanes have been removed. The exact-source runs +below are historical baselines; acceptance of the upgrade requires new evidence. | Environment | Proven contracts | Refusals and gaps | | --- | --- | --- | | X11/Openbox | Exact run `29254936043` passed all 116 declared outcomes: 75 deliveries and 41 exact refusals. GTK3 AT-SPI actions and values, foreground XTest pointer/keyboard routes, capture, desktop scope, and the complete Electron/Tauri catalog all reached fixture-owned oracles. | Xvfb does not prove the real-Xorg MPX/uinput background pointer route. Toolkits that reject XSendEvent retain exact refusals. | -| Sway/wlroots | Runs `29255208927` and `29257961614` passed the complete 116 outcomes: native and capture 36/36, then shared 80/80. The Electron, Tauri, GTK3, capture, and desktop-scope catalogs use live Sway identity/geometry and external fixture oracles. | Stock Wayland cannot target raw focus-bound input at an occluded surface. Those rows retain exact refusals; another wlroots compositor is not assumed equivalent until a material divergence is reported and tested. Exact cursor preservation remains unproven and is tracked in [issue #2194](https://github.com/trycua/cua/issues/2194). | -| GNOME/Mutter | A real GNOME 46 Wayland run passed the full GTK3 matrix, 31/31. The WinRects helper supplies stable window ids, frame and buffer geometry, stacking, verified activation, stage capture, and the compositor cursor. AT-SPI handles semantic actions; persistent portal/libei sessions deliver foreground PX click, right/double click, drag, scroll, type, key, and hotkey. Background rows either deliver through AT-SPI with focus and leak guards or return the declared exact refusal. | The helper requires installation plus one Shell-session restart. Without it, target-bound foreground input refuses. Portal video and a shared Electron/Tauri GNOME run remain open. | -| KDE/KWin | A Plasma 6 session reached GTK AT-SPI discovery, generic toplevel discovery, and portal-interface preflight. Portal input is compiled into release binaries. | No behavioral matrix is accepted. A target-addressable KWin activation adapter is not implemented, so foreground portal/libei input refuses instead of injecting into the wrong focused app. | -| Nested `cua-compositor` | The optional backend owns its nested session and is covered by typed shared, native, and capture catalogs. Run `29197643541` passed native GTK3 31/31; run `29197887387` passed capture/scope 5/5. Full shared run `29199596935` passed 26/36 Electron rows before the occluded-subtree and wheel-frame repairs. | The lane remains experimental. The accepted full shared run still has 10 failures; focused replacement runs reduce that set but do not promote the environment. Unicode text and a canonical parallel-drag row remain unproven. This private route is not evidence of a stock-Wayland capability. | PX background left-click rows may resolve the screen point to an actionable AT-SPI node. Such a pass proves the public PX-addressed behavior and its desktop diff --git a/packages/cua-driver/docs/browser-cross-platform-hardening-journal.md b/packages/cua-driver/docs/browser-cross-platform-hardening-journal.md index 76374fc5f2e..94e33d4f8a8 100644 --- a/packages/cua-driver/docs/browser-cross-platform-hardening-journal.md +++ b/packages/cua-driver/docs/browser-cross-platform-hardening-journal.md @@ -1,5 +1,10 @@ # Browser Cross-Platform Hardening Journal +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + This journal records implementation and validation evidence for the browser support hardening branch. It is not a public support contract. diff --git a/packages/cua-driver/docs/browser-existing-profile-attachment-plan.md b/packages/cua-driver/docs/browser-existing-profile-attachment-plan.md index 56b6561913f..765de802dcb 100644 --- a/packages/cua-driver/docs/browser-existing-profile-attachment-plan.md +++ b/packages/cua-driver/docs/browser-existing-profile-attachment-plan.md @@ -1,5 +1,10 @@ # Existing-Profile Browser Attachment and Reconnect Plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + ## Status - **State:** Implemented and accepted from exact source SHA `7fa6c80f` across macOS, Windows, Linux X11, and Linux Wayland on Sway diff --git a/packages/cua-driver/docs/browser-semantic-state-plan.md b/packages/cua-driver/docs/browser-semantic-state-plan.md index 7b8e39d8cff..2c771bf06f6 100644 --- a/packages/cua-driver/docs/browser-semantic-state-plan.md +++ b/packages/cua-driver/docs/browser-semantic-state-plan.md @@ -1,5 +1,10 @@ # Browser Semantic State Plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + ## Status - **State:** Implemented; cross-platform acceptance in progress diff --git a/packages/cua-driver/docs/browser-tool-implementation-journal.md b/packages/cua-driver/docs/browser-tool-implementation-journal.md index 836a118187b..408b15c6cc3 100644 --- a/packages/cua-driver/docs/browser-tool-implementation-journal.md +++ b/packages/cua-driver/docs/browser-tool-implementation-journal.md @@ -1,5 +1,10 @@ # Browser Tool Implementation Journal +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + This journal records implementation and validation evidence for the first-class browser tools. It intentionally omits machine identities, credentials, private URLs, profile paths, and raw DevTools endpoint identifiers. diff --git a/packages/cua-driver/docs/browser-tool-implementation-plan.md b/packages/cua-driver/docs/browser-tool-implementation-plan.md index 3e89fb46943..bd62963d816 100644 --- a/packages/cua-driver/docs/browser-tool-implementation-plan.md +++ b/packages/cua-driver/docs/browser-tool-implementation-plan.md @@ -1,5 +1,10 @@ # Cua Driver Browser Tool Implementation Plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + ## Status - **State:** Browser-tool v2 roadmap implemented locally; acceptance evidence recorded in the implementation journal diff --git a/packages/cua-driver/docs/cursor-modifier-badge-restructuring-plan.md b/packages/cua-driver/docs/cursor-modifier-badge-restructuring-plan.md index c5d82869681..1ea081fbcb3 100644 --- a/packages/cua-driver/docs/cursor-modifier-badge-restructuring-plan.md +++ b/packages/cua-driver/docs/cursor-modifier-badge-restructuring-plan.md @@ -1,5 +1,10 @@ # Cursor modifier badge restructuring plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Status:** Implemented in [PR #2677](https://github.com/trycua/cua/pull/2677); validation complete **Base:** `main` at `b249df857994ebaebf9f02248e8312b3bd2a5dd2` **Scope:** Move delivery and target modifiers from around the agent pointer into the session badge while preserving action animations beside the pointer. diff --git a/packages/cua-driver/docs/cursor-themes.md b/packages/cua-driver/docs/cursor-themes.md index c0c1ac78022..60b2a88a37e 100644 --- a/packages/cua-driver/docs/cursor-themes.md +++ b/packages/cua-driver/docs/cursor-themes.md @@ -24,11 +24,8 @@ fades over 400 milliseconds while the cursor stays available. Moving the human pointer over the synthetic cursor temporarily reveals the name again, and moving it away hides the name. Hiding and revealing the cursor also shows the badge again. Hover reveal uses -the native global-pointer APIs on macOS, Windows, and X11. Stock Wayland does -not expose another client's global pointer position, so native Wayland keeps -the timed reveal but cannot offer hover reveal without a compositor-owned -adapter. The click-through overlay never captures pointer input to fake this -capability. +the native global-pointer APIs on macOS, Windows, and X11. The click-through +overlay never captures pointer input to fake this capability. Delivery and target context appears as up to two chips inside the same badge. The delivery chip comes first and is filled; the target chip comes second and @@ -48,8 +45,7 @@ move the agent cursor to that target before dispatch. Actions without a spatial target, such as typing into the already focused desktop application, keep the cursor at its current location and only change the semantic animation. During a drag, the native gesture's endpoints, step count, and wall-clock -duration drive the same shared pointer-anchor transform on macOS, Windows, X11, -and supported Wayland input routes, so the synthetic cursor follows the held +duration drive the same shared pointer-anchor transform on macOS, Windows, and X11, so the synthetic cursor follows the held gesture instead of moving only after release. The cursor is a visual aid for people watching an agent. It is not a security diff --git a/packages/cua-driver/docs/driver-permission-modes-and-consent-plan.md b/packages/cua-driver/docs/driver-permission-modes-and-consent-plan.md index 1d3ccaa8835..82647400247 100644 --- a/packages/cua-driver/docs/driver-permission-modes-and-consent-plan.md +++ b/packages/cua-driver/docs/driver-permission-modes-and-consent-plan.md @@ -1,5 +1,10 @@ # Cua Driver Permission Modes, Capability Policy, and Human Consent Plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + ## Status - **State:** Reviewed draft / local; implementation plan, not yet a plan of record diff --git a/packages/cua-driver/docs/driver-permission-modes-implementation-journal.md b/packages/cua-driver/docs/driver-permission-modes-implementation-journal.md index ecf496dc4c0..e22d0f19398 100644 --- a/packages/cua-driver/docs/driver-permission-modes-implementation-journal.md +++ b/packages/cua-driver/docs/driver-permission-modes-implementation-journal.md @@ -1,5 +1,10 @@ # Driver Permission Modes Implementation Journal +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + > Contract update: the shipped canonical middle-mode name is `bounded`, with > `autonomous` retained as a compatibility alias. The interactive CLI danger > flag now selects `unrestricted` as well as acknowledging its risk; embedded diff --git a/packages/cua-driver/docs/e2e-convergence-journal.md b/packages/cua-driver/docs/e2e-convergence-journal.md index b98ecbc06a4..a43bde40b11 100644 --- a/packages/cua-driver/docs/e2e-convergence-journal.md +++ b/packages/cua-driver/docs/e2e-convergence-journal.md @@ -1,5 +1,10 @@ # E2E Convergence Journal +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Historical record.** Use `action-support.md` and the hosted platform support reference for current support claims. diff --git a/packages/cua-driver/docs/linux-desktop-validation.md b/packages/cua-driver/docs/linux-desktop-validation.md index 78c3586d60c..dfbefc2a389 100644 --- a/packages/cua-driver/docs/linux-desktop-validation.md +++ b/packages/cua-driver/docs/linux-desktop-validation.md @@ -1,6 +1,6 @@ # Linux representative desktop validation -Hosted CI owns the canonical Xvfb/Openbox and headless Sway environments. Some +Hosted CI owns the canonical Xvfb/Openbox environment. Some contracts require a real user desktop and therefore run only when a maintainer explicitly provisions the corresponding environment. @@ -8,10 +8,7 @@ explicitly provisions the corresponding environment. | Environment | Required evidence | Current status | Preflight | | --- | --- | --- | --- | -| GNOME/Mutter | WinRects geometry and activation, portal/libei input, portal recording, and shared renderer apps | Native GTK behavior, capture, and desktop scope are accepted; shared renderers and portal video remain open | Wayland user session, enabled WinRects helper, and portal grant | -| KDE/KWin | KWin-specific activation and portal behavior | Plasma 6 session startup, GTK AT-SPI discovery, and portal interfaces were observed; no behavioral matrix is accepted | Plasma/KWin 6 Wayland session; Plasma 5.27 is rejected | -| Real Xorg | MPX/uinput behavior that Xvfb cannot provide | Not yet validated | Non-Wayland Xorg session with `/dev/uinput` access | -| DRM/EGL renderer | Representative WebKitGTK/Tauri accessibility tree | Not available on the hosted Sway or representative desktop hosts used so far | A real `/dev/dri/renderD128`; software-only headless rendering is rejected | +| Real Xorg | MPX/uinput behavior that Xvfb cannot provide | Not yet validated | Xorg session with `/dev/uinput` access | ## Source ownership @@ -32,8 +29,6 @@ equivalent terminal inside that user's session. The wrapper rejects the wrong desktop generation before building fixtures. ```bash -scripts/ci/linux/run-rust-e2e-desktop.sh gnome -scripts/ci/linux/run-rust-e2e-desktop.sh kde scripts/ci/linux/run-rust-e2e-desktop.sh xorg ``` diff --git a/packages/cua-driver/docs/linux-support-completion-plan.md b/packages/cua-driver/docs/linux-support-completion-plan.md index bc6597baee7..b70f480bf6a 100644 --- a/packages/cua-driver/docs/linux-support-completion-plan.md +++ b/packages/cua-driver/docs/linux-support-completion-plan.md @@ -1,5 +1,10 @@ # Linux support completion plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Status:** Historical implementation plan. Current outcomes live in `action-support.md`; remaining work lives in the [public platform roadmap](https://cua.ai/docs/reference/cua-driver/platform-roadmap). diff --git a/packages/cua-driver/docs/permission-adapters-and-session-modes-plan.md b/packages/cua-driver/docs/permission-adapters-and-session-modes-plan.md index 7fa55b67930..edab74d5bc3 100644 --- a/packages/cua-driver/docs/permission-adapters-and-session-modes-plan.md +++ b/packages/cua-driver/docs/permission-adapters-and-session-modes-plan.md @@ -1,5 +1,10 @@ # Active Permission Adapters and Trusted Per-Session Modes +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Status:** Reviewed; implementation sequencing accepted **Issues:** #2385 and #2437 diff --git a/packages/cua-driver/docs/permission-simplification-execution-journal.md b/packages/cua-driver/docs/permission-simplification-execution-journal.md index 0cee9089dad..0212149fcbd 100644 --- a/packages/cua-driver/docs/permission-simplification-execution-journal.md +++ b/packages/cua-driver/docs/permission-simplification-execution-journal.md @@ -1,5 +1,10 @@ # Permission Simplification Execution Journal +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Status:** Active **Branch:** `codex/permission-simplification-0130` diff --git a/packages/cua-driver/docs/pr-2161-landing-plan.md b/packages/cua-driver/docs/pr-2161-landing-plan.md index f8bfb7b8f1a..2b4e9392f20 100644 --- a/packages/cua-driver/docs/pr-2161-landing-plan.md +++ b/packages/cua-driver/docs/pr-2161-landing-plan.md @@ -1,5 +1,10 @@ # PR #2161 Landing Plan - Second-Wave E2E Convergence +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Historical execution plan.** This file records how the original draft was split and landed. It is not the current test or platform support reference. diff --git a/packages/cua-driver/docs/sdk-rust-source-of-truth-and-uniffi-evaluation-plan.md b/packages/cua-driver/docs/sdk-rust-source-of-truth-and-uniffi-evaluation-plan.md index d276f6c36d2..6f86c6a01f2 100644 --- a/packages/cua-driver/docs/sdk-rust-source-of-truth-and-uniffi-evaluation-plan.md +++ b/packages/cua-driver/docs/sdk-rust-source-of-truth-and-uniffi-evaluation-plan.md @@ -1,5 +1,10 @@ # Cua Driver SDK Rust source-of-truth and UniFFI evaluation plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + Status: superseded by the final package-boundary decision in PR #2341 Applies to: PR #2341 and follow-up SDK/runtime work diff --git a/packages/cua-driver/docs/test-harness-convergence-plan.md b/packages/cua-driver/docs/test-harness-convergence-plan.md index d1e19ab694a..16e1d7cc193 100644 --- a/packages/cua-driver/docs/test-harness-convergence-plan.md +++ b/packages/cua-driver/docs/test-harness-convergence-plan.md @@ -1,5 +1,10 @@ # Rust E2E Harness Convergence Plan +> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland +> implementations, helpers, dependencies and validation lanes described below +> have been removed; these historical results do not describe current support. + + **Historical plan.** Use `test-harnesses-guide.md`, `test-matrix.md`, and `e2e-ci-reporting.md` for the implemented contributor workflow. diff --git a/packages/cua-driver/docs/test-harnesses-guide.md b/packages/cua-driver/docs/test-harnesses-guide.md index a9c2f48302f..af335565408 100644 --- a/packages/cua-driver/docs/test-harnesses-guide.md +++ b/packages/cua-driver/docs/test-harnesses-guide.md @@ -222,26 +222,13 @@ Runner: `scripts/ci/linux/run-rust-e2e.sh` | Shared app matrix | `cross_platform_behavior_test.rs` | Electron and Tauri | | Native controls | `harness_gtk3_test.rs` | Repo-local GTK3 app | | Capture contract | `capture_contract_test.rs` | Linux capture backend | -| Desktop scope | `desktop_scope_linux_test.rs` | X11/Wayland desktop scope | +| Desktop scope | `desktop_scope_linux_test.rs` | X11 desktop scope | -Linux has separate X11 and Wayland concerns. Nix supplies the reproducible -build and desktop environment, but the E2E test still needs an actual X11 or -Wayland session. Linux does not need GIF output; MP4, screenshots, accessibility -trees, trajectories, and logs are the useful evidence. - -Wayland results are compositor-specific. The hosted lane uses Sway to prove -wlroots protocols. GNOME requires the optional WinRects Shell helper for -authoritative frame and buffer geometry, observation, capture, and verified -target activation. A portal/libei grant persists until the user revokes it, so -subsequent driver processes do not reopen the consent dialog. -KDE requires a future target-addressable KWin adapter; portal availability by -itself is not evidence that input can be sent safely to a named window. -Standard Wayland does not expose the physical pointer position, so canonical -Wayland rows do not claim the real-cursor preservation oracle. Focus, full -occlusion, sentinel input isolation, liveness, and fixture-state oracles remain -mandatory. [Issue #2194](https://github.com/trycua/cua/issues/2194) tracks -compositor, portal/libei, sentinel, and capture-based ways to add a proven -cursor observer where the environment supports one. +Linux requires an authenticated X11 session and its accessibility session bus. +Xvfb/Openbox can cover capture, AT-SPI and foreground input. Real Xorg is +required to verify MPX/uinput. A DISPLAY with surrounding Wayland environment +variables exercises X11 selection only; it does not prove real XWayland. +Linux evidence includes MP4, screenshots, accessibility trees, trajectories and logs. ## AX, PX, and Delivery @@ -302,10 +289,7 @@ preflight deliberately sends input to the sentinel and deliberately raises the background target. The lane proceeds only if the leaked input and transient focus loss are observed, the sentinel is restored, and it once again fully occludes the target. Windows, macOS, and X11 require the sentinel's live focus -journal to report the loss. Wayland uses the compositor-backed native focus -observer because Electron/Ozone does not reliably emit a DOM `blur` event for -an external surface focus transition. The sentinel heartbeat and leaked-input -journal remain mandatory on Wayland. This positive control prevents a broken +journal to report the loss. This positive control prevents a broken guard from making every background row look green. A focus assertion can prove "no focus steal" while failing to prove that a diff --git a/packages/cua-driver/docs/test-matrix.md b/packages/cua-driver/docs/test-matrix.md index ebf883c6461..818200b0a1c 100644 --- a/packages/cua-driver/docs/test-matrix.md +++ b/packages/cua-driver/docs/test-matrix.md @@ -20,7 +20,7 @@ Every harness E2E result should identify these dimensions: | Dimension | Values | | --- | --- | | OS | `windows`, `macos`, `linux` | -| Window system | Win32/UIA, AppKit/AX, X11/AT-SPI, Wayland/AT-SPI, WebView/CDP | +| Window system | Win32/UIA, AppKit/AX, X11/AT-SPI, WebView/CDP | | Harness | Electron, Tauri, WPF, WinUI3, WebView2, AppKit, SwiftUI, WKWebView, GTK3 | | Action targeting | `ax`, `px`, `page`, `not_applicable` | | Delivery | `background`, `foreground`, `N/A` | @@ -140,13 +140,12 @@ transient-panel AX discovery gap. | Harness | Source test | Window system | Coverage | | --- | --- | --- | --- | -| Electron | `cross_platform_behavior_test.rs` | X11 and hosted Sway | Shared web action matrix | -| Tauri | `cross_platform_behavior_test.rs` | X11 and hosted Sway | Shared web action matrix | -| GTK3 | `harness_gtk3_test.rs` | X11/AT-SPI and Wayland/AT-SPI where configured | Native GTK controls and input | -| Desktop target | `desktop_scope_linux_test.rs` | X11/Wayland | Per-call window/desktop modality, screen-absolute action delivery, and strict rejection | +| Electron | `cross_platform_behavior_test.rs` | X11 | Shared web action matrix | +| Tauri | `cross_platform_behavior_test.rs` | X11 | Shared web action matrix | +| GTK3 | `harness_gtk3_test.rs` | X11/AT-SPI | Native GTK controls and input | +| Desktop target | `desktop_scope_linux_test.rs` | X11 | Per-call window/desktop modality, screen-absolute action delivery, and strict rejection | -Nix provides the Linux build and desktop environment. X11 and Wayland are -separate matrix dimensions because their capture and input contracts differ. +Linux requires an authenticated X11 display and its AT-SPI session bus. Linux runs do not produce GIF output. Every canonical GUI cell retains an MP4 and trajectory alongside screenshots, AX trees, structured results, and driver logs for E2E evidence. @@ -186,9 +185,7 @@ allowed, and the desktop-side-effect oracles pass. | Windows unit/compile | `windows-latest` | Package-scoped `cargo test --all-targets --no-run --locked` | | Windows Harness E2E | Active Windows user session | `scripts/ci/windows/run-rust-e2e.ps1 -RequireGui` | | Linux X11 Harness E2E | Nix X11 session | `scripts/ci/linux/run-rust-e2e.sh` | -| Linux Sway Harness E2E | Controlled wlroots session | `scripts/ci/linux/run-rust-e2e-wayland.sh` | -| Linux nested-compositor E2E | Controlled experimental session | `scripts/ci/linux/run-rust-e2e-inject.sh` | -| Linux representative desktop E2E | Existing GNOME, KDE, or Xorg login | `scripts/ci/linux/run-rust-e2e-desktop.sh ` | +| Linux representative desktop E2E | Existing Xorg login | `scripts/ci/linux/run-rust-e2e-desktop.sh ` | | macOS Harness E2E | Maintainer Lume SIP-off worker with a logged-in session and inherited grants | `packages/cua-driver/tests/runners/macos-lume/run-all.sh` | Workflows select private execution lanes. Rust source owns scenario definitions, diff --git a/packages/cua-driver/docs/wayland-compositor-identity-plan.md b/packages/cua-driver/docs/wayland-compositor-identity-plan.md deleted file mode 100644 index 6bd666c08ae..00000000000 --- a/packages/cua-driver/docs/wayland-compositor-identity-plan.md +++ /dev/null @@ -1,102 +0,0 @@ -# Wayland Compositor Identity Plan - -This is an architecture explanation and evidence plan for expanding exact -browser attachment beyond Sway. It does not claim generic Wayland support. - -## Why Compositor Identity Is Required - -Existing-profile setup can open an internal browser page and press one exact -accessibility control. Before doing so, Cua Driver must prove that the native -toplevel, browser PID, geometry, AT-SPI application/window, product descriptor, -and loopback endpoint all describe the same browser generation. - -Title, application ID, or screen-coordinate similarity alone is insufficient. -An attacker or unrelated window can reuse those values. When the compositor -cannot attest exact identity, mutation must refuse before opening the setup -page or sending input. - -## Accepted Sway Route - -The existing Sway adapter uses compositor IPC to correlate the exact container, -PID, and rectangle with the AT-SPI tree. Setup may briefly focus that attested -container for fixed browser-owned internal navigation, then restores the prior -container. Canonical Sway evidence accepts Google Chrome, Chromium, and -Microsoft Edge after each product completed the same full matrix with playable -video and no unexpected results. - -## Accepted GNOME Route - -The maintained WinRects extension v4 exposes the narrow identity route Cua -Driver needs on GNOME Shell and Mutter: a stable window identifier, PID, -geometry, focus and stacking state, and exact activation. - -The Linux adapter resolves the extension's immutable unique D-Bus owner, -verifies that the owner is the current user's system-installed `gnome-shell` -process, requires helper API v4 for browser-sensitive calls, and addresses that -unique owner directly. This prevents an unrelated same-session process from -replacing the public bus name between verification and activation. Setup and -consent briefly activate one exact window, then restore and verify the prior -Shell focus. - -The route is accepted for Google Chrome after the full standalone-browser -matrix recorded 13 delivered behaviors, 3 policy refusals, 0 failures, and 0 -skips with 16 playable videos at one source commit. Other Chromium-family -products still require their own product-specific evidence on GNOME. - -The adapter refuses when the component is missing, incompatible, disabled, or -unable to distinguish multiple windows for one browser process. Enabling a -generic unsafe-mode switch is not an acceptable production dependency. - -## Candidate Compositors - -### KDE Plasma and KWin - -KWin 6 scripting exposes a per-window internal ID, PID, frame geometry, -stacking order, active state, and workspace activation. See the official -[KWin scripting API](https://develop.kde.org/docs/plasma/kwin/api/). - -A maintained Cua Driver KWin component could return a versioned, read-only -identity record and perform one exact focus-and-restore transition. The adapter -must bind its D-Bus peer, KWin generation, internal window ID, PID, geometry, -and AT-SPI window. A generic user-provided script or title lookup is not enough. - -### Other Compositors - -The standard `ext-foreign-toplevel-list-v1` protocol exposes an opaque stable -toplevel identifier, title, and application ID, but not PID or geometry. See -the [foreign toplevel protocol](https://wayland.app/protocols/ext-foreign-toplevel-list-v1). -It can contribute one part of a proof, but cannot independently authorize -mutation. - -wlroots compositors may expose additional management or compositor IPC. Each -route remains compositor-specific until it proves all required fields. No -screen-coordinate fallback is allowed. - -## Threat Cases - -Each adapter and harness must cover: - -- two same-title browser windows with equal or near-equal geometry; -- PID reuse or browser restart between approval and mutation; -- a decoy application with the same application ID or title; -- a window moving, changing workspace, minimizing, or closing during setup; -- stale compositor object IDs after remap; -- an incomplete or mismatched AT-SPI tree; -- endpoint ownership or generation drift; and -- focus restoration failure after a bounded setup action. - -Any ambiguity produces a stable, non-mutating structured refusal. - -## Release Acceptance Per Compositor - -A compositor is listed as supported only after the canonical matrix records: - -1. exact compositor, distribution, browser product, and version provenance; -2. setup, attach, reconnect, multi-tab, stale-ref, background type, trusted - input policy, and ambiguous-window rows; -3. external page-state oracles rather than driver success responses; -4. continuous focus, z-order or occlusion, and no-leaked-input guards; -5. cursor preservation where the compositor exposes a trustworthy readback; -6. before and after evidence plus playable video at the exact source commit; -7. explicit environment results for missing APIs or accessibility trees; and -8. generic and unknown Wayland sessions refusing before mutation. diff --git a/packages/cua-driver/rust/CHANGELOG.md b/packages/cua-driver/rust/CHANGELOG.md index d44d2876702..d5b84e08083 100644 --- a/packages/cua-driver/rust/CHANGELOG.md +++ b/packages/cua-driver/rust/CHANGELOG.md @@ -1,3 +1,15 @@ +# 0.26.0 (2026-10-16) + +Qwen Computer Use release; no longer follows the upstream version number. The SDK, native payload, browser SDK and Node REPL use this same version. + +- Driver-owned app/window resolution, current short IDs, AX diffs/budgets and structured dispatch errors; compact 19-tool direct MCP surface. +- Background-first platform delivery, guarded foreground fallback on Windows/Linux, and explicit desktop input with session-owned held-key cleanup. Native Wayland removed. +- Runtime documentation and browser-use facade replace the two bundled Computer Use/Browser Use skill entrypoints. Browser functionality is unchanged. +- Windows UIAccess is optional; default installation does not require a signed worker. Native archives remain on-demand downloads with SHA-256 verification. +- Selected upstream MIT fixes are recorded per PR in `.vendored-patches.md` and `docs/design/cua-upgrade-platform-ports.md`, including platform adaptations and runtime evidence gaps. + +This local development checkpoint does not claim published assets, GUI parity or model benchmark improvement. + # Changelog ## [0.20.0](https://github.com/trycua/cua/compare/cua-driver-rs-v0.19.3...cua-driver-rs-v0.20.0) (2026-08-15) diff --git a/packages/cua-driver/rust/Cargo.lock b/packages/cua-driver/rust/Cargo.lock index 6d44b7ca3ca..387fb0c3ed6 100644 --- a/packages/cua-driver/rust/Cargo.lock +++ b/packages/cua-driver/rust/Cargo.lock @@ -46,16 +46,6 @@ dependencies = [ "libc", ] -[[package]] -name = "annotate-snippets" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e" -dependencies = [ - "unicode-width", - "yansi-term", -] - [[package]] name = "anstream" version = "1.0.0" @@ -142,21 +132,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "ashpd" -version = "0.13.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "340e0f6bf7f9ee78549c61454f1460a3ed97c011902ee76b58301bbc6d502a32" -dependencies = [ - "enumflags2", - "futures-util", - "getrandom 0.4.2", - "serde", - "serde_repr", - "tokio", - "zbus", -] - [[package]] name = "askama" version = "0.14.0" @@ -181,7 +156,7 @@ dependencies = [ "memchr", "proc-macro2", "quote", - "rustc-hash 2.1.3", + "rustc-hash", "serde", "serde_derive", "syn", @@ -365,7 +340,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bf601cccedfffec598ec2db1f9d6745885458bccc0e8916d7023f017c94b3d0" dependencies = [ "atspi-common", - "atspi-connection", "atspi-proxies", "zbus", ] @@ -430,27 +404,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "annotate-snippets", - "bitflags 2.11.1", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn", -] - [[package]] name = "bit-set" version = "0.8.0" @@ -582,19 +535,6 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" -[[package]] -name = "calloop" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" -dependencies = [ - "bitflags 2.11.1", - "polling", - "rustix", - "slab", - "tracing", -] - [[package]] name = "camino" version = "1.2.4" @@ -658,25 +598,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom 7.1.3", -] - -[[package]] -name = "cfg-expr" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" -dependencies = [ - "smallvec", - "target-lexicon", -] - [[package]] name = "cfg-if" version = "1.0.4" @@ -717,17 +638,6 @@ dependencies = [ "phf", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.6.1" @@ -781,7 +691,6 @@ dependencies = [ "objc2-foundation 0.3.2", "objc2-ui-kit", "windows 0.59.0", - "wl-clipboard-rs", "x11rb", ] @@ -819,15 +728,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "convert_case" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "cookie" version = "0.18.1" @@ -839,15 +739,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "cookie-factory" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" -dependencies = [ - "futures", -] - [[package]] name = "cookie_store" version = "0.22.1" @@ -939,15 +830,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -981,7 +863,7 @@ dependencies = [ [[package]] name = "cua-driver" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "async-trait", @@ -1017,7 +899,7 @@ dependencies = [ [[package]] name = "cua-driver-bindgen" -version = "0.20.11" +version = "0.26.0" dependencies = [ "cbindgen", "uniffi", @@ -1025,7 +907,7 @@ dependencies = [ [[package]] name = "cua-driver-contract" -version = "0.20.11" +version = "0.26.0" dependencies = [ "schemars", "serde", @@ -1035,7 +917,7 @@ dependencies = [ [[package]] name = "cua-driver-core" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "async-trait", @@ -1053,6 +935,7 @@ dependencies = [ "sha2", "tempfile", "thiserror 1.0.69", + "time", "tokio", "tokio-tungstenite", "tracing", @@ -1064,7 +947,7 @@ dependencies = [ [[package]] name = "cua-driver-sdk" -version = "0.20.11" +version = "0.26.0" dependencies = [ "async-trait", "core-foundation", @@ -1086,7 +969,7 @@ dependencies = [ [[package]] name = "cua-driver-testkit" -version = "0.20.11" +version = "0.26.0" dependencies = [ "core-foundation", "libc", @@ -1100,7 +983,7 @@ dependencies = [ [[package]] name = "cua-driver-uia" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "cua-driver-core", @@ -1115,7 +998,7 @@ dependencies = [ [[package]] name = "cursor-overlay" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "cua-driver-contract", @@ -1132,7 +1015,7 @@ dependencies = [ [[package]] name = "cursor-theme-cli" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "cursor-overlay", @@ -1170,27 +1053,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dispatch2" version = "0.3.1" @@ -1231,24 +1093,12 @@ dependencies = [ "futures-util", ] -[[package]] -name = "downcast-rs" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" - [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "either" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" - [[package]] name = "email_address" version = "0.2.9" @@ -1348,7 +1198,7 @@ dependencies = [ "bitvec", "cfg-if", "libc", - "nix 0.23.2", + "nix", "thiserror 1.0.69", ] @@ -1421,12 +1271,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - [[package]] name = "flate2" version = "1.1.9" @@ -1552,48 +1396,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", - "futures-sink", -] - [[package]] name = "futures-core" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" -[[package]] -name = "futures-executor" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - [[package]] name = "futures-io" version = "0.3.32" @@ -1642,13 +1450,10 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ - "futures-channel", "futures-core", - "futures-io", "futures-macro", "futures-sink", "futures-task", - "memchr", "pin-project-lite", "slab", ] @@ -2008,15 +1813,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -2087,12 +1883,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128fmt" version = "0.1.0" @@ -2105,53 +1895,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "libloading" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" -dependencies = [ - "cfg-if", - "windows-link 0.2.1", -] - -[[package]] -name = "libredox" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" -dependencies = [ - "libc", -] - -[[package]] -name = "libspa" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f3a4b81b2a2d8c7f300643676202debd1b7c929dbf5c9bb89402ea11d19810" -dependencies = [ - "bitflags 2.11.1", - "cc", - "convert_case", - "cookie-factory", - "libc", - "libspa-sys", - "nix 0.27.1", - "nom 7.1.3", - "system-deps", -] - -[[package]] -name = "libspa-sys" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0d9716420364790e85cbb9d3ac2c950bde16a7dd36f3209b7dfdfc4a24d01f" -dependencies = [ - "bindgen", - "cc", - "system-deps", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2325,17 +2068,6 @@ dependencies = [ "memoffset 0.6.5", ] -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.11.1", - "cfg-if", - "libc", -] - [[package]] name = "nom" version = "1.2.4" @@ -2352,15 +2084,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2796,12 +2519,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "ordered-stream" version = "0.2.0" @@ -2812,16 +2529,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "os_pipe" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "outref" version = "0.5.2" @@ -2863,17 +2570,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap", -] - [[package]] name = "phf" version = "0.12.1" @@ -2900,7 +2596,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pip-preview" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "serde_json", @@ -2918,34 +2614,6 @@ dependencies = [ "futures-io", ] -[[package]] -name = "pipewire" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08e645ba5c45109106d56610b3ee60eb13a6f2beb8b74f8dc8186cf261788dda" -dependencies = [ - "anyhow", - "bitflags 2.11.1", - "libc", - "libspa", - "libspa-sys", - "nix 0.27.1", - "once_cell", - "pipewire-sys", - "thiserror 1.0.69", -] - -[[package]] -name = "pipewire-sys" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849e188f90b1dda88fe2bfe1ad31fe5f158af2c98f80fb5d13726c44f3f01112" -dependencies = [ - "bindgen", - "libspa-sys", - "system-deps", -] - [[package]] name = "pkg-config" version = "0.3.33" @@ -2960,53 +2628,39 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "platform-linux" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", - "ashpd", "async-trait", "atspi", + "atspi-connection", "base64", - "calloop", "clipboard-rs", - "crossbeam-channel", "cua-driver-contract", "cua-driver-core", "cursor-overlay", - "dirs", - "enumflags2", "evdev", "image", "libc", - "libspa", "memmap2", "meval", "pip-preview", - "pipewire", - "reis", "serde", "serde_json", + "sha2", "thiserror 1.0.69", "tiny-skia", "tokio", "tracing", - "tracing-subscriber", - "url", "uuid", - "wayland-backend", - "wayland-client", - "wayland-protocols", - "wayland-protocols-wlr", - "wayland-scanner", "x11", "x11rb", - "xkbcommon", "zbus", ] [[package]] name = "platform-macos" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "async-trait", @@ -3042,7 +2696,7 @@ dependencies = [ [[package]] name = "platform-windows" -version = "0.20.11" +version = "0.26.0" dependencies = [ "anyhow", "async-trait", @@ -3056,6 +2710,7 @@ dependencies = [ "pip-preview", "serde", "serde_json", + "tempfile", "thiserror 1.0.69", "tiny-skia", "tokio", @@ -3279,17 +2934,6 @@ dependencies = [ "bitflags 2.11.1", ] -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 1.0.69", -] - [[package]] name = "ref-cast" version = "1.0.25" @@ -3389,19 +3033,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "reis" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3fedd2777cde52c1be5e572efbec485eac7b801c47820eda388d4f13b9c4b" -dependencies = [ - "enumflags2", - "futures-util", - "log", - "rustix", - "tokio", -] - [[package]] name = "ring" version = "0.17.14" @@ -3416,12 +3047,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.3" @@ -3863,19 +3488,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-deps" -version = "6.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" -dependencies = [ - "cfg-expr", - "heck", - "pkg-config", - "toml 0.8.23", - "version-compare", -] - [[package]] name = "tap" version = "1.0.1" @@ -3892,12 +3504,6 @@ dependencies = [ "libc", ] -[[package]] -name = "target-lexicon" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" - [[package]] name = "tempfile" version = "3.27.0" @@ -4209,7 +3815,6 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4265,17 +3870,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "tree_magic_mini" -version = "3.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6" -dependencies = [ - "memchr", - "nom 8.0.0", - "petgraph", -] - [[package]] name = "ttf-parser" version = "0.21.1" @@ -4342,18 +3936,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-xid" version = "0.2.6" @@ -4597,12 +4179,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version-compare" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" - [[package]] name = "version_check" version = "0.9.5" @@ -4738,76 +4314,6 @@ dependencies = [ "semver", ] -[[package]] -name = "wayland-backend" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d" -dependencies = [ - "cc", - "downcast-rs", - "rustix", - "smallvec", - "wayland-sys", -] - -[[package]] -name = "wayland-client" -version = "0.31.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" -dependencies = [ - "bitflags 2.11.1", - "rustix", - "wayland-backend", - "wayland-scanner", -] - -[[package]] -name = "wayland-protocols" -version = "0.32.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f" -dependencies = [ - "bitflags 2.11.1", - "wayland-backend", - "wayland-client", - "wayland-scanner", -] - -[[package]] -name = "wayland-protocols-wlr" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" -dependencies = [ - "bitflags 2.11.1", - "wayland-backend", - "wayland-client", - "wayland-protocols", - "wayland-scanner", -] - -[[package]] -name = "wayland-scanner" -version = "0.31.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" -dependencies = [ - "proc-macro2", - "quick-xml", - "quote", -] - -[[package]] -name = "wayland-sys" -version = "0.31.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" -dependencies = [ - "pkg-config", -] - [[package]] name = "webpki-roots" version = "1.0.7" @@ -4832,28 +4338,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows" version = "0.58.0" @@ -5422,24 +4906,6 @@ dependencies = [ "wasmparser", ] -[[package]] -name = "wl-clipboard-rs" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9651471a32e87d96ef3a127715382b2d11cc7c8bb9822ded8a7cc94072eb0a3" -dependencies = [ - "libc", - "log", - "os_pipe", - "rustix", - "thiserror 2.0.18", - "tree_magic_mini", - "wayland-backend", - "wayland-client", - "wayland-protocols", - "wayland-protocols-wlr", -] - [[package]] name = "writeable" version = "0.6.3" @@ -5482,32 +4948,6 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" -[[package]] -name = "xkbcommon" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a974f48060a14e95705c01f24ad9c3345022f4d97441b8a36beb7ed5c4a02d" -dependencies = [ - "libc", - "memmap2", - "xkeysym", -] - -[[package]] -name = "xkeysym" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" - -[[package]] -name = "yansi-term" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" -dependencies = [ - "winapi", -] - [[package]] name = "yoke" version = "0.8.2" diff --git a/packages/cua-driver/rust/Cargo.toml b/packages/cua-driver/rust/Cargo.toml index 6e6c6f4c94f..66deb760010 100644 --- a/packages/cua-driver/rust/Cargo.toml +++ b/packages/cua-driver/rust/Cargo.toml @@ -17,7 +17,7 @@ members = [ ] [workspace.package] -version = "0.20.11" +version = "0.26.0" edition = "2021" authors = ["trycua"] license = "MIT" diff --git a/packages/cua-driver/rust/README.md b/packages/cua-driver/rust/README.md index acdfb58eb0b..061dd664ade 100644 --- a/packages/cua-driver/rust/README.md +++ b/packages/cua-driver/rust/README.md @@ -11,7 +11,7 @@ implementations, testkit, and helper crates. | `cua-driver-core` | Shared protocol, tool models, config, and CDP helpers | | `platform-macos` | macOS AX, capture, input, browser, and daemon support | | `platform-windows` | Windows UIA, capture, input, overlay, and diagnostics | -| `platform-linux` | Linux AT-SPI, X11/Wayland, capture, and input support | +| `platform-linux` | Linux AT-SPI, X11, capture, and input support | | `cua-driver-testkit` | Test-only helpers for spawning the daemon and parsing responses | | `cua-driver-uia` | Windows UIAccess worker | | `cursor-overlay` | Cross-platform semantic cursor renderer and bounded compiled-theme loader | diff --git a/packages/cua-driver/rust/Skills/cua-driver/BROWSER.md b/packages/cua-driver/rust/Skills/cua-driver/BROWSER.md index c7c9e571618..6c6ea743e4b 100644 --- a/packages/cua-driver/rust/Skills/cua-driver/BROWSER.md +++ b/packages/cua-driver/rust/Skills/cua-driver/BROWSER.md @@ -419,7 +419,6 @@ capabilities, or existing-profile consent. | Chrome / Edge on Windows | Exact binding, refs, navigation, typing, trusted or explicit DOM click | Must run in an interactive user session, not Session 0 | | Chrome / Edge on macOS | Exact binding, refs, navigation, typing, explicit DOM click | Trusted standalone click refuses to preserve background posture | | Chrome / Chromium on Linux X11 | Exact binding, refs, navigation, typing, explicit DOM click | Trusted standalone click refuses to preserve background posture | -| Chromium on validated Wayland setups | Exact binding only when compositor identity is provable | Generic/ambiguous compositor identity refuses mutation | | Electron | Exact single-page routes where endpoint and host relationship are proven | Do not infer support for arbitrary embedded webviews | | Safari / Firefox | Native window state only | Typed page mutation is not supported yet | | WebView2 / Tauri / other embedded webviews | Native AX/PX fallback unless an exact route is reported | Host/renderer correlation may refuse | diff --git a/packages/cua-driver/rust/Skills/cua-driver/LINUX.md b/packages/cua-driver/rust/Skills/cua-driver/LINUX.md index df2cc2df331..f2efd1ac9a5 100644 --- a/packages/cua-driver/rust/Skills/cua-driver/LINUX.md +++ b/packages/cua-driver/rust/Skills/cua-driver/LINUX.md @@ -10,9 +10,6 @@ session recording. On X11, `set_window_frame({pid, window_id, x, y, width, height})` sends an EWMH window-manager request and confirms it against `list_windows` geometry. -Wayland has no portable protocol for setting another application's top-level -geometry, so this tool refuses there unless a future compositor-owned adapter -can provide exact targeting and readback. AT-SPI is talked to natively over D-Bus (the `atspi`/zbus crate) — no `pyatspi` or GObject-introspection typelibs are required at runtime. @@ -73,14 +70,9 @@ Keystroke / XSendEvent / XTest / foreground rungs return `effect:"unverifiable"` unless another publishable readback exists. Confirm those through `verify_state` or multimodal reading. -**Escalation** — action responses use the cross-platform closed -`escalation:{target, reason}` shape. See `SKILL.md` → behavior matrix. On a -standard Wayland compositor the Linux-specific target is **`foreground`** -(raw background pixels cannot target an unfocused window); the opt-in nested -compositor is a separate environment. Use **`pixel` on X11** (an element px -action — background pixel click — lands via -AT-SPI `do_action`-at-point off the screenshot already in the snapshot — the -matrix below). +**Escalation** — use pixels for a non-AX surface when supported. Retry in +foreground only after a pre-dispatch `background_unavailable` refusal. +An unconfirmed dispatched action must be observed before any further action. ## Perception and the ax/px action choice @@ -110,10 +102,7 @@ Modality is chosen at **action time**, by how you address the target: already in the `get_window_state` response. Best-effort; caller-confirmed. `get_window_state` returning `degraded:true` (empty AT-SPI walk) is the cue to -do an **element px action** off that same screenshot (X11) or escalate to -`delivery_mode:"foreground"` (standard Wayland: raw background pixels cannot -target an unfocused window there). The nested compositor has its own -experimental per-surface routes. +use pixels from the same screenshot when the target supports that route. ## Cross-platform schema residuals (Linux) @@ -141,7 +130,7 @@ Use `invoke_menu({pid, window_id, path:[...]})` for a known GTK/Qt application menu command. It activates the exact target only for the duration of the operation, resolves each labelled AT-SPI menu descendant again after the prior menu expands, and refuses missing, duplicate, disabled, or non-actionable -segments. It works through AT-SPI on both X11 and Wayland and never falls back +segments. It works through AT-SPI on X11 and never falls back to coordinates. Verify the command's semantic effect from fresh state; the native `do_action` acknowledgement alone is not task completion. @@ -200,49 +189,23 @@ uinput, escalate to `delivery_mode:"foreground"`. (`type_text` in the `background` rung is focus-dependent for non-editable widgets; that's the one genuine background limitation, and `foreground` is the documented escalation.) -## Wayland +## Display support -Set `CUA_DRIVER_RS_ENABLE_WAYLAND=1` to enable native Wayland support. The -driver selects a backend from compositor capabilities: - -- Sway and other wlroots compositors use foreign-toplevel discovery, - wlr-screencopy, virtual pointer, and virtual keyboard protocols. -- GNOME/Mutter uses the bundled WinRects Shell helper for target geometry and - activation, plus portal/libei for foreground raw input. -- KDE/KWin uses AT-SPI and portal facilities where available. Target-specific - foreground activation remains experimental, so unsafe raw input refuses. -- The optional `cua-compositor` is a separate nested session enabled - explicitly for controlled automation. GNOME and KDE never switch into it. - -Sway recording works through the wlroots recorder path and is exercised by the -canonical harness runner. Portal-backed GNOME recording is still an evidence -gap. Capture and recording availability therefore depend on the compositor, -installed helpers, and portal grant. - -Standard Wayland has no general client protocol for raw input to an arbitrary -occluded surface. Background AX actions can still deliver through AT-SPI, and -a PX left click can deliver when hit-testing resolves to an actionable AT-SPI -control. Other focus-bound background pointer and keyboard shapes return an -exact `background_unavailable` result. They do not report success after a -silent drop. - -Use `delivery_mode:"foreground"` for raw Wayland input. The driver activates -the selected target through a verified compositor adapter before dispatch. If -the compositor has no target-addressable activation or input backend, the call -refuses before sending input. Reconstructing coordinates alone does not make -raw background PX possible on a standard compositor. +Only X11 is supported. Native Wayland backends, compositor helpers, portal +input and their opt-in variables have been removed. X11 applications under +XWayland remain supported through their authenticated DISPLAY; native Wayland +applications cannot be targeted. No DISPLAY produces an explicit X11-only error. ## Quick triage If a tool call surprises you on Linux: -1. `qwen-cua-driver doctor` — reports the display server (X11 / Wayland), +1. `qwen-cua-driver doctor` — reports X11 availability, **whether `org.a11y.Bus` actually answers on the session bus** (not just "is there a bus"), the discovered `DBUS_SESSION_BUS_ADDRESS`, and `ffmpeg` availability (for recording). -2. Check `XDG_SESSION_TYPE` — `x11` is fully supported; `wayland` - needs `CUA_DRIVER_RS_ENABLE_WAYLAND=1` for the native backend, - else XWayland. +2. Check `DISPLAY` and `XAUTHORITY`. A usable X11 display takes precedence + regardless of the surrounding session's `XDG_SESSION_TYPE`. 3. **Empty AT-SPI tree** (`get_window_state` returns `degraded:true`) — in order of likelihood: (a) the daemon isn't on the desktop session bus (headless / container / `runuser` / root-against-user-session — see @@ -269,10 +232,6 @@ ask the user. | Environment | Proven baseline | Main limits | |---|---|---| | X11/Openbox | AT-SPI trees and actions, foreground pointer and keyboard input, window and desktop capture, and video | Raw background delivery remains toolkit-specific; unsupported shapes refuse | -| Sway/wlroots | AT-SPI, native discovery, full-display and cropped-window screencopy, foreground input, semantic background actions, and video | Raw background pointer and keyboard input remains focus-bound | -| GNOME/Mutter | AT-SPI, WinRects geometry and activation, capture, and portal/libei foreground input | Requires the helper and portal grant; portal video parity remains open | -| KDE/KWin | AT-SPI and generic discovery where exposed | Target-specific activation and behavioral coverage remain experimental | -| Nested `cua-compositor` | Versioned direct per-surface input, native GTK 31/31, capture/scope 5/5, and partial Electron coverage | The complete shared matrix remains experimental; do not infer standard-Wayland support | See `SKILL.md` for the cross-platform loop (snapshot-before-AND-after, pixel-click contract, failure modes) and `RECORDING.md` for session diff --git a/packages/cua-driver/rust/Skills/cua-driver/SKILL.md b/packages/cua-driver/rust/Skills/cua-driver/SKILL.md index 08a6956923d..7e75c779da8 100644 --- a/packages/cua-driver/rust/Skills/cua-driver/SKILL.md +++ b/packages/cua-driver/rust/Skills/cua-driver/SKILL.md @@ -1,7 +1,7 @@ --- name: cua-driver description: Drive a native GUI app (macOS, Windows, Linux) via the Qwen Cua Driver CLI (default) or MCP server; snapshot its accessibility tree, act through snapshot-bound element tokens, native menu paths, exact window geometry, or pixel coordinates, and verify from fresh state. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real application on the host. -version: 0.20.11 # x-release-please-version +version: 0.26.0 # x-release-please-version metadata: openclaw: requires: @@ -17,9 +17,6 @@ metadata: - name: CUA_DRIVER_PATH required: false description: Optional path to a qwen-cua-driver binary used by an embedding host. - - name: CUA_DRIVER_RS_ENABLE_WAYLAND - required: false - description: Set to 1 to enable the native Wayland backend. - name: CUA_DRIVER_RS_MCP_HTTP_PORT required: false description: Optional port for the local MCP HTTP endpoint. @@ -52,7 +49,7 @@ directory: ApplicationFrameHost hosting, layered UIA+PostMessage click chain, Session 0 isolation, Windows-specific focus-steal vectors). - **Linux** — read `LINUX.md` (X11 background input via AT-SPI + - XSendEvent and compositor-specific Wayland capabilities). + XSendEvent (including X11 apps under XWayland)). Cross-cutting topics also have their own files: @@ -552,13 +549,6 @@ surface like Electron/Chromium/canvas), returns `degraded: true` plus an observation-specific escalation hint — normally pointing at pixels (you still have the screenshot from the same call to click off). -**Platform nuance for action escalation.** On **Wayland** an unfocused -window cannot be pixel-targeted in the background (libei → -`background_unavailable`), so the action target is -**`foreground`, not `pixel`**. macOS, X11, and most Windows surfaces -can pixel-target in the background, so they target `pixel`. See -`LINUX.md` / `WINDOWS.md`. - ## The verify-then-escalate ladder (algorithm) Every snapshot already hands you both the tree and the screenshot, so @@ -607,8 +597,6 @@ if resp.effect == "suspected_noop" if resp.escalation.target == "foreground" or the px action still did nothing: re-call the same action with delivery_mode:"foreground" - # on Wayland this is the ONLY escalation — px-bg can't target an - # unfocused window there; see LINUX.md verify again # Route 5 — per-call desktop fallback @@ -817,7 +805,7 @@ the response's `snapshot_id` with `element_index`; bare indices fail closed in `list_windows.z_index` uses one portable convention: higher integer values are closer to the front. Select a frontmost candidate with the maximum non-null value. If all values are `null` (as they can be on -native Wayland), use an explicit fallback; never treat `null` as zero +unavailable stacking metadata), use an explicit fallback; never treat `null` as zero or infer stacking from array order. The `windows` records returned by `launch_app` use the same convention. diff --git a/packages/cua-driver/rust/VERSION b/packages/cua-driver/rust/VERSION index 81fb0055204..4e8f395fa5e 100644 --- a/packages/cua-driver/rust/VERSION +++ b/packages/cua-driver/rust/VERSION @@ -1 +1 @@ -0.20.11 +0.26.0 diff --git a/packages/cua-driver/rust/crates/cua-driver-contract/src/bin/cua-contract-gen.rs b/packages/cua-driver/rust/crates/cua-driver-contract/src/bin/cua-contract-gen.rs index 3d0b6864ebf..d8005d30853 100644 --- a/packages/cua-driver/rust/crates/cua-driver-contract/src/bin/cua-contract-gen.rs +++ b/packages/cua-driver/rust/crates/cua-driver-contract/src/bin/cua-contract-gen.rs @@ -20,17 +20,31 @@ fn run() -> Result<(), String> { .find(|arg| !arg.starts_with('-')) .map(String::as_str) .unwrap_or("all"); - if !matches!(target, "all" | "manifest") { - return Err("usage: cua-contract-gen [all|manifest] [--check]".into()); + if !matches!(target, "all" | "manifest" | "computer-use") { + return Err("usage: cua-contract-gen [all|manifest|computer-use] [--check]".into()); } - let mut contents = serde_json::to_string_pretty(&manifest()).map_err(|e| e.to_string())?; - contents.push('\n'); - write_or_check( - &driver_root()?.join("contract/manifest.json"), - &contents, - check, - ) + if target != "computer-use" { + let mut contents = serde_json::to_string_pretty(&manifest()).map_err(|e| e.to_string())?; + contents.push('\n'); + write_or_check( + &driver_root()?.join("contract/manifest.json"), + &contents, + check, + )?; + } + if target != "manifest" { + let mut contents = + serde_json::to_string_pretty(&cua_driver_contract::computer_use::manifest()) + .map_err(|e| e.to_string())?; + contents.push('\n'); + write_or_check( + &driver_root()?.join("typescript/computer-use/interface.json"), + &contents, + check, + )?; + } + Ok(()) } fn driver_root() -> Result { diff --git a/packages/cua-driver/rust/crates/cua-driver-contract/src/computer_use.rs b/packages/cua-driver/rust/crates/cua-driver-contract/src/computer_use.rs new file mode 100644 index 00000000000..2bd57554b33 --- /dev/null +++ b/packages/cua-driver/rust/crates/cua-driver-contract/src/computer_use.rs @@ -0,0 +1,146 @@ +use serde::Serialize; +use serde_json::{json, Map, Value}; + +#[derive(Clone, Debug, Serialize)] +pub struct ComputerUseTool { + pub name: &'static str, + pub method: &'static str, + pub description: &'static str, + pub read_only: bool, + pub input_schema: Value, +} + +fn tool( + name: &'static str, + method: &'static str, + description: &'static str, + read_only: bool, + parameters: Value, + required: &[&str], +) -> ComputerUseTool { + let mut properties = Map::from_iter([ + ( + "app".into(), + json!({"type":"string","minLength":1,"description":"Exact application name, identifier or absolute path from get_state/get_app."}), + ), + ( + "session".into(), + json!({"type":"string","description":crate::MULTI_CALL_SESSION_DESCRIPTION}), + ), + ]); + if name == "get_state" { + properties.remove("app"); + } + properties.extend(parameters.as_object().expect("object schema").clone()); + let desktop = matches!( + name, + "get_screenshot" + | "click" + | "double_click" + | "right_click" + | "scroll" + | "drag" + | "type_text" + | "press_key" + | "hotkey" + | "key_down" + | "key_up" + ); + if desktop { + properties.insert("scope".into(), json!({"type":"string","enum":["desktop"],"description":"Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app."})); + if matches!(name, "key_down" | "key_up") { + properties.remove("app"); + } else { + properties.get_mut("app").unwrap()["description"] = json!( + "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop." + ); + } + } + let schema = json!({ + "type":"object", "properties":properties, + "required":required.iter().filter(|field| !desktop || **field != "app").collect::>(), "additionalProperties":false, + }); + ComputerUseTool { + name, + method, + description, + read_only, + input_schema: schema, + } +} + +pub fn tools() -> Vec { + let observation = json!({ + "force_full":{"type":"boolean","default":false}, + "max_text_chars":{"type":"integer","minimum":512,"maximum":1000000,"default":12000}, + "max_elements":{"type":"integer","minimum":1}, + "max_depth":{"type":"integer","minimum":1}, + }); + let mut initial_observation = observation.clone(); + initial_observation + .as_object_mut() + .unwrap() + .remove("force_full"); + let point = json!({ + "element":{"type":"integer","minimum":0,"description":"Short ID from the latest observation of this app window."}, + "x":{"type":"number","description":"Window screenshot pixel x. Requires a current screenshot."}, + "y":{"type":"number","description":"Window screenshot pixel y. Requires a current screenshot."}, + }); + let mut click = point.clone(); + click["key"] = json!({"type":"array","items":{"type":"string"},"description":"Modifier keys held during the click, such as [\"shift\"] or [\"ctrl\",\"shift\"]."}); + click["button"] = json!({"type":"string","enum":["left","middle","right"],"default":"left"}); + click["count"] = json!({"type":"integer","minimum":1,"maximum":3,"default":1}); + let mut scroll = point; + scroll["direction"] = json!({"type":"string","enum":["up","down","left","right"]}); + scroll["amount"] = json!({"type":"integer","minimum":1,"maximum":50,"default":3}); + let text = json!({"text":{"type":"string"}}); + vec![ + tool("get_state", "getState", "List applications and their windows. Use an exact app selector for subsequent calls.", true, json!({}), &[]), + tool("get_app", "getApp", "Resolve an application and return its full AX state without an image. A stopped, uniquely identified application may be opened for observation; actions never relaunch it.", false, initial_observation, &["app"]), + tool("get_ax_state", "getAXState", "Observe AX text without capturing pixels. Uses a diff after the first observation; force_full requests a complete view. Short IDs belong to this app window's current observation.", true, observation.clone(), &["app"]), + tool("get_screenshot", "getScreenshot", "Observe the current application window screenshot without an AX walk. Coordinates in actions use this image's pixels.", true, json!({}), &["app"]), + tool("get_ax_state_and_screenshot", "getAXStateAndScreenshot", "Observe AX text and the current window image together. Uses an AX diff unless force_full is set.", true, observation, &["app"]), + tool("click", "click", "Click a current short element ID or screenshot point. Optional key holds modifiers during the click. Observe the result before retrying an unconfirmed action.", false, click, &["app"]), + tool("double_click", "doubleClick", "Double-click a current short element ID or screenshot point, then observe the result.", false, json!({"element":{"type":"integer","minimum":0},"x":{"type":"number"},"y":{"type":"number"}}), &["app"]), + tool("right_click", "rightClick", "Right-click a current short element ID or screenshot point, then observe the result.", false, json!({"element":{"type":"integer","minimum":0},"x":{"type":"number"},"y":{"type":"number"}}), &["app"]), + tool("scroll", "scroll", "Scroll at a current element or screenshot point. Observe the result before sending another scroll.", false, scroll, &["app","direction"]), + tool("drag", "drag", "Drag between current window screenshot points. Unsupported background gestures refuse before sending input; use semantic value operations when available.", false, json!({"from_x":{"type":"number"},"from_y":{"type":"number"},"to_x":{"type":"number"},"to_y":{"type":"number"}}), &["app","from_x","from_y","to_x","to_y"]), + tool("type_text", "typeText", "Insert text at the current text caret. This does not replace the whole field; use set_value for replacement.", false, text.clone(), &["app","text"]), + tool("set_value", "setValue", "Replace the entire value of the current short element ID and read back the result. Refuses a non-settable element or a write that was not applied.", false, json!({"element":{"type":"integer","minimum":0},"value":{"type":"string"}}), &["app","element","value"]), + tool("press_key", "pressKey", "Press a key in the selected app. Optional modifiers form a chord. Observe before retrying an unconfirmed dispatch.", false, json!({"key":{"type":"string"},"modifiers":{"type":"array","items":{"type":"string"}}}), &["app","key"]), + tool("hotkey", "hotkey", "Press a modifier/key chord in the selected app.", false, json!({"keys":{"type":"array","minItems":1,"items":{"type":"string"}}}), &["app","keys"]), + tool("key_down", "desktop.keyDown", "Hold a desktop key for this session. Release with key_up in finally. Session end releases owned keys. While keys are held, use key_down/key_up for chords and desktop pointer actions without explicit modifiers.", false, json!({"key":{"type":"string","minLength":1}}), &["scope","key"]), + tool("key_up", "desktop.keyUp", "Release a desktop key held by this session. Another session cannot release it.", false, json!({"key":{"type":"string","minLength":1}}), &["scope","key"]), + tool("paste", "paste", "Paste text at the caret using a temporary clipboard value, then restore the previous clipboard even if the operation fails.", false, text, &["app","text"]), + tool("select_text", "selectText", "Select one unambiguous text match inside a current short element ID. Does not change the field value.", false, json!({"element":{"type":"integer","minimum":0},"text":{"type":"string"}}), &["app","element","text"]), + tool("perform_secondary_action", "performSecondaryAction", "Perform an explicitly named secondary accessibility action on a current short element ID.", false, json!({"element":{"type":"integer","minimum":0},"action":{"type":"string"}}), &["app","element","action"]), + ] +} + +pub fn manifest() -> Value { + json!({"version":1,"tools":tools()}) +} + +/// Versioned dispatch acknowledgement for the app surface; no effect is claimed. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AppActionOutput { + pub computer_use: u8, + pub sent: bool, +} +impl AppActionOutput { + pub fn validate(&self) -> Result<(), String> { + if self.computer_use != 1 || !self.sent { + return Err("invalid Computer Use dispatch acknowledgement".into()); + } + Ok(()) + } +} + +pub fn is_app_call(name: &str, args: &Value) -> bool { + matches!( + name, + "get_state" | "get_app" | "get_ax_state" | "get_screenshot" | "get_ax_state_and_screenshot" + ) || (args.get("app").is_some() || args["scope"] == "desktop") + && tools().iter().any(|tool| tool.name == name) +} diff --git a/packages/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs b/packages/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs index ebcfc427701..743c1422588 100644 --- a/packages/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs +++ b/packages/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs @@ -6,7 +6,7 @@ //! These types are transport-free. The contract generator derives JSON Schema //! from them, and live Rust handlers deserialize the same types before acting. -use crate::CursorThemeSelection; +use crate::{schema_settings, CursorThemeSelection}; use schemars::{json_schema, JsonSchema, Schema, SchemaGenerator}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::Value; @@ -15,42 +15,89 @@ pub trait ToolInput: Serialize + DeserializeOwned + JsonSchema { const TOOL_NAME: &'static str; fn input_schema() -> Value { - let settings = schemars::generate::SchemaSettings::draft2020_12().with(|settings| { - settings.meta_schema = None; - settings.inline_subschemas = true; - }); - let schema = settings.into_generator().into_root_schema_for::(); + let schema = schema_settings() + .into_generator() + .into_root_schema_for::(); let mut value = serde_json::to_value(schema).expect("tool input schema serializes"); - normalize_schema(&mut value); + normalize_input_schema(&mut value); value } } -fn normalize_schema(value: &mut Value) { - match value { - Value::Object(object) => { - object.remove("title"); - if object.get("type").and_then(Value::as_str) == Some("object") { - object - .entry("properties") - .or_insert_with(|| Value::Object(serde_json::Map::new())); - object - .entry("required") - .or_insert_with(|| Value::Array(Vec::new())); - object - .entry("additionalProperties") - .or_insert(Value::Bool(true)); +pub fn normalize_input_schema(value: &mut Value) { + let Some(object) = value.as_object_mut() else { + return; + }; + object.remove("title"); + if object.get("default").is_some_and(Value::is_null) { + object.remove("default"); + } + if let Some(types) = object.get("type").and_then(Value::as_array) { + let kept: Vec<_> = types + .iter() + .filter(|kind| kind.as_str() != Some("null")) + .cloned() + .collect(); + if let [kind] = kept.as_slice() { + object.insert("type".into(), kind.clone()); + } + } + if let Some(variants) = object.get("anyOf").and_then(Value::as_array) { + let kept: Vec<_> = variants + .iter() + .filter(|schema| schema["type"] != "null") + .cloned() + .collect(); + if let [Value::Object(schema)] = kept.as_slice() { + let schema = schema.clone(); + object.remove("anyOf"); + object.extend(schema); + } + } + if let Some(constant) = object.remove("const") { + match constant { + Value::String(text) => { + object.insert("type".into(), serde_json::json!("string")); + object.insert("enum".into(), serde_json::json!([text])); } - for child in object.values_mut() { - normalize_schema(child); + Value::Number(number) => { + object.insert("minimum".into(), serde_json::json!(number)); + object.insert("maximum".into(), serde_json::json!(number)); + } + other => { + object.insert("const".into(), other); } } - Value::Array(values) => { - for child in values { - normalize_schema(child); + } + if object.get("type").and_then(Value::as_str) == Some("object") { + object + .entry("properties") + .or_insert_with(|| serde_json::json!({})); + object + .entry("required") + .or_insert_with(|| serde_json::json!([])); + object + .entry("additionalProperties") + .or_insert(Value::Bool(true)); + } + for key in ["properties", "$defs"] { + if let Some(children) = object.get_mut(key).and_then(Value::as_object_mut) { + for child in children.values_mut() { + normalize_input_schema(child); + } + } + } + for key in ["items", "additionalProperties"] { + if let Some(child) = object.get_mut(key).filter(|child| child.is_object()) { + normalize_input_schema(child); + } + } + for key in ["anyOf", "oneOf", "allOf"] { + if let Some(children) = object.get_mut(key).and_then(Value::as_array_mut) { + for child in children { + normalize_input_schema(child); } } - _ => {} } } @@ -246,23 +293,38 @@ fn desktop_scope_schema(generator: &mut SchemaGenerator) -> Schema { /// `display_id="primary"` is the portable desktop target in this release. /// Platforms that cannot address another display reject it explicitly rather /// than silently changing coordinate spaces. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, uniffi::Enum)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum ActionTarget { Window { pid: u32, window_id: u64 }, Desktop { display_id: String }, } +impl JsonSchema for ActionTarget { + fn schema_name() -> std::borrow::Cow<'static, str> { + "ActionTarget".into() + } + fn json_schema(_: &mut SchemaGenerator) -> Schema { + json_schema!({ + "type": "object", + "description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.", + "required": ["kind"], + "properties": { + "kind": {"type":"string","enum":["window","desktop"]}, + "pid": {"type":"integer","minimum":0}, + "window_id": {"type":"integer","minimum":0}, + "display_id": {"type":"string","description":"Use primary for the primary display."} + } + }) + } +} + pub fn action_target_schema() -> Value { - let settings = schemars::generate::SchemaSettings::draft2020_12().with(|settings| { - settings.meta_schema = None; - settings.inline_subschemas = true; - }); - let schema = settings + let schema = schema_settings() .into_generator() .into_root_schema_for::(); let mut value = serde_json::to_value(schema).expect("action target schema serializes"); - normalize_schema(&mut value); + normalize_input_schema(&mut value); value } @@ -1275,14 +1337,39 @@ mod tests { use super::*; + #[test] + fn input_schema_normalization_keeps_property_names_and_literal_values() { + let mut schema = json!({ + "type":"object", "title":"Drop annotation", "properties": { + "title": {"type":["string","null"], "default":null}, + "const": {"type":"object", "default":{"title":"literal","const":1}}, + "version": {"type":"integer","const":1} + } + }); + normalize_input_schema(&mut schema); + assert_eq!(schema["properties"]["title"], json!({"type":"string"})); + assert_eq!( + schema["properties"]["const"]["default"], + json!({"title":"literal","const":1}) + ); + assert_eq!( + schema["properties"]["version"], + json!({"type":"integer","minimum":1,"maximum":1}) + ); + assert!(schema.get("title").is_none()); + assert!(serde_json::from_value::(json!({"kind":"window","pid":1})).is_err()); + assert!(serde_json::from_value::(json!({"kind":"desktop"})).is_err()); + } + #[test] fn generated_click_schema_matches_driver_dialect() { let schema = ClickInput::input_schema(); assert_eq!(schema["type"], "object"); assert_eq!(schema["additionalProperties"], false); assert_eq!(schema["required"], json!(["x", "y"])); - assert_eq!(schema["properties"]["scope"]["const"], "desktop"); - assert!(schema["properties"]["target"]["anyOf"].is_array()); + assert_eq!(schema["properties"]["scope"]["enum"], json!(["desktop"])); + assert_eq!(schema["properties"]["target"]["type"], "object"); + assert!(schema["properties"]["target"].get("anyOf").is_none()); assert_eq!( schema["properties"]["button"], json!({ "type": "string", "enum": ["left", "right", "middle"] }) diff --git a/packages/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/packages/cua-driver/rust/crates/cua-driver-contract/src/lib.rs index 53a9974e596..eecb174cc40 100644 --- a/packages/cua-driver/rust/crates/cua-driver-contract/src/lib.rs +++ b/packages/cua-driver/rust/crates/cua-driver-contract/src/lib.rs @@ -8,12 +8,14 @@ //! typed inputs/results and versioned declarations used by the live runtime //! and to generate experimental client SDKs. +use schemars::{generate::SchemaSettings, transform::RecursiveTransform, Schema}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; use std::sync::OnceLock; pub mod compatibility; +pub mod computer_use; pub mod cursor; mod cursor_tools; mod desktop; @@ -22,23 +24,41 @@ mod outputs; mod session; mod verification; +pub(crate) fn schema_settings() -> SchemaSettings { + SchemaSettings::draft2020_12() + .with(|settings| { + settings.meta_schema = None; + settings.inline_subschemas = true; + }) + .with_transform(RecursiveTransform(drop_schemars_numeric_format)) +} + +fn drop_schemars_numeric_format(schema: &mut Schema) { + if matches!( + schema.get("format").and_then(Value::as_str), + Some("uint32" | "uint64" | "double") + ) { + schema.remove("format"); + } +} + pub use cursor::{ classify_cursor_semantics, CursorAction, CursorDelivery, CursorPlayback, CursorReducedMotion, CursorSemantics, CursorTarget, CursorThemeSelection, }; pub use inputs::{ - action_target_schema, ActionTarget, CaptureScope, ClickButton, ClickInput, ClipboardReadInput, - ClipboardWriteInput, DeliveryMode, DesktopScope, DoubleClickInput, DragInput, EndSessionInput, - EscalateSessionInput, EscalationReason, GetAgentCursorStateInput, GetCursorPositionInput, - GetDesktopStateInput, GetScreenSizeInput, GetSessionInput, GetSessionStateInput, - GetWindowStateInput, HotkeyInput, InvokeMenuInput, LaunchAppInput, ListAppsInput, - ListSessionsInput, ListWindowsInput, MoveCursorInput, ObservationRevisionInput, PasteFormat, - PasteInput, PerformSecondaryActionInput, PressKeyInput, RightClickInput, ScrollBy, - ScrollDirection, ScrollInput, SelectTextInput, SetAgentCursorEnabledInput, - SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetValueInput, SetWindowFrameInput, - StartSessionInput, TextSelection, ToolInput, TypeTextInput, WindowClickInput, WindowDragInput, - WindowHotkeyInput, WindowPressKeyInput, WindowScrollInput, WindowTypeTextInput, - MULTI_CALL_SESSION_DESCRIPTION, + action_target_schema, normalize_input_schema, ActionTarget, CaptureScope, ClickButton, + ClickInput, ClipboardReadInput, ClipboardWriteInput, DeliveryMode, DesktopScope, + DoubleClickInput, DragInput, EndSessionInput, EscalateSessionInput, EscalationReason, + GetAgentCursorStateInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput, + GetSessionInput, GetSessionStateInput, GetWindowStateInput, HotkeyInput, InvokeMenuInput, + LaunchAppInput, ListAppsInput, ListSessionsInput, ListWindowsInput, MoveCursorInput, + ObservationRevisionInput, PasteFormat, PasteInput, PerformSecondaryActionInput, PressKeyInput, + RightClickInput, ScrollBy, ScrollDirection, ScrollInput, SelectTextInput, + SetAgentCursorEnabledInput, SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetValueInput, + SetWindowFrameInput, StartSessionInput, TextSelection, ToolInput, TypeTextInput, + WindowClickInput, WindowDragInput, WindowHotkeyInput, WindowPressKeyInput, WindowScrollInput, + WindowTypeTextInput, MULTI_CALL_SESSION_DESCRIPTION, }; pub use outputs::{ advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode, @@ -347,6 +367,72 @@ mod tests { ); } + #[derive(Serialize, Deserialize)] + struct NumericFormatFixture; + + impl schemars::JsonSchema for NumericFormatFixture { + fn schema_name() -> std::borrow::Cow<'static, str> { + "NumericFormatFixture".into() + } + + fn json_schema(_: &mut schemars::SchemaGenerator) -> Schema { + schemars::json_schema!({ + "type": "object", + "properties": { + "count": {"type": "integer", "format": "uint32"}, + "frame": {"type": "integer", "format": "uint64"}, + "ratio": {"type": "number", "format": "double"}, + "created_at": {"type": "string", "format": "date-time"}, + "format": {"type": "string"}, + "annotation": { + "const": {"format": "uint32"}, + "default": {"format": "uint64"}, + "examples": [{"format": "double"}] + } + }, + "required": ["format"] + }) + } + } + + impl ToolInput for NumericFormatFixture { + const TOOL_NAME: &'static str = "numeric_format_fixture"; + } + + impl ToolOutput for NumericFormatFixture {} + + #[test] + fn input_and_output_schemas_drop_only_schemars_numeric_formats() { + let schemas = [ + ::input_schema(), + ::output_schema(), + ]; + + for schema in schemas { + for property in ["count", "frame", "ratio"] { + assert!(schema["properties"][property].get("format").is_none()); + } + assert_eq!(schema["properties"]["created_at"]["format"], "date-time"); + assert_eq!( + schema["properties"]["format"], + serde_json::json!({"type": "string"}) + ); + assert_eq!(schema["required"], serde_json::json!(["format"])); + assert_eq!( + schema["properties"]["annotation"]["const"], + serde_json::json!({"format": "uint32"}) + ); + assert_eq!( + schema["properties"]["annotation"]["default"], + serde_json::json!({"format": "uint64"}) + ); + assert_eq!( + schema["properties"]["annotation"]["examples"], + serde_json::json!([{"format": "double"}]) + ); + } + } + #[test] fn manifest_is_sorted_and_versioned() { let manifest = manifest(); @@ -402,8 +488,8 @@ mod tests { ); assert_eq!( contract.input_schema["properties"]["expect"]["items"]["properties"]["element"] - ["properties"]["exists"]["enum"], - serde_json::json!([true]) + ["properties"]["exists"]["type"], + serde_json::json!("boolean") ); } diff --git a/packages/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/packages/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs index bc559b5c719..c295db15f89 100644 --- a/packages/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs +++ b/packages/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 Cua AI, Inc. -use crate::{CaptureScope, EscalationReason, Platform}; -use schemars::{generate::SchemaSettings, JsonSchema}; +use crate::{schema_settings, CaptureScope, EscalationReason, Platform}; +use schemars::JsonSchema; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::{Map, Value}; use std::collections::BTreeMap; @@ -56,11 +56,12 @@ pub fn advertised_output_schema(success: Value) -> Value { } fn output_schema_with_additional_properties(additional_properties: bool) -> Value { - let mut settings = SchemaSettings::draft2020_12(); - settings.inline_subschemas = true; - settings.meta_schema = None; - let mut schema = serde_json::to_value(settings.into_generator().into_root_schema_for::()) - .expect("JSON Schema serializes"); + let mut schema = serde_json::to_value( + schema_settings() + .into_generator() + .into_root_schema_for::(), + ) + .expect("JSON Schema serializes"); strip_schema_titles(&mut schema); if let Some(object) = schema.as_object_mut() { object.insert( diff --git a/packages/cua-driver/rust/crates/cua-driver-contract/src/verification.rs b/packages/cua-driver/rust/crates/cua-driver-contract/src/verification.rs index fb4c068bafe..40ebccc0395 100644 --- a/packages/cua-driver/rust/crates/cua-driver-contract/src/verification.rs +++ b/packages/cua-driver/rust/crates/cua-driver-contract/src/verification.rs @@ -52,7 +52,7 @@ fn nonempty_string_schema(_: &mut SchemaGenerator) -> Schema { } fn true_only_boolean_schema(_: &mut SchemaGenerator) -> Schema { - json_schema!({ "type": "boolean", "enum": [true] }) + json_schema!({ "type": "boolean", "description": "Must be true. Absence cannot be proven; false is rejected." }) } fn nullable_string_schema(_: &mut SchemaGenerator) -> Schema { diff --git a/packages/cua-driver/rust/crates/cua-driver-core/Cargo.toml b/packages/cua-driver/rust/crates/cua-driver-core/Cargo.toml index 8171b241c82..958c8f051f6 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/Cargo.toml +++ b/packages/cua-driver/rust/crates/cua-driver-core/Cargo.toml @@ -39,6 +39,7 @@ uuid = { workspace = true } base64 = { workspace = true } sha2 = "0.10" url = "2.5" +time = "0.3" [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/action_record.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/action_record.rs index 8240c220c5f..6bcade7bffd 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/action_record.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/action_record.rs @@ -59,6 +59,8 @@ pub enum ActualDelivery { #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum ActionTransport { AgentCursorOverlay, + #[cfg(target_os = "macos")] + MacosAppleEvents, MacosAxAction, MacosAxValue, MacosAxWindowFrame, @@ -81,11 +83,9 @@ pub enum ActionTransport { LinuxAtSpiAction, LinuxAtSpiValue, LinuxPty, + LinuxUinput, LinuxXSendEvent, LinuxXTest, - LinuxLibei, - LinuxWaylandVirtualPointer, - LinuxCuaCompositorInject, LinuxX11ConfigureWindow, BrowserCdpInputMouse, BrowserCdpInputKey, @@ -95,6 +95,8 @@ pub enum ActionTransport { impl ActionTransport { pub const ALL: &'static [Self] = &[ Self::AgentCursorOverlay, + #[cfg(target_os = "macos")] + Self::MacosAppleEvents, Self::MacosAxAction, Self::MacosAxValue, Self::MacosAxWindowFrame, @@ -117,11 +119,9 @@ impl ActionTransport { Self::LinuxAtSpiAction, Self::LinuxAtSpiValue, Self::LinuxPty, + Self::LinuxUinput, Self::LinuxXSendEvent, Self::LinuxXTest, - Self::LinuxLibei, - Self::LinuxWaylandVirtualPointer, - Self::LinuxCuaCompositorInject, Self::LinuxX11ConfigureWindow, Self::BrowserCdpInputMouse, Self::BrowserCdpInputKey, @@ -131,6 +131,8 @@ impl ActionTransport { pub const fn route(self) -> ActionRoute { match self { Self::AgentCursorOverlay => ActionRoute::SyntheticEvents, + #[cfg(target_os = "macos")] + Self::MacosAppleEvents => ActionRoute::SystemApi, Self::MacosAxAction | Self::MacosAxValue | Self::MacosAxWindowFrame @@ -147,16 +149,14 @@ impl ActionTransport { Self::MacosCgEventPid | Self::WindowsPostMessage | Self::LinuxPty + | Self::LinuxUinput | Self::LinuxXSendEvent => ActionRoute::SyntheticEvents, Self::MacosCgEventHid | Self::WindowsTargetedInjection | Self::WindowsSendInput | Self::WindowsSetCursorPos | Self::WindowsShellExecute - | Self::LinuxXTest - | Self::LinuxLibei - | Self::LinuxWaylandVirtualPointer - | Self::LinuxCuaCompositorInject => ActionRoute::GlobalInput, + | Self::LinuxXTest => ActionRoute::GlobalInput, Self::WindowsSetWindowPos | Self::LinuxX11ConfigureWindow => ActionRoute::SystemApi, Self::BrowserCdpRuntimeFunction => ActionRoute::Dom, Self::BrowserCdpInputMouse | Self::BrowserCdpInputKey => ActionRoute::TrustedInput, @@ -452,15 +452,13 @@ impl ActionExecutionRecord { if legacy_has_publishable_readback(tool_name, structured) { record.evidence.push(ActionEvidence { - kind: if matches!( - transport, + kind: match transport { + #[cfg(target_os = "macos")] + ActionTransport::MacosAppleEvents => EvidenceKind::ValueReadback, ActionTransport::BrowserCdpInputMouse - | ActionTransport::BrowserCdpInputKey - | ActionTransport::BrowserCdpRuntimeFunction - ) { - EvidenceKind::BrowserReadback - } else { - EvidenceKind::AccessibilityReadback + | ActionTransport::BrowserCdpInputKey + | ActionTransport::BrowserCdpRuntimeFunction => EvidenceKind::BrowserReadback, + _ => EvidenceKind::AccessibilityReadback, }, detail: structured .get("verify") @@ -739,6 +737,8 @@ fn parse_actual_delivery_name(value: &str) -> Result { fn parse_transport_name(value: &str) -> Result { let transport = match value { "agent_cursor_overlay" => ActionTransport::AgentCursorOverlay, + #[cfg(target_os = "macos")] + "macos_apple_events" => ActionTransport::MacosAppleEvents, "macos_ax_action" => ActionTransport::MacosAxAction, "macos_ax_value" => ActionTransport::MacosAxValue, "macos_ax_window_frame" => ActionTransport::MacosAxWindowFrame, @@ -761,11 +761,9 @@ fn parse_transport_name(value: &str) -> Result { "linux_at_spi_action" => ActionTransport::LinuxAtSpiAction, "linux_at_spi_value" => ActionTransport::LinuxAtSpiValue, "linux_pty" => ActionTransport::LinuxPty, + "linux_uinput" => ActionTransport::LinuxUinput, "linux_x_send_event" => ActionTransport::LinuxXSendEvent, "linux_x_test" => ActionTransport::LinuxXTest, - "linux_libei" => ActionTransport::LinuxLibei, - "linux_wayland_virtual_pointer" => ActionTransport::LinuxWaylandVirtualPointer, - "linux_cua_compositor_inject" => ActionTransport::LinuxCuaCompositorInject, "linux_x11_configure_window" => ActionTransport::LinuxX11ConfigureWindow, "browser_cdp_input_mouse" => ActionTransport::BrowserCdpInputMouse, "browser_cdp_input_key" => ActionTransport::BrowserCdpInputKey, @@ -885,8 +883,13 @@ fn browser_refusal_escalation(code: &str) -> Option { /// against the request. A click may do so only when the producer also declares /// explicit accessibility read-back evidence, as the macOS collection-item /// selection fallback does after observing `AXSelected`. +/// Native Apple Events save also compares the document's modified state. fn legacy_has_publishable_readback(tool_name: &str, structured: &serde_json::Value) -> bool { let value_readback_tool = matches!(tool_name, "type_text" | "type_text_chars" | "set_value"); + #[cfg(target_os = "macos")] + let value_readback_tool = value_readback_tool + || (tool_name == "hotkey" + && structured.get("path").and_then(serde_json::Value::as_str) == Some("apple_events")); let click_selection_readback = tool_name == "click" && structured .get("evidence") @@ -913,6 +916,8 @@ fn transport_from_legacy( ) -> Option { let path = raw_path.unwrap_or(""); let transport = match path { + #[cfg(target_os = "macos")] + "apple_events" => ActionTransport::MacosAppleEvents, "ax" | "ax_fg" => { if cfg!(target_os = "macos") { if matches!(tool_name, "type_text" | "type_text_chars" | "set_value") { @@ -941,16 +946,11 @@ fn transport_from_legacy( "post_message" | "PostMessage" => ActionTransport::WindowsPostMessage, "SendInput" => ActionTransport::WindowsSendInput, "SetCursorPos" => ActionTransport::WindowsSetCursorPos, - "atspi" | "wayland_atspi" | "x11_atspi" => ActionTransport::LinuxAtSpiAction, + "atspi" | "x11_atspi" => ActionTransport::LinuxAtSpiAction, "pty" => ActionTransport::LinuxPty, "x11_pixel" | "x11_pixel_fg" | "x11_xtest_fg" | "xtest" | "xtest_desktop" => { ActionTransport::LinuxXTest } - "wayland_activate" | "wayland_focused" => ActionTransport::LinuxLibei, - "wayland_desktop" => ActionTransport::LinuxWaylandVirtualPointer, - "cua_compositor_inject" | "wayland_cua_compositor" => { - ActionTransport::LinuxCuaCompositorInject - } "hid" | "cgevent_hid" | "cgevent_fg" => ActionTransport::MacosCgEventHid, "cgevent" => { if args @@ -1138,9 +1138,7 @@ fn actual_delivery_from_legacy( } match raw_path { Some(path) if path.ends_with("_fg") => Some(ActualDelivery::Foreground), - Some("hid" | "cgevent_hid" | "SendInput" | "wayland_activate" | "wayland_focused") => { - Some(ActualDelivery::Foreground) - } + Some("hid" | "cgevent_hid" | "SendInput") => Some(ActualDelivery::Foreground), Some(_) => Some(ActualDelivery::Background), None => Some(ActualDelivery::Unknown), } @@ -1262,6 +1260,8 @@ fn escalation_kind_name(kind: EscalationKind) -> &'static str { fn transport_name(transport: ActionTransport) -> &'static str { match transport { ActionTransport::AgentCursorOverlay => "agent_cursor_overlay", + #[cfg(target_os = "macos")] + ActionTransport::MacosAppleEvents => "macos_apple_events", ActionTransport::MacosAxAction => "macos_ax_action", ActionTransport::MacosAxValue => "macos_ax_value", ActionTransport::MacosAxWindowFrame => "macos_ax_window_frame", @@ -1284,11 +1284,9 @@ fn transport_name(transport: ActionTransport) -> &'static str { ActionTransport::LinuxAtSpiAction => "linux_at_spi_action", ActionTransport::LinuxAtSpiValue => "linux_at_spi_value", ActionTransport::LinuxPty => "linux_pty", + ActionTransport::LinuxUinput => "linux_uinput", ActionTransport::LinuxXSendEvent => "linux_x_send_event", ActionTransport::LinuxXTest => "linux_x_test", - ActionTransport::LinuxLibei => "linux_libei", - ActionTransport::LinuxWaylandVirtualPointer => "linux_wayland_virtual_pointer", - ActionTransport::LinuxCuaCompositorInject => "linux_cua_compositor_inject", ActionTransport::LinuxX11ConfigureWindow => "linux_x11_configure_window", ActionTransport::BrowserCdpInputMouse => "browser_cdp_input_mouse", ActionTransport::BrowserCdpInputKey => "browser_cdp_input_key", @@ -1443,6 +1441,10 @@ mod tests { .unwrap(); let route = record.stable_projection().unwrap().route; assert_eq!(route, transport.route()); + assert_eq!( + ActionExecutionRecord::from_private_json(&record.debug_json()).unwrap(), + record + ); routes.push(route); } assert!(routes.contains(&ActionRoute::Accessibility)); @@ -1565,7 +1567,7 @@ mod tests { fn projection_contains_only_outcome_information() { let projection = ActionExecutionRecord::builder( ActionEffect::Confirmed, - ActionTransport::LinuxLibei, + ActionTransport::LinuxXTest, RequestedDelivery::Foreground, ) .actual_delivery(ActualDelivery::Foreground) @@ -1730,6 +1732,41 @@ mod tests { } } + #[test] + #[cfg(target_os = "macos")] + fn apple_event_native_readback_preserves_the_actual_background_system_route() { + for tool in ["hotkey", "set_value"] { + let args = serde_json::json!({"pid":101,"window_id":501,"delivery_mode":"foreground"}); + let result = serde_json::json!({ + "path":"apple_events", "verified":true, "effect":"confirmed", + "delivery_mode":"background" + }); + let record = ActionExecutionRecord::from_legacy(tool, &args, &result) + .expect("the native save/rename producer has independent readback"); + assert_eq!(record.requested_delivery, RequestedDelivery::Foreground); + assert_eq!(record.actual_delivery, Some(ActualDelivery::Background)); + let public = record.public_result().unwrap(); + public.validate_invariants().unwrap(); + let public = serde_json::to_value(public).unwrap(); + assert_eq!(public["effect"], "confirmed"); + assert_eq!(public["route"], "system_api"); + assert_eq!(public["delivery"]["mode"], "background"); + assert_eq!(public["evidence"][0]["kind"], "value_readback"); + + let mut unverified = result.clone(); + unverified["verified"] = serde_json::json!(false); + let record = ActionExecutionRecord::from_legacy(tool, &args, &unverified).unwrap(); + assert_eq!(record.effect, ActionEffect::Unverifiable); + assert!(record.evidence.is_empty()); + + let mut acknowledgement = result.clone(); + acknowledgement.as_object_mut().unwrap().remove("effect"); + let record = ActionExecutionRecord::from_legacy(tool, &args, &acknowledgement).unwrap(); + assert_eq!(record.effect, ActionEffect::Unverifiable); + assert!(record.evidence.is_empty()); + } + } + #[test] fn system_api_value_readback_projects_without_transport_details() { let result = ActionExecutionRecord::builder( @@ -2079,7 +2116,6 @@ mod tests { "SendInput", "SetCursorPos", "atspi", - "wayland_atspi", "x11_atspi", "pty", "x11_pixel", @@ -2087,11 +2123,6 @@ mod tests { "x11_xtest_fg", "xtest", "xtest_desktop", - "wayland_activate", - "wayland_focused", - "wayland_desktop", - "cua_compositor_inject", - "wayland_cua_compositor", "hid", "cgevent_hid", "cgevent", diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/authorization.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/authorization.rs index 633664faab2..e201236f763 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/authorization.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/authorization.rs @@ -270,6 +270,8 @@ const DESKTOP_INPUT_OPERATIONS: &[&str] = &[ "type_text", "type_text_chars", "press_key", + "key_down", + "key_up", "hotkey", "set_value", "paste", @@ -859,6 +861,8 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment { | "type_text" | "type_text_chars" | "press_key" + | "key_down" + | "key_up" | "hotkey" | "set_value" | "paste" @@ -882,6 +886,11 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment { "zoom" | "clipboard_read" | "list_apps" + | "get_state" + | "get_app" + | "get_ax_state" + | "get_screenshot" + | "get_ax_state_and_screenshot" | "list_windows" | "debug_window_info" | "check_permissions" @@ -1153,6 +1162,8 @@ fn enforce_hard_invariants( | "type_text" | "type_text_chars" | "press_key" + | "key_down" + | "key_up" | "hotkey" | "set_value" | "paste" @@ -1474,6 +1485,19 @@ mod tests { assert_eq!(isolated.enforcement, RiskEnforcement::Active); } + #[test] + fn app_observation_aliases_have_reviewed_private_content_risk() { + for tool in [ + "get_state", + "get_app", + "get_ax_state", + "get_screenshot", + "get_ax_state_and_screenshot", + ] { + assert_eq!(advertised_risk_for(tool).class, RiskClass::R2, "{tool}"); + } + } + #[test] fn unknown_tools_are_unclassified_and_denied() { let error = authorize_tool_call("new_unreviewed_tool", &serde_json::json!({})) diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs index 903188a1104..04eeeea430c 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs @@ -328,15 +328,9 @@ fn cleanup_created_profile(profile: &PreparedProfile) { } #[cfg(target_os = "linux")] -fn configure_linux_isolated_browser_command( - command: &mut Command, - native_wayland: bool, - no_sandbox: bool, -) { +fn configure_linux_isolated_browser_command(command: &mut Command, no_sandbox: bool) { command.arg("--password-store=basic"); - if native_wayland { - command.arg("--ozone-platform=wayland"); - } + command.arg("--ozone-platform=x11"); if no_sandbox { command.arg("--no-sandbox"); } @@ -364,12 +358,8 @@ fn isolated_browser_command(executable: &str, profile: &Path) -> Command { .arg("--window-size=900,640"); #[cfg(target_os = "linux")] { - let native_wayland = std::env::var("XDG_SESSION_TYPE") - .is_ok_and(|session| session.eq_ignore_ascii_case("wayland")) - && std::env::var_os("WAYLAND_DISPLAY").is_some() - && std::env::var_os("DISPLAY").is_none(); let no_sandbox = std::env::var("CUA_E2E_BROWSER_NO_SANDBOX").as_deref() == Ok("1"); - configure_linux_isolated_browser_command(&mut command, native_wayland, no_sandbox); + configure_linux_isolated_browser_command(&mut command, no_sandbox); } let stderr = if std::env::var_os("CUA_E2E_BROWSER_STDERR").is_some() { Stdio::inherit() @@ -1444,15 +1434,15 @@ mod tests { #[cfg(target_os = "linux")] #[test] - fn isolated_launch_can_select_native_wayland_and_test_vm_sandbox_mode() { + fn isolated_launch_selects_x11_and_test_vm_sandbox_mode() { let mut command = Command::new("chromium-under-test"); - configure_linux_isolated_browser_command(&mut command, true, true); + configure_linux_isolated_browser_command(&mut command, true); let args = command .get_args() .map(|arg| arg.to_string_lossy().into_owned()) .collect::>(); assert!(args.iter().any(|arg| arg == "--password-store=basic")); - assert!(args.iter().any(|arg| arg == "--ozone-platform=wayland")); + assert!(args.iter().any(|arg| arg == "--ozone-platform=x11")); assert!(args.iter().any(|arg| arg == "--no-sandbox")); } } diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/capture_scope.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/capture_scope.rs index ecf5cb787e2..554b283f969 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/capture_scope.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/capture_scope.rs @@ -172,6 +172,8 @@ fn is_scoped_action(tool_name: &str) -> bool { | "type_text" | "type_text_chars" | "press_key" + | "key_down" + | "key_up" | "hotkey" | "set_value" | "paste" diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/computer_use.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/computer_use.rs new file mode 100644 index 00000000000..93e977529d1 --- /dev/null +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/computer_use.rs @@ -0,0 +1,1418 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use serde_json::{json, Value}; + +use crate::protocol::{Content, ToolResult}; +use crate::tool::{DispatchSessionIdentity, Tool, ToolDef, ToolRegistry}; + +pub(crate) struct Entry(ToolDef); + +impl Entry { + pub(crate) fn new(spec: cua_driver_contract::computer_use::ComputerUseTool) -> Self { + Self(ToolDef { + name: spec.name.into(), + description: spec.description.into(), + input_schema: spec.input_schema, + read_only: spec.read_only, + destructive: false, + idempotent: spec.read_only, + open_world: !spec.read_only, + }) + } +} + +#[async_trait::async_trait] +impl Tool for Entry { + fn def(&self) -> &ToolDef { + &self.0 + } + async fn invoke(&self, _args: Value) -> ToolResult { + error( + "unsupported_on_platform", + "This app operation has no native implementation on this platform.", + false, + ) + } +} + +#[derive(Default)] +pub(crate) struct ComputerUseRuntime { + apps: Mutex>>>, + desktops: + Mutex>>>>, +} + +#[derive(Clone, Copy)] +struct DesktopFrame { + image: [f64; 2], + screen: [f64; 2], + scale: f64, +} + +impl DesktopFrame { + fn from_capture(value: &Value) -> Option { + let frame = Self { + image: [ + value["screenshot_width"].as_f64()?, + value["screenshot_height"].as_f64()?, + ], + screen: [ + value["screen_width"].as_f64()?, + value["screen_height"].as_f64()?, + ], + scale: value["scale_factor"].as_f64().unwrap_or(1.0), + }; + let valid = frame + .image + .iter() + .chain(frame.screen.iter()) + .chain([&frame.scale]) + .all(|n| n.is_finite() && *n > 0.0); + valid.then_some(frame) + } + fn map(&self, input: &mut Value, x: &str, y: &str) -> Result<(), ToolResult> { + for (axis, key) in [x, y].into_iter().enumerate() { + let value = input[key] + .as_f64() + .filter(|n| n.is_finite() && *n >= 0.0 && *n < self.image[axis]) + .ok_or_else(|| { + error( + "invalid_arguments", + format!("{key} must be inside the current desktop screenshot."), + false, + ) + })?; + input[key] = json!(value * self.screen[axis] / self.image[axis]); + } + Ok(()) + } +} + +#[derive(Default)] +struct AppState { + window: Option<(u64, u64)>, + revision: Option, + lineage: Option, + ids: HashMap, + next_id: u64, + tokens: HashMap, + screenshot: bool, + last_action: Option, +} + +pub(crate) fn error(code: &str, reason: impl Into, sent: bool) -> ToolResult { + ToolResult::computer_use_error(code, reason, sent) +} + +fn text(result: &ToolResult) -> String { + result + .content + .iter() + .filter_map(|part| match part { + Content::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n") +} + +fn checked(result: ToolResult, mutating: bool) -> Result { + if result.is_error != Some(true) { + return Ok(result); + } + let structured = result.structured_content.as_ref(); + let message = text(&result); + let code = structured + .and_then(|value| value.get("code").or_else(|| value.pointer("/refusal/code"))) + .and_then(Value::as_str) + .or_else(|| { + [ + "no_focused_text_input", + "not_settable", + "write_not_applied", + "user_input_detected", + "foreground_unavailable", + "target_occluded", + "text_delivery_stalled", + ] + .into_iter() + .find(|code| message.contains(&format!("{code}:"))) + }) + .unwrap_or("operation_failed"); + let stale = matches!( + code, + "stale_element_token" + | "invalid_element_token" + | "generation_mismatch" + | "snapshot_id_required" + ); + let pre_dispatch = stale + || matches!( + code, + "no_focused_text_input" | "not_settable" | "foreground_unavailable" | "target_occluded" + ); + let sent = structured + .and_then(|value| value.get("sent")) + .and_then(Value::as_bool) + .unwrap_or(mutating && !pre_dispatch); + Err(error( + if stale { "element_stale" } else { code }, + message, + sent, + )) +} + +fn authorize_app( + name: &str, + args: &Value, + pid: Option, + window: Option, +) -> Result<(), ToolResult> { + let mut resolved = args.clone(); + if let Some(pid) = pid { + resolved["pid"] = json!(pid); + } + if let Some(window) = window { + resolved["window_id"] = json!(window); + } + crate::policy::authorize_tool_call(name, &resolved) + .map_err(|reason| error("permission_denied", reason.to_string(), false)) +} + +fn validate(name: &str, args: &Value) -> Result<(), ToolResult> { + let spec = cua_driver_contract::computer_use::tools() + .into_iter() + .find(|tool| tool.name == name) + .ok_or_else(|| error("unsupported_on_platform", "Unknown app operation.", false))?; + let Some(input) = args.as_object() else { + return Err(error( + "invalid_arguments", + "Arguments must be an object.", + false, + )); + }; + let properties = spec.input_schema["properties"].as_object().unwrap(); + let desktop = input.get("scope").is_some_and(|scope| scope == "desktop"); + if name != "get_state" + && (desktop == input.contains_key("app") + || matches!(name, "key_down" | "key_up") && !desktop) + { + return Err(error( + "invalid_arguments", + "Select exactly one app or scope:desktop. key_down/key_up require desktop scope.", + false, + )); + } + if desktop && input.contains_key("element") { + return Err(error( + "invalid_arguments", + "Desktop input requires screenshot coordinates, not an app element ID.", + false, + )); + } + for field in spec.input_schema["required"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + { + if !input.contains_key(field) { + return Err(error( + "invalid_arguments", + format!("{field} is required."), + false, + )); + } + } + for (field, value) in input { + let schema = properties.get(field).ok_or_else(|| { + error( + "invalid_arguments", + format!("Unknown app option {field}."), + false, + ) + })?; + let typed = match schema["type"].as_str() { + Some("string") => value.is_string(), + Some("integer") => value.is_i64() || value.is_u64(), + Some("number") => value.is_number(), + Some("boolean") => value.is_boolean(), + Some("array") => { + // The JS click API also accepts one modifier as a string. + name == "click" && field == "key" && value.is_string() + || value + .as_array() + .is_some_and(|values| values.iter().all(Value::is_string)) + } + _ => { + value.is_string() + || value + .as_array() + .is_some_and(|values| values.iter().all(Value::is_string)) + } + }; + let range = value.as_f64().is_none_or(|number| { + schema["minimum"] + .as_f64() + .is_none_or(|minimum| number >= minimum) + && schema["maximum"] + .as_f64() + .is_none_or(|maximum| number <= maximum) + }); + let enumeration = schema["enum"] + .as_array() + .is_none_or(|choices| choices.contains(value)); + let length = value.as_str().is_none_or(|text| { + schema["minLength"] + .as_u64() + .is_none_or(|min| text.chars().count() >= min as usize) + }) && value.as_array().is_none_or(|values| { + schema["minItems"] + .as_u64() + .is_none_or(|min| values.len() >= min as usize) + }); + if !typed || !range || !enumeration || !length { + return Err(error( + "invalid_arguments", + format!("Invalid value for {field}."), + false, + )); + } + } + if input.contains_key("element") && (input.contains_key("x") || input.contains_key("y")) { + return Err(error( + "invalid_arguments", + "Use an element ID or coordinates, not both.", + false, + )); + } + Ok(()) +} + +fn normalized_path(value: &str) -> Option { + let windows = value.as_bytes().get(1) == Some(&b':') || value.starts_with("\\\\"); + if !value.starts_with('/') && !windows { + return None; + } + let value = if windows { + value.replace('\\', "/").to_lowercase() + } else { + value.to_owned() + }; + let mut components = Vec::new(); + for component in value.split('/') { + match component { + "." => {} + ".." if components.last().is_some_and(|last| *last != "") => { + components.pop(); + } + ".." => {} + _ => components.push(component), + } + } + Some(components.join("/").trim_end_matches('/').to_owned()) +} + +fn identity(app: &Value) -> String { + let launch = app["launch_path"].as_str().unwrap_or_default(); + let bundle = app["bundle_id"].as_str().unwrap_or_default(); + if opaque_launch(app) { + return launch.to_owned(); + } + normalized_path(launch) + .or_else(|| (!bundle.is_empty()).then(|| bundle.to_owned())) + .unwrap_or_else(|| app["name"].as_str().unwrap_or_default().to_owned()) +} + +fn opaque_launch(app: &Value) -> bool { + let launch = app["launch_path"].as_str().unwrap_or_default(); + let bundle = app["bundle_id"].as_str().unwrap_or_default(); + !bundle.starts_with('/') + && normalized_path(bundle).is_some() + && !launch.is_empty() + && launch != bundle +} + +fn resolve_app(apps: &Value, selector: &str) -> Result { + if selector.trim().is_empty() { + return Err(error( + "app_not_found", + "An exact application selector is required.", + false, + )); + } + let path = normalized_path(selector); + let needle = selector.trim().to_lowercase(); + let matches = apps + .as_array() + .into_iter() + .flatten() + .filter(|app| { + identity(app) == selector + || if let Some(path) = &path { + ["launch_path", "bundle_id"] + .iter() + .filter(|key| **key != "launch_path" || !opaque_launch(app)) + .any(|key| { + app[*key].as_str().and_then(normalized_path).as_ref() == Some(path) + }) + } else { + ["name", "bundle_id"].iter().any(|key| { + app[*key] + .as_str() + .is_some_and(|value| value.to_lowercase() == needle) + }) + } + }) + .collect::>(); + match matches.as_slice() { + [] => Err(error( + "app_not_found", + format!("No application matches {selector:?}."), + false, + )), + [app] => Ok((*app).clone()), + _ => Err(error( + "app_ambiguous", + "Use a unique application identifier or absolute path.", + false, + )), + } +} + +impl AppState { + fn select_window(&mut self, pid: u64, windows: &Value) -> Result { + let selected = windows + .as_array() + .into_iter() + .flatten() + .filter(|window| window["is_app_target"] == true) + .collect::>(); + let [window] = selected.as_slice() else { + self.invalidate(); + return Err(error( + "window_not_found", + "The app has no unambiguous selected window.", + false, + )); + }; + let Some(window_id) = window["window_id"].as_u64() else { + return Err(error( + "window_not_found", + "The selected window has no native identity.", + false, + )); + }; + let key = (window["pid"].as_u64().unwrap_or(pid), window_id); + if self.window != Some(key) { + self.invalidate(); + self.window = Some(key); + } + Ok((*window).clone()) + } + + fn invalidate(&mut self) { + self.window = None; + self.revision = None; + self.lineage = None; + self.ids.clear(); + self.tokens.clear(); + self.screenshot = false; + } + + fn id(&mut self, native: u64) -> u64 { + *self.ids.entry(native).or_insert_with(|| { + self.next_id += 1; + self.next_id + }) + } + + fn render(&mut self, raw: &str, budget: usize) -> String { + let mut lines = Vec::new(); + let mut chars = 0; + for line in raw.lines() { + let start = line.find('[').filter(|start| { + line[..*start] + .chars() + .all(|c| c.is_whitespace() || matches!(c, '+' | '-' | '~')) + }); + let rendered = if let Some(start) = start { + let end = line[start..].find(']').map(|end| end + start); + if let Some((end, native)) = + end.and_then(|end| line[start + 1..end].parse::().ok().map(|id| (end, id))) + { + format!( + "{}[{}]{}", + &line[..start], + self.id(native), + &line[end + 1..] + ) + } else { + line.to_owned() + } + } else { + line.to_owned() + }; + if chars + rendered.chars().count() + 1 > budget.saturating_sub(100) { + lines.push( + "Text truncated. Request force_full:true with a larger max_text_chars.".into(), + ); + break; + } + chars += rendered.chars().count() + 1; + lines.push(rendered); + } + lines.join("\n") + } +} + +impl ComputerUseRuntime { + pub(crate) fn clear_session(&self, session: &str) { + self.apps + .lock() + .unwrap() + .retain(|(owner, _), _| owner.session_id != session); + self.desktops.lock().unwrap().retain(|owner, _| { + owner.session_id != session && owner.transport_session_id != session + }); + } + + pub(crate) async fn invoke( + &self, + registry: &ToolRegistry, + name: &str, + args: Value, + ) -> ToolResult { + match self.invoke_inner(registry, name, args).await { + Ok(result) | Err(result) => result, + } + } + + async fn invoke_inner( + &self, + registry: &ToolRegistry, + name: &str, + args: Value, + ) -> Result { + validate(name, &args)?; + if args["scope"] == "desktop" { + return self.desktop(registry, name, args).await; + } + if name == "get_state" { + authorize_app(name, &args, None, None)?; + } + let session = args.get("session").cloned(); + let call = |name: &'static str, mut input: Value| { + if let Some(session) = &session { + input["session"] = session.clone(); + } + Box::pin(registry.invoke_computer_use_native(name, input)) + }; + let apps = checked(call("list_apps", json!({})).await, false)?; + let apps = apps + .structured_content + .as_ref() + .and_then(|value| value.get("apps")) + .ok_or_else(|| { + error( + "app_not_found", + "The driver returned no application inventory.", + false, + ) + })?; + if name == "get_state" { + let inventory = checked(call("list_windows", json!({})).await, false)?; + let windows = inventory.structured_content.as_ref() + .and_then(|value| value["windows"].as_array()).into_iter().flatten() + .map(|window| { + let app = apps.as_array().into_iter().flatten() + .find(|app| app["pid"] == window["pid"]); + json!({"app":app.map(identity),"title":window["title"],"minimized":window["minimized"]}) + }).collect::>(); + let apps = apps + .as_array() + .into_iter() + .flatten() + .map(|app| { + json!({ + "id":identity(app),"displayName":app["name"],"isRunning":app["running"], + }) + }) + .collect::>(); + let inventory = json!({"apps":apps,"windows":windows}); + return Ok( + ToolResult::text(serde_json::to_string_pretty(&inventory).unwrap()) + .with_structured(inventory), + ); + } + let selector = args + .get("app") + .and_then(Value::as_str) + .ok_or_else(|| error("app_not_found", "Provide an application selector.", false))?; + let observing = matches!( + name, + "get_app" | "get_ax_state" | "get_screenshot" | "get_ax_state_and_screenshot" + ); + let mut app = resolve_app(apps, selector)?; + let owner = crate::tool::current_dispatch_session_identity().ok_or_else(|| { + error( + "session_unavailable", + "A bound driver session is required.", + false, + ) + })?; + let state = self + .apps + .lock() + .unwrap() + .entry((owner, identity(&app))) + .or_default() + .clone(); + let mut state = state.lock().await; + if app["running"] == false || app["pid"].as_u64().unwrap_or(0) == 0 { + state.invalidate(); + if name != "get_app" { + return Err(error( + "app_not_running", + "Actions do not relaunch a stopped app.", + false, + )); + } + authorize_app(name, &args, None, None)?; + let launch = app["launch_path"] + .as_str() + .or_else(|| app["bundle_id"].as_str()) + .or_else(|| app["name"].as_str()) + .unwrap_or(selector); + let mut input = json!({"name":launch}); + if cfg!(target_os = "windows") { + input = json!({"name":app["name"], "launch_path":launch}); + } + checked(call("launch_app", input).await, true)?; + let apps = checked(call("list_apps", json!({"running_only":true})).await, false)?; + app = resolve_app( + &apps + .structured_content + .as_ref() + .map(|value| value["apps"].clone()) + .unwrap_or(Value::Null), + selector, + )?; + } + let pid = app["pid"] + .as_u64() + .filter(|pid| *pid > 0) + .ok_or_else(|| error("app_not_running", "The app has no running process.", false))?; + let readiness_deadline = (observing && name != "get_screenshot") + .then(|| std::time::Instant::now() + crate::tool::AX_READINESS_BUDGET); + let call = |name: &'static str, mut input: Value| { + if let Some(session) = &session { + input["session"] = session.clone(); + } + Box::pin(registry.invoke_computer_use_native_with_readiness( + name, + input, + readiness_deadline, + )) + }; + let mut windows = checked( + call("list_windows", json!({"pid":pid,"app_context":true})).await, + false, + )?; + if observing { + if let Some(action) = state.last_action.take() { + let deadline = action + std::time::Duration::from_millis(400); + let mut quiet = 0; + while std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let next = checked( + call("list_windows", json!({"pid":pid,"app_context":true})).await, + false, + )?; + if next.structured_content == windows.structured_content { + quiet += 1; + } else { + quiet = 0; + } + windows = next; + if quiet >= 2 && action.elapsed() >= std::time::Duration::from_millis(300) { + break; + } + } + } + } + let window = state.select_window( + pid, + &windows + .structured_content + .as_ref() + .map(|value| value["windows"].clone()) + .unwrap_or(Value::Null), + )?; + let (pid, window_id) = state.window.expect("selected window"); + authorize_app(name, &args, Some(pid), Some(window_id))?; + if observing { + let budget = args + .get("max_text_chars") + .and_then(Value::as_u64) + .unwrap_or(12000); + if !(512..=1000000).contains(&budget) { + return Err(error( + "invalid_arguments", + "max_text_chars must be between 512 and 1000000.", + false, + )); + } + let screenshot_only = name == "get_screenshot"; + let mut input = json!({"pid":pid,"window_id":window_id,"app_context":true, + "include_screenshot":name == "get_screenshot" || name == "get_ax_state_and_screenshot", + "include_accessibility":!screenshot_only}); + if !screenshot_only { + input["observation_revision"] = json!({"version":1, + "serializer_version":crate::observation_revision::ACCESSIBILITY_SERIALIZER_VERSION, + "projection_version":crate::observation_revision::APP_ACCESSIBILITY_PROJECTION_VERSION, + "force_full":name == "get_app" || args["force_full"] == true || state.revision.is_none()}); + if let Some(base) = &state.revision { + input["observation_revision"]["base_revision_id"] = json!(base); + } + for field in ["max_elements", "max_depth"] { + if let Some(value) = args.get(field) { + input[field] = value.clone(); + } + } + } + let observed = checked(call("get_window_state", input).await, false)?; + let structured = observed + .structured_content + .as_ref() + .cloned() + .unwrap_or_default(); + if !screenshot_only { + let lineage = structured["observation_revision"]["lineage_id"] + .as_str() + .map(str::to_owned); + if state.lineage != lineage || lineage.is_none() { + state.ids.clear(); + } + state.lineage = lineage; + state.tokens.clear(); + for element in structured["elements"].as_array().into_iter().flatten() { + if let Some((native, token)) = element["element_id"] + .as_u64() + .or_else(|| element["element_index"].as_u64()) + .zip(element["element_token"].as_str()) + { + let id = state.id(native); + state.tokens.insert(id, token.to_owned()); + } + } + state.revision = structured["observation_revision"]["revision_id"] + .as_str() + .map(str::to_owned); + } + state.screenshot = observed + .content + .iter() + .any(|part| matches!(part, Content::Image { .. })) + && structured["screenshot_frame_valid"] != false; + if (screenshot_only || name == "get_ax_state_and_screenshot") && !state.screenshot { + return Err(error( + "screenshot_unavailable", + "The driver could not establish a current image frame.", + false, + )); + } + let mode = structured["observation_revision"]["mode"] + .as_str() + .unwrap_or("full"); + let mut body = if screenshot_only { + "Window screenshot.".to_owned() + } else { + state.render( + structured["tree_markdown"] + .as_str() + .unwrap_or(&text(&observed)), + budget as usize, + ) + }; + if !screenshot_only && structured["capture_complete"] == false { + body = format!("Accessibility capture is incomplete; only current captured IDs are actionable. Use a screenshot or a larger capture budget if needed.\n{body}"); + } + if let Some(note) = structured["off_space_staleness"].as_str() { + body = format!("{note}\n{body}"); + } + let title = window["title"].as_str().unwrap_or_default(); + let mut metadata = + json!({"app":identity(&app),"name":app["name"],"window":title,"mode":mode}); + for field in ["on_current_space", "off_space_staleness"] { + if let Some(value) = structured.get(field) { + metadata[field] = value.clone(); + } + } + let mut result = ToolResult::text(format!( + "{} — {title} ({mode})\n{body}", + app["name"].as_str().unwrap_or(selector) + )) + .with_structured(metadata); + result.content.extend( + observed + .content + .into_iter() + .filter(|part| matches!(part, Content::Image { .. })), + ); + return Ok(result); + } + if let Some(element) = args.get("element") { + let token = element + .as_u64() + .and_then(|id| state.tokens.get(&id)) + .ok_or_else(|| { + error( + "element_stale", + "The ID is not actionable in this window's current observation.", + false, + ) + })?; + let mut input = args.clone(); + input["element_token"] = json!(token); + state.last_action = Some(std::time::Instant::now()); + return self.act(registry, name, input, pid, window_id).await; + } + if name == "drag" || args.get("x").is_some() || args.get("y").is_some() { + if !state.screenshot { + return Err(error( + "screenshot_required", + "Observe a screenshot of this window before using coordinates.", + false, + )); + } + if name != "drag" && (args["x"].as_f64().is_none() || args["y"].as_f64().is_none()) { + return Err(error( + "invalid_arguments", + "Provide both x and y screenshot coordinates.", + false, + )); + } + } else if matches!( + name, + "click" + | "double_click" + | "right_click" + | "set_value" + | "select_text" + | "perform_secondary_action" + ) { + return Err(error( + "element_stale", + "Provide a current element ID or screenshot point.", + false, + )); + } + state.last_action = Some(std::time::Instant::now()); + self.act(registry, name, args, pid, window_id).await + } + + async fn act( + &self, + registry: &ToolRegistry, + name: &str, + mut input: Value, + pid: u64, + window_id: u64, + ) -> Result { + input.as_object_mut().expect("object input").remove("app"); + input.as_object_mut().unwrap().remove("element"); + input["pid"] = json!(pid); + input["window_id"] = json!(window_id); + if let Some(native) = registry.get_def(name) { + for (key, value) in [ + ("app_context", json!(true)), + ("delivery_mode", json!("background")), + ] { + if native.input_schema["properties"].get(key).is_some() { + input[key] = value; + } + } + } + if name == "click" { + if let Some(key) = input.as_object_mut().unwrap().remove("key") { + input["modifier"] = if key.is_string() { json!([key]) } else { key }; + } + } + let mut result = Box::pin(registry.invoke_computer_use_native(name, input.clone())).await; + let refusal = result.structured_content.as_ref(); + if cfg!(any(target_os = "windows", target_os = "linux")) + && result.is_error == Some(true) + && refusal.is_some_and(|value| { + value["code"] == "background_unavailable" && value["sent"] == false + }) + { + input["delivery_mode"] = json!("foreground"); + result = Box::pin(registry.invoke_computer_use_native(name, input)).await; + } + Self::acknowledge(result) + } + + fn acknowledge(result: ToolResult) -> Result { + let result = checked(result, true)?; + let confirmed = result + .action_record + .as_ref() + .is_some_and(|record| record.effect == crate::action_record::ActionEffect::Confirmed); + let mut output = ToolResult::text(if confirmed { + "" + } else { + "Sent but not confirmed. Observe the target state before retrying." + }) + .with_structured(json!({"computer_use":1,"sent":true})); + output.action_record = result.action_record; + Ok(output) + } + + async fn desktop( + &self, + registry: &ToolRegistry, + name: &str, + mut input: Value, + ) -> Result { + authorize_app(name, &input, None, None)?; + let owner = crate::tool::current_dispatch_session_identity().ok_or_else(|| { + error( + "session_unavailable", + "A bound driver session is required.", + false, + ) + })?; + let state = self + .desktops + .lock() + .unwrap() + .entry(owner) + .or_default() + .clone(); + let mut frame = state.lock().await; + let session = input.get("session").cloned(); + let session_args = || { + session + .as_ref() + .map_or_else(|| json!({}), |id| json!({"session":id})) + }; + if name == "get_screenshot" { + *frame = None; + let result = checked( + Box::pin(registry.invoke_computer_use_native("get_desktop_state", session_args())) + .await, + false, + )?; + let captured = result + .structured_content + .as_ref() + .and_then(DesktopFrame::from_capture) + .filter(|_| { + result + .content + .iter() + .any(|part| matches!(part, Content::Image { .. })) + }) + .ok_or_else(|| { + error( + "screenshot_unavailable", + "The driver returned no valid desktop image geometry.", + false, + ) + })?; + *frame = Some(captured); + return Ok(result); + } + if matches!( + name, + "click" | "double_click" | "right_click" | "scroll" | "drag" + ) { + let captured = frame.ok_or_else(|| { + error( + "screenshot_required", + "Observe cua.desktop.getScreenshot() before using desktop coordinates.", + false, + ) + })?; + let geometry = checked( + Box::pin(registry.invoke_computer_use_native("get_screen_size", session_args())) + .await, + false, + )?; + let geometry = geometry.structured_content.unwrap_or_default(); + if geometry["width"].as_f64() != Some(captured.screen[0]) + || geometry["height"].as_f64() != Some(captured.screen[1]) + || geometry["scale_factor"].as_f64().unwrap_or(1.0) != captured.scale + { + *frame = None; + return Err(error( + "screenshot_required", + "Display geometry changed. Observe a fresh desktop screenshot.", + false, + )); + } + if name == "drag" { + captured.map(&mut input, "from_x", "from_y")?; + captured.map(&mut input, "to_x", "to_y")?; + } else { + captured.map(&mut input, "x", "y")?; + } + } + if name == "click" { + if let Some(key) = input.as_object_mut().unwrap().remove("key") { + input["modifier"] = if key.is_string() { json!([key]) } else { key }; + } + } + let native = match name { + "double_click" => { + input["count"] = json!(2); + "click" + } + "right_click" => { + input["button"] = json!("right"); + "click" + } + _ => name, + }; + Self::acknowledge(Box::pin(registry.invoke_computer_use_native(native, input)).await) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::authorization::PermissionMode; + use crate::session_authorization::{SessionAuthorizationRegistry, SessionModeCeiling}; + use crate::tool::TrustedInvocationEvidence; + use std::time::Duration; + + #[test] + fn flat_schema_keeps_target_and_modifier_validation() { + for args in [json!({}), json!({"app":"Fixture","scope":"desktop"})] { + let refusal = validate("click", &args).unwrap_err(); + assert_eq!(refusal.structured_content.unwrap()["sent"], false); + } + for key in [json!("shift"), json!(["ctrl", "shift"])] { + assert!(validate("click", &json!({"app":"Fixture","element":1,"key":key})).is_ok()); + } + assert!(validate("click", &json!({"app":"Fixture","element":1,"key":[7]})).is_err()); + assert!(validate("key_down", &json!({"app":"Fixture","key":"shift"})).is_err()); + assert!(validate("key_down", &json!({"scope":"desktop","key":"shift"})).is_ok()); + } + + #[test] + fn explicit_dispatch_state_survives_error_classification() { + let result = ToolResult::computer_use_error( + "not_settable", + "native write failed after dispatch", + true, + ); + let checked = checked(result, true).unwrap_err(); + assert_eq!(checked.structured_content.unwrap()["sent"], true); + } + + #[derive(Default)] + struct Fixture { + window: Mutex, + screen_width: Mutex, + off_space: Mutex, + calls: Mutex>, + readiness: Mutex)>>, + } + struct Native { + def: ToolDef, + fixture: Arc, + } + #[async_trait::async_trait] + impl Tool for Native { + fn def(&self) -> &ToolDef { + &self.def + } + async fn invoke(&self, args: Value) -> ToolResult { + self.fixture.readiness.lock().unwrap().push(( + self.def.name.clone(), + crate::tool::current_ax_readiness_deadline(), + )); + self.fixture + .calls + .lock() + .unwrap() + .push((self.def.name.clone(), args.clone())); + let window = *self.fixture.window.lock().unwrap(); + match self.def.name.as_str() { + "get_desktop_state" => { + let mut result = ToolResult::text("desktop").with_structured(json!({"platform":"macos","display":"primary","screenshot_mime_type":"image/png","screenshot_width":2000,"screenshot_height":1000,"screen_width":1000,"screen_height":500,"scale_factor":2.0})); + result.content.push(Content::image_png("AQ==".into())); result + }, + "get_screen_size" => ToolResult::text("geometry").with_structured(json!({"width":*self.fixture.screen_width.lock().unwrap(),"height":500,"scale_factor":2.0})), + "list_apps" => ToolResult::text("").with_structured(json!({"apps":[{ + "name":"Fixture", "bundle_id":"org.qwen.fixture", "launch_path":"/Applications/Fixture.app", "pid":42,"running":true, + }]})), + "list_windows" => ToolResult::text("").with_structured(json!({"windows":[{ + "pid":42,"window_id":window,"title":"Fixture window","is_app_target":true,"z_index":1, + }]})), + "get_window_state" => { + let mut metadata = json!({ + "tree_markdown":"[100] button Save", "elements":[{"element_id":100,"element_token":"native-token"}], + }); + if *self.fixture.off_space.lock().unwrap() { + metadata["on_current_space"] = json!(false); + metadata["off_space_staleness"] = json!("This read may show an older state."); + } + let mut result = ToolResult::text("").with_structured(metadata); + if args["include_screenshot"] == true { + result.content.push(Content::image_png("AQ==".into())); + } + result + }, + _ => ToolResult::text("dispatched"), + } + } + } + fn setup() -> ( + ToolRegistry, + Arc, + Arc, + ) { + let fixture = Arc::new(Fixture { + window: Mutex::new(7), + screen_width: Mutex::new(1000), + ..Default::default() + }); + let mut registry = ToolRegistry::new(); + for name in [ + "list_apps", + "list_windows", + "get_window_state", + "get_desktop_state", + "get_screen_size", + "click", + "type_text", + ] { + registry.register(Box::new(Native { + def: ToolDef { name:name.into(), description:"fixture".into(), + input_schema:json!({"type":"object","properties":{"pid":{"type":"integer"},"window_id":{"type":"integer"},"delivery_mode":{"type":"string"},"app_context":{"type":"boolean"}}}), + read_only:name.starts_with("list_") || name == "get_window_state", destructive:false,idempotent:false,open_world:false, + }, fixture:fixture.clone(), + })); + } + registry.enable_computer_use(); + let ceiling = SessionModeCeiling::for_trusted_sessions( + [PermissionMode::Standard], + false, + Duration::from_secs(60), + Duration::from_secs(30), + ) + .unwrap(); + let context = SessionAuthorizationRegistry::with_ceiling(ceiling) + .compatibility_context(PermissionMode::Standard, None) + .unwrap(); + (registry, fixture, context) + } + async fn call( + registry: &ToolRegistry, + context: Arc, + name: &str, + args: Value, + transport: &str, + ) -> ToolResult { + let evidence = TrustedInvocationEvidence::extract_from_adapter_args( + &mut json!({"_transport_session_id":transport}), + ); + registry + .invoke_with_context_and_evidence(name, args, context, evidence) + .await + } + #[tokio::test] + async fn ax_observation_shares_one_readiness_deadline_but_input_and_screenshot_do_not_wait() { + let (registry, fixture, context) = setup(); + let result = call( + ®istry, + context.clone(), + "get_app", + json!({"app":"Fixture"}), + "readiness", + ) + .await; + assert_ne!(result.is_error, Some(true), "{}", text(&result)); + let reads = fixture.readiness.lock().unwrap().clone(); + let selection = reads + .iter() + .find(|(name, _)| name == "list_windows") + .unwrap() + .1; + let walk = reads + .iter() + .find(|(name, _)| name == "get_window_state") + .unwrap() + .1; + assert!(selection.is_some()); + assert_eq!(selection, walk); + assert!(reads + .iter() + .filter(|(name, _)| name == "list_apps") + .all(|(_, deadline)| deadline.is_none())); + for (name, args) in [ + ("click", json!({"app":"Fixture","element":1})), + ("get_screenshot", json!({"app":"Fixture"})), + ] { + fixture.readiness.lock().unwrap().clear(); + let result = call(®istry, context.clone(), name, args, "readiness").await; + assert_ne!(result.is_error, Some(true), "{}", text(&result)); + assert!(fixture + .readiness + .lock() + .unwrap() + .iter() + .all(|(_, deadline)| deadline.is_none())); + } + assert!(crate::tool::current_ax_readiness_deadline().is_none()); + } + #[tokio::test] + async fn desktop_maps_the_observed_frame_once_and_invalidates_changed_geometry() { + let (mut registry, fixture, context) = setup(); + registry.set_coordinate_space_normalized(true); + let result = call( + ®istry, + context.clone(), + "click", + json!({"scope":"desktop","x":1500,"y":600}), + "desktop-a", + ) + .await; + assert_eq!( + result.structured_content.unwrap()["code"], + "screenshot_required" + ); + let result = call( + ®istry, + context.clone(), + "get_screenshot", + json!({"scope":"desktop"}), + "desktop-a", + ) + .await; + assert_ne!(result.is_error, Some(true), "{}", text(&result)); + let result = call( + ®istry, + context.clone(), + "click", + json!({"scope":"desktop","x":1500,"y":600,"key":"shift"}), + "desktop-a", + ) + .await; + assert_ne!(result.is_error, Some(true), "{}", text(&result)); + let action = fixture + .calls + .lock() + .unwrap() + .iter() + .find(|(name, _)| name == "click") + .unwrap() + .1 + .clone(); + assert_eq!(action["x"], 750.0); + assert_eq!(action["y"], 300.0); + assert_eq!(action["modifier"], json!(["shift"])); + assert!(action.get("pid").is_none()); + let result = call( + ®istry, + context.clone(), + "click", + json!({"scope":"desktop","x":2000,"y":0}), + "desktop-a", + ) + .await; + assert_eq!( + result.structured_content.unwrap()["code"], + "invalid_arguments" + ); + let result = call( + ®istry, + context.clone(), + "click", + json!({"scope":"desktop","x":100,"y":100}), + "desktop-b", + ) + .await; + assert_eq!( + result.structured_content.unwrap()["code"], + "screenshot_required" + ); + *fixture.screen_width.lock().unwrap() = 800; + let result = call( + ®istry, + context, + "click", + json!({"scope":"desktop","x":100,"y":100}), + "desktop-a", + ) + .await; + assert_eq!( + result.structured_content.unwrap()["code"], + "screenshot_required" + ); + let calls = fixture.calls.lock().unwrap(); + assert_eq!( + calls + .iter() + .filter(|(name, _)| name == "get_desktop_state") + .count(), + 1 + ); + assert_eq!(calls.iter().filter(|(name, _)| name == "click").count(), 1); + assert!(!calls + .iter() + .any(|(name, _)| matches!(name.as_str(), "list_apps" | "get_window_state"))); + } + + #[tokio::test] + async fn app_calls_resolve_in_the_driver_and_refuse_stale_ids_before_dispatch() { + let (registry, fixture, context) = setup(); + let result = call( + ®istry, + context.clone(), + "get_app", + json!({"app":"Fixture"}), + "a", + ) + .await; + assert_ne!(result.is_error, Some(true), "{}", text(&result)); + assert!(text(&result).contains("[1] button Save")); + assert!(result.structured_content.unwrap().get("elements").is_none()); + let observed = fixture.calls.lock().unwrap().clone(); + assert_eq!( + observed + .iter() + .filter(|(name, _)| name == "get_window_state") + .count(), + 1 + ); + assert_eq!(observed.last().unwrap().1["include_screenshot"], false); + assert_eq!(observed.last().unwrap().1["include_accessibility"], true); + *fixture.window.lock().unwrap() = 8; + let result = call( + ®istry, + context.clone(), + "click", + json!({"app":"Fixture","element":1}), + "a", + ) + .await; + assert_eq!( + result.structured_content.as_ref().unwrap()["code"], + "element_stale" + ); + assert_eq!(result.structured_content.unwrap()["sent"], false); + assert!(!fixture + .calls + .lock() + .unwrap() + .iter() + .any(|(name, _)| name == "click")); + call( + ®istry, + context.clone(), + "get_ax_state", + json!({"app":"Fixture"}), + "a", + ) + .await; + let result = call( + ®istry, + context, + "click", + json!({"app":"Fixture","element":2}), + "a", + ) + .await; + assert_ne!(result.is_error, Some(true), "{}", text(&result)); + let calls = fixture.calls.lock().unwrap(); + let action = &calls.iter().find(|(name, _)| name == "click").unwrap().1; + assert_eq!(action["window_id"], 8); + assert_eq!(action["element_token"], "native-token"); + assert!(action.get("element").is_none()); + } + + #[tokio::test] + async fn app_observation_preserves_off_space_warning_on_all_observation_routes() { + let (registry, fixture, context) = setup(); + *fixture.off_space.lock().unwrap() = true; + for name in [ + "get_app", + "get_ax_state", + "get_screenshot", + "get_ax_state_and_screenshot", + ] { + let result = call( + ®istry, + context.clone(), + name, + json!({"app":"Fixture"}), + "off-space", + ) + .await; + assert_ne!(result.is_error, Some(true), "{name}: {}", text(&result)); + assert!( + text(&result).contains("This read may show an older state."), + "{name}" + ); + let metadata = result.structured_content.unwrap(); + assert_eq!(metadata["on_current_space"], false); + assert_eq!( + metadata["off_space_staleness"], + "This read may show an older state." + ); + } + } + #[tokio::test] + async fn app_state_is_transport_isolated_and_options_are_checked_before_native_calls() { + let (registry, fixture, context) = setup(); + call( + ®istry, + context.clone(), + "get_app", + json!({"app":"Fixture"}), + "a", + ) + .await; + let result = call( + ®istry, + context.clone(), + "click", + json!({"app":"Fixture","element":1}), + "b", + ) + .await; + assert_eq!(result.structured_content.unwrap()["code"], "element_stale"); + let before = fixture.calls.lock().unwrap().len(); + let result = call( + ®istry, + context, + "click", + json!({"app":"Fixture","element":1,"delivery_mode":"foreground"}), + "a", + ) + .await; + assert_eq!( + result.structured_content.unwrap()["code"], + "invalid_arguments" + ); + assert_eq!(fixture.calls.lock().unwrap().len(), before); + } + #[test] + fn app_identity_preserves_opaque_windows_launch_arguments_and_rejects_ambiguity() { + let apps = json!([ + {"name":"Editor","bundle_id":"C:\\Editor.exe","launch_path":"C:\\Editor.exe --Profile ABC"}, + {"name":"Editor","bundle_id":"C:\\Other.exe","launch_path":"C:\\Other.exe"}, + ]); + assert_eq!(identity(&apps[0]), "C:\\Editor.exe --Profile ABC"); + assert!(resolve_app(&apps, "Editor").is_err()); + assert_eq!( + resolve_app(&apps, "c:\\EDITOR.exe").unwrap()["bundle_id"], + "C:\\Editor.exe" + ); + } +} diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/desktop_keyboard.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/desktop_keyboard.rs new file mode 100644 index 00000000000..228f26eb8ab --- /dev/null +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/desktop_keyboard.rs @@ -0,0 +1,274 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use crate::protocol::ToolResult; +use crate::tool::{ + current_dispatch_session_identity, DispatchSessionIdentity, Tool, ToolDef, ToolRegistry, +}; +use serde_json::Value; + +pub trait DesktopKeyboard: Send + Sync { + fn resolve(&self, key: &str) -> Result; + fn transition(&self, code: u32, down: bool, held: &[u32]) -> Result<(), String>; +} + +struct HeldKey { + manager: u64, + owner: DispatchSessionIdentity, + code: u32, + keyboard: Arc, +} + +fn held_keys() -> &'static Mutex> { + static KEYS: OnceLock>> = OnceLock::new(); + KEYS.get_or_init(|| Mutex::new(Vec::new())) +} + +pub fn held_codes() -> Vec { + held_keys() + .lock() + .unwrap() + .iter() + .map(|key| key.code) + .collect() +} + +pub(crate) fn admit_input(name: &str, args: &Value) -> Result<(), ToolResult> { + if matches!(name, "key_down" | "key_up") { + return Ok(()); + } + let held = held_keys().lock().unwrap(); + if held.is_empty() { + return Ok(()); + } + let owner = current_dispatch_session_identity(); + if held.iter().any(|key| Some(&key.owner) != owner.as_ref()) { + return Err(crate::computer_use::error( + "input_busy", + "Another session holds desktop keys.", + false, + )); + } + if args["scope"] != "desktop" + || !matches!( + name, + "click" | "double_click" | "right_click" | "scroll" | "drag" + ) + || args.get("modifier").is_some() + || args.get("key").is_some() + { + return Err(crate::computer_use::error("held_keys_active", "Release held desktop keys before app input, text, chords or clicks with explicit modifiers. Use key_down/key_up for held chords.", false)); + } + Ok(()) +} + +fn transition( + manager: u64, + keyboard: Arc, + owner: DispatchSessionIdentity, + code: u32, + down: bool, +) -> Result<(), ToolResult> { + let mut held = held_keys().lock().unwrap(); + if held.iter().any(|key| key.owner != owner) { + return Err(crate::computer_use::error( + "input_busy", + "Another session holds desktop keys.", + false, + )); + } + let index = held.iter().position(|key| key.code == code); + if down { + if index.is_some() { + return Ok(()); + } + held.push(HeldKey { + manager, + owner, + code, + keyboard: keyboard.clone(), + }); + let codes = held.iter().map(|key| key.code).collect::>(); + keyboard + .transition(code, true, &codes) + .map_err(|reason| crate::computer_use::error("operation_failed", reason, true)) + } else { + let Some(index) = index else { + return Ok(()); + }; + if held[index].manager != manager { + return Err(crate::computer_use::error( + "input_busy", + "The held key belongs to another runtime.", + false, + )); + } + let codes = held + .iter() + .filter(|key| key.code != code) + .map(|key| key.code) + .collect::>(); + keyboard + .transition(code, false, &codes) + .map_err(|reason| crate::computer_use::error("operation_failed", reason, true))?; + held.remove(index); + Ok(()) + } +} + +fn release_matching(manager: u64, session: Option<&str>) -> Result<(), String> { + let mut held = held_keys().lock().unwrap(); + let mut failures = Vec::new(); + for index in (0..held.len()).rev() { + let key = &held[index]; + if key.manager != manager + || session.is_some_and(|session| { + key.owner.session_id != session && key.owner.transport_session_id != session + }) + { + continue; + } + let codes = held + .iter() + .enumerate() + .filter(|(i, _)| *i != index) + .map(|(_, key)| key.code) + .collect::>(); + match key.keyboard.transition(key.code, false, &codes) { + Ok(()) => { + held.remove(index); + } + Err(reason) => failures.push(reason), + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) + } +} + +struct KeyTool { + def: ToolDef, + manager: u64, + keyboard: Arc, + down: bool, +} + +#[async_trait::async_trait] +impl Tool for KeyTool { + fn def(&self) -> &ToolDef { + &self.def + } + async fn invoke(&self, args: Value) -> ToolResult { + let Some(owner) = current_dispatch_session_identity() else { + return crate::computer_use::error( + "session_unavailable", + "Holding keys requires a bound session.", + false, + ); + }; + if args["scope"] != "desktop" { + return crate::computer_use::error( + "invalid_arguments", + "key_down/key_up require scope:desktop.", + false, + ); + } + let Some(key) = args["key"].as_str() else { + return crate::computer_use::error("invalid_arguments", "Provide a key name.", false); + }; + let code = match self.keyboard.resolve(key) { + Ok(code) => code, + Err(reason) => return crate::computer_use::error("invalid_arguments", reason, false), + }; + match transition(self.manager, self.keyboard.clone(), owner, code, self.down) { + Ok(()) => ToolResult::text("Key state sent; observe before retrying.") + .with_structured(serde_json::json!({"computer_use":1,"sent":true})), + Err(result) => result, + } + } +} + +pub fn register(registry: &mut ToolRegistry, keyboard: Arc) { + static NEXT: AtomicU64 = AtomicU64::new(1); + let manager = NEXT.fetch_add(1, Ordering::Relaxed); + registry.retain_session_end_hook(crate::session::register_scoped_fallible_session_end_hook( + "desktop_keyboard", + move |session| release_matching(manager, Some(session)), + )); + registry.retain_runtime_cleanup(move || { + if let Err(reason) = release_matching(manager, None) { + eprintln!("[cua-driver] key cleanup incomplete: {reason}"); + } + }); + for spec in cua_driver_contract::computer_use::tools() + .into_iter() + .filter(|spec| matches!(spec.name, "key_down" | "key_up")) + { + let down = spec.name == "key_down"; + registry.register(Box::new(KeyTool { + def: ToolDef { + name: spec.name.into(), + description: spec.description.into(), + input_schema: spec.input_schema, + read_only: false, + destructive: false, + idempotent: true, + open_world: true, + }, + manager, + keyboard: keyboard.clone(), + down, + })); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicBool; + #[derive(Default)] + struct Keyboard { + events: Mutex>, + fail: AtomicBool, + } + impl DesktopKeyboard for Keyboard { + fn resolve(&self, _: &str) -> Result { + Ok(1) + } + fn transition(&self, code: u32, down: bool, _: &[u32]) -> Result<(), String> { + self.events.lock().unwrap().push((code, down)); + if self.fail.swap(false, Ordering::SeqCst) { + Err("uncertain dispatch".into()) + } else { + Ok(()) + } + } + } + #[test] + fn uncertain_down_is_owned_until_successful_release_and_cleanup_retries() { + let keyboard = Arc::new(Keyboard::default()); + let owner = DispatchSessionIdentity { + runtime_scope: "keyboard-test".into(), + session_id: "keyboard-session".into(), + transport_session_id: "keyboard-transport".into(), + }; + let mut other = owner.clone(); + other.session_id = "other".into(); + keyboard.fail.store(true, Ordering::SeqCst); + assert!(transition(991, keyboard.clone(), owner.clone(), 10001, true).is_err()); + assert!(transition(992, keyboard.clone(), other, 10001, false).is_err()); + assert_eq!(*keyboard.events.lock().unwrap(), vec![(10001, true)]); + assert!(transition(991, keyboard.clone(), owner, 10001, true).is_ok()); + keyboard.fail.store(true, Ordering::SeqCst); + assert!(release_matching(991, Some("keyboard-session")).is_err()); + assert!(held_codes().contains(&10001)); + release_matching(991, Some("keyboard-transport")).unwrap(); + assert!(!held_codes().contains(&10001)); + assert_eq!( + *keyboard.events.lock().unwrap(), + vec![(10001, true), (10001, false), (10001, false)] + ); + } +} diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs index ce8b6c9805d..a9f742aa86e 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/element_cache.rs @@ -1,38 +1,8 @@ -//! Generic locked-HashMap plumbing for per-platform element caches. +//! Generic locked target directory for per-platform element caches. //! -//! Each platform crate stores a per-(pid, window) snapshot of the -//! actionable accessibility elements it last walked, so subsequent -//! tool calls (`click`, `type_text`, etc.) can resolve an -//! `element_index` back to a native handle without re-walking the -//! tree. Before this module the three crates each owned a -//! near-identical `Mutex>` plus -//! the same insert / lookup / count methods — see -//! `docs/dedup-audit.md` item #3. -//! -//! What lives here: the lock + HashMap, generic over the caller's -//! key type `K` and snapshot type `S`. What stays per-platform: -//! -//! - the concrete `CacheKey` (different pid/window-id widths) -//! - the concrete `CachedSnapshot` and its `Drop` impl, which on -//! macOS calls `CFRelease` on every cached AXUIElementRef and on -//! Windows calls COM `Release` on every IUIAutomationElement. -//! Those releases are load-bearing — dropping the wrong way leaks -//! the AX / UIA handles. The generic core just stores `S` by -//! value; when the entry is removed or replaced (via -//! `HashMap::insert`'s replace-return semantics) Rust runs `S`'s -//! destructor, so the platform `Drop` impl still fires exactly as -//! it did pre-refactor. -//! -//! Each platform's `ElementCache` is now a thin wrapper around an -//! `ElementCacheCore`. Specialised -//! accessors (`get_element_retained` on macOS and Windows, -//! `get_element_center` on Windows, `get_element_key` on Linux) call -//! `with_snapshot` and project the field they care about. The macOS and -//! Windows `get_element_retained` accessors retain the element under the -//! lock (CFRetain / COM AddRef) so a concurrent `insert` (snapshot replace) -//! can't free it mid-action — see -//! `platform-macos/src/ax/cache.rs::RetainedElement` and -//! `platform-windows/src/uia/cache.rs::RetainedElement`. +//! Platform caches store weak references here. Native snapshots are owned by +//! the token registry and resolved actions, so replacing a directory entry +//! cannot change the native object or metadata bound to an in-flight action. use std::collections::HashMap; use std::hash::Hash; @@ -54,10 +24,7 @@ impl ElementCacheCore { } } - /// Replace the snapshot for `key`. If an entry already existed - /// its destructor runs here — for macOS that fires `CFRelease` - /// on every retained AXUIElementRef, for Windows it fires COM - /// `Release`, for Linux it's a no-op (just frees the `Vec`). + /// Replace the entry for `key`, dropping the previous stored value. pub fn insert(&self, key: K, snapshot: S) { let mut inner = self.inner.lock().unwrap(); inner.insert(key, snapshot); @@ -150,7 +117,7 @@ mod tests { #[test] fn insert_replaces_existing_and_runs_drop() { // Smoke test that re-inserting under the same key replaces - // the prior snapshot — the platform Drop fires here in real + // the prior stored value — its Drop fires here in this // code (CFRelease/COM Release). We can't exercise FFI in a // unit test, but we can confirm the value was replaced. use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/element_token.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/element_token.rs index 21a20b2883f..69a4025e0f4 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/element_token.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/element_token.rs @@ -35,9 +35,8 @@ //! //! ## Validity contract //! -//! - Snapshot IDs are minted in `register_snapshot` (called by every -//! platform's `get_window_state` implementation immediately after the -//! AX/UIA/AT-SPI walk lands in the per-platform element cache). +//! - Native snapshots publish their ID and owned payload together in +//! `register_snapshot_payload`. Resolution keeps that exact payload alive. //! - A snapshot is valid until either (a) the LRU evicts it, or (b) a //! newer snapshot for the same `pid` pushes it past the LRU cap of //! [`LRU_CAP_PER_PID`]. @@ -48,10 +47,11 @@ //! The LRU is **per runtime and pid**, not global. Two snapshots in different //! lanes never collide even when their numeric counters match. +use std::any::Any; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU32, Ordering}; -use std::sync::Mutex; use std::sync::OnceLock; +use std::sync::{Arc, Mutex}; /// LRU cap of valid snapshots retained per pid. Only the newest snapshot for /// each `(pid, window_id)` remains valid; the cap bounds snapshots retained for @@ -82,6 +82,40 @@ struct SnapshotEntry { element_count: usize, /// Linux window captures retain application-wide indices, including gaps. element_indices: Option>, + binding: Option, + owner_session: Option, +} + +/// An immutable native snapshot retained at the token-resolution boundary. +/// Type erasure keeps the core independent of platform native handle types. +#[derive(Clone)] +pub struct SnapshotBinding { + pid: i32, + window_id: u32, + payload: Arc, +} + +impl std::fmt::Debug for SnapshotBinding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SnapshotBinding") + .field("pid", &self.pid) + .field("window_id", &self.window_id) + .finish_non_exhaustive() + } +} + +impl SnapshotBinding { + pub fn with_payload( + &self, + pid: i32, + window_id: u32, + read: impl FnOnce(&S) -> R, + ) -> Option { + if self.pid != pid || self.window_id != window_id { + return None; + } + self.payload.downcast_ref::().map(read) + } } /// Process-global token registry. Thread-safe; tools resolve from any @@ -114,7 +148,7 @@ impl TokenRegistry { /// in its lane, the oldest is evicted and any token that referenced /// it becomes stale — that's the contract. pub fn register_snapshot(&self, pid: i32, window_id: u32, element_count: usize) -> u32 { - self.register(pid, window_id, element_count, None) + self.register(pid, window_id, element_count, None, None, None) } pub fn register_snapshot_indices( @@ -124,7 +158,40 @@ impl TokenRegistry { indices: impl IntoIterator, ) -> u32 { let indices: HashSet = indices.into_iter().collect(); - self.register(pid, window_id, indices.len(), Some(indices)) + self.register(pid, window_id, indices.len(), Some(indices), None, None) + } + + pub fn register_snapshot_payload( + &self, + pid: i32, + window_id: u32, + indices: impl IntoIterator, + payload: Arc, + ) -> u32 { + self.register_snapshot_payload_for_session(pid, window_id, indices, payload, None) + } + + pub fn register_snapshot_payload_for_session( + &self, + pid: i32, + window_id: u32, + indices: impl IntoIterator, + payload: Arc, + owner_session: Option<&str>, + ) -> u32 { + let indices: HashSet = indices.into_iter().collect(); + self.register( + pid, + window_id, + indices.len(), + Some(indices), + Some(SnapshotBinding { + pid, + window_id, + payload, + }), + owner_session.map(str::to_owned), + ) } fn register( @@ -133,6 +200,8 @@ impl TokenRegistry { window_id: u32, element_count: usize, element_indices: Option>, + binding: Option, + owner_session: Option, ) -> u32 { // Keep the full 32-bit counter. Truncating to 16 bits repeats an id // every 65,536 process-global snapshots. A long-lived daemon can then @@ -142,16 +211,16 @@ impl TokenRegistry { let runtime_scope = current_runtime_scope(); let mut entries = self.by_runtime_and_pid.lock().unwrap(); let lane = entries.entry((runtime_scope, pid)).or_default(); - // Every platform replaces its per-window element cache on the next - // snapshot. Retaining an older token for the same window would resolve - // that token against the new cache and could target a different row. - // Invalidate it at the same boundary as the cache replacement. + // Publication invalidates future resolutions of the preceding window + // snapshot. Already-resolved actions retain their original payload. lane.retain(|entry| entry.window_id != window_id); lane.push(SnapshotEntry { snapshot_id: id, window_id, element_count, element_indices, + binding, + owner_session, }); // Evict oldest. The loop guards against pre-existing over-cap // state from a previous version of the binary; in steady state @@ -174,6 +243,15 @@ impl TokenRegistry { /// - `"element_token element_index out of range"` — the index in /// the token is past the max recorded for the snapshot. pub fn resolve(&self, pid: i32, token: &str) -> Result<(u32, usize), String> { + self.resolve_bound(pid, token) + .map(|(window, index, _)| (window, index)) + } + + pub fn resolve_bound( + &self, + pid: i32, + token: &str, + ) -> Result<(u32, usize, Option), String> { let (sid, idx) = parse_token(token).ok_or_else(|| "element_token has invalid format".to_string())?; let runtime_scope = current_runtime_scope(); @@ -209,7 +287,17 @@ impl TokenRegistry { entry.element_count )); } - Ok((entry.window_id, idx)) + Ok((entry.window_id, idx, entry.binding.clone())) + } + + pub fn clear_session(&self, session_id: &str) -> usize { + let mut entries = self.by_runtime_and_pid.lock().unwrap(); + let before = entries.values().map(Vec::len).sum::(); + entries.retain(|_, snapshots| { + snapshots.retain(|entry| entry.owner_session.as_deref() != Some(session_id)); + !snapshots.is_empty() + }); + before - entries.values().map(Vec::len).sum::() } pub fn clear_runtime_scope(&self, runtime_scope: &str) -> usize { @@ -229,6 +317,18 @@ impl TokenRegistry { before - entries.len() } + /// Invalidate one observation target in the calling runtime only. A failed + /// observation is not proof that another runtime's target has disappeared. + pub fn clear_current_target(&self, pid: i32, window_id: u32) -> usize { + let mut entries = self.by_runtime_and_pid.lock().unwrap(); + let Some(snapshots) = entries.get_mut(&(current_runtime_scope(), pid)) else { + return 0; + }; + let before = snapshots.len(); + snapshots.retain(|entry| entry.window_id != window_id); + before - snapshots.len() + } + /// Invalidate snapshots for one exact process/window target across runtime /// lanes while preserving snapshots for the process's other live windows. pub fn clear_target(&self, pid: i32, window_id: u32) -> usize { @@ -374,9 +474,19 @@ pub enum ResolvedElement { /// through the registry — informational, used by tools that /// want to log "via token" in the success summary. via_token: bool, + binding: Option, }, } +impl ResolvedElement { + pub fn binding(&self) -> Option { + match self { + Self::Element { binding, .. } => binding.clone(), + Self::None => None, + } + } +} + /// Validate tool args that accept `element_index`, `snapshot_id`, and /// `element_token`. Returns either a structured refusal or the resolved /// `(window_id, element_index)` pair. @@ -418,8 +528,8 @@ pub fn resolve_element_args( .resolve_current(pid, tok) .map_err(|error| refusal(error.code(), error.to_string()))?; let snapshot_token = token_for(snapshot_id, element_index); - let (snapshot_window_id, snapshot_element_index) = global() - .resolve(pid, &snapshot_token) + let (snapshot_window_id, snapshot_element_index, binding) = global() + .resolve_bound(pid, &snapshot_token) .map_err(|message| refusal("stale_element_token", message))?; if snapshot_window_id != window_id || snapshot_element_index != element_index { return Err(refusal( @@ -428,9 +538,9 @@ pub fn resolve_element_args( .to_owned(), )); } - return Ok((window_id, element_index)); + return Ok((window_id, element_index, binding)); } - let (wid, idx) = global().resolve(pid, tok).map_err(|message| { + let (wid, idx, binding) = global().resolve_bound(pid, tok).map_err(|message| { let code = if message.contains("another runtime generation") { "generation_mismatch" } else if message == STALE_TOKEN_ERROR { @@ -440,7 +550,7 @@ pub fn resolve_element_args( }; refusal(code, message) })?; - Ok::<_, crate::protocol::ToolResult>((wid, idx)) + Ok::<_, crate::protocol::ToolResult>((wid, idx, binding)) }; match (args_element_index, args_element_token, args_snapshot_id) { @@ -463,7 +573,7 @@ pub fn resolve_element_args( ) })?; let token = format_token(snapshot_id, idx); - let (wid, resolved_idx) = resolve_token(&token)?; + let (wid, resolved_idx, binding) = resolve_token(&token)?; if args_window_id.is_some_and(|arg_wid| arg_wid != wid) { return Err(refusal( "conflicting_element_target", @@ -477,10 +587,11 @@ pub fn resolve_element_args( window_id: Some(wid), element_index: resolved_idx, via_token: false, + binding, }) } (idx_opt, Some(tok), snapshot_opt) => { - let (wid, idx) = resolve_token(tok)?; + let (wid, idx, binding) = resolve_token(tok)?; if idx_opt.is_some_and(|arg_idx| arg_idx != idx) || args_window_id.is_some_and(|arg_wid| arg_wid != wid) || snapshot_opt.is_some_and(|handle| { @@ -499,6 +610,7 @@ pub fn resolve_element_args( window_id: Some(wid), element_index: idx, via_token: true, + binding, }) } } @@ -508,6 +620,118 @@ pub fn resolve_element_args( #[cfg(test)] mod tests { + #[test] + fn resolved_payload_does_not_switch_to_a_replacement_snapshot() { + let registry = super::TokenRegistry::new(); + let first = registry.register_snapshot_payload(42, 7, [130], std::sync::Arc::new("first")); + let (_, _, binding) = registry + .resolve_bound(42, &super::token_for(first, 130)) + .unwrap(); + let binding = binding.unwrap(); + registry.register_snapshot_payload(42, 7, [130], std::sync::Arc::new("replacement")); + assert_eq!( + binding.with_payload(42, 7, |value: &&str| *value), + Some("first") + ); + assert_eq!(binding.with_payload(42, 8, |value: &&str| *value), None); + assert_eq!(binding.with_payload(43, 7, |value: &&str| *value), None); + assert_eq!( + registry.resolve(42, &super::token_for(first, 130)), + Err(super::STALE_TOKEN_ERROR.into()) + ); + } + + #[test] + fn delayed_publication_owns_its_payload_instead_of_the_latest_window_cache() { + let registry = std::sync::Arc::new(super::TokenRegistry::new()); + let (prepared_tx, prepared_rx) = std::sync::mpsc::channel(); + let (publish_tx, publish_rx) = std::sync::mpsc::channel(); + let delayed = registry.clone(); + let first = std::thread::spawn(move || { + let pending = std::sync::Arc::new("first"); + prepared_tx.send(()).unwrap(); + publish_rx.recv().unwrap(); + delayed.register_snapshot_payload(42, 7, [0], pending) + }); + prepared_rx.recv().unwrap(); + let second = registry.register_snapshot_payload(42, 7, [0], std::sync::Arc::new("second")); + publish_tx.send(()).unwrap(); + let first = first.join().unwrap(); + let (_, _, binding) = registry + .resolve_bound(42, &super::token_for(first, 0)) + .unwrap(); + assert_eq!( + binding.unwrap().with_payload(42, 7, |value: &&str| *value), + Some("first") + ); + assert_eq!( + registry.resolve(42, &super::token_for(second, 0)), + Err(super::STALE_TOKEN_ERROR.into()) + ); + } + + #[test] + fn snapshot_cleanup_releases_payload_after_the_last_resolved_action() { + struct Payload(std::sync::Arc); + impl Drop for Payload { + fn drop(&mut self) { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + let drops = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let registry = super::TokenRegistry::new(); + let first = registry.register_snapshot_payload( + 42, + 7, + [0], + std::sync::Arc::new(Payload(drops.clone())), + ); + let (_, _, binding) = registry + .resolve_bound(42, &super::token_for(first, 0)) + .unwrap(); + registry.clear_runtime_scope("legacy"); + assert!(registry.resolve(42, &super::token_for(first, 0)).is_err()); + assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 0); + drop(binding); + assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 1); + registry.register_snapshot_payload(42, 8, [0], std::sync::Arc::new(Payload(drops.clone()))); + registry.clear_target(42, 8); + assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 2); + registry.register_snapshot_payload(42, 9, [0], std::sync::Arc::new(Payload(drops.clone()))); + for window in 10..10 + super::LRU_CAP_PER_PID as u32 { + registry.register_snapshot(42, window, 1); + } + assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 3); + } + + #[test] + fn observation_invalidation_preserves_peer_runtime_and_resolved_payload() { + use crate::tool::with_runtime_scope; + let registry = super::TokenRegistry::new(); + let peer = with_runtime_scope("invalidation-peer".into(), || { + registry.register_snapshot(42, 7, 1) + }); + with_runtime_scope("invalidation-owner".into(), || { + let sid = registry.register_snapshot_payload(42, 7, [0], Arc::new("original")); + let sibling = registry.register_snapshot(42, 8, 1); + let token = super::token_for(sid, 0); + let binding = registry.resolve_bound(42, &token).unwrap().2.unwrap(); + assert_eq!(registry.clear_current_target(42, 7), 1); + assert!(registry.resolve(42, &token).is_err()); + assert!(registry.resolve(42, &super::token_for(sibling, 0)).is_ok()); + assert_eq!(binding.with_payload(42, 7, |s: &&str| *s), Some("original")); + assert_eq!(registry.clear_current_target(42, 7), 0); + }); + with_runtime_scope("invalidation-peer".into(), || { + assert!(registry.resolve(42, &super::token_for(peer, 0)).is_ok()); + }); + // A confirmed target close still revokes every runtime lane. + assert_eq!(registry.clear_target(42, 7), 1); + with_runtime_scope("invalidation-peer".into(), || { + assert!(registry.resolve(42, &super::token_for(peer, 0)).is_err()); + }); + } + #[test] fn sparse_indices_are_membership_checked_and_window_scoped() { let registry = super::TokenRegistry::new(); @@ -797,6 +1021,7 @@ mod tests { window_id, element_index, via_token, + .. } => { assert_eq!(window_id, Some(555), "window_id comes from the snapshot"); assert_eq!(element_index, 2); @@ -829,7 +1054,8 @@ mod tests { ResolvedElement::Element { window_id: Some(888), element_index: 2, - via_token: false + via_token: false, + .. } )); } diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/lib.rs index 929e1003a85..c39cdd70c3e 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/lib.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/lib.rs @@ -54,11 +54,13 @@ pub mod capture_mode; pub mod capture_scope; pub mod cdp; pub mod clipboard; +mod computer_use; pub mod consent; pub mod coord_norm; pub mod cursor_events; pub mod cursor_sampler; pub mod daemon; +pub mod desktop_keyboard; pub mod element_cache; pub mod element_query; pub mod element_token; diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/protocol.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/protocol.rs index 462f0cf040c..28f11980066 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/protocol.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/protocol.rs @@ -326,6 +326,18 @@ pub struct ToolResult { } impl ToolResult { + pub fn computer_use_error(code: &str, reason: impl Into, sent: bool) -> Self { + let reason = reason.into(); + let next = if sent { + "Observe the app state before deciding whether to retry." + } else { + "Observe the app again and use its current target." + }; + Self::error(format!("{code}: {reason} {next}")).with_structured(serde_json::json!({ + "code":code,"reason":reason,"sent":sent,"next":next, + })) + } + pub fn text(msg: impl Into) -> Self { Self { content: vec![Content::text(msg)], @@ -378,6 +390,35 @@ pub fn initialize_result() -> Value { /// Goose uses in its `ComputerController` extension and Open /// Interpreter uses for its system message. fn agent_instructions() -> String { + if std::env::var("CUA_DRIVER_EXTRA_TOOLS").as_deref() == Ok("1") { + legacy_agent_instructions() + } else { + computer_use_instructions() + } +} + +fn computer_use_instructions() -> String { + let platform = if cfg!(target_os = "macos") { + "macOS: application names, bundle IDs or absolute application paths." + } else if cfg!(target_os = "windows") { + "Windows: exact application names or executable paths." + } else { + "Linux: X11 applications only; inherit DISPLAY, XAUTHORITY and the accessibility session bus." + }; + format!( + r#"cua-driver Computer Use. {platform} + +Prefer client-provided APIs, CLI or filesystem operations for non-GUI outcomes. This server has no shell. + +Use `get_state` for inventory, then `get_app(app)` for full AX state. `get_ax_state(app)` returns subsequent diffs; `force_full:true` requests a full tree. Act with `app` and a current short `element` ID. Observe again after actions; an acknowledgement does not prove the UI changed. Never replay uncertain input automatically. + +Request `get_screenshot(app)` before point actions. Coordinates are window screenshot pixels, independent of legacy coordinate settings. For explicit desktop tasks, use `get_screenshot(scope:"desktop")`, then desktop input with `scope:"desktop"` and no `app`. Desktop coordinates are latest primary-display screenshot pixels. `key_down/key_up` are desktop-only; release held keys in finally and before app input or ordinary text/chords. + +`start_session` is optional. For multi-call work, prefer a short `session` label and repeat it on every call that accepts it. Unnamed calls use the transport's implicit session. Only `start_session` revives an ended name; `end_session` explicitly cleans up."# + ) +} + +fn legacy_agent_instructions() -> String { let (tree_kind, platform_skill_pointer) = if cfg!(target_os = "macos") { ( "AX (Accessibility)", @@ -391,7 +432,7 @@ fn agent_instructions() -> String { } else { ( "AT-SPI", - "LINUX.md (X11/Wayland status, AT-SPI bus, BETA-level support)", + "LINUX.md (X11 support, AT-SPI bus, BETA-level support)", ) }; @@ -399,9 +440,9 @@ fn agent_instructions() -> String { coordinate_terms(crate::coord_norm::default_normalized()); format!( - r#"cua-driver: cross-platform background computer-use automation. + r#"cua-driver: extra native tools. App/scope calls always use screenshot pixels; this guidance applies to raw tools. -Before starting UI work, classify the desired postcondition. For a non-GUI outcome, prefer a client-provided app API/SDK, headless/background interface, CLI, or filesystem operation and read the result back in that semantic domain. This server has no shell. +For a non-GUI outcome, prefer a client-provided app API/SDK, headless/background interface, CLI, or filesystem operation and read the result back in that semantic domain. This server has no shell. For an app or window outcome, use the narrowest semantic Cua route first: `set_window_frame` plus `list_windows` readback for geometry, typed browser tools for supported page content, and clipboard tools for clipboard state. Then climb through background `element_index` ({tree_kind}), background {coordinate_term}, foreground delivery, and desktop fallback.{coordinate_note} Never advance on transport success alone. @@ -559,11 +600,11 @@ mod action_record_wire_tests { #[cfg(test)] mod agent_instruction_tests { - use super::{agent_instructions, initialize_result}; + use super::{computer_use_instructions, initialize_result, legacy_agent_instructions}; #[test] fn instructions_route_structured_and_visual_verification_to_the_right_owner() { - let instructions = agent_instructions(); + let instructions = legacy_agent_instructions(); assert!(instructions.contains("verify_state")); assert!(instructions.contains("`unknown` is not success")); assert!(instructions.contains("multimodal agent")); @@ -585,6 +626,33 @@ mod agent_instruction_tests { ); } + #[test] + fn compact_instructions_match_app_desktop_and_pixel_consumers() { + let instructions = computer_use_instructions(); + for required in [ + "get_app(app)", + "get_ax_state(app)", + "short `element` ID", + "window screenshot pixels", + "primary-display screenshot pixels", + "scope:\"desktop\"", + "key_down/key_up", + "Never replay uncertain input", + ] { + assert!(instructions.contains(required), "missing {required}"); + } + for obsolete in [ + "element_index", + "get_window_state", + "verify_state", + "0-1000", + "SKILL.md", + ] { + assert!(!instructions.contains(obsolete), "obsolete {obsolete}"); + } + assert!(instructions.split_whitespace().count() <= 200); + } + #[test] fn initialize_instructions_describe_implicit_session_lifecycle() { let result = initialize_result(); diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/recording.rs index 362f76118ad..999b9edfdf1 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/recording.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/recording.rs @@ -9,9 +9,10 @@ //! { tool, arguments, result_summary, result_error, timestamp, //! t_ms_from_session_start, t_start_ms_from_session_start } +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use std::time::Instant; @@ -109,6 +110,69 @@ pub fn set_ax_snapshot_fn( let _ = AX_SNAPSHOT_FN.set(Box::new(f)); } +/// Cooperative native walk budget; the callback backstop also bounds providers +/// that do not return control to their walker. +pub const RECORDING_AX_WALK_BUDGET: Duration = Duration::from_secs(1); +const AX_CAPTURE_BACKSTOP: Duration = Duration::from_secs(2); +const MAX_AX_CAPTURE_WORKERS: usize = 4; + +fn active_ax_captures() -> &'static Mutex>> { + static ACTIVE: OnceLock>>> = OnceLock::new(); + ACTIVE.get_or_init(|| Mutex::new(HashSet::new())) +} + +struct AxCaptureSlot(Option); + +impl AxCaptureSlot { + fn acquire(pid: Option) -> Option { + let mut active = active_ax_captures() + .lock() + .unwrap_or_else(|p| p.into_inner()); + if active.len() >= MAX_AX_CAPTURE_WORKERS || !active.insert(pid) { + return None; + } + Some(Self(pid)) + } +} + +impl Drop for AxCaptureSlot { + fn drop(&mut self) { + active_ax_captures() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&self.0); + } +} + +fn capture_ax_state( + window_id: Option, + pid: Option, +) -> (Option>, Option<&'static str>) { + let Some(hook) = AX_SNAPSHOT_FN.get() else { + return (None, Some("capture_hook_unavailable")); + }; + let Some(slot) = AxCaptureSlot::acquire(pid) else { + return (None, Some("state_capture_busy")); + }; + let context = crate::tool::CapturedDispatchContext::capture(); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let worker = std::thread::Builder::new() + .name("cua-recording-state".into()) + .spawn(move || { + let result = context.run_sync(|| hook(window_id, pid)); + drop(slot); + let _ = sender.send(result); + }); + if worker.is_err() { + return (None, Some("capture_failed")); + } + match receiver.recv_timeout(AX_CAPTURE_BACKSTOP) { + Ok(Some(bytes)) => (Some(bytes), None), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => (None, Some("state_capture_timeout")), + _ => (None, Some("capture_failed")), + } +} + // ── Platform element-bounds callback ───────────────────────────────────────── // // Resolves an element_index to its center point in window-local screenshot @@ -116,12 +180,19 @@ pub fn set_ax_snapshot_fn( // `CLICK_MARKER_FN`). Used so click.png is also written on element-indexed // clicks, not just pixel-addressed ones. -type ElementBoundsFnBox = Box Option<(f64, f64)> + Send + Sync>; +type ElementBoundsFnBox = Box< + dyn Fn(u64, i64, u32, &crate::element_token::SnapshotBinding) -> Option<(f64, f64)> + + Send + + Sync, +>; static ELEMENT_BOUNDS_FN: OnceLock = OnceLock::new(); -/// Register the platform-specific element-bounds resolver. Args: (window_id, pid, element_index). +/// Register the platform-specific element-bounds resolver. Args: (window_id, pid, element_index, snapshot_binding). pub fn set_element_bounds_fn( - f: impl Fn(u64, i64, u32) -> Option<(f64, f64)> + Send + Sync + 'static, + f: impl Fn(u64, i64, u32, &crate::element_token::SnapshotBinding) -> Option<(f64, f64)> + + Send + + Sync + + 'static, ) { let _ = ELEMENT_BOUNDS_FN.set(Box::new(f)); } @@ -129,6 +200,7 @@ pub fn set_element_bounds_fn( #[derive(Default)] struct TurnCapture { state: Option>, + state_classification: Option<&'static str>, screenshot: Option>, screenshot_classification: Option<&'static str>, } @@ -138,6 +210,7 @@ struct TurnCapture { /// one stable `turn-NNNNN` directory even when calls complete out of order. pub struct PendingTurn { generation: u64, + owner: Option, turn_dir: PathBuf, tool_name: String, args: Value, @@ -150,7 +223,13 @@ pub struct PendingTurn { before: TurnCapture, } -/// Persistent recording session state (singleton per process). +#[derive(Debug, Clone, Copy, Default)] +pub struct RecordingCaller<'a> { + pub session: Option<&'a str>, + pub transport: Option<&'a str>, +} + +/// Persistent recording session state owned by one tool registry. pub struct RecordingSession { inner: Mutex, } @@ -332,9 +411,10 @@ impl RecordingSession { // Write initial session.json — final video metadata is rewritten on // stop. We mark `present` based on whether ffmpeg actually started, // not just whether the caller asked for video. + let started_at_ms = now_ms(); let session_payload = serde_json::json!({ "schema_version": 1, - "started_at_monotonic_ms": now_ms(), + "started_at_monotonic_ms": started_at_ms, "video": video_session_payload(video_present, video_error.as_deref(), None), "cursor": { "present": inner.cursor.is_some(), "sample_count": 0 } }); @@ -350,7 +430,7 @@ impl RecordingSession { inner.enabled = true; inner.output_dir = Some(dir); inner.next_turn = 1; - inner.session_start_ms = now_ms(); + inner.session_start_ms = started_at_ms; inner.session_monotonic_start = Some(monotonic_start); inner.last_error = video_error; inner.last_video = None; @@ -392,6 +472,7 @@ impl RecordingSession { } inner.owner = None; let dir = inner.output_dir.clone(); + let started_at_ms = inner.session_start_ms; let (video_meta, stop_error) = match inner.video.take().map(|rec| rec.stop()) { Some(Ok(meta)) => match validate_video_metadata(meta) { Ok(meta) => (Some(meta), None), @@ -427,7 +508,7 @@ impl RecordingSession { }; let session_payload = serde_json::json!({ "schema_version": 1, - "started_at_monotonic_ms": now_ms(), + "started_at_monotonic_ms": started_at_ms, "video": video_block, "cursor": { "present": cursor_samples > 0, "sample_count": cursor_samples } }); @@ -476,8 +557,14 @@ impl RecordingSession { /// Reserve a turn and capture its target immediately before tool dispatch. /// No-op when recording is disabled. - pub fn begin_turn(&self, tool_name: &str, args: &Value, start_ms: u64) -> Option { - self.begin_turn_with_capture(tool_name, args, start_ms, true) + pub fn begin_turn( + &self, + tool_name: &str, + args: &Value, + start_ms: u64, + caller: RecordingCaller<'_>, + ) -> Option { + self.begin_turn_with_capture(tool_name, args, start_ms, caller, true) } /// Reserve a turn while deliberately suppressing visual and accessibility @@ -489,8 +576,9 @@ impl RecordingSession { tool_name: &str, args: &Value, start_ms: u64, + caller: RecordingCaller<'_>, ) -> Option { - self.begin_turn_with_capture(tool_name, args, start_ms, false) + self.begin_turn_with_capture(tool_name, args, start_ms, caller, false) } fn begin_turn_with_capture( @@ -498,13 +586,19 @@ impl RecordingSession { tool_name: &str, args: &Value, start_ms: u64, + caller: RecordingCaller<'_>, capture_visual_state: bool, ) -> Option { - let (turn_dir, session_start_ms, generation) = { + let (turn_dir, session_start_ms, generation, owner) = { let mut inner = self.inner.lock().unwrap(); if !inner.enabled { return None; } + if inner.owner.as_deref().is_some_and(|owner| { + caller.session != Some(owner) && caller.transport != Some(owner) + }) { + return None; + } let out = inner.output_dir.clone()?; let idx = inner.next_turn; inner.next_turn += 1; @@ -512,6 +606,7 @@ impl RecordingSession { out.join(format!("turn-{idx:05}")), inner.session_start_ms, inner.generation, + inner.owner.clone(), ) }; @@ -525,30 +620,47 @@ impl RecordingSession { let mut window_id = args.opt_u64("window_id"); let pid = args.opt_i64("pid"); let mut element_index = args.opt_u64("element_index"); - if let (Some(pid), Some(token)) = ( - pid.and_then(|pid| i32::try_from(pid).ok()), - args.get("element_token").and_then(Value::as_str), - ) { - if let Ok((resolved_window, resolved_index)) = - crate::element_token::global().resolve(pid, token) - { - window_id = Some(u64::from(resolved_window)); - element_index = u64::try_from(resolved_index).ok(); + let mut binding = None; + if let Some(pid) = pid.and_then(|pid| i32::try_from(pid).ok()) { + if let Ok(resolved) = crate::element_token::resolve_element_args( + pid, + element_index.and_then(|index| usize::try_from(index).ok()), + args.get("element_token").and_then(Value::as_str), + args.get("snapshot_id").and_then(Value::as_str), + window_id.and_then(|window| u32::try_from(window).ok()), + tool_name, + ) { + binding = resolved.binding(); + if let crate::element_token::ResolvedElement::Element { + window_id: Some(window), + element_index: index, + .. + } = resolved + { + window_id = Some(u64::from(window)); + element_index = u64::try_from(index).ok(); + } } } - let click_point = resolve_click_point(tool_name, &args, window_id, pid, element_index); + let click_point = resolve_click_point( + tool_name, + &args, + window_id, + pid, + element_index, + binding.as_ref(), + ); let before = if capture_visual_state { capture_turn(window_id, pid) } else { TurnCapture { - state: None, - screenshot: None, screenshot_classification: Some("privacy_suppressed"), + ..Default::default() } }; let mut inner = self.inner.lock().unwrap(); - if !inner.enabled || inner.generation != generation { + if !inner.enabled || inner.generation != generation || inner.owner != owner { return None; } if let Err(error) = write_phase_artifacts(&turn_dir, "before", &before) { @@ -558,6 +670,7 @@ impl RecordingSession { Some(PendingTurn { generation, + owner, turn_dir, tool_name: tool_name.to_owned(), args, @@ -600,15 +713,27 @@ impl RecordingSession { action_record: Option<&crate::action_record::ActionExecutionRecord>, result_is_error: bool, ) { - let mut inner = self.inner.lock().unwrap(); - if !inner.enabled || inner.generation != pending.generation { - tracing::warn!( - target: "recording", - "discarding a turn from an inactive recording generation" - ); + let is_live = |inner: &RecordingInner| { + inner.enabled && inner.generation == pending.generation && inner.owner == pending.owner + }; + if !is_live(&self.inner.lock().unwrap()) { return; } - if let Err(error) = write_turn(pending, result_text, action_record, result_is_error) { + let after = if pending.capture_visual_state { + capture_turn(pending.window_id, pending.pid) + } else { + TurnCapture { + screenshot_classification: Some("privacy_suppressed"), + ..Default::default() + } + }; + let mut inner = self.inner.lock().unwrap(); + if !is_live(&inner) { + tracing::warn!(target: "recording", "discarding a turn from an inactive recording generation"); + return; + } + if let Err(error) = write_turn(pending, after, result_text, action_record, result_is_error) + { inner.last_error = Some(error.to_string()); } } @@ -616,8 +741,15 @@ impl RecordingSession { /// Compatibility helper for callers that only report completed calls. /// New dispatch paths should use `begin_turn` and `finish_turn` so the /// before phase is captured before the action changes application state. - pub fn record(&self, tool_name: &str, args: &Value, result_text: &str, start_ms: u64) { - let Some(pending) = self.begin_turn(tool_name, args, start_ms) else { + pub fn record( + &self, + tool_name: &str, + args: &Value, + result_text: &str, + start_ms: u64, + caller: RecordingCaller<'_>, + ) { + let Some(pending) = self.begin_turn(tool_name, args, start_ms, caller) else { return; }; self.finish_turn(pending, result_text); @@ -637,10 +769,10 @@ fn capture_turn(window_id: Option, pid: Option) -> TurnCapture { .get() .map(|capture| capture(window_id, pid)) .unwrap_or_else(|| ScreenshotCapture::unavailable("capture_hook_unavailable")); + let (state, state_classification) = capture_ax_state(window_id, pid); TurnCapture { - state: AX_SNAPSHOT_FN - .get() - .and_then(|capture| capture(window_id, pid)), + state, + state_classification, screenshot: screenshot.png, screenshot_classification: screenshot.classification, } @@ -652,6 +784,7 @@ fn resolve_click_point( window_id: Option, pid: Option, element_index: Option, + binding: Option<&crate::element_token::SnapshotBinding>, ) -> Option<(f64, f64)> { use crate::tool_args::ArgsExt; if !matches!(tool_name, "click" | "double_click" | "right_click") { @@ -662,7 +795,7 @@ fn resolve_click_point( _ => match (window_id, pid, element_index, ELEMENT_BOUNDS_FN.get()) { (Some(wid), Some(pid), Some(index), Some(resolve)) => u32::try_from(index) .ok() - .and_then(|index| resolve(wid, pid, index)), + .and_then(|index| resolve(wid, pid, index, binding?)), _ => None, }, } @@ -711,7 +844,7 @@ fn write_evidence_manifest( let manifest = serde_json::json!({ "schema": "cua-turn-evidence/v1", "before": { - "state": capture_status(before.state.is_some(), state_expected, None), + "state": capture_status(before.state.is_some(), state_expected, before.state_classification), "screenshot": capture_status( before.screenshot.is_some(), true, @@ -719,7 +852,7 @@ fn write_evidence_manifest( ), }, "after": { - "state": capture_status(after.state.is_some(), state_expected, None), + "state": capture_status(after.state.is_some(), state_expected, after.state_classification), "screenshot": capture_status( after.screenshot.is_some(), true, @@ -758,18 +891,20 @@ fn strip_internal_keys(args: &Value) -> std::borrow::Cow<'_, Value> { fn write_turn( pending: PendingTurn, + after: TurnCapture, result_text: &str, action_record: Option<&crate::action_record::ActionExecutionRecord>, result_is_error: bool, ) -> anyhow::Result<()> { let PendingTurn { generation: _, + owner: _, turn_dir, tool_name, args, start_ms, session_start_ms, - window_id, + window_id: _, pid, click_point, capture_visual_state, @@ -777,15 +912,6 @@ fn write_turn( } = pending; std::fs::create_dir_all(&turn_dir)?; let now = now_ms(); - let after = if capture_visual_state { - capture_turn(window_id, pid) - } else { - TurnCapture { - state: None, - screenshot: None, - screenshot_classification: Some("privacy_suppressed"), - } - }; let click_family = matches!(tool_name.as_str(), "click" | "double_click" | "right_click"); let action_refused = action_record .is_some_and(|record| record.effect == crate::action_record::ActionEffect::Refused); @@ -803,7 +929,7 @@ fn write_turn( "arguments": args, "result_summary": result_text, "result_error": result_is_error, - "timestamp": iso_now(), + "timestamp": iso8601_utc_from_unix_ms(now), "t_ms_from_session_start": now.saturating_sub(session_start_ms), "t_start_ms_from_session_start": start_ms.saturating_sub(session_start_ms), }); @@ -874,12 +1000,19 @@ pub fn now_ms() -> u64 { .as_millis() as u64 } -fn iso_now() -> String { - let d = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default(); - // Format as fractional Unix seconds (simple, unambiguous, machine-readable). - format!("{:.3}", d.as_secs_f64()) +fn iso8601_utc_from_unix_ms(ms: u64) -> String { + let t = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(ms) * 1_000_000) + .unwrap_or(time::OffsetDateTime::UNIX_EPOCH); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second(), + t.millisecond() + ) } /// Build the `session.json` `video` field. Three shapes: @@ -940,6 +1073,93 @@ fn validate_video_metadata(meta: VideoMetadata) -> anyhow::Result #[cfg(test)] mod tests { + #[test] + fn recording_turns_require_the_owner_session_or_connection() { + let root = tempfile::tempdir().unwrap(); + let session = RecordingSession::new(); + { + let mut inner = session.inner.lock().unwrap(); + inner.enabled = true; + inner.output_dir = Some(root.path().to_owned()); + inner.owner = Some("owner".to_owned()); + } + for caller in [ + RecordingCaller { + session: Some("foreign"), + transport: Some("other"), + }, + RecordingCaller::default(), + ] { + assert!(session + .begin_private_turn("type_text", &serde_json::json!({}), 0, caller) + .is_none()); + assert_eq!(session.current_state().next_turn, 1); + } + for caller in [ + RecordingCaller { + session: Some("owner"), + transport: Some("connection"), + }, + RecordingCaller { + session: Some("named-on-owner-connection"), + transport: Some("owner"), + }, + ] { + assert!(session + .begin_private_turn("type_text", &serde_json::json!({}), 0, caller) + .is_some()); + } + assert_eq!(session.current_state().next_turn, 3); + session.inner.lock().unwrap().owner = None; + assert!(session + .begin_private_turn( + "type_text", + &serde_json::json!({}), + 0, + RecordingCaller::default() + ) + .is_some()); + } + + #[test] + fn recording_timestamps_use_iso8601_and_keep_the_start_baseline() { + for (ms, expected) in [ + (0, "1970-01-01T00:00:00.000Z"), + (951_782_400_123, "2000-02-29T00:00:00.123Z"), + (1_767_225_600_999, "2026-01-01T00:00:00.999Z"), + ] { + assert_eq!(iso8601_utc_from_unix_ms(ms), expected); + } + let root = tempfile::tempdir().unwrap(); + let session = RecordingSession::new(); + session + .start(root.path().to_str().unwrap(), false, None) + .unwrap(); + let manifest = || -> Value { + serde_json::from_slice(&std::fs::read(root.path().join("session.json")).unwrap()) + .unwrap() + }; + let started = manifest()["started_at_monotonic_ms"].as_u64().unwrap(); + let pending = session + .begin_private_turn( + "type_text", + &serde_json::json!({}), + now_ms(), + RecordingCaller::default(), + ) + .unwrap(); + session.finish_turn(pending, "fixture"); + let action: Value = serde_json::from_slice( + &std::fs::read(root.path().join("turn-00001/action.json")).unwrap(), + ) + .unwrap(); + let finished = started + action["t_ms_from_session_start"].as_u64().unwrap(); + assert_eq!(action["timestamp"], iso8601_utc_from_unix_ms(finished)); + std::thread::sleep(std::time::Duration::from_millis(10)); + session.stop_owner(None).unwrap(); + assert_eq!(manifest()["started_at_monotonic_ms"], started); + } + use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -974,7 +1194,7 @@ mod tests { Some(format!(r#"{{"phase":{phase}}}"#).into_bytes()) }); set_click_marker_fn(|_, _, _| Some(b"click".to_vec())); - set_element_bounds_fn(|window_id, pid, element_index| { + set_element_bounds_fn(|window_id, pid, element_index, _binding| { Some((window_id as f64 + element_index as f64, pid as f64)) }); @@ -995,6 +1215,7 @@ mod tests { "click", &serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}), now_ms(), + RecordingCaller::default(), ) .expect("recording should reserve a turn"); let turn = output_dir.join("turn-00001"); @@ -1028,13 +1249,19 @@ mod tests { assert_eq!(action["action_truth"]["route"], "synthetic_events"); assert_eq!(action["action_truth"]["requested_delivery"], "background"); - let snapshot_id = crate::element_token::global().register_snapshot(1, 77, 1); + let snapshot_id = crate::element_token::global().register_snapshot_payload( + 1, + 77, + [0], + std::sync::Arc::new(()), + ); let token = crate::element_token::token_for(snapshot_id, 0); let pending = session .begin_turn( "click", &serde_json::json!({"pid": 1, "element_token": token}), now_ms(), + RecordingCaller::default(), ) .expect("token-only click should reserve a targeted turn"); session.finish_turn(pending, "token click"); @@ -1055,6 +1282,7 @@ mod tests { "click", &serde_json::json!({"pid": 1, "element_token": stale_token}), now_ms(), + RecordingCaller::default(), ) .expect("stale-token refusal should reserve an evidence turn"); session.finish_turn_with_outcome(pending, "stale token", None, true); @@ -1081,6 +1309,7 @@ mod tests { "click", &serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}), now_ms(), + RecordingCaller::default(), ) .expect("resolved refusal should reserve an evidence turn"); let refusal_record = crate::action_record::ActionExecutionRecord::builder( @@ -1162,7 +1391,12 @@ mod tests { inner.session_start_ms = now_ms(); } let pending = session - .begin_turn("click", &serde_json::json!({"x": 1, "y": 2}), now_ms()) + .begin_turn( + "click", + &serde_json::json!({"x": 1, "y": 2}), + now_ms(), + RecordingCaller::default(), + ) .expect("reserve first generation turn"); session.inner.lock().unwrap().generation = 2; session.finish_turn(pending, "must be discarded"); @@ -1195,6 +1429,7 @@ mod tests { "browser_prepare", &serde_json::json!({"pid": 1, "window_id": 2}), now_ms(), + RecordingCaller::default(), ) .expect("reserve private consent turn"); session.finish_turn(pending, "attached"); diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs index 43779ee71ba..ea2509aef8e 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs @@ -45,9 +45,13 @@ impl Tool for StartRecordingTool { fn def(&self) -> &ToolDef { START_REC_DEF.get_or_init(|| ToolDef { name: "start_recording".into(), - description: "Start trajectory recording. Every subsequent action-tool \ + description: "Start trajectory recording for the calling session. Each action-tool \ invocation (click, right_click, scroll, type_text, press_key, hotkey, \ - set_value) writes a turn folder under `output_dir`:\n\n\ + set_value) from that session writes a turn folder under `output_dir`. \ + Without `session`, calls on the same connection are recorded, including \ + named-session calls. Other connections and session lifecycle calls \ + (`start_session` / `end_session`) are not recorded. CLI recordings started \ + with `cua-driver recording start` are daemon-wide.\n\n\ - `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI \ state immediately before and after the action.\n\ - `before.png` / `after.png` — target-window screenshots immediately \ @@ -91,6 +95,10 @@ impl Tool for StartRecordingTool { "description": "Absolute or ~-rooted directory where turn folders \ and (when enabled) the video file are written." }, + "session": { + "type": "string", + "description": "Public session label to record. Omit to record calls on the authenticated connection." + }, "record_video": { "type": "boolean", "description": "Capture the main display to /recording.mp4. \ @@ -117,10 +125,11 @@ impl Tool for StartRecordingTool { return ToolResult::error("`output_dir` is required."); } let record_video = args.bool_or("record_video", false); - // Daemon-injected ownership key (absent for one-shot CLI / anonymous - // sessions). Stamps the recording so a session-scoped teardown - // (session_end) only stops the recording its own session started. - let owner = args.opt_str("_session_id"); + // Use the trusted connection when no named lifecycle was selected. + // CLI recording commands without either key remain daemon-wide. + let owner = args + .opt_str("_session_id") + .or_else(|| args.opt_str("_transport_session_id")); match self.session.start( output_dir.as_deref().unwrap(), diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/session.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/session.rs index 516ce8177fc..55e0da2ed61 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/session.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/session.rs @@ -457,15 +457,28 @@ fn begin_session_dispatch_inner( return Err("session has ended"); } let now = Instant::now(); - { - // Keep tombstone admission and live-record insertion in one critical - // section. Otherwise an end could land between the old pre-check and - // insertion, leaving a tombstoned record that was silently recreated - // by a racing first action. - let ended = ended_sessions().lock().unwrap(); - if ended.contains_key(session_id) { - return Err("session has ended"); - } + let recreated = loop { + // Match termination's lock order and recheck after cleanup: a transport + // close or explicit end racing that work must stay terminal. + let mut ended = ended_sessions().lock().unwrap(); + let recreated = if let Some(episode) = ended.get(session_id) { + if !implicit + || public_label.is_some() + || !episode.recreates_for(session_id, owner_transport) + { + return Err("session has ended"); + } + if cleanup_progress().lock().unwrap().contains_key(session_id) { + drop(ended); + if !retry_session_cleanup(session_id).complete { + return Err("session cleanup is incomplete; retry the call"); + } + continue; + } + true + } else { + false + }; let mut records = lifecycle_records().lock().unwrap(); let record = records .entry(session_id.to_owned()) @@ -490,11 +503,18 @@ fn begin_session_dispatch_inner( record.idle_ttl = idle_ttl; } record.in_flight += 1; - } + if recreated { + ended.remove(session_id); + } + break recreated; + }; activity() .lock() .unwrap() .insert(session_id.to_owned(), now); + if recreated { + fire_session_revive_for_owner(session_id, owner_transport); + } Ok(SessionDispatchGuard { session_id: session_id.to_owned(), }) @@ -612,7 +632,7 @@ pub fn activate_or_revive_session_for_owner( let now = Instant::now(); loop { let mut ended = ended_sessions().lock().unwrap(); - let revived = match ended.get(session_id) { + let revived = match ended.get(session_id).map(|ended| ended.owner.as_deref()) { Some(Some(owner)) if owner != owner_transport => { return Err("session is not available to this transport"); } @@ -707,7 +727,7 @@ fn session_owner_matches(session_id: &str, owner_transport: &str) -> bool { .lock() .unwrap() .get(session_id) - .is_some_and(|owner| owner.as_deref() == Some(owner_transport)) + .is_some_and(|ended| ended.owner.as_deref() == Some(owner_transport)) } /// End a lifecycle episode only when it belongs to the authenticated @@ -773,13 +793,25 @@ pub fn list_session_snapshots_with_prefix( } pub fn end_sessions_for_owner(owner_transport: &str, reason: SessionEndReason) -> usize { - let ids = lifecycle_records() + let mut ids = lifecycle_records() .lock() .unwrap() .iter() .filter(|(_, record)| record.owner_transport == owner_transport) .map(|(id, _)| id.clone()) .collect::>(); + ids.extend( + ended_sessions() + .lock() + .unwrap() + .iter() + .filter(|(_, ended)| { + ended.recreate_after_idle && ended.owner.as_deref() == Some(owner_transport) + }) + .map(|(id, _)| id.clone()), + ); + ids.sort(); + ids.dedup(); for id in &ids { end_session_with_reason(id, reason); } @@ -928,7 +960,20 @@ fn capture_modality_for(tool_name: &str, args: &serde_json::Value) -> Option>>> = OnceLock::new(); +struct EndedSession { + owner: Option, + recreate_after_idle: bool, +} + +impl EndedSession { + fn recreates_for(&self, session_id: &str, owner_transport: &str) -> bool { + self.recreate_after_idle + && session_id == owner_transport + && self.owner.as_deref() == Some(owner_transport) + } +} + +static ENDED_SESSIONS: OnceLock>> = OnceLock::new(); /// Runtime generations that have received terminal revoke-all. /// /// This latch is intentionally independent of grants and public session @@ -948,7 +993,7 @@ fn revive_hooks() -> &'static Mutex> { SESSION_REVIVE_HOOKS.get_or_init(|| Mutex::new(HashMap::new())) } -fn ended_sessions() -> &'static Mutex>> { +fn ended_sessions() -> &'static Mutex> { ENDED_SESSIONS.get_or_init(|| Mutex::new(HashMap::new())) } @@ -1127,7 +1172,7 @@ pub fn fire_session_end(session_id: &str) -> bool { } fn fire_session_end_for_owner(session_id: &str, owner_transport: Option<&str>) -> bool { - let first_fire = mark_session_ended(session_id, owner_transport); + let first_fire = mark_session_ended(session_id, owner_transport, SessionEndReason::Unknown); let _ = retry_session_cleanup(session_id); first_fire } @@ -1137,15 +1182,22 @@ fn fire_session_end_for_owner(session_id: &str, owner_transport: Option<&str>) - /// Hooks run after this short critical section. Keeping this transition under /// the same lock order as dispatch admission prevents a racing first action /// from recreating a record immediately before or after termination. -fn mark_session_ended(session_id: &str, owner_transport: Option<&str>) -> bool { +fn mark_session_ended( + session_id: &str, + owner_transport: Option<&str>, + reason: SessionEndReason, +) -> bool { activity().lock().unwrap().remove(session_id); let mut ended = ended_sessions().lock().unwrap(); - let record_owner = lifecycle_records() - .lock() - .unwrap() - .remove(session_id) - .map(|record| record.owner_transport); - if ended.contains_key(session_id) { + let record = lifecycle_records().lock().unwrap().remove(session_id); + let recreate_after_idle = reason == SessionEndReason::IdleTimeout + && record + .as_ref() + .is_some_and(|record| record.implicit && record.owner_transport == session_id); + if let Some(ended) = ended.get_mut(session_id) { + if reason != SessionEndReason::IdleTimeout { + ended.recreate_after_idle = false; + } false } else { crate::capture_scope::clear_session(session_id); @@ -1167,7 +1219,12 @@ fn mark_session_ended(session_id: &str, owner_transport: Option<&str>) -> bool { ); ended.insert( session_id.to_owned(), - owner_transport.map(str::to_owned).or(record_owner), + EndedSession { + owner: owner_transport + .map(str::to_owned) + .or(record.map(|record| record.owner_transport)), + recreate_after_idle, + }, ); true } @@ -1268,7 +1325,17 @@ pub fn session_cleanup_status(session_id: &str) -> SessionCleanupReport { /// a local operator control path without requiring an authorization grant: /// revocation can only remove authority, never create it. pub fn revoke_all_sessions() -> usize { - let sessions: Vec = activity().lock().unwrap().keys().cloned().collect(); + let mut sessions: Vec = activity().lock().unwrap().keys().cloned().collect(); + sessions.extend( + ended_sessions() + .lock() + .unwrap() + .iter() + .filter(|(_, ended)| ended.recreate_after_idle) + .map(|(id, _)| id.clone()), + ); + sessions.sort(); + sessions.dedup(); for session in &sessions { end_session(session); } @@ -1277,13 +1344,23 @@ pub fn revoke_all_sessions() -> usize { /// Revoke only sessions owned by one runtime-private namespace. pub fn revoke_sessions_with_prefix(prefix: &str) -> usize { - let sessions: Vec = activity() + let mut sessions: Vec = activity() .lock() .unwrap() .keys() .filter(|session| session.starts_with(prefix)) .cloned() .collect(); + sessions.extend( + ended_sessions() + .lock() + .unwrap() + .iter() + .filter(|(id, ended)| ended.recreate_after_idle && id.starts_with(prefix)) + .map(|(id, _)| id.clone()), + ); + sessions.sort(); + sessions.dedup(); for session in &sessions { end_session(session); } @@ -1315,6 +1392,16 @@ pub fn is_session_ended(session_id: &str) -> bool { ended_sessions().lock().unwrap().contains_key(session_id) } +/// Only an idle-reclaimed implicit lifecycle may restart on its owner's call. +/// Admission rechecks this state and requires completed cleanup before revival. +pub fn recreates_on_next_call(session_id: &str, owner_transport: &str) -> bool { + ended_sessions() + .lock() + .unwrap() + .get(session_id) + .is_some_and(|ended| ended.recreates_for(session_id, owner_transport)) +} + /// Revive a previously-ended session id by clearing its tombstone, so a fresh /// `start_session` with a recycled id works as a caller would expect: the id /// becomes live again and its actions stop being rejected by the resurrection @@ -1362,7 +1449,7 @@ pub fn revive_session_for_owner( let Some(ended_owner) = ended.get(session_id) else { return Ok(false); }; - match ended_owner.as_deref() { + match ended_owner.owner.as_deref() { Some(owner) if owner != owner_transport => { return Err("session is not available to this transport"); } @@ -1445,7 +1532,7 @@ fn end_session_with_reason(session_id: &str, reason: SessionEndReason) { } fn finish_session_end(session_id: &str, reason: SessionEndReason) { - let first_fire = mark_session_ended(session_id, None); + let first_fire = mark_session_ended(session_id, None, reason); let mut cursor_readers = CURSOR_OUTCOME_READERS .get() .map(|readers| { @@ -1546,6 +1633,164 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier}; + #[test] + fn idle_recreation_requires_owner_implicit_episode_and_complete_cleanup() { + let base = format!("g08-owner-cleanup-{}", std::process::id()); + let named = format!("{base}-named"); + let begin = |id: &str, label: Option<&str>, owner: &str| { + begin_session_dispatch( + id, + label, + owner, + label.is_none(), + SessionTransport::McpStdio, + SessionClientKind::Mcp, + ) + }; + let failures = Arc::new(AtomicUsize::new(2)); + let remaining = failures.clone(); + let observed = base.clone(); + let _hook = register_scoped_fallible_session_end_hook("g08-cleanup", move |id| { + if id == observed + && remaining + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Err("cleanup fixture incomplete".into()); + } + Ok(()) + }); + drop(begin(&base, None, &base).unwrap()); + drop(begin(&named, Some("named"), &base).unwrap()); + assert_eq!(evict_idle_with_prefix(Duration::ZERO, &base).len(), 2); + assert!(!recreates_on_next_call(&base, "foreign-owner")); + assert_eq!( + begin(&base, None, "foreign-owner").err(), + Some("session has ended") + ); + assert_eq!( + begin(&base, Some("explicit-label"), &base).err(), + Some("session has ended") + ); + assert_eq!( + begin(&named, Some("named"), &base).err(), + Some("session has ended") + ); + assert_eq!( + begin(&base, None, &base).err(), + Some("session cleanup is incomplete; retry the call") + ); + assert!(is_session_ended(&base)); + let guard = begin(&base, None, &base).unwrap(); + assert!(!is_session_ended(&base)); + assert!(evict_idle_with_prefix(Duration::ZERO, &base).is_empty()); + drop(guard); + assert!( + session_snapshot(&base, &base, DEFAULT_SESSION_IDLE_TTL) + .unwrap() + .implicit + ); + end_session(&base); + assert!(!recreates_on_next_call(&base, &base)); + assert!(begin(&base, None, &base).is_err()); + assert!(revive_session(&base)); + assert!(revive_session(&named)); + } + + #[test] + fn transport_close_and_revoke_make_idle_tombstones_terminal() { + for (index, revoke) in [false, true].into_iter().enumerate() { + let id = format!("g08-terminal-{}-{index}", std::process::id()); + drop( + begin_session_dispatch( + &id, + None, + &id, + true, + SessionTransport::McpStdio, + SessionClientKind::Mcp, + ) + .unwrap(), + ); + assert_eq!(evict_idle_with_prefix(Duration::ZERO, &id), [id.clone()]); + assert!(recreates_on_next_call(&id, &id)); + if revoke { + assert_eq!(revoke_sessions_with_prefix(&id), 1); + } else { + assert_eq!( + end_sessions_for_owner(&id, SessionEndReason::ProcessExit), + 1 + ); + } + assert!(!recreates_on_next_call(&id, &id)); + assert!(begin_session_dispatch( + &id, + None, + &id, + true, + SessionTransport::McpStdio, + SessionClientKind::Mcp + ) + .is_err()); + assert!(revive_session(&id)); + } + } + + #[test] + fn explicit_end_during_idle_cleanup_cannot_be_revived_by_a_waiting_call() { + let id = format!("g08-cleanup-race-{}", std::process::id()); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (finish_tx, finish_rx) = std::sync::mpsc::channel(); + let finish_rx = Mutex::new(finish_rx); + let observed = id.clone(); + let _hook = register_scoped_session_end_hook(move |ended| { + if ended == observed { + entered_tx.send(()).unwrap(); + finish_rx.lock().unwrap().recv().unwrap(); + } + }); + drop( + begin_session_dispatch( + &id, + None, + &id, + true, + SessionTransport::McpStdio, + SessionClientKind::Mcp, + ) + .unwrap(), + ); + let swept = id.clone(); + let sweep = std::thread::spawn(move || evict_idle_with_prefix(Duration::ZERO, &swept)); + entered_rx.recv().unwrap(); + assert_eq!( + begin_session_dispatch( + &id, + None, + &id, + true, + SessionTransport::McpStdio, + SessionClientKind::Mcp + ) + .err(), + Some("session cleanup is incomplete; retry the call") + ); + end_session(&id); + finish_tx.send(()).unwrap(); + assert_eq!(sweep.join().unwrap(), [id.clone()]); + assert!(!recreates_on_next_call(&id, &id)); + assert!(begin_session_dispatch( + &id, + None, + &id, + true, + SessionTransport::McpStdio, + SessionClientKind::Mcp + ) + .is_err()); + assert!(revive_session(&id)); + } + #[derive(Default)] struct ProbeObserver { starts: Mutex>, @@ -2026,7 +2271,7 @@ mod tests { Ok(()) }); - assert!(mark_session_ended(sid, None)); + assert!(mark_session_ended(sid, None, SessionEndReason::Explicit)); let pending = session_cleanup_status(sid); assert!( !pending.complete, diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs index b84ff0cdec6..83c1e5414cb 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs @@ -1223,6 +1223,9 @@ fn canonical_manifest_path(path: &Path) -> Result { } Ok(_) => return Err("path contains an unavailable filesystem entry".to_owned()), Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) if error.kind() == std::io::ErrorKind::NotADirectory => { + return Err("an ancestor of the path is not a directory".to_owned()) + } Err(error) => return Err(error.to_string()), } let name = existing @@ -1233,9 +1236,11 @@ fn canonical_manifest_path(path: &Path) -> Result { .parent() .ok_or_else(|| "path has no existing ancestor".to_owned())?; } - let metadata = std::fs::symlink_metadata(existing).map_err(|error| error.to_string())?; + // Follow a symlinked deepest ancestor (macOS `/tmp` -> `/private/tmp`); + // `canonicalize` below records its resolved target. + let metadata = std::fs::metadata(existing).map_err(|error| error.to_string())?; if !metadata.is_dir() { - return Err("existing ancestor is not a directory".to_owned()); + return Err("deepest existing ancestor is not a directory".to_owned()); } let mut canonical = std::fs::canonicalize(existing).map_err(|error| error.to_string())?; for component in suffix.into_iter().rev() { @@ -1673,6 +1678,31 @@ allow: .unwrap(); } + #[cfg(all(feature = "yaml", unix))] + #[test] + fn manifest_paths_resolve_a_symlinked_deepest_ancestor() { + let root = tempfile::tempdir().unwrap(); + let target = root.path().join("private-tmp"); + std::fs::create_dir(&target).unwrap(); + let link = root.path().join("tmp"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + assert_eq!( + canonical_manifest_path(&link.join("out.png")).unwrap(), + std::fs::canonicalize(&target) + .unwrap() + .join("out.png") + .to_string_lossy() + ); + let file = root.path().join("file"); + std::fs::write(&file, b"").unwrap(); + let file_link = root.path().join("file-link"); + std::os::unix::fs::symlink(&file, &file_link).unwrap(); + assert_eq!( + canonical_manifest_path(&file_link.join("out.png")).unwrap_err(), + "an ancestor of the path is not a directory" + ); + } + #[cfg(feature = "yaml")] #[test] fn protected_resources_are_exactly_bounded_by_the_manifest() { diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/tool.rs index edb3d728201..5f8600e11ba 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/tool.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/tool.rs @@ -36,6 +36,27 @@ tokio::task_local! { /// Opaque generation for runtime-owned mutable resources. Nested /// dispatches inherit this key, while public arguments can never select it. static DISPATCH_RUNTIME_SCOPE: String; + static DISPATCH_COMPUTER_USE_NATIVE: bool; + static DISPATCH_AX_READINESS_DEADLINE: Option; +} + +#[doc(hidden)] +pub const AX_READINESS_BUDGET: Duration = Duration::from_millis(500); + +// Read before spawn_blocking, then pass the value explicitly to native code. +#[doc(hidden)] +pub fn current_ax_readiness_deadline() -> Option { + DISPATCH_AX_READINESS_DEADLINE + .try_with(|value| *value) + .ok() + .flatten() +} + +#[doc(hidden)] +pub fn computer_use_native_dispatch() -> bool { + DISPATCH_COMPUTER_USE_NATIVE + .try_with(|native| *native) + .unwrap_or(false) } /// Return the immutable authorization context bound to the current dispatch. @@ -56,6 +77,48 @@ pub fn current_dispatch_runtime_scope() -> Option { .or_else(|| CONSTRUCTION_RUNTIME_SCOPE.with(|scope| scope.borrow().clone())) } +// Recording workers retain the dispatch that requested their read, even when +// the caller times out before the native provider returns. +pub(crate) struct CapturedDispatchContext { + runtime: Option, + authorization: Option>, + evidence: Option, + native: Option, +} + +impl CapturedDispatchContext { + pub(crate) fn capture() -> Self { + Self { + runtime: current_dispatch_runtime_scope(), + authorization: current_dispatch_authorization_context(), + evidence: DISPATCH_TRUSTED_INVOCATION_EVIDENCE + .try_with(Clone::clone) + .ok(), + native: DISPATCH_COMPUTER_USE_NATIVE.try_with(|value| *value).ok(), + } + } + + pub(crate) fn run_sync(self, action: impl FnOnce() -> R) -> R { + fn enter( + key: &'static tokio::task::LocalKey, + value: Option, + action: impl FnOnce() -> R, + ) -> R { + match value { + Some(value) => key.sync_scope(value, action), + None => action(), + } + } + enter(&DISPATCH_RUNTIME_SCOPE, self.runtime, || { + enter(&DISPATCH_AUTHORIZATION_CONTEXT, self.authorization, || { + enter(&DISPATCH_TRUSTED_INVOCATION_EVIDENCE, self.evidence, || { + enter(&DISPATCH_COMPUTER_USE_NATIVE, self.native, action) + }) + }) + }) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub(crate) struct DispatchSessionIdentity { pub runtime_scope: String, @@ -245,6 +308,13 @@ impl ToolDef { fn advertised_runtime_input_schema(tool_name: &str, schema: &Value) -> Value { let mut schema = schema.clone(); + cua_driver_contract::normalize_input_schema(&mut schema); + if tool_name == "set_config" { + if let Some(properties) = schema.get_mut("properties").and_then(Value::as_object_mut) { + properties.remove("key"); + properties.remove("value"); + } + } if !crate::action_target::supports_typed_target(tool_name) { return schema; } @@ -598,6 +668,7 @@ impl TrustedInvocationEvidence { /// Thread-safe collection of all registered tools. pub struct ToolRegistry { + computer_use: Option>, tools: HashMap>, /// Ordered list of tool names for `tools/list`. order: Vec, @@ -640,6 +711,7 @@ impl ToolRegistry { let weak_ownership = Arc::downgrade(&protected_resource_ownership); let session_end_hook = crate::session::register_scoped_session_end_hook(move |session_id| { + crate::element_token::global().clear_session(session_id); if let Some(ownership) = weak_ownership.upgrade() { ownership.remove_session(session_id); } @@ -668,6 +740,7 @@ impl ToolRegistry { }) })); Self { + computer_use: None, tools: HashMap::new(), order: Vec::new(), recording, @@ -776,6 +849,27 @@ impl ToolRegistry { self.tools.insert(name, tool); } + pub fn enable_computer_use(&mut self) { + if self.computer_use.is_some() { + return; + } + let runtime = Arc::new(crate::computer_use::ComputerUseRuntime::default()); + let weak = Arc::downgrade(&runtime); + self.retain_session_end_hook(crate::session::register_scoped_session_end_hook( + move |session| { + if let Some(runtime) = weak.upgrade() { + runtime.clear_session(session); + } + }, + )); + for spec in cua_driver_contract::computer_use::tools() { + if !self.tools.contains_key(spec.name) { + self.register(Box::new(crate::computer_use::Entry::new(spec))); + } + } + self.computer_use = Some(runtime); + } + pub fn retain_session_end_hook( &mut self, registration: crate::session::SessionEndHookRegistration, @@ -802,6 +896,25 @@ impl ToolRegistry { .push(RuntimeCleanup(Some(Box::new(cleanup)))); } + pub(crate) async fn invoke_computer_use_native(&self, name: &str, args: Value) -> ToolResult { + self.invoke_computer_use_native_with_readiness(name, args, None) + .await + } + + pub(crate) async fn invoke_computer_use_native_with_readiness( + &self, + name: &str, + args: Value, + deadline: Option, + ) -> ToolResult { + DISPATCH_AX_READINESS_DEADLINE + .scope( + deadline, + DISPATCH_COMPUTER_USE_NATIVE.scope(true, Box::pin(self.invoke(name, args))), + ) + .await + } + /// Register the four platform-independent recording/replay tools. /// Call this after all platform tools have been registered. pub fn register_recording_tools(&mut self) { @@ -839,13 +952,52 @@ impl ToolRegistry { } pub fn tools_list(&self) -> Value { - let list: Vec = self + let mut list: Vec = self .order .iter() .filter(|name| crate::policy::is_tool_listable(name)) .filter_map(|name| self.tools.get(name)) .map(|tool| tool.def().to_list_entry()) .collect(); + if self.normalized_coordinates { + let mut native = serde_json::json!({"tools":list}); + crate::coord_norm::rewrite_coord_desc(&mut native); + list = native["tools"].as_array().unwrap().clone(); + } + if self.computer_use.is_some() { + let extra = std::env::var("CUA_DRIVER_EXTRA_TOOLS").as_deref() == Ok("1"); + let mut entries = Vec::new(); + for spec in cua_driver_contract::computer_use::tools() { + if !crate::policy::is_tool_listable(spec.name) { + continue; + } + let mut schema = spec.input_schema.clone(); + if extra { + if let Some(native) = list.iter().find(|tool| tool["name"] == spec.name) { + schema = serde_json::json!({"type":"object","anyOf":[schema,native["inputSchema"]]}); + } + } + let risk_tool = match spec.name { + "get_app" => "get_window_state", + "get_state" => "list_apps", + "get_ax_state" | "get_screenshot" | "get_ax_state_and_screenshot" => { + "get_window_state" + } + _ => spec.name, + }; + entries.push(serde_json::json!({ + "name":spec.name,"description":spec.description,"inputSchema":schema, + "annotations":{"readOnlyHint":spec.read_only,"destructiveHint":false,"idempotentHint":spec.read_only,"openWorldHint":!spec.read_only}, + "capabilities":default_capabilities_for(risk_tool), + "risk":crate::authorization::risk_metadata_json(risk_tool), + })); + } + if extra { + list.retain(|entry| !entries.iter().any(|app| app["name"] == entry["name"])); + entries.extend(list); + } + list = entries; + } // `capability_version` is the contract version for the // capability tokens claimed by each tool entry. Bumped on // BREAKING vocabulary changes only; additive changes (new @@ -860,16 +1012,13 @@ impl ToolRegistry { // // Both fields are additive: existing consumers that read only // `tools` keep working unchanged. - let mut result = serde_json::json!({ + let result = serde_json::json!({ "tools": list, "platform": std::env::consts::OS, "capability_version": CAPABILITY_VERSION, "schema_version": TOOLS_LIST_SCHEMA_VERSION, "enforcement_adapters": crate::authorization::enforcement_adapter_inventory_json(), }); - if self.normalized_coordinates { - crate::coord_norm::rewrite_coord_desc(&mut result); - } result } @@ -1071,6 +1220,16 @@ impl ToolRegistry { ); } + if let Some(runtime) = &self.computer_use { + if !DISPATCH_COMPUTER_USE_NATIVE + .try_with(|native| *native) + .unwrap_or(false) + && cua_driver_contract::computer_use::is_app_call(name, &args) + { + return runtime.invoke(self, name, args).await; + } + } + // Deprecated alias: `type_text_chars` → `type_text`. Swift's // ToolRegistry.swift keeps the same alias (with stderr warning) for // backwards compatibility with hermes-agent builds that still emit @@ -1134,8 +1293,13 @@ impl ToolRegistry { .and_then(Value::as_str) .map(str::to_owned); if let Some(session) = runtime_session.as_deref() { + let owner = args + .get("_transport_session_id") + .and_then(Value::as_str) + .unwrap_or(session); if !matches!(resolved_name, "start_session" | "end_session") && crate::session::is_session_ended(session) + && !crate::session::recreates_on_next_call(session, owner) { let mut result = protected_refusal( "session_ended", @@ -1149,7 +1313,7 @@ impl ToolRegistry { // Convert only the private dispatch copy. Authorization, consent, // manifests, recordings, and replay continue to use the caller's // normalized arguments, while platform workers receive pixels. - if self.normalized_coordinates { + if self.normalized_coordinates && !computer_use_native_dispatch() { let pid = args.opt_i64("pid").unwrap_or(0); let window_id = args.opt_u64("window_id").unwrap_or(0); let (width, height) = crate::coord_norm::get_size(pid, window_id).unwrap_or((0, 0)); @@ -1166,8 +1330,9 @@ impl ToolRegistry { // rejected action has no before/after evidence and must not leave a // pending recorder entry behind. if let Err(violation) = crate::capture_scope::enforce_tool(resolved_name, &args) { - let structured = + let mut structured = violation.as_json(args.get("session").and_then(Value::as_str).unwrap_or("")); + structured["sent"] = serde_json::json!(false); let mut result = ToolResult::error(violation.message).with_structured(structured); restore_public_runtime_result(&mut result, &runtime_prefix); return result; @@ -1523,7 +1688,12 @@ impl ToolRegistry { let should_record = !tool.def().read_only && !matches!( resolved_name, - "start_recording" | "stop_recording" | "get_recording_state" | "replay_trajectory" + "start_recording" + | "stop_recording" + | "get_recording_state" + | "replay_trajectory" + | "start_session" + | "end_session" ); let private_consent_turn = is_existing_profile_prepare(resolved_name, &args); let _desktop_action = if is_physical_desktop_action(resolved_name) { @@ -1540,14 +1710,27 @@ impl ToolRegistry { } else { None }; + if is_physical_desktop_action(resolved_name) { + if let Err(result) = crate::desktop_keyboard::admit_input(resolved_name, &args) { + return result; + } + } let pending_turn = should_record .then(|| { + let caller = crate::recording::RecordingCaller { + session: runtime_session.as_deref(), + transport: args.get("_transport_session_id").and_then(Value::as_str), + }; if private_consent_turn { - self.recording - .begin_private_turn(resolved_name, &recording_args, start_ms) + self.recording.begin_private_turn( + resolved_name, + &recording_args, + start_ms, + caller, + ) } else { self.recording - .begin_turn(resolved_name, &recording_args, start_ms) + .begin_turn(resolved_name, &recording_args, start_ms, caller) } }) .flatten(); @@ -1558,7 +1741,9 @@ impl ToolRegistry { crate::coord_norm::ingest_window_size(resolved_name, &args, &result); crate::coord_norm::ingest_screen_size(resolved_name, &result); crate::coord_norm::ingest_zoom_size(resolved_name, &args, &result); - crate::coord_norm::normalize_result(resolved_name, &mut result); + if !computer_use_native_dispatch() { + crate::coord_norm::normalize_result(resolved_name, &mut result); + } } // The platform worker has exited, so another text operation for this // pid may now start even while result projection and evidence capture @@ -1858,7 +2043,7 @@ impl ToolRegistry { // checked against its approved manifest. return Ok(()); } - let delivery_mode = if tool_name == "bring_to_front" { + let delivery_mode = if tool_name == "bring_to_front" || args["scope"] == "desktop" { "foreground" } else { args.get("delivery_mode") @@ -2445,7 +2630,12 @@ fn canonical_existing_file(raw: &str) -> Result { /// The deepest existing ancestor is canonicalized first, so symlinked parents /// are captured in the approved identity. Only normal path components may be /// appended after that ancestor; lexical parent traversal never enters a -/// protected-resource digest. +/// protected-resource digest. The caller replaces the raw argument with this +/// canonical path, so the tool writes exactly the approved location. +/// +/// A deepest existing ancestor that is itself a symbolic link to a directory +/// (macOS `/tmp` -> `/private/tmp`) is resolved like any other symlinked +/// parent. A link to a non-directory is refused. fn canonical_proposed_path(raw: &str) -> Result { let path = expanded_path(raw)?; if path.exists() { @@ -2469,6 +2659,11 @@ fn canonical_proposed_path(raw: &str) -> Result { )) } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) if error.kind() == std::io::ErrorKind::NotADirectory => { + return Err(protected_scope_refusal( + "an ancestor of the output path is not a directory", + )) + } Err(_) => { return Err(protected_scope_refusal( "the output path could not be inspected safely", @@ -2483,11 +2678,13 @@ fn canonical_proposed_path(raw: &str) -> Result { .parent() .ok_or_else(|| protected_scope_refusal("the output path has no existing ancestor"))?; } - let metadata = std::fs::symlink_metadata(existing) + // Follow a symlinked ancestor: `exists()` above already resolved it, and + // `canonicalize` below records its target in the approved identity. + let metadata = std::fs::metadata(existing) .map_err(|_| protected_scope_refusal("the output ancestor is unavailable"))?; if !metadata.is_dir() { return Err(protected_scope_refusal( - "the output path's existing ancestor is not a directory", + "the output path's deepest existing ancestor is not a directory", )); } let mut canonical = std::fs::canonicalize(existing) @@ -2523,6 +2720,8 @@ fn is_physical_desktop_action(tool: &str) -> bool { | "mouse_button_up" | "type_text" | "press_key" + | "key_down" + | "key_up" | "hotkey" | "set_value" | "paste" @@ -2732,6 +2931,37 @@ fn restore_public_runtime_value(value: &mut Value, runtime_prefix: &str) -> bool #[cfg(test)] mod runtime_isolation_tests { + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn readiness_deadline_follows_the_task_and_is_explicitly_captured_for_workers() { + let first = std::time::Instant::now(); + let second = first + std::time::Duration::from_secs(1); + let run = |deadline| { + tokio::spawn(async move { + super::DISPATCH_AX_READINESS_DEADLINE + .scope(Some(deadline), async move { + tokio::task::yield_now().await; + assert_eq!(super::current_ax_readiness_deadline(), Some(deadline)); + let captured = super::current_ax_readiness_deadline(); + let worker = tokio::task::spawn_blocking(move || { + assert!(super::current_ax_readiness_deadline().is_none()); + captured + }) + .await + .unwrap(); + assert_eq!(worker, Some(deadline)); + tokio::task::yield_now().await; + assert_eq!(super::current_ax_readiness_deadline(), Some(deadline)); + }) + .await; + assert!(super::current_ax_readiness_deadline().is_none()); + }) + }; + let (a, b) = tokio::join!(run(first), run(second)); + a.unwrap(); + b.unwrap(); + assert!(super::current_ax_readiness_deadline().is_none()); + } + use super::{ canonical_proposed_path, desktop_action_coordinator, namespace_runtime_args, publish_action_result, restore_public_runtime_result, try_admit_text_input, @@ -2763,6 +2993,74 @@ mod runtime_isolation_tests { .unwrap() } + #[tokio::test] + async fn captured_dispatch_context_preserves_task_scope_and_restores_worker_scope() { + let auth = standard_context(); + let evidence = TrustedInvocationEvidence { + session_id: Some("session-a".into()), + transport_session_id: Some("transport-a".into()), + ..Default::default() + }; + DISPATCH_RUNTIME_SCOPE.scope("runtime-a".into(), async { + super::DISPATCH_AUTHORIZATION_CONTEXT.scope(auth.clone(), async { + super::DISPATCH_TRUSTED_INVOCATION_EVIDENCE.scope(evidence, async { + super::DISPATCH_COMPUTER_USE_NATIVE.scope(true, async { + let context = super::with_runtime_scope("construction-fallback".into(), || { + super::CapturedDispatchContext::capture() + }); + let worker_auth = auth.clone(); + std::thread::spawn(move || { + super::with_runtime_scope("worker-fallback".into(), || { + DISPATCH_RUNTIME_SCOPE.sync_scope("worker-runtime".into(), || { + context.run_sync(|| { + assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("runtime-a")); + assert!(Arc::ptr_eq(&super::current_dispatch_authorization_context().unwrap(), &worker_auth)); + assert!(super::computer_use_native_dispatch()); + assert_eq!(super::current_dispatch_session_identity(), Some(super::DispatchSessionIdentity { + runtime_scope: "runtime-a".into(), + session_id: "__cua_runtime_runtime-a:session-a".into(), + transport_session_id: "__cua_runtime_runtime-a:transport-a".into(), + })); + }); + assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("worker-runtime")); + assert!(super::current_dispatch_authorization_context().is_none()); + assert!(super::current_dispatch_session_identity().is_none()); + assert!(!super::computer_use_native_dispatch()); + }); + assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("worker-fallback")); + }); + assert!(super::current_dispatch_runtime_scope().is_none()); + }).join().unwrap(); + assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("runtime-a")); + assert!(super::computer_use_native_dispatch()); + }).await; + }).await; + }).await; + }).await; + assert!(super::current_dispatch_runtime_scope().is_none()); + assert!(super::current_dispatch_authorization_context().is_none()); + } + + #[test] + fn captured_dispatch_context_preserves_explicit_thread_runtime() { + let context = super::with_runtime_scope("construction-runtime".into(), || { + super::CapturedDispatchContext::capture() + }); + assert!(super::current_dispatch_runtime_scope().is_none()); + std::thread::spawn(move || { + context.run_sync(|| { + assert_eq!( + super::current_dispatch_runtime_scope().as_deref(), + Some("construction-runtime") + ); + assert!(super::current_dispatch_session_identity().is_none()); + }); + assert!(super::current_dispatch_runtime_scope().is_none()); + }) + .join() + .unwrap(); + } + fn unrestricted_context() -> Arc { let ceiling = SessionModeCeiling::for_trusted_sessions( [PermissionMode::Unrestricted], @@ -3124,6 +3422,139 @@ mod runtime_isolation_tests { registry } + fn recording_scope_registry(hits: Arc) -> Arc { + let mut registry = super::ToolRegistry::new(); + registry.register(Box::new(ReplayProbe { + hits, + def: super::ToolDef { + name: "probe".into(), + description: "runtime-local recording-scope probe".into(), + input_schema: serde_json::json!({ + "type": "object", + "properties": { "session": { "type": "string" } }, + }), + read_only: false, + destructive: false, + idempotent: true, + open_world: false, + }, + })); + registry.register_recording_tools(); + registry.register_session_tools(); + let registry = Arc::new(registry); + registry.init_self_weak(); + registry + } + + #[tokio::test] + async fn recording_keeps_only_the_owning_session_and_never_lifecycle_calls() { + let registry = recording_scope_registry(Arc::new(AtomicUsize::new(0))); + let context = unrestricted_context(); + let output = tempfile::tempdir().expect("temp dir"); + + let started = registry + .invoke_with_context( + "start_recording", + serde_json::json!({ + "output_dir": output.path(), + "record_video": false, + "session": "owner", + }), + context.clone(), + ) + .await; + assert_ne!(started.is_error, Some(true), "{started:?}"); + + for (tool, session) in [ + ("probe", "other"), + ("end_session", "other"), + ("end_session", "owner-lifecycle"), + ] { + let result = registry + .invoke_with_context( + tool, + serde_json::json!({ "session": session }), + context.clone(), + ) + .await; + assert_ne!(result.is_error, Some(true), "{tool} {session}: {result:?}"); + } + + let owned = registry + .invoke_with_context( + "probe", + serde_json::json!({ "session": "owner" }), + context.clone(), + ) + .await; + assert_ne!(owned.is_error, Some(true), "{owned:?}"); + registry + .invoke_with_context("stop_recording", serde_json::json!({}), context) + .await; + + let action: serde_json::Value = serde_json::from_slice( + &std::fs::read(output.path().join("turn-00001").join("action.json")) + .expect("the owning session's action is turn-00001"), + ) + .expect("parse action.json"); + assert_eq!(action["tool"], "probe"); + assert_eq!(action["arguments"]["session"], "owner"); + assert!( + !output.path().join("turn-00002").exists(), + "foreign and lifecycle calls must not leave turn folders" + ); + } + + #[tokio::test] + async fn implicit_recording_keeps_only_calls_on_the_owning_connection() { + let registry = recording_scope_registry(Arc::new(AtomicUsize::new(0))); + let context = unrestricted_context(); + let output = tempfile::tempdir().expect("temp dir"); + let evidence = |transport| { + super::TrustedInvocationEvidence::extract_from_adapter_args( + &mut serde_json::json!({"_transport_session_id": transport}), + ) + }; + let started = registry + .invoke_with_context_and_evidence( + "start_recording", + serde_json::json!({"output_dir": output.path(), "record_video": false}), + context.clone(), + evidence("owner-connection"), + ) + .await; + assert_ne!(started.is_error, Some(true), "{started:?}"); + assert!(registry.recording.current_state().owner.is_some()); + for (transport, session) in [ + ("owner-connection", "named-on-owner"), + ("foreign-connection", "foreign"), + ] { + let result = registry + .invoke_with_context_and_evidence( + "probe", + serde_json::json!({"session": session}), + context.clone(), + evidence(transport), + ) + .await; + assert_ne!(result.is_error, Some(true), "{result:?}"); + } + registry + .invoke_with_context_and_evidence( + "stop_recording", + serde_json::json!({}), + context, + evidence("owner-connection"), + ) + .await; + let action: serde_json::Value = serde_json::from_slice( + &std::fs::read(output.path().join("turn-00001/action.json")).unwrap(), + ) + .unwrap(); + assert_eq!(action["arguments"]["session"], "named-on-owner"); + assert!(!output.path().join("turn-00002").exists()); + } + #[tokio::test] async fn replay_dispatches_only_through_the_owning_registry() { let hits_a = Arc::new(AtomicUsize::new(0)); @@ -3172,6 +3603,122 @@ mod runtime_isolation_tests { assert_ne!(result.is_error, Some(true)); } + struct IdleSnapshotProbe(super::ToolDef); + + #[async_trait::async_trait] + impl super::Tool for IdleSnapshotProbe { + fn def(&self) -> &super::ToolDef { + &self.0 + } + async fn invoke(&self, args: serde_json::Value) -> ToolResult { + let owner = args["_session_id"].as_str().expect("admitted session"); + let window = args["window_id"].as_u64().unwrap() as u32; + let id = crate::element_token::global().register_snapshot_payload_for_session( + 42, + window, + [0], + Arc::new(owner.to_owned()), + Some(owner), + ); + ToolResult::text("snapshot").with_structured(serde_json::json!({ + "token": crate::element_token::token_for(id, 0), + })) + } + } + + #[tokio::test] + async fn idle_implicit_recreation_retires_only_its_owned_raw_snapshots() { + let mut registry = super::ToolRegistry::new_with_protected_consent_provider(None); + registry.register(Box::new(IdleSnapshotProbe(super::ToolDef { + name: "get_window_state".into(), + description: "snapshot owner probe".into(), + input_schema: serde_json::json!({"type":"object"}), + read_only: true, + destructive: false, + idempotent: true, + open_world: false, + }))); + registry.register_session_tools(); + let context = standard_context(); + let call = |session: Option<&str>, window: u32| { + let mut args = serde_json::json!({"pid":42,"window_id":window}); + if let Some(session) = session { + args["session"] = session.into(); + } + registry.invoke_with_context("get_window_state", args, context.clone()) + }; + let first = call(None, 7).await; + let peer = call(Some("peer"), 8).await; + assert_ne!(first.is_error, Some(true), "{first:?}"); + assert_ne!(peer.is_error, Some(true), "{peer:?}"); + let token = first.structured_content.unwrap()["token"] + .as_str() + .unwrap() + .to_owned(); + let peer_token = peer.structured_content.unwrap()["token"] + .as_str() + .unwrap() + .to_owned(); + let implicit = context.runtime_session_key("implicit-direct"); + assert_eq!( + crate::session::evict_idle_with_prefix(Duration::ZERO, &implicit), + [implicit.clone()] + ); + super::DISPATCH_RUNTIME_SCOPE + .scope(context.runtime_scope_key(), async { + assert!( + crate::element_token::global().resolve(42, &token).is_err(), + "idle cleanup must retire the old snapshot before any new observation" + ); + assert!(crate::element_token::global() + .resolve(42, &peer_token) + .is_ok()); + }) + .await; + let fresh = call(None, 7).await; + assert_ne!(fresh.is_error, Some(true), "{fresh:?}"); + let new_token = fresh.structured_content.unwrap()["token"] + .as_str() + .unwrap() + .to_owned(); + super::DISPATCH_RUNTIME_SCOPE + .scope(context.runtime_scope_key(), async { + assert!(crate::element_token::global().resolve(42, &token).is_err()); + assert!(crate::element_token::global() + .resolve(42, &new_token) + .is_ok()); + assert!(crate::element_token::global() + .resolve(42, &peer_token) + .is_ok()); + }) + .await; + let peer_id = context.runtime_session_key("peer"); + assert_eq!( + crate::session::evict_idle_with_prefix(Duration::ZERO, &peer_id), + [peer_id] + ); + assert_eq!( + call(Some("peer"), 8).await.structured_content.unwrap()["refusal"]["code"], + "session_ended" + ); + let ended = registry + .invoke_with_context("end_session", serde_json::json!({}), context.clone()) + .await; + assert_ne!(ended.is_error, Some(true), "{ended:?}"); + assert_eq!( + call(None, 7).await.structured_content.unwrap()["refusal"]["code"], + "session_ended" + ); + let restarted = registry + .invoke_with_context("start_session", serde_json::json!({}), context.clone()) + .await; + assert_ne!(restarted.is_error, Some(true), "{restarted:?}"); + assert_ne!(call(None, 7).await.is_error, Some(true)); + let prefix = context.runtime_session_key(""); + crate::session::revoke_sessions_with_prefix(&prefix); + crate::session::forget_ended_sessions_with_prefix(&prefix); + } + #[tokio::test] async fn default_public_session_uses_an_implicit_runtime_lifecycle() { let hits = Arc::new(AtomicUsize::new(0)); @@ -4243,6 +4790,55 @@ resources: assert_eq!(provider.requests.load(Ordering::SeqCst), 0); } + #[cfg(unix)] + #[test] + fn proposed_output_scope_resolves_a_symlinked_deepest_ancestor() { + // Mirrors macOS `/tmp/x.png`, where `/tmp` links to `/private/tmp`. + let root = tempfile::tempdir().unwrap(); + let target = root.path().join("private-tmp"); + std::fs::create_dir(&target).unwrap(); + let link = root.path().join("tmp"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + let proposed = link.join("x.png"); + + let canonical = canonical_proposed_path(proposed.to_str().unwrap()).unwrap(); + + assert_eq!( + canonical, + std::fs::canonicalize(&target) + .unwrap() + .join("x.png") + .to_string_lossy() + ); + assert!(!proposed.exists()); + // The same file already existing was always resolved; both agree. + std::fs::write(target.join("x.png"), b"").unwrap(); + assert_eq!( + canonical_proposed_path(proposed.to_str().unwrap()).unwrap(), + canonical + ); + } + + #[cfg(unix)] + #[test] + fn proposed_output_scope_refuses_a_symlinked_ancestor_that_is_not_a_directory() { + let root = tempfile::tempdir().unwrap(); + let file = root.path().join("file"); + std::fs::write(&file, b"").unwrap(); + let link = root.path().join("link"); + std::os::unix::fs::symlink(&file, &link).unwrap(); + let proposed = link.join("x.png"); + + let refusal = canonical_proposed_path(proposed.to_str().unwrap()).unwrap_err(); + + let text = serde_json::to_string(&refusal).unwrap(); + assert!( + text.contains("an ancestor of the output path is not a directory"), + "{text}" + ); + assert!(!file.with_file_name("x.png").exists()); + } + #[test] fn proposed_output_scope_canonicalizes_the_existing_ancestor_without_creating_output() { let root = tempfile::tempdir().unwrap(); @@ -4585,6 +5181,7 @@ resources: fn permission_denied_result(message: String) -> ToolResult { ToolResult::error(message.clone()).with_structured(serde_json::json!({ "status": "refused", + "sent": false, "refusal": { "code": "permission_denied", "message": message, @@ -4614,6 +5211,7 @@ fn protected_consent_refusal(error: crate::consent::ConsentError) -> ToolResult let message = error.to_string(); ToolResult::error(message.clone()).with_structured(serde_json::json!({ "status": "refused", + "sent": false, "refusal": { "code": code, "message": message, @@ -4628,6 +5226,7 @@ fn protected_scope_refusal(message: &str) -> ToolResult { fn protected_refusal(code: &str, message: &str) -> ToolResult { ToolResult::error(message).with_structured(serde_json::json!({ "status": "refused", + "sent": false, "refusal": { "code": code, "message": message, diff --git a/packages/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs b/packages/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs index ed48f7b6d0b..d4ac3f317ba 100644 --- a/packages/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs +++ b/packages/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs @@ -138,51 +138,47 @@ impl FfmpegVideoBackend { impl VideoBackend for FfmpegVideoBackend { fn stop(mut self: Box) -> anyhow::Result { let elapsed = self.started_at.elapsed(); - let finalized; - if let Some(mut stdin) = self.child.stdin.take() { let _ = stdin.write_all(b"q\n"); let _ = stdin.flush(); } - let deadline = Instant::now() + Duration::from_millis(3000); - loop { + let shutdown_timeout = Duration::from_millis(3000); + let deadline = Instant::now() + shutdown_timeout; + let result = loop { match self.child.try_wait()? { + Some(status) if status.success() => break Ok(()), Some(status) => { - finalized = status.success(); - break; + let cause = status + .code() + .map_or_else(|| status.to_string(), |code| format!("code {code}")); + break Err(anyhow::anyhow!("ffmpeg exited with {cause}")); } - None => { - if Instant::now() > deadline { - // Polite shutdown stalled — force kill. mp4 will lack - // a moov atom and won't be playable; `finalized: - // false` tells the caller. - let _ = self.child.kill(); - let _ = self.child.wait(); - finalized = false; - break; - } - std::thread::sleep(Duration::from_millis(80)); + None if Instant::now() > deadline => { + let _ = self.child.kill(); + let _ = self.child.wait(); + break Err(anyhow::anyhow!( + "ffmpeg shutdown timed out after {} ms", + shutdown_timeout.as_millis() + )); } + None => std::thread::sleep(Duration::from_millis(80)), + } + }; + if let Some(handle) = self.stderr_thread.take() { + let stderr = handle.join().unwrap_or_default(); + if let Err(error) = &result { + tracing::warn!(target: "recording", + %error, + stderr_tail = %String::from_utf8_lossy(&stderr), + "ffmpeg shutdown failed"); } } - - if !finalized { - if let Some(handle) = self.stderr_thread.take() { - if let Ok(buf) = handle.join() { - let tail = String::from_utf8_lossy(&buf); - tracing::warn!(target: "recording", - "ffmpeg did not finalize cleanly. Last stderr tail:\n{tail}"); - } - } - } else if let Some(handle) = self.stderr_thread.take() { - let _ = handle.join(); - } - + result?; Ok(VideoMetadata { path: self.output_path, duration_ms: elapsed.as_millis() as u64, - finalized, + finalized: true, }) } } @@ -333,3 +329,66 @@ fn platform_input_args(cmd: &mut Command) { let _ = cmd; } } + +#[cfg(test)] +mod shutdown_tests { + use super::*; + + #[test] + fn encoder_child() { + let Ok(mode) = std::env::var("CUA_P7_ENCODER_CHILD_MODE") else { + return; + }; + let mut line = String::new(); + std::io::stdin().read_line(&mut line).unwrap(); + assert_eq!(line, "q\n"); + eprintln!("owned encoder fixture received shutdown"); + match mode.as_str() { + "exit23" => std::process::exit(23), + "timeout" => loop { + std::thread::sleep(Duration::from_secs(1)); + }, + _ => {} + } + } + + #[test] + fn stop_reports_encoder_exit_and_timeout_after_reaping_child() { + for (mode, expected) in [ + ("ok", None), + ("exit23", Some("ffmpeg exited with code 23")), + ("timeout", Some("ffmpeg shutdown timed out after 3000 ms")), + ] { + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "video_ffmpeg::shutdown_tests::encoder_child", + "--nocapture", + ]) + .env("CUA_P7_ENCODER_CHILD_MODE", mode) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stderr = child.stderr.take().unwrap(); + let stderr_thread = std::thread::spawn(move || { + use std::io::Read; + let mut bytes = Vec::new(); + stderr.read_to_end(&mut bytes).unwrap(); + bytes + }); + let backend = FfmpegVideoBackend { + child, + output_path: PathBuf::from("unused-fixture.mp4"), + started_at: Instant::now(), + stderr_thread: Some(stderr_thread), + }; + let result = Box::new(backend).stop(); + match expected { + Some(message) => assert_eq!(result.unwrap_err().to_string(), message), + None => assert!(result.unwrap().finalized), + } + } + } +} diff --git a/packages/cua-driver/rust/crates/cua-driver-core/tests/recording_capture_budget.rs b/packages/cua-driver/rust/crates/cua-driver-core/tests/recording_capture_budget.rs new file mode 100644 index 00000000000..0427ee9eda6 --- /dev/null +++ b/packages/cua-driver/rust/crates/cua-driver-core/tests/recording_capture_budget.rs @@ -0,0 +1,221 @@ +//! The real recording API with a controlled native provider; no desktop is opened. +use cua_driver_core::recording::{ + now_ms, set_ax_snapshot_fn, PendingTurn, RecordingCaller, RecordingSession, +}; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +#[derive(Default)] +struct Gate { + state: Mutex<(usize, bool)>, + changed: Condvar, + block_call: usize, +} + +impl Gate { + fn new(block_call: usize) -> Arc { + Arc::new(Self { + block_call, + ..Self::default() + }) + } + + fn capture(&self) -> Option> { + let mut state = self.state.lock().unwrap(); + state.0 += 1; + let call = state.0; + self.changed.notify_all(); + if call == self.block_call { + let (next, timeout) = self + .changed + .wait_timeout_while(state, Duration::from_secs(10), |state| !state.1) + .unwrap(); + state = next; + assert!( + !timeout.timed_out(), + "test failed to release controlled provider" + ); + } + drop(state); + Some(serde_json::to_vec(&json!({"call": call})).unwrap()) + } + + fn wait_for_call(&self, expected: usize) { + let (state, timeout) = self + .changed + .wait_timeout_while( + self.state.lock().unwrap(), + Duration::from_secs(3), + |state| state.0 < expected, + ) + .unwrap(); + assert!( + !timeout.timed_out(), + "provider did not receive call {expected}: {}", + state.0 + ); + } + + fn release(&self) { + self.state.lock().unwrap().1 = true; + self.changed.notify_all(); + } +} + +struct ReleaseAll(Vec>); +impl Drop for ReleaseAll { + fn drop(&mut self) { + for gate in &self.0 { + gate.release(); + } + } +} + +fn begin(session: &RecordingSession, pid: i64, owner: Option<&str>) -> Option { + session.begin_turn( + "budget_probe", + &json!({"pid": pid}), + now_ms(), + RecordingCaller { + session: owner, + transport: owner, + }, + ) +} + +fn evidence(dir: &std::path::Path, turn: usize) -> Value { + serde_json::from_slice( + &std::fs::read(dir.join(format!("turn-{turn:05}/evidence.json"))).unwrap(), + ) + .unwrap() +} + +#[test] +fn bounded_native_capture_releases_locks_and_rejects_late_generations() { + let gates = Arc::new(Mutex::new(HashMap::>::new())); + let provider_gates = gates.clone(); + set_ax_snapshot_fn(move |_, pid| { + let gate = provider_gates.lock().unwrap().get(&pid?).cloned()?; + gate.capture() + }); + let root = tempfile::tempdir().unwrap(); + let mut releases = ReleaseAll(Vec::new()); + + // Four slow providers time out for their callers but retain all four slots. + let mut captures = Vec::new(); + for pid in 1..=4 { + let gate = Gate::new(1); + releases.0.push(gate.clone()); + gates.lock().unwrap().insert(pid, gate.clone()); + let session = Arc::new(RecordingSession::new()); + let dir = root.path().join(format!("timeout-{pid}")); + session.start(dir.to_str().unwrap(), false, None).unwrap(); + let worker_session = session.clone(); + let worker = std::thread::spawn(move || { + let start = Instant::now(); + let pending = begin(&worker_session, pid, None).unwrap(); + (pending, start.elapsed()) + }); + gate.wait_for_call(1); + captures.push((session, dir, worker, gate)); + } + let fifth = Gate::new(0); + gates.lock().unwrap().insert(5, fifth.clone()); + let spare = RecordingSession::new(); + let spare_dir = root.path().join("spare"); + spare + .start(spare_dir.to_str().unwrap(), false, None) + .unwrap(); + let started = Instant::now(); + let pending = begin(&spare, 5, None).unwrap(); + spare.finish_turn(pending, "busy"); + assert!(started.elapsed() < Duration::from_secs(1)); + assert_eq!(fifth.state.lock().unwrap().0, 0); + assert_eq!( + evidence(&spare_dir, 1)["before"]["state"]["classification"], + "state_capture_busy" + ); + + for (session, dir, worker, gate) in captures { + let (pending, elapsed) = worker.join().unwrap(); + assert!(elapsed >= Duration::from_millis(1900)); + assert!(elapsed < Duration::from_secs(4)); + session.finish_turn(pending, "timeout"); + let result = evidence(&dir, 1); + assert_eq!( + result["before"]["state"]["classification"], + "state_capture_timeout" + ); + assert_eq!( + result["after"]["state"]["classification"], + "state_capture_busy" + ); + assert_eq!(gate.state.lock().unwrap().0, 1); + assert!(!dir.join("turn-00001/before_state.json").exists()); + gate.release(); + session.stop_owner(None).unwrap(); + } + // Wait for a slot to be released by the worker, not by its timed-out caller. + let deadline = Instant::now() + Duration::from_secs(2); + loop { + let pending = begin(&spare, 5, None).unwrap(); + spare.finish_turn(pending, "recovered"); + if fifth.state.lock().unwrap().0 > 0 { + break; + } + assert!( + Instant::now() < deadline, + "workers did not release their slots" + ); + std::thread::yield_now(); + } + spare.stop_owner(None).unwrap(); + for pid in 1..=4 { + let turn = root.path().join(format!("timeout-{pid}/turn-00001")); + assert!(!turn.join("before_state.json").exists()); + assert!(!turn.join("after_state.json").exists()); + assert!(!turn.join("app_state.json").exists()); + } + + // An after capture must neither hold the session lock nor outlive its owner/generation. + for (pid, replacement_owner) in [(10, "owner-a"), (11, "owner-b")] { + let gate = Gate::new(2); + releases.0.push(gate.clone()); + gates.lock().unwrap().insert(pid, gate.clone()); + let session = Arc::new(RecordingSession::new()); + let old_dir = root.path().join(format!("old-{pid}")); + let new_dir = root.path().join(format!("new-{pid}")); + session + .start(old_dir.to_str().unwrap(), false, Some("owner-a")) + .unwrap(); + let pending = begin(&session, pid, Some("owner-a")).unwrap(); + let worker_session = session.clone(); + let worker = std::thread::spawn(move || worker_session.finish_turn(pending, "old result")); + gate.wait_for_call(2); + let started = Instant::now(); + assert!(session.current_state().enabled); + session.stop_owner(Some("owner-a")).unwrap(); + session + .start(new_dir.to_str().unwrap(), false, Some(replacement_owner)) + .unwrap(); + assert!(started.elapsed() < Duration::from_secs(1)); + gate.release(); + worker.join().unwrap(); + for dir in [&old_dir, &new_dir] { + for file in [ + "action.json", + "after_state.json", + "app_state.json", + "evidence.json", + ] { + assert!( + !dir.join("turn-00001").join(file).exists(), + "stale write: {dir:?}/{file}" + ); + } + } + session.stop_owner(Some(replacement_owner)).unwrap(); + } +} diff --git a/packages/cua-driver/rust/crates/cua-driver-sdk/Cargo.toml b/packages/cua-driver/rust/crates/cua-driver-sdk/Cargo.toml index 24d6d1f2470..45a27275ee9 100644 --- a/packages/cua-driver/rust/crates/cua-driver-sdk/Cargo.toml +++ b/packages/cua-driver/rust/crates/cua-driver-sdk/Cargo.toml @@ -36,9 +36,6 @@ platform-linux = { path = "../platform-linux" } [features] default = [] -portal-input = ["platform-linux/portal-input"] -portal-capture = ["platform-linux/portal-capture"] -portal-libei = ["portal-input", "portal-capture"] [dev-dependencies] tempfile = "3" diff --git a/packages/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs b/packages/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs index 684e190030c..a806555889d 100644 --- a/packages/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs +++ b/packages/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs @@ -435,6 +435,8 @@ impl EmbeddedCuaDriverHost { Stdio::null() }) .kill_on_drop(true); + #[cfg(target_os = "windows")] + prevent_console_window(&mut command); for variable in safe_environment(&self.options.environment) { command.env(variable.name, variable.value); } @@ -846,12 +848,14 @@ pub(crate) fn allowed_environment_name(name: &str) -> bool { | "LOCALAPPDATA" | "PROGRAMDATA" | "DISPLAY" - | "WAYLAND_DISPLAY" | "XDG_RUNTIME_DIR" | "XDG_SESSION_TYPE" | "DBUS_SESSION_BUS_ADDRESS" | "XAUTHORITY" | "CUA_LOG" + | "CUA_DRIVER_RS_TELEMETRY_ENABLED" + | "CUA_TELEMETRY_ENABLED" + | "CUA_DRIVER_RS_UPDATE_CHECK" ) } @@ -1127,6 +1131,18 @@ async fn terminate_startup_child(child: &mut Child, liveness: &mut ChildStdin) { } } +#[cfg(target_os = "windows")] +fn prevent_console_window(command: &mut Command) { + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + #[link(name = "kernel32")] + unsafe extern "system" { + fn GetConsoleCP() -> u32; + } + if unsafe { GetConsoleCP() } == 0 { + command.creation_flags(CREATE_NO_WINDOW); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1194,12 +1210,37 @@ mod tests { #[test] fn environment_is_allowlisted_and_driver_controls_are_reserved() { assert!(allowed_environment_name("PATH")); - assert!(allowed_environment_name("WAYLAND_DISPLAY")); + assert!(!allowed_environment_name("WAYLAND_DISPLAY")); assert!(!allowed_environment_name("CUA_DRIVER_PERMISSION_MODE")); assert!(!allowed_environment_name("LD_PRELOAD")); assert!(!allowed_environment_name("NODE_OPTIONS")); } + #[test] + fn embedded_environment_preserves_explicit_telemetry_and_update_preferences() { + for name in [ + "CUA_DRIVER_RS_TELEMETRY_ENABLED", + "CUA_TELEMETRY_ENABLED", + "CUA_DRIVER_RS_UPDATE_CHECK", + ] { + let inherited = [(name.to_owned(), "0".to_owned())]; + let values = merge_safe_environment(inherited.clone(), &[]); + assert!(values + .iter() + .any(|variable| variable.name == name && variable.value == "0")); + + let values = merge_safe_environment( + inherited, + &[EmbeddedEnvironmentVariable { + name: name.to_owned(), + value: "false".to_owned(), + }], + ); + assert_eq!(values.len(), 1); + assert_eq!(values[0].value, "false"); + } + } + #[test] fn managed_environment_is_inherited_case_insensitively_but_never_overridden() { assert!(inherited_managed_environment_name( @@ -1239,6 +1280,8 @@ mod tests { fn interactive_linux_session_environment_is_inherited() { let values = merge_safe_environment( [ + ("DISPLAY".into(), ":7".into()), + ("XAUTHORITY".into(), "/tmp/cua-test-cookie".into()), ("WAYLAND_DISPLAY".into(), "wayland-7".into()), ("XDG_RUNTIME_DIR".into(), "/run/user/1000".into()), ("XDG_SESSION_TYPE".into(), "wayland".into()), @@ -1252,7 +1295,8 @@ mod tests { ); for (name, value) in [ - ("WAYLAND_DISPLAY", "wayland-7"), + ("DISPLAY", ":7"), + ("XAUTHORITY", "/tmp/cua-test-cookie"), ("XDG_RUNTIME_DIR", "/run/user/1000"), ("XDG_SESSION_TYPE", "wayland"), ("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"), @@ -1263,7 +1307,8 @@ mod tests { } assert!(!values .iter() - .any(|variable| variable.name == "AT_SPI_BUS_ADDRESS")); + .any(|variable| variable.name == "AT_SPI_BUS_ADDRESS" + || variable.name == "WAYLAND_DISPLAY")); } #[test] @@ -1362,4 +1407,76 @@ mod tests { assert_eq!(host.state(), EmbeddedDriverHostState::Stopped); assert!(!socket_path.exists()); } + + #[cfg(target_os = "windows")] + #[tokio::test] + async fn daemon_console_window_follows_the_host_console() { + use std::os::windows::process::CommandExt as _; + + const ROLE: &str = "CUA_DRIVER_SDK_TEST_CONSOLE_ROLE"; + const HOST_PID: &str = "CUA_DRIVER_SDK_TEST_CONSOLE_HOST_PID"; + const DETACHED_PROCESS: u32 = 0x0000_0008; + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + #[link(name = "kernel32")] + unsafe extern "system" { + fn GetConsoleWindow() -> *mut std::ffi::c_void; + fn GetConsoleProcessList(list: *mut u32, count: u32) -> u32; + } + let rerun = [ + "--exact", + "embedded::tests::daemon_console_window_follows_the_host_console", + "--nocapture", + ]; + let assert_probe_passed = |output: std::process::Output| { + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed"), + "{stdout}{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + + match std::env::var(ROLE).as_deref() { + Ok("probe-without-console") => { + let window = unsafe { GetConsoleWindow() }; + assert!( + window.is_null(), + "a console window was opened for the daemon" + ); + } + Ok("probe-with-console") => { + let host: u32 = std::env::var(HOST_PID).unwrap().parse().unwrap(); + let mut attached = [0u32; 64]; + let count = unsafe { GetConsoleProcessList(attached.as_mut_ptr(), 64) } as usize; + assert!( + attached[..count.min(64)].contains(&host), + "the daemon left its host's console" + ); + } + Ok(host @ ("host-without-console" | "host-with-console")) => { + let mut probe = Command::new(std::env::current_exe().unwrap()); + probe + .args(rerun) + .env(ROLE, host.replace("host-", "probe-")) + .env(HOST_PID, std::process::id().to_string()) + .stdin(Stdio::null()); + prevent_console_window(&mut probe); + assert_probe_passed(probe.output().await.unwrap()); + } + _ => { + for (host, flags) in [ + ("host-without-console", DETACHED_PROCESS), + ("host-with-console", CREATE_NO_WINDOW), + ] { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(rerun) + .env(ROLE, host) + .creation_flags(flags) + .output() + .unwrap(); + assert_probe_passed(output); + } + } + } + } } diff --git a/packages/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs b/packages/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs index 978d2728f78..cba8be13b5a 100644 --- a/packages/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs +++ b/packages/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs @@ -1799,7 +1799,19 @@ fn normalize_result(tool: &str, raw: Value) -> Result { .get("isError") .and_then(Value::as_bool) .unwrap_or(false); - let action = if !is_error && cua_driver_core::action_record::is_action_tool(tool) { + let app_action = + structured.is_some_and(|value| value.get("computer_use") == Some(&serde_json::json!(1))); + if !is_error && app_action { + serde_json::from_value::( + structured.unwrap().clone(), + ) + .and_then(|value| value.validate().map_err(serde::de::Error::custom)) + .map_err(|error| DriverError::Protocol { + reason: format!("{tool} returned an invalid Computer Use acknowledgement: {error}"), + })?; + } + let action = if !is_error && !app_action && cua_driver_core::action_record::is_action_tool(tool) + { let structured = structured.ok_or_else(|| DriverError::Protocol { reason: format!("{tool} response omitted ActionResult"), })?; @@ -2084,9 +2096,11 @@ mod tests { .iter() .filter_map(|tool| tool["name"].as_str()) .collect::>(); - for published in EXPORTED_TOOL_NAMES { - assert!(names.contains(published), "missing {published}"); + for spec in cua_driver_contract::computer_use::tools() { + assert!(names.contains(spec.name), "missing {}", spec.name); } + assert!(!names.contains("start_recording")); + assert!(!names.contains("get_browser_state")); driver.shutdown().await.unwrap(); driver.shutdown().await.unwrap(); @@ -2850,6 +2864,13 @@ mod tests { )); } + #[test] + fn generic_app_action_acknowledgement_does_not_claim_a_legacy_action_effect() { + let result = normalize_result("click", serde_json::json!({"content":[{"type":"text","text":"Sent but not confirmed."}],"structuredContent":{"computer_use":1,"sent":true}})).unwrap(); + assert!(result.action.is_none()); + assert_eq!(result.text, "Sent but not confirmed."); + } + #[tokio::test] #[cfg(unix)] async fn tool_discovery_uses_the_shared_direct_daemon_protocol() { diff --git a/packages/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs b/packages/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs index f0e8b17affa..e0ec539474e 100644 --- a/packages/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs +++ b/packages/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs @@ -559,6 +559,7 @@ fn build_registry(options: &RuntimeOptions) -> ToolRegistry { if let Some(register_host_tools) = options.register_host_tools { register_host_tools(&mut registry); } + registry.enable_computer_use(); let recording = Arc::downgrade(®istry.recording); let recording_session_end = cua_driver_core::session::register_scoped_fallible_session_end_hook( "recording", @@ -596,8 +597,8 @@ fn configure_macos_runtime() { cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| { platform_macos::recording_hooks::app_state_json_for(window_id, pid) }); - cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| { - platform_macos::recording_hooks::element_window_local_xy(window_id, pid, index) + cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index, binding| { + platform_macos::recording_hooks::element_window_local_xy(window_id, pid, index, binding) }); cua_driver_core::video::set_video_backend_factory(Box::new( platform_macos::video_sckit::SckitVideoBackendFactory, @@ -615,8 +616,8 @@ fn configure_windows_runtime() { cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| { platform_windows::recording_hooks::app_state_json_for(window_id, pid) }); - cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| { - platform_windows::recording_hooks::element_window_local_xy(window_id, pid, index) + cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index, binding| { + platform_windows::recording_hooks::element_window_local_xy(window_id, pid, index, binding) }); cua_driver_core::video::set_video_backend_factory(Box::new( cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory, @@ -642,14 +643,10 @@ fn configure_linux_runtime(prepare_desktop_environment: bool) { cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| { platform_linux::recording_hooks::app_state_json_for(window_id, pid) }); - cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| { - platform_linux::recording_hooks::element_window_local_xy(window_id, pid, index) + cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index, binding| { + platform_linux::recording_hooks::element_window_local_xy(window_id, pid, index, binding) }); - if platform_linux::wayland::is_wayland() { - cua_driver_core::video::set_video_backend_factory(Box::new( - platform_linux::video_wayland::WfRecorderVideoBackendFactory, - )); - } else { + { cua_driver_core::video::set_video_backend_factory(Box::new( cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory, )); diff --git a/packages/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/packages/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs index eb07f5c8045..401dfee335e 100644 --- a/packages/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs +++ b/packages/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs @@ -50,7 +50,6 @@ pub enum DisplayServer { Win32, Quartz, X11, - Wayland, } impl DisplayServer { @@ -65,14 +64,7 @@ impl DisplayServer { } #[cfg(target_os = "linux")] { - match std::env::var("XDG_SESSION_TYPE") - .unwrap_or_else(|_| "x11".to_owned()) - .to_ascii_lowercase() - .as_str() - { - "wayland" => Self::Wayland, - _ => Self::X11, - } + Self::X11 } #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))] { @@ -133,9 +125,7 @@ pub enum DriverRoute { LinuxAtSpiValue, LinuxXSendEvent, LinuxXTest, - LinuxLibei, - LinuxWaylandVirtualPointer, - LinuxCuaCompositorInject, + Cdp, Composite, } @@ -146,24 +136,6 @@ pub fn shared_web_route( action: &str, targeting: Targeting, delivery: Delivery, -) -> Result { - shared_web_route_for_environment( - platform, - display_server, - action, - targeting, - delivery, - nested_inject_from_env(), - ) -} - -fn shared_web_route_for_environment( - platform: Platform, - display_server: DisplayServer, - action: &str, - targeting: Targeting, - delivery: Delivery, - nested_inject: bool, ) -> Result { use DriverRoute as Route; @@ -226,30 +198,19 @@ fn shared_web_route_for_environment( Ok(Route::Composite) } - (Platform::Linux, DisplayServer::Wayland, Targeting::Px, _) if nested_inject => - { - Ok(Route::LinuxCuaCompositorInject) - } - (Platform::Linux, DisplayServer::Wayland, Targeting::Px, _) => { - Ok(Route::LinuxWaylandVirtualPointer) - } (Platform::Linux, DisplayServer::X11, Targeting::Px, _) => { pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest) } - (Platform::Linux, DisplayServer::Wayland, Targeting::Ax, "type_text") - if nested_inject => - { - Ok(Route::LinuxCuaCompositorInject) - } + ( Platform::Linux, - DisplayServer::X11 | DisplayServer::Wayland, + DisplayServer::X11, Targeting::Ax, "left_click" | "child_window" | "scroll", ) => Ok(Route::LinuxAtSpiAction), ( Platform::Linux, - DisplayServer::X11 | DisplayServer::Wayland, + DisplayServer::X11, Targeting::Ax, "type_text", ) => Ok(Route::LinuxAtSpiValue), @@ -259,21 +220,10 @@ fn shared_web_route_for_environment( Targeting::Ax, "right_click" | "double_click" | "press_key" | "hotkey", ) => pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest), + ( Platform::Linux, - DisplayServer::Wayland, - Targeting::Ax, - "right_click" | "double_click" | "press_key" | "hotkey", - ) if nested_inject => Ok(Route::LinuxCuaCompositorInject), - ( - Platform::Linux, - DisplayServer::Wayland, - Targeting::Ax, - "right_click" | "double_click" | "press_key" | "hotkey", - ) => Ok(Route::LinuxWaylandVirtualPointer), - ( - Platform::Linux, - DisplayServer::X11 | DisplayServer::Wayland, + DisplayServer::X11, Targeting::Ax, "editor_save", ) => Ok(Route::Composite), @@ -416,10 +366,6 @@ pub fn environment_schema_supported(schema: &str) -> bool { matches!(schema, ENVIRONMENT_SCHEMA | ENVIRONMENT_SCHEMA_V2) } -fn nested_inject_from_env() -> bool { - std::env::var_os("CUA_INJECT_SOCKET").is_some() -} - fn compositor_from_env() -> Option { if let Ok(value) = std::env::var("CUA_E2E_COMPOSITOR") { if !value.trim().is_empty() { @@ -430,23 +376,6 @@ fn compositor_from_env() -> Option { (Platform::Windows, DisplayServer::Win32) => Some("windows-desktop".to_owned()), (Platform::Macos, DisplayServer::Quartz) => Some("windowserver".to_owned()), (Platform::Linux, DisplayServer::X11) => Some("openbox-x11".to_owned()), - (Platform::Linux, DisplayServer::Wayland) if nested_inject_from_env() => { - Some("cua-compositor-nested".to_owned()) - } - (Platform::Linux, DisplayServer::Wayland) => { - let desktop = std::env::var("XDG_CURRENT_DESKTOP") - .unwrap_or_default() - .to_ascii_lowercase(); - if desktop.contains("sway") { - Some("sway".to_owned()) - } else if desktop.contains("gnome") { - Some("gnome-mutter".to_owned()) - } else if desktop.contains("kde") || desktop.contains("plasma") { - Some("kwin".to_owned()) - } else { - Some("wayland-unknown".to_owned()) - } - } _ => None, } } @@ -473,10 +402,7 @@ fn input_backends_from_env() -> Vec { "xsend-event".to_owned(), "xtest".to_owned(), ], - (Platform::Linux, DisplayServer::Wayland) if nested_inject_from_env() => { - vec!["atspi".to_owned(), "cua-compositor-inject".to_owned()] - } - (Platform::Linux, DisplayServer::Wayland) => vec!["atspi".to_owned()], + _ => Vec::new(), } } @@ -629,9 +555,7 @@ pub fn native_background_case( OracleKind::ZOrder, OracleKind::NoLeakedInput, ]; - if DisplayServer::current() != DisplayServer::Wayland { - oracles.push(OracleKind::Cursor); - } + oracles.push(OracleKind::Cursor); CaseSpec::delivered( native_cell_id(toolkit, action, targeting, Delivery::Background), toolkit, @@ -2617,7 +2541,6 @@ mod tests { (Platform::Windows, DisplayServer::Win32), (Platform::Macos, DisplayServer::Quartz), (Platform::Linux, DisplayServer::X11), - (Platform::Linux, DisplayServer::Wayland), ] { for (action, targeting, delivery) in cells.iter().copied() { shared_web_route(platform, display_server, action, targeting, delivery) @@ -2640,44 +2563,12 @@ mod tests { ); } - #[test] - fn nested_wayland_pixel_route_is_distinct_from_stock_wayland() { - let stock = shared_web_route_for_environment( - Platform::Linux, - DisplayServer::Wayland, - "left_click", - Targeting::Px, - Delivery::Background, - false, - ); - let nested = shared_web_route_for_environment( - Platform::Linux, - DisplayServer::Wayland, - "left_click", - Targeting::Px, - Delivery::Background, - true, - ); - assert_eq!(stock, Ok(DriverRoute::LinuxWaylandVirtualPointer)); - assert_eq!(nested, Ok(DriverRoute::LinuxCuaCompositorInject)); - - let nested_text = shared_web_route_for_environment( - Platform::Linux, - DisplayServer::Wayland, - "type_text", - Targeting::Ax, - Delivery::Background, - true, - ); - assert_eq!(nested_text, Ok(DriverRoute::LinuxCuaCompositorInject)); - } - #[test] fn environment_v2_artifacts_remain_readable() { let record: EnvironmentRecord = serde_json::from_value(serde_json::json!({ "schema": ENVIRONMENT_SCHEMA_V2, "platform": "linux", - "display_server": "wayland", + "display_server": "x11", "source_sha": null, "status": "ready", "duration_ms": 1, diff --git a/packages/cua-driver/rust/crates/cua-driver-testkit/src/observer.rs b/packages/cua-driver/rust/crates/cua-driver-testkit/src/observer.rs index 59609430650..2fb4008380a 100644 --- a/packages/cua-driver/rust/crates/cua-driver-testkit/src/observer.rs +++ b/packages/cua-driver/rust/crates/cua-driver-testkit/src/observer.rs @@ -893,13 +893,11 @@ pub mod macos { #[cfg(target_os = "linux")] pub mod linux { - use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::Duration; - use serde::Deserialize; use x11rb::connection::Connection; use x11rb::protocol::xproto::{ AtomEnum, ConnectionExt, MapState, Rectangle, Window, WindowClass, @@ -914,9 +912,6 @@ pub mod linux { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum SessionKind { X11, - Sway, - Gnome, - CuaCompositor, Missing, } @@ -929,21 +924,7 @@ pub mod linux { impl LinuxObserver { pub fn new() -> Self { - let explicit_wayland = std::env::var("XDG_SESSION_TYPE") - .map(|value| value.eq_ignore_ascii_case("wayland")) - .unwrap_or(false) - || std::env::var_os("WAYLAND_DISPLAY").is_some(); - let session = if explicit_wayland { - if cua_compositor_available() { - SessionKind::CuaCompositor - } else if sway_available() { - SessionKind::Sway - } else if gnome_windows().is_ok() { - SessionKind::Gnome - } else { - SessionKind::Missing - } - } else if std::env::var_os("DISPLAY").is_some() { + let session = if std::env::var_os("DISPLAY").is_some() { if x11_window_manager_ready() { SessionKind::X11 } else { @@ -976,14 +957,7 @@ pub mod linux { cursor: true, leaked_input: false, }, - SessionKind::Sway | SessionKind::Gnome | SessionKind::CuaCompositor => { - ObserverCapabilities { - focus: true, - z_order: true, - cursor: false, - leaked_input: false, - } - } + SessionKind::Missing => ObserverCapabilities::default(), } } @@ -991,9 +965,7 @@ pub mod linux { fn snapshot(&self, target: TargetWindow) -> Result { match self.session { SessionKind::X11 => x11_snapshot(target), - SessionKind::Sway => sway_snapshot(target), - SessionKind::Gnome => gnome_snapshot(target), - SessionKind::CuaCompositor => cua_compositor_snapshot(target), + SessionKind::Missing => Ok(DesktopSnapshot { foreground: None, input_focus: None, @@ -1018,9 +990,7 @@ pub mod linux { self.sampler = Some(std::thread::spawn(move || { let focus_identity = || match session { SessionKind::X11 => x11_focus_identity(), - SessionKind::Sway => sway_focus_identity(), - SessionKind::Gnome => gnome_focus_identity(), - SessionKind::CuaCompositor => cua_compositor_focus_identity(), + SessionKind::Missing => Ok(None), }; let mut previous = focus_identity().ok().flatten(); @@ -1068,366 +1038,6 @@ pub mod linux { .map_err(|error| ObserverError::new(format!("X11 connect failed: {error}"))) } - #[derive(Default)] - struct SwayTreeState { - focused: Option, - focused_workspace: Option, - focused_fullscreen: Option, - target: Option<(u64, bool, Option)>, - } - - fn sway_tree() -> Result { - let output = Command::new("swaymsg") - .args(["-r", "-t", "get_tree"]) - .output() - .map_err(|error| ObserverError::new(format!("swaymsg get_tree failed: {error}")))?; - if !output.status.success() { - return Err(ObserverError::new(format!( - "swaymsg get_tree exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ))); - } - serde_json::from_slice(&output.stdout) - .map_err(|error| ObserverError::new(format!("invalid sway tree JSON: {error}"))) - } - - fn sway_available() -> bool { - std::env::var_os("SWAYSOCK").is_some_and(|value| !value.is_empty()) && sway_tree().is_ok() - } - - #[derive(Clone, Copy, Debug, Eq, PartialEq)] - struct CuaCompositorState { - focused_pid: Option, - target_z: TargetZ, - } - - fn parse_cua_compositor_state(line: &str) -> Result { - let mut fields = line.split_whitespace(); - if fields.next() != Some("state") { - return Err(ObserverError::new(format!( - "invalid cua-compositor observer response: {line:?}" - ))); - } - let focused_pid = fields - .next() - .ok_or_else(|| ObserverError::new("cua-compositor state omitted focused pid"))? - .parse::() - .map_err(|error| ObserverError::new(format!("invalid focused pid: {error}")))?; - let target_z = match fields.next() { - Some("foreground") => TargetZ::Foreground, - Some("background_occluded") => TargetZ::BackgroundOccluded, - Some("background_visible") => TargetZ::BackgroundVisible, - Some("not_found") => TargetZ::NotFound, - Some(value) => { - return Err(ObserverError::new(format!( - "invalid cua-compositor target state: {value}" - ))) - } - None => { - return Err(ObserverError::new( - "cua-compositor state omitted target state", - )) - } - }; - if fields.next().is_some() { - return Err(ObserverError::new( - "cua-compositor state contained trailing fields", - )); - } - Ok(CuaCompositorState { - focused_pid: (focused_pid != 0).then_some(focused_pid), - target_z, - }) - } - - fn cua_compositor_state(target_pid: u32) -> Result { - use std::io::{BufRead, BufReader, Write}; - use std::os::unix::net::UnixStream; - - let socket = std::env::var("CUA_INJECT_SOCKET") - .map_err(|_| ObserverError::new("CUA_INJECT_SOCKET is not set"))?; - let stream = UnixStream::connect(&socket).map_err(|error| { - ObserverError::new(format!("connect cua-compositor observer socket: {error}")) - })?; - stream - .set_read_timeout(Some(Duration::from_secs(2))) - .map_err(|error| ObserverError::new(format!("set observer timeout: {error}")))?; - let mut writer = stream - .try_clone() - .map_err(|error| ObserverError::new(format!("clone observer socket: {error}")))?; - let mut reader = BufReader::new(stream); - - writeln!(writer, "cua-inject v1") - .and_then(|_| writer.flush()) - .map_err(|error| ObserverError::new(format!("write observer handshake: {error}")))?; - let mut line = String::new(); - reader - .read_line(&mut line) - .map_err(|error| ObserverError::new(format!("read observer handshake: {error}")))?; - if line.trim() != "cua-inject v1" { - return Err(ObserverError::new(format!( - "cua-compositor observer handshake mismatch: {:?}", - line.trim() - ))); - } - - writeln!(writer, "q {target_pid}") - .and_then(|_| writer.flush()) - .map_err(|error| ObserverError::new(format!("write observer query: {error}")))?; - line.clear(); - let read = reader - .read_line(&mut line) - .map_err(|error| ObserverError::new(format!("read observer query: {error}")))?; - if read == 0 { - return Err(ObserverError::new( - "cua-compositor closed the observer query", - )); - } - parse_cua_compositor_state(&line) - } - - fn cua_compositor_available() -> bool { - std::env::var_os("CUA_INJECT_SOCKET").is_some() && cua_compositor_state(0).is_ok() - } - - fn cua_compositor_snapshot(target: TargetWindow) -> Result { - let state = cua_compositor_state(target.pid)?; - Ok(DesktopSnapshot { - foreground: state.focused_pid, - input_focus: state.focused_pid, - target_z: state.target_z, - cursor_pos: None, - }) - } - - fn cua_compositor_focus_identity() -> Result, ObserverError> { - Ok(cua_compositor_state(0)?.focused_pid) - } - - #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] - struct GnomeWindow { - id: u64, - pid: u32, - title: String, - x: i64, - y: i64, - w: i64, - h: i64, - focused: bool, - minimized: bool, - visible: bool, - stacking: u64, - } - - fn gnome_windows() -> Result, ObserverError> { - let mut child = Command::new("gdbus") - .args([ - "call", - "--session", - "--dest", - "org.cua.WinRects", - "--object-path", - "/org/cua/WinRects", - "--method", - "org.cua.WinRects.GetRects", - ]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|error| ObserverError::new(format!("gdbus GetRects failed: {error}")))?; - let deadline = std::time::Instant::now() + Duration::from_millis(800); - loop { - match child.try_wait() { - Ok(Some(_)) => { - let output = child.wait_with_output().map_err(|error| { - ObserverError::new(format!("gdbus GetRects output failed: {error}")) - })?; - if !output.status.success() { - return Err(ObserverError::new(format!( - "gdbus GetRects exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ))); - } - return parse_gnome_windows(&String::from_utf8_lossy(&output.stdout)); - } - Ok(None) if std::time::Instant::now() < deadline => { - std::thread::sleep(Duration::from_millis(15)) - } - Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(ObserverError::new("gdbus GetRects timed out")); - } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(ObserverError::new(format!( - "gdbus GetRects wait failed: {error}" - ))); - } - } - } - } - - fn parse_gnome_windows(raw: &str) -> Result, ObserverError> { - let start = raw - .find('[') - .ok_or_else(|| ObserverError::new("GetRects response had no JSON array"))?; - let end = raw - .rfind(']') - .filter(|end| *end >= start) - .ok_or_else(|| ObserverError::new("GetRects response had no complete JSON array"))?; - serde_json::from_str(&raw[start..=end]) - .map_err(|error| ObserverError::new(format!("invalid GetRects JSON: {error}"))) - } - - fn gnome_target<'a>( - windows: &'a [GnomeWindow], - target: TargetWindow, - ) -> Option<&'a GnomeWindow> { - let matching = windows.iter().filter(|window| window.pid == target.pid); - matching - .clone() - .find(|window| window.id == target.native_id) - .or_else(|| matching.max_by_key(|window| window.stacking)) - } - - fn fully_covers(cover: &GnomeWindow, target: &GnomeWindow) -> bool { - let valid_rects = target.w > 0 && target.h > 0 && cover.w > 0 && cover.h > 0; - let covers_declared_frame = cover.x <= target.x - && cover.y <= target.y - && cover.x.saturating_add(cover.w) >= target.x.saturating_add(target.w) - && cover.y.saturating_add(cover.h) >= target.y.saturating_add(target.h); - let test_sentinel_covers_visible_frame = cover.title.starts_with("CuaTestHarness Sentinel") - && cover.x == 0 - && cover.y == 0 - && target.x < cover.w - && target.y < cover.h - && target.x.saturating_add(target.w) > 0 - && target.y.saturating_add(target.h) > 0; - valid_rects && (covers_declared_frame || test_sentinel_covers_visible_frame) - } - - fn classify_gnome_target(windows: &[GnomeWindow], target: TargetWindow) -> TargetZ { - let Some(target) = gnome_target(windows, target) else { - return TargetZ::NotFound; - }; - if target.focused { - TargetZ::Foreground - } else if target.minimized || !target.visible { - TargetZ::Minimized - } else if windows.iter().any(|window| { - window.stacking > target.stacking - && window.visible - && !window.minimized - && fully_covers(window, target) - }) { - TargetZ::BackgroundOccluded - } else { - TargetZ::BackgroundVisible - } - } - - fn gnome_focus_identity_from(windows: &[GnomeWindow]) -> Option { - windows - .iter() - .find(|window| window.focused) - .map(|window| window.id) - } - - fn gnome_snapshot(target: TargetWindow) -> Result { - let windows = gnome_windows()?; - let focused = gnome_focus_identity_from(&windows); - Ok(DesktopSnapshot { - foreground: focused, - input_focus: focused, - target_z: classify_gnome_target(&windows, target), - cursor_pos: None, - }) - } - - fn gnome_focus_identity() -> Result, ObserverError> { - Ok(gnome_focus_identity_from(&gnome_windows()?)) - } - - fn walk_sway_tree( - node: &serde_json::Value, - target_pid: u32, - workspace: Option, - state: &mut SwayTreeState, - ) -> bool { - let id = node["id"].as_u64(); - let workspace = if node["type"].as_str() == Some("workspace") { - id - } else { - workspace - }; - let focused = node["focused"].as_bool().unwrap_or(false); - if focused { - state.focused = id; - state.focused_workspace = workspace; - } - if node["pid"].as_u64() == Some(u64::from(target_pid)) { - if let Some(id) = id { - state.target = Some((id, node["visible"].as_bool().unwrap_or(true), workspace)); - } - } - let mut subtree_focused = focused; - for child in ["nodes", "floating_nodes"] - .into_iter() - .flat_map(|key| node[key].as_array().into_iter().flatten()) - { - subtree_focused |= walk_sway_tree(child, target_pid, workspace, state); - } - // Sway commonly puts fullscreen_mode on a container while focus belongs - // to a descendant surface. Treat the focused subtree as fullscreen so a - // sibling hidden behind that container is classified as occluded. - if subtree_focused && node["fullscreen_mode"].as_i64().unwrap_or(0) != 0 { - state.focused_fullscreen = id; - } - subtree_focused - } - - fn classify_sway_target(state: &SwayTreeState) -> TargetZ { - match state.target { - None => TargetZ::NotFound, - Some((id, _, _)) if state.focused == Some(id) => TargetZ::Foreground, - Some((_, _, target_workspace)) - if state.focused_fullscreen.is_some() - && target_workspace == state.focused_workspace => - { - // Sway reports windows hidden behind a fullscreen sibling as - // visible=false. They are occluded, not minimized. - TargetZ::BackgroundOccluded - } - Some((_, false, _)) => TargetZ::Minimized, - Some(_) => TargetZ::BackgroundVisible, - } - } - - fn sway_snapshot(target: TargetWindow) -> Result { - let tree = sway_tree()?; - let mut state = SwayTreeState::default(); - let _ = walk_sway_tree(&tree, target.pid, None, &mut state); - let target_z = classify_sway_target(&state); - Ok(DesktopSnapshot { - foreground: state.focused, - input_focus: state.focused, - target_z, - cursor_pos: None, - }) - } - - fn sway_focus_identity() -> Result, ObserverError> { - let tree = sway_tree()?; - let mut state = SwayTreeState::default(); - let _ = walk_sway_tree(&tree, 0, None, &mut state); - Ok(state.focused) - } - fn x11_snapshot(target: TargetWindow) -> Result { let (connection, screen_index) = x11_connection()?; let root = connection.setup().roots[screen_index].root; @@ -1685,10 +1295,6 @@ pub mod linux { mod tests { use super::*; - fn gnome_payload() -> &'static str { - r#"('[{"id":10,"pid":100,"title":"target","x":10,"y":20,"w":300,"h":200,"focused":false,"minimized":false,"visible":true,"stacking":0},{"id":20,"pid":200,"title":"sentinel","x":0,"y":0,"w":800,"h":600,"focused":true,"minimized":false,"visible":true,"stacking":1}]',)"# - } - #[test] fn occlusion_samples_corners_and_center() { let bounds = Rectangle { @@ -1726,159 +1332,6 @@ pub mod linux { assert!(!rectangle_fully_contains(partial, target, 2)); assert!(rectangle_fully_contains(full, target, 2)); } - - #[test] - fn sway_hidden_target_behind_fullscreen_sibling_is_occluded() { - let state = SwayTreeState { - focused: Some(30), - focused_workspace: Some(10), - focused_fullscreen: Some(30), - target: Some((20, false, Some(10))), - }; - assert_eq!(classify_sway_target(&state), TargetZ::BackgroundOccluded); - } - - #[test] - fn sway_fullscreen_ancestor_of_focused_surface_occludes_target() { - let tree = serde_json::json!({ - "id": 1, - "type": "root", - "nodes": [{ - "id": 10, - "type": "workspace", - "nodes": [ - { - "id": 20, - "pid": 100, - "visible": false, - "focused": false, - "fullscreen_mode": 0, - "nodes": [] - }, - { - "id": 30, - "focused": false, - "fullscreen_mode": 1, - "nodes": [{ - "id": 31, - "pid": 200, - "visible": true, - "focused": true, - "fullscreen_mode": 0, - "nodes": [] - }] - } - ] - }] - }); - let mut state = SwayTreeState::default(); - assert!(walk_sway_tree(&tree, 100, None, &mut state)); - assert_eq!(state.focused, Some(31)); - assert_eq!(state.focused_fullscreen, Some(30)); - assert_eq!(classify_sway_target(&state), TargetZ::BackgroundOccluded); - } - - #[test] - fn sway_hidden_target_on_another_workspace_is_not_occluded() { - let state = SwayTreeState { - focused: Some(30), - focused_workspace: Some(10), - focused_fullscreen: Some(30), - target: Some((20, false, Some(11))), - }; - assert_eq!(classify_sway_target(&state), TargetZ::Minimized); - } - - #[test] - fn cua_compositor_state_is_strict_and_pid_based() { - assert_eq!( - parse_cua_compositor_state("state 200 background_occluded\n") - .expect("valid compositor state"), - CuaCompositorState { - focused_pid: Some(200), - target_z: TargetZ::BackgroundOccluded, - } - ); - assert_eq!( - parse_cua_compositor_state("state 0 not_found") - .expect("zero means no focused client"), - CuaCompositorState { - focused_pid: None, - target_z: TargetZ::NotFound, - } - ); - assert!(parse_cua_compositor_state("ok").is_err()); - assert!(parse_cua_compositor_state("state 1 unknown").is_err()); - assert!(parse_cua_compositor_state("state 1 foreground trailing").is_err()); - } - - #[test] - fn gnome_json_and_focus_identity_are_compositor_derived() { - let windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON"); - assert_eq!(windows.len(), 2); - assert_eq!(gnome_focus_identity_from(&windows), Some(20)); - assert!(parse_gnome_windows(r#"('[{"pid":100}]',)"#).is_err()); - } - - #[test] - fn gnome_only_full_higher_cover_occludes() { - let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON"); - let target = TargetWindow { - pid: 100, - native_id: 10, - }; - assert_eq!( - classify_gnome_target(&windows, target), - TargetZ::BackgroundOccluded - ); - windows[1].w = 200; - assert_eq!( - classify_gnome_target(&windows, target), - TargetZ::BackgroundVisible - ); - windows[1].w = 800; - windows[1].visible = false; - assert_eq!( - classify_gnome_target(&windows, target), - TargetZ::BackgroundVisible - ); - } - - #[test] - fn gnome_fullscreen_test_sentinel_covers_the_visible_part_of_an_oversized_target() { - let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON"); - let target = TargetWindow { - pid: 100, - native_id: 10, - }; - windows[0].x = 66; - windows[0].w = 1020; - windows[1].title = "CuaTestHarness Sentinel [cdp=12345]".to_owned(); - assert_eq!( - classify_gnome_target(&windows, target), - TargetZ::BackgroundOccluded - ); - - windows[1].title = "ordinary window".to_owned(); - assert_eq!( - classify_gnome_target(&windows, target), - TargetZ::BackgroundVisible - ); - } - - #[test] - fn gnome_focused_and_minimized_are_direct() { - let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON"); - let target = TargetWindow { - pid: 100, - native_id: 10, - }; - windows[0].focused = true; - assert_eq!(classify_gnome_target(&windows, target), TargetZ::Foreground); - windows[0].focused = false; - windows[0].minimized = true; - assert_eq!(classify_gnome_target(&windows, target), TargetZ::Minimized); - } } } diff --git a/packages/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs b/packages/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs index a902337f730..c64ad77d261 100644 --- a/packages/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs +++ b/packages/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs @@ -106,13 +106,7 @@ impl ForegroundSentinel { reaper.track_pid(target.pid); let focus_deadline = Instant::now() + Duration::from_secs(10); - if is_wayland_session() { - wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready"); - try_activate_native_foreground(driver, target)?; - // Electron may already be focused before its preload listener is ready. - // The compositor observation is the authoritative Wayland focus gate. - wait_for_native_focus_stable(target); - } else { + { wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready"); try_activate_native_foreground(driver, target)?; wait_for_native_focus_stable(target); @@ -161,7 +155,7 @@ impl ForegroundSentinel { { violations.push("foreground sentinel heartbeat stopped".to_owned()); } - if !is_wayland_session() { + { if events.iter().any(|event| { event_kind(event) == Some("blur") || (event_kind(event) == Some("visibility") @@ -211,22 +205,12 @@ impl ForegroundSentinel { wait_for_event(&self.journal_path, "heartbeat", Duration::from_secs(2))?; reset_journal(&self.journal_path)?; - let canary_key = if std::env::var("CUA_E2E_WAYLAND_SESSION") - .is_ok_and(|session| session == "cua-compositor") - { - // The intentionally small injection protocol exposes navigation - // and control keys, not printable letters. Space still exercises - // the renderer keydown leak detector without broadening it. - "space" - } else { - "a" - }; let leaked_key = driver.call( "press_key", serde_json::json!({ "pid": self.target.pid, "window_id": self.target.native_id, - "key": canary_key, + "key": "a", "delivery_mode": "foreground", }), ); @@ -248,8 +232,6 @@ impl ForegroundSentinel { } reset_journal(&self.journal_path)?; - #[cfg(target_os = "linux")] - set_sway_fullscreen(driver, self.target, false)?; let raised = driver.call( "bring_to_front", serde_json::json!({ @@ -263,11 +245,7 @@ impl ForegroundSentinel { raised.text() )); } - #[cfg(target_os = "linux")] - focus_sway_target(driver, background_target)?; - if is_wayland_session() { - wait_for_native_focus_lost(self.target)?; - } else { + { wait_for_event(&self.journal_path, "blur", Duration::from_secs(3))?; let (_, focus_violations) = self.observe(); if !focus_violations @@ -281,8 +259,6 @@ impl ForegroundSentinel { } activate_native_foreground(driver, self.target); - #[cfg(target_os = "linux")] - set_sway_fullscreen(driver, self.target, true)?; wait_for_native_focus_stable(self.target); std::thread::sleep(Duration::from_millis(150)); reset_journal(&self.journal_path)?; @@ -377,13 +353,7 @@ impl ForegroundSentinel { before.target_z )); } - let mut native_oracles = vec![OracleKind::Focus, OracleKind::ZOrder]; - if std::env::var("XDG_SESSION_TYPE") - .map(|session| !session.eq_ignore_ascii_case("wayland")) - .unwrap_or(true) - { - native_oracles.push(OracleKind::Cursor); - } + let native_oracles = [OracleKind::Focus, OracleKind::ZOrder, OracleKind::Cursor]; let (result, delta) = observer .observe(&native_oracles, action) .map_err(|error| error.to_string())?; @@ -465,12 +435,6 @@ fn wait_for_event(path: &std::path::Path, kind: &str, timeout: Duration) -> Resu } } -fn is_wayland_session() -> bool { - cfg!(target_os = "linux") - && std::env::var("XDG_SESSION_TYPE") - .is_ok_and(|session| session.eq_ignore_ascii_case("wayland")) -} - fn activate_native_foreground(driver: &mut impl Driver, target: TargetWindow) { try_activate_native_foreground(driver, target) .unwrap_or_else(|error| panic!("could not activate foreground sentinel: {error}")); @@ -490,10 +454,6 @@ fn try_activate_native_foreground( if response.is_error() { return Err(response.text().to_owned()); } - #[cfg(target_os = "linux")] - focus_sway_target(driver, target).map_err(|error| { - format!("could not focus foreground sentinel through Sway IPC: {error}") - })?; #[cfg(target_os = "windows")] physically_focus_windows_sentinel(target); #[cfg(target_os = "macos")] @@ -529,385 +489,6 @@ fn focus_macos_sentinel_contents( Ok(()) } -#[cfg(target_os = "linux")] -fn focus_sway_target(driver: &mut impl Driver, target: TargetWindow) -> Result<(), String> { - let is_sway = is_wayland_session() - && std::env::var("CUA_E2E_WAYLAND_SESSION").is_ok_and(|session| session == "sway"); - if !is_sway { - return Ok(()); - } - - let (_, con_id) = sway_tree_and_container_for_target(driver, target)?; - run_sway_container_command(con_id, &["focus"], "focus canary") -} - -#[cfg(target_os = "linux")] -fn set_sway_fullscreen( - driver: &mut impl Driver, - target: TargetWindow, - enabled: bool, -) -> Result<(), String> { - let is_sway = is_wayland_session() - && std::env::var("CUA_E2E_WAYLAND_SESSION").is_ok_and(|session| session == "sway"); - if !is_sway { - return Ok(()); - } - - let (tree, view_id) = sway_tree_and_container_for_target(driver, target)?; - let con_id = if enabled { - view_id - } else { - sway_fullscreen_holder_id(&tree, view_id).unwrap_or(view_id) - }; - let state = if enabled { "enable" } else { "disable" }; - run_sway_container_command(con_id, &["fullscreen", state], "fullscreen canary")?; - if !enabled { - wait_for_sway_fullscreen_cleared(view_id)?; - } - Ok(()) -} - -#[cfg(target_os = "linux")] -fn sway_tree_and_container_for_target( - driver: &mut impl Driver, - target: TargetWindow, -) -> Result<(serde_json::Value, i64), String> { - let tree = read_sway_tree()?; - - let windows = driver.call("list_windows", serde_json::json!({})); - if windows.is_error() { - return Err(format!( - "list windows before Sway focus canary: {}", - windows.text() - )); - } - let title = sway_target_title(windows.structured(), target); - let con_id = title - .and_then(|title| sway_container_id(&tree, title)) - .or_else(|| sway_container_id_by_unique_pid(&tree, target.pid)) - .ok_or_else(|| { - format!( - "Sway focus canary could not resolve container for pid {} window {}", - target.pid, target.native_id - ) - })?; - Ok((tree, con_id)) -} - -#[cfg(target_os = "linux")] -fn read_sway_tree() -> Result { - let tree_output = Command::new("swaymsg") - .args(["-r", "-t", "get_tree"]) - .output() - .map_err(|error| format!("read Sway tree for focus canary: {error}"))?; - if !tree_output.status.success() { - return Err(format!( - "read Sway tree for focus canary: {}", - String::from_utf8_lossy(&tree_output.stderr).trim() - )); - } - serde_json::from_slice(&tree_output.stdout) - .map_err(|error| format!("parse Sway tree for focus canary: {error}")) -} - -#[cfg(target_os = "linux")] -fn sway_path_to_container<'a>( - node: &'a serde_json::Value, - con_id: i64, - path: &mut Vec<&'a serde_json::Value>, -) -> bool { - path.push(node); - if node["id"].as_i64() == Some(con_id) { - return true; - } - for key in ["nodes", "floating_nodes"] { - if let Some(children) = node[key].as_array() { - for child in children { - if sway_path_to_container(child, con_id, path) { - return true; - } - } - } - } - path.pop(); - false -} - -#[cfg(target_os = "linux")] -fn sway_fullscreen_holder_id(tree: &serde_json::Value, view_id: i64) -> Option { - let mut path = Vec::new(); - sway_path_to_container(tree, view_id, &mut path).then_some(())?; - path.into_iter() - .rev() - .find(|node| node["fullscreen_mode"].as_i64().unwrap_or(0) != 0) - .and_then(|node| node["id"].as_i64()) -} - -#[cfg(target_os = "linux")] -fn sway_container_fullscreen_mode(tree: &serde_json::Value, con_id: i64) -> Option { - if tree["id"].as_i64() == Some(con_id) { - return Some(tree["fullscreen_mode"].as_i64().unwrap_or(0)); - } - ["nodes", "floating_nodes"].into_iter().find_map(|key| { - tree[key].as_array().and_then(|children| { - children - .iter() - .find_map(|child| sway_container_fullscreen_mode(child, con_id)) - }) - }) -} - -#[cfg(target_os = "linux")] -fn wait_for_sway_fullscreen_cleared(view_id: i64) -> Result<(), String> { - let deadline = Instant::now() + Duration::from_secs(1); - loop { - let tree = read_sway_tree()?; - let fullscreen_mode = sway_container_fullscreen_mode(&tree, view_id) - .ok_or_else(|| format!("Sway fullscreen canary lost view {view_id}"))?; - if fullscreen_mode == 0 { - return Ok(()); - } - if Instant::now() >= deadline { - return Err(format!( - "Sway fullscreen canary did not clear fullscreen mode {fullscreen_mode} for view {view_id}" - )); - } - std::thread::sleep(Duration::from_millis(25)); - } -} - -#[cfg(target_os = "linux")] -fn run_sway_container_command(con_id: i64, command: &[&str], purpose: &str) -> Result<(), String> { - let criterion = format!("[con_id={con_id}]"); - let output = Command::new("swaymsg") - .arg("-r") - .arg(criterion.as_str()) - .args(command) - .output() - .map_err(|error| format!("run Sway {purpose} for con_id {con_id}: {error}"))?; - let result: serde_json::Value = serde_json::from_slice(&output.stdout) - .map_err(|error| format!("parse Sway {purpose} result for con_id {con_id}: {error}"))?; - let succeeded = output.status.success() - && result.as_array().is_some_and(|results| { - !results.is_empty() && results.iter().all(|item| item["success"] == true) - }); - if !succeeded { - Err(format!( - "Sway {purpose} for con_id {con_id} failed: stdout={} stderr={}", - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim(), - )) - } else { - if std::env::var_os("CUA_E2E_DEBUG_SWAY").is_some() { - let tree = read_sway_tree()?; - let mut rows = Vec::new(); - collect_sway_debug_rows(&tree, &mut rows); - eprintln!( - "[testkit] Sway {purpose}: con_id={con_id} command={command:?} tree={rows:?}" - ); - } - Ok(()) - } -} - -#[cfg(target_os = "linux")] -fn collect_sway_debug_rows(node: &serde_json::Value, rows: &mut Vec) { - let focused = node["focused"].as_bool().unwrap_or(false); - let fullscreen = node["fullscreen_mode"].as_i64().unwrap_or(0); - if focused || fullscreen != 0 || node["pid"].as_u64().is_some() { - rows.push(format!( - "id={} pid={} name={:?} focused={focused} fullscreen={fullscreen}", - node["id"].as_i64().unwrap_or_default(), - node["pid"].as_u64().unwrap_or_default(), - node["name"].as_str().unwrap_or("") - )); - } - for key in ["nodes", "floating_nodes"] { - if let Some(children) = node[key].as_array() { - for child in children { - collect_sway_debug_rows(child, rows); - } - } - } -} - -#[cfg(target_os = "linux")] -fn sway_target_title(windows: &serde_json::Value, target: TargetWindow) -> Option<&str> { - let windows = windows["windows"].as_array()?; - windows - .iter() - .find(|window| { - window["pid"].as_u64() == Some(target.pid as u64) - && window["window_id"].as_u64() == Some(target.native_id) - }) - .and_then(|window| window["title"].as_str()) - .or_else(|| { - let mut titles = windows - .iter() - .filter(|window| window["pid"].as_u64() == Some(target.pid as u64)) - .filter_map(|window| window["title"].as_str()) - .filter(|title| !title.is_empty()); - let title = titles.next()?; - titles.next().is_none().then_some(title) - }) -} - -#[cfg(target_os = "linux")] -fn collect_sway_container_ids_by_pid(node: &serde_json::Value, pid: u32, ids: &mut Vec) { - if node["pid"].as_u64() == Some(pid as u64) { - if let Some(id) = node["id"].as_i64() { - ids.push(id); - } - } - for key in ["nodes", "floating_nodes"] { - if let Some(children) = node[key].as_array() { - for child in children { - collect_sway_container_ids_by_pid(child, pid, ids); - } - } - } -} - -#[cfg(target_os = "linux")] -fn sway_container_id_by_unique_pid(node: &serde_json::Value, pid: u32) -> Option { - let mut ids = Vec::new(); - collect_sway_container_ids_by_pid(node, pid, &mut ids); - (ids.len() == 1).then(|| ids[0]) -} - -#[cfg(target_os = "linux")] -fn sway_container_id(node: &serde_json::Value, title: &str) -> Option { - let raw_title = title - .rsplit_once(" [") - .map(|(candidate, _)| candidate) - .unwrap_or(title); - if node["name"] - .as_str() - .is_some_and(|name| name == title || name == raw_title) - { - return node["id"].as_i64(); - } - ["nodes", "floating_nodes"].into_iter().find_map(|key| { - node[key].as_array().and_then(|children| { - children - .iter() - .find_map(|child| sway_container_id(child, title)) - }) - }) -} - -#[cfg(all(test, target_os = "linux"))] -mod sway_tests { - use super::*; - - #[test] - fn fullscreen_command_targets_the_holder_ancestor() { - let tree = serde_json::json!({ - "id": 1, - "nodes": [{ - "id": 30, - "fullscreen_mode": 1, - "nodes": [{ - "id": 31, - "pid": 1234, - "name": "CuaTestHarness Sentinel", - "focused": true, - "fullscreen_mode": 0 - }] - }] - }); - assert_eq!(sway_fullscreen_holder_id(&tree, 31), Some(30)); - } - - #[test] - fn fullscreen_command_prefers_the_actionable_view() { - let tree = serde_json::json!({ - "id": 1, - "fullscreen_mode": 1, - "nodes": [{ - "id": 30, - "fullscreen_mode": 1, - "nodes": [{ - "id": 31, - "pid": 1234, - "name": "CuaTestHarness Sentinel", - "focused": true, - "fullscreen_mode": 1 - }] - }] - }); - assert_eq!(sway_fullscreen_holder_id(&tree, 31), Some(31)); - } - - #[test] - fn fullscreen_clear_checks_the_view_not_inherited_workspace_state() { - let tree = serde_json::json!({ - "id": 1, - "fullscreen_mode": 1, - "nodes": [{ - "id": 30, - "fullscreen_mode": 1, - "nodes": [{ - "id": 31, - "pid": 1234, - "name": "CuaTestHarness Sentinel", - "focused": true, - "fullscreen_mode": 0 - }] - }] - }); - assert_eq!(sway_container_fullscreen_mode(&tree, 31), Some(0)); - } - - #[test] - fn focus_target_uses_unique_sway_pid_when_title_is_unavailable() { - let tree = serde_json::json!({ - "id": 1, - "nodes": [{ - "id": 9, - "nodes": [{ - "id": 42, - "pid": 1234, - "name": "CuaTestHarness Electron" - }] - }] - }); - assert_eq!(sway_container_id_by_unique_pid(&tree, 1234), Some(42)); - } - - #[test] - fn focus_target_does_not_guess_when_pid_owns_multiple_windows() { - let tree = serde_json::json!({ - "id": 1, - "nodes": [ - {"id": 41, "pid": 1234, "name": "First"}, - {"id": 42, "pid": 1234, "name": "Second"} - ] - }); - assert_eq!(sway_container_id_by_unique_pid(&tree, 1234), None); - assert_eq!(sway_container_id(&tree, "Second"), Some(42)); - } - - #[test] - fn focus_target_uses_unique_same_process_title_when_window_id_changes() { - let windows = serde_json::json!({ - "windows": [{ - "pid": 1234, - "window_id": 88, - "title": "CuaTestHarness Electron [cdp=9223]" - }] - }); - let target = TargetWindow { - pid: 1234, - native_id: 99, - }; - assert_eq!( - sway_target_title(&windows, target), - Some("CuaTestHarness Electron [cdp=9223]") - ); - } -} - #[cfg(target_os = "windows")] fn physically_focus_windows_sentinel(target: TargetWindow) { use windows::Win32::Foundation::{HWND, POINT, RECT}; @@ -988,34 +569,6 @@ fn wait_for_native_focus_stable(target: TargetWindow) { } } -#[cfg(target_os = "linux")] -fn wait_for_native_focus_lost(target: TargetWindow) -> Result<(), String> { - use crate::observer::{ObserverBackend, TargetZ}; - - let backend = NativeObserver::new(); - let deadline = Instant::now() + Duration::from_secs(3); - loop { - let snapshot = backend.snapshot(target).ok(); - let lost = snapshot - .as_ref() - .is_some_and(|snapshot| snapshot.target_z != TargetZ::Foreground); - if lost { - return Ok(()); - } - if Instant::now() >= deadline { - return Err(format!( - "native Wayland focus-loss canary was not detected; last snapshot: {snapshot:?}" - )); - } - std::thread::sleep(Duration::from_millis(25)); - } -} - -#[cfg(not(target_os = "linux"))] -fn wait_for_native_focus_lost(_target: TargetWindow) -> Result<(), String> { - Err("native Wayland focus observation is only available on Linux".to_owned()) -} - #[cfg(not(any(target_os = "windows", target_os = "linux")))] fn wait_for_native_focus_stable(_target: TargetWindow) {} diff --git a/packages/cua-driver/rust/crates/cua-driver/Cargo.toml b/packages/cua-driver/rust/crates/cua-driver/Cargo.toml index 15675283252..d3ad09b3e08 100644 --- a/packages/cua-driver/rust/crates/cua-driver/Cargo.toml +++ b/packages/cua-driver/rust/crates/cua-driver/Cargo.toml @@ -17,6 +17,7 @@ anyhow = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } cua-driver-core = { path = "../cua-driver-core" } +cua-driver-contract = { path = "../cua-driver-contract" } cua-driver-sdk = { path = "../cua-driver-sdk" } cursor-overlay = { path = "../cursor-overlay" } pip-preview = { path = "../pip-preview" } @@ -57,12 +58,6 @@ platform-linux = { path = "../platform-linux" } [features] default = [] -# GNOME/KDE portal input ships in ordinary Linux releases; PipeWire capture -# remains a separate modern-desktop feature. The combined name stays as a -# compatibility alias for Nix and downstream builds. -portal-input = ["platform-linux/portal-input"] -portal-capture = ["platform-linux/portal-capture"] -portal-libei = ["platform-linux/portal-libei"] [target.'cfg(target_os = "windows")'.build-dependencies] # Embed the Windows manifest for DPI awareness (fixes coordinate mismatches @@ -73,7 +68,6 @@ embed-resource = "2" # Shared integration-test harness (MCP/CLI transports, child reaping, paths, # response accessors). Dev-only — never linked into the shipped driver. cua-driver-testkit = { path = "../cua-driver-testkit" } -cua-driver-contract = { path = "../cua-driver-contract" } tokio = { workspace = true, features = ["full"] } image = { workspace = true } base64 = { workspace = true } diff --git a/packages/cua-driver/rust/crates/cua-driver/src/autostart.rs b/packages/cua-driver/rust/crates/cua-driver/src/autostart.rs index feacd66258a..514d4f55af4 100644 --- a/packages/cua-driver/rust/crates/cua-driver/src/autostart.rs +++ b/packages/cua-driver/rust/crates/cua-driver/src/autostart.rs @@ -184,6 +184,31 @@ fn starts_with_drive_letter(path: &str) -> bool { matches!(path.as_bytes(), [drive, b':', ..] if drive.is_ascii_alphabetic()) } +#[cfg(any(target_os = "windows", test))] +const ELEVATE_PS: &str = r#" +$p = $null +try { + $p = Start-Process -FilePath 'powershell.exe' -ArgumentList @('-NoProfile','-NonInteractive','-Command',__INNER_COMMAND__) -Verb RunAs -Wait -PassThru -ErrorAction Stop +} catch { + [Console]::Error.WriteLine("qwen-cua-driver: autostart elevation failed: " + $_.Exception.Message) + exit 1 +} +if ($null -eq $p -or $null -eq $p.ExitCode) { + [Console]::Error.WriteLine("qwen-cua-driver: autostart elevation returned no exit code.") + exit 1 +} +exit $p.ExitCode +"#; + +#[cfg(any(target_os = "windows", test))] +fn elevation_script(exe: &str) -> String { + let inner = format!("& '{}' autostart enable", exe.replace('\'', "''")); + ELEVATE_PS.replace( + "__INNER_COMMAND__", + &format!("'{}'", inner.replace('\'', "''")), + ) +} + // ── Windows impl ────────────────────────────────────────────────────────── #[cfg(target_os = "windows")] @@ -250,8 +275,7 @@ $action = New-ScheduledTaskAction ` $trigger = New-ScheduledTaskTrigger -AtLogOn -User $user $principal = New-ScheduledTaskPrincipal -UserId $user -LogonType Interactive -RunLevel Highest $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1) -ExecutionTimeLimit (New-TimeSpan -Hours 0) -Unregister-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Confirm:$false -ErrorAction SilentlyContinue -Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Description "${env:CUA_DRIVER_AS_CLI}: serve daemon, auto-start at interactive logon, RunLevel=Highest for UWP/AppContainer support" | Out-Null +Register-ScheduledTask -Force -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Description "${env:CUA_DRIVER_AS_CLI}: serve daemon, auto-start at interactive logon, RunLevel=Highest for UWP/AppContainer support" | Out-Null # The uiAccess worker (`qwen-cua-driver-uia.exe`) has no public client route and is # not started by this task. The High-IL daemon is the supported path for @@ -273,7 +297,7 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge .output() .map_err(|e| anyhow!("failed to invoke powershell: {e}"))?; if out.status.success() { - return Ok(()); + return confirm_registered(); } let stderr = String::from_utf8_lossy(&out.stderr); @@ -311,14 +335,7 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge // happens INSIDE the elevated process (where it'll succeed on // the first attempt and not re-enter this branch). -Wait so we // can capture the child's exit code. - let inner = format!("& \"{}\" autostart enable", exe.replace('"', "`\"")); - let outer = format!( - "$p = Start-Process -FilePath '{}' -ArgumentList @('-NoProfile','-NonInteractive','-Command',{}) -Verb RunAs -Wait -PassThru; exit $p.ExitCode", - "powershell.exe", - // Embed the inner command as a single-quoted PowerShell string - // (PS escapes ' as '' inside ''-strings). - format!("'{}'", inner.replace('\'', "''")) - ); + let outer = elevation_script(exe); Command::new("powershell") .args(["-NoProfile", "-NonInteractive", "-Command", &outer]) .creation_flags(CREATE_NO_WINDOW) @@ -327,7 +344,7 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge }; if elevated_status.success() { - Ok(()) + confirm_registered() } else { Err(anyhow!( "self-elevation for autostart registration failed (exit {}). The UAC \ @@ -339,6 +356,19 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge } } + fn confirm_registered() -> Result<()> { + match status() { + Ok(Status::NotRegistered) => Err(anyhow!( + "autostart registration reported success but task '{}' does not exist", + task_name() + )), + Ok(Status::RegisteredIdle | Status::RegisteredRunning) => Ok(()), + Err(error) => Err(anyhow!( + "autostart registration may have completed, but could not be verified: {error}" + )), + } + } + pub fn disable() -> Result<()> { // schtasks /Delete returns 0 on success, 1 on "task not found" // (which we treat as success: the goal is "no task registered" @@ -523,6 +553,36 @@ pub fn run_autostart_cmd(subcommand: &str) { mod tests { use super::*; + #[test] + fn elevation_quotes_the_executable_as_a_literal_argument() { + let script = elevation_script(r"C:\o'brien\$data`files\qwen-cua-driver.exe"); + assert!(script + .contains("'& ''C:\\o''''brien\\$data`files\\qwen-cua-driver.exe'' autostart enable'")); + assert!(!script.contains("__INNER_COMMAND__")); + } + + #[cfg(target_os = "windows")] + #[test] + fn elevation_returns_failure_or_the_actual_child_exit_code() { + for (body, expected) in [ + ("Write-Error 'UAC cancelled'", 1), + ("return $null", 1), + ("return [pscustomobject]@{ ExitCode = $null }", 1), + ("return [pscustomobject]@{ ExitCode = 0 }", 0), + ("return [pscustomobject]@{ ExitCode = 7 }", 7), + ] { + let script = format!( + "function Start-Process {{ [CmdletBinding()] param([string]$FilePath, [string[]]$ArgumentList, [string]$Verb, [switch]$Wait, [switch]$PassThru)\n{body}\n}}\n{}", + elevation_script(r"C:\Qwen\qwen-cua-driver.exe") + ); + let status = std::process::Command::new("powershell") + .args(["-NoProfile", "-NonInteractive", "-Command", &script]) + .status() + .expect("run the elevation payload with a stubbed Start-Process"); + assert_eq!(status.code(), Some(expected), "{body}"); + } + } + #[test] fn windows_task_path_strips_short_extended_drive_prefix() { assert_eq!( diff --git a/packages/cua-driver/rust/crates/cua-driver/src/doctor.rs b/packages/cua-driver/rust/crates/cua-driver/src/doctor.rs index 9cae3decd74..dc41a24a530 100644 --- a/packages/cua-driver/rust/crates/cua-driver/src/doctor.rs +++ b/packages/cua-driver/rust/crates/cua-driver/src/doctor.rs @@ -17,7 +17,7 @@ //! telemetry state. //! - **Windows**: interactive desktop session detection (Session 0 warning), //! UI Automation COM availability, top-level window enumeration count. -//! - **Linux**: `DISPLAY` / `WAYLAND_DISPLAY` presence, X11 connection +//! - **Linux**: `DISPLAY` presence, X11 connection //! reachability, AT-SPI bus availability hint. //! - **macOS**: existing legacy-cleanup steps (LaunchAgent plist + update //! script), plus a hint to run `qwen-cua-driver diagnose` for a full @@ -431,40 +431,16 @@ fn wait_for_child( #[cfg(target_os = "linux")] fn append_platform_probes(report: &mut Report) { - // Display server probe. Order matters: Wayland wins when both are set - // (XWayland leaves DISPLAY pointing at the X server XWayland exposes, - // but the actual session is still Wayland). let display = std::env::var("DISPLAY").ok().filter(|v| !v.is_empty()); - let wayland = std::env::var("WAYLAND_DISPLAY") - .ok() - .filter(|v| !v.is_empty()); - match (display.as_deref(), wayland.as_deref()) { - (None, None) => report.push( - Probe::warn( - "display server", - "neither DISPLAY nor WAYLAND_DISPLAY set — window-driving tools will fail", - ) - .with_detail( - "run from an interactive desktop session (X11 / Wayland with XWayland) or set DISPLAY explicitly.", - ), - ), - (Some(d), None) => report.push(Probe::ok( + match display.as_deref() { + Some(display) => report.push(Probe::ok( "display server", - format!("X11 (DISPLAY={d})"), + format!("X11 (DISPLAY={display})"), )), - (None, Some(w)) => report.push( - Probe::warn( - "display server", - format!("Wayland only (WAYLAND_DISPLAY={w}, DISPLAY unset)"), - ) - .with_detail( - "X11 tools (list_windows, screenshot) need XWayland — start your session with XWayland enabled.", - ), + None => report.push( + Probe::warn("display server", "only X11 is supported; DISPLAY is unset") + .with_detail("Set DISPLAY and XAUTHORITY for an X server, including XWayland."), ), - (Some(d), Some(w)) => report.push(Probe::ok( - "display server", - format!("Wayland+XWayland (WAYLAND_DISPLAY={w}, DISPLAY={d})"), - )), } // X11 window enumeration probe. An empty result could mean either an diff --git a/packages/cua-driver/rust/crates/cua-driver/src/main.rs b/packages/cua-driver/rust/crates/cua-driver/src/main.rs index f7ead0504b1..7736806a79e 100644 --- a/packages/cua-driver/rust/crates/cua-driver/src/main.rs +++ b/packages/cua-driver/rust/crates/cua-driver/src/main.rs @@ -109,6 +109,9 @@ fn maybe_wrap_finite_command() { let Some(command_name) = cli::finite_command_name_from_argv() else { return; }; + if !telemetry::is_enabled() { + return; + } let tool_name = cli::finite_tool_name_from_argv(); let computer_action = cli::finite_computer_action_from_argv(); let operation = cli::finite_operation_from_argv(); @@ -817,6 +820,12 @@ fn main() { #[cfg(not(target_os = "macos"))] fn main() -> anyhow::Result<()> { + #[cfg(windows)] + if let Some(code) = + platform_windows::standard_user_launch::run_shell_launch_helper_if_requested() + { + std::process::exit(code); + } init_logging(); if let Some(generation) = private_worker::requested_generation() { return private_worker::run(generation, None); diff --git a/packages/cua-driver/rust/crates/cua-driver/src/proxy.rs b/packages/cua-driver/rust/crates/cua-driver/src/proxy.rs index df087d1f557..16452799130 100644 --- a/packages/cua-driver/rust/crates/cua-driver/src/proxy.rs +++ b/packages/cua-driver/rust/crates/cua-driver/src/proxy.rs @@ -698,6 +698,19 @@ async fn handle_proxy_request( Err(e) => Response::error(id, -32602, format!("Invalid params: {e}")), Ok(call) => { if let Err(error) = authorize_tool_call(&call.name, &call.args) { + if cua_driver_contract::computer_use::is_app_call(&call.name, &call.args) { + return Response::ok( + id, + serde_json::to_value( + cua_driver_core::protocol::ToolResult::computer_use_error( + "permission_denied", + error.to_string(), + false, + ), + ) + .expect("serializable tool result"), + ); + } return Response::error(id, -32603, error.to_string()); } forward_tool_call( diff --git a/packages/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs b/packages/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs index f835bd482cf..2ed465aa14a 100644 --- a/packages/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs +++ b/packages/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs @@ -53,6 +53,7 @@ pub struct SdkAdapter { runtime_prefix: String, runtime_scope: String, _session_end_hook: cua_driver_core::session::SessionEndHookRegistration, + _session_revive_hook: cua_driver_core::session::SessionReviveHookRegistration, session_lifecycle: tokio::sync::Mutex<()>, } @@ -91,6 +92,14 @@ impl SdkAdapter { sessions.scopes.entry(public.to_owned()).or_default(); sessions.mark_ended(public); }); + let revive_sessions = public_sessions.clone(); + let revive_prefix = runtime_prefix.clone(); + let session_revive_hook = + cua_driver_core::session::register_scoped_session_revive_hook(move |session| { + if let Some(public) = session.strip_prefix(&revive_prefix) { + revive_sessions.lock().unwrap().ended.remove(public); + } + }); Ok(Arc::new(Self { driver, tools_list, @@ -98,6 +107,7 @@ impl SdkAdapter { runtime_prefix, runtime_scope, _session_end_hook: session_end_hook, + _session_revive_hook: session_revive_hook, session_lifecycle: tokio::sync::Mutex::new(()), })) } @@ -107,7 +117,7 @@ impl SdkAdapter { } pub fn is_known_tool(&self, name: &str) -> bool { - name == "type_text_chars" + matches!(name, "type_text_chars" | "start_session" | "end_session") || self .tools_list .get("tools") @@ -228,11 +238,17 @@ impl SdkAdapter { } pub fn is_session_ended(&self, session: &str) -> bool { - self.public_sessions + if !self + .public_sessions .lock() .unwrap() .ended .contains_key(session) + { + return false; + } + let internal = format!("{}{session}", self.runtime_prefix); + !cua_driver_core::session::recreates_on_next_call(&internal, &internal) } pub fn mark_all_sessions_ended(&self) { @@ -553,6 +569,38 @@ mod tests { sdk.shutdown().await.expect("shutdown"); } + #[tokio::test] + async fn idle_implicit_recreation_updates_the_daemon_mirror() { + let _runtime_guard = crate::test_runtime_lock().lock().await; + let sdk = SdkAdapter::load(host_driver()).await.expect("SDK adapter"); + let transport = "g08-adapter-idle-transport"; + let args = || json!({"_session_id": transport, "_transport_session_id": transport}); + let first = sdk.invoke_raw("get_config", args()).await.unwrap(); + assert_ne!(first["isError"], true, "{first}"); + let internal = format!("{}{transport}", sdk.runtime_prefix); + assert_eq!( + cua_driver_core::session::evict_idle_with_prefix(std::time::Duration::ZERO, &internal), + [internal] + ); + assert!(sdk.public_session_observation_state(transport).ended); + assert!( + !sdk.is_session_ended(transport), + "early daemon guard permits core recreation" + ); + let next = sdk.invoke_raw("get_config", args()).await.unwrap(); + assert_ne!(next["isError"], true, "{next}"); + assert!(!sdk.public_session_observation_state(transport).ended); + let ended = sdk.invoke_raw("end_session", args()).await.unwrap(); + assert_ne!(ended["isError"], true, "{ended}"); + assert!(sdk.is_session_ended(transport)); + let refused = sdk.invoke_raw("get_config", args()).await.unwrap(); + assert_eq!( + refused["structuredContent"]["refusal"]["code"], + "session_ended" + ); + sdk.shutdown().await.unwrap(); + } + #[tokio::test] async fn runtime_owned_end_hook_updates_only_its_adapter() { let _runtime_guard = crate::test_runtime_lock().lock().await; diff --git a/packages/cua-driver/rust/crates/cua-driver/src/serve.rs b/packages/cua-driver/rust/crates/cua-driver/src/serve.rs index 92d3aa5a2b0..671c4eb6157 100644 --- a/packages/cua-driver/rust/crates/cua-driver/src/serve.rs +++ b/packages/cua-driver/rust/crates/cua-driver/src/serve.rs @@ -360,6 +360,16 @@ async fn invoke_daemon_tool( // the MCP policy contract now that every call passes through the daemon. if let Err(error) = cua_driver_core::authorization::authorize_tool_call(&tool_name, &args) { observe_daemon_error(observation, 1); + if cua_driver_contract::computer_use::is_app_call(&tool_name, &args) { + return DaemonResponse::ok( + serde_json::to_value(cua_driver_core::protocol::ToolResult::computer_use_error( + "permission_denied", + error.to_string(), + false, + )) + .expect("serializable tool result"), + ); + } return DaemonResponse::err(error.to_string(), 1); } @@ -1318,8 +1328,6 @@ pub async fn run_serve( cua_driver_core::authorization::validate_startup_authorization()?; - eprintln!("Qwen Cua Driver daemon listening on {socket_path}"); - // Build the descriptor once and reuse it for every pipe instance. An // embedded host may explicitly delegate its trusted session to one exact // sandbox account SID; ordinary daemons remain current-user-only. @@ -1366,28 +1374,72 @@ pub async fn run_serve( } let mut first_pipe = true; + let mut consecutive_failures = 0; + let mut fatal_error = None; loop { + if consecutive_failures > 0 { + if consecutive_failures >= PIPE_INSTANCE_FAILURE_LIMIT { + fatal_error = Some(anyhow::anyhow!( + "named pipe {socket_path}: {consecutive_failures} consecutive pipe instance failures" + )); + break; + } + tokio::select! { + _ = tokio::time::sleep(PIPE_INSTANCE_RETRY_DELAY) => {} + _ = &mut shutdown_rx => { + eprintln!("Qwen Cua Driver daemon shutting down."); + break; + } + _ = &mut parent_liveness => { + eprintln!("Qwen Cua Driver embedded host closed its lifetime pipe; shutting down."); + break; + } + } + } + // All daemons use the current-user descriptor. Embedded daemons also // reserve the pipe name with their first instance. let first_pipe_instance = embedded && first_pipe; - let server = if sec_attrs_ptr.is_null() { + let created = if sec_attrs_ptr.is_null() { ServerOptions::new() .first_pipe_instance(first_pipe_instance) .create(socket_path) - .map_err(|e| anyhow::anyhow!("create named pipe {socket_path}: {e}"))? } else { unsafe { ServerOptions::new() .first_pipe_instance(first_pipe_instance) .create_with_security_attributes_raw(socket_path, sec_attrs_ptr) - .map_err(|e| anyhow::anyhow!("create named pipe {socket_path}: {e}"))? } }; - first_pipe = false; + let server = match created { + Ok(server) => server, + Err(error) if first_pipe => { + fatal_error = Some(anyhow::anyhow!("create named pipe {socket_path}: {error}")); + break; + } + Err(error) => { + consecutive_failures += 1; + eprintln!( + "Qwen Cua Driver: create named pipe {socket_path} failed ({consecutive_failures} consecutive): {error}; retrying" + ); + continue; + } + }; + if first_pipe { + first_pipe = false; + eprintln!("Qwen Cua Driver daemon listening on {socket_path}"); + } tokio::select! { result = server.connect() => { - result.map_err(|e| anyhow::anyhow!("named pipe connect: {e}"))?; + if let Err(error) = result { + consecutive_failures += 1; + eprintln!( + "Qwen Cua Driver: named pipe connect on {socket_path} failed ({consecutive_failures} consecutive): {error}; recreating the instance" + ); + continue; + } + consecutive_failures = 0; let client_process_id = unsafe { named_pipe_client_process_id(server.as_raw_handle().cast()) }; let client_sid = unsafe { @@ -1743,9 +1795,18 @@ pub async fn run_serve( sdk.shutdown() .await .map_err(|error| anyhow::anyhow!("shut down SDK runtime: {error}"))?; - Ok(()) + match fatal_error { + Some(error) => Err(error), + None => Ok(()), + } } +#[cfg(target_os = "windows")] +const PIPE_INSTANCE_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(500); + +#[cfg(target_os = "windows")] +const PIPE_INSTANCE_FAILURE_LIMIT: u32 = 120; + #[cfg(all(test, target_os = "windows"))] mod named_pipe_authentication_tests { use super::{ diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/README.md b/packages/cua-driver/rust/crates/cua-driver/tests/README.md index 446a8dae2a0..f26a00a5414 100644 --- a/packages/cua-driver/rust/crates/cua-driver/tests/README.md +++ b/packages/cua-driver/rust/crates/cua-driver/tests/README.md @@ -142,8 +142,7 @@ desktop state outside the repo-local fixtures: through the browser endpoint instead of relying on browser command-line handoff behavior. Run it through the cross-platform `scripts/ci/run-rust-standalone-browser-e2e.sh`, which stages the Electron - foreground sentinel when needed and opts pure Wayland runs into the native - backend. Non-macOS runs prove that standard mode refuses existing-profile + foreground sentinel when needed. Linux requires an authenticated X11 display. Non-macOS runs prove that standard mode refuses existing-profile attachment before running the authorized success rows with a disposable unrestricted daemon. macOS Lume maintainers can add the success rows to the VM acceptance run with `run-all.sh --standalone-browser`. diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/agent_cursor_showcase_test.rs b/packages/cua-driver/rust/crates/cua-driver/tests/agent_cursor_showcase_test.rs index 0e21737c1df..e6d24ab886b 100644 --- a/packages/cua-driver/rust/crates/cua-driver/tests/agent_cursor_showcase_test.rs +++ b/packages/cua-driver/rust/crates/cua-driver/tests/agent_cursor_showcase_test.rs @@ -449,11 +449,5 @@ fn platform_route() -> DriverRoute { #[cfg(target_os = "linux")] fn platform_route() -> DriverRoute { - if std::env::var_os("CUA_INJECT_SOCKET").is_some() { - DriverRoute::LinuxCuaCompositorInject - } else if std::env::var_os("WAYLAND_DISPLAY").is_some() { - DriverRoute::LinuxWaylandVirtualPointer - } else { - DriverRoute::LinuxXTest - } + DriverRoute::LinuxXTest } diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/packages/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs index dc82e033040..8b4cf8e3ac1 100644 --- a/packages/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs +++ b/packages/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs @@ -411,44 +411,9 @@ fn snapshot(fixture: &mut Fixture) -> ToolResponse { "capture_mode": "ax" }), ); - refresh_wayland_window_geometry(fixture); state } -#[cfg(target_os = "linux")] -fn refresh_wayland_window_geometry(fixture: &mut Fixture) { - if std::env::var_os("WAYLAND_DISPLAY").is_none() { - return; - } - // A compositor may place a newly mapped toplevel after its first - // list_windows appearance. Re-read the current compositor-owned geometry - // for every snapshot, matching the native GTK harness, instead of turning - // that launch-time placement into a stale PX coordinate origin. - let windows = fixture.driver.call( - "list_windows", - serde_json::json!({ "pid": fixture.pid as i64 }), - ); - let Some(window) = windows.structured()["windows"] - .as_array() - .and_then(|items| { - items - .iter() - .find(|window| window["pid"].as_u64() == Some(fixture.pid as u64)) - }) - else { - return; - }; - if let Some(window_id) = window["window_id"].as_u64() { - fixture.wid = window_id; - } - let bounds = &window["bounds"]; - fixture.window_x = bounds["x"].as_f64().unwrap_or(fixture.window_x); - fixture.window_y = bounds["y"].as_f64().unwrap_or(fixture.window_y); -} - -#[cfg(not(target_os = "linux"))] -fn refresh_wayland_window_geometry(_fixture: &mut Fixture) {} - fn window_origin(fixture: &Fixture, _state: &ToolResponse) -> (f64, f64) { #[cfg(target_os = "macos")] { @@ -1212,9 +1177,7 @@ fn shared_case(spec: &HostSpec, action: &str, addressing: &str, delivery: &str) // occluded, unfocused toplevel. Focus-free AT-SPI button actions are // the exception: they are externally verified by the fixture and the // focus/z-order/leak sentinels. - if std::env::var_os("CUA_INJECT_SOCKET").is_some() { - Vec::new() - } else { + { match (action, targeting) { ("left_click" | "child_window", Targeting::Ax) => Vec::new(), _ => vec![RefusalCode::BackgroundUnavailable], @@ -1252,11 +1215,7 @@ fn shared_case(spec: &HostSpec, action: &str, addressing: &str, delivery: &str) OracleKind::ZOrder, OracleKind::NoLeakedInput, ]); - if cua_driver_testkit::e2e::DisplayServer::current() - != cua_driver_testkit::e2e::DisplayServer::Wayland - { - oracles.push(OracleKind::Cursor); - } + oracles.push(OracleKind::Cursor); if !expected_background_refusal && cfg!(target_os = "windows") && (matches!(action, "press_key" | "hotkey" | "type_submit") @@ -1587,11 +1546,7 @@ fn embedded_browser_case(spec: &HostSpec) -> CaseSpec { OracleKind::ZOrder, OracleKind::NoLeakedInput, ]; - if cua_driver_testkit::e2e::DisplayServer::current() - != cua_driver_testkit::e2e::DisplayServer::Wayland - { - oracles.push(OracleKind::Cursor); - } + oracles.push(OracleKind::Cursor); let case = CaseSpec::delivered( format!( "{}-{}-browser-tool-roundtrip", diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs b/packages/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs index 5f3d37558bd..5edaffb8562 100644 --- a/packages/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs +++ b/packages/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs @@ -155,13 +155,7 @@ fn marker_value(snap: &serde_json::Value, marker: &str) -> Option { } fn desktop_input_route() -> DriverRoute { - if std::env::var_os("CUA_INJECT_SOCKET").is_some() { - DriverRoute::LinuxCuaCompositorInject - } else if std::env::var_os("WAYLAND_DISPLAY").is_some() { - DriverRoute::LinuxWaylandVirtualPointer - } else { - DriverRoute::LinuxXTest - } + DriverRoute::LinuxXTest } fn start_scope(driver: &mut McpDriver, session: &str, scope: &str) { diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs b/packages/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs index e753251960f..79e30a5375b 100644 --- a/packages/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs +++ b/packages/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs @@ -367,16 +367,7 @@ fn harness_gtk3_query_projects_structured_elements() { #[test] #[ignore] fn harness_gtk3_modified_click_preserves_selection() { - let wayland_refusal = DisplayServer::current() == DisplayServer::Wayland; - let case = if wayland_refusal { - native_background_case( - "gtk3", - "modified_click_selection", - Targeting::Ax, - DriverRoute::LinuxWaylandVirtualPointer, - ) - .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]) - } else { + let case = { native_foreground_case( "gtk3", "modified_click_selection", @@ -386,55 +377,7 @@ fn harness_gtk3_modified_click_preserves_selection() { }; run_case(case, |pid, window_id, driver| { let first = snapshot(driver, pid, window_id); - if wayland_refusal { - let beta = element_token(&first, "selection-beta"); - let (add_beta, mut passed) = run_with_background_oracles( - driver, - TargetWindow { - pid, - native_id: window_id, - }, - |driver| { - driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, - "window_id": window_id, - "element_token": beta, - "delivery_mode": "background", - "modifier": ["ctrl"] - }), - ) - }, - ) - .unwrap_or_else(|error| panic!("Wayland background contract failed: {error}")); - assert!( - add_beta.is_error(), - "native Wayland unexpectedly accepted a modified pointer click: {}", - add_beta.text() - ); - assert!( - add_beta - .text() - .contains("modified element clicks are unavailable on native Wayland"), - "native Wayland returned the wrong refusal: {}", - add_beta.text() - ); - std::thread::sleep(Duration::from_millis(250)); - let post = snapshot(driver, pid, window_id); - assert!( - post.tree_text().contains("selection=none"), - "refused modified click mutated the selection:\n{}", - post.tree_text() - ); - passed.push(OracleKind::FixtureState); - return Observation::refused( - RefusalCode::BackgroundUnavailable, - passed, - "native Wayland cannot pair virtual-pointer delivery with modifier state", - Evidence::default(), - ); - } + {} let alpha = element_token(&first, "selection-alpha"); let select_alpha = driver.call( @@ -880,13 +823,10 @@ fn row_expects_refusal(row: CatalogRow) -> bool { ) { return false; } - let inject_mode = std::env::var_os("CUA_INJECT_SOCKET").is_some(); - if !inject_mode - && matches!( - row.operation, - Operation::PressKey { .. } | Operation::Hotkey { .. } - ) - { + if matches!( + row.operation, + Operation::PressKey { .. } | Operation::Hotkey { .. } + ) { return true; } let focus_bound_pointer = matches!( @@ -899,37 +839,12 @@ fn row_expects_refusal(row: CatalogRow) -> bool { { return true; } - DisplayServer::current() == DisplayServer::Wayland - && !inject_mode - && (focus_bound_pointer || matches!(row.operation, Operation::PxTypeText { .. })) + false } fn run_catalog_row(row: CatalogRow) { let expect_refusal = row_expects_refusal(row); - let route = if DisplayServer::current() == DisplayServer::Wayland - && (row.targeting == Targeting::Px - || matches!( - row.operation, - Operation::PressKey { .. } | Operation::Hotkey { .. } - )) { - if matches!( - row.operation, - Operation::PxClick { - button: "left", - count: 1, - .. - } - ) && row.delivery == Delivery::Background - { - DriverRoute::LinuxAtSpiAction - } else if std::env::var_os("CUA_INJECT_SOCKET").is_some() { - DriverRoute::LinuxCuaCompositorInject - } else { - DriverRoute::LinuxWaylandVirtualPointer - } - } else { - row.route - }; + let route = row.route; let case = match row.delivery { Delivery::Background => native_background_case("gtk3", row.action, row.targeting, route), Delivery::Foreground => native_foreground_case("gtk3", row.action, row.targeting, route), diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/private_worker_test.rs b/packages/cua-driver/rust/crates/cua-driver/tests/private_worker_test.rs index c3128c421b0..ceac05c84d2 100644 --- a/packages/cua-driver/rust/crates/cua-driver/tests/private_worker_test.rs +++ b/packages/cua-driver/rust/crates/cua-driver/tests/private_worker_test.rs @@ -116,11 +116,7 @@ async fn private_worker_inherits_the_interactive_linux_display_scope() { return; } let has_x11 = std::env::var("DISPLAY").is_ok_and(|display| !display.is_empty()); - let has_wayland = std::env::var("WAYLAND_DISPLAY").is_ok_and(|display| !display.is_empty()); - assert!( - has_x11 || has_wayland, - "canonical GUI E2E requires DISPLAY or WAYLAND_DISPLAY" - ); + assert!(has_x11, "canonical GUI E2E requires an X11 DISPLAY"); let driver = cua_driver_sdk::CuaDriver::create_private_worker(worker_options()).unwrap(); let standard = driver @@ -179,27 +175,15 @@ async fn private_worker_inherits_the_interactive_linux_display_scope() { .unwrap(); assert_eq!(started_structured["capture_scope"], "desktop"); assert_eq!(started_structured["effective_scope"], "desktop"); - if has_x11 { - let desktop = session - .call_tool("get_desktop_state".into(), "{}".into()) - .await - .unwrap(); - assert!( - !desktop.images.is_empty(), - "worker inherited no usable X11 capture scope: {}", - desktop.text - ); - } else { - let windows = session - .call_tool("list_windows".into(), "{}".into()) - .await - .unwrap(); - assert!( - windows.error_code.is_none(), - "worker inherited no usable native Wayland session scope: {}", - windows.text - ); - } + let desktop = session + .call_tool("get_desktop_state".into(), "{}".into()) + .await + .unwrap(); + assert!( + !desktop.images.is_empty(), + "worker inherited no usable X11 capture scope: {}", + desktop.text + ); let ended = session .call_tool( "end_session".into(), diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/packages/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs index df332bdaa28..f2d9596ec0e 100644 --- a/packages/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs +++ b/packages/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs @@ -756,26 +756,13 @@ fn spawn_driver(label: &str) -> McpDriver { #[cfg(target_os = "macos")] let driver = McpDriver::spawn_macos_daemon_proxy_named(label); #[cfg(target_os = "linux")] - let driver = if std::env::var("CUA_E2E_WAYLAND_SESSION").as_deref() == Ok("generic") { - // Keep Sway IPC available to the out-of-band test oracle while the - // product under test sees only standard/generic Wayland capabilities. - McpDriver::spawn_named_with_env( - label, - &[ - ("SWAYSOCK", "/dev/null/cua-e2e-withheld"), - ("CUA_DRIVER_PERMISSION_MODE", "unrestricted"), - ("CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS", "1"), - ], - ) - } else { - McpDriver::spawn_named_with_env( - label, - &[ - ("CUA_DRIVER_PERMISSION_MODE", "unrestricted"), - ("CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS", "1"), - ], - ) - }; + let driver = McpDriver::spawn_named_with_env( + label, + &[ + ("CUA_DRIVER_PERMISSION_MODE", "unrestricted"), + ("CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS", "1"), + ], + ); #[cfg(all(not(target_os = "macos"), not(target_os = "linux")))] let driver = McpDriver::spawn_named_with_env( label, @@ -802,12 +789,7 @@ fn spawn_standard_driver(label: &str) -> McpDriver { #[cfg(target_os = "linux")] fn configure_linux_browser_command(command: &mut Command) { command.arg("--password-store=basic"); - let native_wayland = std::env::var("XDG_SESSION_TYPE") - .is_ok_and(|session| session.eq_ignore_ascii_case("wayland")) - && std::env::var_os("WAYLAND_DISPLAY").is_some(); - if native_wayland { - command.arg("--ozone-platform=wayland"); - } + command.arg("--ozone-platform=x11"); } fn configure_test_browser_sandbox(command: &mut Command) { @@ -1432,11 +1414,7 @@ fn case(browser: &str, action: &str) -> CaseSpec { OracleKind::ZOrder, OracleKind::NoLeakedInput, ]; - if cua_driver_testkit::e2e::DisplayServer::current() - != cua_driver_testkit::e2e::DisplayServer::Wayland - { - oracles.push(OracleKind::Cursor); - } + oracles.push(OracleKind::Cursor); CaseSpec::delivered( format!("{}-{browser}-standalone-{action}", std::env::consts::OS), browser, @@ -2736,111 +2714,6 @@ fn run_existing_profile_setup(spec: &BrowserSpec) { ); } -#[cfg(target_os = "linux")] -fn run_generic_wayland_existing_profile_refusal(spec: &BrowserSpec) { - assert!( - std::env::var_os("WAYLAND_DISPLAY").is_some() - && std::env::var("CUA_E2E_WAYLAND_SESSION").as_deref() == Ok("generic"), - "the generic-Wayland refusal row requires its explicit native-Wayland lane" - ); - assert!( - platform_linux::wayland::shell_helper::trusted_window_ids_for_pid(std::process::id()) - .is_none(), - "the generic-Wayland refusal row requires the attested GNOME helper to be unavailable" - ); - let scenario = format!( - "{}-{}-standalone-generic-wayland-existing-profile-refusal", - std::env::consts::OS, - spec.name - ); - execute_case( - refusal_case( - &spec.name, - "browser_prepare_existing_profile_generic_wayland", - RefusalCode::BrowserRouteUnavailable, - ), - |evidence| { - let mut driver = spawn_driver(&scenario); - let server = BrowserFixtureServer::start(&standalone_fixture_html()); - let profile = tempfile::Builder::new() - .prefix("cua-e2e-generic-wayland-browser-") - .tempdir() - .expect("create generic Wayland browser profile"); - let mut command = command_for_unprepared_browser( - spec, - profile.path(), - server.page_url(), - TEST_BROWSER_INITIAL_POSITION, - ); - let child = spawn_in_job(&mut command).expect("launch generic Wayland browser"); - let pid = child.id(); - driver.reaper().push(child); - driver.reaper().track_pid(pid); - wait_for_observed(&server, "WEB_HARNESS_MARKER_v1"); - - let windows = driver.call("list_windows", serde_json::json!({"pid": pid})); - let opaque_window = windows.structured()["windows"] - .as_array() - .and_then(|windows| windows.iter().find(|window| window["pid"] == pid)) - .unwrap_or_else(|| { - panic!( - "generic Wayland did not expose the browser's opaque native window: {}", - windows.raw - ) - }); - let opaque_unattested_window_id = opaque_window["window_id"] - .as_u64() - .expect("opaque generic Wayland window id"); - assert!( - opaque_window["z_index"].is_null(), - "generic Wayland must not treat an AT-SPI window as compositor-attested: {}", - windows.raw - ); - - let sentinel = ForegroundSentinel::launch(&mut driver); - *evidence = recording_evidence(driver.recording_dir()); - driver.start_behavior_recording(); - let (mut observation, passed) = sentinel - .observe_desktop(|| { - let session = format!("generic-wayland-refusal-{pid}"); - let started = - driver.call("start_session", serde_json::json!({ "session": session })); - assert!(!started.is_error(), "start_session failed: {}", started.raw); - let refused = driver.call( - "browser_prepare", - serde_json::json!({ - "pid": pid as i64, - "window_id": opaque_unattested_window_id, - "session": session, - "strategy": {"kind": "existing_profile"}, - }), - ); - assert_eq!( - refused.structured()["refusal"]["code"], - "browser_route_unavailable", - "{}", - refused.raw - ); - assert!( - refused.structured()["refusal"]["detail"]["setup_side_effects"].is_null(), - "generic Wayland must refuse before setup mutation: {}", - refused.raw - ); - wait_for_text(&server, "lbl-counter", "counter=0"); - Observation::refused( - RefusalCode::BrowserRouteUnavailable, - vec![OracleKind::FixtureState], - refused.text(), - Evidence::default(), - ) - }) - .expect("observe generic Wayland refusal desktop effects"); - observation.passed_oracles.extend(passed); - observation - }, - ); -} - fn run_stale_ref(spec: &BrowserSpec) { let scenario = format!( "{}-{}-standalone-stale-ref", @@ -4665,11 +4538,6 @@ standalone_browser_test!( standalone_browser_existing_profile_setup, run_existing_profile_setup ); -#[cfg(target_os = "linux")] -standalone_browser_test!( - standalone_browser_generic_wayland_existing_profile_refusal, - run_generic_wayland_existing_profile_refusal -); standalone_browser_test!(standalone_browser_stale_ref, run_stale_ref); standalone_browser_test!(standalone_browser_frames, run_frame_roundtrip); standalone_browser_test!(standalone_browser_multi_tab, run_multi_tab); diff --git a/packages/cua-driver/rust/crates/cua-driver/tests/wayland_overlay_idle_test.rs b/packages/cua-driver/rust/crates/cua-driver/tests/wayland_overlay_idle_test.rs deleted file mode 100644 index d8583cd7c91..00000000000 --- a/packages/cua-driver/rust/crates/cua-driver/tests/wayland_overlay_idle_test.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! Hosted native-Wayland lifecycle and CPU certification for the layer-shell -//! cursor overlay. Run inside the repository's isolated Sway session. - -#![cfg(target_os = "linux")] - -use std::fs; -use std::thread; -use std::time::{Duration, Instant}; - -use cua_driver_testkit::RawDriver; - -const OVERLAY_THREAD: &str = "cua-overlay-wl"; - -fn call(driver: &mut RawDriver, id: u64, name: &str, arguments: serde_json::Value) { - driver.send(&serde_json::json!({ - "jsonrpc": "2.0", - "id": id, - "method": "tools/call", - "params": { "name": name, "arguments": arguments } - })); - let response = driver.recv(); - assert_eq!(response["id"], id, "unexpected response: {response:?}"); - assert!( - !response["result"]["isError"].as_bool().unwrap_or(false), - "{name} failed: {response:?}" - ); -} - -fn initialize(driver: &mut RawDriver) { - driver.send(&serde_json::json!({ - "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {} - })); - let response = driver.recv(); - assert_eq!(response["id"], 1, "initialize failed: {response:?}"); -} - -fn overlay_tid(pid: u32) -> Option { - let tasks = fs::read_dir(format!("/proc/{pid}/task")).ok()?; - tasks.filter_map(Result::ok).find_map(|task| { - let comm = fs::read_to_string(task.path().join("comm")).ok()?; - (comm.trim() == OVERLAY_THREAD).then(|| { - task.file_name() - .to_string_lossy() - .parse::() - .expect("numeric Linux task id") - }) - }) -} - -fn wait_for_overlay_tid(pid: u32) -> u32 { - let deadline = Instant::now() + Duration::from_secs(5); - while Instant::now() < deadline { - if let Some(tid) = overlay_tid(pid) { - return tid; - } - thread::sleep(Duration::from_millis(25)); - } - panic!("{OVERLAY_THREAD} did not start in daemon {pid}"); -} - -fn cpu_ticks(pid: u32, tid: u32) -> u64 { - let stat = - fs::read_to_string(format!("/proc/{pid}/task/{tid}/stat")).expect("read overlay task stat"); - let suffix = stat.rsplit_once(") ").expect("parse task comm").1; - let fields: Vec<&str> = suffix.split_whitespace().collect(); - let utime = fields[11].parse::().expect("parse task utime"); - let stime = fields[12].parse::().expect("parse task stime"); - utime + stime -} - -fn assert_idle_tick_bound(pid: u32, tid: u32, window: Duration, bound: u64) { - let before = cpu_ticks(pid, tid); - thread::sleep(window); - let after = cpu_ticks(pid, tid); - let delta = after.saturating_sub(before); - eprintln!( - "wayland overlay idle evidence: pid={pid} tid={tid} window_ms={} tick_delta={delta} bound={bound}", - window.as_millis() - ); - assert!( - delta <= bound, - "idle overlay used {delta} ticks (bound {bound})" - ); -} - -#[test] -#[ignore] -fn no_overlay_flag_never_starts_wayland_overlay_thread() { - let Some(mut driver) = RawDriver::spawn_with_env(&[ - ("CUA_DRIVER_PERMISSION_MODE", "unrestricted"), - ("CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS", "1"), - ]) else { - panic!("source-built driver is required"); - }; - let pid = driver.daemon_pid().expect("daemon-backed driver pid"); - initialize(&mut driver); - call( - &mut driver, - 2, - "move_cursor", - serde_json::json!({"x": 300.0, "y": 240.0}), - ); - thread::sleep(Duration::from_millis(300)); - assert_eq!( - overlay_tid(pid), - None, - "--no-overlay daemon unexpectedly started {OVERLAY_THREAD}" - ); -} - -#[test] -#[ignore] -fn wayland_overlay_quiesces_and_recovers_after_capture_and_cursor_activity() { - let Some(mut driver) = RawDriver::spawn_with_overlay_and_env(&[ - ("CUA_DRIVER_PERMISSION_MODE", "unrestricted"), - ("CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS", "1"), - ]) else { - panic!("source-built driver is required"); - }; - let pid = driver.daemon_pid().expect("daemon-backed driver pid"); - initialize(&mut driver); - - // Exercise the reported trigger class: native compositor capture followed - // by daemon-owned cursor motion. Call the Linux capture backend directly: - // `screenshot` is intentionally denied at the MCP authorization boundary - // until that tool has a reviewed risk classification. - let capture = platform_linux::wayland::screenshot_display_dispatch() - .expect("capture native Wayland display before overlay activity"); - assert!(!capture.is_empty(), "native Wayland capture was empty"); - call( - &mut driver, - 3, - "set_agent_cursor_motion", - serde_json::json!({"glide_duration_ms": 700, "idle_hide_ms": 0}), - ); - call( - &mut driver, - 4, - "move_cursor", - serde_json::json!({"x": 500.0, "y": 360.0}), - ); - let tid = wait_for_overlay_tid(pid); - thread::sleep(Duration::from_secs(2)); - assert_idle_tick_bound(pid, tid, Duration::from_secs(2), 1); - - call( - &mut driver, - 5, - "set_agent_cursor_enabled", - serde_json::json!({"enabled": false}), - ); - thread::sleep(Duration::from_millis(250)); - assert_idle_tick_bound(pid, tid, Duration::from_secs(1), 1); - - call( - &mut driver, - 6, - "set_agent_cursor_enabled", - serde_json::json!({"enabled": true}), - ); - let recovery_before = cpu_ticks(pid, tid); - call( - &mut driver, - 7, - "move_cursor", - serde_json::json!({"x": 900.0, "y": 600.0}), - ); - thread::sleep(Duration::from_millis(500)); - let recovery_delta = cpu_ticks(pid, tid).saturating_sub(recovery_before); - eprintln!("wayland overlay recovery evidence: tick_delta={recovery_delta}"); - assert!( - recovery_delta > 0, - "re-enabled overlay did not resume rendering" - ); - - thread::sleep(Duration::from_secs(2)); - assert_idle_tick_bound(pid, tid, Duration::from_secs(2), 1); -} diff --git a/packages/cua-driver/rust/crates/platform-linux/Cargo.toml b/packages/cua-driver/rust/crates/platform-linux/Cargo.toml index 632403fd120..a0409387b58 100644 --- a/packages/cua-driver/rust/crates/platform-linux/Cargo.toml +++ b/packages/cua-driver/rust/crates/platform-linux/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true tokio = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } anyhow = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } @@ -20,9 +21,9 @@ pip-preview = { path = "../pip-preview" } tiny-skia = { version = "0.11", default-features = false, features = ["std"] } [target.'cfg(target_os = "linux")'.dependencies] -clipboard-rs = { version = "0.3.5", features = ["wayland"] } +clipboard-rs = "0.3.5" # X11 background input + window enumeration + MIT-SHM capture -x11rb = { version = "0.13", features = ["xinput", "randr", "xfixes", "composite", "shape", "xtest", "shm"] } +x11rb = { version = "0.13", features = ["xinput", "randr", "xfixes", "composite", "shape", "xtest", "shm", "res"] } x11 = { version = "2.21", features = ["xlib", "xinput", "xtest"] } base64 = { workspace = true } image = { workspace = true } @@ -34,120 +35,15 @@ libc = "0.2" # subprocess bridge so accessibility input/readback needs no Python or GI # typelibs at runtime. `tokio` matches the driver's async runtime; `zbus` # re-exports the bus types we need (fdo::DBusProxy for pid resolution). -atspi = { version = "0.30", features = ["tokio", "zbus"] } +atspi = { version = "0.30", default-features = false, features = ["proxies", "wrappers", "tokio", "zbus"] } +atspi-connection = { version = "0.14", default-features = false, features = ["wrappers"] } evdev = "0.12" # Math-expression evaluator for parallel_mouse_drag `fn` paths (y = f(x)). meval = "0.2" -# Native Wayland backend (window enumeration via wlr-foreign-toplevel-management; -# capture/input protocols added as those slices land). Pure-Rust client; used -# when running under a Wayland compositor with no X11 (no DISPLAY). -wayland-client = "0.31" -wayland-backend = "0.3" -wayland-scanner = "0.31" -wayland-protocols-wlr = { version = "0.3", features = ["client"] } -# Wayland staging protocols: ext-foreign-toplevel-list-v1, -# ext-image-capture-source-v1, ext-image-copy-capture-v1 (per-window -# screenshot on modern wlroots/GNOME/KDE) and wp-fractional-scale-v1 + -# wp-viewporter (HiDPI for the layer-shell overlay). The transitive pull -# from wayland-protocols-wlr does NOT enable `staging`; declaring it here -# unifies the feature globally. -wayland-protocols = { version = "0.32", features = ["client", "staging"] } -# xdg-desktop-portal client: display-screenshot tier -# (org.freedesktop.portal.Screenshot), per-window ScreenCast -# (org.freedesktop.portal.ScreenCast), and the libei input session -# (org.freedesktop.portal.RemoteDesktop.ConnectToEIS). `tokio` shares the -# driver's async runtime. -# -# ashpd's RemoteDesktop feature is pure Rust. Only our ScreenCast frame reader -# pulls pipewire-rs/libspa, whose headers are newer than Debian bullseye. -ashpd = { version = "0.13", default-features = false, features = ["tokio", "screenshot"] } -# zbus is pulled transitively via ashpd + atspi but declared directly so -# the portal-probe path in wayland::portal_screenshot can talk to the -# session bus without going through ashpd's higher-level builders. zbus = { version = "5", default-features = false, features = ["tokio"] } -# Parse `file://` URIs returned by org.freedesktop.portal.Screenshot. -# ashpd re-exports `url` transitively; declared explicitly so ashpd -# version bumps cannot silently break the resolver. -url = "2" -# PipeWire client for the ScreenCast per-window capture path — dequeues -# frames from the node id returned by `open_pipe_wire_remote`. Pinned to -# 0.8 (last well-documented release; 0.10 had docs.rs build failures in -# mid-2026). Needs libpipewire-0.3 headers at build time — gated behind -# the `portal-capture` feature. -pipewire = { version = "0.8", optional = true } -# Companion to pipewire 0.8 — needed to construct the SPA_PARAM_EnumFormat -# pod (SPA_VIDEO_FORMAT_BGRx at announced width/height) we hand to -# `Stream::connect`. Same `portal-capture` gating as `pipewire`. -libspa = { version = "0.8", optional = true } -# Pure-Rust libei/libeis bindings for the GNOME/KDE input path (handshake -# → seat → device → pointer_absolute/button/scroll/keyboard/text). API is -# documented as "incomplete and subject to change" — budget for churn on -# the next bump. Re-exports `enumflags2` we use for `BitFlags`. -reis = { version = "0.7", features = ["tokio"], optional = true } -# Calloop event loop drives the EIS worker thread — reis's example uses -# calloop::EventLoop + Generic source to multiplex EIS protocol reads -# with command-channel polling. reis 0.7 also ships a calloop adapter -# but we use the plain Generic source for clarity. -calloop = { version = "0.14", optional = true } -# Used directly in the ashpd RemoteDesktop call to compose -# `BitFlags::from(DeviceType::Keyboard) | DeviceType::Pointer`. reis also -# re-exports this same major version; pinning explicitly prevents -# duplicate-crate hazards. -enumflags2 = { version = "0.7", optional = true } -# Keymap fallback path for libei keyboard input when the EIS server does -# NOT advertise `ei_text` (libei < 1.6, i.e. older GNOME / KDE Plasma 6.0). -# Mirrors the reis `type-text.rs` example's keycode reverse-mapping. -xkbcommon = { version = "0.9", optional = true } -# Bounded MPSC command/reply channel for the persistent virtual-pointer -# owner thread. EventQueue is !Send, so the persistent device must -# live on its own thread; crossbeam gives us a Send sender and synchronous -# reply channels that preserve the "tool call blocks until compositor -# roundtripped" contract the X11 path provides. -crossbeam-channel = "0.5" -# Resolve `$XDG_STATE_HOME` (with platform fallback) for persisting the -# ScreenCast and RemoteDesktop `restore_token`s so users see the consent -# dialog only once per install, not once per process. -dirs = "5" [features] default = [] -linux-example = [] -# GNOME/KDE input is pure Rust plus libxkbcommon, which is available in the -# Debian 11 release container. Ashpd's RemoteDesktop API exposes ScreenCast -# stream metadata, so its lightweight screencast module is required here too. -# The PipeWire frame reader remains isolated in portal-capture below. -portal-input = [ - "ashpd/remote_desktop", - "ashpd/screencast", - "dep:reis", - "dep:calloop", - "dep:enumflags2", - "dep:xkbcommon", -] -# PipeWire-backed per-window ScreenCast capture keeps its newer system-header -# requirement and remains enabled by Nix/modern desktop builds. -portal-capture = [ - "ashpd/screencast", - "dep:pipewire", - "dep:libspa", -] -# Backward-compatible full desktop feature used by the Nix package. -portal-libei = ["portal-input", "portal-capture"] [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } -# Used by the `screenshot_cascade` example to surface the -# `tracing::debug!` lines emitted by each tier of the dispatch as it -# falls through. -tracing-subscriber = { workspace = true } - -# Cargo example that exercises the libei input path directly. Lives under -# the `portal-input` feature gate so a stock `cargo build --examples` -# doesn't try to compile against the (now-optional) reis/xkbcommon deps. -[[example]] -name = "libei_input" -required-features = ["portal-input"] - -[[example]] -name = "screenshot_cascade" -required-features = ["linux-example"] diff --git a/packages/cua-driver/rust/crates/platform-linux/examples/libei_input.rs b/packages/cua-driver/rust/crates/platform-linux/examples/libei_input.rs deleted file mode 100644 index 16ec7ef1f09..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/examples/libei_input.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! Standalone smoke for the libei input path on platform-linux. -//! -//! Drives `wayland::libei::move_absolute` + `click` and prints what happens. -//! Stress-tests the portal RemoteDesktop handshake plus the reis-based EIS -//! handshake on a real GNOME / KDE-Wayland host. -//! -//! Usage: -//! CUA_DRIVER_RS_ENABLE_WAYLAND=1 \ -//! WAYLAND_DISPLAY=wayland-0 \ -//! cargo run --example libei_input --release - -fn main() { - let _ = tracing_subscriber::FmtSubscriber::builder() - .with_max_level(tracing::Level::DEBUG) - .with_writer(std::io::stderr) - .try_init(); - - eprintln!("== platform-linux libei input smoke =="); - eprintln!( - " WAYLAND_DISPLAY={:?} XDG_RUNTIME_DIR={:?} DBUS_SESSION_BUS_ADDRESS={:?}", - std::env::var("WAYLAND_DISPLAY").ok(), - std::env::var("XDG_RUNTIME_DIR").ok(), - std::env::var("DBUS_SESSION_BUS_ADDRESS").ok(), - ); - - // Try a no-op move first — opens the portal handshake + EIS context. - eprintln!("\n-- move_absolute(500, 400) --"); - let t = std::time::Instant::now(); - match platform_linux::wayland::libei::move_absolute(500.0, 400.0) { - Ok(()) => eprintln!(" OK in {:?}", t.elapsed()), - Err(e) => eprintln!(" ERR after {:?}: {}", t.elapsed(), e), - } - - eprintln!("\n-- click(500, 400, Left) --"); - let t = std::time::Instant::now(); - match platform_linux::wayland::libei::click( - 500.0, - 400.0, - platform_linux::wayland::libei::Button::Left, - ) { - Ok(()) => eprintln!(" OK in {:?}", t.elapsed()), - Err(e) => eprintln!(" ERR after {:?}: {}", t.elapsed(), e), - } - - eprintln!("\n-- type_text('hello world') --"); - let t = std::time::Instant::now(); - match platform_linux::wayland::libei::type_text("hello world") { - Ok(()) => eprintln!(" OK in {:?}", t.elapsed()), - Err(e) => eprintln!(" ERR after {:?}: {}", t.elapsed(), e), - } - - eprintln!("\n-- shutdown --"); - platform_linux::wayland::libei::shutdown(); - // Give the worker a moment to drain. - std::thread::sleep(std::time::Duration::from_millis(500)); - eprintln!("done"); -} diff --git a/packages/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs b/packages/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs deleted file mode 100644 index bf0de170e8c..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! Standalone smoke for the platform-linux screenshot cascade. -//! -//! Calls `capture::screenshot_display_bytes()` and prints the PNG size + -//! magic bytes + which tier produced the result (inferred from -//! tracing-subscriber logs). -//! -//! Usage: -//! CUA_DRIVER_RS_ENABLE_WAYLAND=1 \ -//! WAYLAND_DISPLAY=wayland-0 \ -//! cargo run --example screenshot_cascade --release - -use std::io::Write as _; - -fn main() { - // Bare tracing subscriber so the debug! lines from the cascade tiers - // show up. - let _ = tracing_subscriber::FmtSubscriber::builder() - .with_max_level(tracing::Level::DEBUG) - .with_writer(std::io::stderr) - .try_init(); - - eprintln!("== platform-linux screenshot cascade smoke =="); - eprintln!( - " WAYLAND_DISPLAY={:?} DISPLAY={:?} CUA_DRIVER_RS_ENABLE_WAYLAND={:?}", - std::env::var("WAYLAND_DISPLAY").ok(), - std::env::var("DISPLAY").ok(), - std::env::var("CUA_DRIVER_RS_ENABLE_WAYLAND").ok(), - ); - - let started = std::time::Instant::now(); - match platform_linux::capture::screenshot_display_bytes() { - Ok(bytes) => { - let dt = started.elapsed(); - eprintln!( - "OK {} bytes in {dt:?} magic={:?}", - bytes.len(), - &bytes[..bytes.len().min(8)] - ); - // Write the PNG so we can also visually check it. - let path = std::env::var("CUA_SMOKE_OUT") - .unwrap_or_else(|_| "/tmp/cua-screenshot.png".to_string()); - if let Ok(mut f) = std::fs::File::create(&path) { - let _ = f.write_all(&bytes); - eprintln!(" wrote {}", path); - } - } - Err(e) => { - eprintln!("ERR {}", e); - std::process::exit(1); - } - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/protocol/virtual-keyboard-unstable-v1.xml b/packages/cua-driver/rust/crates/platform-linux/protocol/virtual-keyboard-unstable-v1.xml deleted file mode 100644 index 6b0e1206cf4..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/protocol/virtual-keyboard-unstable-v1.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - Copyright © 2008-2011 Kristian Høgsberg - Copyright © 2010-2013 Intel Corporation - Copyright © 2012-2013 Collabora, Ltd. - Copyright © 2018 Purism SPC - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. - - - - - The virtual keyboard provides an application with requests which emulate - the behaviour of a physical keyboard. - - - - - - - - - - - - - - - - - - - - - - - - - - A virtual keyboard manager allows an application to provide keyboard - input events as if they came from a physical keyboard. - - - - - - - - - - diff --git a/packages/cua-driver/rust/crates/platform-linux/src/at_point_policy.rs b/packages/cua-driver/rust/crates/platform-linux/src/at_point_policy.rs new file mode 100644 index 00000000000..763f96e2e92 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-linux/src/at_point_policy.rs @@ -0,0 +1,104 @@ +// Adapted from trycua/cua #4188, Copyright (c) 2025 Cua AI. MIT license. +//! Pure routing facts for Linux background pixel clicks. +//! +//! Kept free of X11, AT-SPI, and `/proc` I/O so the decisions are unit-tested +//! on every host, not only on Linux. + +/// AT-SPI roles of an application's top-level shell: the application node, +/// its frames and windows. A pixel hit test that resolves only to one of +/// these did not find the control under the point. Chromium on Linux is the +/// common case: until its accessibility tree is populated, the page content +/// is a bare `frame` whose `doDefault` does nothing. Firing that action and +/// reporting a click is a false success, so the shell is never an at-point +/// click target; the caller falls through to its pointer route or refuses. +pub(crate) fn is_top_level_shell_role(role: &str) -> bool { + matches!( + role.trim().to_ascii_lowercase().as_str(), + "application" | "frame" | "window" | "desktop frame" + ) +} + +/// Whether one `/proc//cmdline` belongs to a Chromium helper process +/// (renderer, zygote, or GPU process). +/// +/// The kernel separates arguments with NUL, but Chromium rewrites its own +/// process title with `setproctitle`, which joins the whole command line +/// with spaces into a single NUL-terminated string. Matching only +/// NUL-separated arguments misses every Google Chrome and Chromium helper, +/// so both separators are accepted. +pub(crate) fn cmdline_is_chromium_helper(raw: &[u8]) -> bool { + String::from_utf8_lossy(raw) + .split(|ch: char| ch == '\0' || ch.is_whitespace()) + .any(|arg| { + matches!( + arg, + "--type=renderer" | "--type=zygote" | "--type=gpu-process" + ) + }) +} + +#[cfg(test)] +mod tests { + use super::{cmdline_is_chromium_helper, is_top_level_shell_role}; + + #[test] + fn application_frames_and_windows_are_never_click_targets() { + for role in ["frame", "window", "application", "desktop frame", " Frame "] { + assert!(is_top_level_shell_role(role), "{role:?}"); + } + } + + #[test] + fn controls_and_documents_remain_click_targets() { + for role in [ + "push button", + "button", + "link", + "check box", + "menu item", + "document web", + "internal frame", + "dialog", + "panel", + ] { + assert!(!is_top_level_shell_role(role), "{role:?}"); + } + } + + #[test] + fn nul_separated_chromium_helper_is_detected() { + assert!(cmdline_is_chromium_helper( + b"/usr/lib/chromium/chromium\0--type=renderer\0--lang=en-US\0" + )); + assert!(cmdline_is_chromium_helper( + b"/opt/app/app\0--type=zygote\0--no-zygote-sandbox\0" + )); + } + + #[test] + fn setproctitle_rewritten_chrome_helper_is_detected() { + // Google Chrome 151 renderer as `/proc//cmdline` shows it after + // Chromium's setproctitle: one space-joined string. + assert!(cmdline_is_chromium_helper( + b"/opt/google/chrome/chrome --type=renderer --crashpad-handler-pid=7960 \ + --enable-crash-reporter --lang=en-US --num-raster-threads=4\0" + )); + assert!(cmdline_is_chromium_helper( + b"/opt/google/chrome/chrome --type=gpu-process --ozone-platform=x11\0\0\0" + )); + } + + #[test] + fn unrelated_processes_are_not_chromium_helpers() { + assert!(!cmdline_is_chromium_helper( + b"/usr/bin/gedit\0--new-window\0" + )); + assert!(!cmdline_is_chromium_helper( + b"/usr/bin/python3 tool.py --type=renderer-like\0" + )); + assert!(!cmdline_is_chromium_helper( + b"/usr/lib/webkit2gtk-4.1/WebKitWebProcess 7 12\0" + )); + assert!(!cmdline_is_chromium_helper(b"")); + } +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs b/packages/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs index 9e960856473..e3e66283f51 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/atspi/cache.rs @@ -8,7 +8,9 @@ use super::{AtspiIdentity, AtspiNode}; use cua_driver_core::element_cache::ElementCacheCore; +use cua_driver_core::element_token::{ResolvedElement, SnapshotBinding}; use std::collections::HashMap; +use std::sync::{Arc, Weak}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct CacheKey { @@ -22,13 +24,14 @@ pub struct CachedSnapshot { #[derive(Clone)] pub struct CachedElement { + pub role: String, pub element_key: u64, pub identity: Option, pub actions: Vec, } pub struct ElementCache { - core: ElementCacheCore, + core: ElementCacheCore>, } impl ElementCache { @@ -38,13 +41,24 @@ impl ElementCache { } } - pub fn update(&self, pid: u32, xid: u64, nodes: &[AtspiNode]) { + pub fn update(&self, pid: u32, xid: u64, nodes: &[AtspiNode]) -> u32 { + self.update_for_session(pid, xid, nodes, None) + } + + pub fn update_for_session( + &self, + pid: u32, + xid: u64, + nodes: &[AtspiNode], + session: Option<&str>, + ) -> u32 { let elements = nodes .iter() .filter_map(|node| { Some(( node.element_index?, CachedElement { + role: node.role.clone(), element_key: node.element_key, identity: node.identity.clone(), actions: node.actions.clone(), @@ -52,51 +66,107 @@ impl ElementCache { )) }) .collect(); + let snapshot = Arc::new(CachedSnapshot { elements }); + let id = cua_driver_core::element_token::global().register_snapshot_payload_for_session( + pid as i32, + xid as u32, + snapshot.elements.keys().copied(), + snapshot.clone(), + session, + ); self.core - .insert(CacheKey { pid, xid }, CachedSnapshot { elements }); + .insert(CacheKey { pid, xid }, Arc::downgrade(&snapshot)); + id } - pub fn get_element_key(&self, pid: u32, xid: u64, idx: usize) -> Option { - self.core - .with_snapshot(&CacheKey { pid, xid }, |s| { - s.elements.get(&idx).map(|element| element.element_key) - }) - .flatten() - } - - pub fn get_element_identity(&self, pid: u32, xid: u64, idx: usize) -> Option { + pub fn has_snapshot(&self, pid: u32, xid: u64) -> bool { self.core .with_snapshot(&CacheKey { pid, xid }, |snapshot| { - snapshot - .elements - .get(&idx) - .and_then(|element| element.identity.clone()) + snapshot.strong_count() > 0 }) - .flatten() - } - - pub fn get_element_actions(&self, pid: u32, xid: u64, idx: usize) -> Option> { - self.core - .with_snapshot(&CacheKey { pid, xid }, |snapshot| { - snapshot - .elements - .get(&idx) - .map(|element| element.actions.clone()) - }) - .flatten() + .unwrap_or(false) } pub fn element_count(&self, pid: u32, xid: u64) -> usize { self.core - .with_snapshot(&CacheKey { pid, xid }, |s| s.elements.len()) + .with_snapshot(&CacheKey { pid, xid }, |s| { + s.upgrade().map_or(0, |s| s.elements.len()) + }) .unwrap_or(0) } pub fn clear_target(&self, pid: u32, xid: u64) { + cua_driver_core::element_token::global().clear_current_target(pid as i32, xid as u32); self.core.remove(&CacheKey { pid, xid }); } } +#[derive(Clone)] +pub struct BoundElementCache { + binding: Option, +} + +impl BoundElementCache { + pub fn from_resolved(resolved: &ResolvedElement) -> Self { + Self { + binding: resolved.binding(), + } + } + + pub fn from_binding(binding: SnapshotBinding) -> Self { + Self { + binding: Some(binding), + } + } + + fn with_snapshot( + &self, + key: &CacheKey, + read: impl FnOnce(&CachedSnapshot) -> R, + ) -> Option { + self.binding + .as_ref()? + .with_payload(key.pid as i32, key.xid as u32, read) + } + + pub fn get_element_key(&self, pid: u32, xid: u64, idx: usize) -> Option { + self.with_snapshot(&CacheKey { pid, xid }, |s| { + s.elements.get(&idx).map(|element| element.element_key) + }) + .flatten() + } + + pub fn get_element_identity(&self, pid: u32, xid: u64, idx: usize) -> Option { + self.with_snapshot(&CacheKey { pid, xid }, |snapshot| { + snapshot + .elements + .get(&idx) + .and_then(|element| element.identity.clone()) + }) + .flatten() + } + + pub fn get_element_role(&self, pid: u32, xid: u64, idx: usize) -> Option { + self.with_snapshot(&CacheKey { pid, xid }, |snapshot| { + snapshot + .elements + .get(&idx) + .map(|element| element.role.clone()) + }) + .flatten() + } + + pub fn get_element_actions(&self, pid: u32, xid: u64, idx: usize) -> Option> { + self.with_snapshot(&CacheKey { pid, xid }, |snapshot| { + snapshot + .elements + .get(&idx) + .map(|element| element.actions.clone()) + }) + .flatten() + } +} + impl Default for ElementCache { fn default() -> Self { Self::new() @@ -130,36 +200,106 @@ mod tests { } } + fn bind(pid: i32, id: u32, index: usize) -> BoundElementCache { + let (_, _, binding) = cua_driver_core::element_token::global() + .resolve_bound(pid, &cua_driver_core::element_token::token_for(id, index)) + .unwrap(); + BoundElementCache::from_binding(binding.unwrap()) + } + #[test] fn lookups_preserve_sparse_application_indices_and_window_isolation() { let cache = ElementCache::new(); - cache.update(42, 7, &[node(130), node(132), node(637)]); - cache.update(42, 8, &[node(1), node(2), node(3)]); - assert_eq!(cache.element_count(42, 7), 3); + let first = cache.update(721_511, 7, &[node(130), node(132), node(637)]); + let first = bind(721_511, first, 130); + let sibling = cache.update(721_511, 8, &[node(1), node(2), node(3)]); + let sibling = bind(721_511, sibling, 1); + assert_eq!(cache.element_count(721_511, 7), 3); for index in [130, 132, 637] { assert_eq!( - cache.get_element_key(42, 7, index), + first.get_element_key(721_511, 7, index), Some(1000 + index as u64) ); assert_eq!( - cache.get_element_identity(42, 7, index), + first.get_element_identity(721_511, 7, index), node(index).identity ); assert_eq!( - cache.get_element_actions(42, 7, index), + first.get_element_actions(721_511, 7, index), Some(node(index).actions) ); } for gap in [0, 1, 131, 638] { - assert_eq!(cache.get_element_key(42, 7, gap), None); - assert_eq!(cache.get_element_identity(42, 7, gap), None); - assert_eq!(cache.get_element_actions(42, 7, gap), None); + assert_eq!(first.get_element_key(721_511, 7, gap), None); + assert_eq!(first.get_element_identity(721_511, 7, gap), None); } - cache.update(42, 7, &[node(130)]); - assert_eq!(cache.get_element_key(42, 7, 637), None); - assert_eq!(cache.get_element_key(42, 8, 3), Some(1003)); - cache.clear_target(42, 7); - assert_eq!(cache.get_element_key(42, 7, 130), None); - assert_eq!(cache.get_element_key(42, 8, 3), Some(1003)); + assert_eq!(first.get_element_key(721_511, 8, 130), None); + assert_eq!(sibling.get_element_key(721_511, 8, 3), Some(1003)); + cua_driver_core::element_token::global().clear_pid(721_511); + } + + #[test] + fn target_clear_revokes_token_without_discarding_a_resolved_identity() { + let cache = ElementCache::new(); + let pid = 721_513; + let first = cache.update(pid, 7, &[node(130)]); + let bound = bind(pid as i32, first, 130); + let sibling = cache.update(pid, 8, &[node(1)]); + cache.clear_target(pid, 7); + let registry = cua_driver_core::element_token::global(); + assert!(registry + .resolve( + pid as i32, + &cua_driver_core::element_token::token_for(first, 130) + ) + .is_err()); + assert!(registry + .resolve( + pid as i32, + &cua_driver_core::element_token::token_for(sibling, 1) + ) + .is_ok()); + assert_eq!(bound.get_element_key(pid, 7, 130), Some(1130)); + assert_eq!(cache.element_count(pid, 7), 0); + registry.clear_pid(pid as i32); + } + + #[test] + fn replacement_cannot_mix_native_identity_role_and_actions() { + let cache = ElementCache::new(); + let first = cache.update(721_512, 7, &[node(130)]); + let bound = bind(721_512, first, 130); + let mut replacement = node(130); + replacement.element_key = 9999; + replacement.identity.as_mut().unwrap().unique_owner = ":1.99".into(); + replacement.actions = vec!["replacement-action".into()]; + replacement.role = "entry".into(); + let second = cache.update(721_512, 7, &[replacement]); + assert!(cua_driver_core::element_token::global() + .resolve( + 721_512, + &cua_driver_core::element_token::token_for(first, 130) + ) + .is_err()); + assert_eq!(bound.get_element_key(721_512, 7, 130), Some(1130)); + assert_eq!( + bound.get_element_identity(721_512, 7, 130), + node(130).identity + ); + assert_eq!( + bound.get_element_role(721_512, 7, 130), + Some("button".into()) + ); + assert_eq!( + bound.get_element_actions(721_512, 7, 130), + Some(node(130).actions) + ); + assert_eq!( + bind(721_512, second, 130).get_element_key(721_512, 7, 130), + Some(9999) + ); + cua_driver_core::element_token::global().clear_pid(721_512); + drop(bound); + assert!(!cache.has_snapshot(721_512, 7)); } } diff --git a/packages/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/packages/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs index 4a4dd1a96c2..76025ae5225 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs @@ -106,37 +106,6 @@ pub fn walk_tree(pid: u32, xid: u64, query: Option<&str>) -> AtspiTreeResult { walk_tree_bounded(pid, xid, query, None, None) } -/// Best-effort accessibility snapshot for synchronous trajectory evidence. -/// -/// Recording brackets an action with before/after captures, so it must use a -/// smaller budget than the transport's tool-call deadline. Unlike the -/// interactive tree walker, evidence capture makes one attempt and accepts an -/// unavailable tree when the target renderer is blocked. -pub(crate) fn walk_tree_for_recording( - pid: u32, - xid: u64, - timeout: std::time::Duration, -) -> AtspiTreeResult { - if let Ok(Some(walked)) = native::walk_tree_bounded_with_timeout(pid, xid, None, None, timeout) - { - if !walked.markdown.is_empty() { - return AtspiTreeResult { - tree_markdown: walked.markdown, - nodes: walked.nodes, - bounds: walked.bounds, - trusted: true, - degraded_reason: None, - window_scoped: walked.window_scoped, - backend: AtspiBackend::Atspi, - complete: walked.complete && walked.window_scoped, - truncated: walked.truncated, - incomplete_notes: walked.incomplete_notes, - }; - } - } - walk_via_x11_properties(xid, None) -} - /// Walk the AT-SPI tree with caller-supplied caps. `None` for either cap /// means "use the walker's built-in default" (5 000 nodes; unlimited depth). /// Issue #22865: caps protect against Electron / large web apps that @@ -147,9 +116,20 @@ pub fn walk_tree_bounded( query: Option<&str>, max_elements: Option, max_depth: Option, +) -> AtspiTreeResult { + walk_tree_bounded_with_timeout(pid, xid, query, max_elements, max_depth, native::OP_TIMEOUT) +} + +pub fn walk_tree_bounded_with_timeout( + pid: u32, + xid: u64, + query: Option<&str>, + max_elements: Option, + max_depth: Option, + timeout: std::time::Duration, ) -> AtspiTreeResult { let mut native_failure = None; - match walk_with_cold_start_retries(native::OP_TIMEOUT, |remaining| { + match walk_with_cold_start_retries(timeout, |remaining| { native::walk_tree_bounded_with_timeout(pid, xid, max_elements, max_depth, remaining) }) { Ok(Some(walked)) if !walked.markdown.is_empty() => { @@ -259,15 +239,6 @@ pub fn scroll_element_exact( native::scroll_element(pid, idx, Some(identity), direction, amount) } -/// Enumerate top-level windows from the AT-SPI registry. The window-listing -/// fallback for Wayland compositors without `zwlr_foreign_toplevel_management` -/// (GNOME Mutter / KDE KWin), where native apps have no X11 XID. Returns one -/// entry per application top-level frame with a synthetic, stable `xid` that -/// round-trips into the by-pid AT-SPI element flow. See [`native::list_windows`]. -pub fn list_windows(filter_pid: Option) -> Vec { - native::list_windows(filter_pid) -} - /// Resolve a window-local pixel to the actionable AT-SPI element at that point /// and perform its primary action — the no-focus-steal way to land a *pixel* /// click on toolkits (GTK) that drop synthetic X11 pointer events. Returns @@ -279,7 +250,7 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result { /// True when an ancestor is a web document (e.g. role "document web"), /// i.e. this node is page content rather than browser chrome. in_web_doc: bool, - /// True when this node is exported by a separate WebKit WebProcess bus. - /// Chromium keeps its document on the application's ordinary AT-SPI bus, - /// where descendant Window extents already include the document origin. - on_web_process_bus: bool, /// Position of the application top-level (frame/window) this node descends /// from, in `app.get_children()` order. AT-SPI exposes one application per /// process, so a multi-window app publishes every window's controls in one @@ -358,25 +354,6 @@ async fn accessible_for<'a>( conn: &'a AccessibilityConnection, oref: &RawObjectRef, ) -> Result> { - // Keep the atspi crate's peer-to-peer path when this connection actually - // knows the peer. Late WebKit WebProcess children are not in the initial - // peer snapshot; object_as_accessible's bus fallback omits their destination - // and targets the Accessible interface name instead. Build an explicit bus - // proxy below for those late peers and for well-known references. - if oref.name.starts_with(':') { - let name = atspi::zbus::names::UniqueName::try_from(oref.name.clone()) - .map_err(|e| anyhow!("bad a11y unique name: {e}"))?; - let bus_name = atspi::zbus::names::BusName::Unique(name.as_ref()); - if conn.get_peer(&bus_name).is_some() { - let path = atspi::zbus::zvariant::ObjectPath::try_from(oref.path.clone()) - .map_err(|e| anyhow!("bad a11y path: {e}"))?; - let object = atspi::ObjectRef::new_owned(name, path); - return conn - .object_as_accessible(&object) - .await - .map_err(|e| anyhow!("AccessibleProxy build failed: {e}")); - } - } AccessibleProxy::builder(conn.connection()) .cache_properties(atspi::zbus::proxy::CacheProperties::No) .destination(oref.name.clone()) @@ -421,6 +398,31 @@ async fn canonical_unique_owner( .map(|owner| owner.to_string()) } +async fn raw_children( + conn: &atspi::zbus::Connection, + oref: &RawObjectRef, +) -> Result> { + let proxy = atspi::zbus::Proxy::new( + conn, + oref.name.as_str(), + oref.path.as_str(), + "org.a11y.atspi.Accessible", + ) + .await + .map_err(|e| anyhow!("Accessible proxy unavailable: {e}"))?; + let refs: Vec<(String, atspi::zbus::zvariant::OwnedObjectPath)> = proxy + .call("GetChildren", &()) + .await + .map_err(|e| anyhow!("Accessible.GetChildren failed: {e}"))?; + Ok(refs + .into_iter() + .map(|(name, path)| RawObjectRef { + name, + path: path.to_string(), + }) + .collect()) +} + /// Read one child without deserializing the bus-name field as a /// `UniqueName`. WebKitGTK's embedded WebProcess exposes a well-known name /// containing a UUID; D-Bus can address it, but the stricter AT-SPI wrapper @@ -706,9 +708,8 @@ async fn resolve_window_frame( } let windows = crate::x11::list_windows(Some(pid)); let window = windows.iter().find(|candidate| candidate.xid == xid)?; - let gtk4_x11 = !crate::wayland::is_wayland() - && std::fs::read_to_string(format!("/proc/{pid}/maps")) - .is_ok_and(|maps| maps.contains("libgtk-4.so")); + let gtk4_x11 = std::fs::read_to_string(format!("/proc/{pid}/maps")) + .is_ok_and(|maps| maps.contains("libgtk-4.so")); let mut candidates: Vec<(usize, (i32, i32, i32, i32))> = Vec::new(); let mut titles = Vec::new(); let mut titles_complete = true; @@ -865,6 +866,14 @@ async fn collect_visited_until<'a>( ))) } +fn first_application_walk(pid: u32) -> bool { + static PIDS: OnceLock>> = OnceLock::new(); + PIDS.get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap() + .insert(pid) +} + async fn collect_visited_into<'a>( conn: &'a AccessibilityConnection, pid: u32, @@ -883,6 +892,9 @@ async fn collect_visited_into<'a>( Some(a) => a, None => return Ok(false), }; + if first_application_walk(pid) { + let _ = call(app.get_attributes()).await; + } let zconn = conn.connection(); let dbus = atspi::zbus::fdo::DBusProxy::new(zconn) .await @@ -1147,7 +1159,7 @@ async fn collect_visited_into<'a>( .map(format_value); } } - if has_text { + if has_text && !role.to_ascii_lowercase().contains("password") { if let Some(Ok(tp)) = call(proxies.text()).await { let count = call(tp.character_count()).await.and_then(|r| r.ok()); if let Some(count) = count { @@ -1172,7 +1184,7 @@ async fn collect_visited_into<'a>( if let Some(text) = text_content { // Keep editable contents separate from their accessible label so // edits (including clearing the field) remain observable. - if has_editable { + if has_editable && !has_value { value = Some(text); } else if name.trim().is_empty() { name = text; @@ -1245,7 +1257,6 @@ async fn collect_visited_into<'a>( has_component, focused, in_web_doc, - on_web_process_bus: is_web_process_bus(&oref.name), frame_ordinal, identity, acc, @@ -1563,172 +1574,6 @@ pub(super) fn walk_tree_bounded_with_timeout( }) } -/// Enumerate top-level windows from the AT-SPI registry — the window-listing -/// fallback for Wayland compositors that DON'T implement -/// `zwlr_foreign_toplevel_management` (GNOME Mutter, KDE KWin). Native Wayland -/// apps have no X11 XID and Mutter/KWin expose no foreign-toplevel list, so -/// `wayland::list_windows` comes back empty there and the whole element flow -/// (get_window_state -> click by element_index) is unreachable — even though the -/// AT-SPI tree itself is keyed by PID and works fine (see `walk_tree_bounded`, -/// whose walk ignores the xid). This bridges that gap: it returns one -/// [`WindowInfo`] per application top-level frame, with a SYNTHETIC but stable -/// `xid`. Downstream `get_window_state` / `click` walk the tree by PID and never -/// dereference the xid against X11, so the synthetic value only needs to be -/// non-zero and to round-trip back from the caller. -pub fn list_windows(filter_pid: Option) -> Vec { - if tokio::runtime::Handle::try_current().is_ok() { - return std::thread::spawn(move || list_windows_blocking(filter_pid)) - .join() - .unwrap_or_default(); - } - list_windows_blocking(filter_pid) -} - -fn list_windows_blocking(filter_pid: Option) -> Vec { - use crate::x11::WindowInfo; - runtime().block_on(async { - let work = async { - let conn = shared_connection().await?; - let zconn = conn.connection(); - let root = match call(conn.root_accessible_on_registry()).await { - Some(Ok(r)) => r, - _ => return Ok(Vec::new()), - }; - let dbus = atspi::zbus::fdo::DBusProxy::new(zconn) - .await - .map_err(|e| anyhow!("DBus proxy unavailable: {e}"))?; - let apps = match call(root.get_children()).await { - Some(Ok(a)) => a, - _ => return Ok(Vec::new()), - }; - let mut out: Vec = Vec::new(); - for app_ref in apps { - // Skip apps that can't answer the pid query (modal-grabbed) and - // apps that don't match the filter. - let cpid = match call(pid_of(&dbus, &app_ref)).await { - Some(Some(p)) => p, - _ => continue, - }; - if let Some(want) = filter_pid { - if cpid != want { - continue; - } - } - let app_ref = match RawObjectRef::from_atspi(&app_ref) { - Some(app_ref) => app_ref, - None => continue, - }; - let app = match call(accessible_for(conn, &app_ref)).await { - Some(Ok(a)) => a, - _ => continue, - }; - let app_name = call(app.name()) - .await - .and_then(|r| r.ok()) - .unwrap_or_default(); - let frames = match call(app.get_children()).await { - Some(Ok(c)) => c, - _ => Vec::new(), - }; - let mut emitted = 0usize; - for (i, frame_ref) in frames.iter().enumerate() { - let frame_ref = match RawObjectRef::from_atspi(frame_ref) { - Some(frame_ref) => frame_ref, - None => continue, - }; - let frame = match call(accessible_for(conn, &frame_ref)).await { - Some(Ok(f)) => f, - _ => continue, - }; - let role = call(frame.get_role_name()) - .await - .and_then(|r| r.ok()) - .unwrap_or_default(); - if !matches!( - role.as_str(), - "frame" | "window" | "dialog" | "alert" | "file chooser" - ) { - continue; - } - let title = call(frame.name()) - .await - .and_then(|r| r.ok()) - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| app_name.clone()); - let geometry = match call(frame.proxies()).await { - Some(Ok(proxies)) => match call(proxies.component()).await { - Some(Ok(component)) => call(component.get_extents(CoordType::Screen)) - .await - .and_then(|result| result.ok()), - _ => None, - }, - _ => None, - }; - let (observed_x, observed_y, width, height) = geometry - .filter(|(_, _, width, height)| *width > 0 && *height > 0) - .map(|(x, y, width, height)| { - (x, y, width.max(0) as u32, height.max(0) as u32) - }) - .unwrap_or((0, 0, 0, 0)); - // On native Wayland, AT-SPI frame extents can be a stale - // toolkit default even when the compositor has already - // placed the window elsewhere. Prefer compositor metadata - // by pid/title so list_windows and later element geometry - // share the same screen origin. - let (x, y) = prefer_authoritative_wayland_origin( - authoritative_wayland_origin(cpid, 0, Some(&title)), - Some((observed_x, observed_y)), - ) - .unwrap_or((observed_x, observed_y)); - // Stable, non-zero, unique per (pid, frame ordinal). - let xid = (((cpid as u64) << 16) | (i as u64)).max(1); - out.push(WindowInfo { - xid, - pid: Some(cpid), - app_name: app_name.clone(), - title, - is_on_screen: width > 0 && height > 0, - z_index: None, - x, - y, - width, - height, - }); - emitted += 1; - } - // App with no enumerable top-level frame still gets one handle so - // the by-pid AT-SPI element flow stays reachable. - if emitted == 0 { - out.push(WindowInfo { - xid: (cpid as u64).max(1), - pid: Some(cpid), - app_name: app_name.clone(), - title: app_name, - is_on_screen: true, - z_index: None, - x: 0, - y: 0, - width: 0, - height: 0, - }); - } - } - Ok::<_, anyhow::Error>(out) - }; - match tokio::time::timeout(OP_TIMEOUT, work).await { - Ok(Ok(v)) => v, - Ok(Err(e)) => { - dlog!("atspi list_windows failed: {e}"); - Vec::new() - } - Err(_) => { - dlog!("atspi list_windows timed out"); - Vec::new() - } - } - }) -} - /// Pick the editable node to write into, by priority: /// 1. the focused editable (if the toolkit exposes focus), /// 2. an editable inside web/document content — for a browser this is the @@ -1792,7 +1637,8 @@ async fn write_through_editable_proxies( .map_err(|e| anyhow!("EditableText unavailable: {e}"))?; let off = proxies.text().await?.caret_offset().await?; - let len = text.chars().count() as i32; + let len = i32::try_from(text.len()) + .map_err(|_| anyhow!("AT-SPI text exceeds the supported UTF-8 byte length"))?; attempt.dispatch(); if !et.insert_text(off, text, len).await? { @@ -1821,6 +1667,30 @@ pub fn type_into_editable(pid: u32, text: &str) -> Result<()> { )) } +/// Insert only into the focused editable of the resolved window. Never search a sibling window. +pub fn type_into_focused_window(pid: u32, xid: u64, text: &str) -> Result<()> { + let attempt = InputAttempt::default(); + attempt.finish(bounded( + async { + let conn = shared_connection().await?; + let (visited, frame, _) = collect_visited_bounded(conn, pid, xid, None, None) + .await? + .ok_or_else(|| anyhow!("no AT-SPI application"))?; + let frame = frame.ok_or_else(|| anyhow!("window has no exact AT-SPI frame"))?; + let target = visited + .iter() + .find(|node| node.frame_ordinal == frame && node.focused && node.has_editable) + .ok_or_else(|| anyhow!("no focused editable in the selected window"))?; + if write_into_editable_target(target, text, &attempt).await? { + Ok(()) + } else { + Err(anyhow!("focused element does not support insertion")) + } + }, + || Err(anyhow!("focused window insertion timed out")), + )) +} + /// Write into the exact indexed editable exposed by the caller's snapshot. pub fn type_into_editable_at( pid: u32, @@ -2476,9 +2346,6 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result v, None => return Ok(None), }; - let web_document_origin = web_document_origin_for_visited(&visited, pid) - .await - .unwrap_or((0, 0)); // Collect actionable nodes whose window-local bounds contain the point, // then let `select_click_target` pick the innermost *real actuator* — @@ -2503,19 +2370,7 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result Result v, - None => return Ok(None), + let (visited, frame, _) = + match collect_visited_bounded(conn, pid, xid, None, None).await? { + Some(walked) => walked, + None => return Ok(None), + }; + let Some(frame) = frame else { + return Ok(None); }; - let web_document_origin = web_document_origin_for_visited(&visited, pid) - .await - .unwrap_or((0, 0)); // Reconstruct each indexable element's SCREEN frame the same way // get_window_state does: WINDOW-relative extents (GTK4 reports these // correctly; Screen is (0,0)) plus the window's screen origin (the - // GNOME Shell helper on Wayland, _GTK_FRAME_EXTENTS on X11). When no + // X11 _GTK_FRAME_EXTENTS). When no // offset resolves, fall back to CoordType::Screen (correct on Qt/GTK3). let offset = window_to_screen_offset(pid, xid, None); let coord = if offset.is_some() { @@ -2604,10 +2443,15 @@ pub fn perform_action_at_screen_point( // (element_index, x, y, w, h, is_passive_label) over the SAME indexable // list `perform_action`/`get_window_state` use, so the chosen index // maps straight back to a verified `element_index` actuation. - let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect(); + let action_nodes: Vec<&Visited> = visited + .iter() + .filter(|v| is_indexable(v) && v.frame_ordinal == frame) + .collect(); let mut frames: Vec<(usize, i32, i32, u32, u32, bool)> = Vec::new(); for (idx, node) in action_nodes.iter().enumerate() { - if !node.has_component { + if !node.has_component + || crate::at_point_policy::is_top_level_shell_role(&node.role) + { continue; } let Some(Ok(proxies)) = call(node.acc.proxies()).await else { @@ -2622,15 +2466,10 @@ pub fn perform_action_at_screen_point( if x == i32::MIN || y == i32::MIN || w <= 1 || h <= 1 { continue; } - let (document_x, document_y) = if node.in_web_doc { - web_document_origin - } else { - (0, 0) - }; frames.push(( idx, - x + ox + document_x, - y + oy + document_y, + x + ox, + y + oy, w as u32, h as u32, is_passive_role(&node.role), @@ -2641,6 +2480,12 @@ pub fn perform_action_at_screen_point( return Ok(None); }; let target = action_nodes[idx]; + if matches!( + target.role.to_ascii_lowercase().as_str(), + "entry" | "text" | "password text" | "combo box" | "editable text" + ) { + return Ok(None); + } let Some(chosen) = activation_index(&target.role, &target.actions) else { return Ok(None); }; @@ -2713,6 +2558,108 @@ fn select_click_target( best_active.or(best_passive).map(|(_, idx)| idx) } +const OPTION_SCAN_CAP: usize = 200; + +fn is_option_role(role: &str) -> bool { + matches!( + role.trim().to_ascii_lowercase().as_str(), + "menu item" | "check menu item" | "radio menu item" | "list item" | "option" + ) +} + +/// A combo box's options live in a child menu (GTK) or list (Qt). +fn is_option_container_role(role: &str) -> bool { + matches!( + role.trim().to_ascii_lowercase().as_str(), + "menu" | "popup menu" | "list" | "list box" | "window" + ) +} + +/// Index of the option `value` names: an exact match first, then one that +/// differs only in case or surrounding space. +pub fn match_option(names: &[String], value: &str) -> Option { + let wanted = value.trim().to_lowercase(); + let mut found = names + .iter() + .enumerate() + .filter(|(_, name)| name.trim().to_lowercase() == wanted); + let index = found.next()?.0; + found.next().is_none().then_some(index) +} + +struct OptionNode { + oref: RawObjectRef, + name: String, + /// Position among its container's children: the model row of a GTK + /// combo box, which its `Selection.SelectChild` takes. + index: i32, +} + +/// The options under `oref`: item children, or the items of its popup menu +/// or list child (one level down), in order. +async fn list_options(conn: &AccessibilityConnection, oref: &RawObjectRef) -> Vec { + let zconn = conn.connection(); + let mut options = Vec::new(); + let Some(Ok(children)) = call(raw_children(zconn, oref)).await else { + return options; + }; + for (position, child) in children.into_iter().enumerate() { + if options.len() >= OPTION_SCAN_CAP { + break; + } + let Some(Ok(acc)) = call(accessible_for(conn, &child)).await else { + continue; + }; + let role = call(acc.get_role_name()) + .await + .and_then(|r| r.ok()) + .unwrap_or_default(); + if is_option_role(&role) { + let name = call(acc.name()) + .await + .and_then(|r| r.ok()) + .unwrap_or_default(); + options.push(OptionNode { + oref: child, + name, + index: position as i32, + }); + continue; + } + if !is_option_container_role(&role) { + continue; + } + let Some(Ok(items)) = call(raw_children(zconn, &child)).await else { + continue; + }; + for (index, item) in items.into_iter().enumerate() { + if options.len() >= OPTION_SCAN_CAP { + break; + } + let Some(Ok(item_acc)) = call(accessible_for(conn, &item)).await else { + continue; + }; + let item_role = call(item_acc.get_role_name()) + .await + .and_then(|r| r.ok()) + .unwrap_or_default(); + if !is_option_role(&item_role) { + continue; + } + let name = call(item_acc.name()) + .await + .and_then(|r| r.ok()) + .unwrap_or_default(); + options.push(OptionNode { + oref: item, + name, + index: index as i32, + }); + } + } + options +} + pub fn set_value(pid: u32, idx: usize, identity: Option, value: &str) -> Result<()> { let attempt = InputAttempt::default(); attempt.finish(bounded( @@ -2729,6 +2676,43 @@ pub fn set_value(pid: u32, idx: usize, identity: Option, value: & .await .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?; + if target.role.trim().eq_ignore_ascii_case("combo box") { + let source = RawObjectRef { + name: target.acc.inner().destination().to_string(), + path: target.acc.inner().path().to_string(), + }; + let options = list_options(conn, &source).await; + let names = options + .iter() + .map(|option| option.name.clone()) + .collect::>(); + let index = match_option(&names, value).ok_or_else(|| { + anyhow!( + "not_settable: no unique combo option matching {value:?}; options: {names:?}" + ) + })?; + let selection = proxies + .selection() + .await + .map_err(|e| anyhow!("not_settable: Selection unavailable: {e}"))?; + attempt.dispatch(); + if !selection.select_child(options[index].index).await? { + return Err(anyhow!( + "write_not_applied: Selection.SelectChild was refused" + )); + } + let child = selection.get_selected_child(0).await?; + let selected = RawObjectRef::from_atspi(&child) + .ok_or_else(|| anyhow!("write_not_applied: selected option is unavailable"))?; + let selected = accessible_for(conn, &selected).await?; + if selected.name().await? != options[index].name { + return Err(anyhow!( + "write_not_applied: combo selection readback differs" + )); + } + return Ok(()); + } + // SetValue is a focus-free accessibility operation. Do not call // Component.GrabFocus here: GTK may activate and raise the entire // toplevel in response, violating the background contract. Toolkits @@ -2790,9 +2774,6 @@ pub(crate) fn get_element_bounds_in_window( let (visited, only_frame, _) = collect_visited_bounded(conn, pid, xid, None, None) .await? .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; - let web_document_origin = web_document_origin_for_visited(&visited, pid) - .await - .unwrap_or((0, 0)); if xid != 0 && only_frame.is_none() { return Err(anyhow!( "window {xid} could not be correlated to an AT-SPI frame" @@ -2818,24 +2799,14 @@ pub(crate) fn get_element_bounds_in_window( .map_err(|e| anyhow!("Component unavailable: {e}"))?; // Prefer WINDOW coords + a deterministic screen offset — fixes GTK4, // whose CoordType::Screen collapses every element to (0,0). Fall back to - // Screen on Wayland / when no X11 window resolves (offset is None). + // Screen when no X11 window resolves (offset is None). match window_to_screen_offset(pid, xid, None) { Some((ox, oy)) => { let (x, y, w, h) = comp .get_extents(CoordType::Window) .await .map_err(|e| anyhow!("getExtents failed: {e}"))?; - let (document_x, document_y) = if target.in_web_doc { - web_document_origin - } else { - (0, 0) - }; - Ok(( - x + ox + document_x, - y + oy + document_y, - w.max(0) as u32, - h.max(0) as u32, - )) + Ok((x + ox, y + oy, w.max(0) as u32, h.max(0) as u32)) } None => { let (x, y, w, h) = comp @@ -2871,6 +2842,28 @@ fn x11_window_origin(xid: u64) -> Option<(i32, i32)> { Some((trans.dst_x as i32, trans.dst_y as i32)) } +fn x11_top_level_geometry(xid: u64) -> Option<(i32, i32, i32, i32)> { + use x11rb::protocol::xproto::*; + use x11rb::rust_connection::RustConnection; + + let (conn, _) = RustConnection::connect(None).ok()?; + let mut window = xid as u32; + loop { + let tree = conn.query_tree(window).ok()?.reply().ok()?; + if tree.parent == tree.root || tree.parent == x11rb::NONE { + break; + } + window = tree.parent; + } + let geom = conn.get_geometry(window).ok()?.reply().ok()?; + Some(( + i32::from(geom.x), + i32::from(geom.y), + i32::from(geom.width), + i32::from(geom.height), + )) +} + /// Read the GTK4 client-side-decoration shadow inset from the X11 /// `_GTK_FRAME_EXTENTS` property (`CARDINAL[4]` = left, right, top, bottom). /// @@ -2935,28 +2928,8 @@ fn parse_gtk_frame_extents(vals: &[u32]) -> Option<(i32, i32)> { /// toolkits (Qt, etc.) have no such property *and* report correct Screen /// extents, so we return `None` for them — callers keep the unchanged Screen /// path and the WINDOW reconstruction can never regress a toolkit that was -/// already correct. Also returns `None` on native Wayland (clients may not -/// query screen origins, by design) or when no X11 window resolves. -fn window_to_screen_offset(pid: u32, xid: u64, title: Option<&str>) -> Option<(i32, i32)> { - if crate::wayland::is_wayland() { - // Native Wayland: clients can't query a window's screen origin, and - // AT-SPI CoordType::Screen collapses to (0,0) on Mutter. The bundled - // `org.cua.WinRects` GNOME Shell extension supplies the window's screen - // origin (`meta_window.get_frame_rect()`); combined with the per-widget - // CoordType::Window coords (which GTK4 reports correctly on Wayland too) - // this reconstructs real screen coords — the GNOME analogue of the X11 - // `_GTK_FRAME_EXTENTS` path below. `None` (no extension) keeps the - // legacy Screen path (still (0,0), but no worse than before). - let authoritative = authoritative_wayland_origin(pid, xid, title); - // AT-SPI discovery can only guess a native Wayland origin when the - // compositor exposes no geometry. Keep that observation as the final - // fallback so stale default placement cannot override Sway IPC or a - // shell helper's authoritative frame. - return prefer_authoritative_wayland_origin( - authoritative, - crate::wayland::observed_window_origin(pid), - ); - } +/// already correct. Also returns `None` when no X11 window resolves. +fn window_to_screen_offset(pid: u32, xid: u64, _title: Option<&str>) -> Option<(i32, i32)> { // GTK3 Screen coordinates already include the CSD inset. GTK4's broken // Screen provider still needs the Window reconstruction below. if std::fs::read_to_string(format!("/proc/{pid}/maps")) @@ -2980,130 +2953,16 @@ fn window_to_screen_offset(pid: u32, xid: u64, title: Option<&str>) -> Option<(i Some((ox + fl, oy + ft)) } -/// Resolve a native Wayland window origin from compositor-owned metadata. -/// These sources are authoritative over AT-SPI's observed frame location, -/// which can remain at a toolkit default after Sway places the real window. -fn authoritative_wayland_origin(pid: u32, xid: u64, title: Option<&str>) -> Option<(i32, i32)> { - if !crate::wayland::is_wayland() { - return None; - } - crate::wayland::inject_accessibility_offset(pid) - .or_else(|| crate::wayland::sway_ipc::window_origin_for_pid(pid)) - .or_else(|| { - (xid != 0) - .then(|| crate::wayland::sway_ipc::window_for_id(xid)) - .flatten() - .map(|window| (window.x, window.y)) - }) - .or_else(|| { - (xid != 0) - .then(|| { - crate::wayland::sway_ipc::list_windows().and_then(|_| { - crate::wayland::window_geometry(xid) - .map(|(window_x, window_y, _, _)| (window_x, window_y)) - }) - }) - .flatten() - }) - .or_else(|| crate::wayland::shell_helper::window_origin_for_pid(pid)) - .or_else(|| title.and_then(crate::wayland::sway_ipc::window_origin_for_title)) -} - -fn prefer_authoritative_wayland_origin( - authoritative: Option<(i32, i32)>, - observed: Option<(i32, i32)>, -) -> Option<(i32, i32)> { - authoritative.or(observed) -} - -fn combine_wayland_content_offsets( - compositor: Option<(i32, i32)>, - document: Option<(i32, i32)>, - document_is_separate: bool, -) -> Option<(i32, i32)> { - if !document_is_separate { - // Chromium descendants' CoordType::Window extents are already rooted - // below the document accessible. Adding that document's own (x,y) - // double-counts its renderer inset. WebKitGTK exports page content on - // a distinct WebProcess bus, so only that bridge needs the extra hop. - return compositor; - } - match (compositor, document) { - (Some((cx, cy)), Some((dx, dy))) => Some((cx + dx, cy + dy)), - (Some(offset), None) | (None, Some(offset)) => Some(offset), - (None, None) => None, - } -} - -/// Offset of embedded web content inside a captured Wayland toplevel. -/// Compositor decorations and toolkit document offsets are independent and -/// therefore additive: choosing one or the other leaves WebKit controls one -/// title bar away from the pixels shown to the caller. -async fn web_document_origin_for_visited(visited: &[Visited<'_>], pid: u32) -> Option<(i32, i32)> { - if !crate::wayland::is_wayland() { - return None; - } - let sway_window = crate::wayland::sway_ipc::window_for_pid(pid); - let compositor = sway_window - .as_ref() - .map(|window| (window.content_x, window.content_y)); - let document = visited - .iter() - .filter(|node| node.has_component) - .filter(|node| is_document_role(&node.role) || node.in_web_doc) - .min_by_key(|node| node.depth); - let document = if let Some(document) = document { - match call(document.acc.proxies()).await { - Some(Ok(proxies)) => match call(proxies.component()).await { - Some(Ok(component)) => match call(component.get_extents(CoordType::Window)).await { - Some(Ok((x, y, width, height))) if x >= 0 && y >= 0 => { - let inferred_top = match (compositor, sway_window.as_ref()) { - (Some((_, 0)), Some(window)) - if width > 0 - && height > 0 - && (i64::from(window.width) - i64::from(width)).abs() <= 4 - && i64::from(window.height) > i64::from(height) => - { - (i64::from(window.height) - i64::from(height)) - .min(i64::from(i32::MAX)) as i32 - } - _ => 0, - }; - Some((x, y.max(inferred_top))) - } - _ => None, - }, - _ => None, - }, - _ => None, - } - } else { - None - }; - let document_is_separate = visited.iter().any(|node| node.on_web_process_bus); - let combined = combine_wayland_content_offsets(compositor, document, document_is_separate); - dlog!( - "Wayland web content offset: compositor={compositor:?} document={document:?} separate_process={document_is_separate} combined={combined:?}" - ); - combined -} - fn screen_extent_rebase( x11_origin: (i32, i32), - accessible_frame_origin: (i32, i32), + frame: (i32, i32, i32, i32), + top_level: Option<(i32, i32, i32, i32)>, ) -> Option<(i32, i32)> { - // Chromium's broken "Screen" provider is rooted at the renderer-local - // origin. A legitimate screen provider may differ from the X11 client - // origin by title-bar/CSD extents; rebasing that small decoration delta - // would move otherwise-correct GTK coordinates off their controls. - if accessible_frame_origin.0.abs() <= 2 && accessible_frame_origin.1.abs() <= 2 { - Some(( - x11_origin.0 - accessible_frame_origin.0, - x11_origin.1 - accessible_frame_origin.1, - )) - } else { - None + let (x, y, w, h) = frame; + if w <= 0 || h <= 0 || top_level == Some(frame) { + return None; } + (x.abs() <= 2 && y.abs() <= 2).then_some((x11_origin.0 - x, x11_origin.1 - y)) } fn rebase_renderer_window_offset( @@ -3114,7 +2973,7 @@ fn rebase_renderer_window_offset( // Chromium may expose a negative renderer-local frame origin. Rebase // that shape, but keep positive content insets: its descendants are // already relative to the content origin and subtracting the inset - // moves first-row controls above the captured Wayland window. + // moves first-row controls above the captured window. if frame_x < 0 { offset.0 = offset.0.saturating_sub(frame_x); } @@ -3152,7 +3011,7 @@ async fn element_bounds_for_visited( // Query WINDOW-relative extents and add a deterministic screen offset // (X11 window origin + GTK4 CSD inset). This fixes GTK4 — whose // CoordType::Screen reports every element at (0,0) — by using the - // distinct per-widget WINDOW coords instead. On Wayland / when no X11 + // distinct per-widget WINDOW coords instead. When no X11 // window resolves, `offset` is None and we keep the legacy Screen path // so non-X11 behaviour is unchanged. let window_title = visited.iter().find_map(|node| { @@ -3175,7 +3034,7 @@ async fn element_bounds_for_visited( // produce a zero delta; Chromium's local (0,0) frame produces the // required window-origin delta. GTK's explicit Window-coordinate // path above remains authoritative when available. - let screen_rebase = if offset.is_none() && !crate::wayland::is_wayland() && xid != 0 { + let screen_rebase = if offset.is_none() && xid != 0 { let x11_origin = x11_window_origin(xid); let frame = visited.iter().find(|node| { node.has_component @@ -3189,7 +3048,7 @@ async fn element_bounds_for_visited( Some(Ok(proxies)) => match call(proxies.component()).await { Some(Ok(component)) => { match call(component.get_extents(CoordType::Screen)).await { - Some(Ok((x, y, _, _))) => Some((x, y)), + Some(Ok(extents)) => Some(extents), _ => None, } } @@ -3197,7 +3056,12 @@ async fn element_bounds_for_visited( }, _ => None, }; - accessible_origin.and_then(|frame_origin| screen_extent_rebase(origin, frame_origin)) + accessible_origin.and_then(|frame_origin| { + let top_level = (frame_origin.0.abs() <= 2 && frame_origin.1.abs() <= 2) + .then(|| x11_top_level_geometry(xid)) + .flatten(); + screen_extent_rebase(origin, frame_origin, top_level) + }) } else { None } @@ -3236,14 +3100,6 @@ async fn element_bounds_for_visited( } else { None }; - // Add compositor decorations and the embedded document origin for web - // descendants only. Electron commonly contributes zero for both; WebKitGTK - // under Sway needs the sum. - let web_document_origin = if offset.is_some() { - web_document_origin_for_visited(visited, pid).await - } else { - None - }; let (offset_x, offset_y) = rebase_renderer_window_offset( offset.or(screen_rebase).unwrap_or((0, 0)), window_frame_origin, @@ -3285,18 +3141,7 @@ async fn element_bounds_for_visited( if x == i32::MIN || y == i32::MIN || x < -16384 || y < -16384 || w <= 1 || h <= 1 { continue; } - let (document_x, document_y) = if node.in_web_doc { - web_document_origin.unwrap_or((0, 0)) - } else { - (0, 0) - }; - out.push(( - idx, - x + offset_x + document_x, - y + offset_y + document_y, - w as u32, - h as u32, - )); + out.push((idx, x + offset_x, y + offset_y, w as u32, h as u32)); } } true @@ -3450,9 +3295,8 @@ mod frame_correlation_tests { mod coord_tests { use super::parse_gtk_frame_extents; use super::{ - activation_index, before_snapshot_deadline, combine_wayland_content_offsets, - hidden_native_menu, is_activation_action, is_enabled_state, is_indexable_capabilities, - is_passive_role, is_web_process_bus, prefer_authoritative_wayland_origin, + activation_index, before_snapshot_deadline, hidden_native_menu, is_activation_action, + is_enabled_state, is_indexable_capabilities, is_passive_role, is_web_process_bus, rebase_renderer_window_offset, screen_extent_rebase, select_click_target, ApplicationSelection, }; @@ -3762,9 +3606,35 @@ mod coord_tests { #[test] fn screen_extents_are_rebased_from_accessible_frame_to_x11_origin() { - assert_eq!(screen_extent_rebase((604, 80), (0, 0)), Some((604, 80))); - assert_eq!(screen_extent_rebase((604, 100), (604, 80)), None); - assert_eq!(screen_extent_rebase((604, 80), (604, 80)), None); + assert_eq!( + screen_extent_rebase((604, 80), (0, 0, 800, 600), None), + Some((604, 80)) + ); + assert_eq!( + screen_extent_rebase((604, 100), (604, 80, 800, 600), None), + None + ); + assert_eq!( + screen_extent_rebase((604, 80), (604, 80, 800, 600), None), + None + ); + } + + #[test] + fn corner_screen_frames_keep_their_coordinates_but_renderer_frames_are_rebased() { + assert_eq!( + screen_extent_rebase((1, 16), (0, 0, 422, 301), Some((0, 0, 422, 301))), + None + ); + assert_eq!( + screen_extent_rebase((5, 29), (0, 0, 640, 480), Some((0, 0, 650, 514))), + Some((5, 29)) + ); + assert_eq!( + screen_extent_rebase((5, 29), (0, 0, 640, 480), None), + Some((5, 29)) + ); + assert_eq!(screen_extent_rebase((5, 29), (0, 0, 0, 0), None), None); } #[test] @@ -3779,47 +3649,6 @@ mod coord_tests { ); } - #[test] - fn compositor_origin_wins_over_stale_accessibility_observation() { - assert_eq!( - prefer_authoritative_wayland_origin(Some((0, 0)), Some((120, 120))), - Some((0, 0)) - ); - assert_eq!( - prefer_authoritative_wayland_origin(None, Some((120, 120))), - Some((120, 120)) - ); - } - - #[test] - fn wayland_compositor_and_document_offsets_are_additive() { - assert_eq!( - combine_wayland_content_offsets(Some((0, 47)), Some((0, 0)), true), - Some((0, 47)) - ); - assert_eq!( - combine_wayland_content_offsets(Some((2, 20)), Some((0, 47)), true), - Some((2, 67)) - ); - assert_eq!( - combine_wayland_content_offsets(None, Some((0, 47)), true), - Some((0, 47)) - ); - assert_eq!(combine_wayland_content_offsets(None, None, true), None); - } - - #[test] - fn chromium_window_extents_do_not_double_count_document_origin() { - assert_eq!( - combine_wayland_content_offsets(Some((2, 20)), Some((22, 55)), false), - Some((2, 20)) - ); - assert_eq!( - combine_wayland_content_offsets(None, Some((22, 55)), false), - None - ); - } - #[test] fn webkit_web_process_bus_marks_roleless_document_subtrees() { assert!(is_web_process_bus("org.webkitgtk.WebProcess.1234")); @@ -3959,7 +3788,7 @@ mod coord_tests { #[test] fn chromium_action_names_stay_activatable() { - // Live capture from Chromium on GNOME Wayland. These elements were + // Chromium accessibility capture. These elements were // actuable before this change and must remain so; only // `showContextMenu` is not an activation. assert!(is_activation_action("activate")); // entry diff --git a/packages/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs b/packages/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs index fdfadeded8e..740415779a9 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs @@ -147,7 +147,7 @@ fn exact_allow_button( } fn prove_window_owner(pid: u32, window_id: u64) -> Result<(), BrowserRefusal> { - let owned = crate::wayland::list_windows_dispatch(Some(pid)) + let owned = crate::x11::list_windows(Some(pid)) .into_iter() .any(|window| window.xid == window_id); if !owned { @@ -160,19 +160,11 @@ fn prove_window_owner(pid: u32, window_id: u64) -> Result<(), BrowserRefusal> { } fn with_target_foreground( - pid: u32, + _pid: u32, window_id: u64, body: impl FnOnce() -> anyhow::Result, ) -> anyhow::Result { - if std::env::var_os("WAYLAND_DISPLAY").is_some() { - if let Some(window) = - crate::wayland::sway_ipc::window_for_id(window_id).filter(|window| window.pid == pid) - { - crate::wayland::sway_ipc::with_focused_container(window.id, body) - } else { - crate::wayland::shell_helper::with_focused_window(pid, window_id, body) - } - } else { + { crate::input::with_x11_foreground(window_id, 80, body) } } @@ -203,9 +195,7 @@ fn trusted_allow_click(pid: u32, window_id: u64) -> anyhow::Result<()> { anyhow::anyhow!("the exact Chromium remote-debugging consent action became stale") })?; let (center_x, center_y) = exact_button_center(&tree.bounds, index)?; - if std::env::var_os("WAYLAND_DISPLAY").is_some() { - crate::wayland::click_desktop(center_x, center_y, 1, 1) - } else { + { crate::input::send_click_xtest_desktop(center_x, center_y, 1, 1) } }) diff --git a/packages/cua-driver/rust/crates/platform-linux/src/browser_platform.rs b/packages/cua-driver/rust/crates/platform-linux/src/browser_platform.rs index 6808992d6fc..4dcb45b7e3c 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/browser_platform.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/browser_platform.rs @@ -2,13 +2,15 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::path::PathBuf; +use std::sync::{Arc, Mutex}; use std::time::Duration; use async_trait::async_trait; use cua_driver_core::browser::existing_profile_setup_descriptor; use cua_driver_core::browser::platform::{ - BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, ExistingProfileSetupOutcome, - ExistingProfileSetupRequest, PrepareAction, PrepareOutcome, PrepareRequest, + BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, BrowserVisualAction, + BrowserVisualActionKind, ExistingProfileSetupOutcome, ExistingProfileSetupRequest, + PrepareAction, PrepareOutcome, PrepareRequest, }; use cua_driver_core::browser::refusal::{BrowserRefusal, BrowserRefusalCode}; use cua_driver_core::browser::types::{ @@ -18,8 +20,68 @@ use cua_driver_core::browser::types::{ }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; +// Adapted from trycua/cua #3778 (MIT); only the existing native browser tools use this. +pub struct LinuxBrowserPlatform { + cursor_registry: Arc, + browser_cursors: Mutex, +} +#[derive(Debug, Clone, PartialEq, Eq)] +struct BrowserCursorBinding { + window_id: u64, + cdp_target_id: String, +} + #[derive(Debug, Default)] -pub struct LinuxBrowserPlatform; +struct BrowserCursorTracker { + bindings: HashMap, +} + +impl BrowserCursorTracker { + fn update( + &mut self, + session: &str, + window_id: u64, + cdp_target_id: &str, + tab_is_active: bool, + ) -> Vec<(String, bool)> { + self.bindings.insert( + session.to_owned(), + BrowserCursorBinding { + window_id, + cdp_target_id: cdp_target_id.to_owned(), + }, + ); + + if !tab_is_active { + return vec![(session.to_owned(), false)]; + } + + self.bindings + .iter() + .filter(|(_, binding)| binding.window_id == window_id) + .map(|(key, binding)| { + ( + key.clone(), + key == session && binding.cdp_target_id == cdp_target_id, + ) + }) + .collect() + } +} + +impl LinuxBrowserPlatform { + pub fn new(cursor_registry: Arc) -> Self { + Self { + cursor_registry, + browser_cursors: Mutex::new(BrowserCursorTracker::default()), + } + } +} +impl Default for LinuxBrowserPlatform { + fn default() -> Self { + Self::new(Arc::new(cursor_overlay::CursorRegistry::new())) + } +} fn refusal(code: BrowserRefusalCode, message: impl Into) -> BrowserRefusal { BrowserRefusal::new(code, message) @@ -388,6 +450,76 @@ async fn browser_websocket_url(port: u16) -> Option { #[async_trait] impl BrowserPlatform for LinuxBrowserPlatform { + async fn visualize_browser_action(&self, action: BrowserVisualAction) { + if action.session.is_empty() + || action.cdp_target_id.is_empty() + || cua_driver_core::session::is_session_ended(&action.session) + { + return; + } + + let visibility_updates = self.browser_cursors.lock().unwrap().update( + &action.session, + action.window_id, + &action.cdp_target_id, + action.tab_is_active, + ); + let cursor_enabled = self + .cursor_registry + .get_or_create(&action.session) + .config + .enabled; + for (key, visible) in visibility_updates { + let enabled = if key == action.session { + visible && cursor_enabled + } else { + visible + && self + .cursor_registry + .get(&key) + .is_some_and(|state| state.config.enabled) + }; + crate::overlay::send_command_for( + key, + cursor_overlay::OverlayCommand::SetEnabled(enabled), + ); + } + if !action.tab_is_active || !cursor_enabled { + return; + } + let (Some(screen_x), Some(screen_y)) = (action.screen_x, action.screen_y) else { + return; + }; + if !screen_x.is_finite() || !screen_y.is_finite() { + return; + } + + crate::overlay::send_command_for( + action.session.clone(), + cursor_overlay::OverlayCommand::PinAbove(action.window_id), + ); + crate::overlay::animate_cursor_to_for(action.session.clone(), screen_x, screen_y).await; + self.cursor_registry + .update_position(&action.session, screen_x, screen_y); + + if matches!( + action.kind, + BrowserVisualActionKind::Click + | BrowserVisualActionKind::Type + | BrowserVisualActionKind::RightClick + | BrowserVisualActionKind::DoubleClick + | BrowserVisualActionKind::Drag + ) { + crate::overlay::send_command_for( + action.session, + cursor_overlay::OverlayCommand::ClickPulse { + x: screen_x, + y: screen_y, + }, + ); + } + } + fn standalone_trusted_input_background_limitation(&self) -> Option<&'static str> { Some("Chromium's trusted CDP Input route activates its standalone browser window on Linux") } @@ -450,91 +582,6 @@ impl BrowserPlatform for LinuxBrowserPlatform { format!("pid {pid} is outside the Linux process-id range"), ) })?; - if std::env::var_os("WAYLAND_DISPLAY").is_some() { - if let Some(window) = crate::wayland::sway_ipc::window_for_id(window_id) { - if window.pid != pid_u32 { - return Err(refusal( - BrowserRefusalCode::BrowserWrongTargetRefused, - format!("Sway window {window_id} is not owned by pid {pid}"), - )); - } - return Ok(NativeWindowInfo { - pid, - window_id, - title: window.title, - bounds: Rect::new( - f64::from(window.x), - f64::from(window.y), - f64::from(window.width), - f64::from(window.height), - ), - geometry_exact: true, - ownership: NativeOwnershipProof { - method: NativeOwnershipMethod::WindowServerOwner, - owner_pid: pid, - detail: Some("Sway IPC pid and rect".to_owned()), - }, - }); - } - if let Some(window) = - crate::wayland::shell_helper::trusted_window_for_id(pid_u32, window_id) - { - return Ok(NativeWindowInfo { - pid, - window_id, - title: window.title, - bounds: Rect::new( - f64::from(window.x), - f64::from(window.y), - f64::from(window.width), - f64::from(window.height), - ), - geometry_exact: true, - ownership: NativeOwnershipProof { - method: NativeOwnershipMethod::PlatformAttested, - owner_pid: pid, - detail: Some( - "verified GNOME Shell helper owner, stable window id, pid, and frame rect" - .to_owned(), - ), - }, - }); - } - let window = tokio::task::spawn_blocking(move || { - crate::wayland::list_windows_dispatch(Some(pid_u32)) - .into_iter() - .find(|window| window.xid == window_id) - }) - .await - .ok() - .flatten() - .ok_or_else(|| { - refusal( - BrowserRefusalCode::BrowserRouteUnavailable, - "the active Wayland compositor cannot prove this window's pid and geometry", - ) - })?; - return Ok(NativeWindowInfo { - pid, - window_id, - title: window.title, - bounds: Rect::new( - f64::from(window.x), - f64::from(window.y), - f64::from(window.width), - f64::from(window.height), - ), - geometry_exact: false, - ownership: NativeOwnershipProof { - method: NativeOwnershipMethod::PlatformAttested, - owner_pid: pid, - detail: Some( - "Wayland shell/AT-SPI metadata; geometry may be unavailable, so core permits read-only heuristic binding only" - .to_owned(), - ), - }, - }); - } let window = tokio::task::spawn_blocking(move || { crate::x11::list_windows(Some(pid_u32)) @@ -583,44 +630,7 @@ impl BrowserPlatform for LinuxBrowserPlatform { format!("pid {pid} is outside the Linux process-id range"), ) })?; - if std::env::var_os("WAYLAND_DISPLAY").is_some() { - let Some(windows) = crate::wayland::sway_ipc::list_windows() else { - if let Some(owned) = - crate::wayland::shell_helper::trusted_window_ids_for_pid(pid_u32) - { - return Ok(Some(owned.len() == 1 && owned[0] == window_id)); - } - if crate::wayland::is_inject_mode() { - // The private cua-compositor route owns both the native - // toplevel enumeration and its PID correlation. Unlike a - // generic AT-SPI-only Wayland session, that is sufficient - // to attest singleton native-window cardinality for an - // embedded Chromium endpoint. - let owned = tokio::task::spawn_blocking(move || { - crate::wayland::list_windows_dispatch(Some(pid_u32)) - .into_iter() - .filter(|window| window.pid == Some(pid_u32)) - .map(|window| window.xid) - .collect::>() - }) - .await - .map_err(|error| { - refusal( - BrowserRefusalCode::BrowserRouteUnavailable, - format!("could not enumerate cua-compositor browser windows: {error}"), - ) - })?; - return Ok(Some(owned.len() == 1 && owned[0] == window_id)); - } - return Ok(None); - }; - let owned = windows - .into_iter() - .filter(|window| window.pid == pid_u32) - .map(|window| window.id) - .collect::>(); - return Ok(Some(owned.len() == 1 && owned[0] == window_id)); - } + let owned = tokio::task::spawn_blocking(move || { crate::x11::list_windows(Some(pid_u32)) .into_iter() diff --git a/packages/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs b/packages/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs index a09b44dfb69..faaf310f480 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs @@ -105,28 +105,18 @@ fn setup_not_ready_message(descriptor: &BrowserSetupDescriptor) -> String { } fn with_target_foreground( - pid: u32, + _pid: u32, window_id: u64, body: impl FnOnce() -> anyhow::Result, ) -> anyhow::Result { - if std::env::var_os("WAYLAND_DISPLAY").is_some() { - if let Some(window) = - crate::wayland::sway_ipc::window_for_id(window_id).filter(|window| window.pid == pid) - { - crate::wayland::sway_ipc::with_focused_container(window.id, body) - } else { - crate::wayland::shell_helper::with_focused_window(pid, window_id, body) - } - } else { + { crate::input::with_x11_foreground(window_id, 80, body) } } fn close_tab(pid: u32, window_id: u64) -> anyhow::Result<()> { with_target_foreground(pid, window_id, || { - if std::env::var_os("WAYLAND_DISPLAY").is_some() { - crate::wayland::hotkey(window_id, &["ctrl".to_owned(), "w".to_owned()]) - } else { + { crate::input::send_key_xtest("w", &["ctrl"]) } }) @@ -191,25 +181,19 @@ fn trusted_setup_navigation( window_id: u64, descriptor: &BrowserSetupDescriptor, ) -> anyhow::Result<()> { - let wayland = std::env::var_os("WAYLAND_DISPLAY").is_some(); - // A fresh tab, so the setup page never displaces a page the user was on. // `ctrl+l` then focuses the address field. Both are single letters, so // neither depends on the keyboard layout the way punctuation does. let focus_omnibox = || -> anyhow::Result<()> { with_target_foreground(pid, window_id, || { - if wayland { - crate::wayland::hotkey_focused(&["ctrl".to_owned(), "l".to_owned()]) - } else { + { crate::input::send_key_xtest("l", &["ctrl"]) } }) }; with_target_foreground(pid, window_id, || { - if wayland { - crate::wayland::hotkey_focused(&["ctrl".to_owned(), "t".to_owned()]) - } else { + { crate::input::send_key_xtest("t", &["ctrl"]) } })?; @@ -252,15 +236,9 @@ fn trusted_setup_navigation( .write_text(descriptor.setup_url.to_owned()) .map_err(|error| anyhow::anyhow!("could not stage the fixed setup URL: {error}"))?; let paste = with_target_foreground(pid, window_id, || { - if wayland { - crate::wayland::hotkey_focused(&["ctrl".to_owned(), "a".to_owned()])?; - std::thread::sleep(Duration::from_millis(60)); - crate::wayland::hotkey_focused(&["ctrl".to_owned(), "v".to_owned()]) - } else { - crate::input::send_key_xtest("a", &["ctrl"])?; - std::thread::sleep(Duration::from_millis(60)); - crate::input::send_key_xtest("v", &["ctrl"]) - } + crate::input::send_key_xtest("a", &["ctrl"])?; + std::thread::sleep(Duration::from_millis(60)); + crate::input::send_key_xtest("v", &["ctrl"]) }); // The user's clipboard is theirs; put it back whether or not the paste took. std::thread::sleep(Duration::from_millis(120)); @@ -272,9 +250,7 @@ fn trusted_setup_navigation( // Commit. Enter is the one synthesized keystroke left, and it carries no // layout dependency. with_target_foreground(pid, window_id, || { - if wayland { - crate::wayland::hotkey_focused(&["enter".to_owned()]) - } else { + { crate::input::send_key_xtest("enter", &[]) } })?; @@ -604,9 +580,7 @@ pub fn enable( let center_y = y .checked_add(i32::try_from(height / 2)?) .ok_or_else(|| anyhow::anyhow!("checkbox center y overflowed"))?; - if std::env::var_os("WAYLAND_DISPLAY").is_some() { - crate::wayland::click_desktop(center_x, center_y, 1, 1)?; - } else { + { crate::input::send_click_xtest_desktop(center_x, center_y, 1, 1)?; } Ok(true) diff --git a/packages/cua-driver/rust/crates/platform-linux/src/capture.rs b/packages/cua-driver/rust/crates/platform-linux/src/capture.rs index 0a59d3bd26f..c746950062a 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/capture.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/capture.rs @@ -5,8 +5,7 @@ //! 2. Persistent plain `XGetImage` via x11rb — still in-process //! 3. `import -window png:-` (ImageMagick compatibility fallback) //! -//! Main-display capture keeps its own dispatch (Wayland cascade → ImageMagick → -//! root `XGetImage`). Wayland-native per-window paths are not routed into XShm. +//! Main-display capture uses ImageMagick then root `XGetImage`. //! //! x11rb MIT-SHM API (pinned 0.13.2): //! - https://docs.rs/x11rb/0.13.2/x11rb/protocol/shm/trait.ConnectionExt.html @@ -169,12 +168,13 @@ struct PackedLayout { len: usize, } -#[derive(Clone, Copy)] +#[derive(Clone)] struct PixelDecoder { layout: PackedLayout, red_mask: u32, green_mask: u32, blue_mask: u32, + direct_colors: Option>, } impl PixelCatalog { @@ -235,14 +235,24 @@ impl PixelCatalog { }) } - fn decoder(&self, w: u32, h: u32, depth: u8, visual_id: u32) -> Result { + fn decoder( + &self, + w: u32, + h: u32, + depth: u8, + visual_id: u32, + query_colormap: impl FnOnce(&Visualtype) -> Result>, + ) -> Result { let layout = self.packed_layout(w, h, depth)?; let visual = self .visuals .iter() .find(|visual| visual.visual_id == visual_id) .ok_or_else(|| anyhow!("unknown X11 visual 0x{visual_id:x} for depth {depth}"))?; - if visual.class != VisualClass::TRUE_COLOR { + if !matches!( + visual.class, + VisualClass::TRUE_COLOR | VisualClass::DIRECT_COLOR + ) { bail!( "unsupported X11 visual class {:?} for visual 0x{visual_id:x}", visual.class @@ -261,15 +271,65 @@ impl PixelCatalog { { bail!("overlapping RGB masks for X11 visual 0x{visual_id:x}"); } + let direct_colors = if visual.class == VisualClass::DIRECT_COLOR { + Some(query_colormap(visual)?) + } else { + None + }; Ok(PixelDecoder { layout, red_mask: visual.red_mask, green_mask: visual.green_mask, blue_mask: visual.blue_mask, + direct_colors, }) } } +/// DirectColor masks address separate color ramps, not RGB intensities. Fetch +/// them from the window on every frame: applications can replace the colormap +/// or change its entries without changing the pixels or the visual. +fn query_direct_colors( + conn: &x11rb::rust_connection::RustConnection, + window: u32, + visual: &Visualtype, +) -> Result> { + use x11rb::protocol::xproto::ConnectionExt as _; + + let masks = [visual.red_mask, visual.green_mask, visual.blue_mask]; + let entries = masks + .iter() + .map(|mask| (mask >> mask.trailing_zeros()) + 1) + .max() + .unwrap_or(0); + if entries == 0 || entries > u32::from(visual.colormap_entries) { + bail!("DirectColor masks exceed the visual's colormap entries"); + } + let attributes = conn.get_window_attributes(window)?.reply()?; + if attributes.visual != visual.visual_id { + bail!("X11 capture visual differs from window visual"); + } + let mut colors = Vec::with_capacity(entries as usize); + // Fit ordinary X11 request limits even for 16-bit component maps. + for start in (0..entries).step_by(1024) { + let pixels: Vec<_> = (start..entries.min(start + 1024)) + .map(|index| { + masks.iter().fold(0, |pixel, mask| { + pixel | ((index << mask.trailing_zeros()) & mask) + }) + }) + .collect(); + let reply = conn.query_colors(attributes.colormap, &pixels)?.reply()?; + if reply.colors.len() != pixels.len() { + bail!("X11 QueryColors returned an incomplete DirectColor map"); + } + colors.extend(reply.colors.iter().map(|color| { + [color.red, color.green, color.blue].map(|value| ((u32::from(value) + 128) / 257) as u8) + })); + } + Ok(colors) +} + fn checked_capture_dimensions(w: u32, h: u32) -> Result<()> { if w == 0 || h == 0 { bail!("window geometry is 0x0"); @@ -328,12 +388,17 @@ fn packed_zpixmap_to_png(data: &[u8], w: u32, h: u32, decoder: PixelDecoder) -> } else { bail!("unsupported X11 image byte order"); }; - rgba.extend_from_slice(&[ - component_to_u8(pixel, decoder.red_mask), - component_to_u8(pixel, decoder.green_mask), - component_to_u8(pixel, decoder.blue_mask), - 255, - ]); + let masks = [decoder.red_mask, decoder.green_mask, decoder.blue_mask]; + for (channel, mask) in masks.into_iter().enumerate() { + let value = match &decoder.direct_colors { + Some(colors) => { + colors[((pixel & mask) >> mask.trailing_zeros()) as usize][channel] + } + None => component_to_u8(pixel, mask), + }; + rgba.push(value); + } + rgba.push(255); } } cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h) @@ -647,7 +712,11 @@ impl XShmSession { data, w, h, - decoder: self.pixels.decoder(w, h, reply.depth, reply.visual)?, + decoder: self + .pixels + .decoder(w, h, reply.depth, reply.visual, |visual| { + query_direct_colors(&self.conn, window, visual) + })?, }) } } @@ -873,7 +942,9 @@ impl XGetImageSession { data: img.data, w, h, - decoder: self.pixels.decoder(w, h, img.depth, img.visual)?, + decoder: self.pixels.decoder(w, h, img.depth, img.visual, |visual| { + query_direct_colors(&self.conn, window, visual) + })?, }) } } @@ -890,39 +961,12 @@ pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> { // `CUA_DRIVER_RS_DEDUP_AUDIT.md`. RGBA-encoding callers below now go // through `cua_driver_core::image_utils::encode_rgba_to_png`. -/// Capture the primary display (root window) as raw PNG bytes. -/// -/// Dispatch: -/// - Native Wayland (`CUA_DRIVER_RS_ENABLE_WAYLAND=1` + Wayland session): -/// routes through [`crate::wayland::screenshot_display_dispatch`] which -/// owns the complete GNOME helper → wlroots screencopy → -/// ext-image-copy-capture-v1 → portal Screenshot → X11 cascade. An -/// available GNOME helper's capture failure is terminal. -/// - X11 / Wayland-disabled: ImageMagick `import` → x11rb `XGetImage`. +/// Capture the X11 primary display as raw PNG bytes. pub fn screenshot_display_bytes() -> Result> { - screenshot_display_bytes_with_dispatch( - crate::wayland::is_wayland(), - crate::wayland::screenshot_display_dispatch, - screenshot_display_bytes_x11, - ) + crate::require_x11()?; + screenshot_display_bytes_x11() } -fn screenshot_display_bytes_with_dispatch( - wayland_enabled: bool, - wayland_capture: impl FnOnce() -> Result>, - x11_capture: impl FnOnce() -> Result>, -) -> Result> { - if wayland_enabled { - wayland_capture() - } else { - x11_capture() - } -} - -/// X11-only display capture path — extracted so the wayland cascade in -/// [`crate::wayland::screenshot_display_dispatch`] can call it as a final -/// fallback without re-entering [`screenshot_display_bytes`] (which would -/// loop forever once we're on Wayland). pub(crate) fn screenshot_display_bytes_x11() -> Result> { // Try `import -window root png:-` (ImageMagick). if let Ok(bytes) = capture_import_target("root") { @@ -1023,6 +1067,7 @@ mod tests { red_mask: 0x00ff_0000, green_mask: 0x0000_ff00, blue_mask: 0x0000_00ff, + direct_colors: None, }; // Width 3 at 24bpp with 32-bit scanline padding: 9 payload bytes + // 3 ignored padding bytes per row. Colors are deliberately varied. @@ -1052,6 +1097,7 @@ mod tests { red_mask: 0xf800, green_mask: 0x07e0, blue_mask: 0x001f, + direct_colors: None, }; let packed = [0xf8, 0x00, 0x07, 0xe0]; let png = packed_zpixmap_to_png(&packed, 2, 1, decoder).expect("decode big-endian"); @@ -1078,19 +1124,56 @@ mod tests { }], visuals: vec![valid], }; - let decoder = catalog.decoder(3, 2, 24, 0x21).expect("valid decoder"); + let decoder = catalog + .decoder(3, 2, 24, 0x21, |_| { + panic!("TrueColor does not query a colormap") + }) + .expect("valid decoder"); assert_eq!(decoder.layout.bytes_per_pixel, 3); assert_eq!(decoder.layout.stride, 12); assert_eq!(decoder.layout.len, 24); catalog.visuals[0].red_mask = 0x00f0_f000; let err = catalog - .decoder(3, 2, 24, 0x21) + .decoder(3, 2, 24, 0x21, |_| { + panic!("TrueColor does not query a colormap") + }) .err() .expect("non-contiguous mask must fail"); assert!(format!("{err:#}").contains("not contiguous")); } + #[test] + fn directcolor_decoder_uses_independent_rgb565_indices() { + let catalog = PixelCatalog { + image_byte_order: ImageOrder::MSB_FIRST, + formats: vec![Format { + depth: 16, + bits_per_pixel: 16, + scanline_pad: 32, + }], + visuals: vec![Visualtype { + visual_id: 0x42, + class: VisualClass::DIRECT_COLOR, + bits_per_rgb_value: 8, + colormap_entries: 64, + red_mask: 0xf800, + green_mask: 0x07e0, + blue_mask: 0x001f, + }], + }; + let decoder = catalog + .decoder(1, 1, 16, 0x42, |_| { + Ok((0..64) + .map(|index| [100 + index, 200 - index, index * 2]) + .collect()) + }) + .expect("DirectColor decoder"); + let png = packed_zpixmap_to_png(&[0x08, 0x43, 0xde, 0xad], 1, 1, decoder) + .expect("decode independently indexed channels and skip padding"); + assert_eq!(decode_png_rgba(&png), vec![101, 198, 6, 255]); + } + #[test] fn xid_narrowing_rejects_values_above_x11_range() { assert_eq!(xid_to_window(u64::from(u32::MAX)).unwrap(), u32::MAX); @@ -1215,43 +1298,6 @@ mod tests { )); } - #[test] - fn available_gnome_helper_failure_is_terminal_at_public_boundary() { - let x11_called = Cell::new(false); - - let result = screenshot_display_bytes_with_dispatch( - true, - || Err(anyhow::anyhow!("GNOME compositor helper capture failed")), - || { - x11_called.set(true); - Ok(vec![1, 2, 3]) - }, - ); - - assert_eq!( - result.unwrap_err().to_string(), - "GNOME compositor helper capture failed" - ); - assert!(!x11_called.get(), "public boundary retried X11 capture"); - } - - #[test] - fn wayland_disabled_uses_x11_capture() { - let wayland_called = Cell::new(false); - - let result = screenshot_display_bytes_with_dispatch( - false, - || { - wayland_called.set(true); - Err(anyhow::anyhow!("Wayland capture should not run")) - }, - || Ok(vec![1, 2, 3]), - ); - - assert_eq!(result.unwrap(), vec![1, 2, 3]); - assert!(!wayland_called.get()); - } - #[test] fn xshm_success_short_circuits_other_linux_capture_backends() { use std::rc::Rc; @@ -1463,6 +1509,121 @@ mod tests { .expect("fixture sync reply"); } + /// DirectColor pixels index independently mutable red, green and blue maps. + /// A non-linear map and a map-only update distinguish correct capture from + /// treating the pixel masks as TrueColor intensities or caching the palette. + #[test] + #[ignore = "requires a live X11 server with DirectColor and MIT-SHM 1.2"] + fn live_directcolor_capture_tracks_window_colormap() { + use x11rb::protocol::xproto::{ + ColorFlag, Coloritem, ColormapAlloc, ConnectionExt as _, CreateWindowAux, WindowClass, + }; + use x11rb::rust_connection::RustConnection; + + let display = std::env::var("DISPLAY").expect("DISPLAY must be set"); + let (conn, screen_num) = RustConnection::connect(Some(&display)).expect("connect fixture"); + let screen = &conn.setup().roots[screen_num]; + let visual = screen + .allowed_depths + .iter() + .filter(|depth| depth.depth == 24) + .flat_map(|depth| &depth.visuals) + .find(|visual| { + visual.class == VisualClass::DIRECT_COLOR + && visual.red_mask == 0xff0000 + && visual.green_mask == 0xff00 + && visual.blue_mask == 0xff + }) + .expect("24-bit DirectColor visual"); + let colormap = conn.generate_id().expect("colormap id"); + conn.create_colormap(ColormapAlloc::ALL, colormap, screen.root, visual.visual_id) + .unwrap() + .check() + .unwrap(); + let window = conn.generate_id().expect("window id"); + conn.create_window( + 24, + window, + screen.root, + 0, + 0, + 67, + 43, + 0, + WindowClass::INPUT_OUTPUT, + visual.visual_id, + &CreateWindowAux::new() + .background_pixel(0x175ba8) + .border_pixel(0) + .colormap(colormap), + ) + .unwrap() + .check() + .unwrap(); + conn.map_window(window).unwrap().check().unwrap(); + let fixture = LiveFixture { conn, window }; + paint_live_fixture(&fixture, 67, 43); + let mut shm = XShmSession::connect(display.clone()).expect("connect SHM capture"); + let mut xget = XGetImageSession::connect(display).expect("connect XGetImage capture"); + for phase in 0..2u16 { + let colors: Vec<_> = (0..256u16) + .map(|index| Coloritem { + pixel: u32::from(index) * 0x010101, + red: ((index + 73 + phase * 19) % 256) * 257, + green: (255 - index) * 257, + blue: ((index * 3 + phase * 31) % 256) * 257, + flags: ColorFlag::RED | ColorFlag::GREEN | ColorFlag::BLUE, + }) + .collect(); + fixture + .conn + .store_colors(colormap, &colors) + .unwrap() + .check() + .unwrap(); + for frame in [ + shm.capture_raw(u64::from(window)) + .expect("DirectColor SHM capture"), + xget.capture_raw(u64::from(window)) + .expect("DirectColor XGetImage capture"), + ] { + let png = raw_frame_png(frame); + assert_eq!( + cua_driver_core::image_utils::png_dimensions(&png).unwrap(), + (67, 43) + ); + let rgba = decode_png_rgba(&png); + for (x, indices) in [ + (0, [0x17u16, 0x5b, 0xa8]), + (33, [0xc4, 0x3d, 0x52]), + (66, [0x2d, 0xb8, 0x71]), + ] { + assert_eq!( + &rgba[x * 4..x * 4 + 4], + &[ + ((indices[0] + 73 + phase * 19) % 256) as u8, + (255 - indices[1]) as u8, + ((indices[2] * 3 + phase * 31) % 256) as u8, + 255, + ] + ); + } + } + } + fixture + .conn + .destroy_window(window) + .unwrap() + .check() + .unwrap(); + fixture + .conn + .free_colormap(colormap) + .unwrap() + .check() + .unwrap(); + } + fn raw_frame_png(frame: RawFrame) -> Vec { packed_zpixmap_to_png(&frame.data, frame.w, frame.h, frame.decoder) .expect("encode raw frame") diff --git a/packages/cua-driver/rust/crates/platform-linux/src/health_report.rs b/packages/cua-driver/rust/crates/platform-linux/src/health_report.rs index 0907a352945..787cb03ddea 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/health_report.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/health_report.rs @@ -15,11 +15,6 @@ use cua_driver_core::health_report::{ NAME_SESSION_ACTIVE, NAME_TCC_ACCESSIBILITY, NAME_TCC_SCREEN_RECORDING, }; -/// Doctor entry that surfaces which wlroots manager globals the running -/// Wayland compositor advertises (foreign-toplevel / screencopy / -/// virtual-pointer / wl_shm). Linux-specific; skipped when not on Wayland. -pub const NAME_WAYLAND_BACKEND: &str = "wayland_backend"; - /// Linux canonical check names — same Swift-PR contract, with TCC / /// bundle entries surfaced as `skip("not applicable on Linux")`. The /// full set of entries always appears so cross-platform consumers see @@ -33,7 +28,6 @@ pub const LINUX_CHECK_NAMES: &[&str] = &[ NAME_TCC_SCREEN_RECORDING, NAME_AX_CAPABILITY, NAME_SCREEN_CAPTURE_CAPABILITY, - NAME_WAYLAND_BACKEND, ]; pub struct LinuxHealthProvider; @@ -58,7 +52,6 @@ impl HealthCheckProvider for LinuxHealthProvider { NAME_TCC_SCREEN_RECORDING => skip_not_applicable(NAME_TCC_SCREEN_RECORDING), NAME_AX_CAPABILITY => check_ax_capability().await, NAME_SCREEN_CAPTURE_CAPABILITY => check_screen_capture_capability().await, - NAME_WAYLAND_BACKEND => check_wayland_backend().await, other => CheckEntry::skip( other.to_owned(), "Unknown check name (not implemented on this platform).", @@ -97,28 +90,6 @@ fn skip_not_applicable(name: &str) -> CheckEntry { } async fn check_ax_capability() -> CheckEntry { - // Native Wayland: AT-SPI lives entirely on D-Bus, so the AX prereq is - // `org.a11y.Bus` being reachable on the session bus. Falls back to X11 - // reachability for X / XWayland sessions. - if is_wayland_session() { - let bus_ok = tokio::task::spawn_blocking(probe_a11y_bus) - .await - .unwrap_or(false); - if bus_ok { - return CheckEntry::pass( - NAME_AX_CAPABILITY, - "Wayland session: org.a11y.Bus reachable on the session bus; \ - AT-SPI inspection will work.", - ); - } - return CheckEntry::fail( - NAME_AX_CAPABILITY, - "Wayland session: org.a11y.Bus is not reachable on the session bus; \ - AT-SPI inspection will fail.", - "Start the AT-SPI bus (`/usr/libexec/at-spi-bus-launcher`) or enable \ - accessibility in your desktop's settings.", - ); - } // Mirror the existing `check_permissions` Linux probe: X11 // connectivity is the AX prerequisite (AT-SPI is over D-Bus, but // input/readback need an X server). Cheap and side-effect-free. @@ -167,11 +138,8 @@ async fn check_ax_capability() -> CheckEntry { cookie isn't discoverable, so X11 connects fail 'Authorization required'. \ Point XAUTHORITY at the Xwayland '-auth' cookie file, or restart the \ cua-driver daemon — it auto-discovers the cookie from /proc at startup." - } else if std::env::var_os("WAYLAND_DISPLAY").is_some() { - "Pure Wayland session: opt into the experimental Wayland backend by \ - setting CUA_DRIVER_RS_ENABLE_WAYLAND=1, or run the target under XWayland." } else { - "Set DISPLAY (X11) — under Wayland, run via XWayland or expose XDG_SESSION_TYPE=x11." + "Only X11 is supported. Set DISPLAY and XAUTHORITY for the target X server, including XWayland." }; CheckEntry::fail( NAME_AX_CAPABILITY, @@ -181,62 +149,6 @@ async fn check_ax_capability() -> CheckEntry { } async fn check_screen_capture_capability() -> CheckEntry { - // Native Wayland: capture flows through a cascade — wlroots screencopy - // (sway / labwc / kwin 5.27+ / hyprland) → ext-image-copy-capture-v1 - // (sway 1.10+ / labwc 0.8+ / niri / KDE 6.2+ / GNOME 47+) → - // xdg-desktop-portal Screenshot (GNOME / KDE / COSMIC + portal backend). - // Report which tier is reachable so users on mutter / kwin don't see a - // misleading "screen capture will fail" just because wlroots isn't - // present. - if is_wayland_session() { - let snap = tokio::task::spawn_blocking(probe_wayland_managers) - .await - .ok() - .and_then(|r| r.ok()); - match &snap { - Some(m) if m.screencopy && m.wl_shm => { - return CheckEntry::pass( - NAME_SCREEN_CAPTURE_CAPABILITY, - "Wayland session: zwlr_screencopy_manager_v1 + wl_shm advertised; \ - native output capture is functional.", - ); - } - Some(m) if m.ext_image_copy_capture && m.ext_output_image_capture_source => { - return CheckEntry::pass( - NAME_SCREEN_CAPTURE_CAPABILITY, - "Wayland session: ext-image-copy-capture-v1 + \ - ext-output-image-capture-source-v1 advertised; cross-DE \ - native output capture is functional.", - ); - } - _ => {} - } - // No wlr screencopy AND no ext-image-copy-capture. Probe the - // portal as the last native tier. The probe is read-only — it - // checks for a name owner on the bus, does NOT take a screenshot. - let portal_ok = tokio::task::spawn_blocking(probe_portal_screenshot) - .await - .ok() - .and_then(|r| r.ok()) - .unwrap_or(false); - if portal_ok { - return CheckEntry::pass( - NAME_SCREEN_CAPTURE_CAPABILITY, - "Wayland session: no native screencopy globals, but \ - xdg-desktop-portal Screenshot is reachable; capture will go \ - through the portal (one consent prompt per session).", - ); - } - return CheckEntry::fail( - NAME_SCREEN_CAPTURE_CAPABILITY, - "Wayland session: no native screencopy globals, and \ - xdg-desktop-portal is not reachable on the session bus; \ - screen capture will fail.", - "Use a wlroots-based compositor (sway, labwc, hyprland, kwin 5.27+) \ - or install xdg-desktop-portal-gnome / xdg-desktop-portal-kde / \ - xdg-desktop-portal-wlr.", - ); - } // On Linux the canonical capture path is X11 GetImage / xwd / scrot // — all require an open X11 connection. The probe is the same one // we use for ax_capability, but the consumer-facing message and @@ -251,12 +163,7 @@ async fn check_screen_capture_capability() -> CheckEntry { "X11 reachable; screen capture path is functional.", ); } - let hint = if std::env::var_os("WAYLAND_DISPLAY").is_some() { - "Pure Wayland session: opt into the experimental Wayland backend by \ - setting CUA_DRIVER_RS_ENABLE_WAYLAND=1, or run the target under XWayland." - } else { - "Set DISPLAY (X11). Pure Wayland sessions require an XWayland bridge for capture." - }; + let hint = "Only X11 is supported. Set DISPLAY and XAUTHORITY for capture."; CheckEntry::fail( NAME_SCREEN_CAPTURE_CAPABILITY, "X11 is not reachable; screen capture will fail.", @@ -264,167 +171,6 @@ async fn check_screen_capture_capability() -> CheckEntry { ) } -/// `wayland_backend` doctor entry — surfaces which wlroots manager globals -/// the running compositor advertises. Skipped on non-Wayland sessions and -/// when the experimental Wayland backend isn't opted into, so X11 users see -/// a neutral skip rather than a confusing failure. -async fn check_wayland_backend() -> CheckEntry { - if std::env::var_os("WAYLAND_DISPLAY").is_none() { - return CheckEntry::skip( - NAME_WAYLAND_BACKEND.to_owned(), - "No WAYLAND_DISPLAY in the environment — not a Wayland session.", - ); - } - if !wayland_opt_in_enabled() { - return CheckEntry::skip( - NAME_WAYLAND_BACKEND.to_owned(), - format!( - "Wayland session detected, but the experimental backend is opt-in. \ - Set {}=1 to enable native Wayland and re-run doctor.", - wayland_env_name() - ), - ); - } - let snap = match tokio::task::spawn_blocking(probe_wayland_managers).await { - Ok(Ok(snap)) => snap, - Ok(Err(e)) => { - return CheckEntry::fail( - NAME_WAYLAND_BACKEND, - format!("Failed to connect to the Wayland compositor: {e}"), - "Verify WAYLAND_DISPLAY points at a running compositor socket.", - ); - } - Err(e) => { - return CheckEntry::fail( - NAME_WAYLAND_BACKEND, - format!("Wayland probe task error: {e}"), - "Re-run the doctor; if it persists, file a bug with the doctor output.", - ); - } - }; - let remote_desktop_portal_reachable = if portal_input_enabled() { - tokio::task::spawn_blocking(probe_portal_remote_desktop) - .await - .ok() - .and_then(|r| r.ok()) - .unwrap_or(false) - } else { - false - }; - classify_wayland_backend( - &snap, - portal_input_enabled(), - remote_desktop_portal_reachable, - target_activation_available(), - ) -} - -fn classify_wayland_backend( - snap: &WaylandManagers, - portal_libei_enabled: bool, - remote_desktop_portal_reachable: bool, - target_activation_available: bool, -) -> CheckEntry { - let msg = format!( - "foreign-toplevel={ftl}, screencopy={cap}, ext-image-copy={ext_cap}, \ - ext-output-source={ext_src}, virtual-pointer={vp}, wl_shm={shm}", - ftl = snap.foreign_toplevel, - cap = snap.screencopy, - ext_cap = snap.ext_image_copy_capture, - ext_src = snap.ext_output_image_capture_source, - vp = snap.virtual_pointer, - shm = snap.wl_shm, - ); - if snap.foreign_toplevel && snap.screencopy && snap.virtual_pointer && snap.wl_shm { - return CheckEntry::pass( - NAME_WAYLAND_BACKEND, - format!("All wlroots manager globals advertised ({msg})."), - ); - } - if !snap.virtual_pointer { - if remote_desktop_portal_reachable && target_activation_available { - return CheckEntry::pass( - NAME_WAYLAND_BACKEND, - format!( - "No wlroots virtual-pointer advertised ({msg}), but this \ - portal/libei build can reach the RemoteDesktop portal \ - (proxy reachability only — the full create_session → \ - select_devices → start → connect_to_eis handshake is NOT \ - exercised here, to avoid a consent prompt on every doctor \ - run, so this is not a guarantee that injection succeeds). \ - The compositor helper also provides verified target \ - activation before focus-bound portal input." - ), - ); - } - if remote_desktop_portal_reachable { - return CheckEntry::fail( - NAME_WAYLAND_BACKEND, - format!( - "The RemoteDesktop portal is reachable, but this compositor \ - has no verified target-activation adapter ({msg}). Portal/libei \ - input is global and would otherwise affect whichever window is \ - focused, potentially the wrong application, so cua-driver \ - refuses foreground dispatch." - ), - "On GNOME, install and enable the bundled WinRects Shell helper, then \ - log out and back in. KDE foreground input remains unavailable until \ - a target-addressable KWin activation adapter is installed; AX actions \ - and exact background refusals remain usable.", - ); - } - if portal_libei_enabled { - return CheckEntry::fail( - NAME_WAYLAND_BACKEND, - format!( - "No wlroots virtual-pointer advertised ({msg}) and the \ - portal/libei RemoteDesktop backend is compiled in but not \ - reachable on this session; clicks and key presses have no \ - native Wayland input backend." - ), - "Ensure xdg-desktop-portal and a desktop backend such as \ - xdg-desktop-portal-gnome or xdg-desktop-portal-kde are running \ - on the session bus, or run under XWayland.", - ); - } - return CheckEntry::fail( - NAME_WAYLAND_BACKEND, - format!( - "Input injection has no backend on this compositor ({msg}): it \ - advertises no zwlr_virtual_pointer and this build was compiled \ - without libei/portal support, so clicks and key presses will not \ - be delivered (the agent cursor still renders). list_windows and \ - screen capture are unaffected." - ), - "Use the portal-enabled Linux build (compiled with --features \ - portal-input) for input on KDE Plasma / GNOME, or a wlroots \ - compositor (sway, labwc, hyprland) where zwlr_virtual_pointer exists.", - ); - } - // Partial-pass: list_windows + capture both work, but some optional - // wlroots globals are absent. Require `wl_shm` here too — - // `check_screen_capture_capability` gates on both `screencopy && wl_shm`, - // so excluding `wl_shm` from the partial-pass verdict would let the - // matrices disagree on degenerate compositors that omit it. - if snap.foreign_toplevel && snap.screencopy && snap.wl_shm { - return CheckEntry::pass( - NAME_WAYLAND_BACKEND, - format!( - "Core wlroots manager globals available; some optional globals missing ({msg})." - ), - ); - } - CheckEntry::fail( - NAME_WAYLAND_BACKEND, - format!( - "Compositor does not advertise a complete native Wayland backend \ - set ({msg})." - ), - "Use a wlroots-based compositor (sway, labwc, hyprland), a portal/libei \ - build on GNOME/KDE, or run under XWayland.", - ) -} - // ── Probes ─────────────────────────────────────────────────────────────────── #[cfg(target_os = "linux")] @@ -438,8 +184,7 @@ fn probe_x11_connect() -> bool { } /// Probe whether `org.a11y.Bus` is reachable on the session bus — the -/// canonical AT-SPI prerequisite on Wayland, where there is no X server to -/// stand in. Returns false on any error (no session bus, no a11y service, +/// canonical AT-SPI prerequisite. Returns false on any error (no session bus, no a11y service, /// timeout, etc.) so the doctor message stays simple. #[cfg(target_os = "linux")] pub(crate) fn probe_a11y_bus() -> bool { @@ -452,22 +197,26 @@ pub(crate) fn probe_a11y_bus() -> bool { Err(_) => return false, }; rt.block_on(async { - let bus = match zbus::Connection::session().await { - Ok(b) => b, - Err(_) => return false, - }; - let proxy = match zbus::fdo::DBusProxy::new(&bus).await { - Ok(p) => p, - Err(_) => return false, - }; - // `name_has_owner` is the cheap direct existence check — avoids - // pulling the entire session-bus name registry just to look up one - // entry. Matches the pattern in `wayland::portal_screenshot::probe_portal`. - let bus_name = match "org.a11y.Bus".try_into() { - Ok(n) => n, - Err(_) => return false, - }; - proxy.name_has_owner(bus_name).await.unwrap_or(false) + tokio::time::timeout(std::time::Duration::from_secs(1), async { + let bus = match zbus::Connection::session().await { + Ok(b) => b, + Err(_) => return false, + }; + let proxy = match zbus::fdo::DBusProxy::new(&bus).await { + Ok(p) => p, + Err(_) => return false, + }; + // `name_has_owner` is the cheap direct existence check — avoids + // pulling the entire session-bus name registry just to look up one + // entry. + let bus_name = match "org.a11y.Bus".try_into() { + Ok(n) => n, + Err(_) => return false, + }; + proxy.name_has_owner(bus_name).await.unwrap_or(false) + }) + .await + .unwrap_or(false) }) } @@ -476,150 +225,7 @@ pub(crate) fn probe_a11y_bus() -> bool { false } -/// True when the Wayland backend is opted in and a Wayland session is -/// active. Wrapper around `wayland::is_wayland()` so non-Linux builds of -/// this file compile (the `wayland` module is gated on `target_os = linux`). -#[cfg(target_os = "linux")] -fn is_wayland_session() -> bool { - crate::wayland::is_wayland() -} - -#[cfg(not(target_os = "linux"))] -fn is_wayland_session() -> bool { - false -} - -#[cfg(target_os = "linux")] -fn wayland_opt_in_enabled() -> bool { - crate::wayland::wayland_enabled() -} - -#[cfg(not(target_os = "linux"))] -fn wayland_opt_in_enabled() -> bool { - false -} - -#[cfg(target_os = "linux")] -fn wayland_env_name() -> &'static str { - crate::wayland::ENABLE_WAYLAND_ENV -} - -#[cfg(not(target_os = "linux"))] -fn wayland_env_name() -> &'static str { - "CUA_DRIVER_RS_ENABLE_WAYLAND" -} - -#[cfg(target_os = "linux")] -fn portal_input_enabled() -> bool { - crate::wayland::PORTAL_INPUT_ENABLED -} - -#[cfg(not(target_os = "linux"))] -fn portal_input_enabled() -> bool { - false -} - -#[cfg(target_os = "linux")] -fn target_activation_available() -> bool { - crate::wayland::shell_helper::list_windows(None).is_some() -} - -#[cfg(not(target_os = "linux"))] -fn target_activation_available() -> bool { - false -} - -#[cfg(target_os = "linux")] -type WaylandManagers = crate::wayland::WaylandManagers; - -/// Snapshot of wlroots manager globals. The non-Linux stub returns an empty -/// snapshot so off-platform builds stay green; doctor short-circuits before -/// reaching it via [`is_wayland_session`]. -#[cfg(target_os = "linux")] -fn probe_wayland_managers() -> anyhow::Result { - crate::wayland::probe_managers() -} - -#[cfg(not(target_os = "linux"))] -fn probe_wayland_managers() -> anyhow::Result { - Ok(WaylandManagers::default()) -} - -#[cfg(target_os = "linux")] -fn probe_portal_screenshot() -> anyhow::Result { - crate::wayland::portal_screenshot::probe_portal() -} - -#[cfg(not(target_os = "linux"))] -fn probe_portal_screenshot() -> anyhow::Result { - Ok(false) -} - -#[cfg(target_os = "linux")] -fn probe_portal_remote_desktop() -> anyhow::Result { - #[cfg(feature = "portal-input")] - { - use ashpd::desktop::remote_desktop::RemoteDesktop; - - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| { - anyhow::anyhow!("failed to build tokio runtime for RemoteDesktop probe: {e}") - })?; - - rt.block_on(async { - let probe = tokio::time::timeout(crate::wayland::portal::PROBE_TIMEOUT, async { - let connection = crate::wayland::portal::fresh_session_connection().await?; - RemoteDesktop::with_connection(connection) - .await - .map_err(anyhow::Error::from) - }) - .await; - - match probe { - Err(_) => Err(anyhow::anyhow!("portal RemoteDesktop probe timed out")), - Ok(Ok(_)) => Ok(true), - Ok(Err(e)) => { - let msg = format!("{e}"); - if msg.contains("ServiceUnknown") - || msg.contains("NameHasNoOwner") - || msg.contains("NotFound") - { - Ok(false) - } else { - Err(anyhow::anyhow!("portal RemoteDesktop probe failed: {e}")) - } - } - } - }) - } - #[cfg(not(feature = "portal-input"))] - { - Ok(false) - } -} - -#[cfg(not(target_os = "linux"))] -fn probe_portal_remote_desktop() -> anyhow::Result { - Ok(false) -} - -/// Stub of `wayland::WaylandManagers` so off-Linux builds compile. Always -/// reports nothing advertised — non-Linux code paths never call this. -#[cfg(not(target_os = "linux"))] -#[derive(Default, Clone, Debug)] -struct WaylandManagers { - foreign_toplevel: bool, - screencopy: bool, - ext_image_copy_capture: bool, - ext_output_image_capture_source: bool, - virtual_pointer: bool, - wl_shm: bool, -} - -/// `PRETTY_NAME=` out of `/etc/os-release`. Fails to "Linux" so the -/// downstream message stays human-readable. +/// PRETTY_NAME from /etc/os-release; callers fall back to Linux. fn os_release_pretty_name() -> Option { let s = std::fs::read_to_string("/etc/os-release").ok()?; for line in s.lines() { @@ -675,72 +281,6 @@ mod tests { } } - #[test] - fn wayland_backend_passes_on_non_wlroots_when_portal_libei_backend_is_reachable() { - let snap = WaylandManagers { - foreign_toplevel: false, - screencopy: false, - ext_image_copy_capture: false, - ext_output_image_capture_source: false, - virtual_pointer: false, - wl_shm: true, - }; - - let entry = classify_wayland_backend(&snap, true, true, true); - - assert_eq!(entry.status, CheckStatus::Pass); - assert!(entry.message.contains("portal/libei"), "{}", entry.message); - assert!(entry.message.contains("RemoteDesktop"), "{}", entry.message); - } - - #[test] - fn wayland_backend_fails_on_non_wlroots_when_portal_libei_backend_is_unreachable() { - let snap = WaylandManagers { - foreign_toplevel: false, - screencopy: false, - ext_image_copy_capture: false, - ext_output_image_capture_source: false, - virtual_pointer: false, - wl_shm: true, - }; - - let entry = classify_wayland_backend(&snap, true, false, false); - - assert_eq!(entry.status, CheckStatus::Fail); - assert!(entry.message.contains("portal/libei"), "{}", entry.message); - assert!(entry - .hint - .as_deref() - .unwrap_or("") - .contains("xdg-desktop-portal")); - } - - #[test] - fn wayland_backend_fails_closed_when_portal_input_cannot_target_a_window() { - let snap = WaylandManagers { - foreign_toplevel: false, - screencopy: false, - ext_image_copy_capture: false, - ext_output_image_capture_source: false, - virtual_pointer: false, - wl_shm: true, - }; - - let entry = classify_wayland_backend(&snap, true, true, false); - - assert_eq!(entry.status, CheckStatus::Fail); - assert!( - entry.message.contains("target-activation"), - "{}", - entry.message - ); - assert!( - entry.message.contains("wrong application"), - "{}", - entry.message - ); - } - #[tokio::test] async fn invoke_full_run_produces_linux_check_set() { let provider = Arc::new(LinuxHealthProvider); diff --git a/packages/cua-driver/rust/crates/platform-linux/src/input/delivery.rs b/packages/cua-driver/rust/crates/platform-linux/src/input/delivery.rs index d32bbd58b5a..c8bf31ddd3a 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/input/delivery.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/input/delivery.rs @@ -1,31 +1,7 @@ //! Shared delivery-mode logic for cua-driver Linux input tools. //! -//! Mirrors macOS `tools::DeliveryMode` and Windows `input::delivery`: each -//! input tool accepts an optional `delivery_mode` field with exactly two modes -//! — the agent-selected rung of the best-effort-background ladder, passed per -//! call (never a stored setting): -//! -//! - `background` (DEFAULT) — inject without activating/raising the target. -//! - **X11**: XTEST / XSendEvent / XInput2 MPX master-pointer no-focus-steal -//! path (see `input::mod`). Lands on a backgrounded window without stealing -//! focus — the direct analogue of macOS CGEvent / Windows PostMessage. -//! - **Wayland**: libei via xdg-desktop-portal (`wayland::libei`). Wayland's -//! security model has NO arbitrary per-window background targeting — libei -//! injects to the compositor's input focus, so "background" cannot aim at a -//! specific non-focused window the way X11/macOS/Windows can (this is a -//! platform constraint, reported honestly, like macOS pixel input being -//! driver-unverifiable). When no libei backend is available -//! (`PORTAL_INPUT_ENABLED == false`) the tool returns a structured -//! `background_unavailable` error so the caller can escalate to foreground. -//! -//! - `foreground` — activate the target first, inject, then restore the prior -//! active window. -//! - **X11**: EWMH `_NET_ACTIVE_WINDOW` client message (the `wmctrl -a` -//! equivalent) raise + activate. -//! - **Wayland**: compositor-specific activate where supported; else the -//! libei-to-focus path with an honest note. -//! The agent's explicit last resort; a brief focus swap unless the target was -//! already active. Matches the macOS / Windows `foreground` rung. +//! X11 background delivery uses semantic actions, XSendEvent or MPX. Foreground +//! delivery activates the target and restores the prior active window. use serde_json::Value; @@ -34,7 +10,7 @@ use serde_json::Value; /// macOS `tools::DeliveryMode` and Windows `input::delivery::DeliveryMode`. #[derive(Copy, Clone, Debug, PartialEq, Eq, Default)] pub enum DeliveryMode { - /// Inject without activating the target (X11 XTEST/MPX; Wayland libei-to-focus). + /// Inject without activating the target (X11 XTEST/MPX). #[default] Background, /// Activate the target, inject, restore prior active window. @@ -55,11 +31,7 @@ impl DeliveryMode { /// Parse from a tool's JSON args, reading the `delivery_mode` field. pub fn from_args(args: &Value) -> Self { - if args.get("app_context").and_then(Value::as_bool) == Some(true) { - Self::Foreground - } else { - Self::parse(args.get("delivery_mode").and_then(|v| v.as_str())) - } + Self::parse(args.get("delivery_mode").and_then(|v| v.as_str())) } pub fn is_foreground(self) -> bool { @@ -71,34 +43,16 @@ impl DeliveryMode { /// input tool's `input_schema.properties.delivery_mode`. Two modes, matching /// the macOS / Windows surface. pub fn delivery_mode_schema() -> Value { - // Source the SHAPE (`type` + `enum`) from the shared canon so it can't drift - // from macOS / Windows, while keeping the Linux-specific Wayland/X11 prose - // (the gate compares shape only — descriptions may vary per platform). The - // `default` advertisement is additive on top of the canon shape. let mut v = cua_driver_core::tool_schema::delivery_mode_schema_with( - "Input delivery mode. 'background' (default) never activates or raises \ - the target window. On X11 it injects via XTEST / the XInput2 master \ - pointer (no focus steal). On Wayland it goes through libei + \ - xdg-desktop-portal, which injects to the compositor's input focus — \ - Wayland's security model has no per-window background targeting, so a \ - specific non-focused window cannot be aimed at; when no libei backend \ - is available the tool returns a structured background_unavailable \ - error. 'foreground' is the explicit escalation: activate the target \ - (X11 _NET_ACTIVE_WINDOW; Wayland compositor activate), inject, then \ - restore the prior active window — a brief focus swap unless the \ - target was already active. Matches the macOS / Windows delivery_mode \ - surface.", + "Input delivery mode. 'background' (default) does not activate the target. 'foreground' activates the exact X11 window for input and then restores the previous foreground.", ); v["default"] = serde_json::json!("background"); v } -/// Reason a `background` delivery cannot be performed on Wayland. +/// Reason background input is unavailable on the exact X11 target. #[derive(Copy, Clone, Debug)] pub enum BackgroundUnavailable { - /// No libei backend (built without `portal-input`, or the portal session - /// was denied / unavailable). Input has no actuator at all. - NoLibeiBackend, /// X11/Chromium does not accept synthetic pointer or keyboard input /// addressed to an occluded, unfocused renderer without briefly moving /// focus, which background delivery forbids. @@ -108,24 +62,21 @@ pub enum BackgroundUnavailable { /// WebKitGTK rejects synthetic XSendEvent input and no real target-addressed /// pointer backend is available in this session. WebKitSyntheticInput, + /// Openbox promotes an MPX master's core FocusIn to global activation. + WindowManagerFocus, } impl BackgroundUnavailable { fn code(self) -> &'static str { match self { - Self::NoLibeiBackend => "background_unavailable", Self::ChromiumInput => "background_unavailable", Self::FocusedInputOnly => "background_unavailable", Self::WebKitSyntheticInput => "background_unavailable", + Self::WindowManagerFocus => "background_unavailable", } } fn detail(self) -> &'static str { match self { - Self::NoLibeiBackend => { - "no libei input backend on this Wayland compositor (built without \ - portal-input, or the xdg-desktop-portal RemoteDesktop session was \ - unavailable/denied): synthetic input has no actuator" - } Self::ChromiumInput => { "Chromium/Electron does not accept pointer or keyboard input \ addressed to an occluded, unfocused renderer through X11 \ @@ -137,12 +88,23 @@ impl BackgroundUnavailable { Self::WebKitSyntheticInput => { "WebKitGTK rejects synthetic XSendEvent input and this session has no real target-addressed pointer backend" } + Self::WindowManagerFocus => { + "Openbox treats virtual keyboard focus as global window activation" + } } } } +impl std::fmt::Display for BackgroundUnavailable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.detail()) + } +} + +impl std::error::Error for BackgroundUnavailable {} + /// Build the structured `background_unavailable` error returned when a -/// `delivery_mode:"background"` injection has no Wayland backend. Mirrors the +/// background injection has no usable backend. Mirrors the /// Windows silent-drop error so callers branch on `code` identically. pub fn background_unavailable_error( reason: BackgroundUnavailable, @@ -155,6 +117,7 @@ pub fn background_unavailable_error( )) .with_structured(serde_json::json!({ "code": reason.code(), + "sent": false, "detail": detail, "suggestion": "Retry this action with delivery_mode:\"foreground\".", "escalation": { @@ -228,7 +191,7 @@ mod tests { #[test] fn background_unavailable_error_carries_code() { - let r = background_unavailable_error(BackgroundUnavailable::NoLibeiBackend); + let r = background_unavailable_error(BackgroundUnavailable::FocusedInputOnly); assert_eq!(r.is_error, Some(true)); assert_eq!( r.structured_content.as_ref().unwrap()["code"], diff --git a/packages/cua-driver/rust/crates/platform-linux/src/input/focus_guard.rs b/packages/cua-driver/rust/crates/platform-linux/src/input/focus_guard.rs new file mode 100644 index 00000000000..3fc38fdcede --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-linux/src/input/focus_guard.rs @@ -0,0 +1,1063 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Background non-invasiveness safety net for X11. +//! +//! A `delivery_mode:"background"` action must leave the user's desktop as it +//! found it: the core keyboard focus (`XGetInputFocus`), the window manager's +//! active window (`_NET_ACTIVE_WINDOW`) and the top of the stacking order. +//! The focus-free routes (MPX virtual keyboard, AT-SPI actions) never touch +//! those themselves, but the *application* may: GTK/VCL activate their +//! toplevel when a menu opens, mutter focuses a freshly mapped transient +//! dialog, Nautilus presents its window after a navigation shortcut. +//! +//! [`FocusSnapshot::capture`] records the three signals before an action and +//! [`FocusSnapshot::restore_if_changed`] re-asserts them afterwards when they +//! moved *away from the user*: the previously active window is re-activated +//! through EWMH (so the WM's own bookkeeping and stacking follow) and the +//! previous core focus is re-set. +//! +//! A move that stays inside the target application is not an intrusion: when +//! the target app already owned the focus and it opens a dialog that mutter +//! focuses, the user (or the agent) asked for that dialog. Re-activating the +//! main window there would undo the app's own behaviour and send the next +//! pid-only keystrokes to the wrong window, so the guard leaves it and reports +//! `focus_outcome: same_app_dialog` instead. The outcome is reported, never +//! hidden: a toolkit keyboard grab held by an open menu cannot be undone +//! without closing the menu, so it is surfaced as `grab_held_by`. + +use anyhow::{anyhow, Result}; +use std::collections::HashSet; +use std::time::{Duration, Instant}; +use x11rb::connection::Connection as _; +use x11rb::protocol::xproto::{ + AtomEnum, ClientMessageEvent, ConnectionExt as _, CreateWindowAux, EventMask, InputFocus, + MapState, PropMode, Window, WindowClass, +}; +use x11rb::protocol::Event; +use x11rb::rust_connection::RustConnection; +use x11rb::COPY_DEPTH_FROM_PARENT; + +/// How long the guard watches for a late focus change after delivery (a +/// transient dialog is mapped and focused by the WM a beat after the action). +const SETTLE_WATCH: Duration = Duration::from_millis(220); +/// Longer watch once a new top-level appeared during the short one while the +/// focus belonged to *another* application: a dialog (LibreOffice's take +/// ~1 s to build) is focused by the WM only when mapped, and that steal must +/// be undone. When the target app already owns the focus the new window is +/// its own and there is nothing to wait for. +const SETTLE_WATCH_NEW_WINDOW: Duration = Duration::from_millis(900); +const SETTLE_POLL: Duration = Duration::from_millis(30); +/// Bound on the restore loop: re-activation, verification, one re-send. +// Together with the extended settle watch this bounds the guard at +// ~1.5 s worst case (menubar clicks were spending 4-9 s here while a popup's +// keyboard grab kept every re-activation from sticking). +const RESTORE_BUDGET: Duration = Duration::from_millis(600); +const RESTORE_POLL: Duration = Duration::from_millis(50); +/// Consecutive stable polls before the restore is called done. +const STABLE_POLLS: u32 = 3; +/// Bound on the ancestor / transient walk when attributing a window to a pid. +const OWNER_WALK_LIMIT: usize = 16; + +struct Atoms { + net_active_window: u32, + net_client_list_stacking: u32, + net_wm_pid: u32, + net_wm_name: u32, + utf8_string: u32, +} + +struct X { + conn: RustConnection, + root: Window, + atoms: Atoms, +} + +impl X { + fn open() -> Result { + let (conn, screen) = RustConnection::connect(None) + .map_err(|e| anyhow!("focus guard: cannot open DISPLAY: {e}"))?; + let root = conn.setup().roots[screen].root; + let intern = + |name: &[u8]| -> Result { Ok(conn.intern_atom(false, name)?.reply()?.atom) }; + let atoms = Atoms { + net_active_window: intern(b"_NET_ACTIVE_WINDOW")?, + net_client_list_stacking: intern(b"_NET_CLIENT_LIST_STACKING")?, + net_wm_pid: intern(b"_NET_WM_PID")?, + net_wm_name: intern(b"_NET_WM_NAME")?, + utf8_string: intern(b"UTF8_STRING")?, + }; + Ok(Self { conn, root, atoms }) + } + + fn window_property(&self, window: Window, atom: u32, len: u32) -> Vec { + self.conn + .get_property(false, window, atom, AtomEnum::ANY, 0, len) + .ok() + .and_then(|c| c.reply().ok()) + .and_then(|r| r.value32().map(|v| v.collect())) + .unwrap_or_default() + } + + fn active_window(&self) -> Option { + self.window_property(self.root, self.atoms.net_active_window, 1) + .first() + .copied() + .filter(|w| *w != 0) + } + + fn client_list(&self) -> Vec { + self.window_property(self.root, self.atoms.net_client_list_stacking, u32::MAX) + } + + fn stacking_top(&self) -> Option { + self.client_list().last().copied().filter(|w| *w != 0) + } + + fn core_focus(&self) -> (Window, InputFocus) { + match self + .conn + .get_input_focus() + .ok() + .and_then(|c| c.reply().ok()) + { + Some(reply) => (reply.focus, reply.revert_to), + None => (0, InputFocus::NONE), + } + } + + fn window_pid(&self, window: Window) -> Option { + self.window_property(window, self.atoms.net_wm_pid, 1) + .first() + .copied() + .filter(|p| *p != 0) + } + + fn transient_for(&self, window: Window) -> Option { + self.window_property(window, AtomEnum::WM_TRANSIENT_FOR.into(), 1) + .first() + .copied() + .filter(|w| *w != 0 && *w != self.root) + } + + /// The window still exists and is mapped (a destroyed dialog fails both). + fn window_viewable(&self, window: Window) -> bool { + self.conn + .get_window_attributes(window) + .ok() + .and_then(|c| c.reply().ok()) + .is_some_and(|a| a.map_state == MapState::VIEWABLE) + } + + fn parent(&self, window: Window) -> Option { + let tree = self.conn.query_tree(window).ok()?.reply().ok()?; + (tree.parent != 0 && tree.parent != self.root).then_some(tree.parent) + } + + /// The pid that owns `window`: its own `_NET_WM_PID`, else the one of + /// the window it is transient for, else its ancestors' (the core focus + /// often sits on a child of the client toplevel). + fn owner_pid(&self, window: Window) -> Option { + let mut current = window; + for _ in 0..OWNER_WALK_LIMIT { + if current == 0 || current == self.root { + return None; + } + if let Some(pid) = self.window_pid(current) { + return Some(pid); + } + match self.transient_for(current).or_else(|| self.parent(current)) { + Some(next) if next != current => current = next, + _ => return None, + } + } + None + } + + fn window_title(&self, window: Window) -> String { + let read = |atom: u32, ty: u32| -> Option { + let reply = self + .conn + .get_property(false, window, atom, ty, 0, 256) + .ok()? + .reply() + .ok()?; + (!reply.value.is_empty()).then(|| String::from_utf8_lossy(&reply.value).into_owned()) + }; + read(self.atoms.net_wm_name, self.atoms.utf8_string) + .or_else(|| read(AtomEnum::WM_NAME.into(), AtomEnum::STRING.into())) + .unwrap_or_default() + } + + /// Mapped override-redirect children of the root: menus, combo popups, + /// tooltips. Requests are pipelined so a busy desktop costs one round trip. + fn mapped_popups(&self) -> HashSet { + let tree = match self.conn.query_tree(self.root).map(|c| c.reply()) { + Ok(Ok(tree)) => tree, + _ => return HashSet::new(), + }; + let cookies: Vec<_> = tree + .children + .iter() + .filter_map(|w| self.conn.get_window_attributes(*w).ok().map(|c| (*w, c))) + .collect(); + cookies + .into_iter() + .filter_map(|(w, c)| c.reply().ok().map(|a| (w, a))) + .filter(|(_, a)| a.override_redirect && a.map_state == MapState::VIEWABLE) + .map(|(w, _)| w) + .collect() + } + + /// Current server time (PropertyNotify round trip), bounded at 300 ms; + /// `CurrentTime` (0) would lose to focus-stealing prevention. + fn server_time(&self) -> u32 { + let Ok(probe) = self.conn.generate_id() else { + return 0; + }; + let aux = CreateWindowAux::new().event_mask(EventMask::PROPERTY_CHANGE); + if self + .conn + .create_window( + COPY_DEPTH_FROM_PARENT, + probe, + self.root, + -1, + -1, + 1, + 1, + 0, + WindowClass::INPUT_OUTPUT, + 0, + &aux, + ) + .is_err() + { + return 0; + } + let _ = self.conn.change_property( + PropMode::REPLACE, + probe, + self.atoms.net_wm_pid, + AtomEnum::STRING, + 8, + 1, + &[0u8], + ); + let _ = self.conn.flush(); + let deadline = Instant::now() + Duration::from_millis(300); + let mut time = 0; + while Instant::now() < deadline { + match self.conn.poll_for_event() { + Ok(Some(Event::PropertyNotify(e))) if e.window == probe => { + time = e.time; + break; + } + Ok(Some(_)) => continue, + Ok(None) => std::thread::sleep(Duration::from_millis(5)), + Err(_) => break, + } + } + let _ = self.conn.destroy_window(probe); + let _ = self.conn.flush(); + time + } + + fn activate(&self, target: Window, current: Window) { + let event = ClientMessageEvent::new( + 32, + target, + self.atoms.net_active_window, + [2, self.server_time(), current, 0, 0], + ); + let _ = self.conn.send_event( + false, + self.root, + EventMask::SUBSTRUCTURE_REDIRECT | EventMask::SUBSTRUCTURE_NOTIFY, + event, + ); + let _ = self.conn.flush(); + } + + fn set_focus(&self, window: Window, revert_to: InputFocus) { + // BadWindow / BadMatch (destroyed or unmapped meanwhile) are expected + // and swallowed: the verification loop decides what to do next. + if let Ok(cookie) = self + .conn + .set_input_focus(revert_to, window, x11rb::CURRENT_TIME) + { + let _ = cookie.check(); + } + let _ = self.conn.flush(); + } +} + +/// Where a focus move went, relative to the action's target application. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FocusMove { + /// The target app owned the focus before and still does: it opened one + /// of its own windows (a dialog). Not an intrusion; leave it. + SameAppDialog, + /// The focus left the user's application (or arrived in the target from + /// another app): restore it. + OtherApp, +} + +/// The same-app rule: a move is the app's own dialog only when the target +/// pid owned the focus before the action *and* owns it now. A move from a +/// decoy (another pid) into the target's dialog is a steal, as is a move +/// from the target to anything else. +pub fn classify_focus_move( + target_pid: Option, + previous_owner: Option, + new_owner: Option, +) -> FocusMove { + match (target_pid, previous_owner, new_owner) { + (Some(target), Some(previous), Some(new)) if previous == target && new == target => { + FocusMove::SameAppDialog + } + _ => FocusMove::OtherApp, + } +} + +/// What a background action must not change. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FocusSnapshot { + core_focus: Window, + revert_to: InputFocus, + active: Option, + stacking_top: Option, + popups: HashSet, + clients: HashSet, + /// Pid that owned the active window (else the core focus) before the + /// action: the application the user was working in. + previous_owner: Option, +} + +/// A window of the target application that appeared or took the focus. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SameAppWindow { + pub window: u64, + pub title: String, + /// It received the desktop focus (else it was merely mapped). + pub focused: bool, +} + +/// Outcome of the post-action check and restore. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FocusGuardReport { + /// The core focus, active window or stacking top moved after the action. + pub changed: bool, + /// They were moved back (verified stable) before the tool returned. + pub restored: bool, + /// Human-readable list of what moved (`focus 0x..->0x..`, ...). + pub changes: Vec, + /// A new override-redirect popup (menu, combo list) appeared, so its + /// toolkit now holds a keyboard grab that cannot be released without + /// closing it; the pid that owns it (falls back to the action's target). + pub grab_held_by: Option, + /// The target application opened (and possibly focused) one of its own + /// windows; the guard left it alone. + pub same_app_window: Option, + /// The window that held the focus before the action no longer exists + /// (the action closed a dialog); there is nothing to restore to, the + /// focus went to `same_app_window` / wherever the WM put it. + pub closed_window: Option, + /// Milliseconds spent watching and restoring after delivery. + pub elapsed_ms: u64, +} + +impl FocusGuardReport { + /// `window_closed` / `same_app_dialog` / `restored` / `not_restored` + /// once something moved. + pub fn outcome(&self) -> Option<&'static str> { + if !self.changed { + return None; + } + Some(if self.closed_window.is_some() { + "window_closed" + } else if self.same_app_window.as_ref().is_some_and(|w| w.focused) { + "same_app_dialog" + } else if self.restored { + "restored" + } else if self.grab_held_by.is_some() { + "grab_held" + } else { + "not_restored" + }) + } + + pub fn to_json(&self) -> serde_json::Value { + let mut v = serde_json::json!({ + "focus_changed": self.changed, + "focus_restored": self.restored, + }); + if let Some(outcome) = self.outcome() { + v["focus_outcome"] = serde_json::json!(outcome); + } + if !self.changes.is_empty() { + v["focus_changes"] = serde_json::json!(self.changes); + } + if let Some(pid) = self.grab_held_by { + v["grab_held_by"] = serde_json::json!(pid); + } + if let Some(closed) = self.closed_window { + v["focus_window_closed"] = serde_json::json!(closed); + } + if let Some(window) = &self.same_app_window { + v["app_window_opened"] = serde_json::json!({ + "window_id": window.window, + "title": window.title, + "focused": window.focused, + }); + } + v + } + + /// A `window_change` evidence item when the target application mapped or + /// focused one of its own windows, or closed the one holding the focus: + /// the public action contract keeps `evidence[]`, while the flat guard + /// fields are reduced away. + pub fn evidence_item(&self) -> Option { + if let Some(closed) = self.closed_window { + return Some(serde_json::json!({ + "kind": "window_change", + "detail": format!( + "window {closed}, which held the focus, was closed by the action{}", + self.same_app_window + .as_ref() + .map(|w| format!("; the focus moved to {} \"{}\"", w.window, w.title)) + .unwrap_or_default() + ), + })); + } + let window = self.same_app_window.as_ref()?; + Some(serde_json::json!({ + "kind": "window_change", + "detail": format!( + "the application {} its own window {} \"{}\"", + if window.focused { "opened and focused" } else { "opened" }, + window.window, + window.title + ), + })) + } + + /// One sentence for the tool's text content; empty when nothing moved. + pub fn summary(&self) -> String { + let mut s = String::new(); + match (&self.same_app_window, self.changed) { + _ if self.closed_window.is_some() => s.push_str(&format!( + " The window that held the focus ({}) was closed by this action \ + (focus_outcome=window_closed); the focus moved to {}.", + self.closed_window.unwrap_or(0), + self.same_app_window + .as_ref() + .map(|w| format!("{} \"{}\"", w.window, w.title)) + .unwrap_or_else(|| "another window".to_owned()) + )), + (Some(window), _) if window.focused => s.push_str(&format!( + " The application opened its own window {} \"{}\" and it now holds the \ + focus (focus_outcome=same_app_dialog, left in place); pid-only \ + type_text/press_key go to that window.", + window.window, window.title + )), + (Some(window), false) => s.push_str(&format!( + " The application opened its own window {} \"{}\".", + window.window, window.title + )), + (_, true) if self.grab_held_by.is_some() && !self.restored => s.push_str(&format!( + " The application moved the desktop focus ({}) and its open popup holds \ + the keyboard grab, so it was left in place (focus_outcome=grab_held).", + self.changes.join(", ") + )), + (_, true) => s.push_str(&format!( + " The application moved the desktop focus ({}); {}.", + self.changes.join(", "), + if self.restored { + "it was restored to the previous window (focus_outcome=restored)" + } else { + "restoring it did not hold (focus_outcome=not_restored)" + } + )), + _ => {} + } + if let Some(pid) = self.grab_held_by { + s.push_str(&format!( + " A popup menu is open, so pid {pid} holds a keyboard grab until it closes." + )); + } + s + } +} + +/// Keep late application focus changes off the user's desktop without delaying input replies. +pub struct BackgroundFocusGuard { + snapshot: FocusSnapshot, + input: super::user_input::Observer, + target: u32, + epoch: u64, + session: String, +} +impl BackgroundFocusGuard { + pub fn begin(target: u32, session: String) -> Result> { + let input = super::user_input::Observer::start()?; + let snapshot = + FocusSnapshot::capture().ok_or_else(|| anyhow!("cannot observe X11 focus"))?; + if snapshot.previous_owner == Some(target) { + return Ok(None); + } + Ok(Some(Self { + snapshot, + input, + target, + session, + epoch: super::FOREGROUND_EPOCH.load(std::sync::atomic::Ordering::Acquire), + })) + } + pub fn finish(self, sent: bool) { + if !sent { + return; + } + let _ = std::thread::Builder::new() + .name("cua-background-focus".into()) + .spawn(move || { + let cancelled = || { + self.input.interrupted() + || cua_driver_core::session::is_session_ended(&self.session) + || self.epoch + != super::FOREGROUND_EPOCH.load(std::sync::atomic::Ordering::Acquire) + }; + let report = + self.snapshot + .restore_if_changed_cancelable(Some(self.target), true, cancelled); + if report.changed { + tracing::debug!(?report, "background focus correction"); + } + }); + } +} + +impl FocusSnapshot { + /// Record the current state. `None` when there is no usable X display + /// (the guard then degrades to a no-op rather than failing the action). + pub fn capture() -> Option { + let x = X::open().ok()?; + let (core_focus, revert_to) = x.core_focus(); + let active = x.active_window(); + let previous_owner = active + .or_else(|| (core_focus > 1).then_some(core_focus)) + .and_then(|w| x.owner_pid(w)); + Some(Self { + core_focus, + revert_to, + active, + stacking_top: x.stacking_top(), + popups: x.mapped_popups(), + clients: x.client_list().into_iter().collect(), + previous_owner, + }) + } + + fn diff(&self, x: &X) -> Vec { + let mut changes = Vec::new(); + let (focus, _) = x.core_focus(); + // PointerRoot (1) / None (0) are transient states while the WM moves + // focus; the settle loop sees the final value. + if focus > 1 && focus != self.core_focus { + changes.push(format!("focus 0x{:x}->0x{:x}", self.core_focus, focus)); + } + let active = x.active_window(); + if active != self.active && active.is_some() { + changes.push(format!( + "active 0x{:x}->0x{:x}", + self.active.unwrap_or(0), + active.unwrap_or(0) + )); + } + let top = x.stacking_top(); + if top != self.stacking_top && top.is_some() { + changes.push(format!( + "stacking_top 0x{:x}->0x{:x}", + self.stacking_top.unwrap_or(0), + top.unwrap_or(0) + )); + } + changes + } + + /// Managed toplevels that were not in the client list at capture time, + /// topmost last. + fn new_clients(&self, x: &X) -> Vec { + x.client_list() + .into_iter() + .filter(|w| *w != 0 && !self.clients.contains(w)) + .collect() + } + + /// The window the desktop focus sits on now (the WM's active window when + /// it still exists — `_NET_ACTIVE_WINDOW` lags a destroyed dialog by a + /// beat — else the core focus). + fn current_focus_window(&self, x: &X) -> Option { + x.active_window() + .filter(|w| x.window_viewable(*w)) + .or_else(|| { + let (focus, _) = x.core_focus(); + (focus > 1).then_some(focus) + }) + } + + /// The window that held the focus at capture time (active, else core). + fn previous_focus_window(&self) -> Option { + self.active + .or_else(|| (self.core_focus > 1).then_some(self.core_focus)) + } + + /// Watch briefly for a change, restore if one happened, and report. + /// `target_pid` names the action's application for the grab attribution + /// and the same-app rule. + pub fn restore_if_changed(&self, target_pid: Option) -> FocusGuardReport { + self.restore_if_changed_opts(target_pid, true) + } + + /// [`Self::restore_if_changed`] with the settle watch optional: a body that + /// already observed the desktop for longer than the watch (the MPX pointer + /// route samples the screen ~250 ms after the release and has its own + /// fast-path restore) checks once and returns unless a new top-level is + /// still being mapped. + pub fn restore_if_changed_opts( + &self, + target_pid: Option, + settle_watch: bool, + ) -> FocusGuardReport { + self.restore_if_changed_cancelable(target_pid, settle_watch, || false) + } + + fn restore_if_changed_cancelable( + &self, + target_pid: Option, + settle_watch: bool, + cancelled: impl Fn() -> bool, + ) -> FocusGuardReport { + let started = Instant::now(); + if cancelled() { + return FocusGuardReport::default(); + } + let Ok(x) = X::open() else { + return FocusGuardReport::default(); + }; + let mut report = FocusGuardReport::default(); + // The target app owns the focus: whatever it maps next is its own. + let target_owns_focus = target_pid.is_some() && self.previous_owner == target_pid; + + // Settle watch: stop at the first observed change. A new top-level + // (dialog being mapped) extends the watch when the focus belongs to + // another application, since the WM focuses it only once mapped and + // that steal must be undone; when the target app already owns the + // focus the new window is its own and the watch ends at once. + let mut watch_until = if settle_watch { + started + SETTLE_WATCH + } else { + started + }; + let mut extended = false; + let mut changes = self.diff(&x); + let mut new_clients = self.new_clients(&x); + let mut own_window = self.own_new_window(&x, target_pid, &new_clients); + if !settle_watch && !new_clients.is_empty() && own_window.is_none() { + extended = true; + watch_until = started + SETTLE_WATCH_NEW_WINDOW; + } + while changes.is_empty() && own_window.is_none() && Instant::now() < watch_until { + if cancelled() { + return report; + } + std::thread::sleep(SETTLE_POLL); + changes = self.diff(&x); + new_clients = self.new_clients(&x); + own_window = self.own_new_window(&x, target_pid, &new_clients); + if !extended && !new_clients.is_empty() && own_window.is_none() { + extended = true; + watch_until = started + SETTLE_WATCH_NEW_WINDOW; + } + } + + // Popups opened by the action: an open GTK/VCL menu grabs the keyboard. + let new_popups: Vec = x + .mapped_popups() + .difference(&self.popups) + .copied() + .collect(); + if !new_popups.is_empty() { + report.grab_held_by = new_popups + .iter() + .find_map(|w| x.window_pid(*w)) + .or(target_pid); + } + + if changes.is_empty() { + report.same_app_window = own_window; + report.elapsed_ms = started.elapsed().as_millis() as u64; + return report; + } + report.changed = true; + report.changes = changes; + // The target application mapped one of its own top-levels (a dialog + // opened by a key or a menu item) while another application owned the + // focus: the focus may still be restored below, but the new window is + // the action's effect and is reported as such. + if report.same_app_window.is_none() { + report.same_app_window = self.any_new_window_of(&x, target_pid, &new_clients); + } + + // The action closed the window that held the focus (Escape / OK on + // a dialog): nothing to restore to, the WM already moved on. + let focus_window = self.current_focus_window(&x); + if let Some(previous) = self + .previous_focus_window() + .filter(|w| !x.window_viewable(*w)) + { + report.closed_window = Some(u64::from(previous)); + report.same_app_window = focus_window.map(|w| SameAppWindow { + window: u64::from(w), + title: x.window_title(w), + focused: true, + }); + report.elapsed_ms = started.elapsed().as_millis() as u64; + return report; + } + + // Same-app rule: the focus stayed inside the target application + // (it opened a dialog). Leave it and say so. + let new_owner = focus_window.and_then(|w| x.owner_pid(w)); + if target_owns_focus + && classify_focus_move(target_pid, self.previous_owner, new_owner) + == FocusMove::SameAppDialog + { + let window = focus_window.unwrap_or(0); + report.same_app_window = Some(SameAppWindow { + window: u64::from(window), + title: x.window_title(window), + focused: true, + }); + report.elapsed_ms = started.elapsed().as_millis() as u64; + return report; + } + + // A popup (menu, combo list) that the action opened holds a keyboard + // grab: re-activating the previous window cannot take effect until it + // closes, and closing it would undo the action. Report and return. + if report.grab_held_by.is_some() { + report.elapsed_ms = started.elapsed().as_millis() as u64; + return report; + } + + if new_owner.is_some() && new_owner != self.previous_owner && new_owner != target_pid { + report.elapsed_ms = started.elapsed().as_millis() as u64; + return report; + } + + // Restore. EWMH re-activation makes the WM restore stacking and its + // own focus bookkeeping; the explicit XSetInputFocus covers WMs that + // treat _NET_ACTIVE_WINDOW as raise-only and the no-WM case. + if cancelled() { + return report; + } + let mut stable = 0; + let mut resent = 0; + let deadline = started + RESTORE_BUDGET; + self.reassert(&x); + while Instant::now() < deadline { + if cancelled() { + break; + } + std::thread::sleep(RESTORE_POLL); + if cancelled() { + break; + } + let owner = self.current_focus_window(&x).and_then(|w| x.owner_pid(w)); + if owner.is_some() && owner != self.previous_owner && owner != target_pid { + break; + } + let now = self.diff(&x); + if now.is_empty() { + stable += 1; + if stable >= STABLE_POLLS { + report.restored = true; + break; + } + continue; + } + stable = 0; + if resent < 2 { + resent += 1; + self.reassert(&x); + } + } + if !report.restored { + tracing::warn!( + "background focus guard: could not restore {:?}", + report.changes + ); + } + report.elapsed_ms = started.elapsed().as_millis() as u64; + report + } + + /// The topmost freshly mapped toplevel of the target pid, when the target + /// already owned the focus (so the window is the app's own and the watch + /// need not wait for the WM to focus it). + fn own_new_window( + &self, + x: &X, + target_pid: Option, + new_clients: &[Window], + ) -> Option { + if target_pid.is_none() || self.previous_owner != target_pid { + return None; + } + let window = new_clients + .iter() + .rev() + .copied() + .find(|w| x.owner_pid(*w) == target_pid)?; + Some(SameAppWindow { + window: u64::from(window), + title: x.window_title(window), + focused: false, + }) + } + + /// The topmost freshly mapped toplevel owned by the target pid, whoever + /// held the focus before; `focused` says whether it holds it now. + fn any_new_window_of( + &self, + x: &X, + target_pid: Option, + new_clients: &[Window], + ) -> Option { + target_pid?; + let window = new_clients + .iter() + .rev() + .copied() + .find(|w| x.owner_pid(*w) == target_pid)?; + Some(SameAppWindow { + window: u64::from(window), + title: x.window_title(window), + focused: self.current_focus_window(x) == Some(window), + }) + } + + fn reassert(&self, x: &X) { + if let Some(prev) = self.active { + x.activate(prev, x.active_window().unwrap_or(0)); + } + if self.core_focus > 1 { + x.set_focus(self.core_focus, self.revert_to); + } + } +} + +/// Run `body` (a background delivery) between a snapshot and a restore. +/// The body's error still propagates; the guard only adds its report. +pub fn guarded( + target_pid: Option, + body: impl FnOnce() -> Result, +) -> Result<(T, Option)> { + let snapshot = FocusSnapshot::capture(); + let value = body()?; + let report = snapshot.map(|s| s.restore_if_changed(target_pid)); + Ok((value, report)) +} + +/// [`guarded`] for a body that has already let the desktop settle (the MPX +/// pointer press train and drag gesture): one post-check, no extra settle +/// watch, so the pointer branch's fast path keeps its latency. +pub fn guarded_settled( + target_pid: Option, + body: impl FnOnce() -> Result, +) -> Result<(T, Option)> { + let snapshot = FocusSnapshot::capture(); + let value = body()?; + let report = snapshot.map(|s| s.restore_if_changed_opts(target_pid, false)); + Ok((value, report)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn report_json_and_summary_name_what_moved() { + let report = FocusGuardReport { + changed: true, + restored: true, + changes: vec!["focus 0x1->0x2".into()], + grab_held_by: Some(42), + same_app_window: None, + closed_window: None, + elapsed_ms: 7, + }; + let json = report.to_json(); + assert_eq!(json["focus_changed"], true); + assert_eq!(json["focus_restored"], true); + assert_eq!(json["focus_outcome"], "restored"); + assert_eq!(json["grab_held_by"], 42); + assert_eq!(json["focus_changes"][0], "focus 0x1->0x2"); + assert!(report.evidence_item().is_none()); + let summary = report.summary(); + assert!(summary.contains("restored"), "{summary}"); + assert!(summary.contains("pid 42"), "{summary}"); + } + + #[test] + fn quiet_report_is_silent() { + let report = FocusGuardReport::default(); + assert_eq!(report.summary(), ""); + let json = report.to_json(); + assert_eq!(json["focus_changed"], false); + assert!(json.get("focus_outcome").is_none()); + assert!(json.get("grab_held_by").is_none()); + assert!(json.get("focus_changes").is_none()); + } + + #[test] + fn same_app_dialog_is_reported_not_restored() { + let report = FocusGuardReport { + changed: true, + restored: false, + changes: vec!["active 0x1->0x2".into()], + grab_held_by: None, + same_app_window: Some(SameAppWindow { + window: 2, + title: "Brightness-Contrast".into(), + focused: true, + }), + closed_window: None, + elapsed_ms: 3, + }; + let json = report.to_json(); + assert_eq!(json["focus_outcome"], "same_app_dialog"); + assert_eq!(json["app_window_opened"]["window_id"], 2); + assert_eq!(json["app_window_opened"]["focused"], true); + let evidence = report.evidence_item().expect("window_change evidence"); + assert_eq!(evidence["kind"], "window_change"); + assert!(evidence["detail"] + .as_str() + .unwrap() + .contains("Brightness-Contrast")); + let summary = report.summary(); + assert!(summary.contains("same_app_dialog"), "{summary}"); + assert!(summary.contains("Brightness-Contrast"), "{summary}"); + assert!(!summary.contains("restored"), "{summary}"); + } + + #[test] + fn mapped_but_unfocused_own_window_is_mentioned_without_a_focus_outcome() { + let report = FocusGuardReport { + same_app_window: Some(SameAppWindow { + window: 9, + title: "Format Cells".into(), + focused: false, + }), + ..FocusGuardReport::default() + }; + assert!(report.outcome().is_none()); + assert!(report.summary().contains("Format Cells")); + assert!(report.evidence_item().is_some()); + } + + #[test] + fn a_closed_focus_holder_is_window_closed_evidence_not_a_failed_restore() { + let report = FocusGuardReport { + changed: true, + restored: false, + changes: vec!["focus 0x2->0x1".into()], + closed_window: Some(2), + same_app_window: Some(SameAppWindow { + window: 1, + title: "GNU Image Manipulation Program".into(), + focused: true, + }), + ..FocusGuardReport::default() + }; + assert_eq!(report.outcome(), Some("window_closed")); + assert_eq!(report.to_json()["focus_outcome"], "window_closed"); + assert_eq!(report.to_json()["focus_window_closed"], 2); + let evidence = report.evidence_item().unwrap(); + assert!(evidence["detail"].as_str().unwrap().contains("closed")); + assert!(report.summary().contains("window_closed")); + assert!(!report.summary().contains("not_restored")); + } + + #[test] + fn same_app_rule_only_when_the_target_owned_the_focus_before_and_after() { + // The target app was active and opened its own dialog: leave it. + assert_eq!( + classify_focus_move(Some(7), Some(7), Some(7)), + FocusMove::SameAppDialog + ); + // Decoy case: another app owned the focus and the target's dialog + // stole it -> restore. + assert_eq!( + classify_focus_move(Some(7), Some(3), Some(7)), + FocusMove::OtherApp + ); + // The target was active and something else took the focus -> restore. + assert_eq!( + classify_focus_move(Some(7), Some(7), Some(3)), + FocusMove::OtherApp + ); + // Unknown owners never qualify. + assert_eq!( + classify_focus_move(Some(7), None, Some(7)), + FocusMove::OtherApp + ); + assert_eq!( + classify_focus_move(Some(7), Some(7), None), + FocusMove::OtherApp + ); + assert_eq!( + classify_focus_move(None, Some(7), Some(7)), + FocusMove::OtherApp + ); + } + + #[test] + fn capture_without_display_degrades_to_none() { + let display = crate::test_env::unreachable_x11_display(); + let snapshot = FocusSnapshot::capture(); + drop(display); + assert!(snapshot.is_none()); + } +} + +#[cfg(test)] +mod grab_tests { + use super::*; + + #[test] + fn a_popup_grab_is_reported_instead_of_a_failed_restore() { + let report = FocusGuardReport { + changed: true, + restored: false, + changes: vec!["active 0x1->0x2".into()], + grab_held_by: Some(77), + same_app_window: None, + closed_window: None, + elapsed_ms: 40, + }; + assert_eq!(report.outcome(), Some("grab_held")); + assert_eq!(report.to_json()["focus_outcome"], "grab_held"); + let summary = report.summary(); + assert!(summary.contains("focus_outcome=grab_held"), "{summary}"); + assert!( + summary.contains("pid 77 holds a keyboard grab"), + "{summary}" + ); + } + + #[test] + fn restore_and_settle_budgets_bound_the_guard_under_1500ms() { + assert!(SETTLE_WATCH_NEW_WINDOW + RESTORE_BUDGET <= Duration::from_millis(1500)); + } +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/input/keymap_barrier.rs b/packages/cua-driver/rust/crates/platform-linux/src/input/keymap_barrier.rs new file mode 100644 index 00000000000..24623d96fab --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-linux/src/input/keymap_barrier.rs @@ -0,0 +1,150 @@ +use super::*; +use crate::x11_client_pid::client_base; +use std::time::Instant; +use x11rb::protocol::Event; + +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +struct QueueUnconfirmed(String); + +pub(super) fn unconfirmed(reason: impl Into) -> anyhow::Error { + QueueUnconfirmed(reason.into()).into() +} + +pub(super) fn dispatch(send: impl FnOnce() -> Result) -> Result { + send().map_err(|error| unconfirmed(format!("{error:#}"))) +} + +pub(crate) fn delivery_is_unconfirmed(error: &anyhow::Error) -> bool { + error.is::() +} + +/// A server round-trip does not let a busy client translate queued keycodes. +/// Keep temporary mappings until that same X client services a later ping. +/// This acknowledges its event queue, not the application's resulting text. +pub(super) struct KeymapBarrier<'a> { + conn: &'a RustConnection, + window: u32, + root: u32, + protocols: u32, + words: [u32; 5], + focused_client: Option, +} + +impl<'a> KeymapBarrier<'a> { + pub(super) fn prepare( + conn: &'a RustConnection, + target: Option, + mapping: &GetKeyboardMappingReply, + keysyms: impl IntoIterator, + ) -> Result> { + if !keysyms + .into_iter() + .any(|key| !mapping.keysyms.contains(&key)) + { + return Ok(None); + } + Self::prepare_client(conn, target).map(Some).map_err(|error| { + anyhow!("not_settable: temporary keyboard mapping needs a responsive X11 client queue: {error:#}; no input was sent") + }) + } + + fn prepare_client(conn: &'a RustConnection, target: Option) -> Result { + let mut window = match target { + Some(window) => window, + None => conn.get_input_focus()?.reply()?.focus, + }; + let mask = conn.setup().resource_id_mask; + let owner = client_base(window, mask); + if window <= 1 || owner == 0 { + bail!("no client window owns the target"); + } + let protocols = conn.intern_atom(false, b"WM_PROTOCOLS")?.reply()?.atom; + let ping = conn.intern_atom(false, b"_NET_WM_PING")?.reply()?.atom; + for _ in 0..64 { + if client_base(window, mask) != owner { + break; + } + let tree = conn.query_tree(window)?.reply()?; + let advertised = conn + .get_property(false, window, protocols, AtomEnum::ATOM, 0, 256)? + .reply()?; + if advertised + .value32() + .is_some_and(|mut atoms| atoms.any(|atom| atom == ping)) + { + conn.change_window_attributes( + tree.root, + &ChangeWindowAttributesAux::new().event_mask(EventMask::SUBSTRUCTURE_NOTIFY), + )? + .check()?; + let nonce = uuid::Uuid::new_v4().as_u128(); + return Ok(Self { + conn, + window, + root: tree.root, + protocols, + words: [ + ping, + nonce as u32, + window, + (nonce >> 32) as u32, + (nonce >> 64) as u32, + ], + focused_client: target.is_none().then_some(owner), + }); + } + if tree.parent == tree.root || tree.parent == window { + break; + } + window = tree.parent; + } + bail!("the target's X client does not advertise _NET_WM_PING"); + } + + pub(super) fn check_focus(&self) -> Result<()> { + if let Some(owner) = self.focused_client { + let focus = self.conn.get_input_focus()?.reply()?.focus; + if client_base(focus, self.conn.setup().resource_id_mask) != owner { + bail!("text_delivery_stalled: the input focus changed to another X client; input may have been sent; inspect before retrying"); + } + } + Ok(()) + } + + pub(super) fn wait(&self) -> Result<()> { + self.wait_inner() + .map_err(|error| unconfirmed(format!("{error:#}"))) + } + + fn wait_inner(&self) -> Result<()> { + self.check_focus()?; + let request = ClientMessageEvent::new(32, self.window, self.protocols, self.words); + self.conn + .send_event(false, self.window, EventMask::NO_EVENT, request)? + .check()?; + self.conn.flush()?; + let deadline = Instant::now() + Duration::from_secs(5); + loop { + while let Some(event) = self.conn.poll_for_event()? { + if let Event::ClientMessage(event) = event { + if event.window == self.root + && event.format == 32 + && event.type_ == self.protocols + && event.data.as_data32() == self.words + { + self.check_focus()?; + return Ok(()); + } + } + if Instant::now() >= deadline { + break; + } + } + if Instant::now() >= deadline { + bail!("the X client did not acknowledge its input queue within 5000ms"); + } + sleep(Duration::from_millis(5)); + } + } +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/packages/cua-driver/rust/crates/platform-linux/src/input/mod.rs index a71f840e8b0..832bbd7657d 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/input/mod.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/input/mod.rs @@ -1,3 +1,4 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. //! Background input injection for Linux via X11 XSendEvent. //! //! XSendEvent sends synthetic events directly to a window without changing @@ -14,10 +15,24 @@ /// Shared `delivery_mode` contract (background|foreground) — mirrors macOS /// `tools::DeliveryMode` and Windows `input::delivery`. pub mod delivery; +pub mod focus_guard; +mod keymap_barrier; +mod mpx_keyboard; +mod mpx_owner; +mod user_input; + +pub use focus_guard::{BackgroundFocusGuard, FocusGuardReport, FocusSnapshot, SameAppWindow}; +pub(crate) use keymap_barrier::delivery_is_unconfirmed; +pub(crate) use mpx_keyboard::check_keyboard_background; +pub use mpx_keyboard::{ + real_keyboard_input_available, send_virtual_keyboard_key, send_virtual_keyboard_text, + KeyboardDeliveryReport, MPX_UINPUT_PATH, +}; use anyhow::{anyhow, bail, Context, Result}; use evdev::uinput::VirtualDevice; use evdev::{AttributeSet, EventType, InputEvent, Key, RelativeAxisType}; +use keymap_barrier::KeymapBarrier; use std::collections::HashMap; use std::ffi::{CStr, CString}; use std::fs; @@ -34,6 +49,7 @@ use x11rb::rust_connection::RustConnection; const CLICK_DELAY_MS: u64 = 35; const DOUBLE_CLICK_DELAY_MS: u64 = 50; const KEY_DELAY_MS: u64 = 10; +static KEYMAP_LOCK: Mutex<()> = Mutex::new(()); #[derive(Clone, Debug)] pub struct VirtualPointerDrag { @@ -115,26 +131,49 @@ fn point_on_path(path: &[(i32, i32)], cum: &[f64], total: f64, t: f64) -> (i32, (x.round() as i32, y.round() as i32) } +/// One session's XI2 master pair (XIAddMaster always creates a pointer AND a +/// keyboard) plus the uinput slaves attached to it. The keyboard slave is +/// created lazily by `mpx_keyboard::ensure_master_keyboard`, so a pointer-only +/// click never pays for a second device hotplug. #[derive(Clone, Copy, Debug)] struct MasterPointerIds { pointer_id: i32, - _keyboard_id: i32, + keyboard_id: i32, _slave_pointer_id: i32, + slave_keyboard_id: Option, } static MPX_POINTERS: OnceLock>> = OnceLock::new(); static UINPUT_POINTERS: OnceLock>>>> = OnceLock::new(); static XLIB_THREADS_READY: OnceLock> = OnceLock::new(); +/// Serialises every MPX operation against the idle reaper (and each other), +/// so a retained master pair is never torn down while a call is using it. +static MPX_OP_LOCK: Mutex<()> = Mutex::new(()); +static MPX_LAST_USE: OnceLock>> = OnceLock::new(); +static MPX_IDLE_REAPER: std::sync::Once = std::sync::Once::new(); +/// A session's retained master pair is removed after this much inactivity; +/// `end_session` and the startup reaper cover the explicit and crash cases. +/// `XIRemoveMaster` churns the XInput hierarchy, and mutter 42.x segfaults when +/// it processes that churn (the same reason per-call teardown crashes LibreOffice +/// VCL), so the pair must outlive the inter-action idle of a live session; a +/// long terminal-only stretch leaves the pointer unused far past a few minutes. +/// The window is only a leak backstop for a client that dies without +/// `end_session`; the startup reaper reclaims anything it misses. +const MPX_IDLE_TTL: Duration = Duration::from_secs(1800); +const MPX_IDLE_REAPER_PERIOD: Duration = Duration::from_secs(60); static MPX_NAME_COUNTER: AtomicU64 = AtomicU64::new(1); // evdev 0.12.2 asserts `name.len() + 1 < UINPUT_MAX_NAME_SIZE` while building // a device. Linux defines UINPUT_MAX_NAME_SIZE as 80, leaving 78 usable bytes. const EVDEV_UINPUT_NAME_MAX_BYTES: usize = 78; const UINPUT_POINTER_SUFFIX: &str = " uinput pointer"; pub const UINPUT_UNAVAILABLE_CODE: &str = "uinput_unavailable"; +/// Result `path` for pointer actions delivered as real button events from the +/// session's MPX virtual master pointer. +pub const MPX_POINTER_PATH: &str = "mpx_pointer"; #[derive(Debug, thiserror::Error)] -#[error("Linux uinput pointer unavailable: {reason}")] +#[error("Linux uinput device unavailable: {reason}")] struct UinputUnavailable { reason: String, } @@ -143,12 +182,67 @@ fn mpx_pointers() -> &'static Mutex> { MPX_POINTERS.get_or_init(|| Mutex::new(HashMap::new())) } +fn mpx_last_use() -> &'static Mutex> { + MPX_LAST_USE.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Take the MPX operation lock for one call on `cursor_id`, stamp its last +/// use, and make sure the idle reaper is running. Hold the guard for the +/// whole operation. +fn mpx_op_guard(cursor_id: &str) -> std::sync::MutexGuard<'static, ()> { + let guard = MPX_OP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + mpx_last_use() + .lock() + .unwrap() + .insert(cursor_id.to_owned(), std::time::Instant::now()); + MPX_IDLE_REAPER.call_once(|| { + std::thread::Builder::new() + .name("cua-mpx-idle-reaper".into()) + .spawn(|| loop { + sleep(MPX_IDLE_REAPER_PERIOD); + let _op = MPX_OP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let stale = { + let last_use = mpx_last_use().lock().unwrap(); + stale_cursor_ids(&last_use, std::time::Instant::now(), MPX_IDLE_TTL) + }; + for cursor_id in stale { + tracing::info!(cursor_id, "removing idle MPX master pair"); + forget_master_pointer_inner(&cursor_id); + } + }) + .ok(); + }); + guard +} + +/// Sessions whose last MPX use is older than `ttl`. +fn stale_cursor_ids( + last_use: &HashMap, + now: std::time::Instant, + ttl: Duration, +) -> Vec { + last_use + .iter() + .filter(|(_, at)| now.saturating_duration_since(**at) >= ttl) + .map(|(id, _)| id.clone()) + .collect() +} + fn uinput_pointers() -> &'static Mutex>>> { UINPUT_POINTERS.get_or_init(|| Mutex::new(HashMap::new())) } fn master_pointer_name(cursor_id: &str) -> String { let nonce = MPX_NAME_COUNTER.fetch_add(1, Ordering::Relaxed); + if let Ok(owner) = mpx_owner::Owner::current() { + return owner.master_name(nonce); + } + // Unknown procfs identity cannot safely participate in automatic recovery. + // Preserve ordinary operation, with the legacy name and cleanup behavior. let prefix = "CUA "; let suffix = format!(" mp-{}-{nonce}", std::process::id()); let max_cursor_bytes = EVDEV_UINPUT_NAME_MAX_BYTES @@ -390,11 +484,74 @@ fn real_pointer_capabilities_available( server_supported: bool, xvfb: bool, uinput_accessible: bool, + unsafe_hotplug_session: bool, ) -> bool { - server_supported && !xvfb && uinput_accessible + server_supported && !xvfb && uinput_accessible && !unsafe_hotplug_session } -fn uinput_accessible() -> bool { +fn nonempty(value: Option<&str>) -> bool { + value.is_some_and(|value| !value.trim().is_empty()) +} + +fn desktop_value_is_kde(value: Option<&str>) -> bool { + value.is_some_and(|value| { + value + .split([':', ';', ',']) + .map(str::trim) + .any(|token| token.eq_ignore_ascii_case("kde") || token.eq_ignore_ascii_case("plasma")) + }) +} + +/// KDE Plasma 6 / Qt 6.11 applications on X11 can crash session-wide when an +/// ephemeral uinput pointer is hotplugged into Xorg. Foreground input does not +/// need that device: the click, drag, scroll, and keyboard tools already use +/// XTEST after activating the target window. Disable only the MPX/uinput +/// capability here so callers retain their existing foreground escalation and +/// XSendEvent fallback behavior. +fn kde_x11_uinput_hotplug_is_unsafe( + session_type: Option<&str>, + current_desktop: Option<&str>, + session_desktop: Option<&str>, + desktop_session: Option<&str>, + kde_full_session: Option<&str>, + display: Option<&str>, + _other_display: Option<&str>, +) -> bool { + let explicit_x11 = session_type.is_some_and(|value| value.eq_ignore_ascii_case("x11")); + let x11 = explicit_x11 || nonempty(display); + let kde = desktop_value_is_kde(current_desktop) + || desktop_value_is_kde(session_desktop) + || desktop_value_is_kde(desktop_session) + || kde_full_session.is_some_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" + ) + }); + + x11 && kde +} + +fn kde_x11_uinput_hotplug_is_unsafe_from_env() -> bool { + let session_type = std::env::var("XDG_SESSION_TYPE").ok(); + let current_desktop = std::env::var("XDG_CURRENT_DESKTOP").ok(); + let session_desktop = std::env::var("XDG_SESSION_DESKTOP").ok(); + let desktop_session = std::env::var("DESKTOP_SESSION").ok(); + let kde_full_session = std::env::var("KDE_FULL_SESSION").ok(); + let display = std::env::var("DISPLAY").ok(); + + kde_x11_uinput_hotplug_is_unsafe( + session_type.as_deref(), + current_desktop.as_deref(), + session_desktop.as_deref(), + desktop_session.as_deref(), + kde_full_session.as_deref(), + display.as_deref(), + None, + ) +} + +pub(crate) fn uinput_accessible() -> bool { fs::OpenOptions::new() .read(true) .write(true) @@ -403,6 +560,13 @@ fn uinput_accessible() -> bool { } pub fn real_pointer_input_available() -> bool { + // Do not even probe /dev/uinput on an affected KDE/X11 session. Creating + // the device is itself the dangerous operation; a later fallback is too + // late once Xorg has announced the hotplug to Qt clients. + if kde_x11_uinput_hotplug_is_unsafe_from_env() { + return false; + } + // `ensure_master_pointer` creates an XI2 master before attaching the // uinput slave. If this process cannot open /dev/uinput, attempting that // path on every click/scroll would create and abandon an XInput master @@ -418,12 +582,26 @@ pub fn real_pointer_input_available() -> bool { supports_parallel_pointer_injection(display).is_ok(), is_xvfb_process_running(), true, + false, ); unsafe { x11::xlib::XCloseDisplay(display) }; supported } fn ensure_master_pointer(cursor_id: &str) -> Result { + ensure_master_pointer_for_session(cursor_id, kde_x11_uinput_hotplug_is_unsafe_from_env()) +} + +fn ensure_master_pointer_for_session( + cursor_id: &str, + unsafe_hotplug_session: bool, +) -> Result { + if unsafe_hotplug_session { + return Err(uinput_unavailable( + "disabled on KDE Plasma X11; retry with delivery_mode='foreground'", + )); + } + if let Some(ids) = mpx_pointers().lock().unwrap().get(cursor_id).copied() { return Ok(ids); } @@ -490,15 +668,31 @@ fn ensure_master_pointer(cursor_id: &str) -> Result { let keyboard_id = keyboard_id .ok_or_else(|| anyhow!("failed to locate created master keyboard for '{cursor_id}'"))?; - let slave_pointer_id = wait_for_slave_pointer_id(display, &device_name)?; + let slave_pointer_id = match wait_for_slave_id( + display, + &device_name, + x11::xinput2::XISlavePointer, + Duration::from_secs(5), + ) { + Ok(id) => id, + Err(error) => { + // Never leave the fresh master pair behind when the slave never + // hotplugs (no udev/libinput on this server): a stray master with + // nothing attached is exactly the leak the startup reaper exists for. + let _ = remove_master_pointer(display, pointer_id); + unsafe { x11::xlib::XCloseDisplay(display) }; + return Err(error); + } + }; attach_slave_to_master(display, slave_pointer_id, pointer_id)?; set_flat_pointer_accel(display, slave_pointer_id); unsafe { x11::xlib::XCloseDisplay(display) }; let ids = MasterPointerIds { pointer_id, - _keyboard_id: keyboard_id, + keyboard_id, _slave_pointer_id: slave_pointer_id, + slave_keyboard_id: None, }; mpx_pointers() .lock() @@ -512,7 +706,19 @@ fn ensure_master_pointer(cursor_id: &str) -> Result { } pub fn forget_master_pointer(cursor_id: &str) { + let _op = MPX_OP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + forget_master_pointer_inner(cursor_id); +} + +fn forget_master_pointer_inner(cursor_id: &str) { + mpx_last_use().lock().unwrap().remove(cursor_id); + // Drop the uinput slaves first: closing the fds unplugs them, so the master + // removal below never has to hand a live slave back to the user's core + // devices (XIAttachToMaster only re-homes slaves that still exist). uinput_pointers().lock().unwrap().remove(cursor_id); + mpx_keyboard::forget_uinput_keyboard(cursor_id); let Some(ids) = mpx_pointers().lock().unwrap().remove(cursor_id) else { return; }; @@ -520,11 +726,12 @@ pub fn forget_master_pointer(cursor_id: &str) { let Ok(display) = open_display() else { return; }; + let _ = remove_master_pointer(display, ids.pointer_id); + unsafe { x11::xlib::XCloseDisplay(display) }; +} - let Ok(devices) = xi2_query_devices(display) else { - unsafe { x11::xlib::XCloseDisplay(display) }; - return; - }; +fn remove_master_pointer(display: *mut x11::xlib::Display, pointer_id: i32) -> Result<()> { + let devices = xi2_query_devices(display)?; let mut virtual_core_pointer = None; let mut virtual_core_keyboard = None; @@ -539,22 +746,72 @@ pub fn forget_master_pointer(cursor_id: &str) { let (Some(return_pointer), Some(return_keyboard)) = (virtual_core_pointer, virtual_core_keyboard) else { - unsafe { x11::xlib::XCloseDisplay(display) }; - return; + bail!("cannot remove MPX master without its virtual core return devices"); }; let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default(); unsafe { let remove = change.remove(); (*remove)._type = x11::xinput2::XIRemoveMaster; - (*remove).deviceid = ids.pointer_id; + (*remove).deviceid = pointer_id; (*remove).return_mode = x11::xinput2::XIAttachToMaster; (*remove).return_pointer = return_pointer; (*remove).return_keyboard = return_keyboard; - let _ = x11::xinput2::XIChangeHierarchy(display, &mut change, 1); + let rc = x11::xinput2::XIChangeHierarchy(display, &mut change, 1); + x11::xlib::XSync(display, 0); + if rc != 0 { + bail!("XIChangeHierarchy(XIRemoveMaster) failed with status {rc}"); + } + } + Ok(()) +} + +/// Recover only versioned masters whose local owner is provably gone. A PID +/// alone cannot identify an owner on a shared/remote X server or across restarts. +pub(crate) fn reap_orphaned_master_pointers() { + let Ok(owner) = mpx_owner::Owner::current() else { + return; + }; + let Ok(display) = open_display() else { + return; + }; + // Prevent an ID from being removed/reused by another X client between our + // enumeration and removal. No network or arbitrary filesystem reads occur + // under this grab: owner checks inspect local procfs and kill(pid, 0). + unsafe { + x11::xlib::XGrabServer(display); + } + // A panic while the server is grabbed would freeze every other X client + // (the whole desktop) until this process dies. Catch it so the ungrab + // below always runs, and report it like any other incomplete recovery. + let result = catch_unwind(AssertUnwindSafe(|| -> Result<()> { + for (id, use_, name) in xi2_query_devices(display)? { + if use_ != x11::xinput2::XIMasterPointer { + continue; + } + let Some(candidate) = mpx_owner::Owner::from_pointer_name(&name) else { + continue; + }; + if candidate.stale_in(&owner) { + remove_master_pointer(display, id)?; + tracing::info!(device_id = id, "removed orphaned Cua MPX master pair"); + } + } + Ok(()) + })); + unsafe { + x11::xlib::XUngrabServer(display); x11::xlib::XSync(display, 0); x11::xlib::XCloseDisplay(display); } + match result { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!("MPX orphan recovery incomplete: {error}"), + Err(payload) => tracing::warn!( + "MPX orphan recovery panicked (server grab released): {}", + panic_payload_message(payload.as_ref()) + ), + } } fn create_uinput_pointer(name: &str) -> Result { @@ -582,16 +839,25 @@ fn create_uinput_pointer(name: &str) -> Result { }) } -fn wait_for_slave_pointer_id(display: *mut x11::xlib::Display, device_name: &str) -> Result { - let deadline = std::time::Instant::now() + Duration::from_secs(5); +/// Poll `XIQueryDevice` until the X server has hot-added the uinput device +/// named `device_name` as a slave of kind `use_` (`XISlavePointer` / +/// `XISlaveKeyboard`). udev + xf86-input-libinput do the hotplug on a real +/// Xorg; Xvfb/Xtigervnc never will, which is what the timeout covers. +fn wait_for_slave_id( + display: *mut x11::xlib::Display, + device_name: &str, + use_: i32, + timeout: Duration, +) -> Result { + let deadline = std::time::Instant::now() + timeout; loop { - for (device_id, use_, seen_name) in xi2_query_devices(display)? { - if use_ == x11::xinput2::XISlavePointer && seen_name == device_name { + for (device_id, seen_use, seen_name) in xi2_query_devices(display)? { + if seen_use == use_ && seen_name == device_name { return Ok(device_id); } } if std::time::Instant::now() >= deadline { - bail!("timed out waiting for X input slave pointer '{device_name}'"); + bail!("timed out waiting for X input slave device '{device_name}'"); } sleep(Duration::from_millis(50)); } @@ -705,10 +971,79 @@ fn warp_master_pointer( Ok(()) } -/// XIAnyModifier (1u32 << 31). The x11 crate doesn't export it. -const XI_ANY_MODIFIER: std::os::raw::c_int = 0x8000_0000u32 as std::os::raw::c_int; +/// True when `window` is override-redirect — a menu, tooltip, or other popup +/// the WM does not manage. Such a window takes its own active pointer grab, so +/// the shield-grab-and-replay dance cannot deliver into it; a plain warp+press +/// on the virtual master is what reaches it (and there is no WM focus to steal). +fn is_override_redirect(display: *mut x11::xlib::Display, window: x11::xlib::Window) -> bool { + let mut attrs: x11::xlib::XWindowAttributes = unsafe { std::mem::zeroed() }; + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let rc = unsafe { x11::xlib::XGetWindowAttributes(display, window, &mut attrs) }; + unsafe { x11::xlib::XSetErrorHandler(previous_handler) }; + rc != 0 && attrs.override_redirect != 0 +} -fn xi_mask_len() -> usize { +/// Direct child of the root window under screen point `(x, y)`: the WM frame +/// of a managed toplevel, or an override-redirect popup (menu, tooltip). +fn root_child_under_point( + display: *mut x11::xlib::Display, + x: i32, + y: i32, +) -> Option { + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + let mut child: x11::xlib::Window = 0; + let mut dx = 0; + let mut dy = 0; + let rc = unsafe { + x11::xlib::XTranslateCoordinates(display, root, root, x, y, &mut dx, &mut dy, &mut child) + }; + (rc != 0 && child != 0).then_some(child) +} + +/// The root child (WM frame or the window itself) that contains `window`. +fn root_child_of( + display: *mut x11::xlib::Display, + window: x11::xlib::Window, +) -> Option { + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let mut current = window; + let mut result = None; + for _ in 0..64 { + let mut root_ret: x11::xlib::Window = 0; + let mut parent: x11::xlib::Window = 0; + let mut children: *mut x11::xlib::Window = ptr::null_mut(); + let mut count: std::os::raw::c_uint = 0; + let rc = unsafe { + x11::xlib::XQueryTree( + display, + current, + &mut root_ret, + &mut parent, + &mut children, + &mut count, + ) + }; + if !children.is_null() { + unsafe { x11::xlib::XFree(children as *mut _) }; + } + if rc == 0 || parent == 0 { + break; + } + if parent == root { + result = Some(current); + break; + } + current = parent; + } + unsafe { + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(previous_handler); + } + result +} + +pub(super) fn xi_mask_len() -> usize { (x11::xinput2::XI_LASTEVENT as usize >> 3) + 1 } @@ -732,125 +1067,127 @@ fn xinput_opcode(display: *mut x11::xlib::Display) -> Option Result<()> { - let mut mask_bits = vec![0u8; xi_mask_len()]; - x11::xinput2::XISetMask(&mut mask_bits, x11::xinput2::XI_ButtonPress); - let mut evmask = x11::xinput2::XIEventMask { - deviceid: device_id, - mask_len: mask_bits.len() as std::os::raw::c_int, - mask: mask_bits.as_mut_ptr(), - }; - let mut mods = x11::xinput2::XIGrabModifiers { - modifiers: XI_ANY_MODIFIER, - status: 0, - }; + pointer_id: i32, +) -> (Vec, x11::xinput2::XIModifierState) { + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + let mut root_ret = 0; + let mut child_ret = 0; + let (mut rx, mut ry, mut wx, mut wy) = (0f64, 0f64, 0f64, 0f64); + let mut buttons = x11::xinput2::XIButtonState::default(); + let mut mods = x11::xinput2::XIModifierState::default(); + let mut group = x11::xinput2::XIModifierState::default(); + let prev = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; let rc = unsafe { - x11::xinput2::XIGrabButton( + x11::xinput2::XIQueryPointer( display, - device_id, - button as std::os::raw::c_int, - window, - 0, // cursor: None - x11::xinput2::XIGrabModeSync, // freeze the pointer on press - x11::xinput2::XIGrabModeAsync, // leave the paired keyboard alone - x11::xlib::False, // owner_events: deliver to us - &mut evmask, - 1, + pointer_id, + root, + &mut root_ret, + &mut child_ret, + &mut rx, + &mut ry, + &mut wx, + &mut wy, + &mut buttons, &mut mods, + &mut group, ) }; - unsafe { x11::xlib::XSync(display, 0) }; - if rc != 0 { - bail!("XIGrabButton(shield) failed with status {rc}"); + unsafe { x11::xlib::XSetErrorHandler(prev) }; + let mut held = Vec::new(); + if rc != 0 && !buttons.mask.is_null() && buttons.mask_len > 0 { + let mask = unsafe { std::slice::from_raw_parts(buttons.mask, buttons.mask_len as usize) }; + for button in 1u8..=3 { + let byte = (button / 8) as usize; + if mask.get(byte).is_some_and(|b| b & (1 << (button % 8)) != 0) { + held.push(button); + } + } + unsafe { x11::xlib::XFree(buttons.mask as *mut _) }; } - Ok(()) + (held, mods) } -fn remove_shield_grab( +/// Release anything the session's virtual master still holds from an earlier +/// aborted action: pointer buttons (a shield replay that timed out returned +/// before the release) and keyboard modifiers (a chord interrupted mid-way). +/// +/// A held button is not cosmetic: every key event from the paired virtual +/// keyboard then carries `Button1Mask` (bit 8) in its core `state`, which +/// at-spi2 forwards verbatim and Orca reads as its `ORCA_MODIFIER_MASK` +/// (`1 << 8`) — a plain space becomes "Orca+space" and opens the Screen +/// Reader Preferences. Returns what was released, for the tool's report. +fn release_stuck_virtual_input( + cursor_id: &str, display: *mut x11::xlib::Display, - device_id: i32, - window: x11::xlib::Window, - button: u8, -) { - let mut mods = x11::xinput2::XIGrabModifiers { - modifiers: XI_ANY_MODIFIER, - status: 0, - }; - unsafe { - let prev = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); - x11::xinput2::XIUngrabButton( - display, - device_id, - button as std::os::raw::c_int, - window, - 1, - &mut mods, - ); - x11::xlib::XSync(display, 0); - x11::xlib::XSetErrorHandler(prev); - } -} - -/// Drain the frozen shield presses for `pending_devices` and replay each so -/// it continues to the application. Returns the set of device ids we failed -/// to see within the timeout (their drags still proceed; the focus-restore -/// safety net covers any leak). -fn replay_shielded_presses( - display: *mut x11::xlib::Display, - xi_opcode: std::os::raw::c_int, - pending_devices: &mut std::collections::HashSet, - timeout: Duration, -) { - let deadline = std::time::Instant::now() + timeout; - while !pending_devices.is_empty() && std::time::Instant::now() < deadline { - // Only block on XNextEvent when something is queued, so a missing - // press can't hang us past the deadline. - if unsafe { x11::xlib::XPending(display) } == 0 { - sleep(Duration::from_millis(2)); - continue; - } - let mut ev: x11::xlib::XEvent = unsafe { std::mem::zeroed() }; - unsafe { x11::xlib::XNextEvent(display, &mut ev) }; - if unsafe { ev.type_ } != x11::xlib::GenericEvent { - continue; - } - let mut cookie = unsafe { ev.generic_event_cookie }; - if cookie.extension != xi_opcode || cookie.evtype != x11::xinput2::XI_ButtonPress { - continue; - } - if unsafe { x11::xlib::XGetEventData(display, &mut cookie) } == 0 { - continue; - } - let de = cookie.data as *const x11::xinput2::XIDeviceEvent; - if !de.is_null() { - let device_id = unsafe { (*de).deviceid }; - let time = unsafe { (*de).time }; - if pending_devices.remove(&device_id) { - unsafe { - x11::xinput2::XIAllowEvents( - display, - device_id, - x11::xinput2::XIReplayDevice, - time, - ); - x11::xlib::XSync(display, 0); + ids: MasterPointerIds, + keyboard: &Mutex, +) -> Vec { + let mut released = Vec::new(); + let (held, mods) = virtual_master_input_state(display, ids.pointer_id); + if !held.is_empty() { + if let Some(pointer) = uinput_pointers().lock().unwrap().get(cursor_id).cloned() { + let mut pointer = pointer.lock().unwrap(); + for &button in &held { + if emit_button(&mut pointer, button, false).is_ok() { + released.push(format!("button{button}")); } } } - unsafe { x11::xlib::XFreeEventData(display, &mut cookie) }; + } + if mods.base != 0 { + let mut keyboard = keyboard.lock().unwrap(); + let mut any = false; + for code in MODIFIER_EVDEV_CODES { + any |= keyboard + .emit(&[InputEvent::new(EventType::KEY, code, 0)]) + .is_ok(); + } + if any { + released.push(format!("modifiers(base=0x{:x})", mods.base)); + } + } + if mods.locked != 0 || mods.latched != 0 { + // A locked/latched modifier on the virtual master (CapsLock leaked + // from an interrupted chord) would shift every later character. + unsafe { + let prev = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xlib::XkbLockModifiers(display, ids.keyboard_id as u32, 0xff, 0); + x11::xlib::XkbLatchModifiers(display, ids.keyboard_id as u32, 0xff, 0); + x11::xlib::XSetErrorHandler(prev); + } + released.push(format!( + "locked/latched modifiers (0x{:x}/0x{:x})", + mods.locked, mods.latched + )); + } + if !released.is_empty() { + unsafe { x11::xlib::XSync(display, 0) }; + tracing::warn!(cursor_id, ?released, "released stuck virtual input state"); + sleep(Duration::from_millis(20)); + } + released +} + +/// Release a device frozen by a synchronous grab; harmless when not frozen. +fn thaw_device(display: *mut x11::xlib::Display, device_id: i32) { + unsafe { + let prev = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xinput2::XIAllowEvents( + display, + device_id, + x11::xinput2::XIAsyncDevice, + x11::xlib::CurrentTime, + ); + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(prev); } } @@ -959,6 +1296,7 @@ fn ewmh_activate_window( } } +/// Prepare the exact X11 window without resetting an already focused child. fn prepare_x11_focus(display: *mut x11::xlib::Display, target: x11::xlib::Window) { let active = ewmh_active_window(display); if active != Some(target) { @@ -995,15 +1333,35 @@ fn prepare_x11_focus(display: *mut x11::xlib::Display, target: x11::xlib::Window /// `settle_ms` is retained as a compatibility hint and folded into the bounded /// confirmation timeout; it is no longer an unconditional sleep. /// If the target already holds focus, keep focus changes made by the action. +static FOREGROUND_EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +pub fn note_intentional_foreground() { + FOREGROUND_EPOCH.fetch_add(1, std::sync::atomic::Ordering::AcqRel); +} + +static FOREGROUND_TRANSACTION: Mutex<()> = Mutex::new(()); + pub fn with_x11_foreground( xid: u64, settle_ms: u64, body: impl FnOnce() -> Result, ) -> Result { + note_intentional_foreground(); + let _foreground = FOREGROUND_TRANSACTION + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let display = unsafe { x11::xlib::XOpenDisplay(ptr::null()) }; if display.is_null() { bail!("foreground_unavailable: cannot open DISPLAY to verify exact X11 input focus"); } + let _user_input = match user_input::Guard::start() { + Ok(guard) => guard, + Err(error) => { + unsafe { + x11::xlib::XCloseDisplay(display); + } + return Err(anyhow!("foreground_unavailable: physical-input observer could not start; no input was sent: {error}")); + } + }; let prior = ewmh_active_window(display); let mut prior_core_focus: x11::xlib::Window = 0; let mut prior_revert = 0; @@ -1020,7 +1378,12 @@ pub fn with_x11_foreground( let target_owner = x11_transient_owner(display, target); let restore_focus = prior != Some(target) || !x11_focus_is_within(display, target); prepare_x11_focus(display, target); + let mut interrupted = None; let focused = loop { + if let Err(error) = user_input::checkpoint() { + interrupted = Some(error); + break false; + } let active = ewmh_active_window(display) == Some(target); if active && x11_focus_is_within(display, target) { break true; @@ -1036,8 +1399,12 @@ pub fn with_x11_foreground( } std::thread::sleep(std::time::Duration::from_millis(10)); }; - let result = if focused { - body() + let result = if let Some(error) = interrupted { + Err(error) + } else if focused { + user_input::checkpoint() + .and_then(|()| body()) + .and_then(|value| user_input::checkpoint().map(|()| value)) } else { let active = ewmh_active_window(display).unwrap_or(0); Err(anyhow::anyhow!( @@ -1254,6 +1621,17 @@ fn emit_button(device: &mut VirtualDevice, button: u8, press: bool) -> Result<() Ok(()) } +/// Emit a release for `button` regardless of state; the kernel drops a +/// release for a button that is not held, so this is safe to call on every +/// exit path of a press train or gesture. +fn release_button_best_effort(device: &Arc>, button: u8) { + if let Ok(mut device) = device.lock() { + if let Err(error) = emit_button(&mut device, button, false) { + tracing::warn!("virtual pointer button {button} release failed: {error:#}"); + } + } +} + fn emit_relative_motion(device: &mut VirtualDevice, dx: i32, dy: i32) -> Result<()> { let mut events = Vec::with_capacity(2); if dx != 0 { @@ -1291,9 +1669,20 @@ fn emit_scroll(device: &mut VirtualDevice, horizontal: bool, value: i32) -> Resu } pub fn send_parallel_virtual_pointer_drags(drags: &[(String, VirtualPointerDrag)]) -> Result<()> { + let _op = mpx_op_guard( + drags + .first() + .map(|(id, _)| id.as_str()) + .unwrap_or("default"), + ); + { + let mut last_use = mpx_last_use().lock().unwrap(); + for (cursor_id, _) in drags { + last_use.insert(cursor_id.clone(), std::time::Instant::now()); + } + } let display = open_display()?; supports_parallel_pointer_injection(display)?; - let xi_opcode = xinput_opcode(display); struct ActiveDrag { cursor_id: String, @@ -1313,11 +1702,18 @@ pub fn send_parallel_virtual_pointer_drags(drags: &[(String, VirtualPointerDrag) let start_at = std::time::Instant::now() + Duration::from_millis(120); let mut active = Vec::with_capacity(drags.len()); - // Click-to-focus WMs grab buttons for XIAllMasterDevices, so the drag's - // press activates the target window exactly like a user click would. - // Remember the focus state and hand it back afterwards so parallel - // drags don't steal it. - let saved_focus = save_focus_state(display); + // A click-to-focus WM that grabs buttons for every master device would + // activate the target on the press. Remember the focus state and hand it + // back afterwards so parallel drags don't steal it (mutter grabs for the + // Virtual Core Pointer only, so there the restore is a no-op). + let mut saved_focus = save_focus_state(display); + saved_focus.target_pids = drags + .iter() + .map(|(_, drag)| drag.target_window) + .collect::>() + .into_iter() + .filter_map(crate::x11::window_pid) + .collect(); let result = (|| -> Result<()> { for (cursor_id, drag) in drags { @@ -1355,44 +1751,28 @@ pub fn send_parallel_virtual_pointer_drags(drags: &[(String, VirtualPointerDrag) std::thread::sleep(start_at - now); } - // Shield each drag from the WM's click-to-focus grab, then press. - // Per item: install a device-specific sync grab on the target window, - // warp, press, and immediately replay the frozen press so it reaches - // the app while the WM stays blind to it. We replay each press before - // emitting the next so only ONE device is ever frozen at a time — the - // X server drops replayed presses when several devices are frozen on - // the same window and replayed together. The few-ms stagger this adds - // to the presses is invisible; the concurrency that matters is motion. - // Shielding is mandatory: if install/replay fails, abort instead of - // continuing with a drag that could steal focus and rely on restore. - let mut shielded = std::collections::HashSet::new(); + // Press each drag straight from its virtual master (see + // `send_virtual_pointer_click` for why there is no XI2 shield grab: + // on this server the grab-and-replay swallowed the press). The press + // point must not be covered by another toplevel. for item in &active { - let opcode = xi_opcode.ok_or_else(|| { - anyhow!("parallel_mouse_drag requires XInput/XI2 shield grabs for no-focus-steal operation") - })?; - install_shield_grab( - display, - item.ids.pointer_id, - item.drag.target_window as x11::xlib::Window, - item.drag.button, - ) - .with_context(|| format!("shield grab failed for '{}'", item.cursor_id))?; - shielded.insert(item.ids.pointer_id); let start = *item.drag.path.first().unwrap_or(&(0, 0)); + if let PointCover::Occluded(occluded) = occluding_window( + display, + item.drag.target_window as x11::xlib::Window, + start.0, + start.1, + )? { + return Err(occluded.into()); + } + thaw_device(display, item.ids.pointer_id); warp_master_pointer(display, item.ids, start.0, start.1)?; { let mut device = item.device.lock().unwrap(); emit_button(&mut device, item.drag.button, true)?; } - let mut pending = std::collections::HashSet::from([item.ids.pointer_id]); - replay_shielded_presses(display, opcode, &mut pending, Duration::from_millis(1000)); - if !pending.is_empty() { - return Err(anyhow!( - "shield replay timed out before XI_ButtonPress arrived for '{}'", - item.cursor_id - )); - } } + sleep(Duration::from_millis(40)); while active.iter().any(|item| item.current_step < item.steps) { let now = std::time::Instant::now(); @@ -1450,28 +1830,22 @@ pub fn send_parallel_virtual_pointer_drags(drags: &[(String, VirtualPointerDrag) let mut device = item.device.lock().unwrap(); emit_button(&mut device, item.drag.button, false)?; } - - // Remove the shields now that the drag is done. The button is only - // grabbed for ButtonPress, so the shield is dormant during motion and - // release; this just stops it matching the next gesture's press. - for item in &active { - if shielded.contains(&item.ids.pointer_id) { - remove_shield_grab( - display, - item.ids.pointer_id, - item.drag.target_window as x11::xlib::Window, - item.drag.button, - ); - } - } Ok(()) })(); - // Remove the per-session masters before handing focus back: non-MPX-aware - // WMs (xfwm4, openbox) desync their focus bookkeeping while foreign - // master keyboards linger, and the next call recreates masters cheaply. - for (cursor_id, _) in drags { - forget_master_pointer(cursor_id); + // Whatever happened above, no virtual master may keep a button held + // (a stuck Button1 poisons the core modifier state for later chords). + for (cursor_id, drag) in drags { + if let Some(device) = uinput_pointers().lock().unwrap().get(cursor_id).cloned() { + release_button_best_effort(&device, drag.button); + } } + // The per-session master pair is retained for reuse (torn down on + // end_session / idle / startup reap). Creating and destroying an XI2 master + // plus hot-plugging a uinput slave on every call churns the XInput + // hierarchy hard enough to crash fragile toolkits (LibreOffice VCL); one + // long-lived pair per session avoids that and is cheaper. The focus is + // still saved and restored around each gesture. + let _ = drags; restore_focus_state(display, &saved_focus); unsafe { x11::xlib::XCloseDisplay(display); @@ -1481,16 +1855,22 @@ pub fn send_parallel_virtual_pointer_drags(drags: &[(String, VirtualPointerDrag) /// A discrete no-focus-steal pointer click driven through the same real-input /// pipeline as [`send_parallel_virtual_pointer_drags`] — MPX master pointer + -/// uinput slave + XI2 shield grab — reduced to a press/release (or a short -/// press/release train for `count` > 1) at one screen point. +/// uinput slave — reduced to a press/release (or a short press/release train +/// for `count` > 1) at one screen point. /// /// This is what lands **right / middle / double** clicks (and any left click -/// the AT-SPI path can't actuate) on XInput2 toolkits: GTK3/4 silently drop -/// synthetic `XSendEvent` pointer events and never see XTEST core events, so -/// those clicks are otherwise no-ops. Coordinates are screen-absolute; -/// `target_window` is the X11 window the shield grab is installed on so the WM -/// never sees the press and never steals focus. `button` is an X button number -/// (1=left, 2=middle, 3=right); `count` >= 1 (2 = double-click). +/// the AT-SPI path can't actuate) on XInput2 toolkits: GTK3/4, VCL, Qt and +/// Chromium silently drop synthetic `XSendEvent` pointer events, so those +/// clicks are otherwise no-ops. Coordinates are screen-absolute; +/// `target_window` is the X11 toplevel the caller means to hit. The press is +/// delivered straight from the virtual master: no XI2 shield grab. On GNOME +/// (mutter) the click-to-focus passive grab is installed for the Virtual Core +/// Pointer only, so a second master's press never activates the window; the +/// grab-and-replay "shield" that was meant to hide the press from the WM +/// instead swallowed it on this server (the replayed press never reached the +/// application) and left the device frozen. Focus is still saved and restored +/// around the click for WMs that do grab every master device. `button` is an +/// X button number (1=left, 2=middle, 3=right); `count` >= 1 (2 = double-click). #[derive(Clone, Debug)] pub struct VirtualPointerClick { pub target_window: u64, @@ -1500,20 +1880,792 @@ pub struct VirtualPointerClick { pub count: usize, } +/// The screen point the caller aimed at is covered by another toplevel, so a +/// real pointer press there would land on the covering window, not the +/// target. Refused before any input is sent. +#[derive(Debug, Clone)] +pub struct TargetOccluded { + pub target_window: u64, + pub covering_window: u64, + pub covering_title: String, + pub covering_pid: Option, + pub x: i32, + pub y: i32, +} + +impl std::fmt::Display for TargetOccluded { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "screen point ({}, {}) over window {} is covered by window {}{}{}; a real pointer \ + press there would land on the covering window", + self.x, + self.y, + self.target_window, + self.covering_window, + if self.covering_title.is_empty() { + String::new() + } else { + format!(" \"{}\"", self.covering_title) + }, + self.covering_pid + .map(|pid| format!(" (pid {pid})")) + .unwrap_or_default(), + ) + } +} +impl std::error::Error for TargetOccluded {} + +/// A mapped override-redirect toplevel: a popup menu, popover, combo list +/// or tooltip. The WM does not manage it, so it is absent from +/// `list_windows`; the pointer tools name it so a caller can walk it with +/// `get_window_state(pid, window_id=)`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PopupWindow { + pub window: u64, + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, + pub title: String, + pub pid: Option, +} + +impl PopupWindow { + pub fn to_json(&self) -> serde_json::Value { + serde_json::json!({ + "window_id": self.window, + "bounds": { "x": self.x, "y": self.y, "width": self.width, "height": self.height }, + "title": self.title, + "pid": self.pid, + }) + } + + /// `popup window 12345678 "Edit" (220x340 at 410,220)`. + pub fn describe(&self) -> String { + format!( + "popup window {}{} ({}x{} at {},{})", + self.window, + if self.title.is_empty() { + String::new() + } else { + format!(" \"{}\"", self.title) + }, + self.width, + self.height, + self.x, + self.y + ) + } +} + +/// The caller's window-local point does not lie inside the window: a +/// coordinate-frame mistake (screen pixels passed as window pixels, or the +/// wrong window_id), refused before any input is sent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PointOutsideWindow { + pub target_window: u64, + pub screen_x: i32, + pub screen_y: i32, + pub window_x: i32, + pub window_y: i32, + /// Screen-space `(x, y, width, height)` of the window. + pub bounds: (i32, i32, u32, u32), +} + +impl std::fmt::Display for PointOutsideWindow { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (bx, by, bw, bh) = self.bounds; + write!( + f, + "window-local point ({}, {}) of window {} resolves to screen ({}, {}), outside \ + the window's bounds (x={bx}, y={by}, {bw}x{bh}); no input was sent", + self.window_x, self.window_y, self.target_window, self.screen_x, self.screen_y + ) + } +} +impl std::error::Error for PointOutsideWindow {} + +/// Screen-space `(x, y, width, height)` of `window`, or `None` when it is +/// gone. +fn window_screen_bounds( + display: *mut x11::xlib::Display, + window: x11::xlib::Window, +) -> Option<(i32, i32, u32, u32)> { + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let mut root: x11::xlib::Window = 0; + let (mut gx, mut gy) = (0i32, 0i32); + let (mut width, mut height, mut border, mut depth) = (0u32, 0u32, 0u32, 0u32); + let rc = unsafe { + x11::xlib::XGetGeometry( + display, + window, + &mut root, + &mut gx, + &mut gy, + &mut width, + &mut height, + &mut border, + &mut depth, + ) + }; + let mut result = None; + if rc != 0 && root != 0 { + let (mut dx, mut dy) = (0i32, 0i32); + let mut child: x11::xlib::Window = 0; + let ok = unsafe { + x11::xlib::XTranslateCoordinates( + display, window, root, 0, 0, &mut dx, &mut dy, &mut child, + ) + }; + if ok != 0 { + result = Some((dx, dy, width, height)); + } + } + unsafe { + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(previous_handler); + } + result +} + +fn bounds_contain(bounds: (i32, i32, u32, u32), x: i32, y: i32) -> bool { + let (bx, by, bw, bh) = bounds; + x >= bx && y >= by && x < bx.saturating_add(bw as i32) && y < by.saturating_add(bh as i32) +} + +/// Describe a mapped override-redirect root child as a [`PopupWindow`]. +fn popup_info(display: *mut x11::xlib::Display, child: x11::xlib::Window) -> Option { + let mut attrs: x11::xlib::XWindowAttributes = unsafe { std::mem::zeroed() }; + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let rc = unsafe { x11::xlib::XGetWindowAttributes(display, child, &mut attrs) }; + unsafe { + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(previous_handler); + } + if rc == 0 || attrs.override_redirect == 0 || attrs.map_state != x11::xlib::IsViewable { + return None; + } + if attrs.width <= 1 || attrs.height <= 1 { + return None; + } + let pid = crate::x11::window_pid(child).or_else(|| { + window_children(display, child) + .into_iter() + .find_map(crate::x11::window_pid) + }); + Some(PopupWindow { + window: child, + x: attrs.x, + y: attrs.y, + width: attrs.width as u32, + height: attrs.height as u32, + title: window_title_for_report(display, child), + pid, + }) +} + +/// Every mapped override-redirect child of the root, bottom to top. +fn mapped_popups(display: *mut x11::xlib::Display) -> Vec { + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + window_children(display, root) + .into_iter() + .filter_map(|child| popup_info(display, child)) + .collect() +} + +/// The mapped override-redirect popup (menu, popover, combo list) under the +/// screen point, if any. +pub fn popup_under_screen_point(x: i32, y: i32) -> Option { + let display = open_display().ok()?; + let popup = root_child_under_point(display, x, y).and_then(|child| popup_info(display, child)); + unsafe { + x11::xlib::XCloseDisplay(display); + } + popup +} + +/// `xid` described as a popup when it is a mapped override-redirect window. +pub fn popup_window_info(xid: u64) -> Option { + let display = open_display().ok()?; + let popup = popup_info(display, xid as x11::xlib::Window); + unsafe { + x11::xlib::XCloseDisplay(display); + } + popup +} + +/// Mapped popups on the screen right now (bottom to top), for a caller that +/// wants "the menu that is open" without a window id. +pub fn mapped_popup_windows() -> Vec { + let Ok(display) = open_display() else { + return Vec::new(); + }; + let popups = mapped_popups(display); + unsafe { + x11::xlib::XCloseDisplay(display); + } + popups +} + +/// `_NET_WM_WINDOW_TYPE` says the popup is a tooltip / notification / DND +/// icon: mapped and override-redirect, but nobody's grab. +fn popup_is_passive(display: *mut x11::xlib::Display, window: x11::xlib::Window) -> bool { + let type_atom = intern_atom(display, "_NET_WM_WINDOW_TYPE"); + let passive: Vec = [ + "_NET_WM_WINDOW_TYPE_TOOLTIP", + "_NET_WM_WINDOW_TYPE_NOTIFICATION", + "_NET_WM_WINDOW_TYPE_DND", + ] + .iter() + .map(|name| intern_atom(display, name)) + .collect(); + let mut actual_type: x11::xlib::Atom = 0; + let mut actual_format: std::os::raw::c_int = 0; + let mut nitems: std::os::raw::c_ulong = 0; + let mut bytes_after: std::os::raw::c_ulong = 0; + let mut prop: *mut std::os::raw::c_uchar = ptr::null_mut(); + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let rc = unsafe { + x11::xlib::XGetWindowProperty( + display, + window, + type_atom, + 0, + 8, + 0, + x11::xlib::XA_ATOM, + &mut actual_type, + &mut actual_format, + &mut nitems, + &mut bytes_after, + &mut prop, + ) + }; + unsafe { + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(previous_handler); + } + let mut result = false; + if rc == 0 && !prop.is_null() { + if actual_format == 32 && nitems > 0 { + let atoms = unsafe { + std::slice::from_raw_parts(prop as *const std::os::raw::c_ulong, nitems as usize) + }; + result = atoms + .iter() + .any(|atom| passive.contains(&(*atom as x11::xlib::Atom))); + } + unsafe { x11::xlib::XFree(prop as *mut _) }; + } + result +} + +fn intern_atom(display: *mut x11::xlib::Display, name: &str) -> x11::xlib::Atom { + let cname = CString::new(name).unwrap_or_default(); + unsafe { x11::xlib::XInternAtom(display, cname.as_ptr(), 0) } +} + +/// The topmost mapped override-redirect popup that `pid` owns (a Qt combo +/// list or completer, a GTK/VCL menu) — the toolkit behind it holds an +/// active keyboard grab that makes the X server drop core key events from +/// any *other* master keyboard aimed at that client (`IsInterferingGrab`), +/// so the virtual master keyboard route is silently lost while it is up. +/// Tooltips / notifications are not grabs and do not count. +pub fn popup_of_pid(pid: u32) -> Option { + let display = open_display().ok()?; + let popup = mapped_popups(display) + .into_iter() + .rev() + .filter(|popup| popup.pid == Some(pid)) + .find(|popup| !popup_is_passive(display, popup.window as x11::xlib::Window)); + unsafe { + x11::xlib::XCloseDisplay(display); + } + popup +} + +/// `_NET_WM_PID` of the window holding the core keyboard focus, walking up +/// its X parents (the focus often sits on a child of the client toplevel). +pub fn core_focus_owner_pid() -> Option { + let display = open_display().ok()?; + let mut focus: x11::xlib::Window = 0; + let mut revert: std::os::raw::c_int = 0; + unsafe { x11::xlib::XGetInputFocus(display, &mut focus, &mut revert) }; + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + let mut owner = None; + let mut current = focus; + for _ in 0..8 { + if current == 0 || current == root || current == x11::xlib::PointerRoot as x11::xlib::Window + { + break; + } + if let Some(pid) = crate::x11::window_pid(current as u64) { + owner = Some(pid); + break; + } + let mut parent: x11::xlib::Window = 0; + let mut qroot: x11::xlib::Window = 0; + let mut children: *mut x11::xlib::Window = ptr::null_mut(); + let mut n: std::os::raw::c_uint = 0; + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let rc = unsafe { + x11::xlib::XQueryTree( + display, + current, + &mut qroot, + &mut parent, + &mut children, + &mut n, + ) + }; + unsafe { + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(previous_handler); + } + if !children.is_null() { + unsafe { x11::xlib::XFree(children as *mut _) }; + } + if rc == 0 || parent == current { + break; + } + current = parent; + } + unsafe { + x11::xlib::XCloseDisplay(display); + } + owner +} + +/// Delivery path name for keys sent on the core keyboard while the target's +/// own popup holds the keyboard grab. +pub const XTEST_CORE_GRAB_PATH: &str = "xtest_core_grab"; + +/// A popup of the target pid holds the keyboard grab and the core focus is +/// not the target's, so no route reaches the target without stealing input +/// from the focused application; refused before any key was sent. +#[derive(Debug, Clone)] +pub struct PopupKeyboardGrab { + pub pid: u32, + pub popup: PopupWindow, + pub focus_owner: Option, +} + +impl std::fmt::Display for PopupKeyboardGrab { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "pid {} has {} open, which holds a keyboard grab that drops keys from the virtual \ + keyboard, and the core focus belongs to {}; no key was sent", + self.pid, + self.popup.describe(), + match self.focus_owner { + Some(pid) => format!("pid {pid}"), + None => "no window".to_owned(), + } + ) + } +} +impl std::error::Error for PopupKeyboardGrab {} + +/// Keys for a target whose own popup (`popup`) holds the keyboard grab: the +/// X server routes the *core* keyboard to the grab holder, which is the +/// target itself, so XTest on the core keyboard reaches it without any +/// focus change — verified afterwards on the core focus and the active +/// window. Refused (`PopupKeyboardGrab`) when the core focus belongs to +/// another pid: then a missing grab would send the keys elsewhere. +pub fn send_keys_under_popup_grab( + pid: u32, + popup: PopupWindow, + send: impl FnOnce() -> Result<()>, +) -> Result { + let focus_owner = core_focus_owner_pid(); + if focus_owner != Some(pid) { + return Err(PopupKeyboardGrab { + pid, + popup, + focus_owner, + } + .into()); + } + let display = open_display()?; + let saved = save_focus_state(display); + let sent = send(); + unsafe { x11::xlib::XSync(display, 0) }; + let unchanged = focus_state_unchanged(display, &saved); + unsafe { + x11::xlib::XCloseDisplay(display); + } + sent?; + Ok(KeyboardDeliveryReport { + virtual_focus_held: true, + core_focus_unchanged: unchanged, + delivery_confirmed: true, + key_events: 0, + skipped_characters: Vec::new(), + focus_guard: None, + released_stuck: Vec::new(), + path: XTEST_CORE_GRAB_PATH, + grab_popup: Some(popup), + }) +} + +/// Dispatch metadata. Effects are checked by the next observation, not by input calls. +#[derive(Clone, Debug, Default)] +pub struct PointerEffect { + /// `_NET_ACTIVE_WINDOW` and the core input focus were the same after the + /// action as before it (sampled before the safety-net restore). + pub focus_unchanged: bool, + /// Legacy diagnostic field; actions no longer capture pixels to infer effects. + pub region_diff_pct: Option, + /// Real screen point the pointer acted at. + pub x: i32, + pub y: i32, + /// Mapped override-redirect toplevels (menus, popovers, combo lists, + /// tooltips) that appeared between the press and the post-check. A + /// context menu opening is a window change the contract accepts as + /// evidence of a landed click. + pub popups_appeared: usize, + /// Background focus-guard outcome when the tool ran the action under + /// [`focus_guard`]: whether the application moved the desktop focus after + /// the press (a menu grab, a dialog) and whether it was restored. + pub focus_guard: Option, + /// The point was under another window of the target's own pid (its + /// dialog over its main window) and the press went to that window. + pub retargeted_to: Option, + /// The popups behind `popups_appeared`, so the tool can name them. + pub popups: Vec, + /// The window-local point the caller asked for, when it had one. + pub window_point: Option<(i32, i32)>, + /// A toplevel of ANOTHER pid under the action point (a drop target, the + /// window a drag ended on): what it did during the action. + pub foreign_window: Option, +} + +/// What a toplevel of another pid under the action point did during the +/// action: its title before/after and the windows its pid mapped meanwhile. +/// The focus guard only watches the target pid, so a file dropped onto VLC +/// (auto-played, title changed) would otherwise be invisible. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ForeignWindowEffect { + pub window: u64, + pub pid: u32, + pub title_before: String, + pub title_after: String, + pub new_windows: Vec<(u64, String)>, +} + +impl ForeignWindowEffect { + pub fn changed(&self) -> bool { + self.title_before != self.title_after || !self.new_windows.is_empty() + } + + pub fn to_json(&self) -> serde_json::Value { + serde_json::json!({ + "window_id": self.window, + "pid": self.pid, + "title_before": self.title_before, + "title_after": self.title_after, + "new_windows": self.new_windows.iter().map(|(id, title)| serde_json::json!({"window_id": id, "title": title})).collect::>(), + }) + } + + /// One sentence for the tool text. + pub fn describe(&self) -> String { + let mut parts = Vec::new(); + if self.title_before != self.title_after { + parts.push(format!( + "its title changed from \"{}\" to \"{}\"", + self.title_before, self.title_after + )); + } + if !self.new_windows.is_empty() { + parts.push(format!( + "it opened {}", + self.new_windows + .iter() + .map(|(id, title)| format!("window {id} \"{title}\"")) + .collect::>() + .join(", ") + )); + } + format!( + "The point was over window {} of another application (pid {}); {}.", + self.window, + self.pid, + if parts.is_empty() { + "it did not visibly react (title unchanged, no new window)".to_owned() + } else { + parts.join(" and ") + } + ) + } +} + +impl PointerEffect { + /// A region that changed more than this has plainly reacted to the click + /// (a menu, a caret, a selection, a pressed button). Below it the click + /// may still have landed without a visible reaction. + pub const LANDED_THRESHOLD_PCT: f64 = 0.4; + + pub fn landed(&self) -> bool { + self.popups_appeared > 0 + || self + .region_diff_pct + .is_some_and(|pct| pct >= Self::LANDED_THRESHOLD_PCT) + } +} + +/// Debug overrides for the multi-click cadence (milliseconds), read once. +fn click_cadence() -> (u64, u64) { + fn env_ms(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + } + ( + env_ms("CUA_MPX_PRESS_MS", 12), + env_ms("CUA_MPX_CLICK_GAP_MS", CLICK_DELAY_MS), + ) +} + +/// Best-effort `_NET_WM_NAME` / `WM_NAME` of a window (or of a child, for a +/// WM frame), for naming a covering window in a refusal. +fn window_title_for_report(display: *mut x11::xlib::Display, window: x11::xlib::Window) -> String { + fn name_of(display: *mut x11::xlib::Display, window: x11::xlib::Window) -> Option { + unsafe { + let net_name = + x11::xlib::XInternAtom(display, c"_NET_WM_NAME".as_ptr(), x11::xlib::True); + for prop in [net_name, x11::xlib::XA_WM_NAME] { + if prop == 0 { + continue; + } + let mut type_ret: x11::xlib::Atom = 0; + let mut format_ret = 0; + let mut nitems: std::os::raw::c_ulong = 0; + let mut bytes_after: std::os::raw::c_ulong = 0; + let mut data: *mut std::os::raw::c_uchar = std::ptr::null_mut(); + let rc = x11::xlib::XGetWindowProperty( + display, + window, + prop, + 0, + 256, + x11::xlib::False, + x11::xlib::AnyPropertyType as x11::xlib::Atom, + &mut type_ret, + &mut format_ret, + &mut nitems, + &mut bytes_after, + &mut data, + ); + if rc == x11::xlib::Success as i32 && !data.is_null() { + let text = if format_ret == 8 && nitems > 0 { + Some( + String::from_utf8_lossy(std::slice::from_raw_parts( + data, + nitems as usize, + )) + .into_owned(), + ) + } else { + None + }; + x11::xlib::XFree(data as *mut _); + if let Some(text) = text.filter(|t| !t.trim().is_empty()) { + return Some(text); + } + } + } + } + None + } + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let mut title = name_of(display, window); + if title.is_none() { + title = window_children(display, window) + .into_iter() + .rev() + .find_map(|kid| name_of(display, kid)); + } + unsafe { + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(previous_handler); + } + title.unwrap_or_default() +} + +/// Direct children of `window` (empty on error). Caller installs the error +/// handler. +fn window_children( + display: *mut x11::xlib::Display, + window: x11::xlib::Window, +) -> Vec { + let mut root_ret: x11::xlib::Window = 0; + let mut parent: x11::xlib::Window = 0; + let mut children: *mut x11::xlib::Window = ptr::null_mut(); + let mut count: std::os::raw::c_uint = 0; + let rc = unsafe { + x11::xlib::XQueryTree( + display, + window, + &mut root_ret, + &mut parent, + &mut children, + &mut count, + ) + }; + if rc == 0 || children.is_null() { + return Vec::new(); + } + let kids = unsafe { std::slice::from_raw_parts(children, count as usize) }.to_vec(); + unsafe { x11::xlib::XFree(children as *mut _) }; + kids +} + +/// The toplevel (root child) the virtual pointer would press on at `(x, y)` +/// when the caller means `window`. `Ok(None)` when the point is over the +/// target itself (its WM frame) or over an override-redirect popup (a menu or +/// combo list the click is meant to reach, or the agent-cursor overlay); +/// `Ok(Some(_))` names the window that covers the point instead. +fn occluding_window( + display: *mut x11::xlib::Display, + window: x11::xlib::Window, + x: i32, + y: i32, +) -> Result { + if let Some(bounds) = window_screen_bounds(display, window) { + if !bounds_contain(bounds, x, y) { + return Ok(PointCover::Outside(PointOutsideWindow { + target_window: window, + screen_x: x, + screen_y: y, + window_x: x - bounds.0, + window_y: y - bounds.1, + bounds, + })); + } + } + let under = root_child_under_point(display, x, y); + let frame = root_child_of(display, window); + match (under, frame) { + (Some(under), Some(frame)) if under != frame && !is_override_redirect(display, under) => { + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let covering_client = window_children(display, under) + .into_iter() + .find(|kid| crate::x11::window_pid(*kid).is_some()); + let covering_pid = crate::x11::window_pid(under) + .or_else(|| covering_client.and_then(crate::x11::window_pid)); + unsafe { + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(previous_handler); + } + let target_pid = crate::x11::window_pid(window); + let title = window_title_for_report(display, under); + // The app's own window (a dialog, a file chooser) sits over the + // point: the press goes where the caller can see it, on that + // window, and the effect reports the retarget. + if target_pid.is_some() && covering_pid == target_pid { + return Ok(PointCover::SameApp(SameAppCover { + window: covering_client.unwrap_or(under), + title, + })); + } + Ok(PointCover::Occluded(TargetOccluded { + target_window: window, + covering_window: under, + covering_title: title, + covering_pid, + x, + y, + })) + } + (Some(_), Some(_)) | (None, _) => Ok(PointCover::Clear), + (Some(_), None) => bail!("target window {window} is not mapped on this screen"), + } +} + +/// What sits over the screen point a background pointer action aims at. +#[derive(Debug, Clone)] +enum PointCover { + /// The target (or a popup meant to receive the press). + Clear, + /// A window of the target's own pid; the press is retargeted to it. + SameApp(SameAppCover), + /// Another application's window; refused. + Occluded(TargetOccluded), + /// The point is not inside the target window at all; refused. + Outside(PointOutsideWindow), +} + +/// The target application's own window that received a press aimed at a +/// point of another of its windows (its dialog over its main window). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SameAppCover { + /// Client window id (the `list_windows` id, not the WM frame). + pub window: u64, + pub title: String, +} + +/// True when neither the EWMH active window nor the core focus moved since +/// `saved` was taken. +fn focus_state_unchanged(display: *mut x11::xlib::Display, saved: &SavedFocus) -> bool { + unsafe { x11::xlib::XSync(display, 0) }; + let mut core_focus: x11::xlib::Window = 0; + let mut revert_to: std::os::raw::c_int = 0; + unsafe { + x11::xlib::XGetInputFocus(display, &mut core_focus, &mut revert_to); + } + ewmh_active_window(display) == saved.ewmh_active && core_focus == saved.core_focus +} + +fn pointer_effect( + display: *mut x11::xlib::Display, + saved: &SavedFocus, + x: i32, + y: i32, +) -> PointerEffect { + PointerEffect { + focus_unchanged: focus_state_unchanged(display, saved), + x, + y, + ..Default::default() + } +} + /// Land a discrete click via the MPX real-input pipeline (see -/// [`VirtualPointerClick`]). Mirrors the per-item press/replay logic of -/// `send_parallel_virtual_pointer_drags`: install a device-specific synchronous -/// XI2 shield grab on the target window, warp the master pointer, then for each -/// press freeze→replay it so the application receives a real button event while -/// the WM stays blind to it. The master is torn down and focus restored on exit -/// (matching the drag) to keep non-MPX WMs' focus bookkeeping consistent. +/// [`VirtualPointerClick`]). Refuses with [`TargetOccluded`] when another +/// toplevel covers the point. Returns the post-checks the tool reports. pub fn send_virtual_pointer_click(cursor_id: &str, click: &VirtualPointerClick) -> Result<()> { + send_virtual_pointer_click_report(cursor_id, click).map(|_| ()) +} + +pub fn send_virtual_pointer_click_report( + cursor_id: &str, + click: &VirtualPointerClick, +) -> Result { + let _op = mpx_op_guard(cursor_id); + mpx_last_use() + .lock() + .unwrap() + .insert(cursor_id.to_owned(), std::time::Instant::now()); let display = open_display()?; supports_parallel_pointer_injection(display)?; - let xi_opcode = xinput_opcode(display); - let saved_focus = save_focus_state(display); + let mut saved_focus = save_focus_state(display); + saved_focus.target_pids = vec![click.target_window] + .into_iter() + .filter_map(crate::x11::window_pid) + .collect(); - let result = (|| -> Result<()> { + let result = (|| -> Result { + let window = click.target_window as x11::xlib::Window; + let retargeted_to = match occluding_window(display, window, click.x, click.y)? { + PointCover::Clear => None, + PointCover::SameApp(cover) => Some(cover), + PointCover::Occluded(occluded) => return Err(occluded.into()), + PointCover::Outside(outside) => return Err(outside.into()), + }; let ids = ensure_master_pointer(cursor_id)?; let device = uinput_pointers() .lock() @@ -1521,50 +2673,139 @@ pub fn send_virtual_pointer_click(cursor_id: &str, click: &VirtualPointerClick) .get(cursor_id) .cloned() .ok_or_else(|| anyhow!("missing uinput pointer for '{cursor_id}'"))?; - let opcode = xi_opcode - .ok_or_else(|| anyhow!("no-focus-steal click requires XInput/XI2 shield grabs"))?; - - let window = click.target_window as x11::xlib::Window; - install_shield_grab(display, ids.pointer_id, window, click.button) - .with_context(|| format!("shield grab failed for '{cursor_id}'"))?; - // Run the press train under a guard so the shield is always removed, - // even on an early error mid-train. - let click_result = (|| -> Result<()> { - warp_master_pointer(display, ids, click.x, click.y)?; - let count = click.count.max(1); + let count = click.count.max(1); + // A device left frozen by an earlier synchronous grab (a WM that never + // replayed) would queue this press forever. + thaw_device(display, ids.pointer_id); + warp_master_pointer(display, ids, click.x, click.y)?; + let (press_ms, gap_ms) = click_cadence(); + let train = (|| -> Result<()> { for i in 0..count { { let mut device = device.lock().unwrap(); emit_button(&mut device, click.button, true)?; - } - // The shield grab freezes the device on every press; drain and - // replay this one so it reaches the app (and re-arms for the - // next press in a multi-click train). - let mut pending = std::collections::HashSet::from([ids.pointer_id]); - replay_shielded_presses(display, opcode, &mut pending, Duration::from_millis(1000)); - if !pending.is_empty() { - return Err(anyhow!( - "shield replay timed out before XI_ButtonPress arrived for '{cursor_id}'" - )); - } - { - let mut device = device.lock().unwrap(); + // A real press and release are separate evdev frames; give + // the toolkit a press it can see before the release lands. + sleep(Duration::from_millis(press_ms)); emit_button(&mut device, click.button, false)?; } // Multi-click cadence: keep press→press well under the toolkit // double-click threshold (GTK default 250 ms) so count=2 lands // as a real double-click, not two singles. if count > 1 && i + 1 < count { - sleep(Duration::from_millis(CLICK_DELAY_MS)); + sleep(Duration::from_millis(gap_ms)); } } Ok(()) })(); - remove_shield_grab(display, ids.pointer_id, window, click.button); - click_result + // Never leave the button held: a stuck Button1 on the virtual master + // sets bit 8 of the core modifier state (Orca's modifier), so every + // later virtual-keyboard chord would open Orca's preferences. + release_button_best_effort(&device, click.button); + train?; + let mut effect = pointer_effect(display, &saved_focus, click.x, click.y); + effect.retargeted_to = retargeted_to; + Ok(effect) + })(); + + restore_focus_state(display, &saved_focus); + unsafe { + x11::xlib::XCloseDisplay(display); + } + result +} + +/// One held drag on the session's virtual master pointer: press at +/// `path[0]`, glide through the waypoints, release at the end. Same delivery +/// rules as [`send_virtual_pointer_click`] (no shield grab, occlusion refusal +/// at the press point, focus saved and restored). +pub fn send_virtual_pointer_drag( + cursor_id: &str, + drag: &VirtualPointerDrag, +) -> Result { + let _op = mpx_op_guard(cursor_id); + mpx_last_use() + .lock() + .unwrap() + .insert(cursor_id.to_owned(), std::time::Instant::now()); + let display = open_display()?; + supports_parallel_pointer_injection(display)?; + let mut saved_focus = save_focus_state(display); + saved_focus.target_pids = vec![drag.target_window] + .into_iter() + .filter_map(crate::x11::window_pid) + .collect(); + + let result = (|| -> Result { + if drag.path.len() < 2 { + bail!("drag path needs at least 2 points"); + } + let window = drag.target_window as x11::xlib::Window; + let start = drag.path[0]; + let end = drag.path[drag.path.len() - 1]; + let retargeted_to = match occluding_window(display, window, start.0, start.1)? { + PointCover::Clear => None, + PointCover::SameApp(cover) => Some(cover), + PointCover::Occluded(occluded) => return Err(occluded.into()), + PointCover::Outside(outside) => return Err(outside.into()), + }; + let ids = ensure_master_pointer(cursor_id)?; + let device = uinput_pointers() + .lock() + .unwrap() + .get(cursor_id) + .cloned() + .ok_or_else(|| anyhow!("missing uinput pointer for '{cursor_id}'"))?; + let (cum, total) = path_cumulative(&drag.path); + let steps = drag.steps.max(1); + let step_delay = Duration::from_millis(drag.duration_ms / steps as u64); + thaw_device(display, ids.pointer_id); + warp_master_pointer(display, ids, start.0, start.1)?; + let gesture = (|| -> Result<()> { + { + let mut device = device.lock().unwrap(); + emit_button(&mut device, drag.button, true)?; + } + // Let the toolkit register the press (and arm its drag threshold) + // before the first motion. + sleep(Duration::from_millis(40)); + let (mut last_x, mut last_y) = start; + for step in 1..=steps { + let t = step as f64 / steps as f64; + let (ix, iy) = point_on_path(&drag.path, &cum, total, t); + let (dx, dy) = (ix - last_x, iy - last_y); + if dx != 0 || dy != 0 { + let mut device = device.lock().unwrap(); + emit_relative_motion(&mut device, dx, dy)?; + crate::overlay::send_command_for( + cursor_id.to_owned(), + cursor_overlay::OverlayCommand::SnapTo { + x: ix as f64, + y: iy as f64, + heading_radians: Some((dy as f64).atan2(dx as f64)), + }, + ); + } + last_x = ix; + last_y = iy; + sleep(step_delay); + } + // Relative motion accumulates libinput rounding; pin the release + // to the exact end point before letting go. + warp_master_pointer(display, ids, end.0, end.1)?; + sleep(Duration::from_millis(20)); + let mut device = device.lock().unwrap(); + emit_button(&mut device, drag.button, false)?; + Ok(()) + })(); + // Release on every exit path (see `send_virtual_pointer_click`). + release_button_best_effort(&device, drag.button); + gesture?; + let mut effect = pointer_effect(display, &saved_focus, end.0, end.1); + effect.retargeted_to = retargeted_to; + Ok(effect) })(); - forget_master_pointer(cursor_id); restore_focus_state(display, &saved_focus); unsafe { x11::xlib::XCloseDisplay(display); @@ -1593,9 +2834,14 @@ pub struct VirtualPointerScroll { /// point, then emits `|ticks|` wheel detents on the uinput slave. The master is /// torn down and focus restored on exit, matching the click/drag paths. pub fn send_virtual_pointer_scroll(cursor_id: &str, scroll: &VirtualPointerScroll) -> Result<()> { + let _op = mpx_op_guard(cursor_id); let display = open_display()?; supports_parallel_pointer_injection(display)?; - let saved_focus = save_focus_state(display); + let mut saved_focus = save_focus_state(display); + saved_focus.target_pids = vec![scroll.target_window] + .into_iter() + .filter_map(crate::x11::window_pid) + .collect(); let result = (|| -> Result<()> { let ids = ensure_master_pointer(cursor_id)?; @@ -1622,7 +2868,7 @@ pub fn send_virtual_pointer_scroll(cursor_id: &str, scroll: &VirtualPointerScrol Ok(()) })(); - forget_master_pointer(cursor_id); + let _ = cursor_id; // master pair retained for reuse (see the drag path). restore_focus_state(display, &saved_focus); unsafe { x11::xlib::XCloseDisplay(display); @@ -1636,15 +2882,20 @@ struct SavedFocus { ewmh_active: Option, core_focus: x11::xlib::Window, core_revert_to: std::os::raw::c_int, + target_pids: Vec, + user_input: Option, } fn save_focus_state(display: *mut x11::xlib::Display) -> SavedFocus { + let user_input = user_input::Observer::start().ok(); let mut core_focus: x11::xlib::Window = 0; let mut core_revert_to: std::os::raw::c_int = 0; unsafe { x11::xlib::XGetInputFocus(display, &mut core_focus, &mut core_revert_to); } SavedFocus { + user_input, + target_pids: Vec::new(), ewmh_active: ewmh_active_window(display), core_focus, core_revert_to, @@ -1659,77 +2910,43 @@ unsafe extern "C" fn ignore_x_error( } fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) { - // Let the release/focus events from the drag settle before reading the - // post-drag state, so we don't race the WM's own focus update. - unsafe { x11::xlib::XSync(display, 0) }; - - if let Some(prev) = saved.ewmh_active { - // EWMH path: ask the WM to re-activate, so its active-window - // bookkeeping (decorations, stacking) stays consistent. The WM - // processes its own click-to-focus for the drag asynchronously and - // can re-activate the dragged window even after one re-activation of - // ours has landed — so don't stop at first success: require the - // active window to hold stable for consecutive checks, re-sending on - // every regression, within a bounded budget. - sleep(Duration::from_millis(300)); - let mut stable = 0; - for attempt in 0..15 { - let now = ewmh_active_window(display); - if now == Some(prev) { - stable += 1; - if stable >= 3 { - return; - } - } else { - stable = 0; - // MPX clicks can leave a core-protocol WM believing the - // dragged window is focused while the core focus never moved - // there: its XSetInputFocus for our activation is then a - // no-op, no FocusIn arrives, and its bookkeeping never - // updates. Bounce the core focus onto the window the WM - // believes active so the activation produces a real focus - // transition the WM can observe. - if attempt >= 2 { - if let Some(now_win) = now { - unsafe { - let prev_handler = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); - x11::xlib::XSetInputFocus( - display, - now_win, - x11::xlib::RevertToParent, - x11::xlib::CurrentTime, - ); - x11::xlib::XSync(display, 0); - x11::xlib::XSetErrorHandler(prev_handler); - } - sleep(Duration::from_millis(100)); - } - } - ewmh_activate_window(display, prev, now.unwrap_or(0)); - } - sleep(Duration::from_millis(200)); - } - if stable == 0 { - tracing::warn!("focus restore: WM did not re-activate 0x{prev:x}"); - } + if saved + .user_input + .as_ref() + .is_none_or(user_input::Observer::interrupted) + { return; } - - // No EWMH WM (bare X / minimal WM): restore the core input focus - // directly. The saved window may have been destroyed meanwhile, and - // Xlib's default error handler exits the process on BadWindow, so the - // restore runs under a scoped ignore-errors handler. - if saved.core_focus == 0 { + unsafe { x11::xlib::XSync(display, 0) }; + let belongs_to_target = + |window| crate::x11::window_pid(window).is_some_and(|pid| saved.target_pids.contains(&pid)); + if let Some(previous) = saved.ewmh_active { + let current = ewmh_active_window(display); + if current.is_some_and(|window| window != previous && !belongs_to_target(window)) { + return; + } + if current != Some(previous) { + ewmh_activate_window(display, previous, current.unwrap_or(0)); + } + } + if saved.core_focus <= 1 { return; } unsafe { - let mut now_focus: x11::xlib::Window = 0; - let mut now_revert: std::os::raw::c_int = 0; - x11::xlib::XGetInputFocus(display, &mut now_focus, &mut now_revert); - if now_focus == saved.core_focus { + let mut current = 0; + let mut revert = 0; + x11::xlib::XGetInputFocus(display, &mut current, &mut revert); + if current == saved.core_focus || !belongs_to_target(current) { return; } - let prev_handler = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + if saved + .user_input + .as_ref() + .is_none_or(user_input::Observer::interrupted) + { + return; + } + let previous_handler = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); x11::xlib::XSetInputFocus( display, saved.core_focus, @@ -1737,7 +2954,7 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) { x11::xlib::CurrentTime, ); x11::xlib::XSync(display, 0); - x11::xlib::XSetErrorHandler(prev_handler); + x11::xlib::XSetErrorHandler(previous_handler); } } @@ -1778,11 +2995,58 @@ fn deepest_child_at_point( Ok((window, local_x, local_y)) } -fn resolve_event_target(conn: &RustConnection, xid: u64, x: i32, y: i32) -> Result { +fn window_path_at_point( + conn: &RustConnection, + window: Window, + local_x: i32, + local_y: i32, +) -> Result> { + let attributes = conn.get_window_attributes(window)?.reply()?; + let mut path = vec![(window, local_x, local_y, attributes.all_event_masks)]; + let tree = conn.query_tree(window)?.reply()?; + for child in tree.children.iter().rev() { + let Ok(geom) = conn.get_geometry(*child)?.reply() else { + continue; + }; + if !point_in_rect(local_x, local_y, &geom) { + continue; + } + let child_x = local_x - geom.x as i32; + let child_y = local_y - geom.y as i32; + path.extend(window_path_at_point(conn, *child, child_x, child_y)?); + break; + } + Ok(path) +} + +fn deepest_event_target( + path: &[(Window, i32, i32, EventMask)], + required_masks: EventMask, +) -> (Window, i32, i32) { + path.iter() + .rev() + .find(|(_, _, _, selected)| selected.contains(required_masks)) + .or_else(|| path.last()) + .map(|&(window, x, y, _)| (window, x, y)) + .expect("window path always contains its toplevel") +} + +fn click_event_target(path: &[(Window, i32, i32, EventMask)]) -> (Window, i32, i32) { + deepest_event_target(path, EventMask::BUTTON_PRESS) +} + +fn build_event_target( + conn: &RustConnection, + xid: u64, + x: i32, + y: i32, + window: Window, + local_x: i32, + local_y: i32, +) -> Result { let top = xid as Window; let root = conn.setup().roots[0].root; let root_pos = conn.translate_coordinates(top, root, 0, 0)?.reply()?; - let (window, local_x, local_y) = deepest_child_at_point(conn, top, x, y)?; Ok(EventTarget { window, local_x: local_x as i16, @@ -1792,6 +3056,28 @@ fn resolve_event_target(conn: &RustConnection, xid: u64, x: i32, y: i32) -> Resu }) } +fn resolve_event_target(conn: &RustConnection, xid: u64, x: i32, y: i32) -> Result { + let top = xid as Window; + let (window, local_x, local_y) = deepest_child_at_point(conn, top, x, y)?; + build_event_target(conn, xid, x, y, window, local_x, local_y) +} + +fn resolve_click_event_target( + conn: &RustConnection, + xid: u64, + x: i32, + y: i32, +) -> Result { + let top = xid as Window; + let path = window_path_at_point(conn, top, x, y)?; + // Button bindings commonly select only ButtonPress (Tk's is one + // example). Route the complete synthetic click to that same client-owned + // window: splitting release onto a different geometric leaf does not form a + // coherent toolkit click sequence. + let target = click_event_target(&path); + build_event_target(conn, xid, x, y, target.0, target.1, target.2) +} + fn button_state_mask(button: u8) -> KeyButMask { match button { 1 => KeyButMask::BUTTON1, @@ -1803,16 +3089,9 @@ fn button_state_mask(button: u8) -> KeyButMask { } } -/// Open an X11 connection for background input injection, failing *loudly* and -/// actionably when input cannot be delivered — rather than letting a pure -/// Wayland session fall through to an X11 path that silently no-ops yet reports -/// success (#1921). On a pure Wayland session with the native backend off, this -/// returns a clear error naming the fix; otherwise it connects and, on any -/// connect failure, surfaces DISPLAY so the cause is diagnosable. +/// Open an authenticated X11 connection. XWayland X11 applications use the same path. fn connect_x11_for_input() -> Result<(RustConnection, usize)> { - if let Some(reason) = crate::wayland::wayland_input_unavailable_reason() { - bail!("{reason}"); - } + crate::require_x11()?; RustConnection::connect(None).map_err(|e| { anyhow!( "cannot inject input: X11 connection failed (DISPLAY={:?}): {e}", @@ -1836,16 +3115,26 @@ pub fn send_focus_in(xid: u64) -> Result<()> { mode: NotifyMode::NORMAL, }; - conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_in)?; + conn.send_event(false, window, EventMask::FOCUS_CHANGE, focus_in)?; conn.flush()?; Ok(()) } /// Send a synthetic FocusOut event to restore focus state after send_focus_in. pub fn send_focus_out(xid: u64) -> Result<()> { - let (conn, _) = connect_x11_for_input()?; + let (conn, screen_num) = connect_x11_for_input()?; let window = xid as u32; + let mut focus = conn.get_input_focus()?.reply()?.focus; + for _ in 0..32 { + if focus == xid as u32 { + return Ok(()); + } + if focus == 0 || focus == conn.setup().roots[screen_num].root { + break; + } + focus = conn.query_tree(focus)?.reply()?.parent; + } let focus_out = FocusOutEvent { response_type: FOCUS_OUT_EVENT, detail: NotifyDetail::NONLINEAR, @@ -1854,11 +3143,21 @@ pub fn send_focus_out(xid: u64) -> Result<()> { mode: NotifyMode::NORMAL, }; - conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_out)?; + conn.send_event(false, window, EventMask::FOCUS_CHANGE, focus_out)?; conn.flush()?; Ok(()) } +/// Round-trip to the X server before the short-lived connection that sent +/// synthetic events is dropped. A bare flush is not enough: the server may +/// discard the tail of a client's request stream when the connection closes, +/// which drops the last key or button event (observed under Xvfb). +fn settle_synthetic_events(conn: &RustConnection) { + if let Ok(cookie) = conn.get_input_focus() { + let _ = cookie.reply(); + } +} + /// Send a button click (down + up) to a window at window-local coordinates. pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result<()> { send_click_with_modifiers(xid, x, y, count, button, &[]) @@ -1880,7 +3179,7 @@ pub fn send_click_with_modifiers( let modifier_state = modifiers_to_state(modifiers); for _ in 0..count { - let target = resolve_event_target(&conn, xid, x, y)?; + let target = resolve_click_event_target(&conn, xid, x, y)?; let press = ButtonPressEvent { response_type: BUTTON_PRESS_EVENT, detail: button, @@ -1915,15 +3214,20 @@ pub fn send_click_with_modifiers( same_screen: true, }; - conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?; + // The event mask selects recipients independently of the payload type. + // Select the press client for both payloads so a press-only toolkit + // binding (such as Tk's ) receives one coherent click. + conn.send_event(false, target.window, EventMask::BUTTON_PRESS, press)? + .check()?; sleep(Duration::from_millis(CLICK_DELAY_MS)); - conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?; - conn.flush()?; + conn.send_event(false, target.window, EventMask::BUTTON_PRESS, release)? + .check()?; if count > 1 { sleep(Duration::from_millis(80)); } } + settle_synthetic_events(&conn); Ok(()) } @@ -1934,14 +3238,13 @@ pub fn send_click_with_modifiers( /// `button` — X11 button number (1=left, 2=middle, 3=right). pub fn send_drag( xid: u64, - from_x: i32, - from_y: i32, - to_x: i32, - to_y: i32, + from: (i32, i32), + to: (i32, i32), duration_ms: u64, steps: usize, button: u8, ) -> Result<()> { + let ((from_x, from_y), (to_x, to_y)) = (from, to); let (conn, _) = connect_x11_for_input()?; let root = conn.setup().roots[0].root; let steps = steps.max(1); @@ -1968,7 +3271,7 @@ pub fn send_drag( state: KeyButMask::from(0u16), same_screen: true, }; - conn.send_event(false, press_target.window, EventMask::BUTTON_PRESS, &press)?; + conn.send_event(false, press_target.window, EventMask::BUTTON_PRESS, press)?; conn.flush()?; sleep(Duration::from_millis(CLICK_DELAY_MS)); @@ -1993,7 +3296,7 @@ pub fn send_drag( state: button_state_mask(button), same_screen: true, }; - conn.send_event(false, target.window, EventMask::POINTER_MOTION, &motion)?; + conn.send_event(false, target.window, EventMask::POINTER_MOTION, motion)?; conn.flush()?; if step_delay_ms > 0 { sleep(Duration::from_millis(step_delay_ms)); @@ -2021,9 +3324,10 @@ pub fn send_drag( false, release_target.window, EventMask::BUTTON_RELEASE, - &release, + release, )?; conn.flush()?; + settle_synthetic_events(&conn); Ok(()) } @@ -2046,8 +3350,9 @@ pub fn send_button_down(xid: u64, x: i32, y: i32, button: u8) -> Result<()> { state: KeyButMask::from(0u16), same_screen: true, }; - conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?; - conn.flush()?; + conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)? + .check()?; + settle_synthetic_events(&conn); Ok(()) } @@ -2072,8 +3377,9 @@ pub fn send_motion(xid: u64, x: i32, y: i32, button: Option) -> Result<()> { .unwrap_or_else(|| KeyButMask::from(0u16)), same_screen: true, }; - conn.send_event(false, target.window, EventMask::POINTER_MOTION, &motion)?; + conn.send_event(false, target.window, EventMask::POINTER_MOTION, motion)?; conn.flush()?; + settle_synthetic_events(&conn); Ok(()) } @@ -2096,8 +3402,9 @@ pub fn send_button_up(xid: u64, x: i32, y: i32, button: u8) -> Result<()> { state: button_state_mask(button), same_screen: true, }; - conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?; - conn.flush()?; + conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)? + .check()?; + settle_synthetic_events(&conn); Ok(()) } @@ -2107,64 +3414,149 @@ pub fn send_type_text(xid: u64, text: &str) -> Result<()> { } /// Type a string with an additional `inter_char_ms` delay between each character. +fn keysym_for_char(ch: char) -> u32 { + match ch { + '\n' | '\r' => 0xff0d, + '\t' => 0xff09, + c if (c as u32) < 0x100 => c as u32, + c => 0x0100_0000 | c as u32, + } +} +fn type_text_with_preflight( + text: &str, + keymap: &GetKeyboardMappingReply, + mut remap: impl FnMut(char, &mut GetKeyboardMappingReply) -> Result<(u8, Option)>, + mut send: impl FnMut(u8, bool) -> Result<()>, + finish: impl FnOnce() -> Result<()>, +) -> Result> { + let mut mapping = keymap.clone(); + let mut guards = Vec::new(); + let mut resolved = std::collections::HashMap::new(); + let mut keys = Vec::new(); + for ch in text.chars() { + let key = match resolved.get(&ch).copied() { + Some(key) => key, + None => { + let key = match char_to_keycode_shift(&mapping, keysym_for_char(ch)) { + Some(key) => key, + None => { + let (keycode, guard) = remap(ch, &mut mapping).map_err(|error| { + let needed = text + .chars() + .map(keysym_for_char) + .filter(|keysym| !keymap.keysyms.contains(keysym)) + .collect::>() + .len(); + anyhow::anyhow!( + "typed 0 of {} characters: the text needs {needed} spare keycodes: {error:#}", + text.chars().count() + ) + })?; + guards.extend(guard); + (keycode, false) + } + }; + resolved.insert(ch, key); + key + } + }; + keys.push(key); + } + let mut dispatched = false; + let sent = keys.into_iter().try_for_each(|(keycode, shift)| { + send(keycode, shift)?; + dispatched = true; + Ok(()) + }); + // Even a partial dispatch must keep its mappings while the client drains. + let finished = finish(); + let result = sent.and(finished); + if dispatched { + result.map_err(|error| keymap_barrier::unconfirmed(format!("{error:#}")))?; + } else { + result?; + } + Ok(guards) +} + pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Result<()> { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let (conn, _) = connect_x11_for_input()?; let window = xid as u32; let root = conn.setup().roots[0].root; let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + let barrier = KeymapBarrier::prepare( + &conn, + Some(window), + &mapping, + text.chars().map(keysym_for_char), + )?; + let _guards = type_text_with_preflight( + text, + &mapping, + |ch, mapping| keycode_for_keysym(&conn, mapping, keysym_for_char(ch), &ch.to_string()), + |keycode, needs_shift| { + user_input::checkpoint()?; + keymap_barrier::dispatch(|| { + let state = if needs_shift { + KeyButMask::SHIFT + } else { + KeyButMask::from(0u16) + }; - for ch in text.chars() { - // Resolve the keycode and whether Shift must be held — without it, - // uppercase and shifted symbols would otherwise type their unshifted - // form (e.g. "A" arriving as "a"). - let Some((keycode, needs_shift)) = char_to_keycode_shift(&mapping, ch as u32) else { - continue; - }; - let state = if needs_shift { - KeyButMask::SHIFT - } else { - KeyButMask::from(0u16) - }; + let press = KeyPressEvent { + response_type: KEY_PRESS_EVENT, + detail: keycode, + sequence: 0, + time: x11rb::CURRENT_TIME, + root, + event: window, + child: x11rb::NONE, + root_x: 0, + root_y: 0, + event_x: 0, + event_y: 0, + state, + same_screen: true, + }; + let release = KeyReleaseEvent { + response_type: KEY_RELEASE_EVENT, + detail: keycode, + sequence: 0, + time: x11rb::CURRENT_TIME, + root, + event: window, + child: x11rb::NONE, + root_x: 0, + root_y: 0, + event_x: 0, + event_y: 0, + state, + same_screen: true, + }; - let press = KeyPressEvent { - response_type: KEY_PRESS_EVENT, - detail: keycode, - sequence: 0, - time: x11rb::CURRENT_TIME, - root, - event: window, - child: x11rb::NONE, - root_x: 0, - root_y: 0, - event_x: 0, - event_y: 0, - state, - same_screen: true, - }; - let release = KeyReleaseEvent { - response_type: KEY_RELEASE_EVENT, - detail: keycode, - sequence: 0, - time: x11rb::CURRENT_TIME, - root, - event: window, - child: x11rb::NONE, - root_x: 0, - root_y: 0, - event_x: 0, - event_y: 0, - state, - same_screen: true, - }; - - conn.send_event(false, window, EventMask::KEY_PRESS, &press)?; - sleep(Duration::from_millis(KEY_DELAY_MS)); - conn.send_event(false, window, EventMask::KEY_RELEASE, &release)?; - conn.flush()?; - if inter_char_ms > 0 { - sleep(Duration::from_millis(inter_char_ms)); - } - } + conn.send_event(false, window, EventMask::KEY_PRESS, press)?; + // Start the hold interval after sending the press, not while it is buffered. + conn.flush()?; + sleep(Duration::from_millis(KEY_DELAY_MS)); + conn.send_event(false, window, EventMask::KEY_RELEASE, release)?; + conn.flush()?; + if inter_char_ms > 0 { + sleep(Duration::from_millis(inter_char_ms)); + } + Ok(()) + }) + }, + || { + conn.get_input_focus()?.reply()?; + if let Some(barrier) = &barrier { + barrier.wait()?; + } + Ok(()) + }, + )?; Ok(()) } @@ -2176,9 +3568,13 @@ pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Re /// argument because XTest always delivers to the focused window; the caller /// focuses the target by clicking it first. `\n`/`\t` map to Return/Tab. pub fn send_type_text_xtest(text: &str) -> Result<()> { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); use x11rb::protocol::xtest::ConnectionExt as _; let (conn, _) = connect_x11_for_input()?; let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + let barrier = KeymapBarrier::prepare(&conn, None, &mapping, text.chars().map(keysym_for_char))?; // Shift keycode (modifier index 0) for shifted characters. let modmap = conn.get_modifier_mapping()?.reply()?; let kpm = modmap.keycodes_per_modifier() as usize; @@ -2187,30 +3583,37 @@ pub fn send_type_text_xtest(text: &str) -> Result<()> { .get(..kpm) .and_then(|s| s.iter().copied().find(|&k| k != 0)) .unwrap_or(50); - for ch in text.chars() { - let cp = match ch { - '\n' => 0xff0d, // XK_Return - '\t' => 0xff09, // XK_Tab - c => c as u32, - }; - let Some((keycode, needs_shift)) = char_to_keycode_shift(&mapping, cp) else { - continue; - }; - if needs_shift { - conn.xtest_fake_input(KEY_PRESS_EVENT, shift_kc, 0, x11rb::NONE, 0, 0, 0)?; - } - conn.xtest_fake_input(KEY_PRESS_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; - conn.xtest_fake_input(KEY_RELEASE_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; - if needs_shift { - conn.xtest_fake_input(KEY_RELEASE_EVENT, shift_kc, 0, x11rb::NONE, 0, 0, 0)?; - } - conn.flush()?; - sleep(Duration::from_millis(KEY_DELAY_MS)); - } - // Round-trip so the server delivers the final character's key events before - // this short-lived connection drops (see send_key_xtest — keyboard XTEST - // events queued on a connection that closes immediately can be lost). - let _ = conn.get_input_focus()?.reply(); + let _guards = type_text_with_preflight( + text, + &mapping, + |ch, mapping| keycode_for_keysym(&conn, mapping, keysym_for_char(ch), &ch.to_string()), + |keycode, needs_shift| { + user_input::checkpoint()?; + if let Some(barrier) = &barrier { + barrier.check_focus()?; + } + keymap_barrier::dispatch(|| { + if needs_shift { + conn.xtest_fake_input(KEY_PRESS_EVENT, shift_kc, 0, x11rb::NONE, 0, 0, 0)?; + } + conn.xtest_fake_input(KEY_PRESS_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; + conn.xtest_fake_input(KEY_RELEASE_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; + if needs_shift { + conn.xtest_fake_input(KEY_RELEASE_EVENT, shift_kc, 0, x11rb::NONE, 0, 0, 0)?; + } + conn.flush()?; + sleep(Duration::from_millis(KEY_DELAY_MS)); + Ok(()) + }) + }, + || { + conn.get_input_focus()?.reply()?; + if let Some(barrier) = &barrier { + barrier.wait()?; + } + Ok(()) + }, + )?; Ok(()) } @@ -2229,9 +3632,19 @@ pub fn send_type_text_xtest(text: &str) -> Result<()> { /// that lack a keysym borrow a spare keycode (xdotool-style) via /// [`keycode_for_keysym`]; the returned guards restore the map on drop. pub fn send_key_xtest(key: &str, modifiers: &[&str]) -> Result<()> { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + user_input::checkpoint()?; use x11rb::protocol::xtest::ConnectionExt as _; let (conn, _) = connect_x11_for_input()?; - let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + let mut mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + + let requested = std::iter::once(key) + .chain(modifiers.iter().copied()) + .map(key_name_to_keysym) + .collect::>>()?; + let barrier = KeymapBarrier::prepare(&conn, None, &mapping, requested)?; // Resolve modifier keycodes first. Keep any spare-keycode remap guards alive // until after the events are delivered (drop at end of fn). @@ -2239,7 +3652,7 @@ pub fn send_key_xtest(key: &str, modifiers: &[&str]) -> Result<()> { let mut mod_keycodes = Vec::new(); for m in modifiers { let ks = key_name_to_keysym(m)?; - let (kc, guard) = keycode_for_keysym(&conn, &mapping, ks, m)?; + let (kc, guard) = keycode_for_keysym(&conn, &mut mapping, ks, m)?; if let Some(g) = guard { guards.push(g); } @@ -2257,7 +3670,7 @@ pub fn send_key_xtest(key: &str, modifiers: &[&str]) -> Result<()> { let (keycode, needs_shift) = match char_to_keycode_shift(&mapping, keysym) { Some(found) => found, None => { - let (kc, guard) = keycode_for_keysym(&conn, &mapping, keysym, key)?; + let (kc, guard) = keycode_for_keysym(&conn, &mut mapping, keysym, key)?; if let Some(g) = guard { guards.push(g); } @@ -2281,38 +3694,44 @@ pub fn send_key_xtest(key: &str, modifiers: &[&str]) -> Result<()> { }; // Press modifiers (+ auto-Shift), tap the key, release in reverse order. - for &kc in &mod_keycodes { - conn.xtest_fake_input(KEY_PRESS_EVENT, kc, 0, x11rb::NONE, 0, 0, 0)?; + if let Some(barrier) = &barrier { + barrier.check_focus()?; } - if let Some(sk) = auto_shift_kc { - conn.xtest_fake_input(KEY_PRESS_EVENT, sk, 0, x11rb::NONE, 0, 0, 0)?; - } - conn.xtest_fake_input(KEY_PRESS_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; - sleep(Duration::from_millis(KEY_DELAY_MS)); - conn.xtest_fake_input(KEY_RELEASE_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; - if let Some(sk) = auto_shift_kc { - conn.xtest_fake_input(KEY_RELEASE_EVENT, sk, 0, x11rb::NONE, 0, 0, 0)?; - } - for &kc in mod_keycodes.iter().rev() { - conn.xtest_fake_input(KEY_RELEASE_EVENT, kc, 0, x11rb::NONE, 0, 0, 0)?; - } - conn.flush()?; - - // Round-trip so the server actually PROCESSES (delivers) the injected XTEST - // key events before this function returns and drops its short-lived - // connection. `flush()` only writes the requests to the socket; without a - // following reply-bearing request the connection can close before the server - // routes the KeyPress/KeyRelease to the focused window, and the events are - // dropped. Observed under Xtigervnc: identical raw `xtest_fake_input` calls - // deliver when the connection stays alive but vanish from this short-lived - // one — pointer events (send_click_xtest_desktop) survive, keyboard events do - // not. Unconditional (previously only ran for spare-keycode remaps). - let _ = conn.get_input_focus()?.reply(); - if !guards.is_empty() { - // Spare-keycode remap: give the target a beat to translate the events - // under the temporary mapping before the guards restore it on drop. + keymap_barrier::dispatch(|| { + for &kc in &mod_keycodes { + conn.xtest_fake_input(KEY_PRESS_EVENT, kc, 0, x11rb::NONE, 0, 0, 0)?; + } + if let Some(sk) = auto_shift_kc { + conn.xtest_fake_input(KEY_PRESS_EVENT, sk, 0, x11rb::NONE, 0, 0, 0)?; + } + conn.xtest_fake_input(KEY_PRESS_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)? + .check()?; + conn.flush()?; sleep(Duration::from_millis(KEY_DELAY_MS)); - } + conn.xtest_fake_input(KEY_RELEASE_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; + if let Some(sk) = auto_shift_kc { + conn.xtest_fake_input(KEY_RELEASE_EVENT, sk, 0, x11rb::NONE, 0, 0, 0)?; + } + for &kc in mod_keycodes.iter().rev() { + conn.xtest_fake_input(KEY_RELEASE_EVENT, kc, 0, x11rb::NONE, 0, 0, 0)?; + } + conn.flush()?; + + // Round-trip so the server actually PROCESSES (delivers) the injected XTEST + // key events before this function returns and drops its short-lived + // connection. `flush()` only writes the requests to the socket; without a + // following reply-bearing request the connection can close before the server + // routes the KeyPress/KeyRelease to the focused window, and the events are + // dropped. Observed under Xtigervnc: identical raw `xtest_fake_input` calls + // deliver when the connection stays alive but vanish from this short-lived + // one — pointer events (send_click_xtest_desktop) survive, keyboard events do + // not. Unconditional (previously only ran for spare-keycode remaps). + conn.get_input_focus()?.reply()?; + if let Some(barrier) = &barrier { + barrier.wait()?; + } + Ok(()) + })?; drop(guards); Ok(()) } @@ -2341,21 +3760,29 @@ pub fn send_click_xtest_desktop_with_modifiers( count: usize, modifiers: &[&str], ) -> Result<()> { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + user_input::checkpoint()?; use x11rb::protocol::xtest::ConnectionExt as _; let (conn, screen_num) = connect_x11_for_input()?; let root = conn.setup().roots[screen_num].root; - let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + let mut mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; let mut guards = Vec::new(); let mut modifier_keycodes = Vec::new(); for modifier in modifiers { let keysym = key_name_to_keysym(modifier)?; - let (keycode, guard) = keycode_for_keysym(&conn, &mapping, keysym, modifier)?; + if !mapping.keysyms.contains(&keysym) { + bail!("not_settable: pointer modifiers must exist in the X11 keyboard map; no input was sent"); + } + let (keycode, guard) = keycode_for_keysym(&conn, &mut mapping, keysym, modifier)?; if let Some(guard) = guard { guards.push(guard); } modifier_keycodes.push(keycode); } let mut pressed = Vec::new(); + let mut button_pressed = false; let gesture_result = (|| -> Result<()> { for &keycode in &modifier_keycodes { conn.xtest_fake_input(KEY_PRESS_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?; @@ -2366,8 +3793,15 @@ pub fn send_click_xtest_desktop_with_modifiers( conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?; let count = count.max(1); for click_index in 0..count { - conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?; - conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?; + user_input::checkpoint()?; + conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)? + .check()?; + button_pressed = true; + conn.flush()?; + sleep(Duration::from_millis(CLICK_DELAY_MS)); + conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)? + .check()?; + button_pressed = false; if click_index + 1 < count { // Chromium needs the first pair to reach the server before the // second pair. A zero-gap batch produces two click events but @@ -2383,6 +3817,9 @@ pub fn send_click_xtest_desktop_with_modifiers( // including when a later pointer request fails. A failed gesture must not // leave the desktop with a logically stuck Ctrl/Shift/Alt/Super key. let mut release_result: Result<()> = Ok(()); + if button_pressed { + let _ = conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, 0, 0, 0); + } for &keycode in pressed.iter().rev() { if let Err(error) = conn.xtest_fake_input(KEY_RELEASE_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0) @@ -2406,6 +3843,7 @@ pub fn send_click_xtest_desktop_with_modifiers( /// Move the real X11 pointer to a screen-absolute desktop coordinate. pub fn send_move_xtest_desktop(x: i32, y: i32) -> Result<()> { + user_input::checkpoint()?; use x11rb::protocol::xtest::ConnectionExt as _; let (conn, screen_num) = connect_x11_for_input()?; let root = conn.setup().roots[screen_num].root; @@ -2418,6 +3856,7 @@ pub fn send_move_xtest_desktop(x: i32, y: i32) -> Result<()> { /// Scroll the window under a screen-absolute point via real XTest wheel-button /// events. X11 buttons 4/5 are vertical up/down and 6/7 horizontal left/right. pub fn send_scroll_xtest_desktop(x: i32, y: i32, direction: &str, amount: usize) -> Result<()> { + user_input::checkpoint()?; use x11rb::protocol::xtest::ConnectionExt as _; let button = match direction { "up" => 4, @@ -2430,6 +3869,7 @@ pub fn send_scroll_xtest_desktop(x: i32, y: i32, direction: &str, amount: usize) let root = conn.setup().roots[screen_num].root; conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?; for _ in 0..amount.max(1) { + user_input::checkpoint()?; conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?; conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?; } @@ -2450,6 +3890,7 @@ pub fn send_drag_xtest_desktop( duration_ms: u64, steps: usize, ) -> Result<()> { + user_input::checkpoint()?; use x11rb::protocol::xtest::ConnectionExt as _; let (conn, screen_num) = connect_x11_for_input()?; let root = conn.setup().roots[screen_num].root; @@ -2475,7 +3916,12 @@ pub fn send_drag_xtest_desktop( 0, )?; conn.flush()?; + let mut interrupted = None; for step in 1..=steps { + if let Err(error) = user_input::checkpoint() { + interrupted = Some(error); + break; + } let t = step as f64 / steps as f64; let x = from_x as f64 + (to_x - from_x) as f64 * t; let y = from_y as f64 + (to_y - from_y) as f64 * t; @@ -2493,19 +3939,58 @@ pub fn send_drag_xtest_desktop( sleep(Duration::from_millis(delay)); } } - conn.xtest_fake_input( - BUTTON_RELEASE_EVENT, - button, - 0, - root, - to_x as i16, - to_y as i16, - 0, - )?; + conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, 0, 0, 0)?; conn.flush()?; let _ = conn.get_input_focus()?.reply(); + if let Some(error) = interrupted { + return Err(error); + } Ok(()) } +pub fn send_drag_xtest_desktop_with_modifiers( + from_x: i32, + from_y: i32, + to_x: i32, + to_y: i32, + button: u8, + duration_ms: u64, + steps: usize, + modifiers: &[&str], +) -> Result<()> { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + use x11rb::protocol::xtest::ConnectionExt as _; + let (conn, _) = connect_x11_for_input()?; + let mut mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + let mut guards = Vec::new(); + let mut codes = Vec::new(); + for modifier in modifiers { + let keysym = key_name_to_keysym(modifier)?; + if !mapping.keysyms.contains(&keysym) { + bail!("not_settable: pointer modifiers must exist in the X11 keyboard map; no input was sent"); + } + let (code, guard) = keycode_for_keysym(&conn, &mut mapping, keysym, modifier)?; + guards.extend(guard); + codes.push(code); + } + let mut pressed = Vec::new(); + let result = (|| { + user_input::checkpoint()?; + for &code in &codes { + conn.xtest_fake_input(KEY_PRESS_EVENT, code, 0, x11rb::NONE, 0, 0, 0)?; + pressed.push(code); + } + conn.flush()?; + send_drag_xtest_desktop(from_x, from_y, to_x, to_y, button, duration_ms, steps) + })(); + for code in pressed.into_iter().rev() { + let _ = conn.xtest_fake_input(KEY_RELEASE_EVENT, code, 0, x11rb::NONE, 0, 0, 0); + } + let _ = conn.flush(); + let _ = conn.get_input_focus()?.reply(); + result +} /// Send a named key press to a window. pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> { @@ -2526,6 +4011,9 @@ fn send_key_to_target( key: &str, modifiers: &[&str], ) -> Result<()> { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let (conn, _) = connect_x11_for_input()?; let target = point .map(|(x, y)| resolve_event_target(&conn, xid, x, y)) @@ -2539,8 +4027,13 @@ fn send_key_to_target( // `keycode_for_keysym` instead borrows a spare keycode and hands back a guard // that restores the original mapping once the event has been delivered. let keysym = key_name_to_keysym(key)?; - let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; - let (keycode, remap_guard) = keycode_for_keysym(&conn, &mapping, keysym, key)?; + let mut mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + let requested = std::iter::once(key) + .chain(modifiers.iter().copied()) + .map(key_name_to_keysym) + .collect::>>()?; + let barrier = KeymapBarrier::prepare(&conn, Some(window), &mapping, requested)?; + let (keycode, remap_guard) = keycode_for_keysym(&conn, &mut mapping, keysym, key)?; // XSendEvent's state mask describes modifiers for one key event, but does // not update Chromium's internal modifier state by itself. Emit the @@ -2552,7 +4045,7 @@ fn send_key_to_target( for modifier in modifiers { let modifier_keysym = key_name_to_keysym(modifier)?; let (modifier_keycode, guard) = - keycode_for_keysym(&conn, &mapping, modifier_keysym, modifier)?; + keycode_for_keysym(&conn, &mut mapping, modifier_keysym, modifier)?; if let Some(guard) = guard { remap_guards.push(guard); } @@ -2578,41 +4071,39 @@ fn send_key_to_target( conn.send_event(false, window, event_mask, &event) }; - let mut state_bits = 0u16; - for &(modifier_keycode, modifier_mask) in &modifier_keycodes { - send_key_event( - KEY_PRESS_EVENT, - modifier_keycode, - KeyButMask::from(state_bits), - EventMask::KEY_PRESS, - )?; - state_bits |= u16::from(modifier_mask); - } - let state = KeyButMask::from(state_bits); - send_key_event(KEY_PRESS_EVENT, keycode, state, EventMask::KEY_PRESS)?; - sleep(Duration::from_millis(KEY_DELAY_MS)); - send_key_event(KEY_RELEASE_EVENT, keycode, state, EventMask::KEY_RELEASE)?; - for &(modifier_keycode, modifier_mask) in modifier_keycodes.iter().rev() { - send_key_event( - KEY_RELEASE_EVENT, - modifier_keycode, - KeyButMask::from(state_bits), - EventMask::KEY_RELEASE, - )?; - state_bits &= !u16::from(modifier_mask); - } - conn.flush()?; - - // If we borrowed a spare keycode for this keysym, give the target client a - // moment to translate the synthetic event under the temporary mapping before - // we restore it. The keycode->keysym lookup is client-side, so restoring too - // eagerly would race delivery. A server round-trip (which only returns once - // our queued requests have been processed) plus a short settle keeps that - // race closed; the guard then reinstates the original keysyms on drop. - if remap_guard.is_some() || !remap_guards.is_empty() { - let _ = conn.get_input_focus()?.reply(); + keymap_barrier::dispatch(|| { + let mut state_bits = 0u16; + for &(modifier_keycode, modifier_mask) in &modifier_keycodes { + send_key_event( + KEY_PRESS_EVENT, + modifier_keycode, + KeyButMask::from(state_bits), + EventMask::KEY_PRESS, + )?; + state_bits |= u16::from(modifier_mask); + } + let state = KeyButMask::from(state_bits); + send_key_event(KEY_PRESS_EVENT, keycode, state, EventMask::KEY_PRESS)?.check()?; + conn.flush()?; sleep(Duration::from_millis(KEY_DELAY_MS)); - } + send_key_event(KEY_RELEASE_EVENT, keycode, state, EventMask::KEY_RELEASE)?; + for &(modifier_keycode, modifier_mask) in modifier_keycodes.iter().rev() { + send_key_event( + KEY_RELEASE_EVENT, + modifier_keycode, + KeyButMask::from(state_bits), + EventMask::KEY_RELEASE, + )?; + state_bits &= !u16::from(modifier_mask); + } + conn.flush()?; + + conn.get_input_focus()?.reply()?; + if let Some(barrier) = &barrier { + barrier.wait()?; + } + Ok(()) + })?; drop(remap_guard); Ok(()) } @@ -2640,7 +4131,11 @@ fn char_to_keycode_shift(mapping: &GetKeyboardMappingReply, keysym: u32) -> Opti /// Map a human key name (e.g. "Return", "F5", "a") to its X11 keysym. Pure name /// resolution — no server interaction — split out from keycode lookup so the /// keysym can be remapped onto a spare keycode when the keymap lacks it. -fn key_name_to_keysym(key: &str) -> Result { +/// Resolve a key name in the shared X keysym vocabulary, or error on an +/// unknown name. Exposed crate-wide so input-adjacent paths (e.g. the +/// press_key terminal short-circuit gate) can test key identity against +/// this canonical mapping instead of a local string comparison. +pub(crate) fn key_name_to_keysym(key: &str) -> Result { // Common X11 keysym names. let keysym: u32 = match key.to_lowercase().as_str() { "return" | "enter" => 0xFF0D, @@ -2652,8 +4147,8 @@ fn key_name_to_keysym(key: &str) -> Result { "insert" | "ins" => 0xFF63, "home" => 0xFF50, "end" => 0xFF57, - "pageup" | "pgup" => 0xFF55, - "pagedown" | "pgdn" => 0xFF56, + "pageup" | "pgup" | "page_up" => 0xFF55, + "pagedown" | "pgdn" | "page_down" => 0xFF56, "up" => 0xFF52, "down" => 0xFF54, "left" => 0xFF51, @@ -2673,7 +4168,7 @@ fn key_name_to_keysym(key: &str) -> Result { "shift" => 0xFFE1, "ctrl" | "control" => 0xFFE3, "alt" => 0xFFE9, - "super" | "meta" | "win" => 0xFFEB, + "super" | "meta" | "win" | "cmd" => 0xFFEB, "capslock" => 0xFFE5, "numlock" => 0xFF7F, // Common X keysym names for punctuation. The single-char branch below @@ -2699,12 +4194,305 @@ fn key_name_to_keysym(key: &str) -> Result { Ok(keysym) } +#[cfg(test)] +mod key_name_alias_tests { + use super::{ + char_to_keycode_shift, key_name_to_keysym, reserve_spare_keycode, type_text_with_preflight, + GetKeyboardMappingReply, + }; + + #[test] + fn text_preflight_failure_leaves_target_and_events_unchanged() { + let mapping = GetKeyboardMappingReply { + keysyms_per_keycode: 2, + sequence: 0, + keysyms: vec!['a' as u32, 'A' as u32], + }; + for text in ["約ab", "aa約", "a約a"] { + let mut target = String::from("existing"); + let mut events = Vec::new(); + let result = type_text_with_preflight::<()>( + text, + &mapping, + |_, _| anyhow::bail!("no spare keycode"), + |keycode, shift| { + events.extend([keycode, keycode]); + target.push(if shift { 'A' } else { 'a' }); + Ok(()) + }, + || Ok(()), + ); + assert!(result + .unwrap_err() + .to_string() + .starts_with("typed 0 of 3 characters")); + assert!(events.is_empty()); + assert_eq!(target, "existing"); + } + } + + #[test] + fn text_preflight_requires_a_spare_for_each_distinct_unmapped_character() { + let mapping = GetKeyboardMappingReply { + keysyms_per_keycode: 2, + sequence: 0, + keysyms: vec!['a' as u32, 'A' as u32, 0, 0], + }; + let mut target = String::from("existing"); + let mut events = Vec::new(); + let error = type_text_with_preflight::<()>( + "a你好", + &mapping, + |ch, mapping| { + reserve_spare_keycode( + mapping, + super::mpx_keyboard::keysym_for_char(ch), + |_, _, _, _| Ok(()), + ) + .map(|(keycode, guard)| (keycode, Some(guard))) + }, + |keycode, _| { + events.push(keycode); + target.push('a'); + Ok(()) + }, + || Ok(()), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "typed 0 of 3 characters: the text needs 2 spare keycodes: no spare keycode available to remap keysym 0x{:X}", + super::mpx_keyboard::keysym_for_char('好') + ) + ); + assert!(events.is_empty()); + assert_eq!(target, "existing"); + } + + struct Restore<'a>(&'a std::cell::Cell); + + #[test] + fn text_preflight_marks_errors_after_a_sent_prefix_as_unconfirmed() { + let mapping = GetKeyboardMappingReply { + keysyms_per_keycode: 2, + sequence: 0, + keysyms: vec!['a' as u32, 'A' as u32], + }; + for (fail_at, finish_fails, unconfirmed) in [ + (Some(0), false, false), + (Some(0), true, false), + (Some(1), false, true), + (Some(1), true, true), + (None, true, true), + ] { + let mut sent = 0; + let mut finished = false; + let error = type_text_with_preflight::<()>( + "aa", + &mapping, + |_, _| unreachable!(), + |_, _| { + if fail_at == Some(sent) { + anyhow::bail!("focus changed before next character"); + } + sent += 1; + Ok(()) + }, + || { + finished = true; + if finish_fails { + anyhow::bail!("client queue read failed"); + } + Ok(()) + }, + ) + .unwrap_err(); + assert!(finished, "even partial input must finish its queue barrier"); + assert_eq!( + super::delivery_is_unconfirmed(&error), + unconfirmed, + "sent={sent}, finish_fails={finish_fails}: {error:#}" + ); + } + } + + impl Drop for Restore<'_> { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + #[test] + fn text_preflight_restores_every_remap_after_success_and_failure() { + let mapping = GetKeyboardMappingReply { + keysyms_per_keycode: 2, + sequence: 0, + keysyms: vec!['a' as u32, 'A' as u32, 0, 0, 0, 0], + }; + for (text, fail_send, ok, sent) in [ + ("你好你a", false, true, 4), + ("你好约", false, false, 0), + ("你好", true, false, 1), + ] { + let restored = std::cell::Cell::new(0); + let mut events = 0; + let result = type_text_with_preflight( + text, + &mapping, + |ch, mapping| { + reserve_spare_keycode( + mapping, + super::mpx_keyboard::keysym_for_char(ch), + |_, _, _, _| Ok(Restore(&restored)), + ) + .map(|(keycode, guard)| (keycode, Some(guard))) + }, + |_, _| { + events += 1; + if fail_send { + anyhow::bail!("send failed"); + } + Ok(()) + }, + || { + assert_eq!( + restored.get(), + 0, + "restore must follow client queue completion" + ); + Ok(()) + }, + ); + assert_eq!(result.is_ok(), ok, "{text}"); + drop(result); + assert_eq!(restored.get(), 2, "{text}"); + assert_eq!(events, sent, "{text}"); + } + } + + #[test] + fn text_preflight_preserves_shift_order_and_reuses_remaps() { + let mapping = GetKeyboardMappingReply { + keysyms_per_keycode: 2, + sequence: 0, + keysyms: vec!['a' as u32, 'A' as u32, 0, 0, 0, 0], + }; + let mut remaps = 0; + let mut events = Vec::new(); + type_text_with_preflight::<()>( + "aA你好你", + &mapping, + |ch, mapping| { + remaps += 1; + reserve_spare_keycode( + mapping, + super::mpx_keyboard::keysym_for_char(ch), + |_, _, _, _| Ok(()), + ) + .map(|(keycode, guard)| (keycode, Some(guard))) + }, + |keycode, shift| { + events.push((keycode, shift)); + Ok(()) + }, + || Ok(()), + ) + .unwrap(); + assert_eq!(remaps, 2); + assert_eq!( + events, + [(8, false), (8, true), (10, false), (9, false), (10, false)] + ); + } + + #[test] + fn text_preflight_preserves_keys_with_symbols_in_later_columns() { + let mapping = GetKeyboardMappingReply { + keysyms_per_keycode: 4, + sequence: 0, + keysyms: vec!['a' as u32, 'A' as u32, 'b' as u32, 0], + }; + let mut events = Vec::new(); + type_text_with_preflight::<()>( + "bAa", + &mapping, + |ch, mapping| { + assert_eq!(ch, 'b'); + assert!(mapping.keysyms.contains(&(ch as u32))); + Ok((8, None)) + }, + |keycode, shift| { + events.push((keycode, shift)); + Ok(()) + }, + || Ok(()), + ) + .unwrap(); + assert_eq!(events, [(8, false), (8, true), (8, false)]); + } + + #[test] + fn shifted_column_glyphs_need_shift_and_base_glyphs_do_not() { + let per = 2usize; + let mut keysyms = vec![0u32; (248 - 8 + 1) * per]; + let mut set = |keycode: usize, syms: [u32; 2]| { + let idx = (keycode - 8) * per; + keysyms[idx..idx + per].copy_from_slice(&syms); + }; + set(43, [0x68, 0x48]); + set(11, [0x32, 0x40]); + let mapping = GetKeyboardMappingReply { + keysyms_per_keycode: per as u8, + sequence: 0, + keysyms, + }; + for (ch, keycode, needs_shift) in [ + ('H', 43, true), + ('@', 11, true), + ('h', 43, false), + ('2', 11, false), + ] { + assert_eq!( + char_to_keycode_shift(&mapping, ch as u32), + Some((keycode, needs_shift)), + "{ch:?}" + ); + } + } + + #[test] + fn common_key_name_aliases_resolve_to_the_expected_keysym() { + // Page_Up / Page_Down: the underscore form is the literal X11 keysym + // name and a common cross-platform prompt spelling; only the + // no-underscore "pageup"/"pgup" aliases existed before. + assert_eq!(key_name_to_keysym("Page_Up").unwrap(), 0xFF55); + assert_eq!(key_name_to_keysym("page_up").unwrap(), 0xFF55); + assert_eq!( + key_name_to_keysym("Page_Up").unwrap(), + key_name_to_keysym("pageup").unwrap() + ); + assert_eq!(key_name_to_keysym("Page_Down").unwrap(), 0xFF56); + assert_eq!( + key_name_to_keysym("Page_Down").unwrap(), + key_name_to_keysym("pgdn").unwrap() + ); + // "cmd": common cross-platform prompt name for the Super/Meta/Windows + // key; must alias to the same keysym as "super"/"meta"/"win". + assert_eq!(key_name_to_keysym("cmd").unwrap(), 0xFFEB); + assert_eq!( + key_name_to_keysym("cmd").unwrap(), + key_name_to_keysym("super").unwrap() + ); + } +} + /// A keycode we have *temporarily* rebound to host a keysym that is absent from /// the current X keyboard map (sparse/headless keymaps such as a minimal /// Xwayland). On drop it reinstates the keycode's original keysyms so the -/// server's mapping is left exactly as we found it. Modelled on xdotool's -/// remap-a-spare-keycode trick (`_xdo_charcodemap` / `XChangeKeyboardMapping`). -struct RemappedKeycode<'a> { +/// server's mapping is left exactly as we found it. The remap goes through +/// `XChangeKeyboardMapping` on a spare keycode. +pub(crate) struct RemappedKeycode<'a> { conn: &'a RustConnection, keycode: u8, keysyms_per_keycode: u8, @@ -2713,29 +4501,23 @@ struct RemappedKeycode<'a> { impl Drop for RemappedKeycode<'_> { fn drop(&mut self) { - // Best-effort restore: re-install the original keysyms for this keycode - // and flush. Errors are swallowed deliberately — Drop must not panic in - // the daemon, and the worst case of a failed restore is a single spare - // keycode left mapped (it was unused to begin with), never a crash. let _ = self.conn.change_keyboard_mapping( 1, self.keycode, self.keysyms_per_keycode, &self.original_keysyms, ); - let _ = self.conn.flush(); + let _ = self.conn.get_input_focus().map(|cookie| cookie.reply()); } } -/// Temporarily bind `keysym` onto a spare (fully unused) keycode so it can be -/// injected even when no existing keycode emits it. Returns a guard that -/// restores the original mapping on drop. Errors only if the keymap has no free -/// keycode left to borrow. -fn remap_spare_keycode<'a>( - conn: &'a RustConnection, - mapping: &GetKeyboardMappingReply, +/// Reserve a fully unused keycode for `keysym`, calling `bind` before updating +/// the local mapping. Returns the keycode and the value produced by `bind`. +fn reserve_spare_keycode( + mapping: &mut GetKeyboardMappingReply, keysym: u32, -) -> Result> { + mut bind: impl FnMut(u8, u8, &[u32], &[u32]) -> Result, +) -> Result<(u8, G)> { let per = mapping.keysyms_per_keycode as usize; if per == 0 { bail!("empty keyboard mapping; cannot remap keysym 0x{keysym:X}"); @@ -2759,17 +4541,9 @@ fn remap_spare_keycode<'a>( let idx = (spare as usize - 8) * per; let original_keysyms = mapping.keysyms[idx..idx + per].to_vec(); let new_keysyms = vec![keysym; per]; - conn.change_keyboard_mapping(1, spare, per as u8, &new_keysyms)?; - // Round-trip so the server has installed the new mapping before we emit the - // key event against it. - let _ = conn.get_input_focus()?.reply(); - - Ok(RemappedKeycode { - conn, - keycode: spare, - keysyms_per_keycode: per as u8, - original_keysyms, - }) + let guard = bind(spare, per as u8, &original_keysyms, &new_keysyms)?; + mapping.keysyms[idx..idx + per].copy_from_slice(&new_keysyms); + Ok((spare, guard)) } /// Resolve `keysym` to a keycode usable in a synthetic key event. First scans @@ -2779,24 +4553,34 @@ fn remap_spare_keycode<'a>( /// The guard is `None` when the keysym was already present (no cleanup needed). fn keycode_for_keysym<'a>( conn: &'a RustConnection, - mapping: &GetKeyboardMappingReply, + mapping: &mut GetKeyboardMappingReply, keysym: u32, key: &str, ) -> Result<(u8, Option>)> { let per = mapping.keysyms_per_keycode as usize; if per > 0 { for (i, syms) in mapping.keysyms.chunks(per).enumerate() { - if syms.iter().any(|&s| s == keysym) { + if syms.contains(&keysym) { return Ok(((8 + i) as u8, None)); } } } - // Not in the map — fall back to remapping a spare keycode (xdotool-style). - let guard = remap_spare_keycode(conn, mapping, keysym).with_context(|| { + // Not in the map — fall back to remapping a spare keycode. + let (keycode, guard) = reserve_spare_keycode(mapping, keysym, |spare, per, original, new| { + conn.change_keyboard_mapping(1, spare, per, new)?; + let guard = RemappedKeycode { + conn, + keycode: spare, + keysyms_per_keycode: per, + original_keysyms: original.to_vec(), + }; + let _ = conn.get_input_focus()?.reply(); + Ok(guard) + }) + .with_context(|| { format!("Keysym 0x{keysym:X} not in keyboard map for key '{key}' and no spare keycode could be remapped") })?; - let keycode = guard.keycode; Ok((keycode, Some(guard))) } @@ -2916,12 +4700,93 @@ exit 0"#, #[cfg(test)] mod path_tests { use super::{ - create_uinput_pointer, guarded_uinput_creation, is_uinput_unavailable, master_pointer_name, - modifiers_to_state, normalize_uinput_device_name, path_cumulative, point_on_path, - real_pointer_capabilities_available, sample_function, slave_pointer_name, - EVDEV_UINPUT_NAME_MAX_BYTES, UINPUT_POINTER_SUFFIX, + click_event_target, create_uinput_pointer, ensure_master_pointer_for_session, + guarded_uinput_creation, is_uinput_unavailable, kde_x11_uinput_hotplug_is_unsafe, + master_pointer_name, modifiers_to_state, mpx_op_guard, normalize_uinput_device_name, + path_cumulative, point_on_path, real_pointer_capabilities_available, sample_function, + slave_pointer_name, EVDEV_UINPUT_NAME_MAX_BYTES, UINPUT_POINTER_SUFFIX, }; - use x11rb::protocol::xproto::KeyButMask; + use x11rb::protocol::xproto::{EventMask, KeyButMask}; + + /// Regression for the `XSetErrorHandler` swap race: every caller that + /// could swap the process-global X11 error handler while an MPX + /// operation is mid-swap (`send_virtual_pointer_*`, `deliver`, and now + /// `x11_activate_window_persistent`) must take `MPX_OP_LOCK` first. This + /// doesn't touch X11 (no display in CI), but proves the lock itself is + /// exclusive and not reentrant-by-accident: a second `mpx_op_guard` call + /// blocks until the first is dropped, exactly the property the handler + /// swap needs to be race-free. + #[test] + fn mpx_op_guard_is_mutually_exclusive() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + let entered_second = Arc::new(AtomicBool::new(false)); + let first = mpx_op_guard("test-a"); + let entered_second_clone = entered_second.clone(); + let handle = std::thread::spawn(move || { + let _second = mpx_op_guard("test-b"); + entered_second_clone.store(true, Ordering::SeqCst); + }); + // The second guard cannot have been acquired yet — it is blocked on + // the lock we are still holding. + std::thread::sleep(std::time::Duration::from_millis(50)); + assert!( + !entered_second.load(Ordering::SeqCst), + "second guard acquired while the first was still held" + ); + drop(first); + handle.join().unwrap(); + assert!(entered_second.load(Ordering::SeqCst)); + } + + #[test] + fn synthetic_click_keeps_release_on_press_recipient() { + let path = [ + ( + 10, + 394, + 220, + EventMask::BUTTON_PRESS | EventMask::BUTTON_RELEASE, + ), + (11, 24, 30, EventMask::BUTTON_PRESS), + (12, 6, 8, EventMask::BUTTON_RELEASE), + ]; + + assert_eq!(click_event_target(&path), (11, 24, 30)); + } + + #[test] + fn synthetic_click_uses_deepest_common_target_for_both_events() { + let path = [ + ( + 10, + 394, + 220, + EventMask::BUTTON_PRESS | EventMask::BUTTON_RELEASE, + ), + ( + 11, + 24, + 30, + EventMask::BUTTON_PRESS | EventMask::BUTTON_RELEASE, + ), + (12, 6, 8, EventMask::NO_EVENT), + ]; + + assert_eq!(click_event_target(&path), (11, 24, 30)); + } + + #[test] + fn synthetic_click_preserves_recipient_local_coordinates_and_event_fallback() { + let path = [ + (10, 394, 220, EventMask::BUTTON_PRESS), + (11, 24, 30, EventMask::NO_EVENT), + (12, 6, 8, EventMask::NO_EVENT), + ]; + + assert_eq!(click_event_target(&path), (10, 394, 220)); + } #[test] fn click_modifier_state_combines_canonical_names_and_aliases() { @@ -2937,6 +4802,23 @@ mod path_tests { ); } + #[test] + fn idle_reaper_selects_only_sessions_past_the_ttl() { + use super::{stale_cursor_ids, MPX_IDLE_TTL}; + let now = std::time::Instant::now(); + let mut last_use = std::collections::HashMap::new(); + last_use.insert("fresh".to_owned(), now); + last_use.insert( + "old".to_owned(), + now - (MPX_IDLE_TTL + std::time::Duration::from_secs(60)), + ); + last_use.insert("edge".to_owned(), now - MPX_IDLE_TTL); + let mut stale = stale_cursor_ids(&last_use, now, MPX_IDLE_TTL); + stale.sort(); + assert_eq!(stale, vec!["edge".to_owned(), "old".to_owned()]); + assert!(stale_cursor_ids(&std::collections::HashMap::new(), now, MPX_IDLE_TTL).is_empty()); + } + #[test] fn slave_pointer_name_fits_evdev_uinput_limit() { for cursor_id in ["m".repeat(200), "cursor-鼠".repeat(50)] { @@ -3013,7 +4895,7 @@ mod path_tests { assert!(is_uinput_unavailable(&error)); assert_eq!( error.to_string(), - "Linux uinput pointer unavailable: permission denied" + "Linux uinput device unavailable: permission denied" ); } @@ -3028,10 +4910,76 @@ mod path_tests { #[test] fn real_pointer_capabilities_require_uinput_access() { - assert!(real_pointer_capabilities_available(true, false, true)); - assert!(!real_pointer_capabilities_available(true, false, false)); - assert!(!real_pointer_capabilities_available(false, false, true)); - assert!(!real_pointer_capabilities_available(true, true, true)); + assert!(real_pointer_capabilities_available( + true, false, true, false + )); + assert!(!real_pointer_capabilities_available( + true, false, false, false + )); + assert!(!real_pointer_capabilities_available( + false, false, true, false + )); + assert!(!real_pointer_capabilities_available( + true, true, true, false + )); + assert!(!real_pointer_capabilities_available( + true, false, true, true + )); + } + + #[test] + fn kde_x11_sessions_disable_uinput_pointer_hotplug() { + assert!(kde_x11_uinput_hotplug_is_unsafe( + Some("x11"), + Some("KDE"), + None, + None, + None, + Some(":0"), + None, + )); + assert!(kde_x11_uinput_hotplug_is_unsafe( + Some("x11"), + Some("KDE"), + None, + None, + None, + Some(":0"), + Some("wayland-0"), + )); + assert!(kde_x11_uinput_hotplug_is_unsafe( + None, + None, + Some("plasma"), + None, + Some("true"), + Some(":1"), + None, + )); + + assert!(kde_x11_uinput_hotplug_is_unsafe( + Some("wayland"), + Some("KDE"), + None, + None, + Some("true"), + Some(":0"), + Some("wayland-0"), + )); + assert!(!kde_x11_uinput_hotplug_is_unsafe( + Some("x11"), + Some("GNOME"), + None, + None, + None, + Some(":0"), + None, + )); + + let error = ensure_master_pointer_for_session("regression-test", true) + .expect_err("the creation choke point must refuse before opening X11 or uinput"); + assert!(is_uinput_unavailable(&error)); + assert!(error.to_string().contains("delivery_mode='foreground'")); } #[test] @@ -3050,7 +4998,7 @@ mod path_tests { assert!((y - (2.0 * x + 1.0)).abs() < 1e-9); } // sin(x) parses and yields finite, bounded values. - let s = sample_function("100+50*sin(x)", 0.0, 6.28, 40).unwrap(); + let s = sample_function("100+50*sin(x)", 0.0, std::f64::consts::TAU, 40).unwrap(); assert!(s.iter().all(|(_, y)| (49.9..=150.1).contains(y))); } @@ -3118,3 +5066,58 @@ mod path_tests { assert_eq!(point_on_path(&path, &cum, total, 0.3), (5, 5)); } } + +pub struct DesktopKeyboard; +impl cua_driver_core::desktop_keyboard::DesktopKeyboard for DesktopKeyboard { + fn resolve(&self, key: &str) -> std::result::Result { + let _lock = KEYMAP_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let keysym = key_name_to_keysym(key).map_err(|error| error.to_string())?; + let (connection, _) = connect_x11_for_input().map_err(|error| error.to_string())?; + let mapping = connection + .get_keyboard_mapping(8, 248) + .map_err(|error| error.to_string())? + .reply() + .map_err(|error| error.to_string())?; + let width = mapping.keysyms_per_keycode as usize; + if width == 0 { + return Err("Empty X11 keyboard map".into()); + } + mapping + .keysyms + .chunks(width) + .position(|codes| codes.contains(&keysym)) + .map(|index| index as u32 + 8) + .ok_or_else(|| { + format!("Key {key} is not mapped. Held keys do not borrow temporary keycodes.") + }) + } + fn transition(&self, code: u32, down: bool, _held: &[u32]) -> std::result::Result<(), String> { + use x11rb::protocol::xtest::ConnectionExt as _; + let _lock = KEYMAP_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let result = (|| -> Result<()> { + let (connection, _) = connect_x11_for_input()?; + connection + .xtest_fake_input( + if down { + KEY_PRESS_EVENT + } else { + KEY_RELEASE_EVENT + }, + code as u8, + 0, + x11rb::NONE, + 0, + 0, + 0, + )? + .check()?; + connection.get_input_focus()?.reply()?; + Ok(()) + })(); + result.map_err(|error| error.to_string()) + } +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/input/mpx_keyboard.rs b/packages/cua-driver/rust/crates/platform-linux/src/input/mpx_keyboard.rs new file mode 100644 index 00000000000..9c646644a9b --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-linux/src/input/mpx_keyboard.rs @@ -0,0 +1,1313 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Focus-free keyboard delivery on X11 via XInput2 MPX: a second master +//! keyboard with its own focus, fed by a uinput slave. +//! +//! X11 has exactly one *core* keyboard focus, which is why XTest (real input, +//! but always to the core focus) cannot type into a background window, and +//! why the toolkits that ignore `send_event` (GTK, LibreOffice VCL, Qt, +//! Chromium) drop the XSendEvent alternative. MPX lifts the single-focus +//! limit: every master keyboard carries its own focus window +//! (`XISetFocus`), and key events from a slave attached to that master are +//! delivered — as real, non-synthetic XI2 *and* core events — to that master's +//! focus, while the user's core keyboard, its focus and the pointer stay +//! untouched. XTEST slaves cannot be re-attached to another master, so the +//! slave is a uinput keyboard hot-added through udev/libinput. +//! +//! Lifecycle mirrors the MPX pointer click: the master pair is created (or +//! reused) for the session, the keyboard slave is attached, the target window +//! is focused for that master only, keys are emitted, delivery is confirmed +//! through the server's raw key events. The pair is retained until session +//! cleanup; a keyboard with unconfirmed input cannot be reused meanwhile. + +use super::*; +use x11rb::protocol::xproto::{ConnectionExt as _, GetKeyboardMappingReply}; + +const UINPUT_KEYBOARD_SUFFIX: &str = " uinput keyboard"; +/// Structured `path` reported for a delivery through the virtual master keyboard. +pub const MPX_UINPUT_PATH: &str = "mpx_uinput"; +/// evdev keycodes are X keycodes minus this offset (X reserves 0..=7). +const X_KEYCODE_OFFSET: u16 = 8; +/// Highest evdev key code the uinput keyboard advertises (KEY_* range below +/// the BTN_* block at 0x100, so libinput classifies it as a keyboard only). +const MAX_EVDEV_KEY: u16 = 0xff; +/// Unicode keysyms are the codepoint with this bit set (X11 keysym encoding). +const UNICODE_KEYSYM_BASE: u32 = 0x0100_0000; +/// How long the server gets to report every emitted transition before the +/// retained keyboard becomes unavailable until session cleanup. +const DELIVERY_CONFIRM_TIMEOUT: Duration = Duration::from_millis(1500); + +struct UinputKeyboard { + device: Arc>, + // A timed-out stream can still enter Xserver after a later subscription. + // Keep its device alive but unavailable until normal session cleanup. + pending: bool, +} + +static UINPUT_KEYBOARDS: OnceLock>> = OnceLock::new(); + +fn uinput_keyboards() -> &'static Mutex> { + UINPUT_KEYBOARDS.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub(super) fn forget_uinput_keyboard(cursor_id: &str) { + uinput_keyboards().lock().unwrap().remove(cursor_id); +} + +fn slave_keyboard_name(master_name: &str) -> String { + format!("{master_name}{UINPUT_KEYBOARD_SUFFIX}") +} + +/// The keyboard route needs exactly what the MPX pointer route needs: a real +/// Xorg with udev/libinput hotplug and a writable `/dev/uinput`. +pub fn real_keyboard_input_available() -> bool { + real_pointer_input_available() +} + +pub(crate) fn check_keyboard_background(cursor_id: &str) -> Result<()> { + if uinput_keyboards() + .lock() + .unwrap() + .get(cursor_id) + .is_some_and(|keyboard| keyboard.pending) + { + bail!( + "not_settable: the virtual keyboard has unconfirmed input; no input was sent \ + by this request. Observe the target, then end and recreate the CUA session \ + before sending more keyboard input" + ); + } + // Xorg emits core FocusIn even for a master with send_core disabled. + // Openbox treats that event as global activation; Mutter keeps it local. + if crate::x11::window_manager_name() + .is_some_and(|name| name.trim_end_matches('\0').eq_ignore_ascii_case("Openbox")) + { + return Err(delivery::BackgroundUnavailable::WindowManagerFocus.into()); + } + Ok(()) +} + +/// One evdev key transition for the uinput keyboard. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct KeyStep { + pub evdev_code: u16, + pub press: bool, +} + +impl KeyStep { + fn x_keycode(self) -> u8 { + (self.evdev_code + X_KEYCODE_OFFSET) as u8 + } +} + +/// X keycode → evdev code. X keycodes below 8 have no evdev counterpart. +pub(super) fn evdev_code_for_x_keycode(keycode: u8) -> Option { + let keycode = u16::from(keycode); + (keycode >= X_KEYCODE_OFFSET && keycode - X_KEYCODE_OFFSET <= MAX_EVDEV_KEY) + .then(|| keycode - X_KEYCODE_OFFSET) +} + +/// Keysym for a typed character: Latin-1 keysyms are the codepoint, everything +/// above is the Unicode keysym; `\n` and `\t` become Return and Tab. +pub(super) fn keysym_for_char(ch: char) -> u32 { + match ch { + '\n' | '\r' => 0xff0d, + '\t' => 0xff09, + c if (c as u32) < 0x100 => c as u32, + c => UNICODE_KEYSYM_BASE | c as u32, + } +} + +/// Press/release steps for one key at `keycode`, wrapped in Shift when the +/// keysym lives in the shifted column and Shift is not already held. +fn tap_steps(keycode: u8, needs_shift: bool, shift: Option, steps: &mut Vec) { + let Some(code) = evdev_code_for_x_keycode(keycode) else { + return; + }; + let shift = if needs_shift { shift } else { None }; + if let Some(shift) = shift { + steps.push(KeyStep { + evdev_code: shift, + press: true, + }); + } + steps.push(KeyStep { + evdev_code: code, + press: true, + }); + steps.push(KeyStep { + evdev_code: code, + press: false, + }); + if let Some(shift) = shift { + steps.push(KeyStep { + evdev_code: shift, + press: false, + }); + } +} + +/// Plan the evdev transitions that type `text` under `mapping`, resolving +/// characters missing from the keymap through `remap` (a spare-keycode +/// allocator) INLINE, one character at a time, so a mixed-script string +/// (an in-map prefix, a character needing remap, more in-map text) keeps its +/// original order in `steps` — remapping a character never defers it behind +/// the characters that came after it in `text`. +pub(super) fn plan_text_with_fallback( + mapping: &GetKeyboardMappingReply, + shift_x_keycode: Option, + text: &str, + remap: impl FnMut(char, &mut GetKeyboardMappingReply) -> Result<(u8, Option)>, +) -> Result<(Vec, Vec)> { + let shift = shift_x_keycode.and_then(evdev_code_for_x_keycode); + let mut steps = Vec::with_capacity(text.len() * 2); + let guards = type_text_with_preflight( + text, + mapping, + remap, + |keycode, needs_shift| { + tap_steps(keycode, needs_shift, shift, &mut steps); + Ok(()) + }, + || Ok(()), + )?; + Ok((steps, guards)) +} + +/// Plan a chord: modifiers pressed in order, the key tapped (with Shift when +/// its keysym is shifted and Shift was not requested), modifiers released in +/// reverse order — the same ordering physical input produces. +pub(super) fn plan_chord( + modifier_x_keycodes: &[u8], + key_x_keycode: u8, + key_needs_shift: bool, + shift_x_keycode: Option, +) -> Vec { + let modifiers: Vec = modifier_x_keycodes + .iter() + .filter_map(|&keycode| evdev_code_for_x_keycode(keycode)) + .collect(); + let shift = shift_x_keycode.and_then(evdev_code_for_x_keycode); + let shift_requested = shift.is_some_and(|shift| modifiers.contains(&shift)); + let mut steps = Vec::with_capacity(modifiers.len() * 2 + 4); + for &code in &modifiers { + steps.push(KeyStep { + evdev_code: code, + press: true, + }); + } + tap_steps( + key_x_keycode, + key_needs_shift && !shift_requested, + shift, + &mut steps, + ); + for &code in modifiers.iter().rev() { + steps.push(KeyStep { + evdev_code: code, + press: false, + }); + } + steps +} + +/// What the virtual-keyboard route could observe about its own delivery. +/// Nothing here reads the application's state back (that stays the caller's +/// screenshot/AT-SPI job); it only says the events went where they were aimed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KeyboardDeliveryReport { + /// The target window still owned the virtual master keyboard's focus + /// after the last key event. + pub virtual_focus_held: bool, + /// `XGetInputFocus` (the user's core focus) was identical before and after. + pub core_focus_unchanged: bool, + /// The server reported the last emitted key transition as a raw event. + pub delivery_confirmed: bool, + /// Key transitions emitted through the uinput slave. + pub key_events: usize, + /// Characters that have no keycode in the current keymap and could not be + /// hosted on a spare keycode either. + pub skipped_characters: Vec, + /// What the background focus guard observed and restored around the + /// delivery (attached by the tool layer; `None` when it did not run). + pub focus_guard: Option, + /// Stuck input the virtual master still held from an earlier aborted + /// action and that was released before this delivery (`button1`, + /// `modifiers(...)`); see `release_stuck_virtual_input`. + pub released_stuck: Vec, + /// Route name: [`MPX_UINPUT_PATH`], or [`super::XTEST_CORE_GRAB_PATH`] + /// when the target's own popup held the keyboard grab and the keys went + /// through the core keyboard instead. + pub path: &'static str, + /// The popup whose grab chose the core-keyboard route. + pub grab_popup: Option, +} + +impl KeyboardDeliveryReport { + pub fn to_json(&self) -> serde_json::Value { + let mut json = serde_json::json!({ + "path": self.path, + "virtual_focus_held": self.virtual_focus_held, + "core_focus_unchanged": self.core_focus_unchanged, + "delivery_confirmed": self.delivery_confirmed, + "key_events": self.key_events, + "skipped_characters": self.skipped_characters.iter().collect::(), + }); + if let Some(guard) = &self.focus_guard { + for (key, value) in guard.to_json().as_object().into_iter().flatten() { + json[key] = value.clone(); + } + if let Some(item) = guard.evidence_item() { + json["evidence"] = serde_json::json!([item]); + } + } + if !self.released_stuck.is_empty() { + json["released_stuck_input"] = serde_json::json!(self.released_stuck); + } + if let Some(popup) = &self.grab_popup { + json["grab_popup"] = popup.to_json(); + } + json + } + + /// How the keys travelled, for the tool text: the virtual master + /// keyboard, or the core keyboard under the target's own popup grab. + pub fn route_phrase(&self) -> String { + match &self.grab_popup { + Some(popup) => format!( + "on the core keyboard, which the target's own {} holds grabbed (path={}; \ + core focus and active window verified {})", + popup.describe(), + self.path, + if self.core_focus_unchanged { + "unchanged" + } else { + "CHANGED — check with a screenshot" + } + ), + None => format!( + "through a virtual master keyboard (path={}, focus untouched)", + self.path + ), + } + } + + /// Sentences for the tool text: any focus change the guard saw, and any + /// stuck input released before delivery (empty when nothing to say). + pub fn delivery_notes(&self) -> String { + let mut notes = self + .focus_guard + .as_ref() + .map(|guard| guard.summary()) + .unwrap_or_default(); + if !self.released_stuck.is_empty() { + notes.push_str(&format!( + " Released stuck virtual input left by an earlier action before delivering: {}.", + self.released_stuck.join(", ") + )); + } + notes + } +} + +fn create_uinput_keyboard(name: &str) -> Result { + guarded_uinput_creation(name, |name| { + let mut keys = AttributeSet::::new(); + // Every KEY_* code the X keycode range can address, so any keycode the + // server's keymap resolves to can be emitted, including the modifiers. + for code in 1..=MAX_EVDEV_KEY { + keys.insert(Key::new(code)); + } + Ok(evdev::uinput::VirtualDeviceBuilder::new()? + .name(name) + .with_keys(&keys)? + .build()?) + }) +} + +/// Make sure the session's master pair carries an attached uinput keyboard +/// slave; creates the pair (pointer + keyboard master, uinput pointer) through +/// `ensure_master_pointer` first when needed. +fn ensure_master_keyboard( + cursor_id: &str, +) -> Result<(MasterPointerIds, Arc>)> { + check_keyboard_background(cursor_id)?; + let device = { + let keyboards = uinput_keyboards().lock().unwrap(); + keyboards + .get(cursor_id) + .map(|keyboard| keyboard.device.clone()) + }; + let ids = ensure_master_pointer(cursor_id)?; + if let (Some(_), Some(device)) = (ids.slave_keyboard_id, device) { + return Ok((ids, device)); + } + + let display = open_display()?; + let result = (|| -> Result<(MasterPointerIds, Arc>)> { + // The master keyboard's device name is " keyboard"; recover the + // base to derive the slave's name so the reaper can pair them by eye. + let base = xi2_query_devices(display)? + .into_iter() + .find(|(id, _, _)| *id == ids.keyboard_id) + .and_then(|(_, _, name)| name.strip_suffix(" keyboard").map(str::to_owned)) + .ok_or_else(|| anyhow!("master keyboard {} vanished", ids.keyboard_id))?; + let device_name = slave_keyboard_name(&base); + let device = create_uinput_keyboard(&device_name)?; + let slave_keyboard_id = wait_for_slave_id( + display, + &device_name, + x11::xinput2::XISlaveKeyboard, + Duration::from_secs(5), + )?; + attach_slave_to_master(display, slave_keyboard_id, ids.keyboard_id)?; + let ids = MasterPointerIds { + slave_keyboard_id: Some(slave_keyboard_id), + ..ids + }; + mpx_pointers() + .lock() + .unwrap() + .insert(cursor_id.to_owned(), ids); + let device = Arc::new(Mutex::new(device)); + uinput_keyboards().lock().unwrap().insert( + cursor_id.to_owned(), + UinputKeyboard { + device: device.clone(), + pending: true, + }, + ); + warm_up_keyboard(display, &mut device.lock().unwrap(), slave_keyboard_id)?; + uinput_keyboards() + .lock() + .unwrap() + .get_mut(cursor_id) + .unwrap() + .pending = false; + Ok((ids, device)) + })(); + unsafe { x11::xlib::XCloseDisplay(display) }; + result +} + +/// evdev `KEY_UNKNOWN`: X keycode 248, normally `NoSymbol`. Warm up before +/// the planner can borrow this keycode for a temporary mapping. +const WARM_UP_EVDEV_CODE: u16 = 240; + +// x11 exposes XkbClientMapRec as opaque. These complete C layouts are from +// X11/extensions/XKBstr.h; KeySyms uses width * (group_info & 0x0f) entries. +#[repr(C)] +struct XkbSymMap { + kt_index: [u8; 4], + group_info: u8, + width: u8, + offset: u16, +} + +#[repr(C)] +struct XkbClientMap { + size_types: u8, + num_types: u8, + types: x11::xlib::XkbKeyTypePtr, + size_syms: u16, + num_syms: u16, + syms: *mut x11::xlib::KeySym, + key_sym_map: *mut XkbSymMap, + modmap: *mut u8, +} + +// `keyboard` must be a live XkbGetMap result (including its owned arrays). +unsafe fn warm_up_key_has_no_symbols( + keyboard: x11::xlib::XkbDescPtr, + slave_keyboard_id: i32, +) -> bool { + // XkbExplicitXIDevice allows only the low 8 bits; 0x100 selects core. + if !(1..=255).contains(&slave_keyboard_id) { + return false; + } + let Some(keyboard) = (unsafe { keyboard.as_ref() }) else { + return false; + }; + let keycode = (WARM_UP_EVDEV_CODE + X_KEYCODE_OFFSET) as u8; + if keyboard.device_spec != slave_keyboard_id as u16 + || keyboard.min_key_code < 8 + || keyboard.min_key_code > keycode + || keyboard.max_key_code < keycode + { + return false; + } + let Some(map) = (unsafe { keyboard.map.cast::().as_ref() }) else { + return false; + }; + if map.key_sym_map.is_null() || map.syms.is_null() || map.num_syms > map.size_syms { + return false; + } + let entry = unsafe { &*map.key_sym_map.add(usize::from(keycode)) }; + let groups = usize::from(entry.group_info & 0x0f); + if groups > 4 || (groups > 0 && entry.width == 0) { + return false; + } + let count = groups * usize::from(entry.width); + let offset = usize::from(entry.offset); + if offset + count > usize::from(map.num_syms) { + return false; + } + (0..count).all(|index| unsafe { *map.syms.add(offset + index) == 0 }) +} + +fn validate_warm_up_key(display: *mut x11::xlib::Display, slave_keyboard_id: i32) -> Result<()> { + let safe = if (1..=255).contains(&slave_keyboard_id) { + // A disappearing slave can produce BadDevice. GetMap waits for its + // reply, so restore the process handler after this synchronous call. + let old_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + let keyboard = unsafe { + // XKB.h: XkbKeySymsMask = 1 << 1. + x11::xlib::XkbGetMap(display, 1 << 1, slave_keyboard_id as u32) + }; + unsafe { x11::xlib::XSetErrorHandler(old_handler) }; + let safe = unsafe { warm_up_key_has_no_symbols(keyboard, slave_keyboard_id) }; + if !keyboard.is_null() { + // XKB.h: XkbAllComponentsMask = 0x7f; also free the descriptor. + unsafe { x11::xlib::XkbFreeKeyboard(keyboard, 0x7f, x11::xlib::True) }; + } + safe + } else { + false + }; + if !safe { + bail!( + "not_settable: virtual keyboard warm-up could not verify X248 is symbol-less on its \ + slave; no input was sent. Observe the target, then end and recreate the CUA session \ + before sending more keyboard input" + ); + } + Ok(()) +} + +/// The first key events of a freshly hot-plugged keyboard were observed to +/// vanish (the master switches to the slave's keymap on its first event and +/// clients only learn about the new master from the hierarchy event queued +/// ahead of the keys). Tap a symbol-less key through the whole pipeline and +/// wait until the server reports it back, so the real text starts on a warm +/// device. Unconfirmed warm-up leaves the retained device unavailable. +fn warm_up_keyboard( + display: *mut x11::xlib::Display, + device: &mut VirtualDevice, + slave_keyboard_id: i32, +) -> Result<()> { + validate_warm_up_key(display, slave_keyboard_id)?; + let xi_opcode = xinput_opcode(display) + .ok_or_else(|| anyhow!("virtual keyboard warm-up requires XInput2"))?; + select_raw_key_events(display, slave_keyboard_id)?; + let steps = [ + KeyStep { + evdev_code: WARM_UP_EVDEV_CODE, + press: true, + }, + KeyStep { + evdev_code: WARM_UP_EVDEV_CODE, + press: false, + }, + ]; + for step in steps { + if let Err(error) = device.emit(&[InputEvent::new( + EventType::KEY, + step.evdev_code, + if step.press { 1 } else { 0 }, + )]) { + let _ = device.emit(&[InputEvent::new(EventType::KEY, WARM_UP_EVDEV_CODE, 0)]); + return Err(keymap_barrier::unconfirmed(format!( + "virtual keyboard warm-up failed: {error}; no requested input was sent. \ + Observe the target, then end and recreate the CUA session before sending more keyboard input" + ))); + } + sleep(Duration::from_millis(KEY_DELAY_MS)); + } + let seen = wait_for_raw_keys( + display, + xi_opcode, + slave_keyboard_id, + &steps, + Duration::from_millis(1500), + ); + if !seen { + return Err(keymap_barrier::unconfirmed( + "virtual keyboard warm-up was not confirmed; no requested input was sent. \ + Observe the target, then end and recreate the CUA session before sending more keyboard input", + )); + } + // Give clients a beat to process the hierarchy/keymap change. + sleep(Duration::from_millis(120)); + Ok(()) +} + +/// Shift keycode from the server's modifier map (modifier index 0). +fn shift_keycode(conn: &RustConnection) -> Option { + let modmap = conn.get_modifier_mapping().ok()?.reply().ok()?; + let per = modmap.keycodes_per_modifier() as usize; + modmap + .keycodes + .get(..per) + .and_then(|shift| shift.iter().copied().find(|&keycode| keycode != 0)) +} + +fn xi_get_focus(display: *mut x11::xlib::Display, device_id: i32) -> Option { + let mut focus: x11::xlib::Window = 0; + let rc = unsafe { x11::xinput2::XIGetFocus(display, device_id, &mut focus) }; + (rc == 0).then_some(focus) +} + +fn core_focus(display: *mut x11::xlib::Display) -> x11::xlib::Window { + let mut focus: x11::xlib::Window = 0; + let mut revert_to = 0; + unsafe { x11::xlib::XGetInputFocus(display, &mut focus, &mut revert_to) }; + focus +} + +/// Select raw key events on the root window for `device_id` so the server's +/// processing of our own key transitions can be observed on `display`. +fn select_raw_key_events(display: *mut x11::xlib::Display, device_id: i32) -> Result<()> { + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + let mut mask_bits = vec![0u8; xi_mask_len()]; + x11::xinput2::XISetMask(&mut mask_bits, x11::xinput2::XI_RawKeyPress); + x11::xinput2::XISetMask(&mut mask_bits, x11::xinput2::XI_RawKeyRelease); + let mut evmask = x11::xinput2::XIEventMask { + deviceid: device_id, + mask_len: mask_bits.len() as std::os::raw::c_int, + mask: mask_bits.as_mut_ptr(), + }; + let rc = unsafe { x11::xinput2::XISelectEvents(display, root, &mut evmask, 1) }; + unsafe { x11::xlib::XSync(display, 0) }; + if rc != 0 { + bail!("XISelectEvents(raw keys) failed with status {rc}"); + } + Ok(()) +} + +/// Confirm every transition in this batch, in order. An earlier repeated key +/// release cannot establish that the final uinput event reached the server. +fn wait_for_raw_keys( + display: *mut x11::xlib::Display, + xi_opcode: std::os::raw::c_int, + source_id: i32, + steps: &[KeyStep], + timeout: Duration, +) -> bool { + let mut remaining = steps.iter(); + let Some(mut next) = remaining.next() else { + return true; + }; + let deadline = std::time::Instant::now() + timeout; + while std::time::Instant::now() < deadline { + if unsafe { x11::xlib::XPending(display) } == 0 { + sleep(Duration::from_millis(2)); + continue; + } + let mut ev: x11::xlib::XEvent = unsafe { std::mem::zeroed() }; + unsafe { x11::xlib::XNextEvent(display, &mut ev) }; + if unsafe { ev.type_ } != x11::xlib::GenericEvent { + continue; + } + let mut cookie = unsafe { ev.generic_event_cookie }; + if cookie.extension != xi_opcode + || !matches!( + cookie.evtype, + x11::xinput2::XI_RawKeyPress | x11::xinput2::XI_RawKeyRelease + ) + { + continue; + } + if unsafe { x11::xlib::XGetEventData(display, &mut cookie) } == 0 { + continue; + } + let raw = cookie.data as *const x11::xinput2::XIRawEvent; + let transition = (!raw.is_null()).then(|| { + let raw = unsafe { &*raw }; + ( + if raw.sourceid == 0 { + raw.deviceid + } else { + raw.sourceid + }, + raw.detail, + cookie.evtype == x11::xinput2::XI_RawKeyPress, + ) + }); + unsafe { x11::xlib::XFreeEventData(display, &mut cookie) }; + if let Some((source, keycode, press)) = transition { + if source != source_id { + continue; + } + if keycode != i32::from(next.x_keycode()) || press != next.press { + return false; + } + match remaining.next() { + Some(step) => next = step, + None => return true, + } + } + } + false +} + +/// Everything the planners need from the server's keymap, resolved once per +/// call on a short-lived x11rb connection (the same helpers the XSendEvent and +/// XTest routes use, so keycode choice is identical across routes). +struct Keymap { + conn: RustConnection, + mapping: GetKeyboardMappingReply, + shift: Option, +} + +impl Keymap { + fn load() -> Result { + let (conn, _) = connect_x11_for_input()?; + let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?; + let shift = shift_keycode(&conn); + Ok(Self { + conn, + mapping, + shift, + }) + } +} + +/// Deliver `steps` to `target_window` through the session's virtual master +/// keyboard. Remaps stay live through server delivery and the client queue. +fn deliver( + cursor_id: &str, + target_window: u64, + steps: &[KeyStep], + remap_guards: Vec>, + keymap_conn: &RustConnection, + barrier: Option>, + keyboard: (MasterPointerIds, Arc>), +) -> Result { + let (ids, device) = keyboard; + let display = open_display()?; + let result = (|| -> Result { + supports_parallel_pointer_injection(display)?; + let mut major = 2; + let mut minor = 3; + let rc = unsafe { x11::xinput2::XIQueryVersion(display, &mut major, &mut minor) }; + if rc != 0 { + bail!("XIQueryVersion failed with status {rc}"); + } + let xi_opcode = xinput_opcode(display) + .ok_or_else(|| anyhow!("virtual keyboard delivery requires XInput2"))?; + let slave_keyboard_id = ids + .slave_keyboard_id + .ok_or_else(|| anyhow!("uinput keyboard for '{cursor_id}' is not attached"))?; + + let window = target_window as x11::xlib::Window; + let core_before = core_focus(display); + let released_stuck = release_stuck_virtual_input(cursor_id, display, ids, &device); + select_raw_key_events(display, slave_keyboard_id)?; + // A BadWindow/BadMatch from XISetFocus (target unmapped meanwhile) must + // surface as an error, not take the daemon down through Xlib's default + // handler. + let previous_handler = unsafe { x11::xlib::XSetErrorHandler(Some(ignore_x_error)) }; + // XISetFocus reports async BadWindow/BadMatch (window not viewable) + // through the error handler, so a stale GetFocus is the real signal. + // Retry once — a freshly mapped/raised toplevel can miss the first set. + let mut focus_taken = false; + for attempt in 0..2 { + let rc = unsafe { + x11::xinput2::XISetFocus(display, ids.keyboard_id, window, x11::xlib::CurrentTime) + }; + unsafe { x11::xlib::XSync(display, 0) }; + if rc == 0 && xi_get_focus(display, ids.keyboard_id) == Some(window) { + focus_taken = true; + break; + } + if attempt == 0 { + sleep(Duration::from_millis(30)); + } + } + unsafe { x11::xlib::XSetErrorHandler(previous_handler) }; + if !focus_taken { + bail!( + "not_settable: virtual master keyboard could not take focus on window 0x{target_window:x} \ + (not viewable, or another client owns the master focus); no requested keyboard input was sent" + ); + } + + uinput_keyboards() + .lock() + .unwrap() + .get_mut(cursor_id) + .unwrap() + .pending = true; + + let mut held: Vec = Vec::new(); + for step in steps { + let emitted = { + let mut device = device.lock().unwrap(); + device.emit(&[InputEvent::new( + EventType::KEY, + step.evdev_code, + if step.press { 1 } else { 0 }, + )]) + }; + if let Err(error) = emitted { + // Never abort with keys down: a held modifier would chord + // every later key of this session. + let mut device = device.lock().unwrap(); + if step.press { + let _ = device.emit(&[InputEvent::new(EventType::KEY, step.evdev_code, 0)]); + } + for code in held.iter().rev() { + let _ = device.emit(&[InputEvent::new(EventType::KEY, *code, 0)]); + } + return Err(keymap_barrier::unconfirmed(error.to_string())); + } + if step.press { + held.push(step.evdev_code); + } else { + held.retain(|code| *code != step.evdev_code); + } + sleep(Duration::from_millis(KEY_DELAY_MS)); + } + let delivery_confirmed = wait_for_raw_keys( + display, + xi_opcode, + slave_keyboard_id, + steps, + DELIVERY_CONFIRM_TIMEOUT, + ); + keymap_conn + .get_input_focus() + .map_err(|error| keymap_barrier::unconfirmed(error.to_string()))? + .reply() + .map_err(|error| keymap_barrier::unconfirmed(error.to_string()))?; + let virtual_focus_held = xi_get_focus(display, ids.keyboard_id) == Some(window); + let core_focus_unchanged = core_focus(display) == core_before; + if let Some(barrier) = &barrier { + if !delivery_confirmed || !virtual_focus_held { + return Err(keymap_barrier::unconfirmed( + "virtual key delivery or focus was not confirmed", + )); + } + barrier.wait()?; + } + if !delivery_confirmed { + tracing::warn!( + cursor_id, + "virtual keyboard: server did not confirm the complete key batch within {:?}", + DELIVERY_CONFIRM_TIMEOUT + ); + } + Ok(KeyboardDeliveryReport { + virtual_focus_held, + core_focus_unchanged, + delivery_confirmed, + key_events: steps.len(), + skipped_characters: Vec::new(), + focus_guard: None, + released_stuck, + path: MPX_UINPUT_PATH, + grab_popup: None, + }) + })(); + drop(remap_guards); + // The master pair is retained for reuse. Per-call teardown would hot-plug + // a uinput keyboard and churn the XInput hierarchy on every keystroke + // batch — enough to crash LibreOffice VCL. The pair is cleaned up on + // end_session / idle / startup reap instead. + unsafe { x11::xlib::XCloseDisplay(display) }; + if result + .as_ref() + .is_ok_and(|report| report.delivery_confirmed && report.virtual_focus_held) + { + uinput_keyboards() + .lock() + .unwrap() + .get_mut(cursor_id) + .unwrap() + .pending = false; + } + result +} + +/// Type `text` into `target_window` without touching the core focus or the +/// user's pointer. See the module docs for the mechanism. +pub fn send_virtual_keyboard_text( + cursor_id: &str, + target_window: u64, + text: &str, +) -> Result { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _op = mpx_op_guard(cursor_id); + let keyboard = ensure_master_keyboard(cursor_id)?; + let keymap = Keymap::load()?; + let barrier = KeymapBarrier::prepare( + &keymap.conn, + Some(target_window as u32), + &keymap.mapping, + text.chars().map(keysym_for_char), + )?; + // Characters absent from the keymap (a sparse layout, or a non-Latin + // glyph) are hosted on spare keycodes, with a guard that + // restores the map after delivery. plan_text_with_fallback resolves each + // character in ORIGINAL text order — remapping a character never defers + // it behind the characters that came after it, or a mixed-script string + // (e.g. an ASCII prefix, a character needing remap, then more ASCII) + // would come out with all the in-map characters first, reordering the + // typed text relative to what was asked for. + let (steps, guards) = + plan_text_with_fallback(&keymap.mapping, keymap.shift, text, |ch, mapping| { + keycode_for_keysym(&keymap.conn, mapping, keysym_for_char(ch), &ch.to_string()) + })?; + deliver( + cursor_id, + target_window, + &steps, + guards, + &keymap.conn, + barrier, + keyboard, + ) +} + +/// Press `key` with `modifiers` held, into `target_window`, without touching +/// the core focus or the user's pointer. +pub fn send_virtual_keyboard_key( + cursor_id: &str, + target_window: u64, + key: &str, + modifiers: &[&str], +) -> Result { + let _keymap = KEYMAP_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _op = mpx_op_guard(cursor_id); + let keyboard = ensure_master_keyboard(cursor_id)?; + let mut keymap = Keymap::load()?; + let requested = std::iter::once(key) + .chain(modifiers.iter().copied()) + .map(key_name_to_keysym) + .collect::>>()?; + let barrier = KeymapBarrier::prepare( + &keymap.conn, + Some(target_window as u32), + &keymap.mapping, + requested, + )?; + let mut guards = Vec::new(); + let mut modifier_keycodes = Vec::with_capacity(modifiers.len()); + for modifier in modifiers { + let keysym = key_name_to_keysym(modifier)?; + let (keycode, guard) = + keycode_for_keysym(&keymap.conn, &mut keymap.mapping, keysym, modifier)?; + guards.extend(guard); + modifier_keycodes.push(keycode); + } + let keysym = key_name_to_keysym(key)?; + let (keycode, needs_shift) = match char_to_keycode_shift(&keymap.mapping, keysym) { + Some(found) => found, + None => { + let (keycode, guard) = + keycode_for_keysym(&keymap.conn, &mut keymap.mapping, keysym, key)?; + guards.extend(guard); + (keycode, false) + } + }; + let steps = plan_chord(&modifier_keycodes, keycode, needs_shift, keymap.shift); + deliver( + cursor_id, + target_window, + &steps, + guards, + &keymap.conn, + barrier, + keyboard, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A tiny US-like keymap: keycode 38 = a/A, 10 = 1/!, 36 = Return, + /// 50 = Shift_L, 37 = Control_L, 64 = Alt_L, 250 = spare (NoSymbol). + fn mapping() -> GetKeyboardMappingReply { + let per = 2usize; + let mut keysyms = vec![0u32; (248 - 8 + 1) * per]; + let mut set = |keycode: usize, syms: [u32; 2]| { + let idx = (keycode - 8) * per; + keysyms[idx..idx + per].copy_from_slice(&syms); + }; + set(38, [0x61, 0x41]); + set(10, [0x31, 0x21]); + set(36, [0xff0d, 0xff0d]); + set(50, [0xffe1, 0xffe1]); + set(37, [0xffe3, 0xffe3]); + set(64, [0xffe9, 0xffe9]); + GetKeyboardMappingReply { + keysyms_per_keycode: per as u8, + sequence: 0, + keysyms, + } + } + + fn step(evdev_code: u16, press: bool) -> KeyStep { + KeyStep { evdev_code, press } + } + + #[test] + fn x_keycodes_map_to_evdev_by_subtracting_eight() { + assert_eq!(evdev_code_for_x_keycode(38), Some(30)); // KEY_A + assert_eq!(evdev_code_for_x_keycode(50), Some(42)); // KEY_LEFTSHIFT + assert_eq!(evdev_code_for_x_keycode(37), Some(29)); // KEY_LEFTCTRL + assert_eq!(evdev_code_for_x_keycode(8), Some(0)); + assert_eq!(evdev_code_for_x_keycode(7), None); + assert_eq!(evdev_code_for_x_keycode(0), None); + assert_eq!(step(30, true).x_keycode(), 38); + } + + #[test] + fn characters_resolve_to_latin1_unicode_or_named_keysyms() { + assert_eq!(keysym_for_char('a'), 0x61); + assert_eq!(keysym_for_char('é'), 0xe9); + assert_eq!(keysym_for_char('€'), 0x0100_0000 | 0x20ac); + assert_eq!(keysym_for_char('\n'), 0xff0d); + assert_eq!(keysym_for_char('\t'), 0xff09); + } + + #[test] + fn text_plan_taps_keys_and_wraps_shifted_glyphs_in_shift() { + let (steps, _) = + plan_text_with_fallback::<()>(&mapping(), Some(50), "aA1!\n", |_, _| unreachable!()) + .unwrap(); + assert_eq!( + steps, + vec![ + step(30, true), + step(30, false), + step(42, true), + step(30, true), + step(30, false), + step(42, false), + step(2, true), + step(2, false), + step(42, true), + step(2, true), + step(2, false), + step(42, false), + step(28, true), + step(28, false), + ] + ); + } + + #[test] + fn text_plan_without_a_shift_keycode_types_the_unshifted_level() { + // No Shift in the modifier map: the keycode is still tapped (the + // server types the base glyph) rather than the character being lost. + let (steps, _) = + plan_text_with_fallback::<()>(&mapping(), None, "A", |_, _| unreachable!()).unwrap(); + assert_eq!(steps, vec![step(30, true), step(30, false)]); + } + + #[test] + fn text_plan_with_fallback_preserves_original_character_order() { + // "a" + a char requiring remap (é, missing from this tiny keymap) + + // "1" — a stand-in for a mixed-script string where an ASCII prefix + // is followed by a character needing a spare-keycode remap and then + // more ASCII. The remapped character must land in the MIDDLE of the + // key-event sequence, not be deferred to the end. + const REMAPPED_KEYCODE: u8 = 250; // spare, per the mapping() doc comment + let (steps, _) = plan_text_with_fallback::<()>(&mapping(), Some(50), "aé1", |ch, _| { + assert_eq!( + ch, 'é', + "only the out-of-keymap character should hit the fallback" + ); + Ok((REMAPPED_KEYCODE, None)) + }) + .unwrap(); + let remapped_evdev = evdev_code_for_x_keycode(REMAPPED_KEYCODE).unwrap(); + assert_eq!( + steps, + vec![ + // 'a' + step(30, true), + step(30, false), + // 'é' via the remapped spare keycode — must come BEFORE '1', + // matching its position in the original string. + step(remapped_evdev, true), + step(remapped_evdev, false), + // '1' + step(2, true), + step(2, false), + ] + ); + } + + #[test] + fn text_plan_fails_without_steps_and_restores_remaps_when_spares_run_out() { + let mut keymap = mapping(); + keymap + .keysyms + .iter_mut() + .filter(|sym| **sym == 0) + .skip(2) + .for_each(|sym| *sym = 1); + let restored = std::cell::Cell::new(0); + let error = plan_text_with_fallback(&keymap, Some(50), "aé€a", |ch, mapping| { + reserve_spare_keycode(mapping, keysym_for_char(ch), |_, _, _, _| { + Ok(Restore(&restored)) + }) + .map(|(keycode, guard)| (keycode, Some(guard))) + }) + .err() + .unwrap(); + assert!( + error + .to_string() + .starts_with("typed 0 of 4 characters: the text needs 2 spare keycodes"), + "{error:#}" + ); + assert_eq!(restored.get(), 1); + } + + struct Restore<'a>(&'a std::cell::Cell); + + impl Drop for Restore<'_> { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + #[test] + fn text_plan_gives_distinct_missing_characters_distinct_keycodes() { + let mut remaps = Vec::new(); + let (steps, _) = plan_text_with_fallback(&mapping(), Some(50), "é€é", |ch, mapping| { + let (keycode, ()) = + reserve_spare_keycode(mapping, keysym_for_char(ch), |_, _, _, _| Ok(()))?; + remaps.push(ch); + Ok((keycode, None::<()>)) + }) + .unwrap(); + assert_eq!(remaps, ['é', '€']); + let presses: Vec = steps + .iter() + .filter(|step| step.press) + .map(|step| step.evdev_code) + .collect(); + assert_eq!(presses.len(), 3); + assert_ne!(presses[0], presses[1]); + assert_eq!(presses[0], presses[2]); + } + + #[test] + fn warm_up_keycode_can_be_the_only_spare_for_a_temporary_mapping() { + let mut keymap = mapping(); + keymap.keysyms = vec![1; 248 * usize::from(keymap.keysyms_per_keycode)]; + let keycode = (WARM_UP_EVDEV_CODE + X_KEYCODE_OFFSET) as u8; + let per = keymap.keysyms_per_keycode as usize; + let offset = (usize::from(keycode) - 8) * per; + keymap.keysyms[offset..offset + per].fill(0); + let (borrowed, ()) = reserve_spare_keycode( + &mut keymap, + keysym_for_char('界'), + |borrowed, _, original, _| { + assert_eq!(borrowed, keycode); + assert!(original.iter().all(|symbol| *symbol == 0)); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(borrowed, 248); + } + + fn with_warm_up_map( + check: impl FnOnce( + *mut x11::xlib::_XkbDesc, + *mut XkbClientMap, + *mut XkbSymMap, + *mut x11::xlib::KeySym, + ), + ) { + let mut entries: [XkbSymMap; 256] = std::array::from_fn(|_| XkbSymMap { + kt_index: [0; 4], + group_info: 4, + width: 7, + offset: 3, + }); + let mut symbols = vec![0; 1024]; + let mut map = XkbClientMap { + size_types: 0, + num_types: 0, + types: std::ptr::null_mut(), + size_syms: symbols.len() as u16, + num_syms: symbols.len() as u16, + syms: symbols.as_mut_ptr(), + key_sym_map: entries.as_mut_ptr(), + modmap: std::ptr::null_mut(), + }; + let mut keyboard: x11::xlib::_XkbDesc = unsafe { std::mem::zeroed() }; + keyboard.device_spec = 6; + keyboard.min_key_code = 8; + keyboard.max_key_code = 255; + keyboard.map = (&mut map as *mut XkbClientMap).cast(); + check( + &mut keyboard, + &mut map, + &mut entries[248], + symbols.as_mut_ptr(), + ); + } + + #[test] + fn warm_up_checks_every_group_and_level_in_the_slave_map() { + with_warm_up_map(|keyboard, _, entry, symbols| unsafe { + assert!(warm_up_key_has_no_symbols(keyboard, 6)); + for index in 3..3 + 4 * 7 { + *symbols.add(index) = 0xff0d; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + *symbols.add(index) = 0; + } + (*entry).width = 200; + *symbols.add(3 + 4 * 200 - 1) = 97; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + }); + } + + #[test] + fn warm_up_accepts_verified_empty_groups_with_nonzero_width() { + with_warm_up_map(|keyboard, map, entry, _| unsafe { + // Read-only Xephyr slave6 reply: zero groups, width 2, offset449. + (*entry).group_info = 0; + (*entry).width = 2; + (*entry).offset = 449; + (*map).num_syms = 456; + (*map).size_syms = 640; + assert!(warm_up_key_has_no_symbols(keyboard, 6)); + (*entry).offset = 457; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + }); + } + + #[test] + fn warm_up_rejects_core_wrong_or_invalid_device_ids() { + with_warm_up_map(|keyboard, _, _, _| unsafe { + for id in [-1, 0, 7, 256, 65536, i32::MAX] { + assert!(!warm_up_key_has_no_symbols(keyboard, id)); + } + (*keyboard).device_spec = 256; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + }); + } + + #[test] + fn warm_up_rejects_unknown_or_out_of_bounds_maps() { + assert!(!unsafe { warm_up_key_has_no_symbols(std::ptr::null_mut(), 6) }); + with_warm_up_map(|keyboard, map, entry, _| unsafe { + (*keyboard).min_key_code = 249; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*keyboard).min_key_code = 0; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*keyboard).min_key_code = 8; + (*keyboard).max_key_code = 247; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*keyboard).max_key_code = 255; + (*entry).group_info = 5; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*entry).group_info = 4; + (*entry).width = 0; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*entry).width = 7; + (*entry).offset = u16::MAX; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*entry).offset = 3; + (*map).num_syms = 30; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*map).num_syms = (*map).size_syms + 1; + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*map).num_syms = (*map).size_syms; + (*map).syms = std::ptr::null_mut(); + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + }); + with_warm_up_map(|keyboard, map, _, _| unsafe { + (*map).key_sym_map = std::ptr::null_mut(); + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + (*keyboard).map = std::ptr::null_mut(); + assert!(!warm_up_key_has_no_symbols(keyboard, 6)); + }); + } + + #[test] + fn chord_plan_nests_modifiers_around_the_key_in_physical_order() { + // ctrl+alt+a + assert_eq!( + plan_chord(&[37, 64], 38, false, Some(50)), + vec![ + step(29, true), + step(56, true), + step(30, true), + step(30, false), + step(56, false), + step(29, false), + ] + ); + } + + #[test] + fn chord_plan_adds_shift_only_when_the_key_is_shifted_and_shift_not_held() { + // ctrl+A where 'A' is the shifted level of keycode 38: auto-Shift. + assert_eq!( + plan_chord(&[37], 38, true, Some(50)), + vec![ + step(29, true), + step(42, true), + step(30, true), + step(30, false), + step(42, false), + step(29, false), + ] + ); + // ctrl+shift+A: Shift already requested, never doubled. + assert_eq!( + plan_chord(&[37, 50], 38, true, Some(50)), + vec![ + step(29, true), + step(42, true), + step(30, true), + step(30, false), + step(42, false), + step(29, false), + ] + ); + } + + #[test] + fn report_json_names_the_route() { + let report = KeyboardDeliveryReport { + virtual_focus_held: true, + core_focus_unchanged: true, + delivery_confirmed: false, + key_events: 4, + skipped_characters: vec!['€'], + focus_guard: None, + released_stuck: vec!["button1".into()], + path: MPX_UINPUT_PATH, + grab_popup: None, + }; + let json = report.to_json(); + assert_eq!(json["path"], MPX_UINPUT_PATH); + assert_eq!(json["key_events"], 4); + assert_eq!(json["delivery_confirmed"], false); + assert_eq!(json["skipped_characters"], "€"); + assert_eq!(json["released_stuck_input"][0], "button1"); + assert!(report.delivery_notes().contains("button1")); + } + + #[test] + fn slave_keyboard_name_fits_the_uinput_limit_for_versioned_masters() { + let owner = mpx_owner::Owner::current().expect("procfs identity"); + let name = slave_keyboard_name(&owner.master_name(u64::MAX)); + assert!(name.len() <= EVDEV_UINPUT_NAME_MAX_BYTES, "{name}"); + assert!(name.ends_with(UINPUT_KEYBOARD_SUFFIX)); + } +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/input/mpx_owner.rs b/packages/cua-driver/rust/crates/platform-linux/src/input/mpx_owner.rs new file mode 100644 index 00000000000..562a4d9425f --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-linux/src/input/mpx_owner.rs @@ -0,0 +1,204 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Incarnation-aware ownership encoded atomically in XIAddMaster's name. +//! No separate property-write window can leave an unmarked new master after SIGKILL. +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use sha2::{Digest, Sha256}; +use std::{fs, io}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Owner { + domain: [u8; 16], + pid: u32, + start_ticks: u64, +} + +fn start_ticks(stat: &str) -> io::Result { + crate::proc_fs::process_start_time_from_stat(stat) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid proc stat")) +} + +fn process_start(pid: u32) -> io::Result { + start_ticks(&fs::read_to_string(format!("/proc/{pid}/stat"))?) +} + +impl Owner { + pub(super) fn current() -> io::Result { + let pid = std::process::id(); + let own_stat = fs::read_to_string("/proc/self/stat")?; + if own_stat + .split_whitespace() + .next() + .and_then(|p| p.parse::().ok()) + != Some(pid) + { + return Err(io::Error::other("procfs exposes another PID namespace")); + } + let start_ticks = start_ticks(&own_stat)?; + // A procfs mounted from another PID namespace cannot be used to + // identify peers through namespace-local PIDs. + if process_start(pid)? != start_ticks { + return Err(io::Error::other( + "procfs does not identify this PID namespace", + )); + } + let boot = fs::read_to_string("/proc/sys/kernel/random/boot_id")?; + let namespace = fs::read_link("/proc/self/ns/pid")?; + let mut hash = Sha256::new(); + hash.update(boot.trim().as_bytes()); + hash.update([0]); + hash.update(namespace.as_os_str().as_encoded_bytes()); + hash.update([0]); + hash.update(unsafe { libc::geteuid() }.to_be_bytes()); + let domain = hash.finalize()[..16].try_into().unwrap(); + Ok(Self { + domain, + pid, + start_ticks, + }) + } + + pub(super) fn master_name(&self, nonce: u64) -> String { + let mut process = [0; 12]; + process[..4].copy_from_slice(&self.pid.to_be_bytes()); + process[4..].copy_from_slice(&self.start_ticks.to_be_bytes()); + format!( + "CUA v1.{}.{}.{}", + URL_SAFE_NO_PAD.encode(self.domain), + URL_SAFE_NO_PAD.encode(process), + URL_SAFE_NO_PAD.encode(nonce.to_be_bytes()) + ) + } + + pub(super) fn from_pointer_name(name: &str) -> Option { + let mut parts = name + .strip_prefix("CUA v1.")? + .strip_suffix(" pointer")? + .split('.'); + let domain: [u8; 16] = URL_SAFE_NO_PAD + .decode(parts.next()?) + .ok()? + .try_into() + .ok()?; + let process: [u8; 12] = URL_SAFE_NO_PAD + .decode(parts.next()?) + .ok()? + .try_into() + .ok()?; + let _: [u8; 8] = URL_SAFE_NO_PAD + .decode(parts.next()?) + .ok()? + .try_into() + .ok()?; + if parts.next().is_some() { + return None; + } + let pid = u32::from_be_bytes(process[..4].try_into().ok()?); + if pid == 0 || pid > i32::MAX as u32 { + return None; + } + Some(Self { + domain, + pid, + start_ticks: u64::from_be_bytes(process[4..].try_into().ok()?), + }) + } + + pub(super) fn stale_in(&self, current: &Self) -> bool { + self.stale_with(current, process_start, |pid| { + // hidepid and permission failures can look like absent procfs + // entries. Require the kernel to independently report ESRCH. + (unsafe { libc::kill(pid as i32, 0) }) == -1 + && io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + }) + } + + fn stale_with( + &self, + current: &Self, + read: impl FnOnce(u32) -> io::Result, + absent: impl FnOnce(u32) -> bool, + ) -> bool { + if self.domain != current.domain { + return false; + } + match read(self.pid) { + Ok(start) => start != self.start_ticks, + Err(error) if error.kind() == io::ErrorKind::NotFound => absent(self.pid), + Err(_) => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn owner() -> Owner { + Owner { + domain: [7; 16], + pid: 123, + start_ticks: 456, + } + } + + #[test] + fn ownership_roundtrip_is_bounded_even_at_integer_limits() { + let owner = Owner { + pid: i32::MAX as u32, + start_ticks: u64::MAX, + ..owner() + }; + let name = owner.master_name(u64::MAX); + assert!(format!("{name} uinput pointer").len() <= 78); + assert_eq!( + Owner::from_pointer_name(&format!("{name} pointer")), + Some(owner.clone()) + ); + assert_ne!(owner.master_name(1), owner.master_name(2)); + for suffix in [" keyboard", " XTEST pointer", " pointer extra"] { + assert!(Owner::from_pointer_name(&format!("{name}{suffix}")).is_none()); + } + for name in [ + "CUA legacy mp-123-1 pointer", + "CUA v1.bad.bad.bad pointer", + "ordinary pointer", + ] { + assert!(Owner::from_pointer_name(name).is_none()); + } + } + + #[test] + fn live_peer_and_unverifiable_metadata_are_preserved() { + let owner = owner(); + assert!(!owner.stale_with(&owner, |_| Ok(456), |_| panic!("must not probe"))); + for kind in [ + io::ErrorKind::PermissionDenied, + io::ErrorKind::InvalidData, + io::ErrorKind::Interrupted, + ] { + assert!(!owner.stale_with(&owner, |_| Err(kind.into()), |_| panic!("must not probe"))); + } + assert!(!owner.stale_with(&owner, |_| Err(io::ErrorKind::NotFound.into()), |_| false)); + let foreign = Owner { + domain: [8; 16], + ..owner.clone() + }; + assert!(!foreign.stale_with( + &owner, + |_| panic!("foreign domain"), + |_| panic!("foreign domain") + )); + } + + #[test] + fn dead_owner_and_reused_pid_are_stale() { + let owner = owner(); + assert!(owner.stale_with(&owner, |_| Err(io::ErrorKind::NotFound.into()), |_| true)); + assert!(owner.stale_with(&owner, |_| Ok(999), |_| panic!("must not probe"))); + } + + #[test] + fn actual_current_process_is_not_stale() { + let owner = Owner::current().unwrap(); + assert!(!owner.stale_in(&owner)); + } +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/input/user_input.rs b/packages/cua-driver/rust/crates/platform-linux/src/input/user_input.rs new file mode 100644 index 00000000000..9aadd6b5584 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-linux/src/input/user_input.rs @@ -0,0 +1,140 @@ +//! Observe physical XI2 slave input during foreground transactions. Events always pass through. +use anyhow::{bail, Result}; +use std::cell::RefCell; +use std::collections::HashSet; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::time::Duration; +use x11rb::connection::Connection; +use x11rb::protocol::{ + xinput::{ConnectionExt, Device, DeviceType, EventMask, XIEventMask}, + Event, +}; +use x11rb::rust_connection::RustConnection; + +#[derive(Default)] +struct State { + stopped: AtomicBool, + detected: AtomicBool, +} +thread_local! { static ACTIVE: RefCell>> = const { RefCell::new(None) }; } +pub(super) struct Observer { + state: Arc, + worker: Option>, +} +impl Observer { + pub(super) fn start() -> Result { + let (conn, screen) = RustConnection::connect(None)?; + conn.xinput_xi_query_version(2, 0)?.reply()?; + let devices = conn.xinput_xi_query_device(Device::ALL)?.reply()?; + let physical = devices + .infos + .into_iter() + .filter(|device| { + let name = String::from_utf8_lossy(&device.name).to_ascii_lowercase(); + matches!( + device.type_, + DeviceType::SLAVE_POINTER + | DeviceType::SLAVE_KEYBOARD + | DeviceType::FLOATING_SLAVE + ) && !name.contains("xtest") + && !name.starts_with("cua-") + && !name.starts_with("cua v1.") + && !name.starts_with("qwen-cua") + }) + .map(|device| device.deviceid) + .collect::>(); + conn.xinput_xi_select_events( + conn.setup().roots[screen].root, + &[EventMask { + deviceid: Device::ALL.into(), + mask: vec![ + XIEventMask::RAW_KEY_PRESS + | XIEventMask::RAW_KEY_RELEASE + | XIEventMask::RAW_BUTTON_PRESS + | XIEventMask::RAW_BUTTON_RELEASE + | XIEventMask::RAW_MOTION, + ], + }], + )? + .check()?; + conn.flush()?; + let state = Arc::new(State::default()); + let worker_state = state.clone(); + let worker = std::thread::Builder::new() + .name("cua-user-input".into()) + .spawn(move || { + while !worker_state.stopped.load(Ordering::Acquire) { + match conn.poll_for_event() { + Ok(Some(event)) => { + let source = match event { + Event::XinputRawKeyPress(event) + | Event::XinputRawKeyRelease(event) => Some(event.sourceid), + Event::XinputRawButtonPress(event) + | Event::XinputRawButtonRelease(event) + | Event::XinputRawMotion(event) => Some(event.sourceid), + _ => None, + }; + if source.is_some_and(|source| physical.contains(&source)) { + worker_state.detected.store(true, Ordering::Release); + } + } + Ok(None) => std::thread::sleep(Duration::from_millis(1)), + Err(_) => { + worker_state.detected.store(true, Ordering::Release); + break; + } + } + } + })?; + Ok(Self { + state, + worker: Some(worker), + }) + } + pub(super) fn interrupted(&self) -> bool { + self.state.detected.load(Ordering::Acquire) + } +} +impl Drop for Observer { + fn drop(&mut self) { + self.state.stopped.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} +pub(super) struct Guard { + _observer: Observer, + previous: Option>, +} +impl Guard { + pub(super) fn start() -> Result { + let observer = Observer::start()?; + let previous = ACTIVE.with(|active| active.replace(Some(observer.state.clone()))); + Ok(Self { + _observer: observer, + previous, + }) + } +} +impl Drop for Guard { + fn drop(&mut self) { + ACTIVE.with(|active| { + active.replace(self.previous.take()); + }); + } +} +pub(super) fn checkpoint() -> Result<()> { + if ACTIVE.with(|active| { + active + .borrow() + .as_ref() + .is_some_and(|state| state.detected.load(Ordering::Acquire)) + }) { + bail!("user_input_detected: physical input arrived during foreground delivery; input stopped; observe before retrying"); + } + Ok(()) +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/lib.rs b/packages/cua-driver/rust/crates/platform-linux/src/lib.rs index 75334a31f81..e56540c4b3b 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/lib.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/lib.rs @@ -7,8 +7,7 @@ //! - AT-SPI D-Bus protocol for accessibility tree (via subprocess bridge) //! - xwd/scrot/import for window screenshots //! -//! Wayland: falls back to xdg-output and virtual-keyboard protocols when -//! running under a Wayland compositor with XWayland support. +//! Only X11 is supported, including X11 applications running under XWayland. use cua_driver_core::tool::ToolRegistry; @@ -20,6 +19,9 @@ pub mod tools; #[cfg(target_os = "linux")] pub mod x11; +#[cfg(any(target_os = "linux", test))] +mod x11_client_pid; + #[cfg(target_os = "linux")] pub mod input; @@ -55,12 +57,6 @@ pub mod a11y; #[cfg(target_os = "linux")] pub mod recording_hooks; -#[cfg(target_os = "linux")] -pub mod video_wayland; - -#[cfg(target_os = "linux")] -pub mod wayland; - // `terminal` is OS-independent (pure string matching + a thin x11 hook). // Keeping it un-gated lets the unit tests run on any host. pub mod terminal; @@ -102,31 +98,29 @@ pub fn is_wsl() -> bool { }) } -/// Actionable hint appended to "no usable display" errors. Tailored for WSL, -/// where the common failure is "no real X/Wayland display, or WSLg geometry -/// isn't resolvable". Kept as a trailing fragment so callers can splice it -/// onto a more specific message. See issue #2005. +/// Actionable display diagnostic shared by capture and input failures. pub fn no_display_hint() -> String { - let neither_display_set = - std::env::var_os("DISPLAY").is_none() && std::env::var_os("WAYLAND_DISPLAY").is_none(); - if is_wsl() { - " — no usable display detected (WSL). WSLg provides $DISPLAY/$WAYLAND_DISPLAY \ - under a recent Windows 11 + 'wsl --update'; on older setups install an X server \ - (e.g. VcXsrv) and export DISPLAY. cua-driver needs a real display to capture and \ - report screen size." - .to_owned() - } else if neither_display_set { - " — no usable display detected: neither $DISPLAY (X11) nor $WAYLAND_DISPLAY is set. \ - Start an X server / Wayland compositor (or Xvfb for headless) before running cua-driver." + if std::env::var_os("DISPLAY").is_none() { + " — only X11 is supported. Set DISPLAY and XAUTHORITY for an X server or run the target application under XWayland.".to_owned() + } else if is_wsl() { + " — connect to the WSLg X11 display or an external X server and verify XAUTHORITY." .to_owned() } else { String::new() } } +#[cfg(target_os = "linux")] +pub(crate) fn require_x11() -> anyhow::Result<()> { + if std::env::var_os("DISPLAY").is_none_or(|display| display.is_empty()) { + anyhow::bail!( + "only X11 is supported. Set DISPLAY and XAUTHORITY, including when using XWayland." + ); + } + Ok(()) +} + pub fn register_tools() -> ToolRegistry { - #[cfg(target_os = "linux")] - wayland::ensure_nested_session(); tools::build_registry(false) } @@ -143,10 +137,6 @@ pub fn register_tools_with_cursor_and_provider( cfg: cursor_overlay::CursorConfig, compat: bool, ) -> ToolRegistry { - #[cfg(target_os = "linux")] - wayland::ensure_nested_session(); - #[cfg(target_os = "linux")] - wayland::overlay::set_config_enabled(cfg.enabled); if cfg.enabled { overlay::init(cfg.clone()); overlay::run_on_thread(); @@ -183,3 +173,29 @@ mod display_detection_tests { assert!(!detect_wsl(false, None)); } } + +#[cfg(all(test, target_os = "linux"))] +pub(crate) mod test_env { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + pub(crate) struct DisplayOverride( + Option, + std::sync::MutexGuard<'static, ()>, + ); + pub(crate) fn unreachable_x11_display() -> DisplayOverride { + let lock = LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let prior = std::env::var_os("DISPLAY"); + std::env::set_var("DISPLAY", ":9999999"); + DisplayOverride(prior, lock) + } + impl Drop for DisplayOverride { + fn drop(&mut self) { + match self.0.take() { + Some(prior) => std::env::set_var("DISPLAY", prior), + None => std::env::remove_var("DISPLAY"), + } + } + } +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +mod at_point_policy; diff --git a/packages/cua-driver/rust/crates/platform-linux/src/overlay.rs b/packages/cua-driver/rust/crates/platform-linux/src/overlay.rs index 710aa22e01f..f44b92b9a63 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/overlay.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/overlay.rs @@ -25,8 +25,7 @@ //! as it was at the last paint until the cursor moves or fades. The cutoff //! path had the same limitation over the smaller alpha≥128 silhouette. //! - Z-ordering: `XRaiseWindow` every 80ms to stay above normal windows. -//! - Wayland: when WAYLAND_DISPLAY is set but DISPLAY is also available (XWayland), -//! the X11 path is used. Pure Wayland support is a TODO. +//! - X11 applications under XWayland use the same X11 path. //! //! ## Cross-platform note (2026-05 dedup audit) //! @@ -156,9 +155,7 @@ impl Drop for X11OverlayThreadCleanup { // Disconnect first so an animator racing teardown cannot enqueue after // the waiter sweep. Registrations before release are swept below; // registrations after release observe a disconnected channel and cancel - // themselves. On X11, also make future calls observe the renderer as - // unavailable. A Wayland session may still use its native forwarding - // path even when the optional XWayland owner thread cannot start. + // themselves. Future calls must observe the renderer as unavailable. self.teardown_with_after_disconnect(|| {}); } } @@ -288,9 +285,7 @@ pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) { let _ = try_send_command_for(key, cmd); } -/// Dispatch to every active Linux overlay backend. The result reports only -/// whether the X11 owner accepted the command and can fire `ARRIVAL_TX`; the -/// Wayland layer-shell path does not currently publish arrival notifications. +/// Dispatch to the X11 owner, which acknowledges arrivals via ARRIVAL_TX. fn try_send_command_for(key: CursorKey, cmd: OverlayCommand) -> bool { if key.is_empty() { return false; @@ -307,66 +302,6 @@ fn try_send_command_for(key: CursorKey, cmd: OverlayCommand) -> bool { "overlay: X11 channel rejected command (no sender or queue full)" ); } - // Also forward to the native-Wayland layer-shell overlay when Wayland - // is opted in. The wayland overlay's `forward` is a no-op when its - // owner thread isn't started yet (which is the normal X11-only case). - #[cfg(target_os = "linux")] - { - if crate::wayland::is_wayland() { - if crate::wayland::shell_helper::semantic_cursor_available() { - crate::wayland::shell_helper::set_cursor_color(&cursor_overlay::session_fill_hex( - &key, - )); - // GNOME has no layer-shell. Drive only the final positioning - // commands through the compositor helper; it performs its own - // easing and avoids starting a worker that must fail. - match &cmd { - cursor_overlay::OverlayCommand::ClickPulse { x, y } => { - crate::wayland::shell_helper::click_pulse(*x as i32, *y as i32); - } - cursor_overlay::OverlayCommand::MoveTo { x, y, .. } => { - crate::wayland::shell_helper::move_cursor(*x as i32, *y as i32); - } - cursor_overlay::OverlayCommand::SnapTo { x, y, .. } => { - crate::wayland::shell_helper::move_cursor(*x as i32, *y as i32); - } - cursor_overlay::OverlayCommand::BeginAction { - action, - delivery, - target, - } => { - crate::wayland::shell_helper::set_cursor_state( - action.as_str(), - delivery.as_ref().map_or("", |value| value.as_str()), - target.as_ref().map_or("", |value| value.as_str()), - true, - ); - } - cursor_overlay::OverlayCommand::EndAction(action) => { - crate::wayland::shell_helper::set_cursor_state( - action.as_str(), - "", - "", - false, - ); - } - cursor_overlay::OverlayCommand::SetSessionLabel(label) => { - crate::wayland::shell_helper::set_session_label( - cursor_overlay::sanitize_session_label(label) - .as_deref() - .unwrap_or(""), - ); - } - cursor_overlay::OverlayCommand::SetEnabled(false) => { - crate::wayland::shell_helper::hide_cursor(); - } - _ => {} - } - } else if !crate::wayland::shell_helper::available() { - let _ = crate::wayland::overlay::forward(&msg); - } - } - } x11_queued } @@ -537,15 +472,9 @@ pub fn remove_cursor(key: CursorKey) { if let Some(tx) = CMD_TX.get() { let _ = tx.try_send(msg.clone()); } - #[cfg(target_os = "linux")] - if crate::wayland::is_wayland() && !crate::wayland::shell_helper::available() { - let _ = crate::wayland::overlay::forward(&msg); - } } -/// Clear the X11 render-side tombstone after a successful explicit session -/// revival. Wayland has no keyed tombstone, so forwarding this lifecycle -/// signal there is an accepted no-op. +/// Clear the render-side tombstone after explicit session revival. pub fn revive_cursor(key: CursorKey) { if key.is_empty() { return; @@ -554,10 +483,6 @@ pub fn revive_cursor(key: CursorKey) { if let Some(tx) = CMD_TX.get() { let _ = tx.try_send(msg.clone()); } - #[cfg(target_os = "linux")] - if crate::wayland::is_wayland() && !crate::wayland::shell_helper::available() { - let _ = crate::wayland::overlay::forward(&msg); - } } /// Spawn the overlay on a dedicated thread. Non-blocking. @@ -579,17 +504,6 @@ pub fn run_on_thread() { return; } - // Wayland layer-shell overlay is started LAZILY on the first - // send_command_for() that targets a Wayland session (see the - // wayland::overlay::forward() path). Starting it eagerly here added - // ~100-300ms to cua-driver mcp startup — enough to push the CI - // `cursor-click-gif` test (20s budget for the full launch_app → - // click → type sequence) over its limit, intermittently. The - // forward() path's own ensure_started() OnceLock guarantees the - // thread spins up on demand without losing any commands (the - // first send_command_for that triggers it spawns the thread, - // future commands reuse it). - std::thread::Builder::new() .name("cua-overlay-x11".into()) .spawn(move || { @@ -986,7 +900,6 @@ fn map_x11_overlay_with_empty_input( &[], )? .check()?; - conn.map_window(win)?.check()?; conn.flush()?; Ok(()) } @@ -1004,7 +917,7 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver, win: u32, root: u32, depth: u8, @@ -2534,6 +2449,16 @@ fn paint_x11_tiles( shape_cookie.check()?; } + let should_map = !visible_shape.is_empty(); + if should_map != target.mapped.get() { + if should_map { + conn.map_window(target.win)?.check()?; + } else { + conn.unmap_window(target.win)?.check()?; + } + target.mapped.set(should_map); + } + // Phase D — upload. A composited buffer covers the whole tile rect; pixels // outside the shape are the backdrop copied back and are never displayed. for (bounds, bgra) in payloads { diff --git a/packages/cua-driver/rust/crates/platform-linux/src/pip/mod.rs b/packages/cua-driver/rust/crates/platform-linux/src/pip/mod.rs index 805c0eb4e66..83d9840cbf0 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/pip/mod.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/pip/mod.rs @@ -1,14 +1,3 @@ -//! Linux picture-in-picture preview stub. -//! -//! The Linux implementation is a follow-up to the experimental macOS -//! drop. Plan: GTK4 always-on-top utility window under X11/XWayland, -//! with a Wayland-native fallback via `wlr-layer-shell` where -//! supported. Tracking issue linked in the docs. -//! -//! Until that lands, `start()` returns a clear error so `main.rs` -//! can log "PiP unavailable on this platform" and continue without -//! the window. - use pip_preview::{PipBackend, PipBackendFactory, PipConfig}; pub struct LinuxPipBackendFactory; diff --git a/packages/cua-driver/rust/crates/platform-linux/src/proc_fs.rs b/packages/cua-driver/rust/crates/platform-linux/src/proc_fs.rs index 0935afa9754..ad04ef94de7 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/proc_fs.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/proc_fs.rs @@ -140,3 +140,12 @@ mod tests { assert!(!super::is_process_live(pid), "reaped pid was reported live"); } } + +pub(crate) fn process_start_time_from_stat(stat: &str) -> Option { + stat.rsplit_once(')')? + .1 + .split_whitespace() + .nth(19)? + .parse() + .ok() +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/packages/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs index 30d62d903cb..736af24036e 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs @@ -14,25 +14,8 @@ pub fn app_state_json_for(window_id: Option, pid: Option) -> Option, pid: Option) -> Option> { let pid = u32::try_from(pid?).ok()?; - let window_id = if crate::wayland::is_inject_mode() { - // Most injected actions already carry the protocol-verified window id. - // Process-scoped setup calls such as browser_prepare do not, so resolve - // their single target here instead of classifying required AX evidence - // as a capture failure. - match window_id { - Some(window_id) => window_id, - None => resolve_window_for_recording(pid, None)?.xid, - } - } else { - resolve_window_for_recording(pid, window_id)?.xid - }; - let result = if crate::wayland::is_inject_mode() { - // Evidence capture runs inside the daemon call. Keep it below the - // transport deadline so an unresponsive renderer cannot block input. - crate::atspi::walk_tree_for_recording(pid, window_id, std::time::Duration::from_secs(2)) - } else { - crate::atspi::walk_tree(pid, window_id, None) - }; + let window_id = { resolve_window_for_recording(pid, window_id)?.xid }; + let result = { crate::atspi::walk_tree(pid, window_id, None) }; if result.nodes.is_empty() || result.tree_markdown.trim().is_empty() { return None; } @@ -52,36 +35,35 @@ fn app_state_json_for_blocking(window_id: Option, pid: Option) -> Opti #[cfg(target_os = "linux")] pub fn screenshot_for_recording(window_id: Option, pid: Option) -> Option> { - if crate::wayland::is_inject_mode() { - // A full-output frame is the strongest evidence for the nested - // compositor's background/focus guarantees and needs no slow AT-SPI - // geometry re-resolution. Per-window screenshots elsewhere retain the - // normal crop behavior. - return crate::wayland::screenshot_display_dispatch().ok(); - } if let Some(window_id) = window_id { - crate::wayland::screenshot_dispatch(window_id).ok() + crate::capture::screenshot_window_bytes(window_id).ok() } else if let Some(pid) = pid.and_then(|pid| u32::try_from(pid).ok()) { - let windows = crate::wayland::list_windows_dispatch(Some(pid)); + let windows = crate::x11::list_windows(Some(pid)); windows .first() - .and_then(|window| crate::wayland::screenshot_dispatch(window.xid).ok()) + .and_then(|window| crate::capture::screenshot_window_bytes(window.xid).ok()) } else { crate::capture::screenshot_display_bytes().ok() } } #[cfg(target_os = "linux")] -pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> { +pub fn element_window_local_xy( + window_id: u64, + pid: i64, + element_index: u32, + binding: &cua_driver_core::element_token::SnapshotBinding, +) -> Option<(f64, f64)> { + let binding = binding.clone(); if tokio::runtime::Handle::try_current().is_ok() { return std::thread::spawn(move || { - element_window_local_xy_blocking(window_id, pid, element_index) + element_window_local_xy_blocking(window_id, pid, element_index, &binding) }) .join() .ok() .flatten(); } - element_window_local_xy_blocking(window_id, pid, element_index) + element_window_local_xy_blocking(window_id, pid, element_index, &binding) } #[cfg(target_os = "linux")] @@ -89,10 +71,13 @@ fn element_window_local_xy_blocking( window_id: u64, pid: i64, element_index: u32, + binding: &cua_driver_core::element_token::SnapshotBinding, ) -> Option<(f64, f64)> { let pid = u32::try_from(pid).ok()?; + let cache = crate::atspi::cache::BoundElementCache::from_binding(binding.clone()); + let identity = cache.get_element_identity(pid, window_id, element_index as usize)?; let (screen_x, screen_y, width, height) = - crate::atspi::get_element_bounds(pid, element_index as usize).ok()?; + crate::atspi::get_element_bounds_exact(pid, element_index as usize, identity).ok()?; let window = resolve_window_for_recording(pid, Some(window_id))?; Some(( f64::from(screen_x - window.x) + f64::from(width) / 2.0, @@ -105,13 +90,8 @@ fn resolve_window_for_recording( pid: u32, window_id: Option, ) -> Option { - let windows = crate::wayland::list_windows_dispatch(Some(pid)); - if crate::wayland::is_wayland() { - // Foreign-toplevel protocol object ids are scoped to one Wayland - // connection. Recording hooks open a fresh connection, so re-resolve - // the target by pid instead of comparing an id from the action call. - windows.into_iter().next() - } else if let Some(window_id) = window_id { + let windows = crate::x11::list_windows(Some(pid)); + if let Some(window_id) = window_id { windows.into_iter().find(|window| window.xid == window_id) } else { windows.into_iter().next() @@ -133,6 +113,7 @@ pub fn element_window_local_xy( _window_id: u64, _pid: i64, _element_index: u32, + _binding: &cua_driver_core::element_token::SnapshotBinding, ) -> Option<(f64, f64)> { None } diff --git a/packages/cua-driver/rust/crates/platform-linux/src/terminal.rs b/packages/cua-driver/rust/crates/platform-linux/src/terminal.rs index fe43e5bd0be..0270e88a88b 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/terminal.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/terminal.rs @@ -64,15 +64,9 @@ pub fn wm_class_matches_terminal(instance: &str, class: &str) -> bool { .any(|s| i.contains(s) || c.contains(s)) } -/// Returns `true` if the window with X11 id `xid` belongs to a -/// terminal emulator, based on its `WM_CLASS` property. On native -/// Wayland the dispatcher folds the foreign-toplevel `app_id` into -/// both fields so the substring match still works for terminals like -/// Ghostty / kitty / alacritty. Returns `false` when no connection is -/// available or the property is unset. #[cfg(target_os = "linux")] pub fn is_terminal_window(xid: u64) -> bool { - match crate::wayland::wm_class_dispatch(xid) { + match crate::x11::wm_class_for_window(xid) { Some((instance, class)) => wm_class_matches_terminal(&instance, &class), None => false, } diff --git a/packages/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/packages/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index b03b5e8eb84..fab44efb67c 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -40,7 +40,7 @@ fn pid_window_target_candidates(pid: i64) -> Vec { let Ok(pid) = u32::try_from(pid) else { return Vec::new(); }; - window_target_candidates_for_pid(crate::wayland::list_windows_dispatch(Some(pid)), pid) + window_target_candidates_for_pid(crate::x11::list_windows(Some(pid)), pid) } fn pid_window_guarded( @@ -53,6 +53,110 @@ fn pid_window_guarded( )) } +struct X11Tool(Box); + +#[async_trait] +impl Tool for X11Tool { + fn def(&self) -> &ToolDef { + self.0.def() + } + + async fn protected_resource_ownership( + &self, + adapter_id: &str, + args: &Value, + ) -> cua_driver_core::tool::ProtectedResourceOwnership { + self.0.protected_resource_ownership(adapter_id, args).await + } + + async fn protected_resource_scope( + &self, + adapter_id: &str, + args: &Value, + ) -> Result, String> { + self.0.protected_resource_scope(adapter_id, args).await + } + + async fn validate_protected_resource_scope( + &self, + adapter_id: &str, + args: &Value, + approved_scope: &Value, + ) -> Result<(), String> { + self.0 + .validate_protected_resource_scope(adapter_id, args, approved_scope) + .await + } + + async fn invoke(&self, args: Value) -> ToolResult { + if let Err(error) = crate::require_x11() { + return ToolResult::error(error.to_string()) + .with_structured(json!({"code": "desktop_unavailable"})); + } + let input = matches!( + self.def().name.as_str(), + "click" + | "double_click" + | "right_click" + | "drag" + | "scroll" + | "type_text" + | "press_key" + | "hotkey" + | "set_value" + | "paste" + | "select_text" + | "perform_secondary_action" + | "invoke_menu" + ); + let desktop = args["scope"] == "desktop"; + if input && desktop { + crate::input::note_intentional_foreground(); + } + let guard = if input + && !desktop + && !crate::input::delivery::DeliveryMode::from_args(&args).is_foreground() + { + if let Some(pid) = args["pid"].as_u64().and_then(|pid| u32::try_from(pid).ok()) { + match crate::input::BackgroundFocusGuard::begin(pid, resolve_cursor_key(&args)) { + Ok(guard) => guard, + Err(error) => { + return ToolResult::computer_use_error( + "background_unavailable", + format!("Background focus protection could not start: {error}"), + false, + ) + } + } + } else { + None + } + } else { + None + }; + let result = self.0.invoke(args).await; + if let Some(guard) = guard { + let sent = result + .structured_content + .as_ref() + .is_none_or(|value| value["sent"] != false); + guard.finish(sent); + } + result + } +} + +fn register_native_tool(registry: &mut ToolRegistry, tool: Box) { + if matches!( + tool.def().name.as_str(), + "get_config" | "set_config" | "check_permissions" | "health_report" + ) { + registry.register(tool); + } else { + registry.register(Box::new(X11Tool(tool))); + } +} + // ── DriverConfig + ResizeRegistry + ZoomRegistry ───────────────────────────── #[derive(Clone)] @@ -92,7 +196,7 @@ pub fn load_driver_config() -> DriverConfig { } pub struct ResizeRegistry { - ratios: std::sync::Mutex>, + ratios: std::sync::Mutex>, } impl ResizeRegistry { @@ -101,14 +205,14 @@ impl ResizeRegistry { ratios: std::sync::Mutex::new(Default::default()), } } - pub fn set_ratio(&self, pid: u32, ratio: f64) { - self.ratios.lock().unwrap().insert(pid, ratio); + pub fn set_ratio(&self, pid: u32, window: u64, ratio: f64) { + self.ratios.lock().unwrap().insert((pid, window), ratio); } - pub fn clear_ratio(&self, pid: u32) { - self.ratios.lock().unwrap().remove(&pid); + pub fn clear_ratio(&self, pid: u32, window: u64) { + self.ratios.lock().unwrap().remove(&(pid, window)); } - pub fn ratio(&self, pid: u32) -> Option { - self.ratios.lock().unwrap().get(&pid).copied() + pub fn ratio(&self, pid: u32, window: u64) -> Option { + self.ratios.lock().unwrap().get(&(pid, window)).copied() } } @@ -243,7 +347,7 @@ impl Tool for ListAppsTool { "List Linux apps — both currently running and installed-but-not-running — \ with per-app state flags:\n\n\ - running: does a live process own a top-level window? (pid is 0 when false)\n\ - - active: reserved (Linux X11/Wayland focus model differs from frontmost-app); \ + - active: reserved (Linux X11 focus model differs from frontmost-app); \ always false.\n\ - kind: `\"desktop\"` for XDG `.desktop` launcher entries.\n\ - launch_path: the launcher command from `Exec=` (field codes stripped). \ @@ -486,15 +590,9 @@ fn exec_basename(s: &str) -> String { } fn discovery_windows(filter_pid: Option) -> anyhow::Result> { - let mut windows = if crate::wayland::is_wayland() { - crate::wayland::list_windows_dispatch(filter_pid) - } else { - crate::x11::try_list_windows(filter_pid)? - }; + let mut windows = crate::x11::try_list_windows(filter_pid)?; windows.retain(|window| window.pid.map_or(true, crate::proc_fs::is_process_live)); - if crate::wayland::is_wayland() { - crate::wayland::remember_observed_window_origins(&windows); - } + Ok(windows) } @@ -538,7 +636,7 @@ impl Tool for ListWindowsTool { } let on_screen_only = args.bool_or("on_screen_only", false); let mut windows = match tokio::task::spawn_blocking(move || { - if app_context && !crate::wayland::is_wayland() { + if app_context { crate::x11::try_list_app_windows(filter_pid.expect("validated")) } else { discovery_windows(filter_pid) @@ -554,22 +652,11 @@ impl Tool for ListWindowsTool { windows.retain(|window| window.is_on_screen); } let app_target = if app_context { - if crate::wayland::is_wayland() { - crate::x11::select_app_window( - &windows, - crate::wayland::focused_app_window(filter_pid.expect("validated")), - ) - } else { - crate::x11::resolve_app_window(&windows) - } + crate::x11::resolve_app_window(&windows) } else { None }; - if app_context && crate::wayland::is_wayland() && app_target.is_none() && windows.len() > 1 - { - return ToolResult::error("The Wayland compositor did not expose an unambiguous active app window. This compositor cannot track this app's multiple windows; keep only the intended window open.") - .with_structured(json!({"code":"app_window_unavailable"})); - } + let mut lines = vec![format!("Found {} windows:", windows.len())]; for w in &windows { lines.push(format!( @@ -655,11 +742,11 @@ mod list_windows_tests { } #[test] - fn unavailable_wayland_order_serializes_as_null() { + fn unavailable_z_order_serializes_as_null() { let w = crate::x11::WindowInfo { xid: 43, pid: Some(1234), - app_name: "native-wayland-app".to_owned(), + app_name: "example-app".to_owned(), title: "Example".to_owned(), is_on_screen: true, z_index: None, @@ -727,6 +814,7 @@ impl Tool for GetWindowStateTool { "window_id":{"type":"integer","description":"X11 XID from list_windows."}, "app_context":{"type":"boolean","description":"Use the compact app observation projection."}, "capture_mode": cua_driver_core::capture_mode::capture_mode_schema(), + "include_accessibility":{"type":"boolean","default":true,"description":"Set false for a screenshot without an AT-SPI walk."}, "include_screenshot":{"type":"boolean", "description":"Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return tree only (the cheap path for re-indexing before an element ax action)."}, "screenshot_out_file":{"type":"string", @@ -834,13 +922,7 @@ impl Tool for GetWindowStateTool { .map(|v| v.max(1) as usize); let process_is_live = crate::proc_fs::is_process_live(pid); - let window_matches = if crate::wayland::is_wayland() { - crate::wayland::list_windows_dispatch(Some(pid)) - .iter() - .any(|window| window.xid == xid && window.pid == Some(pid)) - } else { - crate::x11::window_belongs_to_pid(xid, pid) - }; + let window_matches = crate::x11::window_belongs_to_pid(xid, pid); if !process_is_live || !window_matches { return ToolResult::error(format!( "Window target pid {pid}, window_id {xid} is stale or no longer running; refresh list_windows." @@ -884,22 +966,34 @@ impl Tool for GetWindowStateTool { let observation_revisions = state.observation_revisions.clone(); let revision_request_for_capture = observation_revision_request.clone(); let query_for_walk = query.clone(); + let include_accessibility = args["include_accessibility"] != false; + let walk_timeout = + std::time::Duration::from_millis(if !state.element_cache.has_snapshot(pid, xid) { + 2000 + } else { + 1000 + }); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { let capture = || -> anyhow::Result<_> { - let mut tree = crate::atspi::walk_tree_bounded( - pid, - xid, - query_for_walk.as_deref(), - max_elements, - max_depth, - ); - // Compact model observations only. Internal browser setup/consent - // matching needs the original label boundaries from the native tree. - if revision_request_for_capture.is_some() { - tree.nodes = crate::atspi::projection::compact(tree.nodes); - } - let tree_result = Some(tree); + let tree_result = if include_accessibility { + let mut tree = crate::atspi::walk_tree_bounded_with_timeout( + pid, + xid, + query_for_walk.as_deref(), + max_elements, + max_depth, + walk_timeout, + ); + // Compact model observations only. Internal browser setup/consent + // matching needs the original label boundaries from the native tree. + if revision_request_for_capture.is_some() { + tree.nodes = crate::atspi::projection::compact(tree.nodes); + } + Some(tree) + } else { + None + }; // Bounds and element indices come from the same captured AT-SPI // traversal. Joining two live walks by ordinal mis-associated // Chromium controls when its lazy subtree changed between walks. @@ -934,7 +1028,7 @@ impl Tool for GetWindowStateTool { // include_screenshot:false and no disk path was requested. // Tuple: (Option, Option, w, h, Option). let screenshot = if should_capture { - match crate::wayland::screenshot_dispatch(xid) { + match crate::capture::screenshot_window_bytes(xid) { Ok(raw) => { let orig_w = crate::capture::png_dimensions_pub(&raw) .map(|(w, _)| w) @@ -963,11 +1057,7 @@ impl Tool for GetWindowStateTool { }; Ok((tree_result, screenshot, bounds, observation_revision)) }; - if app_context - && should_capture - && !crate::wayland::is_wayland() - && crate::x11::is_minimized(xid) - { + if app_context && should_capture && crate::x11::is_minimized(xid) { crate::input::with_x11_foreground(xid, 1000, || { crate::x11::wait_for_restored_capture(xid)?; capture() @@ -999,25 +1089,18 @@ impl Tool for GetWindowStateTool { content.push(cua_driver_core::protocol::Content::text( header + selected_tree_markdown, )); - if !observation_only { - state.element_cache.update(pid, xid, &tr.nodes); - } + let snapshot_id = (!observation_only).then(|| { + state.element_cache.update_for_session( + pid, + xid, + &tr.nodes, + args.get("_session_id").and_then(Value::as_str), + ) + }); structured["element_count"] = json!(count); structured["elements_complete"] = json!(tr.complete); structured["tree_markdown"] = json!(selected_tree_markdown); - // Surface 6: register a snapshot in the global token - // registry so each actionable element can be addressed - // by an opaque per-snapshot `element_token` alongside - // its existing integer `element_index`. The integer - // surface stays unchanged — the token is additive. - let snapshot_id = (!observation_only).then(|| { - cua_driver_core::element_token::global().register_snapshot_indices( - pid as i32, - xid as u32, - tr.nodes.iter().filter_map(|node| node.element_index), - ) - }); if observation_revision.is_some() && !observation_only { state.watch_target(pid, xid); } @@ -1201,10 +1284,6 @@ impl Tool for GetWindowStateTool { authoritative — verify via the screenshot, and re-snapshot \ after enabling a11y or if the app just launched." ); - // Point the agent at the next rung explicitly: an empty - // tree means element_index has nothing to bind to. The - // recommendation is session-dependent (X11 → px, - // Wayland → foreground) — see non_ax_escalation. structured["escalation"] = non_ax_escalation(); } } @@ -1213,10 +1292,12 @@ impl Tool for GetWindowStateTool { if !observation_only { if let Some(ow) = orig_w { if w > 0 { - state.resize_registry.set_ratio(pid, ow as f64 / w as f64); + state + .resize_registry + .set_ratio(pid, xid, ow as f64 / w as f64); } } else { - state.resize_registry.clear_ratio(pid); + state.resize_registry.clear_ratio(pid, xid); } } // ax mode + screenshot_out_file writes the PNG to disk and @@ -1226,6 +1307,7 @@ impl Tool for GetWindowStateTool { } structured["screenshot_width"] = json!(w); structured["screenshot_height"] = json!(h); + structured["screenshot_frame_valid"] = json!(true); // Surface 7: mirror the MCP image part's `mimeType` onto // the structured payload so consumers don't have to sniff // magic bytes off the base64 to know the format. @@ -1679,7 +1761,7 @@ impl Tool for LaunchAppTool { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); loop { - let windows = crate::wayland::list_windows_dispatch(Some(pid)); + let windows = crate::x11::list_windows(Some(pid)); if !windows.is_empty() || std::time::Instant::now() >= deadline { return windows.iter().map(window_record_json).collect::>(); } @@ -1726,21 +1808,6 @@ fn chromium_family_program(program: &str) -> bool { // ── shared helpers ──────────────────────────────────────────────────────────── -/// Resolve a visible AT-SPI element point in window-local coordinates. -/// -/// Returns `(xid, window_local_x, window_local_y)`. -/// Looks up element bounds via native AT-SPI, finds the owning window -/// (via `xid_hint` or the first window for `pid`), then converts screen-absolute -/// → window-local coords via compositor metadata on Wayland or -/// XTranslateCoordinates on X11. -fn resolve_element_local_coords( - pid: u32, - idx: usize, - xid_hint: Option, -) -> anyhow::Result<(u64, f64, f64)> { - resolve_element_local_coords_exact(pid, idx, xid_hint, None) -} - fn resolve_element_local_coords_exact( pid: u32, idx: usize, @@ -1749,12 +1816,6 @@ fn resolve_element_local_coords_exact( ) -> anyhow::Result<(u64, f64, f64)> { let xid = if let Some(x) = xid_hint { x - } else if crate::wayland::wayland_input_enabled() { - crate::wayland::list_windows_dispatch(Some(pid)) - .into_iter() - .next() - .map(|window| window.xid) - .ok_or_else(|| anyhow::anyhow!("No Wayland windows for pid {pid}"))? } else { crate::x11::list_windows(Some(pid)) .into_iter() @@ -1764,19 +1825,6 @@ fn resolve_element_local_coords_exact( }; let bounds = crate::atspi::native::get_element_bounds_in_window(pid, idx, xid, identity)?; - if crate::wayland::wayland_input_enabled() { - let (window_x, window_y, window_width, window_height) = - crate::wayland::window_geometry(xid) - .ok_or_else(|| anyhow::anyhow!("No Wayland geometry for window {xid}"))?; - let (screen_cx, screen_cy) = - visible_element_point(bounds, (window_x, window_y, window_width, window_height))?; - return Ok(( - xid, - screen_cx - window_x as f64, - screen_cy - window_y as f64, - )); - } - use x11rb::connection::Connection; use x11rb::protocol::xproto::ConnectionExt as _; use x11rb::rust_connection::RustConnection; @@ -1849,24 +1897,18 @@ fn element_screen_center_exact( identity: Option, ) -> anyhow::Result<(f64, f64)> { let (xid, x, y) = resolve_element_local_coords_exact(pid, idx, xid_hint, identity)?; - if crate::wayland::wayland_input_enabled() { - let (ox, oy, _, _) = crate::wayland::window_geometry(xid) - .ok_or_else(|| anyhow::anyhow!("No Wayland geometry for window {xid}"))?; - Ok((x + f64::from(ox), y + f64::from(oy))) - } else { - window_local_to_screen(xid, x, y) - } + window_local_to_screen(xid, x, y) } fn cached_atspi_identity( - state: &ToolState, + element_cache: &crate::atspi::cache::BoundElementCache, pid: u32, xid: u64, idx: usize, - required: bool, + _required: bool, ) -> Result, ToolResult> { - let identity = state.element_cache.get_element_identity(pid, xid, idx); - if required && identity.is_none() { + let identity = element_cache.get_element_identity(pid, xid, idx); + if identity.is_none() { return Err(ToolResult::error( "element_token no longer has a live AT-SPI unique-owner/object-path identity", ) @@ -1925,25 +1967,8 @@ fn parse_mouse_button(name: &str) -> u8 { } } -/// Escalation hint for a non-AX / suspected-no-op surface — the cross-platform -/// `escalation` field mirrored from macOS. The recommended next rung depends on -/// the session type: X11 can pixel-target a specific window in the background, -/// so the deliberate move is an element px action — click by pixel (x,y) off the -/// screenshot already in this response (`px`). Native Wayland CANNOT -/// background-target an unfocused window (libei injects to the compositor's input -/// focus — see `input::delivery`), and the pixel path itself routes through the -/// same coordinate-free AT-SPI actuation that just no-op'd, so the move there is -/// to bring the window to the foreground first (`foreground`). fn non_ax_escalation() -> Value { - if crate::wayland::is_wayland() { - json!({ - "recommended": "foreground", - "reason": "non-AX surface on Wayland: a specific unfocused window can't be \ - pixel-targeted in the background (libei injects to the compositor's \ - focus). bring_to_front, then click by pixel off the screenshot \ - with delivery_mode:\"foreground\"." - }) - } else { + { json!({ "recommended": "px", "reason": "non-AX surface — act by pixel (x,y) off the screenshot \ @@ -1952,13 +1977,6 @@ fn non_ax_escalation() -> Value { } } -/// Structured payload for a `type_text` response. Keeps the legacy -/// `path`/`characters`/`verified` fields for back-compat and adds the cross-tool -/// `effect` tri-state. Linux's AT-SPI `insertText` return value acknowledges -/// the method call but does not read the widget value back, so it and every -/// keystroke / XSendEvent / XTest / Wayland rung are `"unverifiable"` (the -/// caller confirms through a separate observation). An acknowledgement without -/// readback does not establish failed delivery or require another input. fn type_text_structured(path: &str, characters: usize, verified: bool) -> Value { json!({ "path": path, @@ -2027,9 +2045,7 @@ fn type_text_ax_result(pid: u32, text_len: usize, route: &str) -> ToolResult { fn is_chromium_embedder(pid: u32) -> bool { fn argv_is_chromium_helper(p: u32) -> bool { match fs::read(format!("/proc/{p}/cmdline")) { - Ok(raw) => String::from_utf8_lossy(&raw).split('\0').any(|arg| { - arg == "--type=renderer" || arg == "--type=zygote" || arg == "--type=gpu-process" - }), + Ok(raw) => crate::at_point_policy::cmdline_is_chromium_helper(&raw), Err(_) => false, } } @@ -2140,9 +2156,7 @@ fn unavailable_webkit_background( pid: u32, delivery: crate::input::delivery::DeliveryMode, ) -> Option { - (!delivery.is_foreground() - && is_webkitgtk_embedder(pid) - && !crate::wayland::is_inject_mode() + ((!delivery.is_foreground() && is_webkitgtk_embedder(pid)) && !crate::input::real_pointer_input_available()) .then(|| { crate::input::delivery::background_unavailable_error( @@ -2151,38 +2165,46 @@ fn unavailable_webkit_background( }) } +fn background_keyboard_refusal() -> ToolResult { + let mut refusal = crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ); + let available = std::thread::spawn(crate::health_report::probe_a11y_bus) + .join() + .unwrap_or(false); + if !available { + refusal.content.push(cua_driver_core::protocol::Content::text( + "AT-SPI is unavailable. Start the AT-SPI bus (`/usr/libexec/at-spi-bus-launcher`) or enable accessibility in your desktop settings.", + )); + } + refusal +} + fn unavailable_webkit_keyboard_background( pid: u32, delivery: crate::input::delivery::DeliveryMode, ) -> Option { - (!delivery.is_foreground() && is_webkitgtk_embedder(pid) && !crate::wayland::is_inject_mode()) - .then(|| { - crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ) - }) + (!delivery.is_foreground() + && is_webkitgtk_embedder(pid) + && !crate::input::real_keyboard_input_available()) + .then(background_keyboard_refusal) } fn unavailable_gtk_keyboard_background( pid: u32, delivery: crate::input::delivery::DeliveryMode, ) -> Option { - (!delivery.is_foreground() && is_gtk_process(pid) && !crate::wayland::is_inject_mode()).then( - || { - crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ) - }, - ) + (!delivery.is_foreground() + && is_gtk_process(pid) + && !crate::input::real_keyboard_input_available()) + .then(background_keyboard_refusal) } fn unavailable_gtk_pointer_background( pid: u32, delivery: crate::input::delivery::DeliveryMode, ) -> Option { - (!delivery.is_foreground() - && is_gtk_process(pid) - && !crate::wayland::is_inject_mode() + ((!delivery.is_foreground() && is_gtk_process(pid)) && !crate::input::real_pointer_input_available()) .then(|| { crate::input::delivery::background_unavailable_error( @@ -2191,20 +2213,6 @@ fn unavailable_gtk_pointer_background( }) } -fn unavailable_wayland_focused_input_background( - delivery: crate::input::delivery::DeliveryMode, - focus_free_inject_supported: bool, -) -> Option { - (crate::wayland::wayland_input_enabled() - && !(focus_free_inject_supported && crate::wayland::is_inject_mode()) - && !delivery.is_foreground()) - .then(|| { - crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ) - }) -} - /// Chromium's X11 renderer drops synthetic input sent to an occluded, /// unfocused toplevel. Returning success here would be a silent loss, so all /// input tools expose the same typed refusal and leave foreground activation @@ -2213,11 +2221,10 @@ fn unavailable_chromium_background( pid: u32, delivery: crate::input::delivery::DeliveryMode, ) -> Option { - if chromium_background_must_refuse( - delivery.is_foreground(), - crate::wayland::is_inject_mode(), - is_chromium_embedder(pid), - ) { + if !delivery.is_foreground() + && chromium_background_must_refuse(false, is_chromium_embedder(pid)) + && !crate::input::real_pointer_input_available() + { Some(crate::input::delivery::background_unavailable_error( crate::input::delivery::BackgroundUnavailable::ChromiumInput, )) @@ -2226,12 +2233,84 @@ fn unavailable_chromium_background( } } -fn chromium_background_must_refuse( - foreground: bool, - focus_free_inject_mode: bool, - chromium: bool, -) -> bool { - chromium && !foreground && !focus_free_inject_mode +fn chromium_background_must_refuse(foreground: bool, chromium: bool) -> bool { + chromium && !foreground +} + +fn is_qt5_process(pid: u32) -> bool { + fs::read_to_string(format!("/proc/{pid}/maps")) + .is_ok_and(|maps| maps.contains("libQt5Gui") || maps.contains("libQt5Widgets")) +} + +fn mpx_keyboard_refusal( + args: &Value, + delivery: crate::input::delivery::DeliveryMode, +) -> Option { + if delivery.is_foreground() || !crate::input::real_keyboard_input_available() { + return None; + } + crate::input::check_keyboard_background(&resolve_cursor_key(args)) + .err() + .map(linux_input_error) +} + +async fn mpx_text(args: &Value, xid: u64, text: String) -> ToolResult { + if let Some(refusal) = + mpx_keyboard_refusal(args, crate::input::delivery::DeliveryMode::Background) + { + return refusal; + } + let session = resolve_cursor_key(args); + match tokio::task::spawn_blocking(move || { + crate::input::send_virtual_keyboard_text(&session, xid, &text) + }) + .await + { + Ok(Ok(report)) if report.skipped_characters.is_empty() => { + use cua_driver_core::action_record::{ + ActionEffect, ActionExecutionRecord, ActionTransport, ActualDelivery, + RequestedDelivery, + }; + + ToolResult::text("Sent text through the virtual keyboard; observe before retrying.") + .with_structured(json!({"path":"mpx_uinput","verified":false})) + .with_action_record( + ActionExecutionRecord::builder( + ActionEffect::Unverifiable, + ActionTransport::LinuxUinput, + RequestedDelivery::Background, + ) + .actual_delivery(ActualDelivery::Background) + .build() + .expect("background uinput delivery record is valid"), + ) + } + Ok(Ok(_)) => ToolResult::computer_use_error( + "text_delivery_stalled", + "The virtual keyboard could not deliver every character", + true, + ), + Ok(Err(error)) => linux_input_error(error), + Err(error) => ToolResult::computer_use_error("operation_failed", error.to_string(), true), + } +} + +fn background_key( + session: &str, + xid: u64, + x_y: Option<(i32, i32)>, + key: &str, + modifiers: &[&str], +) -> anyhow::Result<()> { + if crate::input::real_keyboard_input_available() { + crate::input::send_virtual_keyboard_key(session, xid, key, modifiers)?; + return Ok(()); + } + if let Some((x, y)) = x_y { + crate::input::send_key_at(xid, x, y, key, modifiers) + } else { + crate::input::send_key(xid, key, modifiers) + } } /// Screen-absolute center of a window (top-left from translate_coordinates plus @@ -2285,7 +2364,7 @@ fn x11_pixel_click_no_focus_steal( ) { Ok(()) => return Ok(()), Err(error) if crate::input::is_uinput_unavailable(&error) => return Err(error), - Err(e) => tracing::warn!("MPX click fell back to XSendEvent: {e}"), + Err(e) => return Err(e), } } } @@ -2293,12 +2372,25 @@ fn x11_pixel_click_no_focus_steal( } fn linux_input_error(error: anyhow::Error) -> ToolResult { - if crate::atspi::native::input_was_dispatched(&error) { + if crate::input::delivery_is_unconfirmed(&error) { + ToolResult::text(format!( + "Sent but not confirmed: {error:#}. Observe the target state before retrying." + )) + .with_structured(json!({ + "code": "text_delivery_unconfirmed", + "sent": true, + "effect": "unverifiable", + })) + } else if crate::atspi::native::input_was_dispatched(&error) { ToolResult::error(format!("{error:#}")).with_structured(json!({ "code": "atspi_input_unconfirmed", "effect": "unverifiable", "input_dispatched": true, })) + } else if let Some(reason) = + error.downcast_ref::() + { + crate::input::delivery::background_unavailable_error(*reason) } else if crate::input::is_uinput_unavailable(&error) { ToolResult::error(error.to_string()).with_structured(json!({ "code": crate::input::UINPUT_UNAVAILABLE_CODE, @@ -2308,6 +2400,17 @@ fn linux_input_error(error: anyhow::Error) -> ToolResult { } } +#[cfg(test)] +#[test] +fn mpx_focus_failure_is_a_pre_dispatch_background_refusal() { + let result = + linux_input_error(crate::input::delivery::BackgroundUnavailable::WindowManagerFocus.into()); + assert_eq!(result.is_error, Some(true)); + let structured = result.structured_content.unwrap(); + assert_eq!(structured["code"], "background_unavailable"); + assert_eq!(structured["sent"], false); +} + #[cfg(test)] #[test] fn uinput_failure_has_a_stable_structured_code() { @@ -2462,14 +2565,7 @@ async fn overlay_glide_to_for(cursor_id: &str, sx: f64, sy: f64) { if !crate::overlay::is_enabled_for(cursor_id) { return; } - // Wayland (Mutter/KDE, no layer-shell): glide the agent cursor via the - // WinRects shell extension. It eases to the target itself, so send the - // destination once here rather than the interpolated stream the X11 render - // loop uses (which is invisible on those compositors anyway). No-op if the - // extension isn't installed. - if crate::wayland::is_wayland() { - crate::wayland::shell_helper::move_cursor(sx as i32, sy as i32); - } + let pos = crate::overlay::current_position_for(cursor_id); if pos.0 < 0.0 && pos.1 < 0.0 { crate::overlay::send_command_for( @@ -2633,22 +2729,6 @@ fn inject_terminal_input(pid: u32, xid: u64, text: &str) -> anyhow::Result // ── click ───────────────────────────────────────────────────────────────────── -fn with_wayland_app_focus( - app_context: bool, - pid: u32, - window_id: u64, - body: impl FnOnce() -> anyhow::Result, -) -> anyhow::Result { - if app_context && crate::wayland::is_inject_mode() { - anyhow::bail!("app_window_unavailable: configured compositor injection addresses a process, not an exact window; App input is unsupported on this route; no input was sent"); - } - if app_context { - crate::wayland::with_target_foreground(pid, window_id, body) - } else { - body() - } -} - pub struct ClickTool { state: Arc, } @@ -2672,11 +2752,8 @@ impl Tool for ClickTool { back to full-window space.\n\n\ button: \"left\" (default), \"right\", or \"middle\". Defaults to left so the \ field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release \ - with the matching button code. Native Wayland: only left-button is supported \ - via the virtual-pointer protocol — right/middle return an error rather than \ - silently degrading to left. `modifier` holds ctrl/shift/alt/super for the \ - click on X11. Native Wayland refuses modified pointer clicks until its input \ - protocol can carry keyboard modifier state.".into(), + with the matching button code. `modifier` holds ctrl/shift/alt/super for the \ + click on X11.".into(), input_schema: json!({ // `pid` is conditionally required (validated in code: needed for // window/element clicks, omitted for windowless scope="desktop"), @@ -2693,10 +2770,7 @@ impl Tool for ClickTool { "element_index": cua_driver_core::tool_schema::element_index_schema(), "element_token": cua_driver_core::tool_schema::element_token_schema(), "snapshot_id": cua_driver_core::tool_schema::snapshot_id_schema(), - // Shape matches the shared button_schema() canon (string + - // [left,right,middle]); kept inline to carry the Linux/Wayland - // back-compat prose the click button-schema test asserts on. - "button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Default: \"left\" (legacy back-compat). X11: routed via ButtonPress/Release with the matching evdev code. Native Wayland: only left-button is supported via the virtual-pointer protocol; right/middle return an error."}, + "button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Default: \"left\" (legacy back-compat). X11: routed via ButtonPress/Release with the matching evdev code."}, "count":{"type":"integer"}, "modifier": cua_driver_core::tool_schema::modifier_schema(), "from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."}, @@ -2751,24 +2825,14 @@ impl Tool for ClickTool { // sees the cursor "click somewhere else." overlay_glide_to_for(&cursor_id, sx as f64, sy as f64).await; let r = tokio::task::spawn_blocking(move || { - if crate::wayland::wayland_input_enabled() { - if !modifiers.is_empty() { - anyhow::bail!( - "modified desktop clicks are unavailable on native Wayland: \ - the virtual-pointer route cannot carry keyboard modifier state" - ); - } - crate::wayland::click_desktop(sx, sy, n as u32, button) - } else { - let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); - crate::input::send_click_xtest_desktop_with_modifiers( - sx, - sy, - button, - n, - &modifier_refs, - ) - } + let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); + crate::input::send_click_xtest_desktop_with_modifiers( + sx, + sy, + button, + n, + &modifier_refs, + ) }) .await; return match r { @@ -2778,7 +2842,7 @@ impl Tool for ClickTool { "✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope)." )) .with_structured( - json!({ "path": if crate::wayland::wayland_input_enabled() { "wayland_desktop" } else { "xtest_desktop" }, "verified": false, "effect": "unverifiable" }), + json!({ "path": "xtest_desktop", "verified": false, "effect": "unverifiable" }), ), Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")), Err(e) => ToolResult::error(format!("task error: {e}")), @@ -2790,7 +2854,6 @@ impl Tool for ClickTool { Err(e) => return e, }; let delivery = crate::input::delivery::DeliveryMode::from_args(&args); - let app_context = args.bool_or("app_context", false); let count = args.u64_or("count", 1) as usize; // Surface 5: reject unknown buttons so a typo can't silently fall through // to a left-click. Empty string keeps back-compat with old clients. @@ -2827,6 +2890,7 @@ impl Tool for ClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let elem_idx_resolved: Option = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -2844,7 +2908,7 @@ impl Tool for ClickTool { let xid_hint = window_id_resolved; let exact_identity = match xid_hint { Some(xid) => match cached_atspi_identity( - &self.state, + &element_cache, pid, xid, idx, @@ -2897,10 +2961,16 @@ impl Tool for ClickTool { // Chromium can execute a genuine AT-SPI action without focus. Try // that route before applying its background synthetic-input gate. - // An entry's AT-SPI "activate" can submit instead of focusing it. - // App clicks use pointer semantics so a subsequent typeText keeps - // the insertion point established by this click. - if !args.bool_or("app_context", false) + let needs_pointer_focus = xid_hint + .and_then(|xid| element_cache.get_element_role(pid, xid, idx)) + .is_some_and(|role| { + matches!( + role.to_ascii_lowercase().as_str(), + "entry" | "text" | "password text" | "combo box" | "editable text" + ) + }); + if !delivery.is_foreground() + && !needs_pointer_focus && button == 1 && count == 1 && modifiers.is_empty() @@ -2937,39 +3007,29 @@ impl Tool for ClickTool { return refusal; } + if !delivery.is_foreground() + && (needs_pointer_focus || !modifiers.is_empty()) + && (is_gtk_process(pid) || is_webkitgtk_embedder(pid)) + && !crate::input::real_pointer_input_available() + { + return crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ); + } // The AX route was unavailable. Fall back to a target-addressed // X11 event for toolkits that accept it. let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let (xid2, lx, ly) = resolve_element_local_coords_exact(pid, idx, xid_hint, exact_identity.clone())?; let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); - if crate::wayland::wayland_input_enabled() && !modifier_refs.is_empty() { - anyhow::bail!( - "modified element clicks are unavailable on native Wayland: \ - the pointer route cannot carry keyboard modifier state" - ); - } + // An explicit X11 foreground request needs the real XTest path // even for a plain click. Some native widgets (notably GTK // selectable rows) expose bounds but no AT-SPI Action and // ignore a targeted XSendEvent; limiting XTest to modified // clicks made those rows addressable but not selectable. - if app_context && crate::wayland::is_inject_mode() { - return with_wayland_app_focus(true, pid, xid2, || { - crate::wayland::inject_click(pid, xid2, lx, ly, count as u32, button) - }); - } - if app_context && crate::wayland::wayland_input_enabled() { - return crate::wayland::with_target_foreground(pid, xid2, || { - let (sx, sy) = crate::wayland::window_local_to_output( - xid2, - lx.round() as i32, - ly.round() as i32, - ); - crate::wayland::click(xid2, sx, sy, count as u32, button) - }); - } - if delivery.is_foreground() && !crate::wayland::wayland_input_enabled() { + + if delivery.is_foreground() { crate::input::with_x11_foreground(xid2, 80, || { let (sx, sy) = element_screen_center_exact(pid, idx, Some(xid2), exact_identity)?; @@ -2981,6 +3041,10 @@ impl Tool for ClickTool { &modifier_refs, ) }) + } else if modifiers.is_empty() { + x11_pixel_click_no_focus_steal( + &cursor_id, xid2, lx as i32, ly as i32, button, count, + ) } else { crate::input::send_click_with_modifiers( xid2, @@ -3038,7 +3102,7 @@ impl Tool for ClickTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, xid) { x *= ratio; y *= ratio; } @@ -3047,21 +3111,7 @@ impl Tool for ClickTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - // Resolve the screen point the cursor glides to. Tool coordinates are - // always window-local screenshot pixels; native Wayland translates - // through compositor/AT-SPI geometry while X11 uses XTranslateCoordinates. - let wayland_output_point = if crate::wayland::wayland_input_enabled() { - Some(crate::wayland::window_local_to_output( - xid, - x.round() as i32, - y.round() as i32, - )) - } else { - None - }; - let glide_target = if let Some((sx, sy)) = wayland_output_point { - Some((sx as f64, sy as f64)) - } else { + let glide_target = { tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)) .await .ok() @@ -3076,64 +3126,21 @@ impl Tool for ClickTool { } let (xi, yi) = (x as i32, y as i32); - let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); let cursor_id_for_task = cursor_id.clone(); let modifiers_for_task = modifiers.clone(); // delivery_mode: background (default) = no-focus-steal injection; // foreground = activate the target window (EWMH) first, then inject, // then restore prior active. Mirrors macOS/Windows. let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> { - if crate::wayland::wayland_input_enabled() { - if !modifiers_for_task.is_empty() { - anyhow::bail!( - "modified coordinate clicks are unavailable on native Wayland: \ - the pointer route cannot carry keyboard modifier state" - ); - } - // Vision/pixel click on native Wayland. Mutter drops synthetic - // virtual-pointer events (the `wayland::click` warp doesn't land), - // so for a plain left single click resolve the screen pixel to the - // covering accessible element and fire its action by - // `element_index` — the coordinate-free path already verified - // working. (x,y) are screen coords here, matching the frames in - // `get_window_state`. Miss → fall through to the injection paths. - if !delivery.is_foreground() && button == 1 && count == 1 { - match crate::atspi::perform_action_at_screen_point(pid, xid, output_x, output_y) - { - Ok(Some(_)) => return Ok("wayland_atspi"), - Err(error) if crate::atspi::native::input_was_dispatched(&error) => { - return Err(error) - } - _ => {} - } - } - if crate::wayland::is_inject_mode() { - with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::inject_click(pid, xid, x, y, count as u32, button) - })?; - return Ok("wayland_cua_compositor"); - } - if !delivery.is_foreground() { - return Ok("background_unavailable"); - } - // Native Wayland: focus+raise the target toplevel - // (foreign-toplevel `activate`), then drive `count` virtual-pointer - // button events. Wayland injection routes to the compositor focus. - with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::click(xid, output_x, output_y, count as u32, button) - })?; - return Ok("wayland_activate"); - } - // X11 injection. Tiered no-focus-steal delivery (background): - // 1. Plain left single-click → AT-SPI doAction at that point. - // 2. Right / middle / double click → real MPX uinput pointer + XI2 - // shield grab (real Xorg only; skipped on Xvfb/Wayland). - // 3. Fallback → synthetic XSendEvent. - // Foreground skips the AT-SPI shortcut and does a real activated pixel - // click (the agent's escalation when background didn't land). let inject = |fg: bool| -> anyhow::Result<&'static str> { if !fg && button == 1 && count == 1 && modifiers_for_task.is_empty() { - match crate::atspi::perform_action_at_point(pid, xi, yi) { + let (sx, sy) = window_local_to_screen(xid, xi as f64, yi as f64)?; + match crate::atspi::perform_action_at_screen_point( + pid, + xid, + sx.round() as i32, + sy.round() as i32, + ) { Ok(Some(_)) => return Ok("x11_atspi"), Err(error) if crate::atspi::native::input_was_dispatched(&error) => { return Err(error) @@ -3141,6 +3148,12 @@ impl Tool for ClickTool { _ => {} } } + if !fg + && !crate::input::real_pointer_input_available() + && (is_gtk_process(pid) || is_webkitgtk_embedder(pid)) + { + return Ok("background_unavailable"); + } if fg { // Foreground: the window is already activated. Deliver a REAL // XTest warp+button click. Synthetic XSendEvent button events @@ -3266,45 +3279,6 @@ async fn focus_by_pixel( Ok(()) } -/// Establish widget-local focus inside a nested-compositor target without -/// changing the compositor's focused toplevel. AX uses Component.GrabFocus; -/// PX sends a private per-surface left click at the requested local point. -async fn focus_nested_inject_target( - pid: u32, - window_id: u64, - element_index: Option, - identity: Option, - pixel: Option<(f64, f64)>, -) -> Result<(), ToolResult> { - if let Some(index) = element_index { - return match tokio::task::spawn_blocking(move || match identity { - Some(identity) => crate::atspi::focus_element_exact(pid, index, identity), - None => crate::atspi::focus_element(pid, index), - }) - .await - { - Ok(Ok(true)) => Ok(()), - Ok(Ok(false)) => Err(ToolResult::error(format!( - "AT-SPI Component.GrabFocus returned false for element {index}" - ))), - Ok(Err(error)) => Err(ToolResult::error(error.to_string())), - Err(error) => Err(ToolResult::error(format!("Task error: {error}"))), - }; - } - if let Some((x, y)) = pixel { - return match tokio::task::spawn_blocking(move || { - crate::wayland::inject_click(pid, window_id, x, y, 1, 1) - }) - .await - { - Ok(Ok(())) => Ok(()), - Ok(Err(error)) => Err(ToolResult::error(error.to_string())), - Err(error) => Err(ToolResult::error(format!("Task error: {error}"))), - }; - } - Ok(()) -} - // ── type_text ───────────────────────────────────────────────────────────────── pub struct TypeTextTool { @@ -3348,22 +3322,17 @@ impl Tool for TypeTextTool { let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&input.text) .into_owned(); - let wayland = crate::wayland::wayland_input_enabled(); - let path = if wayland { "wayland_focused" } else { "xtest" }; - let result = tokio::task::spawn_blocking(move || { - if wayland { - crate::wayland::type_text_focused(&text) - } else { - crate::input::send_type_text_xtest(&text) - } - }) - .await; + + let path = "xtest"; + let result = + tokio::task::spawn_blocking(move || crate::input::send_type_text_xtest(&text)) + .await; return match result { Ok(Ok(())) => ToolResult::text("Typed text into the focused desktop application.") .with_structured( json!({"scope":"desktop","path":path,"effect":"unverifiable"}), ), - Ok(Err(error)) => ToolResult::error(error.to_string()), + Ok(Err(error)) => linux_input_error(error), Err(error) => ToolResult::error(format!("Task error: {error}")), }; } @@ -3394,6 +3363,7 @@ impl Tool for TypeTextTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let (resolved_elem_idx, resolved_window_id) = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, @@ -3424,8 +3394,13 @@ impl Tool for TypeTextTool { }; let exact_identity = match resolved_elem_idx { Some(idx) => { - match cached_atspi_identity(&self.state, pid, xid, idx, revision_identity_required) - { + match cached_atspi_identity( + &element_cache, + pid, + xid, + idx, + revision_identity_required, + ) { Ok(identity) => identity, Err(error) => return error, } @@ -3436,11 +3411,6 @@ impl Tool for TypeTextTool { if let Some(refusal) = unavailable_chromium_background(pid, delivery) { return refusal; } - if resolved_elem_idx.is_none() { - if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { - return refusal; - } - } let px = args.get("x").and_then(|value| value.as_f64()); let py = args.get("y").and_then(|value| value.as_f64()); @@ -3452,6 +3422,11 @@ impl Tool for TypeTextTool { "Pass either element_index (ax) or x,y (px) to type_text, not both.", ); } + if px.is_some() { + if let Some(refusal) = mpx_keyboard_refusal(&args, delivery) { + return refusal; + } + } let cursor_id = resolve_cursor_key(&args); if let Some(idx) = resolved_elem_idx { @@ -3473,69 +3448,6 @@ impl Tool for TypeTextTool { } let text_len = text.chars().count(); - // Native toolkit editables have a stronger focus-free route than raw - // compositor keyboard injection. Keep Chromium/WebKit on real key events - // because their accessibility bridges may echo a write that never reaches - // renderer-owned state. - if crate::wayland::is_inject_mode() - && resolved_elem_idx.is_some() - && !is_chromium_embedder(pid) - && !is_webkitgtk_embedder(pid) - { - let idx = resolved_elem_idx.expect("checked above"); - let text_at = text.clone(); - let identity_for_type = exact_identity.clone(); - let targeted = tokio::task::spawn_blocking(move || { - type_into_atspi_target(pid, idx, identity_for_type, &text_at) - }) - .await; - match targeted { - Ok(Ok(())) => { - return type_text_ax_result(pid, text_len, "via targeted AT-SPI"); - } - Ok(Err(error)) if crate::atspi::native::input_was_dispatched(&error) => { - return linux_input_error(error); - } - Err(error) => { - return ToolResult::error(format!("AT-SPI typing task failed: {error}")) - } - _ => {} - } - } - // The private nested compositor can target the owning Wayland client - // directly. Establish widget-local focus first, without changing the - // compositor's focused toplevel, so keys reach the addressed control. - if crate::wayland::is_inject_mode() { - if let Err(error) = focus_nested_inject_target( - pid, - xid, - resolved_elem_idx, - exact_identity.clone(), - px.zip(py), - ) - .await - { - return error; - } - let text_w = text.clone(); - let result = tokio::task::spawn_blocking(move || { - crate::wayland::inject_type_text(pid, xid, &text_w) - }) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Typed {text_len} character(s) (focus-free via cua-compositor)." - )) - .with_structured(type_text_structured( - "key_events", - text_len, - false, - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), - }; - } - // ── px form: focus by pixel-click, then type into the now-focused element ── // Pass x,y (no element_index/token) for an *element px action*: pixel-click // the field to give the renderer the real keyboard focus the AT-SPI path @@ -3571,46 +3483,6 @@ impl Tool for TypeTextTool { } } - // Renderer EditableText writes can update the accessible value without - // emitting the DOM input event. For an explicit foreground request on - // native Wayland, focus the named field and send real keyboard input so - // Chromium/WebKit observe the same event sequence as a user. - if delivery.is_foreground() - && crate::wayland::wayland_input_enabled() - && (is_chromium_embedder(pid) || is_webkitgtk_embedder(pid)) - { - if let Some(idx) = resolved_elem_idx { - let text_w = text.clone(); - let identity_for_focus = exact_identity.clone(); - let result = tokio::task::spawn_blocking(move || { - crate::wayland::with_target_foreground(pid, xid, || { - if !focus_atspi_target(pid, idx, identity_for_focus)? { - anyhow::bail!( - "AT-SPI Component.GrabFocus returned false for element {idx}" - ); - } - crate::wayland::type_text_focused(&text_w) - }) - }) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Typed {text_len} character(s) (via Wayland virtual-keyboard)." - )) - .with_structured(type_text_structured( - "key_events", - text_len, - false, - )), - Ok(Err(error)) => ToolResult::error(error.to_string()), - Err(error) => ToolResult::error(format!("Task error: {error}")), - }; - } - } - - // AX addressing names one exact editable. Try this focus-free route - // before native Wayland keyboard injection, which can only target the - // compositor's globally focused surface. if let Some(idx) = resolved_elem_idx { let text_at = text.clone(); let identity_for_type = exact_identity.clone(); @@ -3628,57 +3500,11 @@ impl Tool for TypeTextTool { Err(error) => { return ToolResult::error(format!("AT-SPI typing task failed: {error}")) } - Ok(Err(_)) - if !delivery.is_foreground() && crate::wayland::wayland_input_enabled() => - { - return crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ); - } Ok(Err(error)) if !delivery.is_foreground() => return linux_input_error(error), _ => {} } } - // Native Wayland: keys go to the *focused* surface (no pid/window - // targeting in the protocol). Type via the virtual-keyboard tool; pair - // with a prior `click`/`activate` to focus the intended window. - if crate::wayland::wayland_input_enabled() { - if !delivery.is_foreground() { - return crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ); - } - let text_w = text.clone(); - let idx = resolved_elem_idx; - let identity_for_focus = exact_identity.clone(); - let result = tokio::task::spawn_blocking(move || { - crate::wayland::with_target_foreground(pid, xid, || { - if let Some(idx) = idx { - if !focus_atspi_target(pid, idx, identity_for_focus)? { - anyhow::bail!( - "AT-SPI Component.GrabFocus returned false for element {idx}" - ); - } - } - crate::wayland::type_text_focused(&text_w) - }) - }) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Typed {text_len} character(s) (via Wayland virtual-keyboard)." - )) - .with_structured(type_text_structured( - "key_events", - text_len, - false, - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), - }; - } - // Terminal short-circuit: when the target window's WM_CLASS marks it // as a terminal emulator (Ghostty / Alacritty / kitty / …), skip the // AT-SPI path entirely. Terminals expose an editable text area that @@ -3723,7 +3549,7 @@ impl Tool for TypeTextTool { "Typed {text_len} character(s) (terminal emulator: pty/XTest key events)." )) .with_structured(type_text_structured(path, text_len, false)), - Ok(Err(e)) => ToolResult::error(e.to_string()), + Ok(Err(e)) => linux_input_error(e), Err(e) => ToolResult::error(format!("Task error: {e}")), }; } @@ -3732,7 +3558,8 @@ impl Tool for TypeTextTool { // producing the renderer input event, so web embedders use real XTest // key events. Native toolkits keep their verifiable AT-SPI path below. if delivery.is_foreground() - && (resolved_elem_idx.is_some() + && (args.bool_or("app_context", false) + || resolved_elem_idx.is_some() || is_chromium_embedder(pid) || is_webkitgtk_embedder(pid)) { @@ -3761,7 +3588,7 @@ impl Tool for TypeTextTool { text_len, false, )), - Ok(Err(e)) => ToolResult::error(e.to_string()), + Ok(Err(e)) => linux_input_error(e), Err(e) => ToolResult::error(format!("Task error: {e}")), }; } @@ -3780,11 +3607,15 @@ impl Tool for TypeTextTool { .await .ok() .flatten(); + if focus_kind == Some(false) + && !delivery.is_foreground() + && crate::input::real_keyboard_input_available() + { + return mpx_text(&args, xid, text.clone()).await; + } if focus_kind == Some(false) { if !delivery.is_foreground() { - return crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ); + return background_keyboard_refusal(); } let text_f = text.clone(); let result = tokio::task::spawn_blocking(move || { @@ -3809,16 +3640,22 @@ impl Tool for TypeTextTool { text_len, false, )), - Ok(Err(e)) => ToolResult::error(e.to_string()), + Ok(Err(e)) => linux_input_error(e), Err(e) => ToolResult::error(format!("Task error: {e}")), }; } // Try AT-SPI EditableText first (focus-free, works for Qt6/GTK4). let text_clone = text.clone(); - let atspi_result = - tokio::task::spawn_blocking(move || crate::atspi::type_into_editable(pid, &text_clone)) - .await; + let app_bound = args.bool_or("app_context", false); + let atspi_result = tokio::task::spawn_blocking(move || { + if app_bound { + crate::atspi::native::type_into_focused_window(pid, xid, &text_clone) + } else { + crate::atspi::type_into_editable(pid, &text_clone) + } + }) + .await; match atspi_result { Ok(Ok(())) => { @@ -3837,17 +3674,28 @@ impl Tool for TypeTextTool { } } + if !delivery.is_foreground() && crate::input::real_keyboard_input_available() { + return mpx_text(&args, xid, text.clone()).await; + } + // Qt5 workaround: send synthetic FocusIn to make Qt5's AT-SPI bridge // expose the widget tree, type via AT-SPI, then send FocusOut. // This doesn't change the X11 active window, so the test's focus check passes. let text_clone2 = text.clone(); let qt5_result = tokio::task::spawn_blocking(move || { + if !is_qt5_process(pid) { + return Ok::<_, anyhow::Error>((Err(anyhow::anyhow!("not Qt5")), Ok(()))); + } // Send FocusIn to trigger Qt5's bridge crate::input::send_focus_in(xid)?; std::thread::sleep(std::time::Duration::from_millis(100)); // Try AT-SPI again now that widgets should be exposed - let result = crate::atspi::type_into_editable(pid, &text_clone2); + let result = if app_bound { + crate::atspi::native::type_into_focused_window(pid, xid, &text_clone2) + } else { + crate::atspi::type_into_editable(pid, &text_clone2) + }; // Restore state with FocusOut let restore = crate::input::send_focus_out(xid); @@ -3874,6 +3722,12 @@ impl Tool for TypeTextTool { } } + if app_bound { + // The legacy fallbacks search every editable or Tk interpreter. + // App input must stay in its selected window; foreground XTest is the next rung. + return background_keyboard_refusal(); + } + // Track which path the final fallback chain took, so the // structured response stays honest. The closure can't borrow // a local mutably across `spawn_blocking`, so funnel the @@ -3927,11 +3781,7 @@ impl Tool for TypeTextTool { text_len, &format!("via X11, delivery_mode={mode_label}"), ), - Ok(Ok("background_unavailable")) => { - crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ) - } + Ok(Ok("background_unavailable")) => background_keyboard_refusal(), Ok(Ok(path)) => ToolResult::text(format!( "Typed {text_len} character(s) (via X11, delivery_mode={mode_label})." )) @@ -3949,51 +3799,6 @@ pub struct PressKeyTool { } static PRESS_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); -/// Turn a modified `press_key` request into the chord an underlying keyboard -/// path expects when it takes a single key and no separate modifier list. -/// -/// Returns `None` when there is nothing to promote, so the caller keeps the -/// plain single-key route. Dropping the modifiers instead is not a silent -/// no-op: the bare key still reaches the app and is typed as a character. -fn press_key_chord(mods: &[String], key: &str) -> Option> { - if mods.is_empty() { - return None; - } - let mut chord = mods.to_vec(); - chord.push(key.to_owned()); - Some(chord) -} - -#[cfg(test)] -mod press_key_tests { - use super::press_key_chord; - - #[test] - fn unmodified_press_stays_on_the_single_key_route() { - assert_eq!(press_key_chord(&[], "return"), None); - } - - #[test] - fn modifiers_are_promoted_to_a_chord_in_order() { - assert_eq!( - press_key_chord(&["ctrl".to_owned()], "s"), - Some(vec!["ctrl".to_owned(), "s".to_owned()]) - ); - assert_eq!( - press_key_chord(&["ctrl".to_owned(), "shift".to_owned()], "t"), - Some(vec!["ctrl".to_owned(), "shift".to_owned(), "t".to_owned()]) - ); - } - - #[test] - fn the_requested_key_is_last_so_partition_modifiers_can_find_it() { - // `wayland::hotkey` splits the array with `partition_modifiers`, which - // takes the last non-modifier as the key. Appending keeps that true. - let chord = press_key_chord(&["ctrl".to_owned()], "s").expect("chord"); - assert_eq!(chord.last().map(String::as_str), Some("s")); - } -} - #[async_trait] impl Tool for PressKeyTool { fn def(&self) -> &ToolDef { @@ -4030,19 +3835,11 @@ impl Tool for PressKeyTool { let key = input.key; let display = key.clone(); let modifiers = input.modifiers.unwrap_or_default(); - let wayland = crate::wayland::wayland_input_enabled(); - let path = if wayland { "wayland_focused" } else { "xtest" }; + + let path = "xtest"; let result = tokio::task::spawn_blocking(move || { - if wayland && modifiers.is_empty() { - crate::wayland::press_key_focused(&key) - } else if wayland { - let mut keys = modifiers; - keys.push(key); - crate::wayland::hotkey_focused(&keys) - } else { - let refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); - crate::input::send_key_xtest(&key, &refs) - } + let refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); + crate::input::send_key_xtest(&key, &refs) }) .await; return match result { @@ -4050,7 +3847,7 @@ impl Tool for PressKeyTool { .with_structured( json!({"scope":"desktop","path":path,"effect":"unverifiable"}), ), - Ok(Err(error)) => ToolResult::error(error.to_string()), + Ok(Err(error)) => linux_input_error(error), Err(error) => ToolResult::error(format!("Task error: {error}")), }; } @@ -4082,6 +3879,7 @@ impl Tool for PressKeyTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let xid_opt = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => { window_id_arg.or_else(|| window_id.map(|v| v as u64)) @@ -4113,8 +3911,13 @@ impl Tool for PressKeyTool { }; let exact_identity = match resolved_element_idx { Some(idx) => { - match cached_atspi_identity(&self.state, pid, xid, idx, revision_identity_required) - { + match cached_atspi_identity( + &element_cache, + pid, + xid, + idx, + revision_identity_required, + ) { Ok(identity) => identity, Err(error) => return error, } @@ -4127,6 +3930,9 @@ impl Tool for PressKeyTool { // background path (the focus-click already handled fronting when fg). Pass x,y // (no element_index) for Chromium/Electron surfaces the AX path can't focus. let delivery = crate::input::delivery::DeliveryMode::from_args(&args); + if let Some(refusal) = mpx_keyboard_refusal(&args, delivery) { + return refusal; + } if let Some(refusal) = unavailable_chromium_background(pid, delivery) { return refusal; } @@ -4137,9 +3943,6 @@ impl Tool for PressKeyTool { if let Some(refusal) = unavailable_gtk_keyboard_background(pid, delivery) { return refusal; } - if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { - return refusal; - } let px = args.get("x").and_then(|value| value.as_f64()); let py = args.get("y").and_then(|value| value.as_f64()); @@ -4152,44 +3955,6 @@ impl Tool for PressKeyTool { ); } - // Nested cua-compositor addresses the owning Wayland client directly. - // Preserve legacy modifiers by promoting the request to a chord. - if crate::wayland::is_inject_mode() { - if let Err(error) = focus_nested_inject_target( - pid, - xid, - resolved_element_idx, - exact_identity.clone(), - px.zip(py), - ) - .await - { - return error; - } - let result = match press_key_chord(&mods, &key) { - None => { - let key_w = key.clone(); - tokio::task::spawn_blocking(move || { - crate::wayland::inject_press_key(pid, xid, &key_w) - }) - .await - } - Some(chord) => { - tokio::task::spawn_blocking(move || { - crate::wayland::inject_hotkey(pid, xid, &chord) - }) - .await - } - }; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Pressed key '{key}' (focus-free via cua-compositor)." - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), - }; - } - let px_target = { if let (Some(cx), Some(cy)) = (px, py) { let from_zoom = args.bool_or("from_zoom", false); @@ -4213,50 +3978,17 @@ impl Tool for PressKeyTool { } }; - // Native Wayland: send the key to the focused surface via virtual-keyboard. - // `wayland::press_key` carries no modifier list, so a modified request has - // to become a chord here — otherwise the modifiers are dropped and the - // bare key is typed as a character (Ctrl+S inserts a literal "s"). - if crate::wayland::wayland_input_enabled() { - let key_w = key.clone(); - let chord = press_key_chord(&mods, &key); - let idx = resolved_element_idx; - let identity_for_focus = exact_identity.clone(); - let result = tokio::task::spawn_blocking(move || { - crate::wayland::with_target_foreground(pid, xid, || { - if let Some(idx) = idx { - if !focus_atspi_target(pid, idx, identity_for_focus)? { - anyhow::bail!( - "AT-SPI Component.GrabFocus returned false for element {idx}" - ); - } - } - match chord { - Some(keys) => crate::wayland::hotkey_focused(&keys), - None => crate::wayland::press_key_focused(&key_w), - } - }) - }) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Pressed key '{key}' (via Wayland virtual-keyboard)." - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), - }; - } - let key_for_task = key.clone(); // Foreground delivery is one atomic activate-and-XTest transaction. // A preceding PX click establishes internal widget focus, but that // click restores the prior top-level before returning. let deliver_fg = delivery.is_foreground(); let identity_for_focus = exact_identity.clone(); + let session_for_key = resolve_cursor_key(&args); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { if resolved_element_idx.is_none() && mods.is_empty() - && key_for_task.eq_ignore_ascii_case("enter") + && matches!(key_for_task.to_ascii_lowercase().as_str(),"enter"|"return"|"kp_enter"|"kpenter"|"numpadenter") { if inject_terminal_input(pid, xid, "\n")? { return Ok(()); @@ -4287,11 +4019,7 @@ impl Tool for PressKeyTool { ); } } - if let Some((x, y)) = px_target { - crate::input::send_key_at(xid, x, y, &key_for_task, &m) - } else { - crate::input::send_key(xid, &key_for_task, &m) - } + background_key(&session_for_key,xid,px_target,&key_for_task,&m) }) .await; let mode_label = if deliver_fg { @@ -4304,7 +4032,7 @@ impl Tool for PressKeyTool { ToolResult::text(format!("Pressed key '{key}' (delivery_mode={mode_label}).")) .with_structured(json!({ "verified": false, "delivery_mode": mode_label })) } - Ok(Err(e)) => ToolResult::error(e.to_string()), + Ok(Err(e)) => linux_input_error(e), Err(e) => ToolResult::error(format!("Task error: {e}")), } } @@ -4381,15 +4109,11 @@ impl Tool for HotkeyTool { return ToolResult::error("keys must include at least one non-modifier key."); }; let display = keys.join("+"); - let wayland = crate::wayland::wayland_input_enabled(); - let path = if wayland { "wayland_focused" } else { "xtest" }; + + let path = "xtest"; let result = tokio::task::spawn_blocking(move || { - if wayland { - crate::wayland::hotkey_focused(&keys) - } else { - let refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); - crate::input::send_key_xtest(&key, &refs) - } + let refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); + crate::input::send_key_xtest(&key, &refs) }) .await; return match result { @@ -4397,7 +4121,7 @@ impl Tool for HotkeyTool { .with_structured( json!({"scope":"desktop","path":path,"effect":"unverifiable"}), ), - Ok(Err(error)) => ToolResult::error(error.to_string()), + Ok(Err(error)) => linux_input_error(error), Err(error) => ToolResult::error(format!("Task error: {error}")), }; } @@ -4419,6 +4143,7 @@ impl Tool for HotkeyTool { Ok(resolved) => resolved, Err(error) => return error, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let resolved_element_index = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -4452,8 +4177,13 @@ impl Tool for HotkeyTool { }; let exact_identity = match resolved_element_index { Some(idx) => { - match cached_atspi_identity(&self.state, pid, xid, idx, revision_identity_required) - { + match cached_atspi_identity( + &element_cache, + pid, + xid, + idx, + revision_identity_required, + ) { Ok(identity) => identity, Err(error) => return error, } @@ -4483,10 +4213,12 @@ impl Tool for HotkeyTool { }; let key_display = format!("{}+{}", mods.join("+"), key); - let key_for_wayland = key.clone(); - let mods_for_wayland = mods.clone(); + let delivery = crate::input::delivery::DeliveryMode::from_args(&args); + if let Some(refusal) = mpx_keyboard_refusal(&args, delivery) { + return refusal; + } if let Some(refusal) = unavailable_chromium_background(pid, delivery) { return refusal; } @@ -4496,9 +4228,6 @@ impl Tool for HotkeyTool { if let Some(refusal) = unavailable_gtk_keyboard_background(pid, delivery) { return refusal; } - if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { - return refusal; - } let px = args.get("x").and_then(|value| value.as_f64()); let py = args.get("y").and_then(|value| value.as_f64()); @@ -4511,73 +4240,6 @@ impl Tool for HotkeyTool { ); } - if crate::wayland::is_inject_mode() { - if let Err(error) = focus_nested_inject_target( - pid, - xid, - resolved_element_index, - exact_identity.clone(), - px.zip(py), - ) - .await - { - return error; - } - let mut chord = mods.clone(); - chord.push(key.clone()); - let result = tokio::task::spawn_blocking(move || { - crate::wayland::inject_hotkey(pid, xid, &chord) - }) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Pressed hotkey '{key_display}' (focus-free via cua-compositor)." - )), - Ok(Err(error)) => ToolResult::error(error.to_string()), - Err(error) => ToolResult::error(format!("Task error: {error}")), - }; - } - - // Chromium renderer shortcuts need DOM focus established by a real - // pointer event. Component.GrabFocus alone can report success while - // the renderer still drops the virtual-keyboard chord. - if delivery.is_foreground() - && crate::wayland::wayland_input_enabled() - && is_chromium_embedder(pid) - { - if let Some(element_index) = resolved_element_index { - let identity_for_bounds = exact_identity.clone(); - let coordinates = tokio::task::spawn_blocking(move || { - resolve_element_local_coords_exact( - pid, - element_index, - Some(xid), - identity_for_bounds, - ) - }) - .await; - let (_, x, y) = match coordinates { - Ok(Ok(value)) => value, - Ok(Err(error)) => return ToolResult::error(error.to_string()), - Err(error) => return ToolResult::error(format!("Task error: {error}")), - }; - if let Err(error) = focus_by_pixel( - &self.state, - pid, - Some(xid), - x, - y, - true, - args.opt_str("session"), - false, - ) - .await - { - return error; - } - } - } - if let Some(element_index) = resolved_element_index { let identity_for_focus = exact_identity.clone(); let focused = tokio::task::spawn_blocking(move || { @@ -4624,15 +4286,8 @@ impl Tool for HotkeyTool { }; let deliver_fg = delivery.is_foreground(); + let session_for_key = resolve_cursor_key(&args); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::wayland_input_enabled() { - // Native Wayland: route the modifier combo through wtype's - // -M/-k/-m sequence — the closest equivalent to the X11 - // state-mask path. window_id is irrelevant once focused. - let mut combo: Vec = mods_for_wayland.clone(); - combo.push(key_for_wayland.clone()); - return crate::wayland::hotkey(xid, &combo); - } let m: Vec<&str> = mods.iter().map(String::as_str).collect(); // foreground: activate the target first, then inject the accelerator // as REAL key events via XTest. Synthetic XSendEvent keys @@ -4643,11 +4298,7 @@ impl Tool for HotkeyTool { crate::input::send_key_xtest(&key, &m) }); } - if let Some((x, y)) = px_target { - crate::input::send_key_at(xid, x, y, &key, &m) - } else { - crate::input::send_key(xid, &key, &m) - } + background_key(&session_for_key, xid, px_target, &key, &m) }) .await; let mode_label = if deliver_fg { @@ -4660,7 +4311,7 @@ impl Tool for HotkeyTool { "Pressed {key_display} on pid {pid} (delivery_mode={mode_label})." )) .with_structured(json!({ "verified": false, "delivery_mode": mode_label })), - Ok(Err(e)) => ToolResult::error(e.to_string()), + Ok(Err(e)) => linux_input_error(e), Err(e) => ToolResult::error(format!("Task error: {e}")), } } @@ -4669,7 +4320,7 @@ impl Tool for HotkeyTool { // ── set_value ───────────────────────────────────────────────────────────────── pub struct SetValueTool { - state: Arc, + _state: Arc, } static SV_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -4720,6 +4371,7 @@ impl Tool for SetValueTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let (idx, xid) = match resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, @@ -4731,11 +4383,16 @@ impl Tool for SetValueTool { ), _ => return ToolResult::error("set_value element target did not resolve a window"), }; - let exact_identity = - match cached_atspi_identity(&self.state, pid, xid, idx, revision_identity_required) { - Ok(identity) => identity, - Err(error) => return error, - }; + let exact_identity = match cached_atspi_identity( + &element_cache, + pid, + xid, + idx, + revision_identity_required, + ) { + Ok(identity) => identity, + Err(error) => return error, + }; let cursor_id = resolve_cursor_key(&args); let value_for_task = value.clone(); // Pulse the agent cursor onto the target element before writing, so a @@ -4771,7 +4428,7 @@ impl Tool for SetValueTool { } pub struct PerformSecondaryActionTool { - state: Arc, + _state: Arc, } static SECONDARY_ACTION_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -4822,6 +4479,7 @@ impl Tool for PerformSecondaryActionTool { Ok(resolved) => resolved, Err(error) => return error, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let cua_driver_core::element_token::ResolvedElement::Element { window_id: Some(window_id), element_index, @@ -4831,22 +4489,17 @@ impl Tool for PerformSecondaryActionTool { return ToolResult::error("perform_secondary_action requires a current element_token") .with_structured(json!({ "code": "invalid_element_target" })); }; - let identity = match self.state.element_cache.get_element_identity( - pid, - u64::from(window_id), - element_index, - ) { - Some(identity) => identity, - None => { - return ToolResult::error( - "element_token no longer resolves to a native AT-SPI identity", - ) - .with_structured(json!({ "code": "stale_element_token" })) - } - }; - let advertised = self - .state - .element_cache + let identity = + match element_cache.get_element_identity(pid, u64::from(window_id), element_index) { + Some(identity) => identity, + None => { + return ToolResult::error( + "element_token no longer resolves to a native AT-SPI identity", + ) + .with_structured(json!({ "code": "stale_element_token" })) + } + }; + let advertised = element_cache .get_element_actions(pid, u64::from(window_id), element_index) .unwrap_or_default(); if advertised @@ -4897,7 +4550,7 @@ impl Tool for PerformSecondaryActionTool { // ── scroll ──────────────────────────────────────────────────────────────────── pub struct ScrollTool { - state: Arc, + _state: Arc, } static SCROLL_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -4949,12 +4602,10 @@ impl Tool for ScrollTool { let y = input.y.round() as i32; let amount = input.amount.unwrap_or(3).clamp(1, 50) as usize; let display = direction.clone(); - let wayland = crate::wayland::wayland_input_enabled(); - let path = if wayland { "wayland_desktop" } else { "xtest" }; + + let path = "xtest"; let result = tokio::task::spawn_blocking(move || { - if wayland { - crate::wayland::scroll_desktop(x, y, &direction, amount as u32) - } else { + { crate::input::send_scroll_xtest_desktop(x, y, &direction, amount) } }) @@ -4997,6 +4648,7 @@ impl Tool for ScrollTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let xid_opt: Option = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id .map(|v| v as u64) @@ -5030,8 +4682,13 @@ impl Tool for ScrollTool { }; let exact_identity = match resolved_element_index { Some(idx) => { - match cached_atspi_identity(&self.state, pid, xid, idx, revision_identity_required) - { + match cached_atspi_identity( + &element_cache, + pid, + xid, + idx, + revision_identity_required, + ) { Ok(identity) => identity, Err(error) => return error, } @@ -5040,7 +4697,6 @@ impl Tool for ScrollTool { }; let delivery = crate::input::delivery::DeliveryMode::from_args(&args); - let app_context = args.bool_or("app_context", false); if let Some(refusal) = unavailable_chromium_background(pid, delivery) { return refusal; } @@ -5048,11 +4704,7 @@ impl Tool for ScrollTool { &resolved { let idx = *element_index; - // WebKitGTK acknowledges the AT-SPI scroll action without moving - // the DOM scroller. On native Wayland, use a real compositor wheel - // event at the resolved element instead of reporting a silent - // success. Chromium's AT-SPI scroll path remains effective. - if !(crate::wayland::wayland_input_enabled() && is_webkitgtk_embedder(pid)) { + { let direction_for_ax = direction.clone(); let identity_for_scroll = exact_identity.clone(); let ax_result = tokio::task::spawn_blocking(move || match identity_for_scroll { @@ -5112,99 +4764,6 @@ impl Tool for ScrollTool { ); } - if crate::wayland::is_inject_mode() { - let Some((x, y)) = pixel_target else { - return crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, - ); - }; - let direction_for_inject = direction.clone(); - let result = tokio::task::spawn_blocking(move || { - with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::inject_scroll( - pid, - xid, - x, - y, - &direction_for_inject, - amount as u32, - ) - }) - }) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Scrolled {direction} {amount} ticks (focus-free via cua-compositor)." - )) - .with_structured(json!({ - "verified": false, - "delivery_mode": if delivery.is_foreground() { "foreground" } else { "background" }, - "route": "cua_compositor_inject" - })), - Ok(Err(error)) => ToolResult::error(error.to_string()), - Err(error) => ToolResult::error(format!("Task error: {error}")), - }; - } - - if crate::wayland::wayland_input_enabled() { - if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, false) { - return refusal; - } - let direction_for_wayland = direction.clone(); - let local_point = match (pixel_target, &resolved) { - (Some(point), _) => Some(point), - ( - None, - cua_driver_core::element_token::ResolvedElement::Element { - element_index, .. - }, - ) => { - let idx = *element_index; - let identity_for_bounds = exact_identity.clone(); - match tokio::task::spawn_blocking(move || { - resolve_element_local_coords_exact(pid, idx, Some(xid), identity_for_bounds) - }) - .await - { - Ok(Ok((_resolved_xid, x, y))) => Some((x, y)), - Ok(Err(error)) => { - return ToolResult::error(format!( - "Could not resolve scroll element [{idx}] coordinates: {error}" - )) - } - Err(error) => { - return ToolResult::error(format!( - "Scroll element coordinate task failed: {error}" - )) - } - } - } - (None, cua_driver_core::element_token::ResolvedElement::None) => None, - }; - let output_point = local_point.map(|(x, y)| { - crate::wayland::window_local_to_output(xid, x.round() as i32, y.round() as i32) - }); - let result = tokio::task::spawn_blocking(move || { - with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::scroll_at( - xid, - output_point, - &direction_for_wayland, - amount as u32, - ) - }) - }) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Scrolled {direction} {amount} ticks (delivery_mode=foreground)." - )) - .with_structured(json!({ "verified": false, "delivery_mode": "foreground" })), - Ok(Err(error)) => ToolResult::error(error.to_string()), - Err(error) => ToolResult::error(format!("Task error: {error}")), - }; - } - if let Some(refusal) = unavailable_webkit_background(pid, delivery) { return refusal; } @@ -5256,8 +4815,7 @@ impl Tool for ScrollTool { _ => 5, }; let cursor_id_for_task = cursor_id.clone(); - let direction_for_wayland = direction.clone(); - let amount_u32 = amount as u32; + let element_index = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -5265,9 +4823,6 @@ impl Tool for ScrollTool { _ => None, }; let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::wayland_input_enabled() { - return crate::wayland::scroll(xid, &direction_for_wayland, amount_u32); - } // foreground: activate the window, then scroll, then restore — for // surfaces that only route wheel events to the active window. let x11_scroll = || -> anyhow::Result<()> { @@ -5303,7 +4858,7 @@ impl Tool for ScrollTool { Err(error) if crate::input::is_uinput_unavailable(&error) => { return Err(error) } - Err(e) => tracing::warn!("MPX scroll fell back to XSendEvent: {e}"), + Err(error) => return Err(error), } } } @@ -5398,7 +4953,6 @@ impl Tool for DoubleClickTool { Err(e) => return e, }; let delivery = crate::input::delivery::DeliveryMode::from_args(&args); - let app_context = args.bool_or("app_context", false); if let Some(refusal) = unavailable_chromium_background(pid, delivery) { return refusal; } @@ -5408,9 +4962,6 @@ impl Tool for DoubleClickTool { if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) { return refusal; } - if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { - return refusal; - } // Surface 6: element_token / element_index precedence. let element_token_arg = args.opt_str("element_token"); let revision_identity_required = element_token_arg @@ -5427,6 +4978,7 @@ impl Tool for DoubleClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let elem_idx_resolved = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -5443,7 +4995,7 @@ impl Tool for DoubleClickTool { let xid_hint = window_id_resolved; let exact_identity = match xid_hint { Some(xid) => match cached_atspi_identity( - &self.state, + &element_cache, pid, xid, idx, @@ -5479,21 +5031,9 @@ impl Tool for DoubleClickTool { } let lxi = lx as i32; let lyi = ly as i32; - let wayland_point = crate::wayland::wayland_input_enabled() - .then(|| crate::wayland::window_local_to_output(xid, lxi, lyi)); + let cursor_id_for_task = cursor_id.clone(); let click_result = tokio::task::spawn_blocking(move || { - if crate::wayland::is_inject_mode() { - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::inject_click(pid, xid, lx, ly, 2, 1) - }); - } - if crate::wayland::wayland_input_enabled() { - let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi)); - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::click(xid, output_x, output_y, 2, 1) - }); - } if delivery.is_foreground() { return crate::input::with_x11_foreground(xid, 80, || { let (sx, sy) = element_screen_center_exact( @@ -5545,7 +5085,7 @@ impl Tool for DoubleClickTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, xid) { x *= ratio; y *= ratio; } @@ -5553,18 +5093,7 @@ impl Tool for DoubleClickTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - let wayland_output_point = if crate::wayland::wayland_input_enabled() { - Some(crate::wayland::window_local_to_output( - xid, - x.round() as i32, - y.round() as i32, - )) - } else { - None - }; - let glide_target = if let Some((sx, sy)) = wayland_output_point { - Some((sx as f64, sy as f64)) - } else { + let glide_target = { tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)) .await .ok() @@ -5580,17 +5109,6 @@ impl Tool for DoubleClickTool { let (xi, yi) = (x as i32, y as i32); let cursor_id_for_task = cursor_id.clone(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::is_inject_mode() { - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::inject_click(pid, xid, x, y, 2, 1) - }); - } - if crate::wayland::wayland_input_enabled() { - let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::click(xid, output_x, output_y, 2, 1) - }); - } if delivery.is_foreground() { return crate::input::with_x11_foreground(xid, 80, || { // Real XTest double-click at the screen point — synthetic @@ -5668,7 +5186,6 @@ impl Tool for RightClickTool { Err(e) => return e, }; let delivery = crate::input::delivery::DeliveryMode::from_args(&args); - let app_context = args.bool_or("app_context", false); if let Some(refusal) = unavailable_chromium_background(pid, delivery) { return refusal; } @@ -5678,9 +5195,6 @@ impl Tool for RightClickTool { if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) { return refusal; } - if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { - return refusal; - } // Surface 6: element_token / element_index precedence. let element_token_arg = args.opt_str("element_token"); let revision_identity_required = element_token_arg @@ -5697,6 +5211,7 @@ impl Tool for RightClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::atspi::cache::BoundElementCache::from_resolved(&resolved); let elem_idx_resolved = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -5713,7 +5228,7 @@ impl Tool for RightClickTool { let xid_hint = window_id_resolved; let exact_identity = match xid_hint { Some(xid) => match cached_atspi_identity( - &self.state, + &element_cache, pid, xid, idx, @@ -5749,21 +5264,9 @@ impl Tool for RightClickTool { } let lxi = lx as i32; let lyi = ly as i32; - let wayland_point = crate::wayland::wayland_input_enabled() - .then(|| crate::wayland::window_local_to_output(xid, lxi, lyi)); + let cursor_id_for_task = cursor_id.clone(); let click_result = tokio::task::spawn_blocking(move || { - if crate::wayland::is_inject_mode() { - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::inject_click(pid, xid, lx, ly, 1, 3) - }); - } - if crate::wayland::wayland_input_enabled() { - let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi)); - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::click(xid, output_x, output_y, 1, 3) - }); - } if delivery.is_foreground() { return crate::input::with_x11_foreground(xid, 80, || { let (sx, sy) = element_screen_center_exact( @@ -5815,7 +5318,7 @@ impl Tool for RightClickTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, xid) { x *= ratio; y *= ratio; } @@ -5823,18 +5326,7 @@ impl Tool for RightClickTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - let wayland_output_point = if crate::wayland::wayland_input_enabled() { - Some(crate::wayland::window_local_to_output( - xid, - x.round() as i32, - y.round() as i32, - )) - } else { - None - }; - let glide_target = if let Some((sx, sy)) = wayland_output_point { - Some((sx as f64, sy as f64)) - } else { + let glide_target = { tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)) .await .ok() @@ -5850,17 +5342,6 @@ impl Tool for RightClickTool { let (xi, yi) = (x as i32, y as i32); let cursor_id_for_task = cursor_id.clone(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::is_inject_mode() { - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::inject_click(pid, xid, x, y, 1, 3) - }); - } - if crate::wayland::wayland_input_enabled() { - let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); - return with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::click(xid, output_x, output_y, 1, 3) - }); - } if delivery.is_foreground() { return crate::input::with_x11_foreground(xid, 80, || { // Real XTest right-click at the screen point (synthetic @@ -5935,6 +5416,7 @@ impl Tool for DragTool { } async fn invoke(&self, args: Value) -> ToolResult { let cursor_id = resolve_cursor_key(&args); + let drag_modifiers: Vec = args.str_array("modifier"); if args.opt_str("scope").as_deref() == Some("desktop") && args.get("pid").is_none() && args.get("window_id").is_none() @@ -5947,21 +5429,11 @@ impl Tool for DragTool { let button = parse_mouse_button(input.button.unwrap_or(ClickButton::Left).as_str()); let duration_ms = input.duration_ms.unwrap_or(500).min(10_000); let steps = input.steps.unwrap_or(20).clamp(1, 200) as usize; - let wayland = crate::wayland::wayland_input_enabled(); - let path = if wayland { "wayland_desktop" } else { "xtest" }; + + let path = "xtest"; let result = tokio::task::spawn_blocking(move || { - if wayland { - crate::wayland::drag_desktop( - from_x.round() as i32, - from_y.round() as i32, - to_x.round() as i32, - to_y.round() as i32, - steps as u32, - duration_ms, - button, - ) - } else { - crate::input::send_drag_xtest_desktop( + { + crate::input::send_drag_xtest_desktop_with_modifiers( from_x.round() as i32, from_y.round() as i32, to_x.round() as i32, @@ -5969,6 +5441,10 @@ impl Tool for DragTool { button, duration_ms, steps, + &drag_modifiers + .iter() + .map(String::as_str) + .collect::>(), ) } }); @@ -6002,7 +5478,6 @@ impl Tool for DragTool { None => return ToolResult::error("window_id is required on Linux."), }; let delivery = crate::input::delivery::DeliveryMode::from_args(&args); - let app_context = args.bool_or("app_context", false); if let Some(refusal) = unavailable_chromium_background(pid, delivery) { return refusal; } @@ -6012,9 +5487,6 @@ impl Tool for DragTool { if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) { return refusal; } - if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { - return refusal; - } let coerce = |key: &str| -> Option { args.opt_f64(key) @@ -6059,7 +5531,7 @@ impl Tool for DragTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, xid) { from_x *= ratio; from_y *= ratio; to_x *= ratio; @@ -6070,23 +5542,7 @@ impl Tool for DragTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - let wayland_points = crate::wayland::wayland_input_enabled().then(|| { - ( - crate::wayland::window_local_to_output( - xid, - from_x.round() as i32, - from_y.round() as i32, - ), - crate::wayland::window_local_to_output( - xid, - to_x.round() as i32, - to_y.round() as i32, - ), - ) - }); - let screen_from = if let Some((from, _)) = wayland_points { - Some((from.0 as f64, from.1 as f64)) - } else { + let screen_from = { tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)) .await .ok() @@ -6106,75 +5562,40 @@ impl Tool for DragTool { }, ); } - // Native Wayland: emit press + interpolated motion + release as one - // virtual-pointer (wlroots) or libei (GNOME/KDE) sequence, output-relative - // coords. Returns early so we don't fall into the X11 XSendEvent loop below. - if crate::wayland::wayland_input_enabled() { - let steps_u32 = steps as u32; - let drag_result = if crate::wayland::is_inject_mode() { - tokio::task::spawn_blocking(move || { - with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::inject_drag( - pid, - xid, - (from_x, from_y), - (to_x, to_y), - steps, - button as u32, - ) - }) - }) - } else { - let ((fxi, fyi), (txi, tyi)) = wayland_points.unwrap_or(( - (from_x.round() as i32, from_y.round() as i32), - (to_x.round() as i32, to_y.round() as i32), - )); - tokio::task::spawn_blocking(move || { - with_wayland_app_focus(app_context, pid, xid, || { - crate::wayland::drag( - xid, - fxi, - fyi, - txi, - tyi, - steps_u32, - duration_ms, - button, - ) - }) - }) - }; - let ((from_output_x, from_output_y), (to_output_x, to_output_y)) = wayland_points - .unwrap_or(( - (from_x.round() as i32, from_y.round() as i32), - (to_x.round() as i32, to_y.round() as i32), - )); - let visual_drag = track_overlay_drag_for( - cursor_id.clone(), - (f64::from(from_output_x), f64::from(from_output_y)), - (f64::from(to_output_x), f64::from(to_output_y)), - duration_ms, - steps, + + if !delivery.is_foreground() && !drag_modifiers.is_empty() { + return crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, ); - let (drag_result, ()) = tokio::join!(drag_result, visual_drag); - if matches!(&drag_result, Ok(Ok(()))) { - self.state.cursor_registry.update_position( - &cursor_id, - f64::from(to_output_x), - f64::from(to_output_y), - ); - } - return match drag_result { - Ok(Ok(())) => ToolResult::text(format!( - "✅ Posted drag ({button_str}) to pid {pid} \ - from ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0}) \ - in {duration_ms}ms / {steps} steps." - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), + } + if !delivery.is_foreground() && crate::input::real_pointer_input_available() { + let session = cursor_id.clone(); + let result = tokio::task::spawn_blocking(move || { + let from = window_local_to_screen(xid, from_x, from_y)?; + let to = window_local_to_screen(xid, to_x, to_y)?; + crate::input::send_virtual_pointer_drag( + &session, + &crate::input::VirtualPointerDrag { + target_window: xid, + button, + path: vec![ + (from.0.round() as i32, from.1.round() as i32), + (to.0.round() as i32, to.1.round() as i32), + ], + duration_ms, + steps, + }, + ) + }) + .await; + return match result { + Ok(Ok(_)) => { + ToolResult::text("Sent virtual-pointer drag; observe before retrying.") + } + Ok(Err(error)) => linux_input_error(error), + Err(error) => ToolResult::error(error.to_string()), }; } - if delivery.is_foreground() { let screen_points = tokio::task::spawn_blocking(move || { Ok::<_, anyhow::Error>(( @@ -6190,7 +5611,7 @@ impl Tool for DragTool { }; let drag_result = tokio::task::spawn_blocking(move || { crate::input::with_x11_foreground(xid, 80, || { - crate::input::send_drag_xtest_desktop( + crate::input::send_drag_xtest_desktop_with_modifiers( screen_from_x.round() as i32, screen_from_y.round() as i32, screen_to_x.round() as i32, @@ -6198,6 +5619,10 @@ impl Tool for DragTool { button, duration_ms, steps, + &drag_modifiers + .iter() + .map(String::as_str) + .collect::>(), ) }) }); @@ -6412,7 +5837,7 @@ impl Tool for MouseButtonDownTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, xid) { x *= ratio; y *= ratio; } @@ -6433,16 +5858,7 @@ impl Tool for MouseButtonDownTool { let xi = x as i32; let yi = y as i32; - // Native Wayland: route through the persistent virtual-pointer module - // so the held button survives across tool calls; the X11 path keeps - // the existing input::send_button_down behaviour. - let result = if crate::wayland::is_wayland() { - let cid = cursor_id.clone(); - tokio::task::spawn_blocking(move || { - crate::wayland::persistent_vptr::press(&cid, xid, xi, yi, button) - }) - .await - } else { + let result = { tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)) .await }; @@ -6558,7 +5974,7 @@ impl Tool for MouseDragTool { .with_structured(mouse_hold_json(&cursor_id, Some(&hold))) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid, hold.xid) { to_x *= ratio; to_y *= ratio; } @@ -6596,25 +6012,13 @@ impl Tool for MouseDragTool { let mut result: anyhow::Result<()> = Ok(()); let mut prev_x = from_x; let mut prev_y = from_y; - let is_wl = crate::wayland::is_wayland(); + for i in 1..=steps { let t = i as f64 / steps as f64; let ix = from_x + (to_x - from_x) * t; let iy = from_y + (to_y - from_y) * t; - // Native Wayland: route motion through the persistent virtual- - // pointer so the held button stays held across the entire drag; - // X11 keeps the existing input::send_motion path. - let cid_inner = cursor_id.clone(); - let move_result = if is_wl { - tokio::task::spawn_blocking(move || { - crate::wayland::persistent_vptr::move_to( - &cid_inner, - ix.round() as i32, - iy.round() as i32, - ) - }) - .await - } else { + + let move_result = { tokio::task::spawn_blocking(move || { crate::input::send_motion( xid, @@ -6749,26 +6153,32 @@ impl Tool for MouseButtonUpTool { let mut x = args.opt_f64("x").unwrap_or(hold.x); let mut y = args.opt_f64("y").unwrap_or(hold.y); - if args.bool_or("from_zoom", false) { - match self.state.zoom_registry.get(hold.pid) { - Some(ctx) => { - let (wx, wy) = ctx.zoom_to_window(x, y); - x = wx; - y = wy; + if args.get("x").is_some() != args.get("y").is_some() { + return ToolResult::error( + "Provide both x and y, or omit both to release at the held position.", + ); + } + if args.get("x").is_some() { + if args.bool_or("from_zoom", false) { + match self.state.zoom_registry.get(hold.pid) { + Some(ctx) => { + let (wx, wy) = ctx.zoom_to_window(x, y); + x = wx; + y = wy; + } + None => { + return ToolResult::error(format!( + "from_zoom=true but no zoom context for pid {}. Call zoom first.", + hold.pid + )) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))) + } } - None => { - return ToolResult::error(format!( - "from_zoom=true but no zoom context for pid {}. Call zoom first.", - hold.pid - )) - .with_structured(mouse_hold_json(&cursor_id, Some(&hold))) - } + } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid, hold.xid) { + x *= ratio; + y *= ratio; } - } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) { - x *= ratio; - y *= ratio; } - crate::overlay::send_command_for( cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), @@ -6782,16 +6192,7 @@ impl Tool for MouseButtonUpTool { let button = hold.button; let xi = x as i32; let yi = y as i32; - // Native Wayland: release through the persistent virtual-pointer so - // the same vptr device that emitted the press also emits the release - // (single logical drag rather than a click pair). - let result = if crate::wayland::is_wayland() { - let cid = cursor_id.clone(); - tokio::task::spawn_blocking(move || { - crate::wayland::persistent_vptr::release(&cid, button) - }) - .await - } else { + let result = { tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button)) .await }; @@ -6830,81 +6231,6 @@ pub struct ParallelMouseDragTool { } static PMDRAG_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); -/// cua-compositor path for parallel_mouse_drag: build window-local drag paths -/// and run them as concurrent multi-cursor injections over the control socket. -/// Coordinates stay window-local (the compositor maps them per app_id), so no -/// X11 geometry/MPX is needed — the X11 path's hard blocker on Wayland. -async fn parallel_drag_inject(args: &Value) -> ToolResult { - let Some(items) = args.get("drags").and_then(|v| v.as_array()) else { - return ToolResult::error("drags[] is required."); - }; - if items.len() < 2 { - return ToolResult::error("parallel_mouse_drag requires at least two drag items."); - } - let mut drags = Vec::with_capacity(items.len()); - for (i, item) in items.iter().enumerate() { - let Some(xid) = item.get("window_id").and_then(|v| v.as_u64()) else { - return ToolResult::error("each drag item requires window_id."); - }; - let local: Vec<(f64, f64)> = if let Some(pts) = item.get("path").and_then(|v| v.as_array()) - { - pts.iter() - .filter_map(|p| { - let a = p.as_array()?; - Some((a.first()?.as_f64()?, a.get(1)?.as_f64()?)) - }) - .collect() - } else { - let g = |k: &str| item.get(k).and_then(|v| v.as_f64()); - match (g("from_x"), g("from_y"), g("to_x"), g("to_y")) { - (Some(fx), Some(fy), Some(tx), Some(ty)) => vec![(fx, fy), (tx, ty)], - _ => { - return ToolResult::error( - "each drag item requires path[] or from_x/from_y/to_x/to_y.", - ) - } - } - }; - if local.len() < 2 { - return ToolResult::error("drag path needs at least 2 points."); - } - let steps = item - .get("steps") - .and_then(|v| v.as_u64()) - .unwrap_or(60) - .clamp(1, 300) as usize; - let x_button = parse_mouse_button( - item.get("button") - .and_then(|v| v.as_str()) - .unwrap_or("left"), - ) as u32; - let app = match tokio::task::spawn_blocking(move || { - crate::wayland::inject_target_for_window(xid) - }) - .await - { - Ok(Ok(target)) => target, - Ok(Err(error)) => return ToolResult::error(error.to_string()), - Err(error) => return ToolResult::error(format!("Task error: {error}")), - }; - drags.push(crate::wayland::InjectDrag { - app_id: app, - idx: i, - x_button, - path: local, - steps, - }); - } - let n = drags.len(); - match tokio::task::spawn_blocking(move || crate::wayland::inject_parallel_drags(&drags)).await { - Ok(Ok(())) => ToolResult::text(format!( - "Ran {n} concurrent drags (multi-cursor via cua-compositor)." - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), - } -} - #[async_trait] impl Tool for ParallelMouseDragTool { fn def(&self) -> &ToolDef { @@ -6939,21 +6265,6 @@ impl Tool for ParallelMouseDragTool { } async fn invoke(&self, args: Value) -> ToolResult { - // Nested cua-compositor: run the drags as concurrent multi-cursor - // injections (window-local, no X11 MPX/geometry needed). - if crate::wayland::is_inject_mode() { - return parallel_drag_inject(&args).await; - } - // Native Wayland without the inject socket: MPX/XI2 + uinput master - // pointers don't exist on Wayland. Surface a typed error instead of - // silently calling the X11 path that's guaranteed to fail. - if crate::wayland::is_wayland() { - return ToolResult::error( - "parallel_mouse_drag requires the cua-compositor inject socket on Wayland \ - (set CUA_INJECT_SOCKET to the cua-compositor control socket), \ - or run the target under X11.", - ); - } match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await { Ok(Ok(())) => {} Ok(Err(e)) => return ToolResult::error(e.to_string()), @@ -7190,10 +6501,6 @@ impl Tool for GetScreenSizeTool { return result; } let result = tokio::task::spawn_blocking(|| { - // X11 reports pixel dimensions; scale factor on X11 is not - // well-defined per-monitor, so report 1.0 (matches DPI-unaware - // assumption). Wayland/HiDPI X11 callers should query - // `xrandr --query` for true scale. let (w, h) = x11_screen_size()?; Ok::<(u32, u32, f64), anyhow::Error>((w, h, 1.0)) }) @@ -7272,20 +6579,7 @@ impl Tool for GetDesktopStateTool { // so screen-absolute pixels land exactly. let png = crate::capture::screenshot_display_bytes()?; let (shot_w, shot_h) = crate::capture::png_dimensions_pub(&png)?; - // True screen size. On a pure-Wayland session (native backend - // opted in, no X11 DISPLAY) the capture above came from the - // wlroots `zwlr_screencopy` cascade, whose full-display buffer is - // the whole output at native (physical) pixels — so the PNG - // dimensions ARE the true screen size. Querying the X11 root - // window here would fail with "$DISPLAY variable not set" and - // abort the tool even though the screenshot already succeeded. - // Only fall back to the X11 root-window geometry off Wayland, so - // the X11 / XWayland path is unchanged. See #2017 / Sway testing. - let (screen_w, screen_h) = if crate::wayland::is_wayland() { - (shot_w, shot_h) - } else { - x11_screen_size()? - }; + let (screen_w, screen_h) = x11_screen_size()?; // Optional: write PNG to disk instead of returning base64. let written = if let Some(path) = out_file.as_deref() { std::fs::write(path, &png)?; @@ -7370,21 +6664,7 @@ impl Tool for GetCursorPositionTool { { return result; } - // Native Wayland: there's no protocol for clients to query the real - // global cursor position. Fall back to the synthetic registry that - // records every `motion_absolute` this process emits. - if crate::wayland::is_wayland() { - return match crate::wayland::last_synth_cursor_pos() { - Some((x, y)) => ToolResult::text( - format!("✅ Cursor at ({x}, {y}) (synthetic — last move_cursor in this process)") - ).with_structured(json!({ - "x": x, "y": y, "source": "synthetic" - })), - None => ToolResult::text( - "Cursor position unknown on Wayland — no move_cursor has been issued in this process yet.".to_string() - ).with_structured(json!({ "source": "synthetic", "available": false })), - }; - } + let result = tokio::task::spawn_blocking(|| { use x11rb::connection::Connection; use x11rb::protocol::xproto::ConnectionExt as _; @@ -7435,18 +6715,9 @@ impl Tool for MoveCursorTool { let (x, y) = (input.x, input.y); let xi = x.round() as i32; let yi = y.round() as i32; - let wayland = crate::wayland::wayland_input_enabled(); - let path = if wayland { - "wayland_desktop" - } else { - "xtest_desktop" - }; - let result = if wayland { - tokio::task::spawn_blocking(move || { - crate::wayland::move_cursor_absolute(None, xi, yi) - }) - .await - } else { + + let path = "xtest_desktop"; + let result = { tokio::task::spawn_blocking(move || crate::input::send_move_xtest_desktop(xi, yi)) .await }; @@ -7463,7 +6734,6 @@ impl Tool for MoveCursorTool { } let x = args.f64_or("x", 0.0); let y = args.f64_or("y", 0.0); - let window_id = args.get("window_id").and_then(|v| v.as_u64()); let cursor_id = resolve_cursor_key(&args); self.state.cursor_registry.update_position(&cursor_id, x, y); // End pointing upper-left (45°) — matches Swift's @@ -7472,26 +6742,8 @@ impl Tool for MoveCursorTool { // Use the acknowledged animation path so a first-ever move seeds and // displays the session cursor just as reliably as a coordinate click. crate::overlay::animate_cursor_to_for(cursor_id.clone(), x, y).await; - // Native Wayland: also warp the real cursor via zwlr_virtual_pointer. - // Off-thread because the wayland-client roundtrip is blocking. Best-effort - // — overlay update + registry write already succeeded; surface a warning - // only if the warp itself failed. - let real_warp_note = if crate::wayland::wayland_input_enabled() { - let xi = x.round() as i32; - let yi = y.round() as i32; - match tokio::task::spawn_blocking(move || { - crate::wayland::move_cursor_absolute(window_id, xi, yi) - }) - .await - { - Ok(Ok(())) => " (real cursor warped via virtual-pointer)", - Ok(Err(_)) | Err(_) => " (overlay updated; real-cursor warp failed)", - } - } else { - "" - }; ToolResult::text(format!( - "Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).{real_warp_note}" + "Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})." )) } } @@ -7759,7 +7011,6 @@ impl Tool for CheckPermissionsTool { .await .unwrap_or(false); - let wayland_display = std::env::var("WAYLAND_DISPLAY").ok(); let atspi_status = if atspi_ok { match &dbus_address { Some(a) => format!("✅ org.a11y.Bus reachable (DBUS_SESSION_BUS_ADDRESS={a})"), @@ -7776,23 +7027,19 @@ impl Tool for CheckPermissionsTool { .to_string() }; let status_text = format!( - "X11 display: {}\nWayland: {}\nAT-SPI (D-Bus): {}\nXSendEvent injection: {}", - if x11_ok { "✅ connected" } else { "❌ DISPLAY not set or X11 unavailable" }, - match &wayland_display { - Some(s) if crate::wayland::wayland_enabled() => - format!("✅ native Wayland session (WAYLAND_DISPLAY={s}) — experimental backend ENABLED"), - Some(s) => format!( - "⚠️ native Wayland session (WAYLAND_DISPLAY={s}) — experimental backend OFF; \ - set {}=1 to enable it", - crate::wayland::ENABLE_WAYLAND_ENV - ), - None => "❌ not a Wayland session".to_string(), + "X11 display: {}\nAT-SPI (D-Bus): {}\nXSendEvent injection: {}", + if x11_ok { + "connected" + } else { + "only X11 is supported; set DISPLAY and XAUTHORITY" }, atspi_status, - if x11_ok { "✅ available" } else { "❌ requires X11" } + if x11_ok { "available" } else { "requires X11" } ); - ToolResult::text(status_text) - .with_structured(json!({ "x11": x11_ok, "wayland": wayland_display.is_some(), "wayland_enabled": crate::wayland::wayland_enabled(), "atspi": atspi_ok, "dbus_session_bus_address": dbus_address, "xsend_event": x11_ok })) + ToolResult::text(status_text).with_structured(json!({ + "x11": x11_ok, "atspi": atspi_ok, + "dbus_session_bus_address": dbus_address, "xsend_event": x11_ok + })) } } @@ -7845,18 +7092,15 @@ impl Tool for SetConfigTool { name: "set_config".into(), description: "Update cua-driver-rs configuration. capture_mode / \ max_image_dimension take effect immediately.\n\n\ - Two input shapes (both accepted, matching Windows/Swift):\n\ - - **{key, value}** (preferred): `{\"key\": \"max_image_dimension\", \"value\": 800}` \ - — single leaf write.\n\ - - **Legacy per-field**: `{\"capture_mode\": \"som\", \"max_image_dimension\": 0}`.\n\n\ + Pass named configuration fields, for example max_image_dimension.\n\n\ The experimental_pip keys persist to ~/.cua-driver/config.json and apply on next \ daemon restart (the PiP backend is initialised once at startup; \ Linux ships only the trait stub today — see issue #1729).".into(), input_schema: json!({"type":"object","properties":{ "key":{"type":"string","description":"Name of a single config field to write ({key, value} shape). Pair with `value`."}, "value":{"description":"New value for `key`. JSON type depends on the key."}, - "capture_mode":{"type":"string","enum":["ax","vision"],"description":"Legacy per-field shape. Default capture mode for get_window_state. (\"som\"/\"screenshot\" still decode as deprecated aliases.)"}, - "max_image_dimension":{"type":"integer","description":"Legacy per-field shape. Max dimension for screenshot resizing (0 = no limit)."}, + "capture_mode":{"type":"string","enum":["ax","vision"],"description":"Configuration field. Default capture mode for get_window_state. (\"som\"/\"screenshot\" still decode as deprecated aliases.)"}, + "max_image_dimension":{"type":"integer","description":"Configuration field. Max dimension for screenshot resizing (0 = no limit)."}, "experimental_pip":{"type":"boolean","description":"Enable the experimental PiP preview window (applies next restart; Linux backend stubbed)."}, "experimental_pip_geometry":{"type":"string","description":"PiP window size + optional position in `WxH` or `WxH+X+Y` form."} },"additionalProperties":false}), @@ -7935,7 +7179,7 @@ impl Tool for SetConfigTool { )), } } - // Legacy per-field shape. + // Configuration field. if let Some(mode) = args.opt_str("capture_mode") { if let Err(e) = pip_preview::write_config_key("capture_mode", Value::String(mode.clone())) @@ -8123,11 +7367,7 @@ impl Tool for ZoomTool { let state = self.state.clone(); let result = tokio::task::spawn_blocking(move || { - // Route through the Wayland-aware window capture dispatcher so - // pure-Wayland sessions surface a typed "per-window capture not - // supported yet" error instead of accidentally calling the - // X11-only path with a foreign-toplevel id. - let png = crate::wayland::screenshot_window_dispatch(xid)?; + let png = crate::capture::screenshot_window_bytes(xid)?; cursor_overlay::capture_utils::crop_png_to_jpeg(&png, x1, y1, x2, y2, 500) }) .await; @@ -8230,20 +7470,6 @@ impl Tool for TypeTextCharsTool { }; let text_len = text.chars().count(); let result = tokio::task::spawn_blocking(move || { - if crate::wayland::wayland_input_enabled() { - // Per-char `wtype` loop with the requested delay — mirrors the - // X11 XSendEvent per-char path. Sleeping here is fine because - // we're inside spawn_blocking. - let mut buf = [0u8; 4]; - for ch in text.chars() { - let s = ch.encode_utf8(&mut buf); - crate::wayland::type_text(xid, s)?; - if delay_ms > 0 { - std::thread::sleep(std::time::Duration::from_millis(delay_ms)); - } - } - return Ok(()); - } crate::input::send_type_text_with_delay(xid, &text, delay_ms) }) .await; @@ -8295,7 +7521,7 @@ impl Tool for KillAppTool { .and_then(Value::as_i64) .filter(|pid| *pid > 0) .ok_or_else(|| "kill_app requires a positive integer pid".to_owned())?; - let fingerprint = crate::browser_platform::LinuxBrowserPlatform + let fingerprint = crate::browser_platform::LinuxBrowserPlatform::default() .process_fingerprint(pid) .await .map_err(|error| error.message)?; @@ -8335,7 +7561,23 @@ impl Tool for KillAppTool { // SAFETY: libc::kill is a thin syscall wrapper, no thread-safety concerns. let rc = unsafe { libc::kill(pid_i, libc::SIGKILL) }; if rc == 0 { - ToolResult::text(format!("✅ Sent SIGKILL to pid {pid_i}.")) + let exited = tokio::task::spawn_blocking(move || { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + while crate::proc_fs::is_process_live(pid_i as u32) { + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + true + }) + .await + .unwrap_or(false); + if exited { + ToolResult::text(format!("Terminated pid {pid_i}; process exit confirmed.")) + } else { + ToolResult::error(format!("Sent SIGKILL to {pid_i}, but process exit was not confirmed; observe before retrying")) + } } else { let err = std::io::Error::last_os_error(); ToolResult::error(format!( @@ -8420,30 +7662,14 @@ impl Tool for InvokeMenuTool { }; let pid = input.pid; let window_id = input.window_id; - if !crate::wayland::list_windows_dispatch(Some(pid)) + if !crate::x11::list_windows(Some(pid)) .iter() .any(|window| window.xid == window_id && window.pid == Some(pid)) { return menu_refusal("invoke_menu: window_id does not belong to pid".into()); } - let activation = if crate::wayland::is_wayland() { - let result = tokio::task::spawn_blocking(move || { - crate::wayland::activate_window_for_input_target(window_id, Some(pid)) - }) - .await; - match result { - Ok(Ok(())) => None, - Ok(Err(error)) => { - return menu_refusal(format!( - "invoke_menu: target window could not be activated: {error}" - )) - } - Err(error) => { - return menu_refusal(format!("invoke_menu: activation task failed: {error}")) - } - } - } else { + let activation = { match tokio::task::spawn_blocking(move || { crate::input::x11_activate_window_persistent(window_id) }) @@ -8531,11 +7757,7 @@ impl Tool for SetWindowFrameTool { Ok(input) => input, Err(result) => return result, }; - if crate::wayland::is_wayland() { - return ToolResult::error( - "set_window_frame: this Wayland compositor exposes no protocol that permits a client to set another top-level window's geometry", - ); - } + let values = [input.x, input.y, input.width, input.height]; if values.iter().any(|value| !value.is_finite()) || input.width <= 0.0 @@ -8657,10 +7879,7 @@ impl Tool for BringToFrontTool { activation prevents the remote surface from accepting input. \ X11: EWMH _NET_ACTIVE_WINDOW activation (the `wmctrl -a` equivalent, \ with proper timestamp handling to beat focus-stealing prevention). \ - Wayland: activates through a target-addressable compositor adapter \ - (wlroots foreign-toplevel or the GNOME Shell helper) and refuses \ - when the compositor offers no safe adapter. Matches the macOS / \ - Windows bring_to_front rung." + Matches the macOS / Windows bring_to_front rung." .into(), input_schema: serde_json::json!({ "type":"object","required":["pid"],"properties":{ @@ -8675,35 +7894,6 @@ impl Tool for BringToFrontTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; let pid = args.u64_or("pid", 0) as u32; - if crate::wayland::is_wayland() { - let window_id = match args.opt_u64("window_id") { - Some(window_id) => window_id, - None => match crate::wayland::list_windows_dispatch(Some(pid)).first() { - Some(window) => window.xid, - None => { - return ToolResult::error(format!( - "bring_to_front: no window_id given and no Wayland windows found for pid {pid}." - )) - } - }, - }; - let result = tokio::task::spawn_blocking(move || { - crate::wayland::activate_window_for_input_target(window_id, Some(pid)) - }) - .await; - return match result { - Ok(Ok(())) => { - ToolResult::text(format!("Brought Wayland window {window_id} to front.")) - .with_structured(serde_json::json!({ - "window_id": window_id, - "platform": "linux", - "session": "wayland", - })) - } - Ok(Err(error)) => ToolResult::error(error.to_string()), - Err(error) => ToolResult::error(format!("Task error: {error}")), - }; - } // X11: resolve the target xid (window_id, else first window for pid). let xid = match args.opt_u64("window_id") { @@ -8751,6 +7941,8 @@ pub fn build_registry_with_provider( compat: bool, provider: Option>, ) -> ToolRegistry { + static REAP: std::sync::Once = std::sync::Once::new(); + REAP.call_once(crate::input::reap_orphaned_master_pointers); let state = ToolState::new(); let cursor_outcome_reader = { let cursor_registry = state.cursor_registry.clone(); @@ -8830,108 +8022,158 @@ pub fn build_registry_with_provider( cua_driver_core::observation_revision::revision_tokens().clear_runtime(&runtime_scope); }); } - r.register(Box::new(ListAppsTool)); - r.register(Box::new(ListWindowsTool)); - r.register(Box::new(GetWindowStateTool { - state: state.clone(), - })); - r.register(Box::new( - cua_driver_core::expectation::VerifyStateTool::new(std::sync::Arc::new( - cua_driver_core::expectation::ToolObservationProvider::new( + cua_driver_core::desktop_keyboard::register(&mut r, Arc::new(crate::input::DesktopKeyboard)); + register_native_tool(&mut r, Box::new(ListAppsTool)); + register_native_tool(&mut r, Box::new(ListWindowsTool)); + register_native_tool( + &mut r, + Box::new(GetWindowStateTool { + state: state.clone(), + }), + ); + register_native_tool( + &mut r, + Box::new(cua_driver_core::expectation::VerifyStateTool::new( + std::sync::Arc::new(cua_driver_core::expectation::ToolObservationProvider::new( std::sync::Arc::new(ListWindowsTool), std::sync::Arc::new(GetWindowStateTool { state: state.clone(), }), - ), + )), )), - )); - r.register(Box::new(LaunchAppTool)); - r.register(Box::new(KillAppTool)); + ); + register_native_tool(&mut r, Box::new(LaunchAppTool)); + register_native_tool(&mut r, Box::new(KillAppTool)); let pid_window_candidates: WindowTargetCandidates = Arc::new(pid_window_target_candidates); - r.register(pid_window_guarded(BringToFrontTool, &pid_window_candidates)); - r.register(Box::new(SetWindowFrameTool)); - r.register(Box::new(InvokeMenuTool)); - r.register(pid_window_guarded( - ClickTool { + register_native_tool( + &mut r, + pid_window_guarded(BringToFrontTool, &pid_window_candidates), + ); + register_native_tool(&mut r, Box::new(SetWindowFrameTool)); + register_native_tool(&mut r, Box::new(InvokeMenuTool)); + register_native_tool( + &mut r, + pid_window_guarded( + ClickTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + DoubleClickTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + RightClickTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + DragTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + MouseButtonDownTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + MouseDragTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + MouseButtonUpTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + Box::new(ParallelMouseDragTool { state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - DoubleClickTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - RightClickTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - DragTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - MouseButtonDownTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - MouseDragTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - MouseButtonUpTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(Box::new(ParallelMouseDragTool { - state: state.clone(), - })); - r.register(pid_window_guarded( - TypeTextTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - PressKeyTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - HotkeyTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - SetValueTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - PerformSecondaryActionTool { - state: state.clone(), - }, - &pid_window_candidates, - )); - r.register(pid_window_guarded( - ScrollTool { - state: state.clone(), - }, - &pid_window_candidates, - )); + }), + ); + register_native_tool( + &mut r, + pid_window_guarded( + TypeTextTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + PressKeyTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + HotkeyTool { + state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + SetValueTool { + _state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + PerformSecondaryActionTool { + _state: state.clone(), + }, + &pid_window_candidates, + ), + ); + register_native_tool( + &mut r, + pid_window_guarded( + ScrollTool { + _state: state.clone(), + }, + &pid_window_candidates, + ), + ); cua_driver_core::clipboard::register_clipboard_tools( &mut r, Arc::new(crate::clipboard::LinuxClipboard::new()), @@ -8940,49 +8182,73 @@ pub fn build_registry_with_provider( // platform-windows/src/tools/impl_.rs::build_registry. Canonical // screenshot path is `get_window_state` (it always returns a screenshot now). let _ = compat; - r.register(Box::new(GetScreenSizeTool)); - r.register(Box::new(GetDesktopStateTool)); - r.register(Box::new(GetCursorPositionTool)); - r.register(Box::new(MoveCursorTool { - state: state.clone(), - })); - r.register(Box::new(SetAgentCursorEnabledV2Tool { - state: state.clone(), - })); - r.register(Box::new(SetAgentCursorMotionV2Tool)); - r.register(Box::new(GetAgentCursorStateV2Tool)); - r.register(Box::new(SetAgentCursorThemeTool { - state: state.clone(), - })); - r.register(Box::new(CheckPermissionsTool)); + register_native_tool(&mut r, Box::new(GetScreenSizeTool)); + register_native_tool(&mut r, Box::new(GetDesktopStateTool)); + register_native_tool(&mut r, Box::new(GetCursorPositionTool)); + register_native_tool( + &mut r, + Box::new(MoveCursorTool { + state: state.clone(), + }), + ); + register_native_tool( + &mut r, + Box::new(SetAgentCursorEnabledV2Tool { + state: state.clone(), + }), + ); + register_native_tool(&mut r, Box::new(SetAgentCursorMotionV2Tool)); + register_native_tool(&mut r, Box::new(GetAgentCursorStateV2Tool)); + register_native_tool( + &mut r, + Box::new(SetAgentCursorThemeTool { + state: state.clone(), + }), + ); + register_native_tool(&mut r, Box::new(CheckPermissionsTool)); // `health_report` — single-call cross-platform driver diagnostics. // Stable schema_version="1" contract for downstream consumers. Linux skips // tcc_* and bundle_identity with "not applicable on Linux". - r.register(Box::new( - cua_driver_core::health_report::HealthReportTool::new(std::sync::Arc::new( - crate::health_report::LinuxHealthProvider, + register_native_tool( + &mut r, + Box::new(cua_driver_core::health_report::HealthReportTool::new( + std::sync::Arc::new(crate::health_report::LinuxHealthProvider), )), - )); - r.register(Box::new(GetConfigTool { - state: state.clone(), - })); - r.register(Box::new(SetConfigTool { - state: state.clone(), - })); - r.register(Box::new(GetAccessibilityTreeTool)); - r.register(Box::new(ZoomTool { - state: state.clone(), - })); + ); + register_native_tool( + &mut r, + Box::new(GetConfigTool { + state: state.clone(), + }), + ); + register_native_tool( + &mut r, + Box::new(SetConfigTool { + state: state.clone(), + }), + ); + register_native_tool(&mut r, Box::new(GetAccessibilityTreeTool)); + register_native_tool( + &mut r, + Box::new(ZoomTool { + state: state.clone(), + }), + ); // `type_text_chars` is a deprecated invoke-time alias for `type_text`. // Keep it out of tools/list, matching the macOS and Windows registries. let _: &TypeTextCharsTool = &TypeTextCharsTool; // Cross-platform `page` tool definition lives in mcp-server; Linux plugs // in its AT-SPI + CDP backend here. - r.register(Box::new(cua_driver_core::page::PageTool::new(Arc::new( - super::page::LinuxPageBackend::new(), - )))); + register_native_tool( + &mut r, + Box::new(cua_driver_core::page::PageTool::new(Arc::new( + super::page::LinuxPageBackend::new(), + ))), + ); let browser_engine = cua_driver_core::browser::BrowserEngine::new_with_runtime_services( - Arc::new(crate::browser_platform::LinuxBrowserPlatform), + Arc::new(crate::browser_platform::LinuxBrowserPlatform::new( + state.cursor_registry.clone(), + )), r.approval_broker(), r.protected_resource_ownership(), ); @@ -9026,17 +8292,16 @@ mod click_button_schema_tests { let lc = desc.to_ascii_lowercase(); assert!(lc.contains("left"), "description should mention default"); assert!( - lc.contains("wayland"), - "description should call out wayland fallback" + lc.contains("x11"), + "description should describe X11 delivery" ); } #[test] - fn chromium_background_requires_focus_free_inject_mode() { - assert!(chromium_background_must_refuse(false, false, true)); - assert!(!chromium_background_must_refuse(false, true, true)); - assert!(!chromium_background_must_refuse(true, false, true)); - assert!(!chromium_background_must_refuse(false, false, false)); + fn chromium_background_refuses_synthetic_input() { + assert!(chromium_background_must_refuse(false, true)); + assert!(!chromium_background_must_refuse(true, true)); + assert!(!chromium_background_must_refuse(false, false)); } #[test] @@ -9190,26 +8455,3 @@ mod app_discovery_tests { assert!(message.contains("restart")); } } - -#[cfg(test)] -mod app_inject_tests { - use super::*; - #[test] - #[ignore = "changes process environment; run alone"] - fn app_inject_refuses_process_addressing_before_dispatch() { - let old = std::env::var_os("CUA_INJECT_SOCKET"); - std::env::set_var("CUA_INJECT_SOCKET", "/tmp/cua-fixture-unused.sock"); - let called = std::cell::Cell::new(false); - let result = with_wayland_app_focus(true, 42, 7, || { - called.set(true); - Ok(()) - }); - if let Some(value) = old { - std::env::set_var("CUA_INJECT_SOCKET", value); - } else { - std::env::remove_var("CUA_INJECT_SOCKET"); - } - assert!(result.unwrap_err().to_string().contains("exact window")); - assert!(!called.get()); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs b/packages/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs index bc8a05e093b..a7ad67f4276 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs @@ -183,10 +183,6 @@ stub_tool!(scroll_m, ScrollTool, "scroll", "Scroll the target pid's focused region. direction required; by defaults to line, amount defaults to 3.", serde_json::json!({"type":"object","required":["direction"],"properties":{"session": cua_driver_core::tool_schema::session_schema(),"cursor_id":{"type":"string"},"pid":{"type":"integer"},"direction":{"type":"string","enum":["up","down","left","right"]},"by":{"type":"string","enum":["line","page"]},"amount":{"type":"integer","minimum":1,"maximum":50},"window_id":{"type":"integer"},"element_index": cua_driver_core::tool_schema::element_index_schema()},"additionalProperties":false})); -stub_tool!(screenshot_m, ScreenshotTool, "screenshot", - "Capture a screenshot. Without window_id captures the full display. Supports png and jpeg formats.", - serde_json::json!({"type":"object","properties":{"window_id":{"type":"integer"},"format":{"type":"string","enum":["png","jpeg"]},"quality":{"type":"integer","minimum":1,"maximum":95}},"additionalProperties":false})); - stub_tool!( get_screen_size_m, GetScreenSizeTool, @@ -223,7 +219,28 @@ contract_stub_tool!( SetAgentCursorThemeTool, "set_agent_cursor_theme" ); -contract_stub_tool!(invoke_menu_m, InvokeMenuTool, "invoke_menu"); +pub struct InvokeMenuTool; +#[async_trait] +impl Tool for InvokeMenuTool { + fn def(&self) -> &ToolDef { + static DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + DEF.get_or_init(|| { + let c = cua_driver_contract::tool_contract("invoke_menu").expect("menu contract"); + ToolDef { + name: c.name, + description: c.description, + input_schema: c.input_schema, + read_only: c.annotations.read_only, + destructive: c.annotations.destructive, + idempotent: c.annotations.idempotent, + open_world: c.annotations.open_world, + } + }) + } + async fn invoke(&self, _args: Value) -> ToolResult { + not_impl("invoke_menu") + } +} stub_tool!( check_perms_m, @@ -327,7 +344,6 @@ pub fn build_registry() -> cua_driver_core::tool::ToolRegistry { r.register(Box::new(SetValueTool)); r.register(Box::new(InvokeMenuTool)); r.register(Box::new(ScrollTool)); - r.register(Box::new(ScreenshotTool)); r.register(Box::new(GetScreenSizeTool)); r.register(Box::new(GetCursorPositionTool)); r.register(Box::new(MoveCursorTool)); diff --git a/packages/cua-driver/rust/crates/platform-linux/src/video_wayland.rs b/packages/cua-driver/rust/crates/platform-linux/src/video_wayland.rs deleted file mode 100644 index 0a525d88655..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/video_wayland.rs +++ /dev/null @@ -1,287 +0,0 @@ -//! Native Wayland full-desktop video capture. -//! -//! wlroots compositors use `wf-recorder`. GNOME does not expose the wlroots -//! screencopy protocol, so its attested Shell helper supplies compositor-owned -//! PNG frames which are encoded into the same MP4 artifact through FFmpeg. - -use std::io::Read; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; -use std::sync::mpsc::{self, RecvTimeoutError}; -use std::time::{Duration, Instant}; - -use cua_driver_core::video::{VideoBackend, VideoBackendFactory, VideoMetadata}; - -pub struct WfRecorderVideoBackendFactory; - -impl VideoBackendFactory for WfRecorderVideoBackendFactory { - fn start(&self, output_path: &Path) -> anyhow::Result> { - if let Some(first_frame) = crate::wayland::shell_helper::trusted_screenshot_display() { - return GnomeShellVideoBackend::start(output_path, first_frame) - .map(|backend| Box::new(backend) as Box); - } - WfRecorderVideoBackend::start(output_path) - .map(|backend| Box::new(backend) as Box) - } -} - -struct GnomeShellVideoBackend { - stop_tx: mpsc::Sender<()>, - worker: std::thread::JoinHandle>, - output_path: PathBuf, - started_at: Instant, -} - -impl GnomeShellVideoBackend { - fn start(output_path: &Path, first_frame: Vec) -> anyhow::Result { - let available = Command::new("ffmpeg") - .arg("-version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|status| status.success()) - .unwrap_or(false); - if !available { - anyhow::bail!("ffmpeg is required for GNOME Wayland video capture"); - } - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent)?; - } - - let output_path = output_path.to_path_buf(); - let worker_output = output_path.clone(); - let (stop_tx, stop_rx) = mpsc::channel(); - let (ready_tx, ready_rx) = mpsc::sync_channel(1); - let worker = std::thread::spawn(move || { - let result = run_gnome_shell_encoder(&worker_output, first_frame, stop_rx, &ready_tx); - if result.is_err() { - let _ = ready_tx.send(result.as_ref().map(|_| ()).map_err(ToString::to_string)); - } - result - }); - match ready_rx.recv_timeout(Duration::from_secs(8)) { - Ok(Ok(())) => Ok(Self { - stop_tx, - worker, - output_path, - started_at: Instant::now(), - }), - Ok(Err(error)) => { - let _ = worker.join(); - anyhow::bail!("GNOME Wayland video failed to start: {error}") - } - Err(error) => { - let _ = stop_tx.send(()); - let _ = worker.join(); - anyhow::bail!("GNOME Wayland video startup timed out: {error}") - } - } - } -} - -fn run_gnome_shell_encoder( - output_path: &Path, - first_frame: Vec, - stop_rx: mpsc::Receiver<()>, - ready_tx: &mpsc::SyncSender>, -) -> anyhow::Result<()> { - let mut child = Command::new("ffmpeg") - .args([ - "-hide_banner", - "-loglevel", - "error", - "-f", - "image2pipe", - "-framerate", - "5", - "-vcodec", - "png", - "-i", - "-", - "-an", - "-vf", - "pad=ceil(iw/2)*2:ceil(ih/2)*2", - "-c:v", - "libx264", - "-pix_fmt", - "yuv420p", - "-movflags", - "+faststart", - "-y", - ]) - .arg(output_path) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|error| anyhow::anyhow!("failed to start GNOME FFmpeg encoder: {error}"))?; - let mut stdin = child - .stdin - .take() - .ok_or_else(|| anyhow::anyhow!("GNOME FFmpeg encoder exposed no stdin"))?; - stdin.write_all(&first_frame)?; - stdin.flush()?; - let _ = ready_tx.send(Ok(())); - - loop { - match stop_rx.recv_timeout(Duration::from_millis(200)) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => break, - Err(RecvTimeoutError::Timeout) => { - let frame = crate::wayland::shell_helper::trusted_screenshot_display() - .ok_or_else(|| anyhow::anyhow!("trusted GNOME compositor capture stopped"))?; - stdin.write_all(&frame)?; - stdin.flush()?; - } - } - } - drop(stdin); - let output = child.wait_with_output()?; - if !output.status.success() { - anyhow::bail!( - "GNOME FFmpeg encoder exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - } - if output_path.metadata().map(|meta| meta.len()).unwrap_or(0) == 0 { - anyhow::bail!("GNOME FFmpeg encoder produced an empty artifact"); - } - Ok(()) -} - -impl VideoBackend for GnomeShellVideoBackend { - fn stop(self: Box) -> anyhow::Result { - let elapsed = self.started_at.elapsed(); - let _ = self.stop_tx.send(()); - match self.worker.join() { - Ok(result) => result?, - Err(_) => anyhow::bail!("GNOME Wayland video worker panicked"), - } - Ok(VideoMetadata { - path: self.output_path, - duration_ms: elapsed.as_millis() as u64, - finalized: true, - }) - } -} - -struct WfRecorderVideoBackend { - child: Child, - output_path: PathBuf, - started_at: Instant, - stderr_thread: Option>>, -} - -impl WfRecorderVideoBackend { - fn start(output_path: &Path) -> anyhow::Result { - if std::env::var_os("WAYLAND_DISPLAY").is_none() { - anyhow::bail!("wf-recorder requires WAYLAND_DISPLAY"); - } - let available = Command::new("wf-recorder") - .arg("--help") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|status| status.success()) - .unwrap_or(false); - if !available { - anyhow::bail!( - "wf-recorder not found on PATH. Install wf-recorder for native Wayland video." - ); - } - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent)?; - } - - let started_at = Instant::now(); - let mut command = Command::new("wf-recorder"); - command - .arg("-f") - .arg(output_path) - .args(["--no-damage", "-c", "libx264", "-x", "yuv420p"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()); - if let Some(output) = std::env::var_os("CUA_WAYLAND_RECORDING_OUTPUT") { - command.arg("-o").arg(output); - } - let mut child = command - .spawn() - .map_err(|error| anyhow::anyhow!("failed to start wf-recorder: {error}"))?; - let stderr_thread = child.stderr.take().map(|mut stderr| { - std::thread::spawn(move || { - let mut output = Vec::new(); - let _ = stderr.read_to_end(&mut output); - if output.len() > 4096 { - let excess = output.len() - 4096; - output.drain(..excess); - } - output - }) - }); - - let probe_deadline = Instant::now() + Duration::from_millis(1500); - while Instant::now() < probe_deadline { - if let Some(status) = child.try_wait()? { - let stderr = stderr_thread - .map(|worker| worker.join().unwrap_or_default()) - .unwrap_or_default(); - anyhow::bail!( - "wf-recorder exited during startup ({status}): {}", - String::from_utf8_lossy(&stderr) - ); - } - std::thread::sleep(Duration::from_millis(100)); - } - - Ok(Self { - child, - output_path: output_path.to_path_buf(), - started_at, - stderr_thread, - }) - } -} - -impl VideoBackend for WfRecorderVideoBackend { - fn stop(mut self: Box) -> anyhow::Result { - let elapsed = self.started_at.elapsed(); - unsafe { - libc::kill(self.child.id() as i32, libc::SIGINT); - } - let deadline = Instant::now() + Duration::from_secs(10); - let finalized = loop { - if let Some(status) = self.child.try_wait()? { - break status.success(); - } - if Instant::now() >= deadline { - let _ = self.child.kill(); - let _ = self.child.wait(); - break false; - } - std::thread::sleep(Duration::from_millis(80)); - }; - let stderr = self - .stderr_thread - .take() - .and_then(|worker| worker.join().ok()) - .unwrap_or_default(); - let has_video = self - .output_path - .metadata() - .map(|metadata| metadata.len() > 0) - .unwrap_or(false); - if !finalized || !has_video { - anyhow::bail!( - "wf-recorder did not finalize a playable artifact: {}", - String::from_utf8_lossy(&stderr) - ); - } - Ok(VideoMetadata { - path: self.output_path, - duration_ms: elapsed.as_millis() as u64, - finalized: true, - }) - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs deleted file mode 100644 index b103a219480..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs +++ /dev/null @@ -1,422 +0,0 @@ -//! Output capture via `ext-image-copy-capture-v1` (staging protocol). -//! -//! The wlroots `zwlr_screencopy_manager_v1` capture path in `mod.rs` works -//! on wlroots compositors (sway, labwc) but is missing from compositors -//! that don't speak wlroots protocols (GNOME mutter, KDE Plasma <6.2). -//! The staging `ext-image-copy-capture-v1` protocol replaces wlr-screencopy -//! across the ecosystem — sway 1.10+, labwc 0.8+, niri 0.1.6+, hyprland -//! 0.45+, KDE Plasma 6.2+, GNOME mutter 47+ all implement it. -//! -//! This module captures the FIRST `wl_output` advertised by the compositor -//! via the chain: -//! ext_output_image_capture_source_manager_v1.create_source(wl_output) -//! → ext_image_copy_capture_manager_v1.create_session(source, options) -//! → session emits buffer_size + done -//! → allocate wl_shm pool + wl_buffer matching the size/format -//! → session.create_frame() → frame.attach_buffer(buffer) → frame.capture() -//! → frame emits ready (success) or failed (error) -//! → read pixels out of the mmap'd buffer, PNG-encode, destroy -//! -//! Per-window capture (ext_foreign_toplevel_image_capture_source_manager_v1) -//! is a follow-up — needs the existing `list_windows` to migrate from -//! `zwlr_foreign_toplevel_manager_v1` to `ext_foreign_toplevel_list_v1` so -//! the window_id handles match up. - -use std::time::Duration; - -use wayland_client::{ - protocol::{ - wl_buffer::WlBuffer, - wl_output::WlOutput, - wl_registry, - wl_shm::{self, WlShm}, - wl_shm_pool::WlShmPool, - }, - Connection, Dispatch, Proxy, QueueHandle, -}; -use wayland_protocols::ext::image_capture_source::v1::client::{ - ext_image_capture_source_v1::ExtImageCaptureSourceV1, - ext_output_image_capture_source_manager_v1::ExtOutputImageCaptureSourceManagerV1, -}; -use wayland_protocols::ext::image_copy_capture::v1::client::{ - ext_image_copy_capture_frame_v1::{self, ExtImageCopyCaptureFrameV1}, - ext_image_copy_capture_manager_v1::{self, ExtImageCopyCaptureManagerV1}, - ext_image_copy_capture_session_v1::{self, ExtImageCopyCaptureSessionV1}, -}; - -#[derive(Default)] -struct CapState { - output: Option, - source_mgr: Option, - copy_mgr: Option, - shm: Option, - // Buffer specs reported by the session before we allocate. - fmt: Option, - width: u32, - height: u32, - stride: u32, - done: bool, - // Frame outcome - ready: bool, - failed: bool, - fail_reason: Option, -} - -/// Capture the first `wl_output` via ext-image-copy-capture-v1, return raw -/// PNG bytes. Falls back to the caller's next tier when the compositor -/// doesn't expose the protocol — surface a typed anyhow::Error so the -/// cascade in `screenshot_display_dispatch` can branch. -pub fn screenshot_via_ext_copy() -> anyhow::Result> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - - let mut state = CapState::default(); - queue.roundtrip(&mut state)?; - for _ in 0..3 { - queue.roundtrip(&mut state)?; - } - - let output = state - .output - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output"))?; - let source_mgr = state.source_mgr.clone().ok_or_else(|| { - anyhow::anyhow!("compositor does not expose ext_output_image_capture_source_manager_v1") - })?; - let copy_mgr = state.copy_mgr.clone().ok_or_else(|| { - anyhow::anyhow!("compositor does not expose ext_image_copy_capture_manager_v1") - })?; - let shm = state - .shm - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?; - - // Create source + session. - let source: ExtImageCaptureSourceV1 = source_mgr.create_source(&output, &qh, ()); - let session: ExtImageCopyCaptureSessionV1 = copy_mgr.create_session( - &source, - ext_image_copy_capture_manager_v1::Options::empty(), - &qh, - (), - ); - - // Wait for the session to announce buffer dimensions + format + done. - let deadline = std::time::Instant::now() + Duration::from_secs(3); - while !state.done && std::time::Instant::now() < deadline { - queue.roundtrip(&mut state)?; - if state.width > 0 && state.height > 0 && state.fmt.is_some() { - // Some compositors don't emit `done` explicitly — proceed once - // we have geometry + format. - state.done = true; - } - } - if !state.done || state.width == 0 || state.height == 0 || state.fmt.is_none() { - anyhow::bail!( - "ext-image-copy-capture session never reported buffer specs (w={}, h={}, fmt={:?})", - state.width, - state.height, - state.fmt - ); - } - - // Checked arithmetic on compositor-provided sizes. A malicious or - // buggy compositor could announce dimensions that overflow `usize` on - // multiplication — we refuse to allocate in that case rather than - // wrapping and reading past the buffer. - let stride = if state.stride > 0 { - state.stride - } else { - state.width.checked_mul(4).ok_or_else(|| { - anyhow::anyhow!( - "compositor advertised width that overflows stride: {}", - state.width - ) - })? - }; - if stride < state.width.saturating_mul(4) { - anyhow::bail!( - "compositor stride {} is smaller than width*4 {}; refusing to allocate (would alias rows)", - stride, state.width.saturating_mul(4) - ); - } - let size = (stride as usize) - .checked_mul(state.height as usize) - .ok_or_else(|| { - anyhow::anyhow!( - "compositor buffer size overflows usize: stride={} height={}", - stride, - state.height - ) - })?; - - // Allocate the wl_shm buffer. - let (fd, ptr) = super::anon_shm(size)?; - use std::os::fd::AsFd as _; - let pool_fd = unsafe { super::borrowed_fd(fd) }; - let pool: WlShmPool = shm.create_pool(pool_fd.as_fd(), size as i32, &qh, ()); - let fmt: wl_shm::Format = wl_shm::Format::try_from(state.fmt.unwrap()).map_err(|_| { - anyhow::anyhow!( - "compositor advertised unsupported wl_shm format {:#x}", - state.fmt.unwrap() - ) - })?; - let buffer: WlBuffer = pool.create_buffer( - 0, - state.width as i32, - state.height as i32, - stride as i32, - fmt, - &qh, - (), - ); - - // Create + run the frame. - let frame: ExtImageCopyCaptureFrameV1 = session.create_frame(&qh, ()); - frame.attach_buffer(&buffer); - frame.capture(); - queue.roundtrip(&mut state)?; - - let deadline = std::time::Instant::now() + Duration::from_secs(5); - while !state.ready && !state.failed && std::time::Instant::now() < deadline { - queue.roundtrip(&mut state)?; - } - if state.failed { - super::cleanup_mmap(ptr, size, fd); - anyhow::bail!( - "ext-image-copy-capture frame failed (reason={:?})", - state.fail_reason - ); - } - if !state.ready { - super::cleanup_mmap(ptr, size, fd); - anyhow::bail!("ext-image-copy-capture frame timed out without ready event"); - } - - // Copy + encode + cleanup. - let pixels: &[u8] = unsafe { std::slice::from_raw_parts(ptr as *const u8, size) }; - let png = encode_buffer_to_png(pixels, state.width, state.height, stride, fmt)?; - - frame.destroy(); - session.destroy(); - source.destroy(); - pool.destroy(); - queue.roundtrip(&mut state).ok(); - super::cleanup_mmap(ptr, size, fd); - - Ok(png) -} - -fn encode_buffer_to_png( - pixels: &[u8], - width: u32, - height: u32, - stride: u32, - fmt: wl_shm::Format, -) -> anyhow::Result> { - // The image crate expects RGBA8888 row-packed. wl_shm gives us Argb8888 - // / Xrgb8888 which in little-endian memory layout is BGRA / BGRX. Swap - // channels into RGBA and pack rows. - let mut rgba: Vec = Vec::with_capacity((width as usize) * (height as usize) * 4); - for y in 0..(height as usize) { - let row_start = y * (stride as usize); - for x in 0..(width as usize) { - let i = row_start + x * 4; - let b = pixels[i]; - let g = pixels[i + 1]; - let r = pixels[i + 2]; - let a = match fmt { - wl_shm::Format::Xrgb8888 => 0xFF, - _ => pixels[i + 3], - }; - rgba.extend_from_slice(&[r, g, b, a]); - } - } - - use image::{codecs::png::PngEncoder, ImageBuffer, Rgba}; - let img: ImageBuffer, _> = ImageBuffer::from_raw(width, height, rgba) - .ok_or_else(|| anyhow::anyhow!("internal: buffer dims mismatch ({}x{})", width, height))?; - let mut out: Vec = Vec::new(); - use image::ImageEncoder; - PngEncoder::new(&mut out).write_image( - img.as_raw(), - width, - height, - image::ExtendedColorType::Rgba8, - )?; - Ok(out) -} - -// ── Wayland Dispatch impls ─────────────────────────────────────────────── - -impl Dispatch for CapState { - fn event( - state: &mut Self, - registry: &wl_registry::WlRegistry, - event: wl_registry::Event, - _: &(), - _: &Connection, - qh: &QueueHandle, - ) { - if let wl_registry::Event::Global { - name, - interface, - version, - } = event - { - match interface.as_str() { - "wl_output" => { - if state.output.is_none() { - state.output = - Some(registry.bind::(name, version.min(4), qh, ())); - } - } - "wl_shm" => { - state.shm = Some(registry.bind::(name, 1, qh, ())); - } - "ext_output_image_capture_source_manager_v1" => { - state.source_mgr = - Some(registry.bind::( - name, - 1, - qh, - (), - )); - } - "ext_image_copy_capture_manager_v1" => { - state.copy_mgr = - Some(registry.bind::(name, 1, qh, ())); - } - _ => {} - } - } - } -} - -impl Dispatch for CapState { - fn event( - _: &mut Self, - _: &WlOutput, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} -impl Dispatch for CapState { - fn event( - _: &mut Self, - _: &WlShm, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} -impl Dispatch for CapState { - fn event( - _: &mut Self, - _: &WlShmPool, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} -impl Dispatch for CapState { - fn event( - _: &mut Self, - _: &WlBuffer, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} -impl Dispatch for CapState { - fn event( - _: &mut Self, - _: &ExtOutputImageCaptureSourceManagerV1, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} -impl Dispatch for CapState { - fn event( - _: &mut Self, - _: &ExtImageCaptureSourceV1, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} -impl Dispatch for CapState { - fn event( - _: &mut Self, - _: &ExtImageCopyCaptureManagerV1, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} -impl Dispatch for CapState { - fn event( - state: &mut Self, - _: &ExtImageCopyCaptureSessionV1, - event: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - use ext_image_copy_capture_session_v1::Event as Ev; - match event { - Ev::BufferSize { width, height } => { - state.width = width; - state.height = height; - } - Ev::ShmFormat { format } => { - if state.fmt.is_none() { - if let Ok(f) = format.into_result() { - state.fmt = Some(f as u32); - } - } - } - Ev::Done => { - state.done = true; - } - _ => {} - } - } -} -impl Dispatch for CapState { - fn event( - state: &mut Self, - _: &ExtImageCopyCaptureFrameV1, - event: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - use ext_image_copy_capture_frame_v1::Event as Ev; - match event { - Ev::Ready => state.ready = true, - Ev::Failed { reason } => { - state.failed = true; - state.fail_reason = reason.into_result().ok().map(|r| r as u32); - } - _ => {} - } - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs deleted file mode 100644 index e5e6071a255..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs +++ /dev/null @@ -1,431 +0,0 @@ -//! Generic staging `ext-foreign-toplevel-list-v1` window enumeration. - -use std::collections::HashMap; -use std::sync::{Mutex, OnceLock}; - -use wayland_client::{ - event_created_child, protocol::wl_registry, Connection, Dispatch, Proxy, QueueHandle, -}; -use wayland_protocols::ext::foreign_toplevel_list::v1::client::{ - ext_foreign_toplevel_handle_v1::{self as ext_handle, ExtForeignToplevelHandleV1}, - ext_foreign_toplevel_list_v1::{ - self as ext_list, ExtForeignToplevelListV1, EVT_TOPLEVEL_OPCODE, - }, -}; - -use crate::x11::WindowInfo; - -/// Synthetic ext-toplevel IDs must survive the existing `window_id as u32` -/// element-token paths. Keep them in a high, nonzero u32 namespace and reserve -/// `u32::MAX` as an exhaustion sentinel rather than emitting it. -const EXT_ID_NAMESPACE_START: u32 = 0xF000_0000; -const EXT_ID_NAMESPACE_END: u32 = 0xFFFF_FFFE; - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct ToplevelRecord { - identifier: String, - title: String, - app_id: String, -} - -#[derive(Default)] -struct PendingRecord { - identifier: Option, - title: Option, - app_id: Option, -} - -#[derive(Default)] -struct State { - manager: Option, - pending: HashMap, - toplevels: HashMap, -} - -impl Dispatch for State { - fn event( - state: &mut Self, - registry: &wl_registry::WlRegistry, - event: wl_registry::Event, - _: &(), - _: &Connection, - qh: &QueueHandle, - ) { - if let wl_registry::Event::Global { - name, - interface, - version, - } = event - { - if interface == ExtForeignToplevelListV1::interface().name { - state.manager = Some(registry.bind::( - name, - version.min(1), - qh, - (), - )); - } - } - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &ExtForeignToplevelListV1, - _: ext_list::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } - - event_created_child!(State, ExtForeignToplevelListV1, [ - EVT_TOPLEVEL_OPCODE => (ExtForeignToplevelHandleV1, ()), - ]); -} - -impl Dispatch for State { - fn event( - state: &mut Self, - handle: &ExtForeignToplevelHandleV1, - event: ext_handle::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - let protocol_id = handle.id().protocol_id(); - match event { - ext_handle::Event::Title { title } => { - state.pending.entry(protocol_id).or_default().title = Some(title); - } - ext_handle::Event::AppId { app_id } => { - state.pending.entry(protocol_id).or_default().app_id = Some(app_id); - } - ext_handle::Event::Identifier { identifier } => { - state.pending.entry(protocol_id).or_default().identifier = Some(identifier); - } - ext_handle::Event::Done => { - let pending = state.pending.remove(&protocol_id).unwrap_or_default(); - let record = state.toplevels.entry(protocol_id).or_default(); - if let Some(identifier) = pending.identifier { - record.identifier = identifier; - } - if let Some(title) = pending.title { - record.title = title; - } - if let Some(app_id) = pending.app_id { - record.app_id = app_id; - } - } - ext_handle::Event::Closed => { - state.pending.remove(&protocol_id); - state.toplevels.remove(&protocol_id); - handle.destroy(); - } - _ => {} - } - } -} - -struct IdRegistry { - by_identifier: HashMap, - by_id: HashMap, - next: Option, -} - -impl Default for IdRegistry { - fn default() -> Self { - Self { - by_identifier: HashMap::new(), - by_id: HashMap::new(), - next: Some(EXT_ID_NAMESPACE_START), - } - } -} - -impl IdRegistry { - fn id_for(&mut self, identifier: &str) -> anyhow::Result { - if let Some(id) = self.by_identifier.get(identifier) { - if self.by_id.get(id).is_some_and(|known| known == identifier) { - return Ok(u64::from(*id)); - } - anyhow::bail!("ext toplevel id registry collision for identifier {identifier:?}"); - } - - let mut id = self - .next - .ok_or_else(|| anyhow::anyhow!("ext toplevel numeric id space exhausted"))?; - while self.by_id.contains_key(&id) { - id = id - .checked_add(1) - .filter(|candidate| *candidate <= EXT_ID_NAMESPACE_END) - .ok_or_else(|| anyhow::anyhow!("ext toplevel numeric id space exhausted"))?; - } - self.next = (id < EXT_ID_NAMESPACE_END).then_some(id + 1); - self.by_identifier.insert(identifier.to_owned(), id); - self.by_id.insert(id, identifier.to_owned()); - Ok(u64::from(id)) - } -} - -fn id_registry() -> &'static Mutex { - static REGISTRY: OnceLock> = OnceLock::new(); - REGISTRY.get_or_init(|| Mutex::new(IdRegistry::default())) -} - -fn stable_id(identifier: &str) -> anyhow::Result { - id_registry() - .lock() - .map_err(|_| anyhow::anyhow!("ext toplevel id registry lock poisoned"))? - .id_for(identifier) -} - -/// Enumerate ext toplevels and enrich them with AT-SPI pid and geometry data. -pub fn list_windows() -> anyhow::Result> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - - let mut state = State::default(); - queue.roundtrip(&mut state)?; - if state.manager.is_none() { - anyhow::bail!("compositor does not expose ext_foreign_toplevel_list_v1"); - } - for _ in 0..3 { - queue.roundtrip(&mut state)?; - } - - let mut native = Vec::with_capacity(state.toplevels.len()); - for record in state.toplevels.into_values() { - if record.identifier.is_empty() { - tracing::warn!("ignoring ext toplevel whose done batch had no identifier"); - continue; - } - native.push((stable_id(&record.identifier)?, record)); - } - native.sort_unstable_by_key(|(id, _)| *id); - Ok(merge_atspi_records( - native, - crate::atspi::list_windows(None), - )) -} - -fn merge_atspi_records( - native: Vec<(u64, ToplevelRecord)>, - atspi: Vec, -) -> Vec { - let mut claimed = vec![false; atspi.len()]; - let mut out = Vec::with_capacity(native.len() + atspi.len()); - - for (id, record) in native { - let exact_title = atspi.iter().enumerate().find_map(|(index, window)| { - (!claimed[index] && !record.title.is_empty() && window.title == record.title) - .then_some(index) - }); - let matching_atspi = exact_title.or_else(|| { - unique_match(&atspi, &claimed, |window| { - !record.app_id.is_empty() && window.app_name == record.app_id - }) - }); - - let mut window = WindowInfo { - xid: id, - pid: None, - app_name: record.app_id, - title: record.title, - is_on_screen: true, - z_index: None, - x: 0, - y: 0, - width: 0, - height: 0, - }; - if let Some(index) = matching_atspi { - claimed[index] = true; - let enrichment = &atspi[index]; - window.pid = enrichment.pid; - window.is_on_screen = enrichment.is_on_screen; - window.z_index = enrichment.z_index; - window.x = enrichment.x; - window.y = enrichment.y; - window.width = enrichment.width; - window.height = enrichment.height; - if window.app_name.is_empty() { - window.app_name = enrichment.app_name.clone(); - } - if window.title.is_empty() { - window.title = enrichment.title.clone(); - } - } - out.push(window); - } - - out.extend( - atspi - .into_iter() - .enumerate() - .filter_map(|(index, window)| (!claimed[index]).then_some(window)), - ); - out -} - -fn unique_match( - windows: &[WindowInfo], - claimed: &[bool], - predicate: impl Fn(&WindowInfo) -> bool, -) -> Option { - let mut matches = windows - .iter() - .enumerate() - .filter(|(index, window)| !claimed[*index] && predicate(window)); - let (index, _) = matches.next()?; - matches.next().is_none().then_some(index) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn record(identifier: &str, title: &str, app_id: &str) -> ToplevelRecord { - ToplevelRecord { - identifier: identifier.to_owned(), - title: title.to_owned(), - app_id: app_id.to_owned(), - } - } - - fn atspi(xid: u64, pid: u32, title: &str, app_name: &str, x: i32) -> WindowInfo { - WindowInfo { - xid, - pid: Some(pid), - app_name: app_name.to_owned(), - title: title.to_owned(), - is_on_screen: true, - z_index: None, - x, - y: 20, - width: 800, - height: 600, - } - } - - #[test] - fn id_registry_is_stable_and_bijective() { - let mut registry = IdRegistry::default(); - let first = registry.id_for("opaque-a").unwrap(); - let second = registry.id_for("opaque-b").unwrap(); - assert_eq!(registry.id_for("opaque-a").unwrap(), first); - assert_ne!(first, second); - assert_eq!( - registry - .by_id - .get(&u32::try_from(first).unwrap()) - .map(String::as_str), - Some("opaque-a") - ); - assert_eq!( - registry - .by_id - .get(&u32::try_from(second).unwrap()) - .map(String::as_str), - Some("opaque-b") - ); - } - - #[test] - fn merge_recovers_metadata_and_preserves_unmatched_records() { - let native = vec![ - ( - u64::from(EXT_ID_NAMESPACE_START), - record("a", "Editor", "org.editor"), - ), - ( - u64::from(EXT_ID_NAMESPACE_START + 1), - record("b", "Other", "org.viewer"), - ), - ( - u64::from(EXT_ID_NAMESPACE_START + 2), - record("c", "Native only", "org.native"), - ), - ]; - let merged = merge_atspi_records( - native, - vec![ - atspi(11, 101, "Editor", "unrelated-name", 10), - atspi(12, 202, "Different title", "org.viewer", 30), - atspi(13, 303, "AT-SPI only", "org.extra", 50), - ], - ); - - assert_eq!(merged.len(), 4); - assert_eq!( - (merged[0].xid, merged[0].pid, merged[0].x), - (u64::from(EXT_ID_NAMESPACE_START), Some(101), 10) - ); - assert_eq!(merged[1].pid, Some(202)); - assert_eq!(merged[2].pid, None); - assert_eq!((merged[3].xid, merged[3].pid), (13, Some(303))); - } - - #[test] - fn merge_does_not_guess_ambiguous_app_ids() { - let merged = merge_atspi_records( - vec![( - u64::from(EXT_ID_NAMESPACE_START), - record("a", "", "org.same"), - )], - vec![ - atspi(11, 101, "One", "org.same", 10), - atspi(12, 202, "Two", "org.same", 30), - ], - ); - assert_eq!(merged.len(), 3); - assert_eq!(merged[0].pid, None); - assert_eq!(merged[1].pid, Some(101)); - assert_eq!(merged[2].pid, Some(202)); - } - - #[test] - fn merge_pairs_repeated_exact_titles_one_to_one() { - let merged = merge_atspi_records( - vec![ - ( - u64::from(EXT_ID_NAMESPACE_START), - record("a", "Document", ""), - ), - ( - u64::from(EXT_ID_NAMESPACE_START + 1), - record("b", "Document", ""), - ), - ], - vec![ - atspi(11, 101, "Document", "first", 10), - atspi(12, 202, "Document", "second", 30), - ], - ); - assert_eq!(merged.len(), 2); - assert_eq!(merged[0].pid, Some(101)); - assert_eq!(merged[1].pid, Some(202)); - } - - #[test] - fn id_registry_skips_collisions_and_reports_exhaustion() { - let mut registry = IdRegistry::default(); - registry - .by_id - .insert(EXT_ID_NAMESPACE_START, "already-used".to_owned()); - assert_eq!( - registry.id_for("next").unwrap(), - u64::from(EXT_ID_NAMESPACE_START + 1) - ); - - registry.next = Some(EXT_ID_NAMESPACE_END); - assert_eq!( - registry.id_for("last").unwrap(), - u64::from(EXT_ID_NAMESPACE_END) - ); - assert!(registry.id_for("exhausted").is_err()); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs deleted file mode 100644 index 6933a64eb72..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs +++ /dev/null @@ -1,1421 +0,0 @@ -//! libei-based input synthesis via the `reis` crate + xdg-desktop-portal -//! RemoteDesktop. -//! -//! Reaches GNOME/Mutter (47+, mutter shipped EIS in 45, ei_text in 46) -//! and KDE/KWin (Plasma 6.0+, ei_text in 6.1) — the cross-DE input -//! complement that doesn't depend on the wlroots-only -//! `zwlr_virtual_pointer_v1` / `zwlr_virtual_keyboard_v1` protocols. -//! -//! Sequence: -//! 1. `ei::Context::connect_to_env()` — fast path when a compositor or test -//! environment already exports `$LIBEI_SOCKET`. -//! 2. Fallback: ashpd `RemoteDesktop::create_session` → -//! `select_devices(KEYBOARD|POINTER)` → -//! `start(session, parent)` (user consent dialog the first time) → -//! `connect_to_eis()` returns a Unix fd → `ei::Context::new(fd)`. -//! 3. Worker thread runs a calloop event loop on the ei::Context; -//! handshake → connection.seat events → seat.bind(capabilities) → -//! device.frame + emulate events. -//! 4. Public API sends commands over a crossbeam-channel and blocks -//! until the worker reports the request was flushed to the EIS -//! server. -//! -//! Persistence: ashpd `PersistMode::ExplicitlyRevoked` keeps the user's consent -//! until they revoke it in desktop settings. The restore token is stored at -//! `~/.config/cua-driver/libei-persistent.token` and reused across daemon -//! restarts. -//! -//! Coordinates: ei_pointer_absolute uses LOGICAL PIXELS inside an -//! announced ei_device.Region — collected between device creation and -//! the first device.Done event. Multi-monitor: pick the region -//! containing the target (x, y). - -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::OnceLock; -use std::thread; - -use crossbeam_channel::{bounded, Receiver, Sender}; -use xkbcommon::xkb; - -/// Buttons the public API exposes. Mapped to evdev codes in the worker -/// thread so the libei surface only sees evdev integers. -#[derive(Debug, Clone, Copy)] -pub enum Button { - Left, - Right, - Middle, -} - -impl Button { - fn to_evdev(self) -> u32 { - match self { - Button::Left => 0x110, // BTN_LEFT - Button::Right => 0x111, // BTN_RIGHT - Button::Middle => 0x112, // BTN_MIDDLE - } - } -} - -/// One evdev keyboard state transition in a [`key_sequence`] request. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum KeyTransition { - Press(u32), - Release(u32), -} - -const MAX_KEY_SEQUENCE_TRANSITIONS: usize = 64; - -/// EIS frame timestamps are CLOCK_MONOTONIC microseconds, not per-command -/// sequence numbers. Mutter discards transactions carrying the old 0,1,2... -/// placeholders. Keep them strictly increasing even when several frames are -/// emitted inside one microsecond. -fn event_time_us() -> u64 { - use std::sync::atomic::{AtomicU64, Ordering}; - - static LAST: AtomicU64 = AtomicU64::new(0); - let mut ts = unsafe { - let mut value: libc::timespec = std::mem::zeroed(); - if libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut value) == 0 { - (value.tv_sec as u64) - .saturating_mul(1_000_000) - .saturating_add((value.tv_nsec as u64) / 1_000) - } else { - 0 - } - }; - loop { - let previous = LAST.load(Ordering::Relaxed); - ts = ts.max(previous.saturating_add(1)); - if LAST - .compare_exchange_weak(previous, ts, Ordering::Relaxed, Ordering::Relaxed) - .is_ok() - { - return ts; - } - } -} - -/// Commands the worker thread accepts. Each carries a reply channel so -/// the caller blocks until the EIS server has received the event. -enum Cmd { - WaitReady { - interface: &'static str, - reply: Sender>, - }, - Click { - x: f64, - y: f64, - button: Button, - reply: Sender>, - }, - MoveAbsolute { - x: f64, - y: f64, - reply: Sender>, - }, - Scroll { - dx: f64, - dy: f64, - reply: Sender>, - }, - TypeText { - text: String, - reply: Sender>, - }, - PressKey { - keycode: u32, - reply: Sender>, - }, - KeySequence { - transitions: Vec, - reply: Sender>, - }, - Drag { - from_x: f64, - from_y: f64, - to_x: f64, - to_y: f64, - steps: u32, - duration_ms: u64, - button: Button, - reply: Sender>, - }, - Shutdown, -} - -static TX: OnceLock> = OnceLock::new(); - -fn tx() -> anyhow::Result<&'static Sender> { - TX.get() - .ok_or_else(|| anyhow::anyhow!("libei worker not started; call ensure_started() first")) -} - -fn wait_for_reply(rx: Receiver>) -> anyhow::Result<()> { - rx.recv_timeout(std::time::Duration::from_secs(20)) - .map_err(|error| match error { - crossbeam_channel::RecvTimeoutError::Timeout => anyhow::anyhow!( - "libei input backend did not become ready within 20s; the desktop portal may be waiting for Remote Desktop consent or its EIS session may be wedged" - ), - crossbeam_channel::RecvTimeoutError::Disconnected => { - anyhow::anyhow!("libei worker reply channel closed") - } - })? -} - -fn restore_token_path() -> Option { - let base = dirs::config_dir()?; - Some(base.join("qwen-cua-driver").join("libei-persistent.token")) -} - -fn read_restore_token() -> Option { - let path = restore_token_path()?; - std::fs::read_to_string(path) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -fn write_restore_token(token: &str) -> anyhow::Result<()> { - let path = restore_token_path() - .ok_or_else(|| anyhow::anyhow!("no config dir available for libei restore_token"))?; - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).map_err(|e| { - anyhow::anyhow!( - "failed to create {} for restore_token: {e}", - parent.display() - ) - })?; - } - use std::io::Write; - use std::os::unix::fs::OpenOptionsExt; - - let mut file = std::fs::OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .mode(0o600) - .open(&path) - .map_err(|e| { - anyhow::anyhow!( - "failed to open libei restore_token at {}: {e}", - path.display() - ) - })?; - file.write_all(token.as_bytes()).map_err(|e| { - anyhow::anyhow!( - "failed to write libei restore_token to {}: {e}", - path.display() - ) - }) -} - -/// Spawn the libei worker thread (idempotent — safe to call from every -/// MCP tool invocation; subsequent calls are no-ops). -pub fn ensure_started() -> anyhow::Result<()> { - TX.get_or_init(|| { - let (tx, rx) = bounded::(64); - thread::Builder::new() - .name("cua-libei-worker".into()) - .spawn(move || { - if let Err(e) = worker(rx) { - tracing::warn!("cua-libei-worker exited with error: {e}"); - } - }) - .expect("spawn cua-libei-worker thread"); - tx - }); - Ok(()) -} - -fn wait_until_ready(interface: &'static str) -> anyhow::Result<()> { - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::WaitReady { - interface, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -/// Negotiate a resumed absolute-pointer device without emitting input. -/// Callers can then restore target activation after a portal consent dialog. -pub fn wait_pointer_ready() -> anyhow::Result<()> { - wait_until_ready("ei_pointer_absolute") -} - -/// Negotiate a resumed scroll device without emitting input. -pub fn wait_scroll_ready() -> anyhow::Result<()> { - wait_until_ready("ei_scroll") -} - -/// Negotiate a resumed keyboard device without emitting input. -pub fn wait_keyboard_ready() -> anyhow::Result<()> { - wait_until_ready("ei_keyboard") -} - -// ── Public API ─────────────────────────────────────────────────────────── - -/// Move the cursor to absolute (x, y) within the announced device region, -/// then press + release `button`. Blocks until the EIS server has -/// received the event sequence. -pub fn click(x: f64, y: f64, button: Button) -> anyhow::Result<()> { - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::Click { - x, - y, - button, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -/// Move the cursor to absolute (x, y) inside the device region (no -/// button press). -pub fn move_absolute(x: f64, y: f64) -> anyhow::Result<()> { - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::MoveAbsolute { x, y, reply: tx_r }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -/// Scroll by (dx, dy) logical units. Positive y scrolls down. -pub fn scroll(dx: f64, dy: f64) -> anyhow::Result<()> { - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::Scroll { - dx, - dy, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -/// Inject a UTF-8 string via the `ei_text` interface (libei 1.6+). -/// Bypasses keycode reverse-mapping — works correctly for any layout. -pub fn type_text(text: &str) -> anyhow::Result<()> { - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::TypeText { - text: text.to_string(), - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -/// Press + release a key by evdev code (e.g. `KEY_ENTER` = 28). For -/// modifier combos use the keyboard interface's modifier state explicitly. -pub fn press_key(keycode: u32) -> anyhow::Result<()> { - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::PressKey { - keycode, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -/// Submit an ordered sequence of evdev key press/release transitions. -/// -/// The worker emits every transition in one emulation session and frames each -/// transition separately, allowing callers to hold modifiers while pressing a -/// key. Requests are capped to keep the synchronous worker responsive. -pub fn key_sequence(transitions: &[KeyTransition]) -> anyhow::Result<()> { - validate_key_sequence(transitions)?; - if transitions.is_empty() { - return Ok(()); - } - - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::KeySequence { - transitions: transitions.to_vec(), - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -fn validate_key_sequence(transitions: &[KeyTransition]) -> anyhow::Result<()> { - if transitions.len() > MAX_KEY_SEQUENCE_TRANSITIONS { - anyhow::bail!( - "libei key sequence has {} transitions; maximum is {}", - transitions.len(), - MAX_KEY_SEQUENCE_TRANSITIONS - ); - } - Ok(()) -} - -fn emit_key_transitions(transitions: &[KeyTransition], mut emit: impl FnMut(KeyTransition, u64)) { - for (frame, transition) in transitions.iter().copied().enumerate() { - emit(transition, frame as u64); - } -} - -/// Press `button` at (from_x, from_y), move through `steps` interpolated points -/// to (to_x, to_y), then release — a genuine button-held drag (text selection / -/// drag-and-drop / slider). Coordinates are absolute within the announced -/// device region, like [`click`]. -pub fn drag( - from_x: f64, - from_y: f64, - to_x: f64, - to_y: f64, - steps: u32, - duration_ms: u64, - button: Button, -) -> anyhow::Result<()> { - ensure_started()?; - let (tx_r, rx_r) = bounded(1); - tx()? - .send(Cmd::Drag { - from_x, - from_y, - to_x, - to_y, - steps, - duration_ms, - button, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - wait_for_reply(rx_r) -} - -/// Cleanly stop the worker thread. -pub fn shutdown() { - if let Some(tx) = TX.get() { - let _ = tx.send(Cmd::Shutdown); - } -} - -// ── Worker thread ──────────────────────────────────────────────────────── -// -// The worker owns the `reis::ei::Context`, the per-seat device map, and -// the negotiated input region. It runs a calloop event loop that handles -// both the EIS protocol and the inbound command channel. - -/// Owns the resources that must outlive a single libei call so the portal -/// RemoteDesktop + EIS session stays alive for the whole worker-thread lifetime. -/// -/// GNOME/Mutter (and KDE) tie the RemoteDesktop session — and therefore the -/// handed-off EIS fd — to the D-Bus connection that created it. The previous -/// code dropped the ashpd proxy/session (and the tokio runtime backing their -/// zbus connection) at the end of `open_eis_context`, before the calloop loop -/// even started, so the session died immediately on those compositors (#2105). -/// Holding these for the worker's lifetime keeps the connection — and the -/// session — open. The direct `$LIBEI_SOCKET` fast path has no portal -/// session, so it carries `None`. -#[allow(dead_code)] // fields are keep-alive only; never read -enum PortalKeepAlive { - None, - Portal { - _rt: tokio::runtime::Runtime, - _proxy: ashpd::desktop::remote_desktop::RemoteDesktop, - _session: ashpd::desktop::Session, - }, -} - -fn worker(rx: Receiver) -> anyhow::Result<()> { - // Phase 1 — acquire an EIS context (plus, on the portal path, a keep-alive - // that owns the ashpd RemoteDesktop session + its tokio runtime). - let (context, _portal_keepalive) = - open_eis_context().map_err(|e| anyhow::anyhow!("failed to obtain EIS connection: {e}"))?; - - // Phase 2 — handshake. The reis API requires we call context.handshake() - // and flush before the event loop starts polling. - let _handshake = context.handshake(); - let _ = context.flush(); - - // Phase 3 — run the calloop event loop. We use a calloop Generic source - // wrapping the ei::Context's underlying socket so the loop wakes up on - // EIS messages. The command channel is polled at the top of each loop - // iteration. `_portal_keepalive` stays bound until this fn returns — i.e. - // across the entire loop — so the portal session is never dropped early (#2105). - run_calloop(context, rx) -} - -/// Open the EIS context. Fast path: `$LIBEI_SOCKET` supplied by the environment. -/// Fallback: ashpd portal RemoteDesktop handshake. -fn open_eis_context() -> anyhow::Result<(reis::ei::Context, PortalKeepAlive)> { - if let Some(ctx) = reis::ei::Context::connect_to_env() - .map_err(|e| anyhow::anyhow!("env ei socket open failed: {e}"))? - { - // Direct $LIBEI_SOCKET fast path: no portal session to keep alive. - return Ok((ctx, PortalKeepAlive::None)); - } - - // Portal RemoteDesktop fallback. - use ashpd::desktop::{ - remote_desktop::{ - ConnectToEISOptions, DeviceType, RemoteDesktop, SelectDevicesOptions, StartOptions, - }, - CreateSessionOptions, PersistMode, - }; - use ashpd::enumflags2::BitFlags; - use std::os::unix::net::UnixStream; - - // A multi-threaded runtime (1 worker) so the zbus connection backing the - // RemoteDesktop session keeps being driven after this fn returns. The - // session must stay live for the whole worker lifetime (#2105); a - // current-thread runtime would freeze the connection the moment we stop - // calling `block_on`, and GNOME/KDE would tear the session down. - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .build() - .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?; - - let (fd, proxy, session) = rt.block_on(async { - let connection = super::portal::fresh_session_connection().await?; - let proxy = RemoteDesktop::with_connection(connection) - .await - .map_err(|e| anyhow::anyhow!("portal RemoteDesktop proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?; - let session = proxy - .create_session(CreateSessionOptions::default()) - .await - .map_err(|e| anyhow::anyhow!("portal create_session failed: {e}"))?; - - let mut select_opts = SelectDevicesOptions::default() - .set_devices(BitFlags::::from(DeviceType::Keyboard) | DeviceType::Pointer) - .set_persist_mode(PersistMode::ExplicitlyRevoked); - if let Some(tok) = read_restore_token() { - select_opts = select_opts.set_restore_token(Some(tok.as_str())); - } - - proxy - .select_devices(&session, select_opts) - .await - .map_err(|e| anyhow::anyhow!("portal select_devices failed: {e}"))? - .response() - .map_err(|e| anyhow::anyhow!("portal select_devices response error: {e}"))?; - - let started = proxy - .start(&session, None, StartOptions::default()) - .await - .map_err(|e| anyhow::anyhow!("portal start failed: {e}"))? - .response() - .map_err(|e| anyhow::anyhow!("portal start response error (user denied?): {e}"))?; - - // Best-effort persist of the restore_token. Without this, every - // process restart re-prompts the user for consent — with it, ashpd - // 0.13's explicitly-revoked mode and the stored token together - // skip the dialog for the lifetime of the persistence grant - // until the user explicitly revokes the grant in desktop settings. - if let Some(tok) = started.restore_token() { - let _ = write_restore_token(tok); - } - - let fd = proxy - .connect_to_eis(&session, ConnectToEISOptions::default()) - .await - .map_err(|e| anyhow::anyhow!("portal connect_to_eis failed: {e}"))?; - anyhow::Ok((fd, proxy, session)) - })?; - - let stream = UnixStream::from(fd); - let ctx = reis::ei::Context::new(stream) - .map_err(|e| anyhow::anyhow!("reis ei::Context::new failed: {e}"))?; - Ok(( - ctx, - PortalKeepAlive::Portal { - _rt: rt, - _proxy: proxy, - _session: session, - }, - )) -} - -// ── calloop dispatch ──────────────────────────────────────────────────── -// -// The EIS protocol is event-driven and non-trivial — handshake → connection -// → seat → device → frame negotiation. We model the worker as a calloop -// EventLoop driving the ei::Context Generic source, with a separate -// channel source for inbound Cmd messages. -// -// For v1 we implement the handshake + seat-bind dance and provide -// click/move/type via the negotiated pointer/keyboard/text interfaces. -// Region selection picks the first announced ei_device::Region. - -fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result<()> { - use calloop::{generic::Generic, EventLoop, Interest, Mode}; - - let mut event_loop: EventLoop = EventLoop::try_new() - .map_err(|e| anyhow::anyhow!("calloop EventLoop::try_new failed: {e}"))?; - let handle = event_loop.handle(); - - // EIS protocol source. - let ctx_source = Generic::new(context, Interest::READ, Mode::Level); - handle - .insert_source(ctx_source, |_event, ctx, state: &mut EisState| { - state.handle_eis_readable(unsafe { ctx.get_mut() }) - }) - .map_err(|e| anyhow::anyhow!("calloop insert_source(eis) failed: {e}"))?; - - let mut state = EisState::default(); - - // Inbound commands can arrive before the EIS handshake has negotiated a - // usable input device — seat → device → interface → Resumed is async and - // only advances while `dispatch` runs, and the portal consent-and-connect - // path (#2105) returns before the device is live. Running a command against - // a not-yet-negotiated device fails with "no EIS device negotiated yet". So - // queue commands and run each only once `input_ready()`; fail any that wait - // longer than READY_TIMEOUT (handle_command then replies with the natural - // "no device" error) so a caller blocked on the reply never hangs forever. - let mut pending: Vec<(Cmd, std::time::Instant)> = Vec::new(); - const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); - - loop { - match rx.try_recv() { - Ok(Cmd::Shutdown) => break, - Ok(cmd) => pending.push((cmd, std::time::Instant::now())), - Err(crossbeam_channel::TryRecvError::Empty) => {} - Err(crossbeam_channel::TryRecvError::Disconnected) => break, - } - - event_loop - .dispatch(Some(std::time::Duration::from_millis(20)), &mut state) - .map_err(|e| anyhow::anyhow!("calloop dispatch failed: {e}"))?; - - // Run each queued command once the device IT needs is negotiated; time - // stale ones out. Per-command (not one global gate) so a keyboard-only - // session doesn't make keyboard commands wait 20s for a pointer device. - if !pending.is_empty() { - let mut still = Vec::with_capacity(pending.len()); - for (cmd, queued) in pending.drain(..) { - if state.cmd_ready(&cmd) || queued.elapsed() >= READY_TIMEOUT { - state.handle_command(cmd); - } else { - still.push((cmd, queued)); - } - } - pending = still; - } - - // The handle_command path may have queued frame() requests; flush - // them once per loop iteration so the EIS server sees them. - if let Some(ctx) = state.context.as_ref() { - let _ = ctx.flush(); - } - } - - Ok(()) -} - -#[derive(Default)] -struct EisState { - context: Option, - seats: HashMap, - devices: HashMap, - sequence: u32, - /// The latest serial emitted by the EIS connection. Request serials are - /// connection-global (libei's `ei_get_serial`), while readiness is tracked - /// per device below. - last_serial: u32, - /// Last device lifecycle event. Mutter 46 may advertise a provisional - /// absolute pointer before seat binding, then a second live device. Waiting - /// briefly for this stream to settle avoids dispatching to the transient one. - last_device_event: Option, - /// `char -> (evdev keycode, needs_shift)` built from the compositor's active - /// xkb keymap (delivered by the `ei_keyboard` device). Lets keycode typing - /// follow the user's real layout instead of assuming US. Empty until the - /// `Keymap` event arrives; typing then falls back to [`char_to_evdev`]. - keymap_chars: HashMap, - /// evdev keycode that produces `Shift_L` in the active keymap (else 42). - keymap_shift: Option, -} - -#[derive(Default)] -struct SeatData { - capabilities: HashMap, -} - -/// One announced ei_device::Region — a single monitor's pixel rectangle -/// inside the seat's logical coordinate space. Multi-monitor sessions -/// announce one Region per output; we route each absolute (x, y) to the -/// region containing it so the cursor lands on the correct monitor. -#[derive(Clone, Copy, Debug)] -struct RegionRect { - offset_x: f32, - offset_y: f32, - width: f32, - height: f32, -} - -impl RegionRect { - fn contains(&self, x: f64, y: f64) -> bool { - let xf = x as f32; - let yf = y as f32; - xf >= self.offset_x - && xf < self.offset_x + self.width - && yf >= self.offset_y - && yf < self.offset_y + self.height - } -} - -#[derive(Default)] -struct DeviceData { - device_type: Option, - /// Every Region the device announces between its creation and its - /// first Done event. Empty until the EIS server has sent at least - /// one Region. - regions: Vec, - interfaces: HashMap, - resumed: bool, - resumed_serial: Option, -} - -impl EisState { - fn handle_eis_readable( - &mut self, - context: &mut reis::ei::Context, - ) -> std::io::Result { - use reis::PendingRequestResult; - if self.context.is_none() { - self.context = Some(context.clone()); - } - if context.read().is_err() { - return Ok(calloop::PostAction::Remove); - } - while let Some(result) = context.pending_event() { - let request = match result { - PendingRequestResult::Request(r) => r, - _ => continue, - }; - self.dispatch_eis_event(request); - } - let _ = context.flush(); - Ok(calloop::PostAction::Continue) - } - - fn dispatch_eis_event(&mut self, event: reis::ei::Event) { - use reis::ei::Event; - match event { - Event::Handshake(handshake, ev) => { - if let reis::ei::handshake::Event::HandshakeVersion { .. } = ev { - handshake.handshake_version(1); - handshake.name("cua-driver"); - handshake.context_type(reis::ei::handshake::ContextType::Sender); - // Advertise the interface set + versions that reis 0.7's own - // EIS client examples use against real compositors. The - // previous list advertised ei_device v2 and omitted - // ei_pingpong, which Mutter rejects — it closes the EIS - // connection immediately after our handshake `finish()` - // (observed: one Handshake event, then read() errors). reis's - // handshaker also requires ei_pingpong / ei_callback / - // ei_connection. See reis 0.7 examples/type-text.rs. - for (iface, ver) in [ - ("ei_callback", 1u32), - ("ei_connection", 1), - ("ei_pingpong", 1), - ("ei_seat", 1), - ("ei_device", 1), - ("ei_pointer", 1), - ("ei_pointer_absolute", 1), - ("ei_button", 1), - ("ei_scroll", 1), - ("ei_keyboard", 1), - ("ei_text", 1), - ] { - handshake.interface_version(iface, ver); - } - handshake.finish(); - } - } - Event::Connection(_conn, ev) => match ev { - reis::ei::connection::Event::Seat { seat } => { - self.seats.insert(seat, SeatData::default()); - } - // The EIS server pings to check liveness; failing to answer - // makes it drop the connection. Mirror reis's examples. - reis::ei::connection::Event::Ping { ping } => { - ping.done(0); - } - _ => {} - }, - Event::Seat(seat, ev) => { - let data = self.seats.entry(seat.clone()).or_default(); - match ev { - reis::ei::seat::Event::Capability { mask, interface } => { - data.capabilities.insert(interface, mask); - } - reis::ei::seat::Event::Done => { - // Bind every input capability the seat advertises - // (pointer / pointer_absolute / button / scroll / - // keyboard / text). The server then announces - // matching devices. - let mut mask = 0u64; - for k in [ - "ei_pointer", - "ei_pointer_absolute", - "ei_button", - "ei_scroll", - "ei_keyboard", - "ei_text", - ] { - if let Some(m) = data.capabilities.get(k) { - mask |= *m; - } - } - if mask != 0 { - seat.bind(mask); - } - } - reis::ei::seat::Event::Device { device } => { - self.devices.insert(device, DeviceData::default()); - } - _ => {} - } - } - Event::Device(device, ev) => { - self.last_device_event = Some(std::time::Instant::now()); - if let reis::ei::device::Event::Destroyed { serial } = &ev { - self.last_serial = *serial; - self.devices.remove(&device); - return; - } - let data = self.devices.entry(device.clone()).or_default(); - match ev { - reis::ei::device::Event::DeviceType { device_type } => { - data.device_type = Some(device_type); - } - reis::ei::device::Event::Interface { object } => { - data.interfaces - .insert(object.interface().to_owned(), object); - } - reis::ei::device::Event::Region { - offset_x, - offset_y, - width, - hight, - scale: _, - } => { - // reis 0.7 keeps the misspelled "hight" field name - // for ABI compatibility — see the protocol comment. - if width > 0 && hight > 0 { - data.regions.push(RegionRect { - offset_x: offset_x as f32, - offset_y: offset_y as f32, - width: width as f32, - height: hight as f32, - }); - } - } - reis::ei::device::Event::Resumed { serial } => { - data.resumed = true; - data.resumed_serial = Some(serial); - self.last_serial = serial; - // A RemoteDesktop session is one emulation transaction, - // not one transaction per click. Mutter can neutralize a - // device stopped in the same batch as its frame. Match - // libei's reference client: start on Resumed and keep it - // active until the device is paused or disconnected. - self.sequence = self.sequence.wrapping_add(1); - device.start_emulating(serial, self.sequence); - } - reis::ei::device::Event::Paused { serial } => { - data.resumed = false; - data.resumed_serial = None; - self.last_serial = serial; - } - _ => {} - } - } - Event::Keyboard( - _kb, - reis::ei::keyboard::Event::Keymap { - keymap_type, - size, - keymap, - }, - ) => { - if keymap_type == reis::ei::keyboard::KeymapType::Xkb { - self.load_xkb_keymap(keymap, size); - } - } - _ => {} - } - } - - /// Compile the compositor's xkb keymap (from the `ei_keyboard` `Keymap` fd) - /// into a `char -> (evdev keycode, shift)` table + the Shift keycode, so - /// keycode typing follows the user's real layout instead of assuming US. - /// Best-effort: on any failure the table stays empty and typing falls back - /// to [`char_to_evdev`]. - fn load_xkb_keymap(&mut self, fd: std::os::unix::io::OwnedFd, size: u32) { - use std::io::{Read, Seek, SeekFrom}; - const XKB_EVDEV_OFFSET: u32 = 8; - const SHIFT_L: u32 = 0xffe1; - - // Sanity-cap the size from the (trusted, but possibly buggy) compositor - // event: real xkb keymaps are a few KiB; refuse absurd values rather - // than attempt a multi-GB allocation. - if size == 0 || size as usize > 16 * 1024 * 1024 { - return; - } - let mut file = std::fs::File::from(fd); - let mut buf = vec![0u8; size as usize]; - let _ = file.seek(SeekFrom::Start(0)); - if file.read_exact(&mut buf).is_err() { - return; - } - // The keymap buffer is NUL-terminated text. - let text = match buf.iter().position(|&b| b == 0) { - Some(n) => String::from_utf8_lossy(&buf[..n]).into_owned(), - None => String::from_utf8_lossy(&buf).into_owned(), - }; - let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); - let keymap = match xkb::Keymap::new_from_string( - &ctx, - text, - xkb::KEYMAP_FORMAT_TEXT_V1, - xkb::KEYMAP_COMPILE_NO_FLAGS, - ) { - Some(k) => k, - None => return, - }; - - let mut chars: HashMap = HashMap::new(); - let mut shift = None; - for kc in keymap.min_keycode().raw()..=keymap.max_keycode().raw() { - let keycode = xkb::Keycode::new(kc); - let evdev = kc.wrapping_sub(XKB_EVDEV_OFFSET); - // Level 0 = unshifted, 1 = shifted; scan 0 first so the unshifted - // mapping wins when a char is reachable both ways. - for level in 0..=1u32 { - for sym in keymap.key_get_syms_by_level(keycode, 0, level) { - if shift.is_none() && sym.raw() == SHIFT_L { - shift = Some(evdev); - } - if let Some(ch) = sym.key_char() { - chars.entry(ch).or_insert((evdev, level == 1)); - } - } - } - } - self.keymap_chars = chars; - self.keymap_shift = shift; - } - - /// True once the device THIS command needs has been negotiated and - /// `Resumed`. Gating per-command (rather than on one global pointer check) - /// means a keyboard-only session doesn't stall keyboard commands waiting for - /// an absolute-pointer device that never arrives — and vice versa. Commands - /// run before their device exists fail with "no EIS device negotiated yet", - /// so `run_calloop` queues them until this holds (or READY_TIMEOUT elapses). - fn cmd_ready(&self, cmd: &Cmd) -> bool { - const DEVICE_QUIET_PERIOD: std::time::Duration = std::time::Duration::from_millis(200); - if self - .last_device_event - .is_none_or(|last| last.elapsed() < DEVICE_QUIET_PERIOD) - { - return false; - } - match cmd { - Cmd::WaitReady { interface, .. } => self.device_with_interface(interface).is_some(), - Cmd::Click { .. } | Cmd::MoveAbsolute { .. } | Cmd::Drag { .. } => { - self.device_with_interface("ei_pointer_absolute").is_some() - } - Cmd::Scroll { .. } => self.device_with_interface("ei_scroll").is_some(), - Cmd::PressKey { .. } | Cmd::KeySequence { .. } => { - self.device_with_interface("ei_keyboard").is_some() - } - Cmd::TypeText { .. } => { - self.device_with_interface("ei_text").is_some() - || self.device_with_interface("ei_keyboard").is_some() - } - Cmd::Shutdown => true, - } - } - - fn handle_command(&mut self, cmd: Cmd) { - let result = self.run_command(&cmd); - // Send the reply on whichever channel this command carries. - match cmd { - Cmd::WaitReady { reply, .. } => { - let _ = reply.send(result); - } - Cmd::Click { reply, .. } => { - let _ = reply.send(result); - } - Cmd::MoveAbsolute { reply, .. } => { - let _ = reply.send(result); - } - Cmd::Scroll { reply, .. } => { - let _ = reply.send(result); - } - Cmd::TypeText { reply, .. } => { - let _ = reply.send(result); - } - Cmd::PressKey { reply, .. } => { - let _ = reply.send(result); - } - Cmd::KeySequence { reply, .. } => { - let _ = reply.send(result); - } - Cmd::Drag { reply, .. } => { - let _ = reply.send(result); - } - Cmd::Shutdown => {} - } - } - - fn run_command(&mut self, cmd: &Cmd) -> anyhow::Result<()> { - match cmd { - Cmd::WaitReady { interface, .. } => { - self.device_with_interface(interface).ok_or_else(|| { - anyhow::anyhow!("no resumed EIS {interface} device negotiated within 20s") - })?; - } - Cmd::Click { x, y, button, .. } => { - let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?; - let serial = self.last_serial; - let ptr_abs = - require_device_interface::(&self.devices, &device)?; - let btn = require_device_interface::(&self.devices, &device)?; - - ptr_abs.motion_absolute(rel_x, rel_y); - btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press); - device.frame(serial, event_time_us()); - btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released); - device.frame(serial, event_time_us()); - } - Cmd::Drag { - from_x, - from_y, - to_x, - to_y, - steps, - duration_ms, - button, - .. - } => { - let (device, from_rx, from_ry) = self.pointer_device_for(*from_x, *from_y)?; - let serial = self.last_serial; - let ptr_abs = - require_device_interface::(&self.devices, &device)?; - let btn = require_device_interface::(&self.devices, &device)?; - // Map the drag end into the SAME device's region (a single EIS - // emulation session can't span devices). Fall back to the start - // region's offset when no region of this device contains the end - // (single-region device, or a point just off-screen) — this is - // correct only within one region, but avoids the cross-monitor - // mis-mapping of blindly reusing the start offset. - let (to_rx, to_ry) = - self.region_local_on(&device, *to_x, *to_y) - .unwrap_or_else(|| { - let off_x = *from_x as f32 - from_rx; - let off_y = *from_y as f32 - from_ry; - (*to_x as f32 - off_x, *to_y as f32 - off_y) - }); - - ptr_abs.motion_absolute(from_rx, from_ry); - device.frame(serial, event_time_us()); - btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press); - device.frame(serial, event_time_us()); - if let Some(context) = self.context.as_ref() { - let _ = context.flush(); - } - // Bound the interpolation: run_command executes synchronously in - // the worker loop, so a huge step count would block EIS event - // processing (incl. Ping) and balloon the unflushed request queue. - let n = (*steps).max(1).min(500); - let delay = drag_step_delay(*duration_ms, n); - for s in 1..=n { - let t = s as f32 / n as f32; - let ix = from_rx + (to_rx - from_rx) * t; - let iy = from_ry + (to_ry - from_ry) * t; - ptr_abs.motion_absolute(ix, iy); - device.frame(serial, event_time_us()); - if let Some(context) = self.context.as_ref() { - let _ = context.flush(); - } - if !delay.is_zero() { - std::thread::sleep(delay); - } - } - btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released); - device.frame(serial, event_time_us()); - } - Cmd::MoveAbsolute { x, y, .. } => { - let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?; - let serial = self.last_serial; - let ptr_abs = - require_device_interface::(&self.devices, &device)?; - - ptr_abs.motion_absolute(rel_x, rel_y); - device.frame(serial, event_time_us()); - } - Cmd::Scroll { dx, dy, .. } => { - let device = self.device_with_interface("ei_scroll").ok_or_else(|| { - anyhow::anyhow!("no EIS scroll device negotiated yet — wait for handshake") - })?; - let serial = self.last_serial; - let scroll = require_device_interface::(&self.devices, &device)?; - - scroll.scroll(*dx as f32, *dy as f32); - device.frame(serial, event_time_us()); - } - Cmd::TypeText { text, .. } => { - // Prefer ei_text (libei 1.6+ — a UTF-8 string, layout-correct). - // Mutter does NOT advertise ei_text, so fall back to keycode - // typing via ei_keyboard: map each char to a US-layout evdev - // keycode + shift and emit press/release. Chars with no US-layout - // keycode are skipped (a documented limitation — full layout - // support needs an xkb keymap, like reis's type-text example). - if let Some(device) = self.device_with_interface("ei_text") { - let serial = self.last_serial; - let text_iface = - require_device_interface::(&self.devices, &device)?; - text_iface.utf8(text); - device.frame(serial, event_time_us()); - } else { - use reis::ei::keyboard::KeyState; - let device = self.device_with_interface("ei_keyboard").ok_or_else(|| { - anyhow::anyhow!( - "no EIS keyboard device negotiated yet — wait for handshake" - ) - })?; - let serial = self.last_serial; - let kb = - require_device_interface::(&self.devices, &device)?; - let shift_code = self.keymap_shift.unwrap_or(42); - let mut skipped = 0usize; - for ch in text.chars() { - // Prefer the compositor's actual keymap (layout-correct); - // fall back to the US-layout table if no keymap arrived. - let mapped = self - .keymap_chars - .get(&ch) - .copied() - .or_else(|| char_to_evdev(ch)); - let Some((code, shift)) = mapped else { - // No keycode for this char in the keymap/US table - // (e.g. CJK/emoji via the keycode path). Skip it, but - // don't pretend the whole string was typed. - skipped += 1; - continue; - }; - if shift { - kb.key(shift_code, KeyState::Press); - device.frame(serial, event_time_us()); - } - kb.key(code, KeyState::Press); - device.frame(serial, event_time_us()); - kb.key(code, KeyState::Released); - device.frame(serial, event_time_us()); - if shift { - kb.key(shift_code, KeyState::Released); - device.frame(serial, event_time_us()); - } - } - if skipped > 0 { - tracing::warn!( - "libei keycode typing skipped {skipped} char(s) with no \ - keycode in the active keymap (e.g. non-Latin/emoji) — \ - the ei_text path (libei 1.6+) would type these verbatim" - ); - } - } - } - Cmd::PressKey { keycode, .. } => { - let device = self.device_with_interface("ei_keyboard").ok_or_else(|| { - anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake") - })?; - let serial = self.last_serial; - let kb = require_device_interface::(&self.devices, &device)?; - - kb.key(*keycode, reis::ei::keyboard::KeyState::Press); - device.frame(serial, event_time_us()); - kb.key(*keycode, reis::ei::keyboard::KeyState::Released); - device.frame(serial, event_time_us()); - } - Cmd::KeySequence { transitions, .. } => { - let device = self.device_with_interface("ei_keyboard").ok_or_else(|| { - anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake") - })?; - let serial = self.last_serial; - let kb = require_device_interface::(&self.devices, &device)?; - - emit_key_transitions(transitions, |transition, _frame| { - let (keycode, state) = match transition { - KeyTransition::Press(keycode) => { - (keycode, reis::ei::keyboard::KeyState::Press) - } - KeyTransition::Release(keycode) => { - (keycode, reis::ei::keyboard::KeyState::Released) - } - }; - kb.key(keycode, state); - device.frame(serial, event_time_us()); - }); - } - Cmd::Shutdown => {} - } - if let Some(ctx) = self.context.as_ref() { - let _ = ctx.flush(); - } - Ok(()) - } - - /// Pick a pointer device whose announced Region contains `(x, y)`, - /// and translate the absolute screen coordinates into the device's - /// region-local pixel coordinates. - /// - /// Multi-monitor seats announce one Region per output; routing each - /// (x, y) by containment is what lets a click on monitor #2 actually - /// land on monitor #2. When no device's region contains the point — - /// e.g. coords just past the edge of an output, or a session with - /// a single device that hasn't announced any regions yet — we fall - /// back to the first pointer device with the raw coordinates, which - /// matches the pre-multi-monitor behaviour. - fn pointer_device_for(&self, x: f64, y: f64) -> anyhow::Result<(reis::ei::Device, f32, f32)> { - // Absolute motion needs the device that actually carries - // ei_pointer_absolute — Mutter/KWin split the relative and absolute - // pointers onto separate devices, and only one of them accepts an - // absolute warp. - for (device, data) in self.devices.iter() { - if !data.resumed || !data.interfaces.contains_key("ei_pointer_absolute") { - continue; - } - for region in &data.regions { - if region.contains(x, y) { - let rel_x = x as f32 - region.offset_x; - let rel_y = y as f32 - region.offset_y; - return Ok((device.clone(), rel_x, rel_y)); - } - } - } - // Fallback: first absolute-pointer device, raw coords (single-region). - let device = self - .device_with_interface("ei_pointer_absolute") - .ok_or_else(|| { - anyhow::anyhow!( - "no EIS absolute-pointer device negotiated yet — wait for handshake" - ) - })?; - Ok((device, x as f32, y as f32)) - } - - /// Region-local coordinates for absolute `(x, y)` within one of `device`'s - /// announced regions, or `None` when no region of that device contains the - /// point. Used to map a drag endpoint into the same device the drag start - /// resolved to (a single EIS emulation session can't span devices). - fn region_local_on(&self, device: &reis::ei::Device, x: f64, y: f64) -> Option<(f32, f32)> { - let data = self.devices.get(device)?; - data.regions - .iter() - .find(|r| r.contains(x, y)) - .map(|r| (x as f32 - r.offset_x, y as f32 - r.offset_y)) - } - - /// The negotiated device that exposes `iface` (e.g. `ei_pointer_absolute`, - /// `ei_scroll`, `ei_keyboard`, `ei_text`). Mutter/KWin announce one device - /// per capability group — a relative pointer, an absolute pointer, a - /// keyboard — so a command must pick the device carrying the interface it - /// needs, not just "the first virtual device" (which may be the relative - /// pointer that has no `ei_pointer_absolute`). - fn device_with_interface(&self, iface: &str) -> Option { - self.devices - .iter() - .filter(|(_, data)| data.resumed && data.interfaces.contains_key(iface)) - .max_by_key(|(_, data)| data.resumed_serial.unwrap_or(0)) - .map(|(d, _)| d.clone()) - } -} - -/// Map a character to a US-layout evdev keycode + whether Shift is needed. -/// Used by the keyboard-typing fallback when the compositor's EIS exposes no -/// `ei_text` interface (e.g. Mutter). Returns `None` for characters outside the -/// US-ASCII printable set — those are skipped rather than mistyped. Full -/// layout-correct typing would need an xkb keymap (see reis `type-text` example). -fn char_to_evdev(c: char) -> Option<(u32, bool)> { - // Letter codes follow the QWERTY scancode layout (input-event-codes.h). - fn letter(c: char) -> u32 { - // KEY_A..KEY_Z follow the QWERTY scancode layout, not the alphabet. - match c { - 'a' => 30, - 'b' => 48, - 'c' => 46, - 'd' => 32, - 'e' => 18, - 'f' => 33, - 'g' => 34, - 'h' => 35, - 'i' => 23, - 'j' => 36, - 'k' => 37, - 'l' => 38, - 'm' => 50, - 'n' => 49, - 'o' => 24, - 'p' => 25, - 'q' => 16, - 'r' => 19, - 's' => 31, - 't' => 20, - 'u' => 22, - 'v' => 47, - 'w' => 17, - 'x' => 45, - 'y' => 21, - 'z' => 44, - _ => 0, - } - } - // (keycode, needs_shift). Digit/symbol codes and their shifted variants - // follow the US QWERTY row (input-event-codes.h). - Some(match c { - 'a'..='z' => (letter(c), false), - 'A'..='Z' => (letter(c.to_ascii_lowercase()), true), - '1' => (2, false), - '2' => (3, false), - '3' => (4, false), - '4' => (5, false), - '5' => (6, false), - '6' => (7, false), - '7' => (8, false), - '8' => (9, false), - '9' => (10, false), - '0' => (11, false), - '!' => (2, true), - '@' => (3, true), - '#' => (4, true), - '$' => (5, true), - '%' => (6, true), - '^' => (7, true), - '&' => (8, true), - '*' => (9, true), - '(' => (10, true), - ')' => (11, true), - ' ' => (57, false), - '\n' => (28, false), - '\t' => (15, false), - '-' => (12, false), - '_' => (12, true), - '=' => (13, false), - '+' => (13, true), - '[' => (26, false), - '{' => (26, true), - ']' => (27, false), - '}' => (27, true), - '\\' => (43, false), - '|' => (43, true), - ';' => (39, false), - ':' => (39, true), - '\'' => (40, false), - '"' => (40, true), - '`' => (41, false), - '~' => (41, true), - ',' => (51, false), - '<' => (51, true), - '.' => (52, false), - '>' => (52, true), - '/' => (53, false), - '?' => (53, true), - _ => return None, - }) -} - -fn require_device_interface( - devices: &HashMap, - device: &reis::ei::Device, -) -> anyhow::Result { - device_interface::(devices, device) - .ok_or_else(|| anyhow::anyhow!("EIS device lacks required {} interface", T::NAME)) -} - -fn device_interface( - devices: &HashMap, - device: &reis::ei::Device, -) -> Option { - devices - .get(device)? - .interfaces - .get(T::NAME)? - .clone() - .downcast() -} - -fn drag_step_delay(duration_ms: u64, steps: u32) -> std::time::Duration { - let frames = u64::from(steps.max(1).min(500)); - std::time::Duration::from_micros( - duration_ms - .min(10_000) - .saturating_mul(1_000) - .checked_div(frames) - .unwrap_or(0), - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn key_sequence_accepts_empty_and_maximum_length_requests() { - assert!(validate_key_sequence(&[]).is_ok()); - - let transitions = vec![KeyTransition::Press(29); MAX_KEY_SEQUENCE_TRANSITIONS]; - assert!(validate_key_sequence(&transitions).is_ok()); - } - - #[test] - fn key_sequence_rejects_requests_over_the_worker_bound() { - let transitions = vec![KeyTransition::Release(29); MAX_KEY_SEQUENCE_TRANSITIONS + 1]; - let error = validate_key_sequence(&transitions).unwrap_err(); - - assert_eq!( - error.to_string(), - "libei key sequence has 65 transitions; maximum is 64" - ); - } - - #[test] - fn drag_duration_is_distributed_across_motion_frames() { - let delay = drag_step_delay(500, 30); - assert_eq!(delay, std::time::Duration::from_micros(16_666)); - assert!(delay.saturating_mul(30) <= std::time::Duration::from_millis(500)); - assert_eq!(drag_step_delay(0, 30), std::time::Duration::from_millis(0)); - } - - #[test] - fn key_sequence_preserves_transition_order_and_assigns_frames() { - let transitions = [ - KeyTransition::Press(29), - KeyTransition::Press(46), - KeyTransition::Release(46), - KeyTransition::Release(29), - ]; - let mut emitted = Vec::new(); - - emit_key_transitions(&transitions, |transition, frame| { - emitted.push((transition, frame)); - }); - - assert_eq!( - emitted, - vec![ - (KeyTransition::Press(29), 0), - (KeyTransition::Press(46), 1), - (KeyTransition::Release(46), 2), - (KeyTransition::Release(29), 3), - ] - ); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs deleted file mode 100644 index 78c74694e57..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs +++ /dev/null @@ -1,3496 +0,0 @@ -//! Native-Wayland backend. -//! -//! Used when the experimental backend is enabled under a Wayland compositor. -//! Enumerates toplevels via `zwlr_foreign_toplevel_manager_v1` or the generic -//! staging `ext_foreign_toplevel_list_v1`, captures per-output screenshots via -//! `zwlr_screencopy_manager_v1` + `wl_shm` (native — `grim` remains a -//! fallback), and synthesises pointer / scroll / drag input via -//! `zwlr_virtual_pointer_v1`. Per-window image capture is deferred until -//! `ext-foreign-toplevel-image-capture-source-v1` lands in -//! `wayland-protocols-wlr`; until then `screenshot_window_dispatch` returns a -//! typed error on pure Wayland. - -pub mod ext_screencopy; -pub mod ext_toplevel; -pub mod overlay; -pub mod persistent_vptr; -pub(crate) mod portal; -pub mod portal_screenshot; -pub mod shell_helper; -pub mod sway_ipc; -mod virtual_keyboard; -// RemoteDesktop/libei input is portable and ships in release binaries. -// PipeWire ScreenCast capture remains separately gated for modern/Nix builds. -#[cfg(feature = "portal-input")] -pub mod libei; -#[cfg(feature = "portal-capture")] -pub mod portal_screencast; - -/// Whether the GNOME/KDE RemoteDesktop + libei input backend is compiled in. -pub const PORTAL_INPUT_ENABLED: bool = cfg!(feature = "portal-input"); - -use std::collections::{HashMap, HashSet}; -use std::sync::{Mutex, OnceLock}; - -use wayland_client::{ - event_created_child, - protocol::{ - wl_buffer::WlBuffer, - wl_output::{self, WlOutput}, - wl_pointer::{Axis, AxisSource, ButtonState}, - wl_registry, - wl_seat::WlSeat, - wl_shm::{self, WlShm}, - wl_shm_pool::WlShmPool, - }, - Connection, Dispatch, Proxy, QueueHandle, WEnum, -}; -use wayland_protocols_wlr::foreign_toplevel::v1::client::{ - zwlr_foreign_toplevel_handle_v1::{self as ftl_handle, ZwlrForeignToplevelHandleV1}, - zwlr_foreign_toplevel_manager_v1::{ - self as ftl_manager, ZwlrForeignToplevelManagerV1, EVT_TOPLEVEL_OPCODE, - }, -}; -use wayland_protocols_wlr::screencopy::v1::client::{ - zwlr_screencopy_frame_v1::{self as scrcopy_frame, ZwlrScreencopyFrameV1}, - zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1, -}; -use wayland_protocols_wlr::virtual_pointer::v1::client::{ - zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1, - zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1, -}; - -/// Linux evdev BTN_LEFT — the button code the virtual-pointer protocol expects. -const BTN_LEFT: u32 = 0x110; - -use crate::x11::WindowInfo; - -/// Name of the opt-in env var that unlocks the experimental native-Wayland -/// backend. -pub const ENABLE_WAYLAND_ENV: &str = "CUA_DRIVER_RS_ENABLE_WAYLAND"; - -/// Whether the user has opted into the experimental native-Wayland backend. -/// -/// The Wayland backend covers toplevel enumeration, per-output capture -/// (native screencopy + a `grim` fallback), and virtual-pointer / -/// virtual-keyboard input via the wlroots protocols. Per-window image -/// capture still depends on the staging `ext-image-copy-capture-v1` -/// protocol, so the backend stays OFF by default and a pure-Wayland -/// session is reported as unsupported unless the user explicitly sets -/// `CUA_DRIVER_RS_ENABLE_WAYLAND=1`. Any value other than empty / `0` / -/// `false` enables it. -pub fn wayland_enabled() -> bool { - match std::env::var(ENABLE_WAYLAND_ENV) { - Ok(v) => { - let v = v.trim(); - !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false") - } - Err(_) => false, - } -} - -/// True when this is an opted-in Wayland desktop session. -/// -/// GNOME and KDE export `DISPLAY` for XWayland even when the target and the -/// desktop are native Wayland. Treating that compatibility variable as proof -/// of an X11 session routed native windows, capture, video, geometry, and focus -/// through invalid XIDs. Backend selection below is capability based; the mere -/// presence of `DISPLAY` must not disable Wayland. -pub fn is_wayland() -> bool { - wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() -} - -/// True when input tools should attempt the Wayland input path (wlroots -/// virtual-pointer, falling back to libei/portal via [`with_libei_fallback`]). -/// -/// Input-specific alias retained to make dispatch intent explicit. The -/// wlroots-vs-portal decision is made from live compositor capabilities inside -/// the `wayland::*` input functions, not from XWayland's `DISPLAY` variable. -pub fn wayland_input_enabled() -> bool { - wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() -} - -/// Reason string when X11 input injection cannot possibly work, so callers -/// **fail loudly** instead of falling through to an X11 path that no-ops yet -/// reports success. Triggers only on a *pure* Wayland session — `WAYLAND_DISPLAY` -/// set, no X11 `DISPLAY` — with the native-Wayland backend NOT opted in (so -/// [`is_wayland`] is false and the X11 path would be chosen). XWayland sessions -/// (where `DISPLAY` is set) and X11 sessions return `None` and proceed normally. -/// See #1921. -pub fn wayland_input_unavailable_reason() -> Option { - if std::env::var_os("WAYLAND_DISPLAY").is_some() - && std::env::var_os("DISPLAY").is_none() - && !wayland_enabled() - { - Some(format!( - "input cannot be delivered: pure Wayland session (no X11 DISPLAY) and \ - the native-Wayland input backend is not enabled. Set {}=1 to enable \ - the Wayland backend (wlroots compositors: sway, labwc, hyprland), or \ - run the target under XWayland so an X11 DISPLAY is available.", - ENABLE_WAYLAND_ENV - )) - } else { - None - } -} - -fn wl_sockets(dir: &str) -> std::collections::HashSet { - std::fs::read_dir(dir) - .into_iter() - .flatten() - .flatten() - .filter_map(|e| e.file_name().into_string().ok()) - .filter(|n| { - n.strip_prefix("wayland-") - .is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit())) - }) - .collect() -} - -/// "Bring your own compositor": if `CUA_WAYLAND_NEST` is set, spawn a private -/// **headless wlroots compositor** (labwc by default) and point this process — -/// and therefore every app it launches (`launch_app`), every capture (`grim`), -/// and all enumeration/injection — at it via `WAYLAND_DISPLAY`. This lets -/// cua-driver automate apps in its **own** Wayland session on ANY host, -/// including KDE (kwin) and GNOME (mutter) which expose no client protocols for -/// this, without ever touching the host compositor or its focus. Idempotent. -pub fn ensure_nested_session() { - use std::sync::OnceLock; - static DONE: OnceLock<()> = OnceLock::new(); - if std::env::var_os("CUA_WAYLAND_NEST").is_none() { - return; - } - DONE.get_or_init(|| { - let xdg = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/run/user/0".into()); - let comp = std::env::var("CUA_WAYLAND_NEST_COMPOSITOR").unwrap_or_else(|_| "labwc".into()); - let before = wl_sockets(&xdg); - let spawned = std::process::Command::new(&comp) - .env("WLR_BACKENDS", "headless") - .env("WLR_RENDERER", "pixman") - .env("WLR_RENDERER_ALLOW_SOFTWARE", "1") - .env("WLR_LIBINPUT_NO_DEVICES", "1") - .env("WLR_HEADLESS_OUTPUTS", "1") - .env_remove("WAYLAND_DISPLAY") // headless: do not nest into the host compositor - .env_remove("DISPLAY") - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .spawn(); - match spawned { - Ok(child) => { - std::mem::forget(child); // keep the compositor alive for our lifetime - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); - loop { - if let Some(sock) = wl_sockets(&xdg).difference(&before).min().cloned() { - std::env::set_var("WAYLAND_DISPLAY", &sock); - std::env::remove_var("DISPLAY"); - // Publish the nested socket so external tools (e.g. a - // `grim` recorder) can target the same session we drive. - let _ = std::fs::write(format!("{xdg}/.cua-nested-display"), &sock); - tracing::info!("cua nested compositor '{comp}' up: WAYLAND_DISPLAY={sock}"); - break; - } - if std::time::Instant::now() >= deadline { - tracing::error!( - "cua nested compositor '{comp}': no Wayland socket appeared" - ); - break; - } - std::thread::sleep(std::time::Duration::from_millis(200)); - } - } - Err(e) => tracing::error!("cua nested compositor '{comp}' spawn failed: {e}"), - } - }); -} - -#[derive(Default)] -struct Toplevel { - title: String, - app_id: String, - closed: bool, -} - -#[derive(Clone, Debug)] -struct ToplevelIdentity { - title: String, - app_id: String, -} - -fn identity_registry() -> &'static Mutex> { - static REGISTRY: OnceLock>> = OnceLock::new(); - REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) -} - -fn observed_origin_registry() -> &'static Mutex> { - static REGISTRY: OnceLock>> = OnceLock::new(); - REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) -} - -pub fn remember_observed_window_origins(windows: &[WindowInfo]) { - if let Ok(mut registry) = observed_origin_registry().lock() { - for window in windows { - if let Some(pid) = window.pid { - // Generic foreign-toplevel and AT-SPI fallbacks use (0,0) when - // they do not know compositor geometry. Do not let that - // placeholder erase a previously observed real origin or - // prevent the caller from falling through to Sway/GNOME data. - if (window.x, window.y) != (0, 0) { - registry.insert(pid, (window.x, window.y)); - } - } - } - } -} - -pub fn observed_window_origin(pid: u32) -> Option<(i32, i32)> { - observed_origin_registry() - .lock() - .ok() - .and_then(|registry| registry.get(&pid).copied()) -} - -fn remember_identity(id: u64, toplevel: &Toplevel) { - if let Ok(mut registry) = identity_registry().lock() { - registry.insert( - id, - ToplevelIdentity { - title: toplevel.title.clone(), - app_id: toplevel.app_id.clone(), - }, - ); - } -} - -fn identity_for(id: u64) -> Option { - identity_registry() - .lock() - .ok() - .and_then(|registry| registry.get(&id).cloned()) - .or_else(|| { - sway_ipc::window_for_id(id).map(|window| ToplevelIdentity { - title: window.title, - app_id: window.app_id, - }) - }) - .or_else(|| { - crate::atspi::list_windows(None) - .into_iter() - .find(|window| window.xid == id || u64::from(window.xid as u32) == id) - .map(|window| ToplevelIdentity { - title: window.title, - app_id: window.app_name, - }) - }) -} - -fn matching_handle(state: &State, id: u64) -> Option { - if let Some(identity) = identity_for(id) { - let by_title = state.toplevels.iter().find_map(|(protocol_id, toplevel)| { - (!identity.title.is_empty() && toplevel.title == identity.title) - .then(|| state.handles.get(protocol_id).cloned()) - .flatten() - }); - return by_title.or_else(|| { - state.toplevels.iter().find_map(|(protocol_id, toplevel)| { - (!identity.app_id.is_empty() && toplevel.app_id == identity.app_id) - .then(|| state.handles.get(protocol_id).cloned()) - .flatten() - }) - }); - } - - let protocol_id = u32::try_from(id).ok()?; - state.handles.get(&protocol_id).cloned() -} - -/// Per-capture in-flight state populated by the screencopy frame Dispatch. -#[derive(Default)] -struct CaptureState { - /// wl_shm format code (Argb8888 / Xrgb8888 / …). - format: Option, - width: u32, - height: u32, - stride: u32, - y_invert: bool, - ready: bool, - failed: bool, -} - -#[derive(Default)] -struct State { - manager: Option, - toplevels: HashMap, - // Live handles + a seat, kept so `click` can `activate` a target toplevel by - // its window_id (foreign-toplevel protocol id) — the focus-based input model. - handles: HashMap, - seat: Option, - // Virtual-pointer manager + output dimensions, so `click` can land a real - // button press at the output centre (over the just-activated window). - vptr_manager: Option, - output: Option, - output_w: u32, - output_h: u32, - // Native screencopy capture state. - scrcopy_manager: Option, - shm: Option, - capture: CaptureState, -} - -impl Dispatch for State { - fn event( - state: &mut Self, - registry: &wl_registry::WlRegistry, - event: wl_registry::Event, - _: &(), - _: &Connection, - qh: &QueueHandle, - ) { - if let wl_registry::Event::Global { - name, - interface, - version, - } = event - { - if interface == ZwlrForeignToplevelManagerV1::interface().name { - let v = version.min(3); - state.manager = - Some(registry.bind::(name, v, qh, ())); - } else if interface == WlSeat::interface().name { - let v = version.min(7); - state.seat = Some(registry.bind::(name, v, qh, ())); - } else if interface == ZwlrVirtualPointerManagerV1::interface().name { - state.vptr_manager = Some(registry.bind::( - name, - version.min(2), - qh, - (), - )); - } else if interface == WlOutput::interface().name { - let out = registry.bind::(name, version.min(4), qh, ()); - if state.output.is_none() { - state.output = Some(out); - } - } else if interface == ZwlrScreencopyManagerV1::interface().name { - state.scrcopy_manager = Some(registry.bind::( - name, - version.min(3), - qh, - (), - )); - } else if interface == WlShm::interface().name { - state.shm = Some(registry.bind::(name, version.min(1), qh, ())); - } - } - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &WlSeat, - _: wayland_client::protocol::wl_seat::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - // Seat name/capabilities events are irrelevant here — we only need the - // seat object to pass to foreign-toplevel `activate`. - } -} - -impl Dispatch for State { - fn event( - state: &mut Self, - _: &WlOutput, - event: wl_output::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - // Remember the output resolution so `click` can aim at its centre. - if let wl_output::Event::Mode { width, height, .. } = event { - state.output_w = width.max(0) as u32; - state.output_h = height.max(0) as u32; - } - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &ZwlrVirtualPointerManagerV1, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &ZwlrVirtualPointerV1, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &WlShm, - _: wl_shm::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - // wl_shm advertises supported formats via `format` events; we don't - // need to track them — screencopy tells us exactly which format to use - // for the frame buffer. - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &WlShmPool, - _: wayland_client::protocol::wl_shm_pool::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &WlBuffer, - _: wayland_client::protocol::wl_buffer::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} - -impl Dispatch for State { - fn event( - _: &mut Self, - _: &ZwlrScreencopyManagerV1, - _: ::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - } -} - -impl Dispatch for State { - fn event( - state: &mut Self, - _: &ZwlrScreencopyFrameV1, - event: scrcopy_frame::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - match event { - scrcopy_frame::Event::Buffer { - format, - width, - height, - stride, - } => { - if let WEnum::Value(fmt) = format { - state.capture.format = Some(fmt as u32); - } - state.capture.width = width; - state.capture.height = height; - state.capture.stride = stride; - } - scrcopy_frame::Event::Flags { flags } => { - if let WEnum::Value(f) = flags { - state.capture.y_invert = f.contains(scrcopy_frame::Flags::YInvert); - } - } - scrcopy_frame::Event::Ready { .. } => { - state.capture.ready = true; - } - scrcopy_frame::Event::Failed => { - state.capture.failed = true; - } - _ => {} - } - } -} - -impl Dispatch for State { - fn event( - _state: &mut Self, - _: &ZwlrForeignToplevelManagerV1, - _event: ftl_manager::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - // The `toplevel` event creates a handle object (see event_created_child!); - // the handle's own events carry the title/app_id we collect below. - } - - event_created_child!(State, ZwlrForeignToplevelManagerV1, [ - EVT_TOPLEVEL_OPCODE => (ZwlrForeignToplevelHandleV1, ()), - ]); -} - -impl Dispatch for State { - fn event( - state: &mut Self, - handle: &ZwlrForeignToplevelHandleV1, - event: ftl_handle::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - let id = handle.id().protocol_id(); - state.handles.entry(id).or_insert_with(|| handle.clone()); - let tl = state.toplevels.entry(id).or_default(); - match event { - ftl_handle::Event::Title { title } => tl.title = title, - ftl_handle::Event::AppId { app_id } => tl.app_id = app_id, - ftl_handle::Event::Closed => tl.closed = true, - _ => {} - } - } -} - -/// Enumerate native Wayland toplevels via wlr-foreign-toplevel-management. -/// `xid` begins as the foreign-toplevel handle's connection-scoped protocol id. -/// The dispatcher replaces it with a stable compositor or AT-SPI identity when -/// available. pid is unknown (not exposed by the protocol); geometry is 0 -/// (the protocol does not surface position/size). app_id is folded into the -/// title (`" [<app_id>]"`) so callers matching on either still match. -pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - - let mut state = State::default(); - queue.roundtrip(&mut state)?; // registry globals -> bind manager - if state.manager.is_none() { - anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1"); - } - // Manager emits a `toplevel` per window; each handle then emits title/app_id - // and a `done`. A few roundtrips drain the initial enumeration. - for _ in 0..4 { - queue.roundtrip(&mut state)?; - } - - let sway_windows = sway_ipc::list_windows().unwrap_or_default(); - let mut used_sway_ids = HashSet::new(); - let mut out = Vec::new(); - for (id, tl) in &state.toplevels { - if tl.closed { - continue; - } - let title = if tl.app_id.is_empty() { - tl.title.clone() - } else { - format!("{} [{}]", tl.title, tl.app_id) - }; - let sway = sway_windows - .iter() - .find(|window| { - !used_sway_ids.contains(&window.id) - && !tl.title.is_empty() - && window.title == tl.title - }) - .or_else(|| { - sway_windows.iter().find(|window| { - !used_sway_ids.contains(&window.id) - && !tl.app_id.is_empty() - && window.app_id == tl.app_id - }) - }); - let stable_id = sway.map(|window| window.id).unwrap_or(*id as u64); - if let Some(window) = sway { - used_sway_ids.insert(window.id); - } - remember_identity(stable_id, tl); - out.push(WindowInfo { - xid: stable_id, - pid: sway.map(|window| window.pid), - app_name: tl.app_id.clone(), - title, - is_on_screen: sway.map(|window| window.visible).unwrap_or(true), - z_index: None, - x: sway.map(|window| window.x).unwrap_or(0), - y: sway.map(|window| window.y).unwrap_or(0), - width: sway.map(|window| window.width).unwrap_or(0), - height: sway.map(|window| window.height).unwrap_or(0), - }); - } - Ok(out) -} - -// ── Capture (native screencopy + grim fallback) ────────────────────────────── - -/// Capture the Wayland output as PNG bytes via `zwlr_screencopy_manager_v1`. -/// -/// Binds the screencopy manager plus `wl_shm`, asks the compositor to copy the -/// next frame of the first advertised output into a shm buffer, channel-swaps -/// from the compositor's pixel format to RGBA, and encodes a PNG via the -/// existing `image` crate. Falls back to shelling out to `grim` when the -/// screencopy manager or `wl_shm` is unavailable so users on lighter wlroots -/// builds stay supported. -pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> { - match capture_via_screencopy() { - Ok(bytes) => return Ok(bytes), - Err(e) => tracing::warn!("native screencopy failed, falling back to grim: {e}"), - } - capture_via_grim() -} - -/// Shell out to `grim -t png -` — the wlroots reference screenshot tool. Kept -/// as the last-resort fallback for compositors that hide screencopy. -fn capture_via_grim() -> anyhow::Result<Vec<u8>> { - let out = std::process::Command::new("grim") - .args(["-t", "png", "-"]) - .output()?; - if !out.status.success() { - anyhow::bail!("grim failed: {}", String::from_utf8_lossy(&out.stderr)); - } - if out.stdout.is_empty() { - anyhow::bail!("grim produced no output"); - } - Ok(out.stdout) -} - -/// Native screencopy path: bind manager + shm, allocate an anon mmap buffer, -/// request a copy, wait for Ready, swap channels, encode PNG. Returns an error -/// if any global is missing or the compositor flags the capture as failed. -fn capture_via_screencopy() -> anyhow::Result<Vec<u8>> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - - let mut state = State::default(); - queue.roundtrip(&mut state)?; - queue.roundtrip(&mut state)?; // outputs report their Mode - - let manager = state - .scrcopy_manager - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_screencopy_manager_v1"))?; - let shm = state - .shm - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?; - let output = state - .output - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output to capture"))?; - - let frame = manager.capture_output(0, &output, &qh, ()); - // Drain Buffer / Flags events; spin until Ready or Failed (or timeout). - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - let mut buffer: Option<WlBuffer> = None; - let mut pool: Option<WlShmPool> = None; - let mut mmap_ptr: *mut libc::c_void = std::ptr::null_mut(); - let mut mmap_len: usize = 0; - let mut fd: i32 = -1; - - loop { - queue.roundtrip(&mut state)?; - if state.capture.failed { - anyhow::bail!("compositor signalled screencopy failure"); - } - if state.capture.ready { - break; - } - // Once we know the buffer params, allocate + send copy exactly once. - if buffer.is_none() - && state.capture.format.is_some() - && state.capture.stride > 0 - && state.capture.height > 0 - { - let size = (state.capture.stride as usize) - .checked_mul(state.capture.height as usize) - .ok_or_else(|| anyhow::anyhow!("screencopy buffer size overflow"))?; - let (anon_fd, p) = anon_shm(size)?; - fd = anon_fd; - mmap_ptr = p; - mmap_len = size; - use std::os::fd::AsFd as _; - let pool_fd = unsafe { borrowed_fd(fd) }; - let p = shm.create_pool(pool_fd.as_fd(), size as i32, &qh, ()); - let fmt_raw = state.capture.format.unwrap(); - let fmt: wl_shm::Format = match wl_shm::Format::try_from(fmt_raw) { - Ok(f) => f, - Err(_) => { - cleanup_mmap(mmap_ptr, mmap_len, fd); - anyhow::bail!("compositor advertised unsupported wl_shm format {fmt_raw:#x}"); - } - }; - let b = p.create_buffer( - 0, - state.capture.width as i32, - state.capture.height as i32, - state.capture.stride as i32, - fmt, - &qh, - (), - ); - frame.copy(&b); - buffer = Some(b); - pool = Some(p); - } - if std::time::Instant::now() >= deadline { - cleanup_mmap(mmap_ptr, mmap_len, fd); - anyhow::bail!("screencopy timed out waiting for frame"); - } - } - - let result = (|| -> anyhow::Result<Vec<u8>> { - let w = state.capture.width; - let h = state.capture.height; - let stride = state.capture.stride as usize; - let format = state.capture.format.unwrap_or(0); - if mmap_ptr.is_null() || mmap_len == 0 { - anyhow::bail!("screencopy ready without a backing buffer"); - } - let raw = unsafe { std::slice::from_raw_parts(mmap_ptr as *const u8, mmap_len) }; - let mut rgba = Vec::with_capacity((w as usize) * (h as usize) * 4); - for row in 0..(h as usize) { - let src_row = if state.capture.y_invert { - (h as usize) - 1 - row - } else { - row - }; - let base = src_row * stride; - for col in 0..(w as usize) { - let px = &raw[base + col * 4..base + col * 4 + 4]; - let (r, g, b, a) = match wl_shm::Format::try_from(format).ok() { - // Argb8888 / Xrgb8888 over wl_shm are little-endian BGRA / BGRX. - Some(wl_shm::Format::Argb8888) => (px[2], px[1], px[0], px[3]), - Some(wl_shm::Format::Xrgb8888) => (px[2], px[1], px[0], 255), - Some(wl_shm::Format::Abgr8888) => (px[0], px[1], px[2], px[3]), - Some(wl_shm::Format::Xbgr8888) => (px[0], px[1], px[2], 255), - _ => (px[2], px[1], px[0], px[3]), - }; - rgba.extend_from_slice(&[r, g, b, a]); - } - } - cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h) - })(); - - // Always tear down regardless of result. - if let Some(b) = buffer { - b.destroy(); - } - if let Some(p) = pool { - p.destroy(); - } - frame.destroy(); - let _ = queue.roundtrip(&mut state); - cleanup_mmap(mmap_ptr, mmap_len, fd); - - result -} - -/// Allocate an anonymous shared-memory file of `size` bytes and mmap it RW. -/// Returns the raw fd and the mmap pointer; the caller is responsible for -/// passing both to [`cleanup_mmap`] when done. -pub(crate) fn anon_shm(size: usize) -> anyhow::Result<(i32, *mut libc::c_void)> { - // memfd_create is Linux-only and is the cleanest path; fall back to - // shm_open if memfd isn't available for any reason. - let name = b"cua-scrcopy\0"; - let fd = unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) }; - if fd < 0 { - return Err(anyhow::anyhow!( - "memfd_create failed: {}", - std::io::Error::last_os_error() - )); - } - let rc = unsafe { libc::ftruncate(fd, size as libc::off_t) }; - if rc != 0 { - let err = std::io::Error::last_os_error(); - unsafe { libc::close(fd) }; - return Err(anyhow::anyhow!("ftruncate failed: {err}")); - } - let p = unsafe { - libc::mmap( - std::ptr::null_mut(), - size, - libc::PROT_READ | libc::PROT_WRITE, - libc::MAP_SHARED, - fd, - 0, - ) - }; - if p == libc::MAP_FAILED { - let err = std::io::Error::last_os_error(); - unsafe { libc::close(fd) }; - return Err(anyhow::anyhow!("mmap failed: {err}")); - } - Ok((fd, p)) -} - -/// Unmap and close the screencopy backing buffer; safe to call with the -/// sentinel values left from a never-allocated buffer. -pub(crate) fn cleanup_mmap(ptr: *mut libc::c_void, len: usize, fd: i32) { - if !ptr.is_null() && len > 0 { - unsafe { libc::munmap(ptr, len) }; - } - if fd >= 0 { - unsafe { libc::close(fd) }; - } -} - -/// Borrow a raw fd as an `OwnedFd` for wl_shm.create_pool. The pool keeps -/// its own reference; we close our copy via [`cleanup_mmap`]. -/// -/// SAFETY: caller must guarantee `fd` is a valid open file descriptor. -/// `libc::dup` may fail (returning -1, errno set), in which case we panic -/// instead of constructing an `OwnedFd` from -1 (which would have UB on -/// drop). Callers that need fallible behaviour should use the dup syscall -/// directly and check the result before wrapping. -pub(crate) unsafe fn borrowed_fd(fd: i32) -> std::os::fd::OwnedFd { - use std::os::fd::FromRawFd; - let dup = libc::dup(fd); - if dup < 0 { - let err = std::io::Error::last_os_error(); - panic!("dup({fd}) failed: {err}"); - } - std::os::fd::OwnedFd::from_raw_fd(dup) -} - -/// Capture dispatcher: native Wayland (screencopy with grim fallback) when -/// applicable, else X11. Mirrors `screenshot_window_dispatch` for the -/// output-level path used by `get_window_state`'s vision payload. -pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> { - if is_wayland() { - let bytes = screenshot_display_dispatch()?; - if let Some((x, y, width, height)) = window_geometry(xid) { - crop_png_to_rect( - &bytes, - x, - y, - width, - height, - &format!("Wayland window {xid}"), - ) - } else { - Ok(bytes) - } - } else { - crate::capture::screenshot_window_bytes(xid) - } -} - -fn crop_png_to_rect( - output_png: &[u8], - rect_x: i32, - rect_y: i32, - rect_width: u32, - rect_height: u32, - label: &str, -) -> anyhow::Result<Vec<u8>> { - let image = image::load_from_memory(output_png)?; - let image_width = image.width(); - let image_height = image.height(); - let x = rect_x.max(0) as u32; - let y = rect_y.max(0) as u32; - if x >= image_width || y >= image_height { - anyhow::bail!( - "{label} origin ({x},{y}) is outside captured output {image_width}x{image_height}" - ); - } - let width = rect_width.min(image_width - x); - let height = rect_height.min(image_height - y); - if width == 0 || height == 0 { - anyhow::bail!("{label} has empty capture geometry"); - } - let cropped = image.crop_imm(x, y, width, height); - let mut cursor = std::io::Cursor::new(Vec::new()); - cropped.write_to(&mut cursor, image::ImageFormat::Png)?; - Ok(cursor.into_inner()) -} - -/// Display-level capture dispatcher. Cascade: -/// 1. Opt-in GNOME compositor helper. If available, capture failure is -/// terminal rather than cascading into GNOME's portal implementation. -/// 2. Native Wayland on wlroots: zwlr_screencopy_manager_v1 (fast, zero -/// consent). -/// 3. ext-image-copy-capture-v1 on supported compositors. -/// 4. xdg-desktop-portal Screenshot via ashpd. Triggers a consent prompt on -/// first use per session. -/// 5. X11: existing root-window path. -pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> { - if is_wayland() { - // Tier 1: the opt-in GNOME compositor helper. It avoids probing - // wlroots-only protocols and captures the Shell stage without consent. - // If the helper is present but capture fails, do not fall through to - // GNOME's portal implementation: on GNOME 50 a malformed 0x0 cursor - // sprite can crash Shell in GNOME's unsafe stage-content capture path. - if let Some(result) = checked_shell_helper_capture( - shell_helper::available(), - shell_helper::screenshot_display, - ) { - return result; - } - // Tier 2: native wlroots screencopy (fast, zero consent). - match screenshot_bytes() { - Ok(bytes) => return Ok(bytes), - Err(e) => { - tracing::debug!( - "wlroots screencopy unavailable ({e}); trying ext-image-copy-capture-v1" - ); - } - } - // Tier 3: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri, - // hyprland, KDE 6.2+, GNOME 47+). - match ext_screencopy::screenshot_via_ext_copy() { - Ok(bytes) => return Ok(bytes), - Err(e) => { - tracing::debug!( - "ext-image-copy-capture-v1 unavailable ({e}); trying xdg-desktop-portal" - ); - } - } - // Tier 4: xdg-desktop-portal (GNOME, KDE, COSMIC fallback). - match portal_screenshot::screenshot_via_portal() { - Ok(bytes) => return Ok(bytes), - Err(e) => { - tracing::debug!( - "xdg-desktop-portal Screenshot unavailable ({e}); falling through to X11" - ); - } - } - } - // Final fallback: X11 root window. Call the X11-only path explicitly - // so we don't re-enter screenshot_display_bytes (which routes back here - // on Wayland — would loop forever). - crate::capture::screenshot_display_bytes_x11() -} - -fn checked_shell_helper_capture( - available: bool, - capture: impl FnOnce() -> Option<Vec<u8>>, -) -> Option<anyhow::Result<Vec<u8>>> { - available - .then(|| capture().ok_or_else(|| anyhow::anyhow!("GNOME compositor helper capture failed"))) -} - -/// Per-window capture dispatcher. On X11 forwards to the existing window -/// capture path; on pure Wayland returns a typed error pointing at the -/// staging `ext-image-copy-capture-v1` protocol — wlr-screencopy is -/// output-only, and `foreign-toplevel` exposes no per-window geometry to -/// crop with. -pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> { - if is_wayland() { - if let Some((x, y, width, height)) = window_geometry(xid) { - return crop_png_to_rect( - &screenshot_display_dispatch()?, - x, - y, - width, - height, - &format!("Wayland window {xid}"), - ); - } - anyhow::bail!( - "per-window screenshot is not yet supported on native Wayland — \ - zwlr_screencopy_manager_v1 is output-only and ext-image-copy-capture-v1 \ - is not yet shipped in wayland-protocols-wlr. Run under XWayland to crop \ - to a single window, or capture the full output instead." - ); - } - crate::capture::screenshot_window_bytes(xid) -} - -// ── Input session helper ───────────────────────────────────────────────────── - -/// Sentinel substring carried by the `open_vptr_session` error when the -/// compositor exposes no `zwlr_virtual_pointer_manager_v1` (KWin/Plasma, -/// Mutter/GNOME). The input dispatch matches on this to decide whether the -/// libei/portal fallback ([`libei`]) can recover the call. Kept as a string -/// marker (rather than a typed error) so the existing `anyhow::Result` -/// signatures of every input fn are unchanged. See #1982. -pub const NO_VPTR_MARKER: &str = "no-zwlr-virtual-pointer"; - -/// True when `err` is the "compositor has no wlroots virtual-pointer" failure -/// from [`open_vptr_session`] — i.e. the point where a non-wlroots compositor -/// needs the libei fallback rather than a hard error. -fn is_no_vptr(err: &anyhow::Error) -> bool { - err.to_string().contains(NO_VPTR_MARKER) -} - -/// Run the wlroots virtual-pointer closure `f`; if it fails specifically -/// because the compositor exposes no `zwlr_virtual_pointer_manager_v1` and this -/// binary carries the `portal-input` feature, run the libei `fallback` instead. -/// Any other wlroots error (and the no-vptr error in a build without the -/// feature) propagates unchanged. This is the single seam through which #1982's -/// KDE/GNOME input recovery flows. -fn with_libei_fallback<T>( - f: impl FnOnce() -> anyhow::Result<T>, - #[allow(unused_variables)] fallback: impl FnOnce() -> anyhow::Result<T>, -) -> anyhow::Result<T> { - match f() { - Ok(v) => Ok(v), - Err(e) if is_no_vptr(&e) => { - #[cfg(feature = "portal-input")] - { - tracing::info!( - "wlroots virtual-pointer unavailable ({e}); falling back to libei/portal" - ); - return fallback(); - } - #[cfg(not(feature = "portal-input"))] - { - Err(e) - } - } - Err(e) => Err(e), - } -} - -/// Live virtual-pointer session: connection + queue + the bound objects every -/// pointer op (click, scroll, drag) needs. Returned by [`open_vptr_session`]. -pub struct VptrSession { - pub conn: Connection, - queue: wayland_client::EventQueue<State>, - state: State, - pub seat: WlSeat, - pub vptr: ZwlrVirtualPointerV1, - pub output_w: u32, - pub output_h: u32, -} - -/// Bind manager + seat + virtual-pointer + first output, optionally activate a -/// foreign-toplevel by `window_id` so the synthesised events land on it, and -/// return the live session that scroll / drag / click reuse. Wayland forbids a -/// client from knowing another window's on-screen geometry, so we drive every -/// pointer event in *output* coordinates and rely on the activated toplevel -/// covering the centre. -pub fn open_vptr_session(activate_window_id: Option<u64>) -> anyhow::Result<VptrSession> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - - let mut state = State::default(); - queue.roundtrip(&mut state)?; - for _ in 0..4 { - queue.roundtrip(&mut state)?; - } - - // Evaluate the virtual-pointer / NO_VPTR_MARKER path FIRST: on compositors - // that expose neither zwlr_virtual_pointer nor zwlr_foreign_toplevel - // (KWin/Plasma, Mutter/GNOME) we must surface the marker so - // `with_libei_fallback` re-routes through libei/portal. Requiring - // foreign-toplevel up front would mask the marker and leave the libei - // fallback dead. See #1982. - let mgr = state.vptr_manager.clone().ok_or_else(|| { - if PORTAL_INPUT_ENABLED { - // The caller (via `with_libei_fallback`) recognises NO_VPTR_MARKER - // and re-routes the op through the libei/portal backend, which DOES - // reach KWin/Plasma and Mutter/GNOME. Keep the marker in the text. - anyhow::anyhow!( - "compositor does not expose zwlr_virtual_pointer_manager_v1 \ - ({NO_VPTR_MARKER})" - ) - } else { - // KWin/Plasma and Mutter/GNOME don't implement zwlr_virtual_pointer, - // and this build has no libei/portal fallback — so input has no - // backend at all rather than silently no-op'ing. The marker still - // lets the dispatch layer classify the failure uniformly. See #1982. - anyhow::anyhow!( - "no input backend for this compositor ({NO_VPTR_MARKER}): it \ - exposes no zwlr_virtual_pointer_manager_v1 and this build was \ - compiled without libei/portal support (#1982). Use the \ - portal-enabled Linux build for input on KDE Plasma / GNOME, or \ - a wlroots compositor (sway, labwc, hyprland)." - ) - } - })?; - - // foreign-toplevel is only needed to activate a specific window before - // synthesising input; require it only when a caller actually asks for that. - if activate_window_id.is_some() && state.manager.is_none() { - anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1"); - } - - let seat = state.seat.clone().ok_or_else(|| { - anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input") - })?; - - if let Some(id) = activate_window_id { - let handle = matching_handle(&state, id) - .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {id}"))?; - handle.activate(&seat); - queue.roundtrip(&mut state)?; - } - - let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ()); - let (output_w, output_h) = (state.output_w.max(1), state.output_h.max(1)); - Ok(VptrSession { - conn, - queue, - state, - seat, - vptr, - output_w, - output_h, - }) -} - -/// Focus and raise a specific native Wayland toplevel before focus-bound -/// keyboard or portal/libei input. wlroots exposes an activation request on its -/// foreign-toplevel protocol; GNOME uses the bundled compositor helper. Other -/// compositors must refuse until they provide an equally target-addressable -/// adapter, because global injection without this gate can affect the wrong app. -pub fn activate_window_for_input(window_id: u64) -> anyhow::Result<()> { - let pid = crate::atspi::list_windows(None) - .into_iter() - .find(|window| window.xid == window_id) - .and_then(|window| window.pid); - activate_window_for_input_target(window_id, pid) -} - -/// Activate a Wayland target with an explicit process identity when available. -/// The bundled compositor does not depend on connection-local Wayland object -/// ids: its control protocol resolves the one mapped toplevel owned by `pid`. -pub fn activate_window_for_input_target( - window_id: u64, - target_pid: Option<u32>, -) -> anyhow::Result<()> { - if is_inject_mode() { - let pid = target_pid.ok_or_else(|| { - anyhow::anyhow!( - "foreground_unavailable: cua-compositor activation requires a verified target pid" - ) - })?; - inject_send(&[format!("f {pid}")])?; - remember_inject_focused_target(pid, window_id); - std::thread::sleep(std::time::Duration::from_millis(60)); - return Ok(()); - } - - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - let mut state = State::default(); - queue.roundtrip(&mut state)?; - for _ in 0..4 { - queue.roundtrip(&mut state)?; - } - - if let (Some(_), Some(seat), Some(handle)) = ( - state.manager.as_ref(), - state.seat.clone(), - matching_handle(&state, window_id), - ) { - handle.activate(&seat); - queue.roundtrip(&mut state)?; - std::thread::sleep(std::time::Duration::from_millis(60)); - return Ok(()); - } - - if shell_helper::activate_window(window_id) { - std::thread::sleep(std::time::Duration::from_millis(60)); - return Ok(()); - } - - anyhow::bail!( - "foreground_unavailable: this Wayland compositor does not expose a verified, \ - target-addressable activation adapter for window {window_id}; refusing global \ - input because it could affect the wrong application" - ) -} - -pub(crate) fn focused_app_window(pid: u32) -> Option<u64> { - sway_ipc::list_windows() - .and_then(|windows| { - windows - .into_iter() - .find(|window| window.pid == pid && window.focused) - }) - .map(|window| window.id) - .or_else(|| shell_helper::trusted_focused_window_for_pid(pid)) -} - -/// Run a focus-bound keyboard transaction only after a compositor adapter has -/// confirmed the exact PID/window pair, and restore the previously focused -/// toplevel afterward. Global virtual-keyboard/libei input is refused when the -/// compositor cannot provide this read-back contract. -pub fn with_target_foreground<T>( - pid: u32, - window_id: u64, - body: impl FnOnce() -> anyhow::Result<T>, -) -> anyhow::Result<T> { - if let Some(window) = sway_ipc::window_for_id(window_id) { - if window.pid != pid { - anyhow::bail!( - "foreground_unavailable: Sway window {window_id} belongs to pid {}, not pid {pid}", - window.pid - ); - } - return sway_ipc::with_focused_container(window_id, body); - } - if shell_helper::trusted_window_for_id(pid, window_id).is_some() { - return shell_helper::with_focused_window(pid, window_id, body); - } - anyhow::bail!( - "foreground_unavailable: no trusted Wayland compositor adapter can confirm exact \ - target window {window_id} for pid {pid}; no global keyboard input was sent" - ) -} - -/// Query the first `wl_output`'s pixel dimensions via a short Wayland -/// roundtrip, independent of the virtual-pointer protocol. Used by the libei -/// fallback (which never opens a `VptrSession`) to reproduce the vptr path's -/// default-to-centre and clamp behaviour so both backends treat coordinates -/// identically. Falls back to `(1, 1)` when no output reports a mode. -#[cfg(feature = "portal-input")] -fn output_dimensions() -> anyhow::Result<(u32, u32)> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - let mut state = State::default(); - queue.roundtrip(&mut state)?; - for _ in 0..4 { - queue.roundtrip(&mut state)?; - } - Ok((state.output_w.max(1), state.output_h.max(1))) -} - -/// Reproduce the wlroots vptr path's coordinate handling for the libei -/// fallback: `(0, 0)` defaults to the output centre, and any value is clamped -/// to `[0, dim-1]`. Keeps `click(.., 0, 0, ..)` landing on centre rather than -/// the top-left corner across both backends. -#[cfg(feature = "portal-input")] -fn normalize_click_xy(x: i32, y: i32, w: u32, h: u32) -> (i32, i32) { - let (px, py) = if x == 0 && y == 0 { - ((w / 2) as i32, (h / 2) as i32) - } else { - (x, y) - }; - ( - px.clamp(0, (w as i32).saturating_sub(1)), - py.clamp(0, (h as i32).saturating_sub(1)), - ) -} - -/// Map a cua/X11 pointer button (1=left / 2=middle / 3=right) to its evdev -/// code, which is what `zwlr_virtual_pointer_v1::button` expects. -pub fn evdev_pointer_button(button: u8) -> u32 { - match button { - 2 => 0x112, // BTN_MIDDLE - 3 => 0x111, // BTN_RIGHT - _ => 0x110, // BTN_LEFT - } -} - -fn event_time_ms() -> u32 { - static START: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new(); - START - .get_or_init(std::time::Instant::now) - .elapsed() - .as_millis() - .clamp(1, u32::MAX as u128) as u32 -} - -/// Click a native Wayland toplevel identified by its `window_id` (the -/// foreign-toplevel protocol id from `list_windows`) at output-relative -/// `(x, y)`, with `button` (1/2/3 = left/middle/right) emitted `count` times. -/// Coordinates default to the output centre when both x and y are zero so -/// the legacy focus-based behaviour is preserved when callers can't supply -/// real coords. A short delay between iterations gives the compositor time -/// to discriminate single vs. double clicks. -pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> { - with_libei_fallback( - || click_vptr(Some(window_id), x, y, count, button), - || { - libei_wait_pointer_ready()?; - activate_window_for_input(window_id)?; - libei_click(x, y, count, button) - }, - ) -} - -/// Click a desktop-absolute point without selecting or activating a toplevel. -/// This is the Wayland peer of an XTest root-window click and is used only by -/// the explicit desktop capture scope. -pub fn click_desktop(x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> { - if is_inject_mode() { - let btn = evdev_button(button as u32); - return inject_send(&[format!("d {x} {y} {} {btn}", count.max(1))]); - } - with_libei_fallback( - || click_vptr(None, x, y, count, button), - || libei_click(x, y, count, button), - ) -} - -/// wlroots virtual-pointer implementation of [`click`]. Falls back to libei via -/// [`with_libei_fallback`] when the compositor exposes no virtual-pointer. -fn click_vptr( - window_id: Option<u64>, - x: i32, - y: i32, - count: u32, - button: u8, -) -> anyhow::Result<()> { - let mut sess = open_vptr_session(window_id)?; - std::thread::sleep(std::time::Duration::from_millis(40)); - let (w, h) = (sess.output_w, sess.output_h); - let (px, py) = if x == 0 && y == 0 { - ((w / 2) as i32, (h / 2) as i32) - } else { - (x, y) - }; - let px = px.clamp(0, w as i32 - 1) as u32; - let py = py.clamp(0, h as i32 - 1) as u32; - let btn = evdev_pointer_button(button); - for i in 0..count.max(1) { - if i > 0 { - std::thread::sleep(std::time::Duration::from_millis(80)); - } - sess.vptr.motion_absolute(event_time_ms(), px, py, w, h); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - std::thread::sleep(std::time::Duration::from_millis(15)); - sess.vptr.button(event_time_ms(), btn, ButtonState::Pressed); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - std::thread::sleep(std::time::Duration::from_millis(20)); - sess.vptr - .button(event_time_ms(), btn, ButtonState::Released); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - } - // Keep the synthetic-cursor registry in sync with the warp we just - // performed so a subsequent `get_cursor_position` reflects reality. - record_synth_cursor(px as i32, py as i32); - sess.vptr.destroy(); - sess.queue.roundtrip(&mut sess.state)?; - Ok(()) -} - -/// Synthesize a vertical or horizontal scroll on the activated toplevel. Each -/// tick emits an `axis_source(wheel)` + `axis_discrete(1)` pair through the -/// virtual-pointer protocol, mirroring how a real wheel notch decomposes. The -/// magnitude follows wl_pointer convention: ±10 (in wl_fixed = ×256) per tick. -pub fn scroll(window_id: u64, direction: &str, amount: u32) -> anyhow::Result<()> { - scroll_at(window_id, None, direction, amount) -} - -/// Translate window-local screenshot coordinates into compositor output -/// coordinates when the active compositor exposes the target geometry. -pub fn window_local_to_output(window_id: u64, x: i32, y: i32) -> (i32, i32) { - window_geometry(window_id) - .map(|(window_x, window_y, _, _)| (window_x.saturating_add(x), window_y.saturating_add(y))) - .unwrap_or((x, y)) -} - -/// Resolve geometry through stable title/app identity when a foreign-toplevel -/// object ID came from an earlier Wayland connection. Protocol object IDs are -/// connection-local, so direct equality is only a fast path. -pub fn window_geometry(window_id: u64) -> Option<(i32, i32, u32, u32)> { - if let Some(window) = sway_ipc::window_for_id(window_id) { - return Some((window.x, window.y, window.width, window.height)); - } - - let identity = identity_for(window_id); - if let Some(identity) = identity.as_ref() { - if let Some(windows) = sway_ipc::list_windows() { - let title_matches = windows - .iter() - .filter(|window| !identity.title.is_empty() && window.title == identity.title) - .collect::<Vec<_>>(); - if title_matches.len() == 1 { - let window = title_matches[0]; - return Some((window.x, window.y, window.width, window.height)); - } - let app_matches = windows - .iter() - .filter(|window| !identity.app_id.is_empty() && window.app_id == identity.app_id) - .collect::<Vec<_>>(); - if app_matches.len() == 1 { - let window = app_matches[0]; - return Some((window.x, window.y, window.width, window.height)); - } - } - } - - let windows = list_windows_dispatch(None); - if let Some(window) = windows - .iter() - .find(|window| window.xid == window_id && window.width > 0 && window.height > 0) - { - return Some((window.x, window.y, window.width, window.height)); - } - let identity = identity?; - let title_matches = windows - .iter() - .filter(|window| { - window.width > 0 - && window.height > 0 - && !identity.title.is_empty() - && undecorated_native_title(window) == identity.title - }) - .collect::<Vec<_>>(); - if title_matches.len() == 1 { - let window = title_matches[0]; - return Some((window.x, window.y, window.width, window.height)); - } - let app_matches = windows - .iter() - .filter(|window| { - window.width > 0 - && window.height > 0 - && !identity.app_id.is_empty() - && window.app_name == identity.app_id - }) - .collect::<Vec<_>>(); - (app_matches.len() == 1).then(|| { - let window = app_matches[0]; - (window.x, window.y, window.width, window.height) - }) -} - -/// Scroll after positioning the synthetic pointer over an output-relative -/// target. Wayland routes wheel events to the surface beneath the pointer, so -/// pixel-addressed scrolls must not inherit an unrelated cursor position. -pub fn scroll_at( - window_id: u64, - point: Option<(i32, i32)>, - direction: &str, - amount: u32, -) -> anyhow::Result<()> { - let direction = direction.to_string(); - with_libei_fallback( - || scroll_vptr(Some(window_id), point, &direction, amount), - || { - libei_wait_scroll_ready()?; - activate_window_for_input(window_id)?; - if let Some((x, y)) = point { - libei_move_absolute(x, y)?; - } - libei_scroll(&direction, amount) - }, - ) -} - -/// Scroll at a desktop-absolute point without activating a named toplevel. -pub fn scroll_desktop(x: i32, y: i32, direction: &str, amount: u32) -> anyhow::Result<()> { - if is_inject_mode() { - return inject_scroll_desktop(x, y, direction, amount); - } - let direction = direction.to_string(); - with_libei_fallback( - || scroll_vptr(None, Some((x, y)), &direction, amount), - || { - libei_wait_scroll_ready()?; - libei_move_absolute(x, y)?; - libei_scroll(&direction, amount) - }, - ) -} - -/// wlroots virtual-pointer implementation of [`scroll`]. -fn scroll_vptr( - window_id: Option<u64>, - point: Option<(i32, i32)>, - direction: &str, - amount: u32, -) -> anyhow::Result<()> { - let mut sess = open_vptr_session(window_id)?; - if let Some((x, y)) = point { - let px = x.clamp(0, (sess.output_w as i32).saturating_sub(1)) as u32; - let py = y.clamp(0, (sess.output_h as i32).saturating_sub(1)) as u32; - sess.vptr - .motion_absolute(event_time_ms(), px, py, sess.output_w, sess.output_h); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - record_synth_cursor(px as i32, py as i32); - std::thread::sleep(std::time::Duration::from_millis(15)); - } - let (axis, sign): (Axis, i32) = match direction.to_ascii_lowercase().as_str() { - "up" => (Axis::VerticalScroll, -1), - "down" => (Axis::VerticalScroll, 1), - "left" => (Axis::HorizontalScroll, -1), - "right" => (Axis::HorizontalScroll, 1), - other => anyhow::bail!("unknown scroll direction: {other}"), - }; - // axis_discrete: `value` is logical units (the wayland-rs wrapper - // converts to wl_fixed internally); `discrete` is the tick count. - let value: f64 = (sign as f64) * 10.0; - for i in 0..amount.max(1) { - if i > 0 { - std::thread::sleep(std::time::Duration::from_millis(25)); - } - sess.vptr.axis_source(AxisSource::Wheel); - sess.vptr.axis_discrete(event_time_ms(), axis, value, sign); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - } - sess.vptr.destroy(); - sess.queue.roundtrip(&mut sess.state)?; - Ok(()) -} - -/// Last cursor position the agent warped to via `move_cursor_absolute`. -/// Wayland exposes no protocol for clients to read the real global cursor -/// position; a Wayland-conformant `get_cursor_position` can therefore only -/// report what THIS process synthesized. Updated every `motion_absolute` -/// emitted from `move_cursor_absolute` / `click` / `drag`. -static SYNTH_CURSOR_POS: std::sync::OnceLock<std::sync::Mutex<Option<(i32, i32)>>> = - std::sync::OnceLock::new(); - -fn record_synth_cursor(x: i32, y: i32) { - let cell = SYNTH_CURSOR_POS.get_or_init(|| std::sync::Mutex::new(None)); - if let Ok(mut g) = cell.lock() { - *g = Some((x, y)); - } -} - -/// Returns the last `(x, y)` this process warped the cursor to via the -/// Wayland virtual-pointer protocol, or `None` if no warp has happened in -/// this process. The reading is "synthetic": Wayland forbids clients from -/// querying the real cursor position, so this value diverges from reality -/// the moment the user moves their physical mouse. Callers should surface -/// `source: "synthetic"` in the structured payload. -pub fn last_synth_cursor_pos() -> Option<(i32, i32)> { - SYNTH_CURSOR_POS - .get_or_init(|| std::sync::Mutex::new(None)) - .lock() - .ok() - .and_then(|g| *g) -} - -/// Warp the cursor to absolute output coordinates `(x, y)` using -/// `zwlr_virtual_pointer_v1::motion_absolute`. Clamps to the output bounds -/// reported by `open_vptr_session`. Emits a motion + frame and roundtrips so -/// the compositor commits the warp before returning. Records the position in -/// the synthetic-cursor registry so `last_synth_cursor_pos` can report it. -pub fn move_cursor_absolute(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> { - with_libei_fallback( - || move_cursor_absolute_vptr(window_id, x, y), - || libei_move_absolute(x, y), - ) -} - -/// wlroots virtual-pointer implementation of [`move_cursor_absolute`]. -fn move_cursor_absolute_vptr(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> { - let mut sess = open_vptr_session(window_id)?; - let (w, h) = (sess.output_w, sess.output_h); - let px = x.clamp(0, (w as i32).saturating_sub(1)) as u32; - let py = y.clamp(0, (h as i32).saturating_sub(1)) as u32; - sess.vptr.motion_absolute(event_time_ms(), px, py, w, h); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - record_synth_cursor(px as i32, py as i32); - sess.vptr.destroy(); - sess.queue.roundtrip(&mut sess.state)?; - Ok(()) -} - -/// Press-drag-release on a native Wayland toplevel. Emits one button press at -/// `(from_x, from_y)`, then `steps` interpolated motion events along the -/// straight segment to `(to_x, to_y)`, then a release. Coordinates are -/// output-relative; window-local coords need the nested cua-compositor -/// injection socket (`CUA_INJECT_SOCKET`). -pub fn drag( - window_id: u64, - from_x: i32, - from_y: i32, - to_x: i32, - to_y: i32, - steps: u32, - duration_ms: u64, - button: u8, -) -> anyhow::Result<()> { - with_libei_fallback( - || drag_vptr(Some(window_id), from_x, from_y, to_x, to_y, steps, button), - || { - libei_wait_pointer_ready()?; - activate_window_for_input(window_id)?; - libei_drag(from_x, from_y, to_x, to_y, steps, duration_ms, button) - }, - ) -} - -/// Drag through desktop-absolute points without activating a named toplevel. -pub fn drag_desktop( - from_x: i32, - from_y: i32, - to_x: i32, - to_y: i32, - steps: u32, - duration_ms: u64, - button: u8, -) -> anyhow::Result<()> { - with_libei_fallback( - || drag_vptr(None, from_x, from_y, to_x, to_y, steps, button), - || { - libei_wait_pointer_ready()?; - libei_drag(from_x, from_y, to_x, to_y, steps, duration_ms, button) - }, - ) -} - -/// wlroots virtual-pointer implementation of [`drag`]. -#[allow(clippy::too_many_arguments)] -fn drag_vptr( - window_id: Option<u64>, - from_x: i32, - from_y: i32, - to_x: i32, - to_y: i32, - steps: u32, - button: u8, -) -> anyhow::Result<()> { - let mut sess = open_vptr_session(window_id)?; - std::thread::sleep(std::time::Duration::from_millis(40)); - let (w, h) = (sess.output_w, sess.output_h); - let btn = evdev_pointer_button(button); - let clamp_xy = |x: i32, y: i32| -> (u32, u32) { - ( - x.clamp(0, w as i32 - 1) as u32, - y.clamp(0, h as i32 - 1) as u32, - ) - }; - let (fx, fy) = clamp_xy(from_x, from_y); - sess.vptr.motion_absolute(event_time_ms(), fx, fy, w, h); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - std::thread::sleep(std::time::Duration::from_millis(15)); - sess.vptr.button(event_time_ms(), btn, ButtonState::Pressed); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - let n = steps.max(1); - for s in 1..=n { - let t = s as f64 / n as f64; - let ix = (from_x as f64 + (to_x - from_x) as f64 * t).round() as i32; - let iy = (from_y as f64 + (to_y - from_y) as f64 * t).round() as i32; - let (cx, cy) = clamp_xy(ix, iy); - sess.vptr.motion_absolute(event_time_ms(), cx, cy, w, h); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - std::thread::sleep(std::time::Duration::from_millis(8)); - } - let (tx, ty) = clamp_xy(to_x, to_y); - sess.vptr.motion_absolute(event_time_ms(), tx, ty, w, h); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - sess.vptr - .button(event_time_ms(), btn, ButtonState::Released); - sess.vptr.frame(); - // Sync the synthetic-cursor registry with the drag endpoint so a - // subsequent `get_cursor_position` reports where we left the pointer. - record_synth_cursor(tx as i32, ty as i32); - sess.queue.roundtrip(&mut sess.state)?; - sess.vptr.destroy(); - sess.queue.roundtrip(&mut sess.state)?; - Ok(()) -} - -/// Type Unicode text into the focused Wayland surface via `wtype` (the -/// virtual-keyboard tool — `zwp_virtual_keyboard_v1` under the hood; it builds -/// the xkb keymap and resolves shift levels for us). This mirrors the X11 -/// backend's XSendEvent typing and the capture slice's shell-out to `grim`. -/// foreign-toplevel exposes no pid and Wayland delivers keys to the *focused* -/// surface, so this is window_id-free; pair it with `click`/`activate` to put -/// the intended window in focus first. -pub fn type_text(window_id: u64, text: &str) -> anyhow::Result<()> { - if text.is_empty() { - return Ok(()); - } - activate_window_for_input(window_id)?; - // Lead with a no-op Shift_L tap: on a freshly-focused window under a headless - // seat (notably sway), the compositor needs the first virtual-keyboard event - // to wire up keyboard routing, and that first key is dropped. Sacrificing a - // modifier tap (no character) absorbs the drop so the real text lands intact; - // it's harmless where routing is already live (labwc). - let result = std::process::Command::new("wtype") - .args(["-k", "Shift_L", "--"]) - .arg(text) - .output(); - match result { - Ok(out) if out.status.success() => Ok(()), - // `wtype` relies on `zwp_virtual_keyboard_v1`, which KWin/Plasma and - // Mutter/GNOME don't implement (and the binary may be missing wtype - // entirely). On a portal-input build, route typing through libei's - // `ei_text` interface instead. See #1982. - other => with_wtype_libei_fallback( - || { - libei_wait_keyboard_ready()?; - activate_window_for_input(window_id)?; - libei_type_text(text) - }, - other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()), - ), - } -} - -/// Type into a surface whose exact Sway container is already held focused by -/// the caller. This avoids re-resolving a title that can change mid-sequence -/// (for example after opening a Chromium tab). -pub fn type_text_focused(text: &str) -> anyhow::Result<()> { - if text.is_empty() { - return Ok(()); - } - let result = std::process::Command::new("wtype") - .args(["-k", "Shift_L", "--"]) - .arg(text) - .output(); - match result { - Ok(out) if out.status.success() => Ok(()), - other => with_wtype_libei_fallback( - || { - libei_wait_keyboard_ready()?; - libei_type_text(text) - }, - other.map(|out| String::from_utf8_lossy(&out.stderr).into_owned()), - ), - } -} - -/// Type text and press one key while an outer exact-container focus guard is -/// active. Keeping both operations in one virtual-keyboard lifetime avoids a -/// headless wlroots seat dropping the first event from a second `wtype` -/// process after the text has landed. -pub fn type_text_then_key_focused(text: &str, key: &str) -> anyhow::Result<()> { - let keysym = key_to_keysym(key); - let result = std::process::Command::new("wtype") - .args(["-k", "Shift_L", "-s", "30"]) - .arg(text) - .args(["-s", "50", "-k", &keysym]) - .output(); - match result { - Ok(out) if out.status.success() => Ok(()), - other => with_wtype_libei_fallback( - || { - libei_wait_keyboard_ready()?; - libei_type_text(text)?; - libei_press_key(key) - }, - other.map(|out| String::from_utf8_lossy(&out.stderr).into_owned()), - ), - } -} - -/// Press a single named key into the focused Wayland surface via `wtype -k`. -pub fn press_key(window_id: u64, key: &str) -> anyhow::Result<()> { - activate_window_for_input(window_id)?; - let keysym = key_to_keysym(key); - // Keep the sacrificial modifier and requested key in one virtual-keyboard - // lifetime. Starting a second wtype process creates a fresh protocol object, - // causing headless seats to drop the requested key as their first event. - let result = std::process::Command::new("wtype") - .args(["-k", "Shift_L", "-k", &keysym]) - .output(); - match result { - Ok(out) if out.status.success() => Ok(()), - other => with_wtype_libei_fallback( - || { - libei_wait_keyboard_ready()?; - activate_window_for_input(window_id)?; - libei_press_key(key) - }, - other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()), - ), - } -} - -/// Press one key while an outer exact-container focus guard is active. -pub fn press_key_focused(key: &str) -> anyhow::Result<()> { - if is_inject_mode() { - let (pid, window_id) = inject_focused_target()?; - return inject_press_key(pid, window_id, key); - } - let keysym = key_to_keysym(key); - let result = std::process::Command::new("wtype") - .args(["-k", "Shift_L", "-k", &keysym]) - .output(); - match result { - Ok(out) if out.status.success() => Ok(()), - other => with_wtype_libei_fallback( - || { - libei_wait_keyboard_ready()?; - libei_press_key(key) - }, - other.map(|out| String::from_utf8_lossy(&out.stderr).into_owned()), - ), - } -} - -/// Press a key combination (modifiers + final key) via `wtype`. Each modifier -/// is pressed before the key, then released after, exactly like -/// `wtype -M ctrl -M shift -k key -m shift -m ctrl`. Unknown values pass -/// straight to wtype's `-k` so single-character keys and X keysym names work -/// as-is. This is the Wayland equivalent of the X11 `send_key` modifier mask. -pub fn hotkey(window_id: u64, keys: &[String]) -> anyhow::Result<()> { - activate_window_for_input(window_id)?; - let (mods, final_key) = partition_modifiers(keys)?; - if let Ok(()) = virtual_keyboard::hotkey(&mods, &final_key) { - return Ok(()); - } - let keysym = key_to_keysym(&final_key); - let args = wtype_hotkey_args(&mods, &keysym); - let result = std::process::Command::new("wtype").args(&args).output(); - match result { - Ok(out) if out.status.success() => Ok(()), - other => { - let stderr = other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()); - #[cfg(feature = "portal-input")] - { - return with_wtype_libei_fallback( - || { - libei::wait_keyboard_ready()?; - activate_window_for_input(window_id)?; - libei_hotkey(&mods, &final_key) - }, - stderr, - ); - } - #[cfg(not(feature = "portal-input"))] - { - anyhow::bail!( - "wtype {} failed: {}", - args.join(" "), - stderr.unwrap_or_else(|_| "wtype unavailable".into()) - ); - } - } - } -} - -/// Send a chord while an outer exact-container focus guard is active. -pub fn hotkey_focused(keys: &[String]) -> anyhow::Result<()> { - if is_inject_mode() { - let (pid, window_id) = inject_focused_target()?; - return inject_hotkey(pid, window_id, keys); - } - let (mods, final_key) = partition_modifiers(keys)?; - if let Ok(()) = virtual_keyboard::hotkey(&mods, &final_key) { - return Ok(()); - } - let keysym = key_to_keysym(&final_key); - let args = wtype_hotkey_args(&mods, &keysym); - let result = std::process::Command::new("wtype").args(&args).output(); - match result { - Ok(out) if out.status.success() => Ok(()), - other => { - let stderr = other.map(|out| String::from_utf8_lossy(&out.stderr).into_owned()); - #[cfg(feature = "portal-input")] - { - return with_wtype_libei_fallback( - || { - libei::wait_keyboard_ready()?; - libei_hotkey(&mods, &final_key) - }, - stderr, - ); - } - #[cfg(not(feature = "portal-input"))] - { - anyhow::bail!( - "wtype {} failed: {}", - args.join(" "), - stderr.unwrap_or_else(|_| "wtype unavailable".into()) - ); - } - } - } -} - -fn wtype_hotkey_args(mods: &[String], keysym: &str) -> Vec<String> { - // Keep the same harmless first-event primer used by `press_key`. A fresh - // virtual-keyboard object on headless seats can drop its first event. Give - // wlroots one event cycle after the primer and modifier transitions; - // otherwise Chromium can miss a coalesced shortcut even though wtype exits - // successfully. - let mut args: Vec<String> = vec!["-k".into(), "Shift_L".into(), "-s".into(), "30".into()]; - for m in mods { - args.push("-M".into()); - args.push(m.clone()); - } - args.push("-s".into()); - args.push("20".into()); - args.push("-k".into()); - args.push(keysym.to_owned()); - args.push("-s".into()); - args.push("20".into()); - // Release modifiers in reverse press order. - for m in mods.iter().rev() { - args.push("-m".into()); - args.push(m.clone()); - } - args -} - -/// Split a `keys` array into wtype-compatible modifier names and a single -/// final key. Recognised modifier inputs: ctrl/control, alt, shift, -/// super/meta/cmd/command/win/windows. The final key must be the one -/// non-modifier in the list. -fn partition_modifiers(keys: &[String]) -> anyhow::Result<(Vec<String>, String)> { - let mut mods: Vec<String> = Vec::new(); - let mut non_mods: Vec<String> = Vec::new(); - for k in keys { - match k.to_ascii_lowercase().as_str() { - "ctrl" | "control" => mods.push("ctrl".into()), - "alt" => mods.push("alt".into()), - "shift" => mods.push("shift".into()), - "super" | "meta" | "cmd" | "command" | "win" | "windows" => mods.push("logo".into()), - _ => non_mods.push(k.clone()), - } - } - let final_key = non_mods - .last() - .cloned() - .ok_or_else(|| anyhow::anyhow!("hotkey requires at least one non-modifier key"))?; - Ok((mods, final_key)) -} - -/// Map cua key names to X keysym names that `wtype -k` understands. Unknown -/// values pass through (single characters and valid keysym names work as-is). -fn key_to_keysym(key: &str) -> String { - match key.to_lowercase().as_str() { - "enter" | "return" => "Return", - "tab" => "Tab", - "esc" | "escape" => "Escape", - "space" => "space", - "backspace" => "BackSpace", - "delete" | "del" => "Delete", - "up" => "Up", - "down" => "Down", - "left" => "Left", - "right" => "Right", - "home" => "Home", - "end" => "End", - "pageup" | "page_up" => "Prior", - "pagedown" | "page_down" => "Next", - _ => return key.to_string(), - } - .to_string() -} - -// ── libei / portal fallback adapters ─────────────────────────────────────── -// -// These bridge the wlroots-shaped public input API (output-relative integer -// coordinates, cua button codes, X-keysym key names) onto the libei worker -// (`libei` module), which speaks logical device-region floats and evdev -// codes. They are the recovery path for compositors with no -// `zwlr_virtual_pointer_v1` (KWin/Plasma, Mutter/GNOME) — see #1982. -// -// In a build WITHOUT the `portal-input` feature the `libei` module does not -// exist, so each adapter compiles to an error stub. The dispatch seams above -// only ever CALL these inside `#[cfg(feature = "portal-input")]` branches, so -// the stubs are dead in that build; they exist purely so the closures passed -// to `with_libei_fallback` / `with_wtype_libei_fallback` type-check. - -/// libei recovery wrapper for the `wtype`-based typing/key functions: when the -/// virtual-keyboard shell-out failed (`wtype_err`), try the libei `run` on a -/// portal-input build, otherwise surface the original wtype failure. -fn with_wtype_libei_fallback( - #[allow(unused_variables)] run: impl FnOnce() -> anyhow::Result<()>, - wtype_err: Result<String, std::io::Error>, -) -> anyhow::Result<()> { - #[cfg(feature = "portal-input")] - { - match wtype_err { - Ok(stderr) => { - tracing::info!("wtype failed ({stderr}); falling back to libei/portal typing") - } - Err(e) => { - tracing::info!("wtype unavailable ({e}); falling back to libei/portal typing") - } - } - run() - } - #[cfg(not(feature = "portal-input"))] - { - let _ = run; - match wtype_err { - Ok(stderr) => anyhow::bail!("wtype failed: {stderr}"), - Err(e) => anyhow::bail!("wtype unavailable: {e}"), - } - } -} - -// Stubs for the no-feature build: the dispatch seams never call these (the -// libei branch in `with_libei_fallback` / `with_wtype_libei_fallback` is -// `#[cfg]`-d out), but the closures still need them to exist to type-check. -#[cfg(not(feature = "portal-input"))] -fn libei_wait_pointer_ready() -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -fn libei_wait_scroll_ready() -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -fn libei_wait_keyboard_ready() -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -fn libei_click(_x: i32, _y: i32, _count: u32, _button: u8) -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -fn libei_scroll(_direction: &str, _amount: u32) -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -fn libei_move_absolute(_x: i32, _y: i32) -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -#[allow(clippy::too_many_arguments)] -fn libei_drag( - _from_x: i32, - _from_y: i32, - _to_x: i32, - _to_y: i32, - _steps: u32, - _duration_ms: u64, - _button: u8, -) -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -fn libei_type_text(_text: &str) -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(not(feature = "portal-input"))] -fn libei_press_key(_key: &str) -> anyhow::Result<()> { - unreachable!("libei fallback compiled out (no portal-input feature)") -} -#[cfg(feature = "portal-input")] -fn cua_button_to_libei(button: u8) -> libei::Button { - match button { - 2 => libei::Button::Middle, - 3 => libei::Button::Right, - _ => libei::Button::Left, - } -} - -#[cfg(feature = "portal-input")] -fn libei_wait_pointer_ready() -> anyhow::Result<()> { - libei::wait_pointer_ready() -} - -#[cfg(feature = "portal-input")] -fn libei_wait_scroll_ready() -> anyhow::Result<()> { - libei::wait_scroll_ready() -} - -#[cfg(feature = "portal-input")] -fn libei_wait_keyboard_ready() -> anyhow::Result<()> { - libei::wait_keyboard_ready() -} - -#[cfg(feature = "portal-input")] -fn libei_click(x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> { - let btn = cua_button_to_libei(button); - let (w, h) = output_dimensions()?; - let (px, py) = normalize_click_xy(x, y, w, h); - libei::move_absolute(px as f64, py as f64)?; - for i in 0..count.max(1) { - if i > 0 { - std::thread::sleep(std::time::Duration::from_millis(80)); - } - libei::click(px as f64, py as f64, btn)?; - } - record_synth_cursor(px, py); - Ok(()) -} - -#[cfg(feature = "portal-input")] -fn libei_scroll(direction: &str, amount: u32) -> anyhow::Result<()> { - // libei scroll is logical-unit deltas; mirror the wlroots ±10/tick step. - let (dx, dy): (f64, f64) = match direction.to_ascii_lowercase().as_str() { - "up" => (0.0, -10.0), - "down" => (0.0, 10.0), - "left" => (-10.0, 0.0), - "right" => (10.0, 0.0), - other => anyhow::bail!("unknown scroll direction: {other}"), - }; - for i in 0..amount.max(1) { - if i > 0 { - std::thread::sleep(std::time::Duration::from_millis(25)); - } - libei::scroll(dx, dy)?; - } - Ok(()) -} - -#[cfg(feature = "portal-input")] -fn libei_move_absolute(x: i32, y: i32) -> anyhow::Result<()> { - // Match `move_cursor_absolute_vptr`: clamp to output bounds (no - // default-to-centre — an explicit (0,0) move means the top-left corner). - let (w, h) = output_dimensions()?; - let px = x.clamp(0, (w as i32).saturating_sub(1)); - let py = y.clamp(0, (h as i32).saturating_sub(1)); - libei::move_absolute(px as f64, py as f64)?; - record_synth_cursor(px, py); - Ok(()) -} - -#[cfg(feature = "portal-input")] -fn libei_drag( - from_x: i32, - from_y: i32, - to_x: i32, - to_y: i32, - steps: u32, - duration_ms: u64, - button: u8, -) -> anyhow::Result<()> { - // ei_button exposes separate Press/Released states, so the libei worker can - // hold the button across the interpolated motion — a genuine - // press→move→release drag. Clamp both endpoints to the output — but NOT via - // `normalize_click_xy`, whose (0,0)→centre convention (for coordinate-free - // clicks) is wrong here: a drag endpoint is always explicit and (0,0) is a - // valid top-left corner target. - let btn = cua_button_to_libei(button); - let (w, h) = output_dimensions()?; - let cx = |x: i32| x.clamp(0, (w as i32).saturating_sub(1)); - let cy = |y: i32| y.clamp(0, (h as i32).saturating_sub(1)); - libei::drag( - cx(from_x) as f64, - cy(from_y) as f64, - cx(to_x) as f64, - cy(to_y) as f64, - steps, - duration_ms, - btn, - )?; - record_synth_cursor(cx(to_x), cy(to_y)); - Ok(()) -} - -#[cfg(feature = "portal-input")] -fn libei_type_text(text: &str) -> anyhow::Result<()> { - if text.is_empty() { - return Ok(()); - } - libei::type_text(text) -} - -#[cfg(feature = "portal-input")] -fn libei_press_key(key: &str) -> anyhow::Result<()> { - let keycode = key_to_evdev(key) - .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}' (libei path)"))?; - libei::press_key(keycode) -} - -#[cfg(feature = "portal-input")] -fn libei_hotkey(mods: &[String], key: &str) -> anyhow::Result<()> { - use libei::KeyTransition::{Press, Release}; - - let mut modifier_codes = Vec::with_capacity(mods.len()); - for modifier in mods { - modifier_codes.push(match modifier.as_str() { - "ctrl" => 29, - "shift" => 42, - "alt" => 56, - "logo" => 125, - other => anyhow::bail!("no evdev keycode mapping for modifier '{other}'"), - }); - } - let keycode = key_to_evdev(key) - .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}' (libei path)"))?; - let mut transitions = Vec::with_capacity(modifier_codes.len() * 2 + 2); - transitions.extend(modifier_codes.iter().copied().map(Press)); - transitions.push(Press(keycode)); - transitions.push(Release(keycode)); - transitions.extend(modifier_codes.iter().rev().copied().map(Release)); - libei::key_sequence(&transitions) -} - -/// Map cua key names to Linux evdev keycodes for the libei `press_key` path -/// (libei emulates raw evdev, not X keysyms). Mirrors [`key_to_keysym`] but -/// emits `linux/input-event-codes.h` values. Returns `None` for keys with no -/// known mapping so the caller can fail loudly. -fn key_to_evdev(key: &str) -> Option<u32> { - let code = match key.to_lowercase().as_str() { - "enter" | "return" => 28, // KEY_ENTER - "tab" => 15, // KEY_TAB - "esc" | "escape" => 1, // KEY_ESC - "space" => 57, // KEY_SPACE - "backspace" => 14, // KEY_BACKSPACE - "delete" | "del" => 111, // KEY_DELETE - "up" => 103, // KEY_UP - "down" => 108, // KEY_DOWN - "left" => 105, // KEY_LEFT - "right" => 106, // KEY_RIGHT - "home" => 102, // KEY_HOME - "end" => 107, // KEY_END - "pageup" | "page_up" => 104, // KEY_PAGEUP - "pagedown" | "page_down" => 109, // KEY_PAGEDOWN - // Letters a-z. evdev codes follow the QWERTY scancode layout, not the - // alphabet, so each is listed explicitly (linux/input-event-codes.h). - "a" => 30, // KEY_A - "b" => 48, // KEY_B - "c" => 46, // KEY_C - "d" => 32, // KEY_D - "e" => 18, // KEY_E - "f" => 33, // KEY_F - "g" => 34, // KEY_G - "h" => 35, // KEY_H - "i" => 23, // KEY_I - "j" => 36, // KEY_J - "k" => 37, // KEY_K - "l" => 38, // KEY_L - "m" => 50, // KEY_M - "n" => 49, // KEY_N - "o" => 24, // KEY_O - "p" => 25, // KEY_P - "q" => 16, // KEY_Q - "r" => 19, // KEY_R - "s" => 31, // KEY_S - "t" => 20, // KEY_T - "u" => 22, // KEY_U - "v" => 47, // KEY_V - "w" => 17, // KEY_W - "x" => 45, // KEY_X - "y" => 21, // KEY_Y - "z" => 44, // KEY_Z - // Digits. KEY_1=2 .. KEY_9=10, KEY_0=11 (input-event-codes.h). - "1" => 2, // KEY_1 - "2" => 3, // KEY_2 - "3" => 4, // KEY_3 - "4" => 5, // KEY_4 - "5" => 6, // KEY_5 - "6" => 7, // KEY_6 - "7" => 8, // KEY_7 - "8" => 9, // KEY_8 - "9" => 10, // KEY_9 - "0" => 11, // KEY_0 - // Function keys. KEY_F1=59 .. KEY_F10=68, then KEY_F11=87, KEY_F12=88. - "f1" => 59, - "f2" => 60, - "f3" => 61, - "f4" => 62, - "f5" => 63, - "f6" => 64, - "f7" => 65, - "f8" => 66, - "f9" => 67, - "f10" => 68, - "f11" => 87, - "f12" => 88, - _ => return None, - }; - Some(code) -} - -// ── Nested cua-compositor injection ──────────────────────────────────────── -// -// When cua-driver's nested compositor is `cua-compositor` (our patched wlroots, -// see nix/cua-driver/compositor/), it exposes a line-protocol control socket at -// $CUA_INJECT_SOCKET for what stock Wayland forbids: focus-FREE per-surface -// keyboard injection and MULTI-cursor pointer injection, routed by stable PID -// when available and xdg app_id only as a fallback. These helpers speak that -// protocol. -// -// The protocol is a simple line-based v1 exchange with per-command -// acknowledgement: the client sends `INJECT_PROTO_HELLO` and the compositor -// echoes it (or replies `err ...`), then every command line is answered by -// exactly one `ok` / `err <reason>` line. The client fails on a protocol -// mismatch, a read timeout, an EOF, or any compositor error line — an -// acknowledgement is transport evidence only, not proof the target changed. - -/// Version banner exchanged at connect time: the client sends this line and the -/// compositor must echo it back verbatim to confirm both speak v1. -const INJECT_PROTO_HELLO: &str = "cua-inject v1"; - -/// The exact named keys the nested compositor's `k` command can emit — the -/// whitelist in `cua_key_named` (cua_compositor_patch.py). Compared -/// case-insensitively, matching the compositor's `strcasecmp`. -const INJECT_NAMED_KEYS: &[&str] = &[ - "enter", - "return", - "tab", - "escape", - "esc", - "backspace", - "space", - "up", - "down", - "left", - "right", - "f1", - "f2", - "f3", - "f4", - "f5", - "f6", - "f7", - "f8", - "f9", - "f10", - "f11", - "f12", -]; - -/// The control socket path, when running against the nested cua-compositor. -pub fn inject_socket_path() -> Option<String> { - std::env::var("CUA_INJECT_SOCKET") - .ok() - .filter(|s| !s.is_empty()) -} - -/// True when input should be routed through the nested cua-compositor's control -/// socket (focus-free / multi-cursor) rather than wtype / virtual-pointer. -pub fn is_inject_mode() -> bool { - inject_socket_path().is_some() -} - -static INJECT_FOCUSED_TARGET: OnceLock<Mutex<Option<(u32, u64)>>> = OnceLock::new(); - -fn remember_inject_focused_target(pid: u32, window_id: u64) { - let target = INJECT_FOCUSED_TARGET.get_or_init(|| Mutex::new(None)); - if let Ok(mut target) = target.lock() { - *target = Some((pid, window_id)); - } -} - -fn inject_focused_target() -> anyhow::Result<(u32, u64)> { - INJECT_FOCUSED_TARGET - .get_or_init(|| Mutex::new(None)) - .lock() - .ok() - .and_then(|target| *target) - .ok_or_else(|| { - anyhow::anyhow!( - "foreground_unavailable: cua-compositor has no verified foreground target; \ - call bring_to_front before desktop keyboard input" - ) - }) -} - -fn inject_scroll_desktop(x: i32, y: i32, direction: &str, amount: u32) -> anyhow::Result<()> { - let windows = crate::atspi::list_windows(None); - let target = windows - .iter() - .filter(|window| { - window.is_on_screen - && x >= window.x - && y >= window.y - && x < window.x.saturating_add(window.width as i32) - && y < window.y.saturating_add(window.height as i32) - }) - .max_by_key(|window| window.z_index.unwrap_or_default()) - .or_else(|| { - let (pid, _) = inject_focused_target().ok()?; - windows.iter().find(|window| window.pid == Some(pid)) - }) - .ok_or_else(|| { - anyhow::anyhow!( - "foreground_unavailable: no cua-compositor window contains desktop point ({x},{y})" - ) - })?; - let pid = target.pid.ok_or_else(|| { - anyhow::anyhow!( - "foreground_unavailable: desktop point ({x},{y}) resolved to a window without a pid" - ) - })?; - inject_scroll( - pid, - target.xid, - f64::from(x.saturating_sub(target.x)), - f64::from(y.saturating_sub(target.y)), - direction, - amount, - ) -} - -/// Reject any character the nested compositor cannot type before it reaches the -/// wire. The compositor's chartab (`cua_init_keymap`) only covers printable -/// ASCII (`0x20..=0x7E`) plus newline and tab; anything else — Unicode, other -/// control bytes — would be silently dropped, so fail loudly instead. -fn validate_injectable_text(text: &str) -> anyhow::Result<()> { - for ch in text.chars() { - let ok = ch == '\n' || ch == '\t' || (ch.is_ascii() && !ch.is_ascii_control()); - if !ok { - anyhow::bail!( - "cua-compositor cannot type {ch:?}: only printable ASCII plus newline and tab \ - are supported in the v1 injection protocol" - ); - } - } - Ok(()) -} - -/// Reject any key name outside the compositor's named-key whitelist before -/// sending. Mirrors `cua_key_named`; unsupported names must fail here rather -/// than being silently ignored by the compositor. -fn validate_injectable_key(key: &str) -> anyhow::Result<()> { - let normalized = key.trim().to_ascii_lowercase(); - if INJECT_NAMED_KEYS.contains(&normalized.as_str()) { - Ok(()) - } else { - anyhow::bail!( - "cua-compositor does not support key {key:?}; supported keys: {}", - INJECT_NAMED_KEYS.join(", ") - ); - } -} - -fn validate_injectable_hotkey(keys: &[String]) -> anyhow::Result<(String, String)> { - let (key, modifiers) = keys - .split_last() - .ok_or_else(|| anyhow::anyhow!("cua-compositor hotkey requires a non-modifier key"))?; - let key = key.trim().to_ascii_lowercase(); - if !(key.len() == 1 && key.is_ascii()) && !INJECT_NAMED_KEYS.contains(&key.as_str()) { - anyhow::bail!("cua-compositor does not support hotkey key {key:?}"); - } - let mut normalized = Vec::with_capacity(modifiers.len()); - for modifier in modifiers { - let modifier = modifier.trim().to_ascii_lowercase(); - let canonical = match modifier.as_str() { - "ctrl" | "control" => "ctrl", - "shift" => "shift", - "alt" | "option" => "alt", - "meta" | "super" | "win" | "cmd" => "meta", - _ => anyhow::bail!("cua-compositor does not support modifier {modifier:?}"), - }; - normalized.push(canonical); - } - if normalized.is_empty() { - anyhow::bail!("cua-compositor hotkey requires at least one modifier"); - } - Ok((normalized.join(","), key)) -} - -/// Interpret the compositor's handshake reply. Accepts only the verbatim v1 -/// banner; a compositor `err ...` line or anything else is a protocol mismatch. -fn parse_inject_hello(line: &str) -> anyhow::Result<()> { - let trimmed = line.trim(); - if trimmed == INJECT_PROTO_HELLO { - Ok(()) - } else if let Some(reason) = trimmed.strip_prefix("err") { - anyhow::bail!( - "cua-compositor rejected the v1 handshake:{}", - if reason.trim().is_empty() { - String::new() - } else { - format!(" {}", reason.trim()) - } - ) - } else { - anyhow::bail!( - "cua-compositor protocol mismatch: expected {INJECT_PROTO_HELLO:?}, got {trimmed:?}" - ) - } -} - -/// Interpret a single per-command acknowledgement line. `ok` succeeds; `err -/// <reason>` and any unrecognised line fail. -fn parse_inject_reply(line: &str) -> anyhow::Result<()> { - let trimmed = line.trim(); - if trimmed == "ok" { - Ok(()) - } else if let Some(reason) = trimmed.strip_prefix("err") { - let reason = reason.trim(); - if reason.is_empty() { - anyhow::bail!("cua-compositor rejected the command"); - } - anyhow::bail!("cua-compositor rejected the command: {reason}") - } else { - anyhow::bail!("unexpected cua-compositor response: {trimmed:?}") - } -} - -/// Read one newline-terminated response line, mapping timeout and EOF to clear -/// errors so the caller never blocks forever on an unresponsive compositor. -fn read_inject_line(reader: &mut impl std::io::BufRead) -> anyhow::Result<String> { - let mut line = String::new(); - let n = reader.read_line(&mut line).map_err(|e| { - if matches!( - e.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) { - anyhow::anyhow!("cua-compositor did not respond within the timeout") - } else { - anyhow::anyhow!("cua-compositor read failed: {e}") - } - })?; - if n == 0 { - anyhow::bail!("cua-compositor closed the connection before responding"); - } - Ok(line) -} - -/// Connect to the nested cua-compositor control socket, perform the v1 -/// handshake, then send each command line and require exactly one -/// acknowledgement per command. Fails on protocol mismatch, timeout, EOF, or a -/// compositor error line — the earlier fire-and-forget path hid all of these. -fn inject_exchange(lines: &[String]) -> anyhow::Result<Vec<String>> { - use std::io::{BufReader, Write}; - use std::os::unix::net::UnixStream; - let path = inject_socket_path().ok_or_else(|| anyhow::anyhow!("CUA_INJECT_SOCKET not set"))?; - // The nested compositor may still be starting; retry the connect briefly. - let mut stream = None; - for _ in 0..60 { - match UnixStream::connect(&path) { - Ok(s) => { - stream = Some(s); - break; - } - Err(_) => std::thread::sleep(std::time::Duration::from_millis(50)), - } - } - let stream = - stream.ok_or_else(|| anyhow::anyhow!("could not connect to inject socket {path}"))?; - stream.set_read_timeout(Some(std::time::Duration::from_secs(5)))?; - let mut writer = stream.try_clone()?; - let mut reader = BufReader::new(stream); - - // v1 handshake: send our banner and require the compositor to echo it. - writeln!(writer, "{INJECT_PROTO_HELLO}")?; - writer.flush()?; - parse_inject_hello(&read_inject_line(&mut reader)?)?; - - let mut replies = Vec::with_capacity(lines.len()); - // One command per line; block on its response before the next. - for l in lines { - writeln!(writer, "{l}")?; - writer.flush()?; - replies.push(read_inject_line(&mut reader)?); - } - Ok(replies) -} - -fn inject_send(lines: &[String]) -> anyhow::Result<()> { - for reply in inject_exchange(lines)? { - parse_inject_reply(&reply)?; - } - Ok(()) -} - -fn parse_inject_geometry(line: &str) -> anyhow::Result<((i32, i32), (i32, i32))> { - let fields = line.split_whitespace().collect::<Vec<_>>(); - if fields.len() == 5 && fields[0] == "geometry" { - return Ok(( - (fields[1].parse()?, fields[2].parse()?), - (fields[3].parse()?, fields[4].parse()?), - )); - } - if let Some(reason) = line.trim().strip_prefix("err") { - anyhow::bail!("cua-compositor geometry query failed: {}", reason.trim()); - } - anyhow::bail!( - "unexpected cua-compositor geometry response: {:?}", - line.trim() - ) -} - -/// Return the offset that rebases native Wayland accessibility coordinates into -/// the nested compositor's root-surface/output coordinate space. -pub fn inject_accessibility_offset(pid: u32) -> Option<(i32, i32)> { - if !is_inject_mode() || pid == 0 { - return None; - } - let replies = inject_exchange(&[format!("g {pid}")]).ok()?; - replies - .first() - .and_then(|line| parse_inject_geometry(line).ok()) - .map(|geometry| geometry.0) -} - -fn inject_window_origin(pid: u32) -> Option<(i32, i32)> { - if !is_inject_mode() || pid == 0 { - return None; - } - let replies = inject_exchange(&[format!("g {pid}")]).ok()?; - replies - .first() - .and_then(|line| parse_inject_geometry(line).ok()) - .map(|geometry| geometry.1) -} - -/// Resolve a window_id to its xdg app_id via the stable identity registry that -/// [`list_windows`] populates (falling back to sway IPC / AT-SPI through -/// [`identity_for`]) — the nested cua-compositor injection protocol addresses -/// windows by app_id. Returns `None` when no identity is registered or the -/// resolved app_id is empty, so callers can surface a clear error. -pub fn app_id_for_window(window_id: u64) -> Option<String> { - identity_for(window_id) - .map(|identity| identity.app_id) - .filter(|s| !s.is_empty()) -} - -/// Resolve the WM_CLASS-equivalent (instance, class) pair for a window. On -/// X11 reads `WM_CLASS`; on Wayland reuses [`app_id_for_window`] and returns -/// `(app_id, app_id)` — the closest analogue, since foreign-toplevel exposes -/// a single app_id and not the X11 instance/class split. Used by terminal -/// emulator detection on `is_terminal_window` so Ghostty / kitty / alacritty -/// are recognised on Wayland too. -pub fn wm_class_dispatch(window_id: u64) -> Option<(String, String)> { - if is_wayland() { - let app = app_id_for_window(window_id)?; - return Some((app.clone(), app)); - } - crate::x11::wm_class_for_window(window_id) -} - -fn to_hex(s: &str) -> String { - s.bytes().map(|b| format!("{b:02x}")).collect() -} - -/// Map a cua/X11 mouse-button number to its evdev (wl_pointer) button code. -fn evdev_button(x_button: u32) -> u32 { - match x_button { - 3 => 0x111, // BTN_RIGHT - 2 => 0x112, // BTN_MIDDLE - _ => 0x110, // BTN_LEFT - } -} - -/// Sentinel error when a window_id has no registered cua-compositor identity. -fn no_app_id(window_id: u64) -> anyhow::Error { - anyhow::anyhow!( - "no known cua-compositor app_id for window {window_id}; call list_windows first so its \ - Wayland identity is registered" - ) -} - -/// Resolve the strongest target token understood by the private nested -/// compositor. AT-SPI window IDs are synthetic on Wayland, but its process ID -/// is the same credential the compositor observes on the owning wl_client. -/// Fall back to app_id for clients whose accessibility metadata has no PID. -pub fn inject_target_for_window(window_id: u64) -> anyhow::Result<String> { - inject_target_for_window_with_pid(window_id, None) -} - -fn inject_target_for_window_with_pid( - window_id: u64, - target_pid: Option<u32>, -) -> anyhow::Result<String> { - if let Some(pid) = target_pid { - anyhow::ensure!(pid > 0, "cua-compositor target pid must be positive"); - // Electron/Chromium may create the xdg_toplevel from a renderer child - // rather than the public tool target. The compositor verifies the - // wl_client owner is this process or one of its descendants. - return Ok(format!("root:{pid}")); - } - let atspi = crate::atspi::list_windows(None); - let direct_pid = atspi - .iter() - .find(|window| window.xid == window_id) - .and_then(|window| window.pid); - let correlated_pid = identity_for(window_id) - .as_ref() - .and_then(|identity| unique_atspi_pid_for_identity(identity, &atspi)); - if let Some(pid) = direct_pid.or(correlated_pid) { - return Ok(format!("pid:{pid}")); - } - app_id_for_window(window_id).ok_or_else(|| no_app_id(window_id)) -} - -/// Correlate a connection-local native toplevel with its AT-SPI process. Exact -/// titles are the same bridge used by window enumeration; requiring one unique -/// PID prevents a shared toolkit app_id from silently selecting another app. -fn unique_atspi_pid_for_identity( - identity: &ToplevelIdentity, - windows: &[WindowInfo], -) -> Option<u32> { - if identity.title.is_empty() { - return None; - } - let mut pids = windows - .iter() - .filter(|window| window.title == identity.title) - .filter_map(|window| window.pid) - .collect::<Vec<_>>(); - pids.sort_unstable(); - pids.dedup(); - (pids.len() == 1).then(|| pids[0]) -} - -/// Focus-free type into the window's surface (no focus change). Rejects any -/// character the compositor cannot emit before touching the socket. -pub fn inject_type_text(target_pid: u32, window_id: u64, text: &str) -> anyhow::Result<()> { - validate_injectable_text(text)?; - let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; - inject_send(&[format!("t {app} {}", to_hex(text))]) -} - -/// Focus-free named-key press into the window's surface. Rejects any key -/// outside the compositor's whitelist before touching the socket. -pub fn inject_press_key(target_pid: u32, window_id: u64, key: &str) -> anyhow::Result<()> { - validate_injectable_key(key)?; - let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; - inject_send(&[format!("k {app} {}", key.trim())]) -} - -/// Focus-free modifier chord into the target surface. -pub fn inject_hotkey(target_pid: u32, window_id: u64, keys: &[String]) -> anyhow::Result<()> { - let (modifiers, key) = validate_injectable_hotkey(keys)?; - let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; - inject_send(&[format!("h {app} {modifiers} {key}")]) -} - -/// Focus-free wheel/axis input at one target-local point. -pub fn inject_scroll( - target_pid: u32, - window_id: u64, - x: f64, - y: f64, - direction: &str, - amount: u32, -) -> anyhow::Result<()> { - let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; - let (axis, value) = match direction.to_ascii_lowercase().as_str() { - "up" => (0, -15.0), - "down" | "page" => (0, 15.0), - "left" => (1, -15.0), - "right" => (1, 15.0), - _ => anyhow::bail!("unsupported cua-compositor scroll direction {direction:?}"), - }; - let mut lines = vec![format!("m {app} 0 {x:.1} {y:.1}")]; - lines.extend((0..amount.max(1)).map(|_| format!("a {app} 0 {axis} {value:.1}"))); - inject_send(&lines) -} - -/// Focus-free click into the window's surface via the nested cua-compositor. -/// Coordinates are window-local, matching the rest of the inject protocol. -pub fn inject_click( - target_pid: u32, - window_id: u64, - x: f64, - y: f64, - count: u32, - button: u8, -) -> anyhow::Result<()> { - let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; - let btn = evdev_button(button as u32); - let n = count.max(1); - let mut lines = Vec::with_capacity((n as usize) * 4); - for i in 0..n { - if i > 0 { - // The line protocol is batch-oriented, so use a tiny move-only - // separator between clicks to give the compositor a frame boundary - // without introducing a protocol-level sleep primitive. - lines.push(format!("m {app} 0 {x:.1} {y:.1}")); - } - lines.push(format!("m {app} 0 {x:.1} {y:.1}")); - lines.push(format!("b {app} 0 {btn} 1")); - lines.push(format!("b {app} 0 {btn} 0")); - } - inject_send(&lines) -} - -/// A single pointer drag for `inject_parallel_drags`: window-local waypoints, -/// driven by cursor `idx` so several run concurrently on one window. -pub struct InjectDrag { - pub app_id: String, - pub idx: usize, - pub x_button: u32, - pub path: Vec<(f64, f64)>, - pub steps: usize, -} - -fn resample(path: &[(f64, f64)], steps: usize) -> Vec<(f64, f64)> { - if path.len() < 2 || steps == 0 { - return path.to_vec(); - } - let seglen: Vec<f64> = path - .windows(2) - .map(|w| ((w[1].0 - w[0].0).powi(2) + (w[1].1 - w[0].1).powi(2)).sqrt()) - .collect(); - let total: f64 = seglen.iter().sum(); - if total == 0.0 { - return vec![path[0]; steps + 1]; - } - let mut out = Vec::with_capacity(steps + 1); - for s in 0..=steps { - let target = total * (s as f64) / (steps as f64); - let mut acc = 0.0; - let mut pt = path[path.len() - 1]; - for (i, &l) in seglen.iter().enumerate() { - if acc + l >= target || i == seglen.len() - 1 { - let f = if l > 0.0 { (target - acc) / l } else { 0.0 }; - pt = ( - path[i].0 + (path[i + 1].0 - path[i].0) * f, - path[i].1 + (path[i + 1].1 - path[i].1) * f, - ); - break; - } - acc += l; - } - out.push(pt); - } - out -} - -/// Run N pointer drags concurrently on their target windows: each cursor presses -/// at its start, glides through its (interleaved) waypoints, and releases. This -/// is true multi-cursor — each `idx` is an independent cursor in the compositor. -pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> { - if drags.is_empty() { - return Ok(()); - } - let resampled: Vec<Vec<(f64, f64)>> = drags - .iter() - .map(|d| resample(&d.path, d.steps.max(1))) - .collect(); - let max_steps = resampled.iter().map(|p| p.len()).max().unwrap_or(0); - let mut lines = Vec::new(); - // Press each cursor at its start point. - for (d, pts) in drags.iter().zip(&resampled) { - let (x, y) = pts[0]; - lines.push(format!("m {} {} {x:.1} {y:.1}", d.app_id, d.idx)); - lines.push(format!( - "b {} {} {} 1", - d.app_id, - d.idx, - evdev_button(d.x_button) - )); - } - // Glide all cursors together, one interleaved step at a time. - for s in 1..max_steps { - for (d, pts) in drags.iter().zip(&resampled) { - let (x, y) = pts[s.min(pts.len() - 1)]; - lines.push(format!("m {} {} {x:.1} {y:.1}", d.app_id, d.idx)); - } - } - // Release each cursor. - for (d, _) in drags.iter().zip(&resampled) { - lines.push(format!( - "b {} {} {} 0", - d.app_id, - d.idx, - evdev_button(d.x_button) - )); - } - inject_send(&lines) -} - -/// Focus-free single drag using the same per-surface path as parallel drags. -pub fn inject_drag( - target_pid: u32, - window_id: u64, - from: (f64, f64), - to: (f64, f64), - steps: usize, - x_button: u32, -) -> anyhow::Result<()> { - let app_id = inject_target_for_window_with_pid(window_id, Some(target_pid))?; - inject_parallel_drags(&[InjectDrag { - app_id, - idx: 0, - x_button, - path: vec![from, to], - steps, - }]) -} - -fn wayland_atspi_windows(filter_pid: Option<u32>) -> Vec<WindowInfo> { - let mut windows = crate::atspi::list_windows(filter_pid); - // AT-SPI can retain a toolkit's default placement (commonly 120,120) - // after Sway has placed the real toplevel at another origin. Reconcile the - // fallback records with compositor-owned metadata before exposing them to - // callers; element bounds already use this same authoritative Sway tree. - for window in &mut windows { - let sway = window - .pid - .and_then(sway_ipc::window_for_pid) - .or_else(|| sway_ipc::window_for_title(&window.title)) - .or_else(|| sway_ipc::window_for_app_id(&window.app_name)); - if let Some(sway) = sway { - window.xid = sway.id; - window.x = sway.x; - window.y = sway.y; - window.width = sway.width; - window.height = sway.height; - window.is_on_screen = sway.visible && sway.width > 0 && sway.height > 0; - } - } - if is_inject_mode() { - for window in &mut windows { - if let Some(pid) = window.pid { - if let Some((window_x, window_y)) = inject_window_origin(pid) { - window.x = window_x; - window.y = window_y; - } - } - } - } - windows -} - -/// Window-enumeration dispatcher: native Wayland when available, else X11. -pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> { - if wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() { - // Prefer the richer wlroots protocol. The generic staging protocol is - // only consulted when wlroots yields no windows (including when its - // manager global is absent). - let native = match list_windows() { - Ok(ws) if !ws.is_empty() => Ok(enrich_native_windows( - ws, - wayland_atspi_windows(filter_pid), - is_inject_mode(), - )), - Ok(_) => ext_toplevel::list_windows(), - Err(wlr_error) => ext_toplevel::list_windows().map_err(|ext_error| { - anyhow::anyhow!( - "wlr provider failed ({wlr_error}); ext provider failed ({ext_error})" - ) - }), - }; - match native { - Ok(ws) if !ws.is_empty() => { - if let Some(pid) = filter_pid { - if let Some(filtered) = native_windows_for_pid(ws, pid) { - return filtered; - } - } else { - return ws; - } - // A compositor window without pid metadata cannot satisfy a - // pid-scoped request. Continue to the AT-SPI registry. - let ws = wayland_atspi_windows(filter_pid); - if !ws.is_empty() { - return ws; - } - } - Ok(_) => { - if let Some(ws) = shell_helper::list_windows(filter_pid).filter(|ws| !ws.is_empty()) - { - return ws; - } - let ws = wayland_atspi_windows(filter_pid); - if !ws.is_empty() { - return ws; - } - } - Err(e) => { - if let Some(ws) = shell_helper::list_windows(filter_pid).filter(|ws| !ws.is_empty()) - { - tracing::debug!( - "native Wayland protocols unavailable ({e}); using compositor helper" - ); - return ws; - } - tracing::warn!("native Wayland list_windows failed: {e}; trying AT-SPI registry"); - let ws = wayland_atspi_windows(filter_pid); - if !ws.is_empty() { - return ws; - } - } - } - // Last resort under Wayland: an Xwayland app may still have an X11 XID. - } - // If native enumeration and its AT-SPI fallback found nothing, X11 may still - // expose XWayland clients. Merge one final AT-SPI snapshot so native windows - // remain visible on hybrid sessions even when neither foreign-toplevel - // protocol is advertised (#1978). Gated on the native-Wayland opt-in. - // - // Caveats for the merged AT-SPI entries: they carry a synthetic (non-X11) - // xid and zero geometry (x/y/w/h = 0), like the existing wlroots AT-SPI - // fallback — so `bring_to_front` / `screenshot_window` / pixel translation - // against them error cleanly rather than acting (input on GNOME/KDE routes - // by pid + screen coords, not xid, so it's unaffected). Dedup is per-pid, so - // the rare app owning BOTH an XWayland window and a separate native-Wayland - // toplevel would list only the XWayland one. - let mut ws = crate::x11::list_windows(filter_pid); - if wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() { - let seen: std::collections::HashSet<u32> = ws.iter().filter_map(|w| w.pid).collect(); - // A specific pid already resolved via X11 needs no AT-SPI walk (a full - // D-Bus enumeration of every registered app): it can only add duplicates. - let already_covered = filter_pid.map_or(false, |p| seen.contains(&p)); - if !already_covered { - merge_atspi_windows(&mut ws, &seen, wayland_atspi_windows(filter_pid)); - } - } - ws -} - -fn merge_atspi_windows( - windows: &mut Vec<WindowInfo>, - x11_pids: &std::collections::HashSet<u32>, - atspi_windows: Vec<WindowInfo>, -) { - for window in atspi_windows { - // XWayland apps appear in both lists; keep the X11 entry (real XID + - // geometry) and retain every native frame whose pid X11 did not expose. - if window.pid.is_none_or(|pid| !x11_pids.contains(&pid)) { - windows.push(window); - } - } -} - -fn enrich_native_windows( - mut native: Vec<WindowInfo>, - atspi: Vec<WindowInfo>, - adopt_atspi_ids: bool, -) -> Vec<WindowInfo> { - let mut claimed = std::collections::HashSet::new(); - for window in &mut native { - if window.pid.is_some() { - continue; - } - let native_title = undecorated_native_title(window); - let title_match = atspi.iter().enumerate().find_map(|(index, candidate)| { - (!claimed.contains(&index) - && !native_title.is_empty() - && candidate.title == native_title) - .then_some(index) - }); - let app_match = title_match.or_else(|| { - let matches = atspi - .iter() - .enumerate() - .filter(|(index, candidate)| { - !claimed.contains(index) - && !window.app_name.is_empty() - && candidate.app_name == window.app_name - }) - .map(|(index, _)| index) - .collect::<Vec<_>>(); - (matches.len() == 1).then(|| matches[0]) - }); - let Some(index) = app_match else { continue }; - claimed.insert(index); - let candidate = &atspi[index]; - window.pid = candidate.pid; - if adopt_atspi_ids { - let toplevel = Toplevel { - title: undecorated_native_title(window).to_owned(), - app_id: window.app_name.clone(), - closed: false, - }; - window.xid = candidate.xid; - remember_identity(window.xid, &toplevel); - } - if window.width == 0 || window.height == 0 { - window.x = candidate.x; - window.y = candidate.y; - window.width = candidate.width; - window.height = candidate.height; - } - if adopt_atspi_ids { - if let Some(pid) = candidate.pid { - if let Some((window_x, window_y)) = inject_window_origin(pid) { - window.x = window_x; - window.y = window_y; - } - } - } - window.is_on_screen = candidate.is_on_screen; - } - native -} - -fn undecorated_native_title(window: &WindowInfo) -> &str { - if window.app_name.is_empty() { - return &window.title; - } - let suffix = format!(" [{}]", window.app_name); - window.title.strip_suffix(&suffix).unwrap_or(&window.title) -} - -/// Return native records only when they contain a real match for a pid-scoped -/// request. Ext records whose AT-SPI merge left pid unknown must not suppress -/// the later AT-SPI and X11 fallback providers. -fn native_windows_for_pid(windows: Vec<WindowInfo>, pid: u32) -> Option<Vec<WindowInfo>> { - let matching: Vec<_> = windows - .into_iter() - .filter(|window| window.pid == Some(pid)) - .collect(); - (!matching.is_empty()).then_some(matching) -} - -/// Snapshot of which wlroots manager globals the running compositor advertises. -/// Used by the `health_report` Wayland backend probe to distinguish a working -/// session from one missing screencopy or virtual-pointer support. -#[derive(Default, Clone, Debug)] -pub struct WaylandManagers { - pub foreign_toplevel: bool, - pub screencopy: bool, - pub virtual_pointer: bool, - pub wl_shm: bool, - /// Staging `ext-image-copy-capture-v1` manager — sway 1.10+, labwc - /// 0.8+, niri, KDE 6.2+, GNOME mutter 47+. - pub ext_image_copy_capture: bool, - /// Companion `ext-output-image-capture-source-v1` source manager — - /// required to capture a `wl_output` via the staging protocol. - pub ext_output_image_capture_source: bool, -} - -/// Perform a single registry roundtrip and report which of the manager -/// interfaces the doctor cares about advertise themselves. Returns `Err` only -/// when we can't even open a Wayland connection — a successful connect with -/// no managers still resolves to an all-false snapshot. -pub fn probe_managers() -> anyhow::Result<WaylandManagers> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - let mut state = State::default(); - queue.roundtrip(&mut state)?; - // Reuse the existing State for wlroots managers, then do a parallel - // probe for staging ext-image-copy-capture interfaces by walking the - // raw registry events (no binding required — we only need presence). - let ext_probe = probe_ext_interfaces().unwrap_or_default(); - Ok(WaylandManagers { - foreign_toplevel: state.manager.is_some(), - screencopy: state.scrcopy_manager.is_some(), - virtual_pointer: state.vptr_manager.is_some(), - wl_shm: state.shm.is_some(), - ext_image_copy_capture: ext_probe.image_copy_capture, - ext_output_image_capture_source: ext_probe.output_image_capture_source, - }) -} - -#[derive(Default, Clone, Copy, Debug)] -struct ExtInterfaceProbe { - image_copy_capture: bool, - output_image_capture_source: bool, -} - -/// Probe registry for `ext-image-copy-capture-v1` + companion source manager -/// presence without binding them. Cheap (one roundtrip) and side-effect -/// free. -fn probe_ext_interfaces() -> anyhow::Result<ExtInterfaceProbe> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<ExtProbeState>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - let mut state = ExtProbeState::default(); - queue.roundtrip(&mut state)?; - Ok(state.probe) -} - -#[derive(Default)] -struct ExtProbeState { - probe: ExtInterfaceProbe, -} - -impl Dispatch<wl_registry::WlRegistry, ()> for ExtProbeState { - fn event( - state: &mut Self, - _: &wl_registry::WlRegistry, - event: wl_registry::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - if let wl_registry::Event::Global { interface, .. } = event { - match interface.as_str() { - "ext_image_copy_capture_manager_v1" => { - state.probe.image_copy_capture = true; - } - "ext_output_image_capture_source_manager_v1" => { - state.probe.output_image_capture_source = true; - } - _ => {} - } - } - } -} - -// Suppress dead-code warning for the unused BTN_LEFT alias kept for backward -// compatibility with earlier slice constants. -#[allow(dead_code)] -const _BTN_LEFT_ALIAS: u32 = BTN_LEFT; - -#[cfg(test)] -mod tests { - use super::*; - - fn window(xid: u64, pid: Option<u32>, title: &str) -> WindowInfo { - WindowInfo { - xid, - pid, - app_name: String::new(), - title: title.to_owned(), - is_on_screen: true, - z_index: None, - x: 0, - y: 0, - width: 0, - height: 0, - } - } - - #[test] - fn atspi_merge_keeps_x11_geometry_owner_and_native_only_frames() { - let mut windows = vec![window(10, Some(100), "XWayland")]; - let x11_pids = std::collections::HashSet::from([100]); - merge_atspi_windows( - &mut windows, - &x11_pids, - vec![ - window(100 << 16, Some(100), "XWayland duplicate"), - window(200 << 16, Some(200), "Native Wayland"), - window(1, None, "Unknown native frame"), - ], - ); - assert_eq!(windows.len(), 3); - assert_eq!(windows[0].xid, 10); - assert_eq!(windows[1].pid, Some(200)); - assert_eq!(windows[2].pid, None); - } - - #[test] - fn zero_geometry_does_not_replace_a_real_observed_origin() { - let pid = u32::MAX - 17; - let mut observed = window(1, Some(pid), "Observed"); - observed.x = 120; - observed.y = 80; - remember_observed_window_origins(&[observed]); - assert_eq!(observed_window_origin(pid), Some((120, 80))); - - remember_observed_window_origins(&[window(2, Some(pid), "Unknown")]); - assert_eq!(observed_window_origin(pid), Some((120, 80))); - } - - #[test] - fn native_enrichment_matches_plain_atspi_title() { - let mut native = window(42, None, "CUA Fixture [cua-fixture]"); - native.app_name = "cua-fixture".into(); - let mut accessible = window(123 << 16, Some(123), "CUA Fixture"); - accessible.x = 20; - accessible.y = 30; - accessible.width = 800; - accessible.height = 600; - - let enriched = enrich_native_windows(vec![native], vec![accessible], false); - - assert_eq!(enriched[0].xid, 42); - assert_eq!(enriched[0].pid, Some(123)); - assert_eq!((enriched[0].x, enriched[0].y), (20, 30)); - assert_eq!((enriched[0].width, enriched[0].height), (800, 600)); - } - - #[test] - fn unmatched_ext_windows_do_not_satisfy_pid_filter() { - let windows = vec![window(0xF000_0000, None, "Protocol-only")]; - assert!(native_windows_for_pid(windows, 4242).is_none()); - } - - #[test] - fn native_title_match_recovers_pid_without_replacing_native_id() { - let native = vec![window(77, None, "CuaTestHarness")]; - let mut accessible = window(123 << 16, Some(123), "CuaTestHarness"); - accessible.x = 20; - accessible.y = 30; - accessible.width = 800; - accessible.height = 600; - let enriched = enrich_native_windows(native, vec![accessible], false); - assert_eq!(enriched[0].xid, 77); - assert_eq!(enriched[0].pid, Some(123)); - assert_eq!( - ( - enriched[0].x, - enriched[0].y, - enriched[0].width, - enriched[0].height - ), - (20, 30, 800, 600) - ); - } - - #[test] - fn nested_enrichment_adopts_stable_atspi_id() { - let native = vec![window(77, None, "CuaTestHarness")]; - let accessible = window(123 << 16, Some(123), "CuaTestHarness"); - let enriched = enrich_native_windows(native, vec![accessible], true); - assert_eq!(enriched[0].xid, 123 << 16); - assert_eq!(enriched[0].pid, Some(123)); - assert_eq!( - identity_for(enriched[0].xid).unwrap().title, - "CuaTestHarness" - ); - } - - #[test] - fn inject_target_correlates_native_identity_to_unique_atspi_pid() { - let identity = ToplevelIdentity { - title: "Unique sentinel".into(), - app_id: "electron".into(), - }; - let windows = vec![ - window(10, Some(100), "Background fixture"), - window(20, Some(200), "Unique sentinel"), - ]; - assert_eq!( - unique_atspi_pid_for_identity(&identity, &windows), - Some(200) - ); - } - - #[test] - fn inject_target_prefers_explicit_positive_pid() { - assert_eq!( - inject_target_for_window_with_pid(99, Some(123)).unwrap(), - "root:123" - ); - assert!(inject_target_for_window_with_pid(99, Some(0)).is_err()); - } - - #[test] - fn inject_target_refuses_ambiguous_title_pid_correlation() { - let identity = ToplevelIdentity { - title: "Shared title".into(), - app_id: "electron".into(), - }; - let windows = vec![ - window(10, Some(100), "Shared title"), - window(20, Some(200), "Shared title"), - ]; - assert_eq!(unique_atspi_pid_for_identity(&identity, &windows), None); - } - - #[test] - fn sway_window_capture_is_cropped_to_compositor_geometry() { - let source = image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel( - 8, - 6, - image::Rgba([20, 40, 60, 255]), - )); - let mut encoded = std::io::Cursor::new(Vec::new()); - source - .write_to(&mut encoded, image::ImageFormat::Png) - .expect("encode fixture PNG"); - let cropped = - crop_png_to_rect(encoded.get_ref(), 2, 1, 3, 4, "fixture").expect("crop fixture PNG"); - let decoded = image::load_from_memory(&cropped).expect("decode cropped PNG"); - assert_eq!((decoded.width(), decoded.height()), (3, 4)); - } - - #[test] - fn shell_helper_capture_failure_is_terminal() { - let result = checked_shell_helper_capture(true, || None) - .expect("available helper must produce a terminal result"); - assert_eq!( - result.unwrap_err().to_string(), - "GNOME compositor helper capture failed" - ); - } - - #[test] - fn unavailable_shell_helper_does_not_attempt_capture() { - let called = std::cell::Cell::new(false); - let result = checked_shell_helper_capture(false, || { - called.set(true); - Some(vec![1, 2, 3]) - }); - assert!(result.is_none()); - assert!(!called.get()); - } - - #[test] - fn injectable_text_accepts_printable_ascii_newline_and_tab() { - validate_injectable_text("Hello, World! 123 @#$%\t\n").expect("printable ASCII is typable"); - // The full printable ASCII span the compositor chartab covers. - let printable: String = (0x20u8..=0x7e).map(|b| b as char).collect(); - validate_injectable_text(&printable).expect("every printable ASCII byte is typable"); - } - - #[test] - fn injectable_text_rejects_unicode_and_other_controls() { - for bad in [ - "café", - "emoji 😀", - "bell\u{07}", - "null\u{00}", - "delete\u{7f}", - "cr\r", - ] { - assert!( - validate_injectable_text(bad).is_err(), - "{bad:?} must be rejected before it reaches the compositor" - ); - } - } - - #[test] - fn injectable_key_accepts_whitelist_case_insensitively() { - for good in [ - "enter", "Enter", "RETURN", "tab", "Escape", "esc", "space", "up", "Left", "f1", "F12", - ] { - validate_injectable_key(good).unwrap_or_else(|e| panic!("{good:?} should pass: {e}")); - } - } - - #[test] - fn injectable_key_rejects_unsupported_names() { - for bad in ["f13", "ctrl", "a", "delete", "home", "pageup", ""] { - assert!( - validate_injectable_key(bad).is_err(), - "{bad:?} is not in the compositor whitelist" - ); - } - } - - #[test] - fn injectable_hotkey_normalizes_supported_chords() { - let keys = vec!["control".to_owned(), "SHIFT".to_owned(), "7".to_owned()]; - assert_eq!( - validate_injectable_hotkey(&keys).expect("supported chord"), - ("ctrl,shift".to_owned(), "7".to_owned()) - ); - assert!(validate_injectable_hotkey(&["7".to_owned()]).is_err()); - assert!(validate_injectable_hotkey(&["hyper".to_owned(), "k".to_owned()]).is_err()); - } - - #[test] - fn wtype_hotkey_keeps_primer_chord_and_releases_in_order() { - let args = wtype_hotkey_args(&["ctrl".into(), "shift".into()], "7"); - assert_eq!( - args, - [ - "-k", "Shift_L", "-s", "30", "-M", "ctrl", "-M", "shift", "-s", "20", "-k", "7", - "-s", "20", "-m", "shift", "-m", "ctrl", - ] - ); - } - - #[test] - fn hello_reply_parses_exact_banner_and_rejects_mismatch() { - parse_inject_hello("cua-inject v1\n").expect("verbatim banner is accepted"); - parse_inject_hello(" cua-inject v1 ").expect("surrounding whitespace is tolerated"); - assert!(parse_inject_hello("cua-inject v2").is_err()); - assert!(parse_inject_hello("err unsupported-version").is_err()); - assert!(parse_inject_hello("garbage").is_err()); - } - - #[test] - fn command_reply_parses_ok_and_surfaces_error_reason() { - parse_inject_reply("ok\n").expect("ok is success"); - parse_inject_reply("ok").expect("ok without newline is success"); - let err = parse_inject_reply("err ambiguous-app-id\n").unwrap_err(); - assert!(err.to_string().contains("ambiguous-app-id")); - assert!(parse_inject_reply("err").is_err()); - assert!(parse_inject_reply("maybe").is_err()); - } - - #[test] - fn geometry_reply_is_strict_and_signed() { - assert_eq!( - parse_inject_geometry("geometry -4 23 10 20\n").unwrap(), - ((-4, 23), (10, 20)) - ); - assert!(parse_inject_geometry("geometry 1").is_err()); - assert!(parse_inject_geometry("err target-not-found").is_err()); - assert!(parse_inject_geometry("ok").is_err()); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs deleted file mode 100644 index 088846188e7..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs +++ /dev/null @@ -1,868 +0,0 @@ -//! Native-Wayland agent-cursor overlay via `zwlr_layer_shell_v1`. -//! -//! Replaces the X11-only `overlay.rs` render loop on wlroots compositors -//! (sway, labwc, kwin 5.27+, hyprland) by creating a full-screen, -//! click-through, always-on-top `wl_surface` anchored to the first output -//! via `zwlr_layer_shell_v1`. The surface renders the same gradient-arrow -//! cursor as the X11 path by sharing `cursor_overlay::RenderStateCore` — -//! bloom, click-pulse, idle-fade, and motion all work identically. -//! -//! GNOME mutter does not expose `zwlr_layer_shell_v1` — those sessions -//! either fall through to the X11 path (XWayland) or the nested-compositor -//! mode that spawns labwc internally. -//! -//! Architecture mirrors the existing `wayland/persistent_vptr.rs`: one -//! owner thread (`cua-overlay-wl`) holds the wayland Connection + -//! EventQueue + layer surface; commands flow in over a `crossbeam-channel`. -//! The render core wakes at frame cadence only while pixels can change; -//! stable or hidden state blocks on the command channel without polling. - -use std::collections::HashMap; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - OnceLock, -}; -use std::thread; -use std::time::{Duration, Instant}; - -use crossbeam_channel::{bounded, Receiver, Sender}; -use cursor_overlay::{CursorConfig, OverlayCommand, OverlayMsg, RenderStateCore}; -use wayland_client::{ - protocol::{ - wl_buffer::WlBuffer, - wl_compositor::WlCompositor, - wl_output::WlOutput, - wl_region::WlRegion, - wl_registry, - wl_shm::{self, WlShm}, - wl_shm_pool::WlShmPool, - wl_surface::WlSurface, - }, - Connection, Dispatch, Proxy, QueueHandle, -}; -use wayland_protocols_wlr::layer_shell::v1::client::{ - zwlr_layer_shell_v1::{Layer, ZwlrLayerShellV1}, - zwlr_layer_surface_v1::{self, Anchor, KeyboardInteractivity, ZwlrLayerSurfaceV1}, -}; - -/// Commands the overlay owner thread accepts. The richer commands the -/// cross-platform [`RenderStateCore`] understands (MoveTo, ClickPulse, -/// SetPressed) are forwarded as-is so the layer-shell overlay matches the -/// X11 visual: bloom + animated arrow + click pulse + press ring. -enum WlOverlayCmd { - Cmd { cmd: OverlayCommand }, - Remove, - Shutdown, -} - -static TX: OnceLock<Sender<WlOverlayCmd>> = OnceLock::new(); -// Starts false deliberately: the platform registration path must explicitly -// opt the native Wayland overlay in with the daemon's CursorConfig. This keeps -// lazy forwarding from bypassing --no-overlay before any window exists. -static CONFIG_ENABLED: AtomicBool = AtomicBool::new(false); - -pub fn set_config_enabled(enabled: bool) { - CONFIG_ENABLED.store(enabled, Ordering::Release); -} - -fn tx() -> Option<&'static Sender<WlOverlayCmd>> { - TX.get() -} - -/// Lazily start the owner thread. Idempotent — safe to call from every -/// MCP tool invocation; subsequent calls are no-ops. -pub fn ensure_started() -> bool { - if !CONFIG_ENABLED.load(Ordering::Acquire) { - return false; - } - TX.get_or_init(|| { - let (tx, rx) = bounded::<WlOverlayCmd>(64); - thread::Builder::new() - .name("cua-overlay-wl".into()) - .spawn(move || { - if let Err(e) = owner_thread(rx) { - tracing::warn!("cua-overlay-wl thread exited with error: {e}"); - } - }) - .expect("spawn cua-overlay-wl thread"); - tx - }); - true -} - -/// Translate a generic [`OverlayMsg`] (the cross-platform command shape) -/// to the layer-shell owner thread. The owner-thread render core consumes -/// every variant the X11 path handles; only `ShowFocusRect` (macOS-only) -/// is silently dropped here. -pub fn forward(msg: &OverlayMsg) -> bool { - if !should_forward(CONFIG_ENABLED.load(Ordering::Acquire), msg) { - return false; - } - // The native Wayland path owns one cursor and does not keep keyed session - // tombstones. Accept revival without starting the compositor thread. - if matches!(msg, OverlayMsg::Revive(_)) { - return true; - } - // Lazy startup: spawning the layer-shell owner thread + connecting to - // the Wayland compositor takes 100-300ms. Doing that at cua-driver mcp - // boot (the old eager-init path) was tipping the borderline CI - // cursor-click-gif test over its 20s budget. ensure_started is - // idempotent so calling it on every forward is fine — the OnceLock - // bypasses the spawn after the first call. - if !ensure_started() { - return false; - } - let Some(tx) = tx() else { return false }; - match msg { - OverlayMsg::Remove(k) => { - let _ = k; - let _ = tx.try_send(WlOverlayCmd::Remove); - true - } - OverlayMsg::Cmd(kc) => { - if matches!(&kc.cmd, OverlayCommand::ShowFocusRect(_)) { - return false; - } - let _ = tx.try_send(WlOverlayCmd::Cmd { - cmd: kc.cmd.clone(), - }); - true - } - OverlayMsg::Revive(_) => true, - } -} - -fn should_forward(config_enabled: bool, msg: &OverlayMsg) -> bool { - config_enabled - && !matches!( - msg, - OverlayMsg::Cmd(kc) if matches!(&kc.cmd, OverlayCommand::ShowFocusRect(_)) - ) -} - -/// Cleanly stop the owner thread. Tests use this; production code typically -/// lets the thread die at process exit. -pub fn shutdown() { - if let Some(tx) = tx() { - let _ = tx.send(WlOverlayCmd::Shutdown); - } -} - -// ── owner thread ───────────────────────────────────────────────────────── - -struct OverlayState { - compositor: Option<WlCompositor>, - shm: Option<WlShm>, - layer_shell: Option<ZwlrLayerShellV1>, - output: Option<WlOutput>, - output_w: u32, - output_h: u32, - surface: Option<WlSurface>, - layer_surface: Option<ZwlrLayerSurfaceV1>, - configured: bool, - /// Cross-platform render core: position, animation, gradient arrow, - /// bloom, click pulse, idle-fade. Shared verbatim with the X11 path. - core: RenderStateCore, - /// In-flight wl_shm buffers awaiting `wl_buffer.release` from the - /// compositor. Keyed by `WlBuffer` object id; value is the - /// `(mmap ptr, mmap size, memfd fd)` triple that must be unmapped + - /// closed once the compositor signals it's done with the buffer. - /// Replaces the per-redraw `mem::forget` leak: the previous frame's - /// memory is reclaimed as soon as the compositor releases it. - pending_buffers: HashMap<u32, (*mut libc::c_void, usize, i32)>, -} - -// SAFETY: the raw pointers in pending_buffers point at mmap regions owned -// exclusively by this thread (the owner thread). OverlayState is never -// shared across threads — wayland-client's EventQueue<State> is !Send so -// it stays pinned to the owner thread. The Send/Sync bounds wayland-client -// requires for State types apply to the struct as a whole, hence the -// explicit assertion. -unsafe impl Send for OverlayState {} - -impl Default for OverlayState { - fn default() -> Self { - Self { - compositor: None, - shm: None, - layer_shell: None, - output: None, - output_w: 0, - output_h: 0, - surface: None, - layer_surface: None, - configured: false, - core: RenderStateCore::new(CursorConfig::default()), - pending_buffers: HashMap::new(), - } - } -} - -fn dbg(msg: &str) { - if std::env::var_os("CUA_OVERLAY_DEBUG").is_some() { - eprintln!("[cua-overlay-wl] {msg}"); - } -} - -fn owner_thread(rx: Receiver<WlOverlayCmd>) -> anyhow::Result<()> { - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<OverlayState>(); - let qh = queue.handle(); - let _registry = conn.display().get_registry(&qh, ()); - - let mut state = OverlayState::default(); - queue.roundtrip(&mut state)?; - for _ in 0..3 { - queue.roundtrip(&mut state)?; - } - - let compositor = state - .compositor - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_compositor"))?; - let shm = state - .shm - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?; - let layer_shell = state - .layer_shell - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_layer_shell_v1"))?; - let output = state - .output - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output"))?; - - // Build the layer surface: fullscreen, overlay layer, click-through. - let surface = compositor.create_surface(&qh, ()); - let layer_surface = layer_shell.get_layer_surface( - &surface, - Some(&output), - Layer::Overlay, - "cua-agent-cursor".to_string(), - &qh, - (), - ); - // Anchor to all four edges = full screen. - layer_surface.set_anchor(Anchor::Top | Anchor::Bottom | Anchor::Left | Anchor::Right); - layer_surface.set_size(0, 0); - layer_surface.set_exclusive_zone(-1); - layer_surface.set_keyboard_interactivity(KeyboardInteractivity::None); - - // Click-through: empty input region. Standard Wayland intentionally does - // not expose another client's global pointer position, so this surface - // cannot implement the macOS/Windows/X11 badge-hover reveal without a - // compositor-owned adapter. Giving it an input region would steal the - // user's pointer events instead of observing them. - let region: WlRegion = compositor.create_region(&qh, ()); - surface.set_input_region(Some(®ion)); - region.destroy(); - - state.surface = Some(surface); - state.layer_surface = Some(layer_surface); - - // First commit kicks off the configure handshake. - if let Some(s) = state.surface.as_ref() { - s.commit(); - } - - // Wait for the first configure event so we know the output dimensions - // before drawing. - for _ in 0..10 { - queue.roundtrip(&mut state)?; - if state.configured && state.output_w > 0 && state.output_h > 0 { - break; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - - if !state.configured { - anyhow::bail!("layer surface never received configure event"); - } - dbg(&format!( - "configured: w={} h={}", - state.output_w, state.output_h - )); - - // Demand-driven main loop. A stable cursor blocks on the command channel; - // only active motion, click/fade animation, or the exact idle-fade - // deadline schedules a wake. This avoids display-sized SHM allocation and - // conversion at 60Hz when no pixel can change while preserving immediate - // command wakeups. - let mut last_tick = Instant::now(); - let mut frame_tick_needed = false; - loop { - let wait = next_wait(&state.core, frame_tick_needed); - let (first_cmd, timed_out) = match wait_for_work(&rx, wait) { - WlWake::Command(cmd) => (Some(cmd), None), - WlWake::Timeout => (None, Some(wait)), - WlWake::Disconnected => break, - }; - - let now = Instant::now(); - let elapsed = now.duration_since(last_tick).as_secs_f64(); - last_tick = now; - - let mut dirty = false; - let mut shutdown = false; - let mut pending = first_cmd; - loop { - let received = pending.take().map(Ok).unwrap_or_else(|| rx.try_recv()); - match received { - Ok(WlOverlayCmd::Shutdown) => { - shutdown = true; - break; - } - Ok(WlOverlayCmd::Cmd { cmd }) => { - // Seed: if the cursor is still at the off-screen sentinel - // `(-200, -200)` from `RenderStateCore::new`, snap to a - // point near the MoveTo / SnapTo target so the spring - // animation starts on-screen. Mirrors X11 overlay.rs's - // `seed_start_if_sentinel` helper — without it, the - // spring oscillates around the sentinel and the cursor - // never reaches the screen. - let seed_target = match &cmd { - OverlayCommand::MoveTo { x, y, .. } - | OverlayCommand::SnapTo { x, y, .. } - | OverlayCommand::ClickPulse { x, y } => Some((*x, *y)), - _ => None, - }; - if let Some((tx, ty)) = seed_target { - if state.core.pos.0 < -50.0 { - const SEED_OFFSET: f64 = 16.0; - let sx = (tx - SEED_OFFSET).max(2.0); - let sy = (ty - SEED_OFFSET).max(2.0); - state.core.pos = (sx, sy); - } - } - // apply_command_base consumes every variant the X11 - // path handles. `move_to_snap_sentinel` / `click_pulse - // _sentinel_only` are both `false` here — same as X11. - let disabling = matches!(&cmd, OverlayCommand::SetEnabled(false)); - dirty |= state.core.apply_command_base(cmd, false, false); - if disabling { - quiesce_hidden(&mut state.core); - } - } - Ok(WlOverlayCmd::Remove) => { - // Single-cursor overlay: removing the active cursor - // hides it. Multi-cursor wlroots support can layer on - // top of this in a follow-up if needed. - dirty |= state.core.visible || state.core.pos.0 >= -100.0; - state.core.visible = false; - quiesce_hidden(&mut state.core); - } - Err(crossbeam_channel::TryRecvError::Empty) => break, - Err(crossbeam_channel::TryRecvError::Disconnected) => { - shutdown = true; - break; - } - } - } - if shutdown { - break; - } - - // Advance only on a scheduled animation/fade wake. A command wake - // applies the new state at dt=0, avoiding a jump proportional to how - // long the loop was parked. - if let Some(timeout_kind) = timed_out { - let dt = match timeout_kind { - WlWait::Frame => elapsed.min(0.05), - WlWait::Deadline(_) => elapsed, - WlWait::Block => unreachable!("a blocking receive cannot time out"), - }; - state.core.tick_motion(dt); - dirty = true; - } - let next_frame_tick_needed = needs_frame_tick(&state.core); - if state.configured && (dirty || frame_tick_needed || next_frame_tick_needed) { - redraw(&mut state, &shm, &qh)?; - // Flush the committed frame and dispatch wl_buffer.release before - // parking. The buffer map remains authoritative until release, so - // no mmap/fd can be reclaimed while the compositor still uses it. - queue.roundtrip(&mut state)?; - } - frame_tick_needed = next_frame_tick_needed; - } - - if let Some(ls) = state.layer_surface.take() { - ls.destroy(); - } - if let Some(s) = state.surface.take() { - s.destroy(); - } - queue.roundtrip(&mut state)?; - Ok(()) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum WlWait { - Block, - Frame, - Deadline(Duration), -} - -enum WlWake { - Command(WlOverlayCmd), - Timeout, - Disconnected, -} - -fn wait_for_work(rx: &Receiver<WlOverlayCmd>, wait: WlWait) -> WlWake { - match wait { - WlWait::Block => match rx.recv() { - Ok(cmd) => WlWake::Command(cmd), - Err(_) => WlWake::Disconnected, - }, - WlWait::Frame => match rx.recv_timeout(Duration::from_millis(16)) { - Ok(cmd) => WlWake::Command(cmd), - Err(crossbeam_channel::RecvTimeoutError::Timeout) => WlWake::Timeout, - Err(crossbeam_channel::RecvTimeoutError::Disconnected) => WlWake::Disconnected, - }, - WlWait::Deadline(timeout) => match rx.recv_timeout(timeout) { - Ok(cmd) => WlWake::Command(cmd), - Err(crossbeam_channel::RecvTimeoutError::Timeout) => WlWake::Timeout, - Err(crossbeam_channel::RecvTimeoutError::Disconnected) => WlWake::Disconnected, - }, - } -} - -fn next_wait(core: &RenderStateCore, frame_tick_needed: bool) -> WlWait { - if frame_tick_needed || needs_frame_tick(core) { - return WlWait::Frame; - } - idle_fade_wait(core) - .map(WlWait::Deadline) - .unwrap_or(WlWait::Block) -} - -fn needs_frame_tick(core: &RenderStateCore) -> bool { - if !core.visible || core.pos.0 < -100.0 { - return false; - } - let fade_start = core.motion.idle_hide_ms / 1000.0; - core.path.is_some() - || core.spring.is_some() - || core.click_t.is_some() - || core.session_badge_needs_frame_tick() - || (core.motion.idle_hide_ms > 0.0 - && core.idle_secs >= fade_start - && core.idle_alpha >= 0.004) -} - -fn idle_fade_wait(core: &RenderStateCore) -> Option<Duration> { - if !core.visible - || core.pos.0 < -100.0 - || core.motion.idle_hide_ms <= 0.0 - || core.path.is_some() - || core.spring.is_some() - || core.click_t.is_some() - { - return None; - } - let remaining = core.motion.idle_hide_ms / 1000.0 - core.idle_secs; - (remaining.is_finite() && remaining > 0.0).then(|| Duration::from_secs_f64(remaining)) -} - -fn quiesce_hidden(core: &mut RenderStateCore) { - core.path = None; - core.spring = None; - core.spring_tgt = None; - core.click_t = None; -} - -/// Render one cursor frame into a fresh wl_shm ARGB8888 buffer and attach -/// it to the layer surface. -/// -/// Pipeline: -/// 1. Allocate a memfd-backed wl_shm pool sized at output_w × output_h. -/// 2. Paint the cross-platform cursor (bloom + click pulse + gradient -/// arrow) into a `tiny_skia::Pixmap` via `cursor_overlay::paint_cursor` -/// — same call the X11 path uses. -/// 3. Channel-swap RGBA → BGRA into the wl_shm buffer (wl_shm Argb8888 -/// is little-endian BGRA in memory). This is the inverse of the swap -/// in `ext_screencopy::encode_buffer_to_png`. -/// 4. Attach + damage + commit on the layer surface. -/// -/// When the cursor is hidden (`core.visible == false`, idle-faded, or -/// off-screen sentinel) the pixmap is all zeros — the surface remains -/// transparent and click-through. -fn redraw( - state: &mut OverlayState, - shm: &WlShm, - qh: &QueueHandle<OverlayState>, -) -> anyhow::Result<()> { - let Some(surface) = state.surface.as_ref() else { - return Ok(()); - }; - let w = state.output_w.max(1); - let h = state.output_h.max(1); - let stride = w as i32 * 4; - let size = (stride as usize) * (h as usize); - - // Reuses the same anon_shm pattern as the screencopy path in mod.rs. - let (fd, ptr) = - super::anon_shm(size).map_err(|e| anyhow::anyhow!("overlay shm allocation failed: {e}"))?; - - // SAFETY: ptr came from mmap of `size` bytes, lifetime bounded to this - // function. - let pixels: &mut [u8] = unsafe { std::slice::from_raw_parts_mut(ptr as *mut u8, size) }; - - // Paint the cursor into a tiny_skia pixmap. paint_cursor early-returns - // when the cursor is hidden / off-screen / idle-faded, so the pixmap - // is left fully transparent in those cases (which is also what we want - // for the click-through layer surface). - let pm_result = tiny_skia::Pixmap::new(w, h); - let mut pm = match pm_result { - Some(p) => p, - // tiny_skia::Pixmap::new only fails on OOM at sizes that fit u32. - // We refuse to fall back to a 1x1 pixmap because the subsequent - // RGBA → BGRA loop indexes `src[i+3]` over the full `size` range — - // a 1x1 source would crash. Surface the allocation failure - // properly instead. - None => anyhow::bail!( - "tiny_skia::Pixmap::new({w}, {h}) failed — out of memory for the overlay buffer" - ), - }; - // backing_scale=1.0 matches the X11 path; per-output Wayland scale is - // a follow-up (would consume `wl_output.scale` and `preferred_buffer - // _scale` from wl_surface v6). - cursor_overlay::paint_cursor(&mut pm, &state.core, 0.0, 0.0, None, 1.0); - - // CUA_OVERLAY_DEBUG=1 paints a 60x60 magenta square at the cursor's - // current pos on top of the gradient arrow. Useful when validating - // layer-shell visibility on a new compositor — the gradient arrow is - // small at native scale and easy to miss in a screenshot, while the - // magenta block is impossible to miss. - if std::env::var_os("CUA_OVERLAY_DEBUG").is_some() { - let (cx, cy) = state.core.pos; - let cx = cx as i32; - let cy = cy as i32; - let half = 30i32; - for dy in -half..half { - for dx in -half..half { - let px = cx + dx; - let py = cy + dy; - if px < 0 || py < 0 || px >= w as i32 || py >= h as i32 { - continue; - } - let off = ((py as usize) * (w as usize) + (px as usize)) * 4; - pm.data_mut()[off] = 0xFF; // R - pm.data_mut()[off + 1] = 0x00; // G - pm.data_mut()[off + 2] = 0xFF; // B - pm.data_mut()[off + 3] = 0xFF; // A - } - } - } - - // RGBA → BGRA channel swap. tiny_skia stores pixels as RGBA8888 - // (premultiplied); wl_shm Argb8888 is little-endian = BGRA in memory. - // Mirrors the inverse swap in ext_screencopy::encode_buffer_to_png. - let src = pm.data(); - for i in (0..size).step_by(4) { - // pm.data() is already RGBA premultiplied; just swap R↔B. - pixels[i] = src[i + 2]; // B ← R - pixels[i + 1] = src[i + 1]; // G - pixels[i + 2] = src[i]; // R ← B - pixels[i + 3] = src[i + 3]; // A - } - - use std::os::fd::AsFd as _; - let pool_fd = unsafe { super::borrowed_fd(fd) }; - let pool: WlShmPool = shm.create_pool(pool_fd.as_fd(), size as i32, qh, ()); - let buffer: WlBuffer = pool.create_buffer( - 0, - w as i32, - h as i32, - stride, - wl_shm::Format::Argb8888, - qh, - (), - ); - - // Track the (mmap, fd) by buffer object id so the wl_buffer.release - // event Dispatch handler can clean up exactly when the compositor is - // done with the underlying memory — no leak, no use-after-free. - let buffer_id = buffer.id().protocol_id(); - state.pending_buffers.insert(buffer_id, (ptr, size, fd)); - - dbg(&format!( - "redraw w={w} h={h} stride={stride} buf_id={buffer_id} pos=({:.1},{:.1}) visible={}", - state.core.pos.0, state.core.pos.1, state.core.visible - )); - surface.attach(Some(&buffer), 0, 0); - surface.damage_buffer(0, 0, w as i32, h as i32); - surface.commit(); - pool.destroy(); - Ok(()) -} - -// ── Wayland Dispatch impls ─────────────────────────────────────────────── - -impl Dispatch<wl_registry::WlRegistry, ()> for OverlayState { - fn event( - state: &mut Self, - registry: &wl_registry::WlRegistry, - event: wl_registry::Event, - _data: &(), - _conn: &Connection, - qh: &QueueHandle<Self>, - ) { - if let wl_registry::Event::Global { - name, - interface, - version, - } = event - { - match interface.as_str() { - "wl_compositor" => { - state.compositor = - Some(registry.bind::<WlCompositor, _, _>(name, version.min(6), qh, ())); - } - "wl_shm" => { - state.shm = Some(registry.bind::<WlShm, _, _>(name, version.min(1), qh, ())); - } - "wl_output" => { - if state.output.is_none() { - state.output = - Some(registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ())); - } - } - "zwlr_layer_shell_v1" => { - state.layer_shell = - Some(registry.bind::<ZwlrLayerShellV1, _, _>(name, version.min(4), qh, ())); - } - _ => {} - } - } - } -} - -impl Dispatch<WlCompositor, ()> for OverlayState { - fn event( - _state: &mut Self, - _: &WlCompositor, - _: <WlCompositor as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -impl Dispatch<WlShm, ()> for OverlayState { - fn event( - _state: &mut Self, - _: &WlShm, - _: <WlShm as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -impl Dispatch<WlOutput, ()> for OverlayState { - fn event( - state: &mut Self, - _: &WlOutput, - event: <WlOutput as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - use wayland_client::protocol::wl_output; - if let wl_output::Event::Mode { width, height, .. } = event { - if width > 0 && height > 0 { - state.output_w = width as u32; - state.output_h = height as u32; - } - } - } -} - -impl Dispatch<ZwlrLayerShellV1, ()> for OverlayState { - fn event( - _state: &mut Self, - _: &ZwlrLayerShellV1, - _: <ZwlrLayerShellV1 as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -impl Dispatch<ZwlrLayerSurfaceV1, ()> for OverlayState { - fn event( - state: &mut Self, - layer: &ZwlrLayerSurfaceV1, - event: <ZwlrLayerSurfaceV1 as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - if let zwlr_layer_surface_v1::Event::Configure { - serial, - width, - height, - } = event - { - layer.ack_configure(serial); - if width > 0 { - state.output_w = width; - } - if height > 0 { - state.output_h = height; - } - state.configured = true; - } - } -} - -impl Dispatch<WlSurface, ()> for OverlayState { - fn event( - _: &mut Self, - _: &WlSurface, - _: <WlSurface as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -impl Dispatch<WlShmPool, ()> for OverlayState { - fn event( - _: &mut Self, - _: &WlShmPool, - _: <WlShmPool as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -impl Dispatch<WlBuffer, ()> for OverlayState { - fn event( - state: &mut Self, - buffer: &WlBuffer, - event: <WlBuffer as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - use wayland_client::protocol::wl_buffer; - if matches!(event, wl_buffer::Event::Release) { - // Compositor is done with the underlying mmap. Free it + - // close the memfd + destroy the wayland object. - let id = buffer.id().protocol_id(); - if let Some((ptr, size, fd)) = state.pending_buffers.remove(&id) { - super::cleanup_mmap(ptr, size, fd); - } - buffer.destroy(); - } - } -} - -impl Dispatch<WlRegion, ()> for OverlayState { - fn event( - _: &mut Self, - _: &WlRegion, - _: <WlRegion as wayland_client::Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -#[cfg(test)] -mod tests { - use super::*; - use cursor_overlay::KeyedOverlayCommand; - - fn message(cmd: OverlayCommand) -> OverlayMsg { - OverlayMsg::Cmd(KeyedOverlayCommand { - key: "test".to_owned(), - cmd, - }) - } - - fn positioned_core() -> RenderStateCore { - let mut core = RenderStateCore::new(CursorConfig::default()); - core.pos = (100.0, 100.0); - core.motion.idle_hide_ms = 1_000.0; - core - } - - #[test] - fn fresh_sentinel_overlay_blocks_without_frame_polling() { - let core = RenderStateCore::new(CursorConfig::default()); - assert_eq!(next_wait(&core, false), WlWait::Block); - assert!(!needs_frame_tick(&core)); - } - - #[test] - fn stable_visible_overlay_sleeps_until_idle_fade_deadline() { - let mut core = positioned_core(); - core.idle_secs = 0.25; - assert_eq!( - next_wait(&core, false), - WlWait::Deadline(Duration::from_millis(750)) - ); - assert!(!needs_frame_tick(&core)); - } - - #[test] - fn animation_and_fade_use_frame_cadence() { - let mut core = positioned_core(); - core.click_t = Some(0.0); - assert!(needs_frame_tick(&core)); - assert_eq!(next_wait(&core, false), WlWait::Frame); - - core.click_t = None; - core.idle_secs = 1.0; - core.idle_alpha = 1.0; - assert!(needs_frame_tick(&core)); - assert_eq!(next_wait(&core, false), WlWait::Frame); - } - - #[test] - fn hidden_and_disabled_state_quiesces_deterministically() { - let mut core = positioned_core(); - core.click_t = Some(0.25); - core.visible = false; - quiesce_hidden(&mut core); - assert!(core.click_t.is_none()); - assert!(core.path.is_none()); - assert!(core.spring.is_none()); - assert_eq!(next_wait(&core, false), WlWait::Block); - } - - #[test] - fn blocked_scheduler_wakes_on_command_arrival() { - let (tx, rx) = bounded(1); - tx.send(WlOverlayCmd::Remove).unwrap(); - assert!(matches!( - wait_for_work(&rx, WlWait::Block), - WlWake::Command(WlOverlayCmd::Remove) - )); - } - - #[test] - fn disabled_config_refuses_forwarding_and_thread_startup() { - set_config_enabled(false); - let msg = message(OverlayCommand::SnapTo { - x: 10.0, - y: 20.0, - heading_radians: None, - }); - let had_thread = TX.get().is_some(); - assert!(!should_forward(false, &msg)); - assert!(!ensure_started()); - assert_eq!(TX.get().is_some(), had_thread); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs deleted file mode 100644 index 9ee2718e12a..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs +++ /dev/null @@ -1,336 +0,0 @@ -//! Persistent virtual-pointer for stateful `mouse_button_down` / `mouse_drag` / -//! `mouse_button_up`. -//! -//! The non-persistent path in [`crate::wayland::click`] opens its own -//! `ZwlrVirtualPointerV1`, presses, releases, drops the connection — useful -//! for one-shot clicks but useless for held-button drags: each tool call -//! emits a fresh device whose press/release pair is matched by the -//! compositor, so apps that distinguish a real drag (press, motion+, release) -//! from a series of clicks (press, release, press, release, …) miss the -//! drag entirely. -//! -//! This module keeps the virtual-pointer alive across tool calls. A single -//! owner thread per process owns one Wayland `Connection`, one `EventQueue`, -//! and a map of `cursor_id -> ActivePointer`. Commands are sent over a -//! `crossbeam-channel`; replies come back on a per-call reply channel so the -//! caller blocks until the compositor has roundtripped. -//! -//! Lifecycle: -//! - First `press` for a cursor_id binds a fresh `ZwlrVirtualPointerV1`, -//! activates the foreign-toplevel target window once, presses the button, -//! adds to the held-button set, roundtrips. -//! - Subsequent `move_to` calls emit `motion_absolute` on the same vptr (no -//! activate — would steal focus mid-drag) and roundtrip. -//! - `release` emits a button release, removes from the held set; if the -//! set is empty the vptr is destroyed and the map entry dropped. -//! - On `Connection` roundtrip failure (compositor restart / disconnect) -//! the owner thread tears down its connection and accepts the next -//! command on a fresh one, emitting a typed error for the in-flight call. - -use std::collections::{HashMap, HashSet}; -use std::sync::OnceLock; -use std::thread; - -use crossbeam_channel::{bounded, Receiver, Sender}; -use wayland_client::{protocol::wl_pointer::ButtonState, Connection}; -use wayland_protocols_wlr::virtual_pointer::v1::client::zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1; - -use super::{evdev_pointer_button, open_vptr_session}; - -/// One in-flight command from the public API to the owner thread. -enum Cmd { - Press { - cursor_id: String, - window_id: u64, - x: i32, - y: i32, - button: u8, - reply: Sender<anyhow::Result<()>>, - }, - MoveTo { - cursor_id: String, - x: i32, - y: i32, - reply: Sender<anyhow::Result<()>>, - }, - Release { - cursor_id: String, - button: u8, - reply: Sender<anyhow::Result<()>>, - }, - /// Drop the entry for a cursor_id without sending wire events — used to - /// recover when the compositor disconnected mid-life. - Forget { - cursor_id: String, - reply: Sender<anyhow::Result<()>>, - }, -} - -/// State held inside the owner thread for one cursor_id. -struct ActivePointer { - vptr: ZwlrVirtualPointerV1, - /// evdev codes of buttons currently held down. When this set becomes - /// empty the vptr is destroyed and the entry dropped from the map. - held: HashSet<u32>, - /// Output extent at session open time — needed for motion_absolute. - out_w: u32, - out_h: u32, -} - -/// Process-global command channel into the owner thread. Lazily started on -/// first use. -static TX: OnceLock<Sender<Cmd>> = OnceLock::new(); - -fn tx() -> &'static Sender<Cmd> { - TX.get_or_init(|| { - let (tx, rx) = bounded::<Cmd>(32); - thread::Builder::new() - .name("cua-persistent-vptr".into()) - .spawn(move || owner_thread(rx)) - .expect("spawn cua-persistent-vptr thread"); - tx - }) -} - -fn owner_thread(rx: Receiver<Cmd>) { - let mut active: HashMap<String, ActivePointer> = HashMap::new(); - while let Ok(cmd) = rx.recv() { - match cmd { - Cmd::Press { - cursor_id, - window_id, - x, - y, - button, - reply, - } => { - let r = handle_press(&mut active, &cursor_id, window_id, x, y, button); - let _ = reply.send(r); - } - Cmd::MoveTo { - cursor_id, - x, - y, - reply, - } => { - let r = handle_move(&mut active, &cursor_id, x, y); - let _ = reply.send(r); - } - Cmd::Release { - cursor_id, - button, - reply, - } => { - let r = handle_release(&mut active, &cursor_id, button); - let _ = reply.send(r); - } - Cmd::Forget { cursor_id, reply } => { - if let Some(p) = active.remove(&cursor_id) { - p.vptr.destroy(); - } - let _ = reply.send(Ok(())); - } - } - } -} - -fn handle_press( - active: &mut HashMap<String, ActivePointer>, - cursor_id: &str, - window_id: u64, - x: i32, - y: i32, - button: u8, -) -> anyhow::Result<()> { - // Open a fresh session for this press — this binds the seat, the foreign- - // toplevel manager, activates the target window, and creates a new vptr. - // Keep the (out_w, out_h) but drop the queue + state at end of scope; the - // vptr itself remains alive (Wayland objects survive their original queue - // as long as the Connection is alive). - let mut sess = open_vptr_session(Some(window_id))?; - let (w, h) = (sess.output_w, sess.output_h); - let px = x.clamp(0, w as i32 - 1) as u32; - let py = y.clamp(0, h as i32 - 1) as u32; - let btn = evdev_pointer_button(button); - - sess.vptr.motion_absolute(0, px, py, w, h); - sess.vptr.frame(); - sess.vptr.button(0, btn, ButtonState::Pressed); - sess.vptr.frame(); - sess.queue.roundtrip(&mut sess.state)?; - - // Take ownership of the vptr handle by extracting it from the session. - // ZwlrVirtualPointerV1 is a Wayland proxy — cloning it gives another - // handle to the same wire object; destroying it sends the destructor. - let vptr = sess.vptr.clone(); - // Persist the live connection so the proxy stays valid after this fn returns - // (the session goes out of scope; we need the conn alive). - // We do this by leaking the connection into a process-static slot keyed by - // cursor_id. Subsequent commands on the same cursor reuse this conn. - persist_conn(cursor_id, sess.conn); - - let mut held = HashSet::new(); - held.insert(btn); - active.insert( - cursor_id.to_string(), - ActivePointer { - vptr, - held, - out_w: w, - out_h: h, - }, - ); - Ok(()) -} - -fn handle_move( - active: &mut HashMap<String, ActivePointer>, - cursor_id: &str, - x: i32, - y: i32, -) -> anyhow::Result<()> { - let entry = active.get_mut(cursor_id).ok_or_else(|| { - anyhow::anyhow!( - "no held mouse button for cursor '{cursor_id}'; call mouse_button_down first" - ) - })?; - let px = x.clamp(0, entry.out_w as i32 - 1) as u32; - let py = y.clamp(0, entry.out_h as i32 - 1) as u32; - entry - .vptr - .motion_absolute(0, px, py, entry.out_w, entry.out_h); - entry.vptr.frame(); - roundtrip_on_persistent(cursor_id)?; - Ok(()) -} - -fn handle_release( - active: &mut HashMap<String, ActivePointer>, - cursor_id: &str, - button: u8, -) -> anyhow::Result<()> { - let btn = evdev_pointer_button(button); - let drop_entry = { - let entry = active - .get_mut(cursor_id) - .ok_or_else(|| anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'"))?; - entry.vptr.button(0, btn, ButtonState::Released); - entry.vptr.frame(); - roundtrip_on_persistent(cursor_id)?; - entry.held.remove(&btn); - entry.held.is_empty() - }; - if drop_entry { - if let Some(p) = active.remove(cursor_id) { - p.vptr.destroy(); - roundtrip_on_persistent(cursor_id).ok(); - } - forget_conn(cursor_id); - } - Ok(()) -} - -// Process-static slots for Connection + EventQueue keyed by cursor_id. The -// EventQueue is !Send but we only touch these on the owner thread, so wrap -// in a thread-local-by-construction pattern: store inside the same map so -// the owner thread is the sole accessor. -// -// We use a per-thread static rather than a Mutex<HashMap> because the owner -// thread is the only accessor (no contention possible). -thread_local! { - static CONNS: std::cell::RefCell<HashMap<String, (Connection, wayland_client::EventQueue<super::State>)>> - = std::cell::RefCell::new(HashMap::new()); -} - -fn persist_conn(cursor_id: &str, conn: Connection) { - let queue = conn.new_event_queue::<super::State>(); - CONNS.with(|c| { - c.borrow_mut().insert(cursor_id.to_string(), (conn, queue)); - }); -} - -fn forget_conn(cursor_id: &str) { - CONNS.with(|c| { - c.borrow_mut().remove(cursor_id); - }); -} - -fn roundtrip_on_persistent(cursor_id: &str) -> anyhow::Result<()> { - CONNS.with(|c| { - let mut b = c.borrow_mut(); - let (_conn, queue) = b - .get_mut(cursor_id) - .ok_or_else(|| anyhow::anyhow!("no persistent connection for cursor '{cursor_id}'"))?; - let mut tmp = super::State::default(); - queue - .roundtrip(&mut tmp) - .map_err(|e| anyhow::anyhow!("compositor roundtrip failed: {e}"))?; - Ok(()) - }) -} - -// ── public API ──────────────────────────────────────────────────────────── - -/// Press and HOLD `button` (evdev code) at output coordinates `(x, y)` on the -/// toplevel identified by `window_id`. Subsequent `move_to` / `release` calls -/// targeting the same `cursor_id` reuse the same virtual-pointer device, so -/// the compositor treats the sequence as one logical drag rather than as -/// independent clicks. Errors if `cursor_id` already has a held button. -pub fn press(cursor_id: &str, window_id: u64, x: i32, y: i32, button: u8) -> anyhow::Result<()> { - let (tx_r, rx_r) = bounded(1); - tx().send(Cmd::Press { - cursor_id: cursor_id.to_string(), - window_id, - x, - y, - button, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?; - rx_r.recv() - .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))? -} - -/// Emit motion_absolute on the held cursor's virtual-pointer. Errors if there -/// is no held button for `cursor_id`. -pub fn move_to(cursor_id: &str, x: i32, y: i32) -> anyhow::Result<()> { - let (tx_r, rx_r) = bounded(1); - tx().send(Cmd::MoveTo { - cursor_id: cursor_id.to_string(), - x, - y, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?; - rx_r.recv() - .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))? -} - -/// Release `button` on the held cursor. If no other buttons remain held the -/// virtual-pointer is destroyed and its Wayland connection torn down. -pub fn release(cursor_id: &str, button: u8) -> anyhow::Result<()> { - let (tx_r, rx_r) = bounded(1); - tx().send(Cmd::Release { - cursor_id: cursor_id.to_string(), - button, - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?; - rx_r.recv() - .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))? -} - -/// Drop the entry for `cursor_id` without emitting any Wayland events. -/// Useful for recovery — if the agent thinks a button is held but the -/// compositor disagrees, this clears the local state without trying to -/// send a release that would error. -pub fn forget(cursor_id: &str) -> anyhow::Result<()> { - let (tx_r, rx_r) = bounded(1); - tx().send(Cmd::Forget { - cursor_id: cursor_id.to_string(), - reply: tx_r, - }) - .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?; - rx_r.recv() - .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))? -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal.rs deleted file mode 100644 index 166158bc5b4..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Shared xdg-desktop-portal connection helpers. -//! -//! ashpd's convenience constructors cache their first session-bus connection -//! process-wide. That is unsafe for our portal callers because several of them -//! deliberately use short-lived Tokio runtimes: once the first runtime is -//! dropped, the cached zbus connection no longer has a running executor and a -//! later portal call can wait forever. Always give ashpd an explicit -//! connection whose lifetime is owned by the caller's runtime instead. - -use std::time::Duration; - -pub(crate) const PROBE_TIMEOUT: Duration = Duration::from_secs(2); - -pub(crate) async fn fresh_session_connection() -> anyhow::Result<zbus::Connection> { - zbus::Connection::session() - .await - .map_err(|e| anyhow::anyhow!("xdg-desktop-portal session bus unreachable: {e}")) -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs deleted file mode 100644 index 14f1712dce2..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs +++ /dev/null @@ -1,465 +0,0 @@ -//! Per-window capture via `xdg-desktop-portal` ScreenCast + PipeWire. -//! -//! Reaches GNOME/Mutter and KDE/KWin per-window selection — the -//! cross-DE complement to wlroots `ext-image-copy-capture-v1`. Sequence: -//! -//! 1. ashpd ScreenCast::create_session. -//! 2. select_sources with types=Window|Monitor, cursor_mode=Embedded, -//! persist_mode=Application (consent dialog fires ONCE per process, -//! cached for the lifetime of the requesting binary). -//! 3. start(session, parent_window) — user picks the source window in -//! the portal's system dialog the first time; subsequent calls within -//! the same process reuse the cached choice. -//! 4. open_pipe_wire_remote returns a Unix file descriptor; Streams -//! carries the published node ids. -//! 5. Connect a pipewire client to the fd, open a Stream targeting the -//! advertised node id, negotiate Video/Raw format, dequeue one frame -//! with `STREAM_FLAG_MAP_BUFFERS`, copy pixels, PNG-encode, stop. -//! -//! The session + restore_token are persisted in a process-local -//! `OnceLock<PortalScreencastSession>` so only the first call across the -//! process lifetime triggers the consent dialog; later calls reuse the -//! same PipeWire node id. - -use std::os::fd::AsRawFd; -use std::sync::OnceLock; - -use ashpd::desktop::screencast::{ - CursorMode, OpenPipeWireRemoteOptions, Screencast, SelectSourcesOptions, SourceType, - StartCastOptions, Streams, -}; -use ashpd::desktop::{CreateSessionOptions, PersistMode, Session}; -use ashpd::enumflags2::BitFlags; - -/// A live portal ScreenCast session held across calls. Expensive to -/// create (system consent dialog) and cheap to reuse — the OnceLock -/// keeps it alive until the requesting process exits. -struct PortalScreencastSession { - _session: Session<Screencast>, - streams: Streams, - fd: std::os::fd::OwnedFd, -} - -unsafe impl Send for PortalScreencastSession {} -unsafe impl Sync for PortalScreencastSession {} - -static SESSION: OnceLock<PortalScreencastSession> = OnceLock::new(); - -/// Capture the user-selected window via `xdg-desktop-portal` ScreenCast. -/// On first invocation per process the portal dialog asks the user to -/// pick a window or monitor; subsequent calls reuse the same node -/// transparently. Returns PNG bytes of the latest frame on the stream. -pub fn screenshot_window_via_portal() -> anyhow::Result<Vec<u8>> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?; - - if SESSION.get().is_none() { - let sess = rt.block_on(create_portal_session())?; - let _ = SESSION.set(sess); - } - let session = SESSION - .get() - .ok_or_else(|| anyhow::anyhow!("portal ScreenCast session unavailable"))?; - - let node_id = session - .streams - .streams() - .iter() - .next() - .map(|s| s.pipe_wire_node_id()) - .ok_or_else(|| anyhow::anyhow!("portal advertised no streams"))?; - - capture_one_frame(session.fd.as_raw_fd(), node_id) -} - -async fn create_portal_session() -> anyhow::Result<PortalScreencastSession> { - let proxy = Screencast::new() - .await - .map_err(|e| anyhow::anyhow!("portal ScreenCast proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?; - - let session = proxy - .create_session(CreateSessionOptions::default()) - .await - .map_err(|e| anyhow::anyhow!("portal create_session failed: {e}"))?; - - let select_opts = SelectSourcesOptions::default() - .set_sources(BitFlags::<SourceType>::from(SourceType::Window) | SourceType::Monitor) - .set_cursor_mode(CursorMode::Embedded) - .set_persist_mode(PersistMode::Application); - - proxy - .select_sources(&session, select_opts) - .await - .map_err(|e| anyhow::anyhow!("portal select_sources failed: {e}"))? - .response() - .map_err(|e| anyhow::anyhow!("portal select_sources response error: {e}"))?; - - let streams = proxy - .start(&session, None, StartCastOptions::default()) - .await - .map_err(|e| anyhow::anyhow!("portal start failed: {e}"))? - .response() - .map_err(|e| anyhow::anyhow!("portal start response error (user denied?): {e}"))?; - - let fd = proxy - .open_pipe_wire_remote(&session, OpenPipeWireRemoteOptions::default()) - .await - .map_err(|e| anyhow::anyhow!("portal open_pipe_wire_remote failed: {e}"))?; - - Ok(PortalScreencastSession { - _session: session, - streams, - fd, - }) -} - -/// Pull one frame off the PipeWire stream advertised by the portal, -/// encode it as PNG, and return the bytes. -/// -/// Pipeline: -/// 1. `pipewire::MainLoop` + `Context` + `Core::connect_fd(portal_fd)` — -/// join the portal-published PipeWire remote on the same fd ashpd -/// returned from `open_pipe_wire_remote`. -/// 2. `Stream::new` with `media.type=Video, media.category=Capture, -/// media.role=Screen` so PipeWire knows to route the cast properly. -/// 3. Build an `EnumFormat` SPA pod listing the channel-orders we accept -/// (BGRx, BGRA, RGBx, RGBA) at any size up to 8K — the compositor -/// picks one and announces it via `param_changed`. We track w/h/fmt -/// from that callback. -/// 4. The `process` callback dequeues exactly one buffer, copies the -/// pixels out into a shared `Arc<Mutex<...>>`, and quits the main -/// loop. Subsequent buffers are dropped (queued back via Buffer::Drop) -/// once `frame.is_some()`. -/// 5. After `mainloop.run()` returns, channel-swap the captured bytes -/// into RGBA and PNG-encode them via the `image` crate. -fn capture_one_frame(pipewire_fd: i32, node_id: u32) -> anyhow::Result<Vec<u8>> { - use std::os::fd::FromRawFd; - use std::sync::{Arc, Mutex}; - - use pipewire as pw; - use pw::{properties::properties, spa}; - use spa::pod::Pod; - - pw::init(); - - let mainloop = pw::main_loop::MainLoop::new(None) - .map_err(|e| anyhow::anyhow!("pipewire MainLoop::new failed: {e}"))?; - let context = pw::context::Context::new(&mainloop) - .map_err(|e| anyhow::anyhow!("pipewire Context::new failed: {e}"))?; - - // dup the portal fd so the OwnedFd we hand to connect_fd has an - // independent lifecycle from the PortalScreencastSession's master copy. - let dup_fd = unsafe { libc::dup(pipewire_fd) }; - if dup_fd < 0 { - anyhow::bail!( - "dup of portal PipeWire fd failed: {}", - std::io::Error::last_os_error() - ); - } - let owned_fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(dup_fd) }; - let core = context - .connect_fd(owned_fd, None) - .map_err(|e| anyhow::anyhow!("pipewire connect_fd failed: {e}"))?; - - // Shared state populated by the param_changed + process callbacks. - // FrameBuf carries the negotiated geometry alongside the pixel data so - // the PNG encoder picks the right stride / channel order on exit. - struct FrameBuf { - bytes: Vec<u8>, - width: u32, - height: u32, - stride: u32, - format: spa::param::video::VideoFormat, - } - let frame: Arc<Mutex<Option<FrameBuf>>> = Arc::new(Mutex::new(None)); - let neg_format: Arc<Mutex<spa::param::video::VideoInfoRaw>> = - Arc::new(Mutex::new(Default::default())); - - let stream = pw::stream::Stream::new( - &core, - "cua-driver-capture", - properties! { - *pw::keys::MEDIA_TYPE => "Video", - *pw::keys::MEDIA_CATEGORY => "Capture", - *pw::keys::MEDIA_ROLE => "Screen", - }, - ) - .map_err(|e| anyhow::anyhow!("pipewire Stream::new failed: {e}"))?; - - let mainloop_for_process = mainloop.clone(); - let neg_format_for_param = neg_format.clone(); - let frame_for_process = frame.clone(); - let neg_format_for_process = neg_format.clone(); - - let _listener = stream - .add_local_listener::<()>() - .param_changed(move |_stream, _user, id, param| { - // The compositor confirms the negotiated VideoInfoRaw via the - // Format param event right after stream.connect — capture it - // so the process callback knows the width/height/format. - let Some(param) = param else { return }; - if id != spa::param::ParamType::Format.as_raw() { - return; - } - let (media_type, media_subtype) = match spa::param::format_utils::parse_format(param) { - Ok(v) => v, - Err(_) => return, - }; - if media_type != spa::param::format::MediaType::Video - || media_subtype != spa::param::format::MediaSubtype::Raw - { - return; - } - let mut info = spa::param::video::VideoInfoRaw::default(); - if info.parse(param).is_ok() { - if let Ok(mut g) = neg_format_for_param.lock() { - *g = info; - } - } - }) - .process(move |stream, _user| { - // Already captured? Drop further buffers (they'll be queued - // back via Buffer::Drop) so the run loop can finish. - if frame_for_process - .lock() - .ok() - .map(|g| g.is_some()) - .unwrap_or(false) - { - let _ = stream.dequeue_buffer(); - return; - } - let mut buffer = match stream.dequeue_buffer() { - Some(b) => b, - None => return, - }; - let datas = buffer.datas_mut(); - if datas.is_empty() { - return; - } - let info = match neg_format_for_process.lock() { - Ok(g) => *g, - Err(_) => return, - }; - let size_struct = info.size(); - let width = size_struct.width; - let height = size_struct.height; - if width == 0 || height == 0 { - return; - } - let chunk_stride = datas[0].chunk().stride(); - let chunk_size = datas[0].chunk().size(); - // PipeWire sometimes leaves stride at 0 (when the producer - // doesn't fill it in); fall back to width*4 for the packed - // BGRx/BGRA/RGBx/RGBA formats we negotiate. - let stride = if chunk_stride > 0 { - chunk_stride as u32 - } else if chunk_size > 0 && height > 0 { - chunk_size / height - } else { - width * 4 - }; - let payload = match datas[0].data() { - Some(p) => p, - None => return, - }; - let payload_len = (stride as usize) * (height as usize); - if payload.len() < payload_len { - return; - } - let bytes = payload[..payload_len].to_vec(); - if let Ok(mut g) = frame_for_process.lock() { - *g = Some(FrameBuf { - bytes, - width, - height, - stride, - format: info.format(), - }); - } - // Quit the main loop so the caller can encode + return. - mainloop_for_process.quit(); - }) - .register() - .map_err(|e| anyhow::anyhow!("pipewire stream listener register failed: {e}"))?; - - // Build an EnumFormat SPA pod listing the formats + sizes we accept. - // Compositors typically pick BGRx/BGRA on Linux desktops; we list a - // few alternates so negotiation succeeds on more sources. - let obj = pw::spa::pod::object!( - pw::spa::utils::SpaTypes::ObjectParamFormat, - pw::spa::param::ParamType::EnumFormat, - pw::spa::pod::property!( - pw::spa::param::format::FormatProperties::MediaType, - Id, - pw::spa::param::format::MediaType::Video - ), - pw::spa::pod::property!( - pw::spa::param::format::FormatProperties::MediaSubtype, - Id, - pw::spa::param::format::MediaSubtype::Raw - ), - pw::spa::pod::property!( - pw::spa::param::format::FormatProperties::VideoFormat, - Choice, - Enum, - Id, - pw::spa::param::video::VideoFormat::BGRx, - pw::spa::param::video::VideoFormat::BGRx, - pw::spa::param::video::VideoFormat::BGRA, - pw::spa::param::video::VideoFormat::RGBx, - pw::spa::param::video::VideoFormat::RGBA, - ), - pw::spa::pod::property!( - pw::spa::param::format::FormatProperties::VideoSize, - Choice, - Range, - Rectangle, - pw::spa::utils::Rectangle { - width: 1920, - height: 1080 - }, - pw::spa::utils::Rectangle { - width: 1, - height: 1 - }, - pw::spa::utils::Rectangle { - width: 7680, - height: 4320 - } - ), - pw::spa::pod::property!( - pw::spa::param::format::FormatProperties::VideoFramerate, - Choice, - Range, - Fraction, - pw::spa::utils::Fraction { num: 30, denom: 1 }, - pw::spa::utils::Fraction { num: 0, denom: 1 }, - pw::spa::utils::Fraction { - num: 1000, - denom: 1 - } - ), - ); - let values: Vec<u8> = pw::spa::pod::serialize::PodSerializer::serialize( - std::io::Cursor::new(Vec::new()), - &pw::spa::pod::Value::Object(obj), - ) - .map_err(|e| anyhow::anyhow!("EnumFormat pod serialization failed: {e}"))? - .0 - .into_inner(); - let mut params = [Pod::from_bytes(&values) - .ok_or_else(|| anyhow::anyhow!("EnumFormat pod parse-back failed"))?]; - - stream - .connect( - spa::utils::Direction::Input, - Some(node_id), - pw::stream::StreamFlags::AUTOCONNECT | pw::stream::StreamFlags::MAP_BUFFERS, - &mut params, - ) - .map_err(|e| anyhow::anyhow!("pipewire stream.connect failed: {e}"))?; - - // Install a 10-second timeout source so a producer that never delivers - // a frame (eg the user closed the consent dialog mid-cast) doesn't - // hang the main loop forever. - let timeout_loop = mainloop.clone(); - let _timeout_source = mainloop - .loop_() - .add_timer(move |_expirations: u64| timeout_loop.quit()); - _timeout_source - .update_timer(Some(std::time::Duration::from_secs(10)), None) - .into_result() - .map_err(|e| anyhow::anyhow!("pipewire timeout source failed: {e:?}"))?; - - mainloop.run(); - - // Tear down the stream before we read the frame, so the producer - // can't keep mutating the buffer underneath us. - drop(_listener); - drop(stream); - - let frame = frame - .lock() - .map_err(|_| anyhow::anyhow!("frame mutex poisoned"))? - .take() - .ok_or_else(|| anyhow::anyhow!("portal ScreenCast: no frame arrived within timeout"))?; - - encode_frame_to_png( - &frame.bytes, - frame.width, - frame.height, - frame.stride, - frame.format, - ) -} - -/// Channel-swap a PipeWire video frame into RGBA8888 and PNG-encode it. -/// -/// Mirrors `ext_screencopy::encode_buffer_to_png` — the input layout -/// depends on the negotiated VideoFormat: BGRx/BGRA are little-endian -/// BGRA (so swap R↔B), RGBx/RGBA are already RGBA in memory. -fn encode_frame_to_png( - pixels: &[u8], - width: u32, - height: u32, - stride: u32, - fmt: libspa::param::video::VideoFormat, -) -> anyhow::Result<Vec<u8>> { - use libspa::param::video::VideoFormat; - let mut rgba: Vec<u8> = Vec::with_capacity((width as usize) * (height as usize) * 4); - for y in 0..(height as usize) { - let row_start = y * (stride as usize); - for x in 0..(width as usize) { - let i = row_start + x * 4; - let (r, g, b, a) = match fmt { - VideoFormat::BGRx => (pixels[i + 2], pixels[i + 1], pixels[i], 0xFF), - VideoFormat::BGRA => (pixels[i + 2], pixels[i + 1], pixels[i], pixels[i + 3]), - VideoFormat::RGBx => (pixels[i], pixels[i + 1], pixels[i + 2], 0xFF), - VideoFormat::RGBA => (pixels[i], pixels[i + 1], pixels[i + 2], pixels[i + 3]), - // Fall through: treat unknown formats as BGRA (the most - // common compositor output on Linux). - _ => (pixels[i + 2], pixels[i + 1], pixels[i], pixels[i + 3]), - }; - rgba.extend_from_slice(&[r, g, b, a]); - } - } - use image::{codecs::png::PngEncoder, ImageBuffer, ImageEncoder, Rgba}; - let img: ImageBuffer<Rgba<u8>, _> = ImageBuffer::from_raw(width, height, rgba) - .ok_or_else(|| anyhow::anyhow!("internal: buffer dims mismatch ({}x{})", width, height))?; - let mut out: Vec<u8> = Vec::new(); - PngEncoder::new(&mut out).write_image( - img.as_raw(), - width, - height, - image::ExtendedColorType::Rgba8, - )?; - Ok(out) -} - -/// Probe whether the portal ScreenCast interface is reachable on the -/// session bus without taking an actual capture (avoids cached consent). -pub fn probe_screencast_portal() -> anyhow::Result<bool> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| anyhow::anyhow!("failed to build tokio runtime: {e}"))?; - rt.block_on(async { - let conn = zbus::Connection::session() - .await - .map_err(|e| anyhow::anyhow!("session bus unreachable: {e}"))?; - let proxy = zbus::fdo::DBusProxy::new(&conn) - .await - .map_err(|e| anyhow::anyhow!("dbus proxy creation failed: {e}"))?; - let bus_name: zbus::names::BusName = "org.freedesktop.portal.Desktop" - .try_into() - .map_err(|e| anyhow::anyhow!("bus name parse failed: {e}"))?; - let has_owner = proxy - .name_has_owner(bus_name) - .await - .map_err(|e| anyhow::anyhow!("name_has_owner failed: {e}"))?; - Ok(has_owner) - }) -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs deleted file mode 100644 index 8c317fb0920..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! `xdg-desktop-portal` Screenshot capture path. -//! -//! Reaches GNOME/Mutter, KDE/KWin, and any compositor that ships an -//! xdg-desktop-portal backend — superset of the wlroots-only -//! `zwlr_screencopy_manager_v1` path. The portal asks the user for consent -//! on first invocation per session (GNOME caches via `Settings → Privacy → -//! Screenshot`; KDE Plasma 6 caches per requesting binary); subsequent -//! calls in the same session are silent. -//! -//! ashpd is async (zbus under the hood); the rest of `wayland::screenshot_*` -//! is sync because it's called from `tokio::task::spawn_blocking`. We build -//! a dedicated single-threaded tokio runtime per call so the async ashpd -//! work has somewhere to live without colliding with the caller's runtime. - -use std::path::PathBuf; - -use ashpd::desktop::screenshot::Screenshot; - -/// Capture the current display via `org.freedesktop.portal.Screenshot.Screenshot`. -/// Returns the PNG/JPEG bytes the portal wrote to disk, or a typed error -/// when the portal isn't reachable (e.g. wlroots compositor without -/// `xdg-desktop-portal-wlr`, KDE without `xdg-desktop-portal-kde` running, -/// or no D-Bus session bus). -/// -/// `interactive=false` skips the customization dialog (region select, -/// post-shot editor) — the user still consents to the screenshot itself -/// on the first call per session unless they've pre-approved the -/// requesting binary in their DE's privacy settings. -pub fn screenshot_via_portal() -> anyhow::Result<Vec<u8>> { - if tokio::runtime::Handle::try_current().is_ok() { - return std::thread::spawn(screenshot_via_portal_blocking) - .join() - .map_err(|_| anyhow::anyhow!("xdg-desktop-portal Screenshot worker panicked"))?; - } - screenshot_via_portal_blocking() -} - -fn screenshot_via_portal_blocking() -> anyhow::Result<Vec<u8>> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?; - let uri = rt.block_on(async { - let connection = super::portal::fresh_session_connection().await?; - Screenshot::request() - .interactive(false) - .modal(false) - .connection(Some(connection)) - .send() - .await - .map_err(map_ashpd_err)? - .response() - .map_err(map_ashpd_err) - .map(|resp| resp.uri().clone()) - })?; - - // The portal returns a file:// URI pointing at a PNG (typically in - // /tmp or $XDG_RUNTIME_DIR/doc/...). Decode it via the `url` crate. - let path = uri_to_path(&uri.to_string()) - .ok_or_else(|| anyhow::anyhow!("portal returned a non-file URI: {uri}"))?; - let bytes = std::fs::read(&path) - .map_err(|e| anyhow::anyhow!("portal screenshot at {} unreadable: {e}", path.display()))?; - // Best-effort cleanup: the portal places the file in a process-readable - // location that can outlive our process. Removing avoids the disk - // accumulating one PNG per call. If removal fails (permissions, file - // already gone) we silently drop the error — the bytes are already - // captured. - let _ = std::fs::remove_file(&path); - Ok(bytes) -} - -/// Convert a `file://...` URI string to a local PathBuf. Returns None for -/// non-file schemes (e.g. portal `doc://...` URIs in some sandboxed -/// configurations, which would need a second resolution step through the -/// document portal). -fn uri_to_path(uri_str: &str) -> Option<PathBuf> { - let url = url::Url::parse(uri_str).ok()?; - if url.scheme() != "file" { - return None; - } - url.to_file_path().ok() -} - -fn map_ashpd_err<E: std::fmt::Display>(e: E) -> anyhow::Error { - let msg = format!("xdg-desktop-portal Screenshot failed: {e}"); - // Pattern-match common failure modes to produce a typed-ish error - // the caller can pivot on. - if msg.contains("ServiceUnknown") || msg.contains("NotFound") { - anyhow::anyhow!("xdg-desktop-portal is not running on this session ({msg}). Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde / xdg-desktop-portal-wlr for your compositor.") - } else if msg.contains("Cancelled") || msg.contains("denied") { - anyhow::anyhow!( - "xdg-desktop-portal Screenshot consent dialog was cancelled or denied: {msg}" - ) - } else { - anyhow::anyhow!(msg) - } -} - -/// Probe whether the portal *service* is reachable on the session bus. -/// Returns Ok(true) when the `org.freedesktop.portal.Desktop` name has an -/// owner, Ok(false) otherwise, and Err(...) for transport failures -/// (no session bus, etc.). -/// -/// Read-only and side-effect free — does NOT actually invoke the -/// Screenshot interface, so it doesn't cache a consent grant on GNOME / -/// KDE backends that gate Screenshot on user prompts. The doctor uses -/// this to surface portal availability without burning an interactive -/// dialog every time `hermes computer-use doctor` runs. -/// -/// Note: a true return here means the portal *service* is present. The -/// Screenshot *interface* on that service may still be missing or -/// disabled — for that, an actual `Screenshot::request().send().await` -/// is needed. We accept the false-positive risk in exchange for the -/// no-consent, no-burn-dialog property. -pub fn probe_portal() -> anyhow::Result<bool> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd probe: {e}"))?; - rt.block_on(async { - tokio::time::timeout(super::portal::PROBE_TIMEOUT, async { - // Cheapest possible probe: open a session bus and check that the - // portal name has an owner. We avoid Screenshot::request() because - // that would cache a (potentially undesired) consent grant. - let conn = super::portal::fresh_session_connection().await?; - let proxy = zbus::fdo::DBusProxy::new(&conn) - .await - .map_err(|e| anyhow::anyhow!("dbus proxy creation failed: {e}"))?; - let has_owner = proxy - .name_has_owner( - "org.freedesktop.portal.Desktop" - .try_into() - .map_err(|e| anyhow::anyhow!("bus name parse failed: {e}"))?, - ) - .await - .map_err(|e| anyhow::anyhow!("name_has_owner failed: {e}"))?; - Ok(has_owner) - }) - .await - .map_err(|_| anyhow::anyhow!("xdg-desktop-portal availability probe timed out"))? - }) -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs deleted file mode 100644 index 3f3a539a35e..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs +++ /dev/null @@ -1,695 +0,0 @@ -//! Client for the bundled **cua WinRects** GNOME Shell extension -//! (`org.cua.WinRects`, see `wayland-helper/winrects@cua/`). -//! -//! On GNOME Mutter (and other non-wlroots compositors) a normal client cannot -//! query a window's on-screen origin (`org.gnome.Shell.Introspect.GetWindows` -//! is privacy-denied; Wayland exposes no global coordinates) nor position an -//! overlay surface at a screen coordinate (no `zwlr_layer_shell_v1`). Both are -//! solvable only from *inside* the compositor — which is what the extension is -//! for. It runs in the shell's privileged context and exposes: -//! -//! - `GetRects() -> json` — every window's `meta_window.get_frame_rect()` (screen -//! geometry), so `screen_xy = window_origin + AT-SPI CoordType::Window xy` -//! (the GNOME analogue of the X11 `_GTK_FRAME_EXTENTS` reconstruction). -//! - `MoveCursor(x,y)` / `ClickPulse(x,y)` / `HideCursor()` — draw the agent -//! cursor as a Clutter actor on the compositor stage. -//! - `SetCursorState(...)` / `SetCursorColor(...)` — keep the compositor cursor -//! aligned with the shared semantic theme and active session identity. -//! -//! Everything here is **best-effort**: if the extension isn't installed/enabled -//! the calls return `None` / no-op and callers keep the prior behaviour (no -//! screen coords, no Wayland cursor). Uses a short-lived `gdbus` subprocess so -//! there's no zbus blocking-feature or async-context coupling — the calls are -//! infrequent (once per `get_window_state`, a few per click). - -use std::os::unix::fs::{MetadataExt, PermissionsExt}; -use std::process::Command; -use std::time::Duration; - -use crate::x11::WindowInfo; - -const DEST: &str = "org.cua.WinRects"; -const PATH: &str = "/org/cua/WinRects"; -const IFACE: &str = "org.cua.WinRects"; -const DBUS_DEST: &str = "org.freedesktop.DBus"; -const DBUS_PATH: &str = "/org/freedesktop/DBus"; -const DBUS_IFACE: &str = "org.freedesktop.DBus"; -const BROWSER_HELPER_API_VERSION: u32 = 4; -const SEMANTIC_CURSOR_API_VERSION: u32 = 8; - -#[derive(Debug, Clone)] -struct ShellWindow { - info: WindowInfo, - focused: bool, -} - -pub fn available() -> bool { - shell_owner(false).is_some() -} - -pub fn semantic_cursor_available() -> bool { - shell_owner_with_min_version(Some(SEMANTIC_CURSOR_API_VERSION)).is_some() -} - -fn gdbus_call(method: &str, args: &[String]) -> Option<String> { - let owner = shell_owner(false)?; - gdbus_call_to( - &owner, - PATH, - &format!("{IFACE}.{method}"), - args, - Duration::from_millis(800), - ) -} - -fn gdbus_call_with_timeout(method: &str, args: &[String], timeout: Duration) -> Option<String> { - let owner = shell_owner(false)?; - gdbus_call_to(&owner, PATH, &format!("{IFACE}.{method}"), args, timeout) -} - -fn trusted_gdbus_call(method: &str, args: &[String]) -> Option<String> { - let owner = shell_owner(true)?; - gdbus_call_to( - &owner, - PATH, - &format!("{IFACE}.{method}"), - args, - Duration::from_millis(800), - ) -} - -fn gdbus_call_to( - destination: &str, - object_path: &str, - method: &str, - args: &[String], - timeout: Duration, -) -> Option<String> { - let mut cmd = Command::new("gdbus"); - cmd.arg("call") - .arg("--session") - .arg("--dest") - .arg(destination) - .arg("--object-path") - .arg(object_path) - .arg("--method") - .arg(method); - for a in args { - cmd.arg(a); - } - // gdbus is local IPC; cap it so a wedged shell can't stall the caller. - let child = cmd - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::null()) - .spawn() - .ok()?; - let out = wait_timeout(child, timeout)?; - if !out.status.success() { - return None; - } - Some(String::from_utf8_lossy(&out.stdout).into_owned()) -} - -/// Resolve the helper's immutable unique bus name and prove that it is hosted -/// by this user's system-installed GNOME Shell process. Browser-sensitive -/// callers additionally require the current helper API. Calling the unique -/// name closes the race where another process replaces the well-known name -/// after ownership is checked. -fn shell_owner(require_browser_api: bool) -> Option<String> { - shell_owner_with_min_version(require_browser_api.then_some(BROWSER_HELPER_API_VERSION)) -} - -fn shell_owner_with_min_version(min_version: Option<u32>) -> Option<String> { - let owner_raw = gdbus_call_to( - DBUS_DEST, - DBUS_PATH, - &format!("{DBUS_IFACE}.GetNameOwner"), - &[DEST.to_owned()], - Duration::from_millis(800), - )?; - let owner = parse_quoted_string(&owner_raw)?; - if !owner.starts_with(':') { - return None; - } - - let pid_raw = gdbus_call_to( - DBUS_DEST, - DBUS_PATH, - &format!("{DBUS_IFACE}.GetConnectionUnixProcessID"), - &[owner.clone()], - Duration::from_millis(800), - )?; - let uid_raw = gdbus_call_to( - DBUS_DEST, - DBUS_PATH, - &format!("{DBUS_IFACE}.GetConnectionUnixUser"), - &[owner.clone()], - Duration::from_millis(800), - )?; - let pid = parse_first_u32(&pid_raw)?; - let uid = parse_first_u32(&uid_raw)?; - if uid != current_uid() || !is_trusted_gnome_shell(pid) { - return None; - } - - if let Some(min_version) = min_version { - let version_raw = gdbus_call_to( - &owner, - PATH, - &format!("{IFACE}.GetVersion"), - &[], - Duration::from_millis(800), - )?; - if parse_first_u32(&version_raw)? < min_version { - return None; - } - } - Some(owner) -} - -fn parse_quoted_string(raw: &str) -> Option<String> { - let start = raw.find('\'')? + 1; - let end = raw[start..].find('\'')? + start; - (end > start).then(|| raw[start..end].to_owned()) -} - -fn parse_first_u32(raw: &str) -> Option<u32> { - // `gdbus call` renders typed scalars as `(uint32 6079,)`. Searching the - // whole string would incorrectly return the `32` in the type annotation. - let payload = raw.split_once("uint32").map_or(raw, |(_, payload)| payload); - payload - .split(|character: char| !character.is_ascii_digit()) - .find(|part| !part.is_empty())? - .parse() - .ok() -} - -fn current_uid() -> u32 { - std::fs::metadata("/proc/self") - .map(|meta| meta.uid()) - .unwrap_or(u32::MAX) -} - -fn is_trusted_gnome_shell(pid: u32) -> bool { - let comm = std::fs::read_to_string(format!("/proc/{pid}/comm")).ok(); - if comm.as_deref().map(str::trim) != Some("gnome-shell") { - return false; - } - let executable = std::fs::read_link(format!("/proc/{pid}/exe")).ok(); - let metadata = executable - .as_ref() - .and_then(|path| std::fs::metadata(path).ok()); - executable - .as_ref() - .and_then(|path| path.file_name()) - .and_then(|name| name.to_str()) - == Some("gnome-shell") - && metadata - .as_ref() - .is_some_and(|meta| meta.uid() == 0 && meta.permissions().mode() & 0o022 == 0) -} - -/// Capture the GNOME stage through the compositor helper. -/// -/// Mutter does not expose wlroots screencopy protocols, and its one-shot -/// Screenshot portal may reject an unregistered command-line process. The -/// opt-in helper already runs inside Shell for geometry and activation, so it -/// can use Shell's screenshot API without confusing a stable Wayland window id -/// for an X11 drawable. -pub fn screenshot_display() -> Option<Vec<u8>> { - let raw = gdbus_call_with_timeout("Capture", &[], Duration::from_secs(5))?; - decode_capture(&raw) -} - -/// Capture the GNOME stage only when the helper owner and current browser API -/// have passed the same compositor-attestation checks used for mutation. -pub fn trusted_screenshot_display() -> Option<Vec<u8>> { - let owner = shell_owner(true)?; - let raw = gdbus_call_to( - &owner, - PATH, - &format!("{IFACE}.Capture"), - &[], - Duration::from_secs(5), - )?; - decode_capture(&raw) -} - -fn decode_capture(raw: &str) -> Option<Vec<u8>> { - use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; - - let start = raw.find('\'')? + 1; - let end = raw.rfind('\'')?; - if end <= start { - return None; - } - B64.decode(&raw[start..end]).ok() -} - -/// `Child::wait` with a deadline (no extra crates). Kills + reaps on timeout. -fn wait_timeout(mut child: std::process::Child, dur: Duration) -> Option<std::process::Output> { - use std::io::Read; - - // Drain stdout while the child is running. Capture() returns a base64 PNG - // that readily exceeds a pipe's ~64 KiB capacity; waiting for exit before - // reading deadlocks the child on a full pipe and turns a healthy Shell - // response into a false timeout. - let stdout = child.stdout.take()?; - let reader = std::thread::spawn(move || { - let mut stdout = stdout; - let mut bytes = Vec::new(); - stdout.read_to_end(&mut bytes).ok()?; - Some(bytes) - }); - let deadline = std::time::Instant::now() + dur; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break status, - Ok(None) => { - if std::time::Instant::now() >= deadline { - let _ = child.kill(); - let status = child.wait().ok()?; - let _ = reader.join(); - if !status.success() { - return None; - } - return None; - } - std::thread::sleep(Duration::from_millis(15)); - } - Err(_) => { - let _ = child.kill(); - let _ = child.wait(); - let _ = reader.join(); - return None; - } - } - }; - let stdout = reader.join().ok().flatten()?; - Some(std::process::Output { - status, - stdout, - stderr: Vec::new(), - }) -} - -/// Screen origin of the compositor frame backing `pid`. -/// -/// `screenshot_window_dispatch` crops the Shell stage to this same frame -/// rectangle, and GTK's AT-SPI `CoordType::Window` coordinates are relative to -/// that frame. Using Mutter's larger surface-buffer rectangle here shifts -/// elements up and left by the client-side shadow extents whenever the window -/// is floating, so pixel actions derived from the returned screenshot miss -/// their target. -pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> { - let raw = gdbus_call("GetRects", &[])?; - parse_window_origin(&raw, pid) -} - -fn parse_window_origin(raw: &str, pid: u32) -> Option<(i32, i32)> { - // gdbus prints a GVariant tuple like `('[{"pid":..,"x":..}]',)`. Pull the - // JSON array out robustly (first '[' .. last ']') rather than parsing the - // GVariant wrapper, so an apostrophe in a window title can't break it. - let start = raw.find('[')?; - let end = raw.rfind(']')?; - let json = &raw[start..=end]; - let arr: Vec<serde_json::Value> = serde_json::from_str(json).ok()?; - for w in &arr { - if w.get("pid").and_then(|p| p.as_u64()) == Some(pid as u64) { - let x = w.get("x").and_then(serde_json::Value::as_i64)? as i32; - let y = w.get("y").and_then(serde_json::Value::as_i64)? as i32; - return Some((x, y)); - } - } - None -} - -/// Enumerate GNOME Shell toplevels when the compositor helper is available. -/// -/// AT-SPI remains the source of accessibility elements, but it is a poor -/// source of truth for desktop window discovery: one unresponsive application -/// can exhaust the bounded registry walk and hide every healthy toplevel. The -/// shell already owns the authoritative stacking list, geometry, visibility, -/// title, and PID, so use that metadata directly for `list_windows`. -pub fn list_windows(filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> { - let raw = gdbus_call("GetRects", &[])?; - parse_windows(&raw, filter_pid) -} - -/// Return one compositor-attested GNOME window only when the current helper -/// API is hosted by the verified Shell owner. -pub fn trusted_window_for_id(pid: u32, window_id: u64) -> Option<WindowInfo> { - trusted_shell_windows(Some(pid))? - .into_iter() - .find(|window| window.info.xid == window_id) - .map(|window| window.info) -} - -/// Enumerate exact browser-window ids for one process through the verified -/// GNOME Shell helper. `None` means no trusted helper is available; an empty -/// vector means the trusted helper found no owned windows. -pub fn trusted_window_ids_for_pid(pid: u32) -> Option<Vec<u64>> { - Some( - trusted_shell_windows(Some(pid))? - .into_iter() - .map(|window| window.info.xid) - .collect(), - ) -} - -pub(crate) fn trusted_focused_window_for_pid(pid: u32) -> Option<u64> { - trusted_shell_windows(Some(pid))? - .into_iter() - .find(|window| window.focused) - .map(|window| window.info.xid) -} - -/// Briefly activate an exact compositor-owned window, execute one bounded -/// focus-sensitive operation, and restore the prior exact Shell focus. -pub fn with_focused_window<T>( - pid: u32, - window_id: u64, - body: impl FnOnce() -> anyhow::Result<T>, -) -> anyhow::Result<T> { - let before = trusted_shell_windows(None) - .ok_or_else(|| anyhow::anyhow!("the verified GNOME Shell helper API is unavailable"))?; - let target = before - .iter() - .find(|window| window.info.pid == Some(pid) && window.info.xid == window_id) - .ok_or_else(|| anyhow::anyhow!("no exact GNOME Shell window owns the approved target"))?; - let previous = before - .iter() - .find(|window| window.focused) - .map(|window| window.info.xid) - .ok_or_else(|| anyhow::anyhow!("GNOME Shell did not expose a restorable focused window"))?; - - if target.focused { - return body(); - } - trusted_activate_window(window_id) - .then_some(()) - .ok_or_else(|| anyhow::anyhow!("GNOME Shell did not confirm exact target activation"))?; - let body_result = body(); - let restored = trusted_activate_window(previous); - match (body_result, restored) { - (Ok(value), true) => Ok(value), - (Err(error), true) => Err(error), - (Ok(_), false) => { - anyhow::bail!("GNOME Shell did not restore the previously focused window") - } - (Err(error), false) => Err(anyhow::anyhow!( - "{error}; GNOME Shell also failed to restore the previously focused window" - )), - } -} - -fn trusted_shell_windows(filter_pid: Option<u32>) -> Option<Vec<ShellWindow>> { - let raw = trusted_gdbus_call("GetRects", &[])?; - parse_shell_windows(&raw, filter_pid) -} - -/// Ask GNOME Shell to focus and raise one stable-sequence window. -/// -/// Returns `false` when the helper is absent, the id is unknown, or Shell did -/// not confirm focus. Callers must not inject global libei input unless this -/// returns true: portal input is focus-bound and otherwise targets whichever -/// application the user happened to be using. -pub fn activate_window(window_id: u64) -> bool { - let Ok(window_id) = u32::try_from(window_id) else { - return false; - }; - let accepted = gdbus_call("Activate", &[window_id.to_string()]) - .is_some_and(|output| output.trim_start().starts_with("(true,")); - if !accepted { - return false; - } - let deadline = std::time::Instant::now() + Duration::from_millis(500); - loop { - if window_is_focused(window_id) { - return true; - } - if std::time::Instant::now() >= deadline { - return false; - } - std::thread::sleep(Duration::from_millis(10)); - } -} - -fn trusted_activate_window(window_id: u64) -> bool { - let Ok(window_id) = u32::try_from(window_id) else { - return false; - }; - let accepted = trusted_gdbus_call("Activate", &[window_id.to_string()]) - .is_some_and(|output| output.trim_start().starts_with("(true,")); - if !accepted { - return false; - } - let deadline = std::time::Instant::now() + Duration::from_millis(500); - loop { - if trusted_shell_windows(None).is_some_and(|windows| { - windows - .into_iter() - .any(|window| window.info.xid == u64::from(window_id) && window.focused) - }) { - return true; - } - if std::time::Instant::now() >= deadline { - return false; - } - std::thread::sleep(Duration::from_millis(10)); - } -} - -fn window_is_focused(window_id: u32) -> bool { - let Some(raw) = gdbus_call("GetRects", &[]) else { - return false; - }; - let (Some(start), Some(end)) = (raw.find('['), raw.rfind(']')) else { - return false; - }; - serde_json::from_str::<Vec<serde_json::Value>>(&raw[start..=end]) - .ok() - .and_then(|windows| { - windows.into_iter().find(|window| { - window.get("id").and_then(serde_json::Value::as_u64) == Some(window_id as u64) - }) - }) - .and_then(|window| window.get("focused").and_then(serde_json::Value::as_bool)) - .unwrap_or(false) -} - -fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> { - Some( - parse_shell_windows(raw, filter_pid)? - .into_iter() - .map(|window| window.info) - .collect(), - ) -} - -fn parse_shell_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<ShellWindow>> { - let start = raw.find('[')?; - let end = raw.rfind(']')?; - let windows: Vec<serde_json::Value> = serde_json::from_str(&raw[start..=end]).ok()?; - - Some( - windows - .into_iter() - .filter_map(|window| { - let pid = u32::try_from(window.get("pid")?.as_u64()?).ok()?; - if filter_pid.is_some_and(|wanted| wanted != pid) { - return None; - } - let id = window.get("id")?.as_u64()?.max(1); - let x = i32::try_from(window.get("x")?.as_i64()?).ok()?; - let y = i32::try_from(window.get("y")?.as_i64()?).ok()?; - let width = u32::try_from(window.get("w")?.as_u64()?).ok()?; - let height = u32::try_from(window.get("h")?.as_u64()?).ok()?; - let visible = window - .get("visible") - .and_then(serde_json::Value::as_bool) - .unwrap_or(width > 0 && height > 0); - let minimized = window - .get("minimized") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); - let title = window - .get("title") - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .to_owned(); - let z_index = window - .get("stacking") - .and_then(serde_json::Value::as_u64) - .and_then(|value| usize::try_from(value).ok()); - - Some(ShellWindow { - info: WindowInfo { - xid: id, - pid: Some(pid), - app_name: title.clone(), - title, - is_on_screen: visible && !minimized && width > 0 && height > 0, - z_index, - x, - y, - width, - height, - }, - focused: window - .get("focused") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false), - }) - }) - .collect(), - ) -} - -/// Glide the agent cursor to screen `(x, y)`. -pub fn move_cursor(x: i32, y: i32) { - let _ = gdbus_call("MoveCursor", &[x.to_string(), y.to_string()]); -} - -/// Snap + pulse the agent cursor at screen `(x, y)` (a click indicator). -pub fn click_pulse(x: i32, y: i32) { - let _ = gdbus_call("ClickPulse", &[x.to_string(), y.to_string()]); -} - -/// Set the stable session-specific fill color for the compositor cursor. -pub fn set_cursor_color(fill_color: &str) { - let _ = gdbus_call("SetCursorColor", &[fill_color.to_owned()]); -} - -/// Update the compositor-owned cursor's semantic action state. -/// -/// Callers gate this method on helper v8 so an older helper cannot silently -/// render the retired cursor artwork. -pub fn set_cursor_state(action: &str, delivery: &str, target: &str, active: bool) { - let _ = gdbus_call( - "SetCursorState", - &[ - action.to_owned(), - delivery.to_owned(), - target.to_owned(), - active.to_string(), - ], - ); -} - -/// Set the renderer-visible public session label for the compositor cursor. -pub fn set_session_label(label: &str) { - let _ = gdbus_call("SetSessionLabel", &[label.to_owned()]); -} - -/// Hide the agent cursor. -pub fn hide_cursor() { - let _ = gdbus_call("HideCursor", &[]); -} - -#[cfg(test)] -mod tests { - use super::*; - - const EXTENSION_SOURCE: &str = - include_str!("../../../../../wayland-helper/winrects@cua/extension.js"); - const EXTENSION_METADATA: &str = - include_str!("../../../../../wayland-helper/winrects@cua/metadata.json"); - - #[test] - fn parses_and_filters_shell_windows() { - let raw = r#"('[{"id":46,"pid":6079,"title":"Sentinel's window","x":66,"y":32,"w":958,"h":736,"focused":true,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#; - let windows = parse_windows(raw, Some(6079)).expect("valid helper response"); - assert_eq!(windows.len(), 1); - assert_eq!(windows[0].xid, 46); - assert_eq!(windows[0].pid, Some(6079)); - assert_eq!(windows[0].title, "Sentinel's window"); - assert_eq!((windows[0].x, windows[0].y), (66, 32)); - assert_eq!((windows[0].width, windows[0].height), (958, 736)); - assert!(windows[0].is_on_screen); - assert_eq!(windows[0].z_index, Some(2)); - } - - #[test] - fn accessibility_origin_matches_the_frame_cropped_screenshot() { - let raw = r#"('[{"id":46,"pid":6079,"title":"Floating GTK","x":14,"y":12,"w":560,"h":736,"buffer_x":0,"buffer_y":0}]',)"#; - - assert_eq!(parse_window_origin(raw, 6079), Some((14, 12))); - } - - #[test] - fn marks_minimized_shell_windows_off_screen() { - let raw = r#"('[{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#; - let windows = parse_windows(raw, None).expect("valid helper response"); - assert_eq!(windows.len(), 1); - assert!(!windows[0].is_on_screen); - } - - #[test] - fn parses_dbus_owner_and_numeric_identity() { - assert_eq!( - parse_quoted_string("(':1.204',)"), - Some(":1.204".to_owned()) - ); - assert_eq!(parse_first_u32("(uint32 6079,)"), Some(6079)); - assert_eq!(parse_first_u32("(uint32 4,)"), Some(4)); - assert_eq!(parse_first_u32("(6079,)"), Some(6079)); - assert_eq!(parse_quoted_string("(nothing,)"), None); - assert_eq!(parse_first_u32("(nothing,)"), None); - } - - #[test] - fn preserves_exact_focus_from_shell_snapshot() { - let raw = r#"('[{"id":46,"pid":6079,"title":"Target","x":66,"y":32,"w":958,"h":736,"focused":false,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Sentinel","x":0,"y":0,"w":100,"h":100,"focused":true,"minimized":false,"visible":true,"stacking":3}]',)"#; - let windows = parse_shell_windows(raw, None).expect("valid helper response"); - assert_eq!(windows.len(), 2); - assert!(!windows[0].focused); - assert!(windows[1].focused); - assert_eq!(windows[1].info.xid, 47); - } - - #[test] - fn bundled_helper_v8_uses_host_owned_modifier_badge_chips() { - assert!(EXTENSION_SOURCE.contains("GetVersion()")); - assert!(EXTENSION_SOURCE.contains("return 8;")); - assert!(EXTENSION_SOURCE.contains("SetCursorState")); - assert!(EXTENSION_SOURCE.contains("SetCursorColor")); - assert!(EXTENSION_SOURCE.contains("SetSessionLabel")); - assert!(EXTENSION_SOURCE.contains("const DISPLAY_SIZE = 42;")); - assert!(EXTENSION_SOURCE.contains("const GLOW_PADDING = 24;")); - assert!(EXTENSION_SOURCE.contains("function drawCursorGlowShape")); - assert!(EXTENSION_SOURCE.contains("function glowPath")); - assert!(EXTENSION_SOURCE.contains("strokePath(cr, width, alpha, fillColor)")); - assert!(EXTENSION_SOURCE.contains("width + 1.5")); - assert!(EXTENSION_SOURCE.contains("width - 1")); - assert!(EXTENSION_SOURCE.contains("createGlowSurface(this._fillColor)")); - assert!(EXTENSION_SOURCE.contains("cr.translate(-GLOW_PADDING, -GLOW_PADDING);")); - assert!(EXTENSION_SOURCE.contains("function drawBadgeChip")); - assert!(EXTENSION_SOURCE.contains("function badgeStyle(fillColor)")); - assert!(EXTENSION_SOURCE.contains("this._badge.add_child(this._badgeLabel)")); - assert!(EXTENSION_SOURCE.contains("this._deliveryChip")); - assert!(EXTENSION_SOURCE.contains("this._targetChip")); - assert!(EXTENSION_SOURCE.contains("if (labelAlpha > 0.001 || chipAlpha > 0.001)")); - assert!(EXTENSION_SOURCE.contains("this._badgeLabel.hide()")); - assert!(!EXTENSION_SOURCE.contains("this._badgeIdentity")); - assert!(!EXTENSION_SOURCE.contains("this._badgeDot")); - assert!(!EXTENSION_SOURCE.contains("function drawModifiers")); - let metadata: serde_json::Value = - serde_json::from_str(EXTENSION_METADATA).expect("valid bundled helper metadata"); - assert_eq!(metadata["version"], 8); - - for action in [ - "idle", "observe", "click", "drag", "scroll", "text", "key", "navigate", "app", - "transfer", "record", "system", - ] { - assert!( - EXTENSION_SOURCE.contains(&format!("'{action}'")), - "missing semantic cursor state {action}" - ); - } - - assert!(!EXTENSION_SOURCE.contains("const VERTS")); - assert!(!EXTENSION_SOURCE.contains("setSourceRGBA(0.10, 0.75, 1.00")); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs deleted file mode 100644 index bc090b94cba..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! Best-effort Sway/i3-compatible compositor metadata. -//! -//! The wlroots foreign-toplevel protocol exposes titles and app ids, but not -//! process ids or geometry. Sway's IPC tree supplies those missing fields and -//! a compositor-stable container id. Other compositors simply return no data. - -use std::process::{Command, Stdio}; - -use serde::Deserialize; - -#[derive(Clone, Debug, Default, Deserialize)] -struct Rect { - x: i32, - y: i32, - width: i32, - height: i32, -} - -#[derive(Clone, Debug, Default, Deserialize)] -struct Node { - id: u64, - #[serde(default)] - name: String, - #[serde(default)] - app_id: String, - pid: Option<u32>, - #[serde(default)] - rect: Rect, - #[serde(default)] - window_rect: Rect, - #[serde(default)] - deco_rect: Rect, - #[serde(default)] - focused: bool, - #[serde(default = "default_visible")] - visible: bool, - #[serde(default)] - fullscreen_mode: i32, - #[serde(default)] - nodes: Vec<Node>, - #[serde(default)] - floating_nodes: Vec<Node>, -} - -fn default_visible() -> bool { - true -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Window { - pub id: u64, - pub pid: u32, - pub title: String, - pub app_id: String, - pub x: i32, - pub y: i32, - pub width: u32, - pub height: u32, - pub content_x: i32, - pub content_y: i32, - pub focused: bool, - pub visible: bool, - pub fullscreen: bool, -} - -fn collect(node: &Node, windows: &mut Vec<Window>) { - if let Some(pid) = node.pid { - if !node.name.is_empty() || !node.app_id.is_empty() { - // Sway normally reports the client surface origin in - // `window_rect`. Some server-decorated Wayland clients instead - // leave that origin at zero and expose the title-bar inset only - // through `deco_rect`; use it as the content origin in that shape. - let inferred_top_inset = if node.window_rect.height > 0 { - node.rect - .height - .saturating_sub(node.window_rect.height) - .max(0) - } else { - 0 - }; - let content_y = if node.window_rect.y != 0 { - node.window_rect.y - } else { - node.deco_rect - .y - .saturating_add(node.deco_rect.height.max(0)) - .max(inferred_top_inset) - }; - windows.push(Window { - id: node.id, - pid, - title: node.name.clone(), - app_id: node.app_id.clone(), - x: node.rect.x, - y: node.rect.y, - width: node.rect.width.max(0) as u32, - height: node.rect.height.max(0) as u32, - content_x: node.window_rect.x, - content_y, - focused: node.focused, - visible: node.visible, - fullscreen: node.fullscreen_mode != 0, - }); - } - } - for child in node.nodes.iter().chain(&node.floating_nodes) { - collect(child, windows); - } -} - -fn parse_tree(bytes: &[u8]) -> Option<Vec<Window>> { - let root: Node = serde_json::from_slice(bytes).ok()?; - let mut windows = Vec::new(); - collect(&root, &mut windows); - Some(windows) -} - -pub fn list_windows() -> Option<Vec<Window>> { - if std::env::var_os("SWAYSOCK").is_none() { - return None; - } - let output = Command::new("swaymsg") - .args(["-r", "-t", "get_tree"]) - .stdin(Stdio::null()) - .stderr(Stdio::null()) - .output() - .ok()?; - if !output.status.success() { - return None; - } - parse_tree(&output.stdout) -} - -pub fn window_for_id(id: u64) -> Option<Window> { - list_windows()?.into_iter().find(|window| window.id == id) -} - -pub fn window_for_pid(pid: u32) -> Option<Window> { - list_windows()? - .into_iter() - .filter(|window| window.pid == pid && window.width > 0 && window.height > 0) - .max_by_key(|window| { - ( - window.focused, - window.visible, - u64::from(window.width) * u64::from(window.height), - ) - }) -} - -pub fn window_for_title(title: &str) -> Option<Window> { - list_windows()? - .into_iter() - .filter(|window| { - window.width > 0 - && window.height > 0 - && (window.title == title - || (!window.title.is_empty() && title.starts_with(&window.title)) - || (!title.is_empty() && window.title.starts_with(title))) - }) - .max_by_key(|window| (window.focused, window.visible)) -} - -pub fn window_for_app_id(app_id: &str) -> Option<Window> { - if app_id.is_empty() { - return None; - } - list_windows()? - .into_iter() - .filter(|window| window.width > 0 && window.height > 0 && window.app_id == app_id) - .max_by_key(|window| (window.focused, window.visible)) -} - -fn focus_container(id: u64) -> bool { - let selector = format!("[con_id={id}]"); - Command::new("swaymsg") - .args([selector.as_str(), "focus"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .is_ok_and(|status| status.success()) -} - -fn wait_for_container_focus(id: u64, timeout: std::time::Duration) -> bool { - let deadline = std::time::Instant::now() + timeout; - loop { - if window_for_id(id).is_some_and(|window| window.focused) { - return true; - } - if std::time::Instant::now() >= deadline { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(10)); - } -} - -/// Briefly focus one compositor-attested container, run `body`, then restore -/// the previously focused container. The caller must already have verified the -/// target container's PID and id through [`window_for_id`]. -pub fn with_focused_container<T>( - id: u64, - body: impl FnOnce() -> anyhow::Result<T>, -) -> anyhow::Result<T> { - let prior = list_windows() - .ok_or_else(|| anyhow::anyhow!("Sway IPC tree is unavailable"))? - .into_iter() - .find(|window| window.focused) - .map(|window| window.id); - if !focus_container(id) { - anyhow::bail!("Sway refused to focus exact container {id}"); - } - if !wait_for_container_focus(id, std::time::Duration::from_millis(500)) { - anyhow::bail!("Sway did not confirm focus on exact container {id}"); - } - let result = body(); - let restore = prior - .filter(|prior| *prior != id) - .map(|prior| { - if !focus_container(prior) { - anyhow::bail!("Sway could not restore prior container {prior}"); - } - if !wait_for_container_focus(prior, std::time::Duration::from_millis(500)) { - anyhow::bail!("Sway did not confirm restored container {prior}"); - } - Ok(()) - }) - .transpose(); - match (result, restore) { - (Ok(value), Ok(_)) => Ok(value), - (Err(error), Ok(_)) => Err(error), - (Ok(_), Err(error)) => Err(error), - (Err(error), Err(restore)) => Err(error.context(format!( - "the prior Sway focus also could not be restored: {restore}" - ))), - } -} - -pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> { - window_for_pid(pid).map(|window| (window.x, window.y)) -} - -/// Offset of the application content inside Sway's captured toplevel. -/// Server-side decorations are part of `rect`/window screenshots but not of -/// WebKitGTK's AT-SPI `CoordType::Window` descendants. -pub fn window_content_offset_for_pid(pid: u32) -> Option<(i32, i32)> { - window_for_pid(pid).map(|window| (window.content_x, window.content_y)) -} - -pub fn window_origin_for_title(title: &str) -> Option<(i32, i32)> { - let window = window_for_title(title)?; - Some((window.x, window.y)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_nested_and_floating_windows() { - let tree = br#"{ - "id": 1, - "nodes": [{ - "id": 2, - "nodes": [{ - "id": 10, - "name": "Editor", - "app_id": "org.example.Editor", - "pid": 123, - "rect": {"x": 20, "y": 30, "width": 800, "height": 600}, - "window_rect": {"x": 0, "y": 47, "width": 800, "height": 553}, - "focused": true, - "visible": true, - "fullscreen_mode": 1 - }], - "floating_nodes": [{ - "id": 11, - "name": "Dialog", - "pid": 124, - "rect": {"x": 100, "y": 120, "width": 300, "height": 200} - }] - }] - }"#; - let windows = parse_tree(tree).expect("parse Sway tree"); - assert_eq!(windows.len(), 2); - assert_eq!(windows[0].pid, 123); - assert_eq!((windows[0].x, windows[0].y), (20, 30)); - assert_eq!((windows[0].content_x, windows[0].content_y), (0, 47)); - assert!(windows[0].focused); - assert!(windows[0].fullscreen); - assert_eq!(windows[1].title, "Dialog"); - } - - #[test] - fn decoration_height_fills_zero_window_content_origin() { - let tree = br#"{ - "id": 1, - "nodes": [{ - "id": 10, - "name": "Tauri", - "app_id": "cua-test-harness", - "pid": 123, - "rect": {"x": 0, "y": 0, "width": 940, "height": 780}, - "window_rect": {"x": 0, "y": 0, "width": 940, "height": 733}, - "deco_rect": {"x": 0, "y": 0, "width": 940, "height": 47} - }] - }"#; - let windows = parse_tree(tree).expect("parse decorated Sway tree"); - assert_eq!((windows[0].content_x, windows[0].content_y), (0, 47)); - } - - #[test] - fn outer_client_height_delta_fills_missing_decoration_metadata() { - let tree = br#"{ - "id": 1, - "nodes": [{ - "id": 10, - "name": "Tauri", - "app_id": "cua-test-harness", - "pid": 123, - "rect": {"x": 0, "y": 0, "width": 940, "height": 780}, - "window_rect": {"x": 0, "y": 0, "width": 940, "height": 733}, - "deco_rect": {"x": 0, "y": 0, "width": 0, "height": 0} - }] - }"#; - let windows = parse_tree(tree).expect("parse Sway tree with implicit decoration"); - assert_eq!((windows[0].content_x, windows[0].content_y), (0, 47)); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/wayland/virtual_keyboard.rs b/packages/cua-driver/rust/crates/platform-linux/src/wayland/virtual_keyboard.rs deleted file mode 100644 index 5a57436ace9..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/src/wayland/virtual_keyboard.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! Stable physical-key injection for wlroots virtual-keyboard compositors. - -use std::ffi::CString; -use std::fs::File; -use std::io::{Seek, SeekFrom, Write}; -use std::os::fd::{AsFd, FromRawFd}; - -use wayland_client::protocol::{wl_registry, wl_seat::WlSeat}; -use wayland_client::{Connection, Dispatch, Proxy, QueueHandle}; - -use self::protocol::zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1; -use self::protocol::zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1; - -mod protocol { - use wayland_client; - use wayland_client::protocol::*; - - pub mod __interfaces { - use wayland_client::protocol::__interfaces::*; - wayland_scanner::generate_interfaces!("protocol/virtual-keyboard-unstable-v1.xml"); - } - use self::__interfaces::*; - wayland_scanner::generate_client_code!("protocol/virtual-keyboard-unstable-v1.xml"); -} - -const XKB_KEYMAP: &str = r#"xkb_keymap { - xkb_keycodes { include "evdev+aliases(qwerty)" }; - xkb_types { include "complete" }; - xkb_compatibility { include "complete" }; - xkb_symbols { include "pc+us+inet(evdev)" }; - xkb_geometry { include "pc(pc105)" }; -}; -"#; - -const KEY_RELEASED: u32 = 0; -const KEY_PRESSED: u32 = 1; - -#[derive(Default)] -struct State { - seat: Option<WlSeat>, - manager: Option<ZwpVirtualKeyboardManagerV1>, -} - -impl Dispatch<wl_registry::WlRegistry, ()> for State { - fn event( - state: &mut Self, - registry: &wl_registry::WlRegistry, - event: wl_registry::Event, - _: &(), - _: &Connection, - qh: &QueueHandle<Self>, - ) { - if let wl_registry::Event::Global { - name, - interface, - version, - } = event - { - if interface == WlSeat::interface().name { - state.seat = Some(registry.bind(name, version.min(7), qh, ())); - } else if interface == ZwpVirtualKeyboardManagerV1::interface().name { - state.manager = Some(registry.bind(name, version.min(1), qh, ())); - } - } - } -} - -impl Dispatch<WlSeat, ()> for State { - fn event( - _: &mut Self, - _: &WlSeat, - _: <WlSeat as Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -impl Dispatch<ZwpVirtualKeyboardManagerV1, ()> for State { - fn event( - _: &mut Self, - _: &ZwpVirtualKeyboardManagerV1, - _: <ZwpVirtualKeyboardManagerV1 as Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -impl Dispatch<ZwpVirtualKeyboardV1, ()> for State { - fn event( - _: &mut Self, - _: &ZwpVirtualKeyboardV1, - _: <ZwpVirtualKeyboardV1 as Proxy>::Event, - _: &(), - _: &Connection, - _: &QueueHandle<Self>, - ) { - } -} - -pub(super) fn hotkey(modifiers: &[String], key: &str) -> anyhow::Result<()> { - let transitions = hotkey_transitions(modifiers, key)?; - let conn = Connection::connect_to_env()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - let mut state = State::default(); - queue.roundtrip(&mut state)?; - queue.roundtrip(&mut state)?; - - let seat = state - .seat - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor exposes no wl_seat"))?; - let manager = state - .manager - .clone() - .ok_or_else(|| anyhow::anyhow!("compositor exposes no zwp_virtual_keyboard_manager_v1"))?; - let keyboard = manager.create_virtual_keyboard(&seat, &qh, ()); - let keymap = keymap_file()?; - keyboard.keymap(1, keymap.as_fd(), XKB_KEYMAP.len() as u32 + 1); - queue.roundtrip(&mut state)?; - - // A harmless Shift tap absorbs the first-event loss seen on fresh - // headless wlroots seats without changing the requested chord. - send_key(&keyboard, &mut queue, &mut state, 42, KEY_PRESSED)?; - send_key(&keyboard, &mut queue, &mut state, 42, KEY_RELEASED)?; - for transition in transitions { - send_key( - &keyboard, - &mut queue, - &mut state, - transition.keycode, - if transition.pressed { - KEY_PRESSED - } else { - KEY_RELEASED - }, - )?; - keyboard.modifiers(transition.modifier_mask, 0, 0, 0); - queue.roundtrip(&mut state)?; - } - keyboard.destroy(); - queue.roundtrip(&mut state)?; - Ok(()) -} - -fn send_key( - keyboard: &ZwpVirtualKeyboardV1, - queue: &mut wayland_client::EventQueue<State>, - state: &mut State, - keycode: u32, - key_state: u32, -) -> anyhow::Result<()> { - keyboard.key(super::event_time_ms(), keycode, key_state); - queue.roundtrip(state)?; - std::thread::sleep(std::time::Duration::from_millis(4)); - Ok(()) -} - -fn keymap_file() -> anyhow::Result<File> { - let name = CString::new("cua-driver-keymap")?; - let fd = unsafe { libc::memfd_create(name.as_ptr(), libc::MFD_CLOEXEC) }; - if fd < 0 { - return Err(std::io::Error::last_os_error().into()); - } - let mut file = unsafe { File::from_raw_fd(fd) }; - file.write_all(XKB_KEYMAP.as_bytes())?; - file.write_all(&[0])?; - file.seek(SeekFrom::Start(0))?; - Ok(file) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct KeyTransition { - keycode: u32, - pressed: bool, - modifier_mask: u32, -} - -fn hotkey_transitions(modifiers: &[String], key: &str) -> anyhow::Result<Vec<KeyTransition>> { - let mut modifier_mask = 0; - let mut modifier_keys = Vec::with_capacity(modifiers.len()); - for modifier in modifiers { - let (keycode, mask) = match modifier.as_str() { - "ctrl" => (29, 4), - "shift" => (42, 1), - "alt" => (56, 8), - "logo" => (125, 64), - other => anyhow::bail!("unsupported Wayland modifier '{other}'"), - }; - modifier_keys.push((keycode, mask)); - } - let keycode = super::key_to_evdev(key) - .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}'"))?; - let mut transitions = Vec::with_capacity(modifier_keys.len() * 2 + 2); - for (modifier_key, mask) in &modifier_keys { - modifier_mask |= mask; - transitions.push(KeyTransition { - keycode: *modifier_key, - pressed: true, - modifier_mask, - }); - } - transitions.push(KeyTransition { - keycode, - pressed: true, - modifier_mask, - }); - transitions.push(KeyTransition { - keycode, - pressed: false, - modifier_mask, - }); - for (modifier_key, mask) in modifier_keys.iter().rev() { - modifier_mask &= !mask; - transitions.push(KeyTransition { - keycode: *modifier_key, - pressed: false, - modifier_mask, - }); - } - Ok(transitions) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn chord_uses_physical_digit7_and_balanced_modifiers() { - assert_eq!( - hotkey_transitions(&["ctrl".into(), "shift".into()], "7").unwrap(), - [ - KeyTransition { - keycode: 29, - pressed: true, - modifier_mask: 4 - }, - KeyTransition { - keycode: 42, - pressed: true, - modifier_mask: 5 - }, - KeyTransition { - keycode: 8, - pressed: true, - modifier_mask: 5 - }, - KeyTransition { - keycode: 8, - pressed: false, - modifier_mask: 5 - }, - KeyTransition { - keycode: 42, - pressed: false, - modifier_mask: 4 - }, - KeyTransition { - keycode: 29, - pressed: false, - modifier_mask: 0 - }, - ] - ); - } -} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/packages/cua-driver/rust/crates/platform-linux/src/x11/mod.rs index 3b669fd415f..5c72aa392ff 100644 --- a/packages/cua-driver/rust/crates/platform-linux/src/x11/mod.rs +++ b/packages/cua-driver/rust/crates/platform-linux/src/x11/mod.rs @@ -5,6 +5,7 @@ use anyhow::Result; use x11rb::connection::Connection; +use x11rb::protocol::res::{ClientIdMask, ClientIdSpec, ConnectionExt as _}; use x11rb::protocol::xproto::*; use x11rb::rust_connection::RustConnection; @@ -47,7 +48,7 @@ pub(crate) fn window_owner_pid(xid: u64) -> Option<u32> { let Ok((conn, _)) = RustConnection::connect(None) else { return None; }; - get_window_pid(&conn, xid).ok().flatten() + OwnerResolver::new(&conn).owner_pid(xid) } pub(crate) fn is_minimized(xid: u64) -> bool { @@ -115,6 +116,7 @@ fn try_list_windows_inner( filter_pid: Option<u32>, include_untitled: bool, ) -> Result<Vec<WindowInfo>> { + crate::require_x11()?; let (conn, screen_num) = RustConnection::connect(None)?; let screen = &conn.setup().roots[screen_num]; let root = screen.root; @@ -122,9 +124,10 @@ fn try_list_windows_inner( // Get _NET_CLIENT_LIST_STACKING (or fallback to _NET_CLIENT_LIST). let (windows, stacking_known) = get_window_list(&conn, root)?; + let owners = OwnerResolver::new(&conn); let mut result = Vec::new(); for (z_index, xid) in windows.into_iter().enumerate() { - let pid = get_window_pid(&conn, xid).ok().flatten(); + let pid = owners.owner_pid(xid); if let Some(fp) = filter_pid { if pid != Some(fp) { continue; @@ -173,6 +176,24 @@ fn try_list_windows_inner( Ok(result) } +pub(crate) fn window_manager_name() -> Option<String> { + let (conn, screen) = RustConnection::connect(None).ok()?; + let root = conn.setup().roots[screen].root; + let atom = get_atom(&conn, "_NET_SUPPORTING_WM_CHECK").ok()?; + let read = |window| { + conn.get_property(false, window, atom, AtomEnum::WINDOW, 0, 1) + .ok()? + .reply() + .ok()? + .value32()? + .next() + }; + let window = read(root).filter(|window| *window != 0)?; + (read(window) == Some(window)) + .then(|| get_window_title(&conn, window).ok()) + .flatten() +} + pub fn resolve_app_window(windows: &[WindowInfo]) -> Option<u64> { let (conn, screen) = RustConnection::connect(None).ok()?; let atom = get_atom(&conn, "_NET_ACTIVE_WINDOW").ok()?; @@ -336,11 +357,10 @@ pub fn set_window_frame( let xid = u32::try_from(xid).map_err(|_| anyhow::anyhow!("window_id is out of X11 range"))?; let (conn, screen_num) = RustConnection::connect(None)?; let root = conn.setup().roots[screen_num].root; - let owner = get_window_pid(&conn, xid)?; - match owner { + match OwnerResolver::new(&conn).owner_pid(xid) { Some(owner) if owner == pid => {} Some(owner) => anyhow::bail!("window_id {xid} belongs to pid {owner}, not pid {pid}"), - None => anyhow::bail!("window_id {xid} has no verifiable _NET_WM_PID owner"), + None => anyhow::bail!("window_id {xid} has no verifiable owner pid ({UNATTRIBUTED_OWNER})"), } let atom = get_atom(&conn, "_NET_MOVERESIZE_WINDOW")?; @@ -397,6 +417,118 @@ fn get_window_pid(conn: &RustConnection, window: Window) -> Result<Option<u32>> Ok(reply.value32().and_then(|mut i| i.next())) } +/// Why an X11 toplevel can be listed with no owner pid. +const UNATTRIBUTED_OWNER: &str = "the client publishes no _NET_WM_PID and the X server's \ +X-Resource extension could not attribute it to a local process: XRes 1.2 is unavailable, \ +the client is remote or forwarded, or the server is outside this PID namespace"; + +/// Owner pid of client-list toplevels: `_NET_WM_PID` when published, else +/// the X-Resource `LocalClientPID` of the connection that created the window, +/// subject to the fail-closed rules in [`crate::x11_client_pid`]. The XRes +/// probe runs at most once per connection and only for windows that lack +/// `_NET_WM_PID`. +struct OwnerResolver<'c> { + conn: &'c RustConnection, + xres: std::cell::OnceCell<bool>, + hostname: std::cell::OnceCell<Option<String>>, +} + +impl<'c> OwnerResolver<'c> { + fn new(conn: &'c RustConnection) -> Self { + Self { + conn, + xres: std::cell::OnceCell::new(), + hostname: std::cell::OnceCell::new(), + } + } + + fn owner_pid(&self, window: Window) -> Option<u32> { + match get_window_pid(self.conn, window) { + Ok(Some(pid)) => Some(pid), + Ok(None) => self.xres_pid(window), + Err(_) => None, + } + } + + fn xres_pid(&self, window: Window) -> Option<u32> { + use crate::x11_client_pid::{ + client_base, client_machine_is_local, select_owner_pid, xres_supports_client_ids, + XresClientValue, + }; + let available = *self.xres.get_or_init(|| { + self.conn + .res_query_version(1, 2) + .ok() + .and_then(|cookie| cookie.reply().ok()) + .is_some_and(|v| xres_supports_client_ids(v.server_major, v.server_minor)) + }); + if !available { + return None; + } + let machine = get_client_machine(self.conn, window).ok()?; + let hostname = self.hostname.get_or_init(|| { + std::fs::read_to_string("/proc/sys/kernel/hostname") + .ok() + .map(|h| h.trim().to_owned()) + }); + if !client_machine_is_local(machine.as_deref(), hostname.as_deref()) { + return None; + } + let setup = self.conn.setup(); + let own_base = setup.resource_id_base; + let specs = [ + ClientIdSpec { + client: own_base, + mask: ClientIdMask::LOCAL_CLIENT_PID, + }, + ClientIdSpec { + client: window, + mask: ClientIdMask::LOCAL_CLIENT_PID, + }, + ]; + let reply = self.conn.res_query_client_ids(&specs).ok()?.reply().ok()?; + let values: Vec<XresClientValue<'_>> = reply + .ids + .iter() + .map(|id| XresClientValue { + client: id.spec.client, + mask: u32::from(id.spec.mask), + value: &id.value, + }) + .collect(); + let pid = select_owner_pid( + &values, + own_base, + std::process::id(), + client_base(window, setup.resource_id_mask), + )?; + // The server keys the query by the client slot in the XID. A window + // that still exists after the reply proves its creator held that slot + // when the server answered, so a recycled slot cannot be attributed. + self.conn.get_window_attributes(window).ok()?.reply().ok()?; + Some(pid) + } +} + +/// `WM_CLIENT_MACHINE`, `None` when unset. Errors when the window is gone. +fn get_client_machine(conn: &RustConnection, window: Window) -> Result<Option<String>> { + let reply = conn + .get_property( + false, + window, + AtomEnum::WM_CLIENT_MACHINE, + AtomEnum::ANY, + 0, + 256, + )? + .reply()?; + if reply.type_ == x11rb::NONE { + return Ok(None); + } + let value = String::from_utf8_lossy(&reply.value); + Ok(Some(value.trim_end_matches('\0').to_owned())) +} + fn get_window_title(conn: &RustConnection, window: Window) -> Result<String> { // Try _NET_WM_NAME (UTF-8) first. if let Ok(atom) = get_atom(conn, "_NET_WM_NAME") { @@ -544,3 +676,40 @@ mod tests { assert!(indices[2] > indices[0]); } } + +pub fn window_info(xid: u64) -> Option<WindowInfo> { + let window = u32::try_from(xid).ok()?; + let (conn, screen_num) = RustConnection::connect(None).ok()?; + let root = conn.setup().roots[screen_num].root; + let attributes = conn.get_window_attributes(window).ok()?.reply().ok()?; + let geom = conn.get_geometry(window).ok()?.reply().ok()?; + let trans = conn + .translate_coordinates(window, root, 0, 0) + .ok()? + .reply() + .ok()?; + let pid = OwnerResolver::new(&conn).owner_pid(window); + let title = get_window_title(&conn, window).unwrap_or_default(); + let app_name = get_window_class(&conn, window) + .map(|(instance, class)| if class.is_empty() { instance } else { class }) + .unwrap_or_default(); + Some(WindowInfo { + xid, + pid, + app_name, + title, + is_on_screen: attributes.map_state == MapState::VIEWABLE, + z_index: None, + x: i32::from(trans.dst_x), + y: i32::from(trans.dst_y), + width: u32::from(geom.width), + height: u32::from(geom.height), + }) +} + +// Parent/child walks must not attribute a window-manager frame to the manager. +pub fn window_pid(xid: u64) -> Option<u32> { + let xid = u32::try_from(xid).ok()?; + let (conn, _) = RustConnection::connect(None).ok()?; + get_window_pid(&conn, xid).ok().flatten() +} diff --git a/packages/cua-driver/rust/crates/platform-linux/src/x11_client_pid.rs b/packages/cua-driver/rust/crates/platform-linux/src/x11_client_pid.rs new file mode 100644 index 00000000000..ee5d57e2bb0 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-linux/src/x11_client_pid.rs @@ -0,0 +1,291 @@ +//! Pure selection rules for attributing an X11 toplevel to a local process +//! when the client does not publish `_NET_WM_PID`. +//! +//! Tk, many Java/AWT builds, Wine, and legacy Xlib/Xt clients never set +//! `_NET_WM_PID`. The X-Resource extension (`XResQueryClientIds` with +//! `XRES_CLIENT_ID_PID_MASK`, XRes >= 1.2) lets the X server report the PID +//! of the connection that created a resource, which it learns from the Unix +//! socket peer credentials. That answer is only trustworthy when all of the +//! following hold, and these rules fail closed otherwise: +//! +//! - the server reports our own connection's PID as our real PID, which proves +//! the driver is a local client in the same PID namespace as the server (a +//! remote `DISPLAY`, a TCP connection, or a container boundary fails this); +//! - the server reports exactly one PID for the client that created the +//! window, never the server itself (client 0 owns the root window); +//! - the window does not name a different host in `WM_CLIENT_MACHINE`, which +//! is how an SSH-forwarded client (whose socket peer is the local `ssh` +//! process, not the application) identifies itself. +//! +//! The X11 I/O lives in `crate::x11`; everything here is platform-independent +//! so it is unit-tested on every host. + +/// `XRES_CLIENT_ID_PID_MASK` (X-Resource 1.2, `LocalClientPID`). +pub(crate) const LOCAL_CLIENT_PID_MASK: u32 = 1 << 1; + +/// One `ClientIdValue` of an `XResQueryClientIds` reply. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct XresClientValue<'a> { + /// The client's resource-id base (`clientAsMask`), as echoed by the server. + pub client: u32, + pub mask: u32, + pub value: &'a [u32], +} + +/// The XRes version that introduced `QueryClientIds`. +pub(crate) fn xres_supports_client_ids(major: u16, minor: u16) -> bool { + (major, minor) >= (1, 2) +} + +/// Resource-id base of the client that created `xid`: the server allocates +/// every client a disjoint id range, so clearing the client-local bits of +/// `resource_id_mask` (from the connection setup) names the creator. +pub(crate) fn client_base(xid: u32, resource_id_mask: u32) -> u32 { + xid & !resource_id_mask +} + +/// The PID the server reported for `client`, if it reported exactly one. +fn reported_pid(values: &[XresClientValue<'_>], client: u32) -> Option<u32> { + let mut pids = values + .iter() + .filter(|v| v.client == client && v.mask == LOCAL_CLIENT_PID_MASK) + .map(|v| match v.value { + [pid] if *pid != 0 && i32::try_from(*pid).is_ok() => Some(*pid), + _ => None, + }); + let first = pids.next()??; + pids.next().is_none().then_some(first) +} + +/// Select the owner PID of the window created by the client at `target_base` +/// from a reply that also covered our own client at `own_base`. +/// +/// Returns `None` unless the server reported `own_pid` for our own +/// connection (so its PIDs are local and in our namespace) and exactly one +/// valid PID for the target client. The server's own resources (client base +/// 0, e.g. the root window) are never attributed. +pub(crate) fn select_owner_pid( + values: &[XresClientValue<'_>], + own_base: u32, + own_pid: u32, + target_base: u32, +) -> Option<u32> { + if target_base == 0 || reported_pid(values, own_base) != Some(own_pid) { + return None; + } + reported_pid(values, target_base) +} + +/// Whether `WM_CLIENT_MACHINE` allows a local-socket PID to stand for the +/// application. An absent or empty property is accepted (most clients that +/// omit `_NET_WM_PID` omit it too); a present one must name this host, and an +/// unknown local hostname fails closed. +pub(crate) fn client_machine_is_local(machine: Option<&str>, local_hostname: Option<&str>) -> bool { + let Some(machine) = machine.map(str::trim).filter(|m| !m.is_empty()) else { + return true; + }; + let Some(local) = local_hostname.map(str::trim).filter(|h| !h.is_empty()) else { + return false; + }; + let short = |name: &str| name.split('.').next().unwrap_or(name).to_ascii_lowercase(); + machine.eq_ignore_ascii_case(local) || short(machine) == short(local) +} + +#[cfg(test)] +mod tests { + use super::*; + + const OWN_BASE: u32 = 0x0060_0000; + const OWN_PID: u32 = 4242; + const TK_BASE: u32 = 0x0120_0000; + const MASK: u32 = 0x001f_ffff; + + #[test] + fn a_local_client_without_net_wm_pid_is_attributed_to_its_creator() { + let own = [OWN_PID]; + let tk = [777]; + let values = [ + XresClientValue { + client: OWN_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &own, + }, + XresClientValue { + client: TK_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &tk, + }, + ]; + let window = TK_BASE | 0x0000_0017; + assert_eq!(client_base(window, MASK), TK_BASE); + assert_eq!( + select_owner_pid(&values, OWN_BASE, OWN_PID, client_base(window, MASK)), + Some(777) + ); + } + + #[test] + fn a_remote_or_forwarded_target_without_a_pid_value_stays_unattributed() { + // The server returns no LocalClientPID value for TCP clients. + let own = [OWN_PID]; + let values = [XresClientValue { + client: OWN_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &own, + }]; + assert_eq!(select_owner_pid(&values, OWN_BASE, OWN_PID, TK_BASE), None); + } + + #[test] + fn a_server_in_another_pid_namespace_or_host_is_never_trusted() { + // Our own connection reports a different PID (container boundary) or + // none at all (the driver itself reached the server over TCP). + let own = [1]; + let tk = [777]; + let foreign = [ + XresClientValue { + client: OWN_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &own, + }, + XresClientValue { + client: TK_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &tk, + }, + ]; + assert_eq!(select_owner_pid(&foreign, OWN_BASE, OWN_PID, TK_BASE), None); + let remote = [XresClientValue { + client: TK_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &tk, + }]; + assert_eq!(select_owner_pid(&remote, OWN_BASE, OWN_PID, TK_BASE), None); + } + + #[test] + fn ambiguous_malformed_or_invalid_pid_values_fail_closed() { + let own = [OWN_PID]; + let a = [777]; + let b = [778]; + let two = [777, 778]; + let zero = [0]; + let negative = [u32::MAX]; + let base = XresClientValue { + client: OWN_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &own, + }; + for target in [&two[..], &zero[..], &negative[..], &[][..]] { + let values = [ + base, + XresClientValue { + client: TK_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: target, + }, + ]; + assert_eq!( + select_owner_pid(&values, OWN_BASE, OWN_PID, TK_BASE), + None, + "{target:?}" + ); + } + let duplicated = [ + base, + XresClientValue { + client: TK_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &a, + }, + XresClientValue { + client: TK_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &b, + }, + ]; + assert_eq!( + select_owner_pid(&duplicated, OWN_BASE, OWN_PID, TK_BASE), + None + ); + // A CLIENT_XID value is not a PID. + let wrong_mask = [ + base, + XresClientValue { + client: TK_BASE, + mask: 1, + value: &a, + }, + ]; + assert_eq!( + select_owner_pid(&wrong_mask, OWN_BASE, OWN_PID, TK_BASE), + None + ); + } + + #[test] + fn server_owned_resources_are_never_attributed() { + let own = [OWN_PID]; + let server = [1]; + let values = [ + XresClientValue { + client: OWN_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &own, + }, + XresClientValue { + client: 0, + mask: LOCAL_CLIENT_PID_MASK, + value: &server, + }, + ]; + assert_eq!(select_owner_pid(&values, OWN_BASE, OWN_PID, 0), None); + } + + #[test] + fn our_own_window_resolves_to_our_pid() { + let own = [OWN_PID]; + let values = [XresClientValue { + client: OWN_BASE, + mask: LOCAL_CLIENT_PID_MASK, + value: &own, + }]; + assert_eq!( + select_owner_pid(&values, OWN_BASE, OWN_PID, OWN_BASE), + Some(OWN_PID) + ); + } + + #[test] + fn query_client_ids_requires_xres_1_2() { + assert!(!xres_supports_client_ids(1, 0)); + assert!(!xres_supports_client_ids(1, 1)); + assert!(xres_supports_client_ids(1, 2)); + assert!(xres_supports_client_ids(2, 0)); + } + + #[test] + fn wm_client_machine_must_be_absent_or_this_host() { + assert!(client_machine_is_local(None, Some("ci-runner"))); + assert!(client_machine_is_local(Some(""), None)); + assert!(client_machine_is_local( + Some("ci-runner"), + Some("ci-runner") + )); + assert!(client_machine_is_local( + Some("CI-Runner.example.com"), + Some("ci-runner") + )); + assert!(client_machine_is_local( + Some("ci-runner"), + Some("ci-runner.local\n") + )); + // An SSH-forwarded client names its remote host. + assert!(!client_machine_is_local( + Some("build-box"), + Some("ci-runner") + )); + // Unknown local hostname: cannot prove a named machine is local. + assert!(!client_machine_is_local(Some("ci-runner"), None)); + } +} diff --git a/packages/cua-driver/rust/crates/platform-linux/tests/wayland_app.rs b/packages/cua-driver/rust/crates/platform-linux/tests/wayland_app.rs deleted file mode 100644 index f2662744134..00000000000 --- a/packages/cua-driver/rust/crates/platform-linux/tests/wayland_app.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Explicitly run on an isolated headless Sway desktop, never the user's seat. -#![cfg(target_os = "linux")] -use serde_json::json; -use std::process::{Child, Command}; -use std::time::{Duration, Instant}; -struct Fixture(Child); -impl Drop for Fixture { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "requires isolated Sway, native Wayland GTK3 and a session bus"] -async fn live_wayland_app_follows_compositor_focus() { - let child = Command::new("/usr/bin/python3") - .args([ - "-c", - r#" -import gi -gi.require_version('Gtk','3.0') -from gi.repository import Gtk -windows=[] -for title in ['CUA first','CUA second']: - w=Gtk.Window(title=title);w.add(Gtk.Entry());w.show_all();windows.append(w) -Gtk.main() -"#, - ]) - .spawn() - .unwrap(); - let fixture = Fixture(child); - let deadline = Instant::now() + Duration::from_secs(10); - let windows = loop { - let windows = platform_linux::wayland::sway_ipc::list_windows() - .unwrap_or_default() - .into_iter() - .filter(|w| w.pid == fixture.0.id()) - .collect::<Vec<_>>(); - if windows.len() == 2 { - break windows; - } - assert!(Instant::now() < deadline, "two Wayland windows must appear"); - std::thread::sleep(Duration::from_millis(100)); - }; - let registry = platform_linux::tools::build_registry(false); - for window in windows { - let output = Command::new("swaymsg") - .arg(format!("[con_id={}] focus", window.id)) - .output() - .unwrap(); - assert!(output.status.success()); - let result = registry - .invoke( - "list_windows", - json!({"pid":fixture.0.id(),"app_context":true,"on_screen_only":false}), - ) - .await; - assert_ne!(result.is_error, Some(true), "{result:?}"); - let state = result.structured_content.unwrap(); - let selected = state["windows"] - .as_array() - .unwrap() - .iter() - .filter(|w| w["is_app_target"] == true) - .collect::<Vec<_>>(); - assert_eq!(selected.len(), 1, "{state}"); - assert_eq!(selected[0]["window_id"], window.id, "{state}"); - } - println!("App selection followed both native Sway windows in the same process"); -} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/apple_events.rs b/packages/cua-driver/rust/crates/platform-macos/src/apple_events.rs new file mode 100644 index 00000000000..00a734e5bff --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-macos/src/apple_events.rs @@ -0,0 +1,577 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Background routes through an app's scripting interface (Apple Events). +//! +//! Some commands cannot run in a background app through accessibility or key +//! events. TextEdit keeps File > Save disabled while it is not the active app +//! (AppKit validates Save against the main window, and a background app has +//! none), and background key events never reach a menu's key equivalents. +//! Finder closes an inline rename field when it is not active, and an AXValue +//! write to a file's name cell changes only the list. An Apple Event runs the +//! same command inside the app without activating it, raising a window, or +//! touching the pointer, so these routes keep the user's front app untouched. +//! +//! Apple Events need the user's Automation consent for each target app. The +//! permission is read first without asking. When macOS has never asked, it +//! shows its consent prompt once, on the first route that needs that app. +//! When the user has turned it off, the caller gets an honest refusal. + +use std::ffi::c_void; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Whether this process may send Apple Events to an app. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AutomationPermission { + Granted, + /// macOS has not asked the user yet; the first event shows its prompt. + NeedsConsent, + /// The user turned it off in System Settings > Privacy & Security > + /// Automation. + Denied, + /// The target app is not running. + NotRunning, + Unknown(i32), +} + +#[repr(C)] +struct AeDesc { + descriptor_type: u32, + data_handle: *mut c_void, +} + +#[link(name = "CoreServices", kind = "framework")] +extern "C" { + fn AECreateDesc( + type_code: u32, + data_ptr: *const c_void, + data_size: isize, + result: *mut AeDesc, + ) -> i16; + fn AEDisposeDesc(desc: *mut AeDesc) -> i16; + fn AEDeterminePermissionToAutomateTarget( + target: *const AeDesc, + event_class: u32, + event_id: u32, + ask_user_if_needed: u8, + ) -> i32; +} + +const TYPE_APPLICATION_BUNDLE_ID: u32 = u32::from_be_bytes(*b"bund"); +const TYPE_WILDCARD: u32 = u32::from_be_bytes(*b"****"); + +fn permission_from_status(status: i32) -> AutomationPermission { + match status { + 0 => AutomationPermission::Granted, + -1743 => AutomationPermission::Denied, + -1744 => AutomationPermission::NeedsConsent, + -600 => AutomationPermission::NotRunning, + other => AutomationPermission::Unknown(other), + } +} + +/// Read this process's Automation permission for `bundle_id` without asking. +pub fn automation_permission(bundle_id: &str) -> AutomationPermission { + let mut desc = AeDesc { + descriptor_type: 0, + data_handle: std::ptr::null_mut(), + }; + let bytes = bundle_id.as_bytes(); + let created = unsafe { + AECreateDesc( + TYPE_APPLICATION_BUNDLE_ID, + bytes.as_ptr().cast(), + bytes.len() as isize, + &mut desc, + ) + }; + if created != 0 { + return AutomationPermission::Unknown(created as i32); + } + let status = + unsafe { AEDeterminePermissionToAutomateTarget(&desc, TYPE_WILDCARD, TYPE_WILDCARD, 0) }; + unsafe { AEDisposeDesc(&mut desc) }; + permission_from_status(status) +} + +/// A route that did not run, in words a caller can act on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScriptRefusal { + pub code: &'static str, + pub reason: String, +} + +/// Run `script` with `osascript`, passing `args` to its `on run argv` +/// handler. Arguments never become script source, so a file name cannot +/// inject AppleScript. Returns the script's result text. +pub fn run_script(script: &str, args: &[&str], timeout: Duration) -> Result<String, ScriptRefusal> { + let mut child = Command::new("/usr/bin/osascript") + .arg("-") + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| ScriptRefusal { + code: "automation_failed", + reason: format!("osascript could not start: {error}"), + })?; + if let Some(mut stdin) = child.stdin.take() { + let _ = stdin.write_all(script.as_bytes()); + } + let deadline = Instant::now() + timeout; + loop { + match child.try_wait() { + Ok(Some(_)) => break, + Ok(None) if Instant::now() >= deadline => { + let _ = child.kill(); + let _ = child.wait(); + return Err(ScriptRefusal { + code: "automation_timeout", + reason: format!( + "the app did not answer within {} s (a macOS Automation prompt may be \ + waiting for the user)", + timeout.as_secs() + ), + }); + } + Ok(None) => std::thread::sleep(Duration::from_millis(25)), + Err(error) => { + return Err(ScriptRefusal { + code: "automation_failed", + reason: format!("osascript wait failed: {error}"), + }) + } + } + } + let output = child.wait_with_output().map_err(|error| ScriptRefusal { + code: "automation_failed", + reason: format!("osascript output could not be read: {error}"), + })?; + if output.status.success() { + return Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + Err(script_error_refusal(&stderr)) +} + +/// Map an osascript error line ("... (-1743)") to a refusal. +fn script_error_refusal(stderr: &str) -> ScriptRefusal { + if stderr.contains("(-1743)") { + ScriptRefusal { + code: "automation_denied", + reason: "macOS does not allow Cua Driver to control this app (System Settings > \ + Privacy & Security > Automation)" + .into(), + } + } else { + // osascript prints "<offset>: execution error: <message> (<code>)". + let message = stderr + .split_once("execution error: ") + .map(|(_, rest)| rest) + .unwrap_or(stderr); + ScriptRefusal { + code: "automation_failed", + reason: message.to_owned(), + } + } +} + +/// Check the permission before a route runs: refuse when the user turned it +/// off, so no event is sent. +fn ensure_permitted(bundle_id: &str, app_name: &str) -> Result<(), ScriptRefusal> { + match automation_permission(bundle_id) { + AutomationPermission::Denied => Err(ScriptRefusal { + code: "automation_denied", + reason: format!( + "macOS does not allow Cua Driver to control {app_name}. The user can allow it in \ + System Settings > Privacy & Security > Automation" + ), + }), + AutomationPermission::NotRunning => Err(ScriptRefusal { + code: "automation_failed", + reason: format!("{app_name} is not running"), + }), + // NeedsConsent: macOS shows its prompt once, on this event. + _ => Ok(()), + } +} + +const FINDER_BUNDLE_ID: &str = "com.apple.finder"; + +const FINDER_RENAME_SCRIPT: &str = r#"on run argv + set theItem to (POSIX file (item 1 of argv)) as alias + tell application id "com.apple.finder" + set name of theItem to (item 2 of argv) + end tell + return "ok" +end run +"#; + +/// Whether `name` can be a single file name (no path separators, not empty +/// or a dot entry). +pub fn valid_file_name(name: &str) -> bool { + !name.is_empty() && name != "." && name != ".." && !name.contains('/') && !name.contains('\0') +} + +/// Rename the file or folder at `path` to `new_name` through Finder, then +/// read the folder back. Returns the new path once it exists and the old +/// name is gone. +pub fn finder_rename(path: &Path, new_name: &str) -> Result<std::path::PathBuf, ScriptRefusal> { + if !valid_file_name(new_name) { + return Err(ScriptRefusal { + code: "invalid_file_name", + reason: format!("\"{new_name}\" is not a single file name"), + }); + } + let parent = path.parent().ok_or_else(|| ScriptRefusal { + code: "automation_failed", + reason: "the item has no parent folder".into(), + })?; + let target = parent.join(new_name); + let same_item = path.file_name().and_then(|n| n.to_str()) == Some(new_name); + if same_item { + return Ok(target); + } + // A name that differs only in case is the same entry on a case- + // insensitive volume; anything else must not exist yet. + let case_only = path + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|old| old.eq_ignore_ascii_case(new_name)); + if !case_only && target.exists() { + return Err(ScriptRefusal { + code: "name_taken", + reason: format!( + "\"{new_name}\" already exists in {}; nothing was renamed", + parent.display() + ), + }); + } + ensure_permitted(FINDER_BUNDLE_ID, "Finder")?; + let path_text = path.to_string_lossy(); + run_script( + FINDER_RENAME_SCRIPT, + &[&path_text, new_name], + Duration::from_secs(30), + )?; + // Read the folder back: the new name is listed and the old one is gone. + let deadline = Instant::now() + Duration::from_secs(2); + loop { + if folder_lists(parent, new_name) && (case_only || !path.exists()) { + return Ok(target); + } + if Instant::now() >= deadline { + return Err(ScriptRefusal { + code: "rename_unconfirmed", + reason: format!( + "Finder accepted the rename, but {} does not list \"{new_name}\"", + parent.display() + ), + }); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn folder_lists(folder: &Path, name: &str) -> bool { + std::fs::read_dir(folder).is_ok_and(|entries| { + entries + .flatten() + .any(|entry| entry.file_name().to_str() == Some(name)) + }) +} + +/// Save the open document whose file is `path` in the app `bundle_id` +/// through the Cocoa Standard Suite `save` command, then read its modified +/// flag back. Raw codes keep the script independent of the app's +/// terminology: `docu` document, `ppth` its path (TextEdit) or `file` its +/// file (the Standard Suite), `imod` modified, `core/save` save. Returns `false` +/// when the app reports no unsaved changes. +const SAVE_DOCUMENT_SCRIPT: &str = r#"on run argv + set thePath to item 1 of argv + tell application id (item 2 of argv) + repeat with theDocument in (every «class docu») + set theDocumentPath to missing value + try + set theDocumentPath to «property ppth» of theDocument + end try + if theDocumentPath is missing value then + try + set theDocumentPath to POSIX path of ((«property file» of theDocument) as alias) + end try + end if + if theDocumentPath is thePath then + «event coresave» theDocument + return («property imod» of theDocument) as text + end if + end repeat + end tell + return "no-document" +end run +"#; + +/// Read the modified flag of the open document whose file is `path`, the +/// same document lookup as [`SAVE_DOCUMENT_SCRIPT`] without saving. +const DOCUMENT_MODIFIED_SCRIPT: &str = r#"on run argv + set thePath to item 1 of argv + tell application id (item 2 of argv) + repeat with theDocument in (every «class docu») + set theDocumentPath to missing value + try + set theDocumentPath to «property ppth» of theDocument + end try + if theDocumentPath is missing value then + try + set theDocumentPath to POSIX path of ((«property file» of theDocument) as alias) + end try + end if + if theDocumentPath is thePath then + return («property imod» of theDocument) as text + end if + end repeat + end tell + return "no-document" +end run +"#; + +// Excel exposes neither the Standard Suite file nor modified properties. +// Resolve its workbook by name, then guard the full path before the one save. +const EXCEL_DOCUMENT_SCRIPT: &str = r#"on run argv + set thePath to item 1 of argv + set theBundle to item 2 of argv + set theName to item 3 of argv + set theAction to item 4 of argv + tell application id theBundle + considering case + if («property 1773» of («class X141» theName)) is not thePath then return "no-document" + end considering + if theAction is "save" then + «event coresave» («class X141» theName) + end if + return (not («property 1842» of («class X141» theName))) as text + end tell +end run +"#; + +fn run_document_script( + bundle_id: &str, + path: &Path, + save: bool, + timeout: Duration, +) -> Result<String, ScriptRefusal> { + let path_text = path.to_string_lossy(); + if bundle_id == "com.microsoft.Excel" { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + run_script( + EXCEL_DOCUMENT_SCRIPT, + &[ + &path_text, + bundle_id, + &name, + if save { "save" } else { "read" }, + ], + timeout, + ) + } else { + run_script( + if save { + SAVE_DOCUMENT_SCRIPT + } else { + DOCUMENT_MODIFIED_SCRIPT + }, + &[&path_text, bundle_id], + timeout, + ) + } +} + +/// Whether the app reports unsaved changes in the document at `path`. +/// Only reads when Automation is already granted, so a readback never shows +/// the consent prompt; `None` when it cannot tell. +pub fn document_modified(bundle_id: &str, path: &Path) -> Option<bool> { + if automation_permission(bundle_id) != AutomationPermission::Granted { + return None; + } + match run_document_script(bundle_id, path, false, Duration::from_secs(5)) + .ok()? + .as_str() + { + "true" => Some(true), + "false" => Some(false), + _ => None, + } +} + +/// Outcome of a background save. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SavedDocument { + /// The file's modification time moved during the save. + pub file_written: bool, +} + +pub fn save_document( + bundle_id: &str, + app_name: &str, + path: &Path, +) -> Result<SavedDocument, ScriptRefusal> { + ensure_permitted(bundle_id, app_name)?; + let before = modified_time(path); + let result = run_document_script(bundle_id, path, true, Duration::from_secs(30))?; + match result.as_str() { + "false" => {} + "true" => { + return Err(ScriptRefusal { + code: "save_unconfirmed", + reason: format!( + "{app_name} accepted the save, but still reports unsaved changes in {}", + path.display() + ), + }) + } + "no-document" => { + return Err(ScriptRefusal { + code: "automation_failed", + reason: format!("{app_name} lists no open document for {}", path.display()), + }) + } + other => { + return Err(ScriptRefusal { + code: "save_unconfirmed", + reason: format!("{app_name} answered \"{other}\" to the save"), + }) + } + } + let file_written = wait_for_new_mtime(path, before, Duration::from_millis(1500)); + Ok(SavedDocument { file_written }) +} + +pub fn modified_time(path: &Path) -> Option<std::time::SystemTime> { + std::fs::metadata(path).and_then(|m| m.modified()).ok() +} + +/// Wait up to `timeout` for `path`'s modification time to move past +/// `before`. +pub fn wait_for_new_mtime( + path: &Path, + before: Option<std::time::SystemTime>, + timeout: Duration, +) -> bool { + let deadline = Instant::now() + timeout; + loop { + let now = modified_time(path); + if now.is_some() && now != before { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// The file path of a `file://` URL string, as `AXDocument` and `AXURL` +/// report them. +pub fn file_url_to_path(url: &str) -> Option<std::path::PathBuf> { + use core_foundation::{base::TCFType, string::CFString, url::CFURL}; + if !url.starts_with("file://") { + return None; + } + let text = CFString::new(url); + let raw = unsafe { + core_foundation::url::CFURLCreateWithString( + std::ptr::null(), + text.as_concrete_TypeRef(), + std::ptr::null(), + ) + }; + if raw.is_null() { + return None; + } + let url = unsafe { CFURL::wrap_under_create_rule(raw) }; + url.to_path() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn permission_codes_map_to_states() { + assert_eq!(permission_from_status(0), AutomationPermission::Granted); + assert_eq!(permission_from_status(-1743), AutomationPermission::Denied); + assert_eq!( + permission_from_status(-1744), + AutomationPermission::NeedsConsent + ); + assert_eq!( + permission_from_status(-600), + AutomationPermission::NotRunning + ); + assert_eq!( + permission_from_status(-50), + AutomationPermission::Unknown(-50) + ); + } + + #[test] + fn script_errors_name_a_denied_permission() { + let denied = script_error_refusal( + "0:120: execution error: Not authorized to send Apple events to Finder. (-1743)", + ); + assert_eq!(denied.code, "automation_denied"); + let other = script_error_refusal( + "0:98: execution error: Finder got an error: The name \"b.txt\" is already taken. (-48)", + ); + assert_eq!(other.code, "automation_failed"); + assert!(other.reason.starts_with("Finder got an error")); + } + + #[test] + fn file_names_are_single_path_components() { + assert!(valid_file_name("renamed.txt")); + assert!(valid_file_name("with space é.md")); + for bad in ["", ".", "..", "a/b", "x\0y"] { + assert!(!valid_file_name(bad), "{bad:?}"); + } + } + + #[test] + fn file_urls_decode_to_paths() { + assert_eq!( + file_url_to_path("file:///Users/lume/xfb/te/a%20main.txt"), + Some("/Users/lume/xfb/te/a main.txt".into()) + ); + assert_eq!(file_url_to_path("https://example.com/a"), None); + } + + #[test] + fn a_rename_onto_an_existing_name_is_refused_before_any_event() { + let dir = std::env::temp_dir().join(format!("cua-ae-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("a.txt"), "a").unwrap(); + std::fs::write(dir.join("b.txt"), "b").unwrap(); + let refused = finder_rename(&dir.join("a.txt"), "b.txt").unwrap_err(); + assert_eq!(refused.code, "name_taken"); + assert_eq!( + finder_rename(&dir.join("a.txt"), "x/y").unwrap_err().code, + "invalid_file_name" + ); + // Renaming to the current name is already done; nothing is sent. + assert_eq!( + finder_rename(&dir.join("a.txt"), "a.txt").unwrap(), + dir.join("a.txt") + ); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn scripts_pass_values_as_arguments_only() { + // Values reach the scripts through argv, never through the source. + assert!(FINDER_RENAME_SCRIPT.contains("item 1 of argv")); + assert!(SAVE_DOCUMENT_SCRIPT.contains("item 2 of argv")); + assert!(EXCEL_DOCUMENT_SCRIPT.contains("item 3 of argv")); + assert!(!FINDER_RENAME_SCRIPT.contains('{')); + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/apps/mod.rs b/packages/cua-driver/rust/crates/platform-macos/src/apps/mod.rs index c254d549f7b..0989daa4f83 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/apps/mod.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/apps/mod.rs @@ -42,13 +42,14 @@ pub struct AppInfo { /// Query process membership directly, then read metadata through AppKit. This /// does not require System Events or a host-driven AppKit main run loop. pub fn list_running_apps() -> Vec<AppInfo> { - list_running_apps_native() + list_running_apps_native().0 } -fn list_running_apps_native() -> Vec<AppInfo> { +fn list_running_apps_native() -> (Vec<AppInfo>, Vec<AppInfo>) { use objc2_app_kit::{NSApplicationActivationPolicy, NSRunningApplication}; let mut apps = Vec::new(); + let mut accessory = Vec::new(); let active_pid = frontmost_pid(); unsafe { for pid in process_manager::running_pids() { @@ -56,7 +57,7 @@ fn list_running_apps_native() -> Vec<AppInfo> { else { continue; }; - if app.activationPolicy() != NSApplicationActivationPolicy::Regular { + if app.isTerminated() { continue; } let Some(name) = app.localizedName().map(|value| value.to_string()) else { @@ -65,7 +66,7 @@ fn list_running_apps_native() -> Vec<AppInfo> { if name.is_empty() || pid <= 0 { continue; } - apps.push(AppInfo { + let info = AppInfo { name, pid, bundle_id: app.bundleIdentifier().map(|value| value.to_string()), @@ -77,10 +78,15 @@ fn list_running_apps_native() -> Vec<AppInfo> { .map(|path| canonical_app_path(&path.to_string())), kind: Some("desktop".to_owned()), last_used: None, - }); + }; + if app.activationPolicy() == NSApplicationActivationPolicy::Regular { + apps.push(info); + } else { + accessory.push(info); + } } } - apps + (apps, accessory) } /// Launch an app by bundle ID via NSWorkspace, background only (no focus @@ -437,12 +443,25 @@ fn bundle_id_for_app_path(app_path: &str) -> Option<String> { /// * `launch_path` (filesystem `.app` path when known, else `None`), /// * `kind` (`"desktop"` on macOS). pub fn list_all_apps() -> Vec<AppInfo> { - let mut running = list_running_apps(); + let (running, accessory) = list_running_apps_native(); let installed = scan_installed_apps(); + merge_app_lists(running, accessory, installed) +} + +fn merge_app_lists( + mut running: Vec<AppInfo>, + accessory: Vec<AppInfo>, + installed: Vec<AppInfo>, +) -> Vec<AppInfo> { let installed_by_path: std::collections::HashMap<_, _> = installed .iter() .filter_map(|app| app.launch_path.as_ref().map(|path| (path, app))) .collect(); + running.extend(accessory.into_iter().filter(|app| { + app.launch_path + .as_ref() + .is_some_and(|path| installed_by_path.contains_key(path)) + })); for app in running.iter_mut() { if let Some(installed) = app .launch_path @@ -616,25 +635,50 @@ pub fn activate_pid(pid: i32) -> bool { } } -pub fn with_app_observation<R>(pid: i32, body: impl FnOnce() -> R) -> R { - let prior = frontmost_pid(); - let needs_reopen = crate::ax::bindings::app_window_id_of_pid(pid).is_none(); +pub fn with_app_observation<R>( + pid: i32, + readiness_deadline: Option<std::time::Instant>, + body: impl FnOnce() -> R, +) -> R { + let mut attempt_started = std::time::Instant::now(); + let needs_reopen = + crate::ax::bindings::app_window_id_of_pid_with_readiness(pid, readiness_deadline).is_none(); if needs_reopen { - if let Some(app) = list_running_apps().into_iter().find(|app| app.pid == pid) { + let launch_deadline = crate::ax::launch::lookup_deadline(readiness_deadline); + if launch_deadline.is_some() && crate::ax::launch::is_still_launching(pid) { + while let Some(pause) = + crate::ax::launch::retry_pause(pid, launch_deadline, attempt_started) + { + std::thread::sleep(pause); + if launch_deadline.is_some_and(|limit| std::time::Instant::now() >= limit) { + break; + } + attempt_started = std::time::Instant::now(); + if crate::ax::bindings::app_window_id_of_pid_with_readiness(pid, readiness_deadline) + .is_some() + { + break; + } + } + } else if let Some(app) = list_running_apps().into_iter().find(|app| app.pid == pid) { let app_ref = app .bundle_id .clone() .or(app.launch_path) .unwrap_or(app.name); let config = nsworkspace::OpenConfig { - activates: true, + activates: false, reopens_running_application: true, apple_event_bundle_id: app.bundle_id, ..Default::default() }; if nsworkspace::open_application(&app_ref, &config).is_ok() { let deadline = std::time::Instant::now() + Duration::from_millis(400); - while crate::ax::bindings::app_window_id_of_pid(pid).is_none() + while crate::ax::bindings::app_window_id_of_pid_with_readiness( + pid, + readiness_deadline, + ) + .is_none() && std::time::Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); @@ -643,11 +687,6 @@ pub fn with_app_observation<R>(pid: i32, body: impl FnOnce() -> R) -> R { } } let result = body(); - if needs_reopen { - if let Some(prior) = prior.filter(|prior| *prior != pid) { - let _ = activate_pid(prior); - } - } result } @@ -717,7 +756,46 @@ pub fn format_app_list(apps: &[AppInfo]) -> String { #[cfg(test)] mod tests { - use super::{finder_folder_handoff, read_app_plist, unix_secs_to_rfc3339}; + use super::{ + finder_folder_handoff, merge_app_lists, read_app_plist, unix_secs_to_rfc3339, AppInfo, + }; + + #[test] + fn installed_accessories_keep_all_live_pids_without_matching_other_bundle_copies() { + let app = |pid: i32, path: &str| AppInfo { + name: "Fixture".to_owned(), + pid, + bundle_id: Some("test.shared.bundle".to_owned()), + running: pid > 0, + active: pid == 42, + launch_path: Some(path.to_owned()), + kind: Some("desktop".to_owned()), + last_used: (pid == 0).then(|| "2026-10-18T00:00:00Z".to_owned()), + }; + let merged = merge_app_lists( + vec![app(31, "/Apps/Editor.app")], + vec![ + app(41, "/Apps/Proxy.app"), + app(42, "/Apps/Proxy.app"), + app(9, "/Uninstalled/Helper.app"), + ], + vec![ + app(0, "/Apps/Editor.app"), + app(0, "/Apps/Proxy.app"), + app(0, "/Apps/Copy.app"), + ], + ); + assert_eq!(merged.len(), 4); + assert_eq!( + merged.iter().map(|app| app.pid).collect::<Vec<_>>(), + [31, 41, 42, 0] + ); + assert!(merged[..3].iter().all(|app| app.running)); + assert!(merged.iter().all(|app| app.last_used.is_some())); + assert!(merged[2].active); + assert!(!merged[3].running); + assert_eq!(merged[3].launch_path.as_deref(), Some("/Apps/Copy.app")); + } #[test] fn plist_reader_supports_xml_binary_fallbacks_and_invalid_metadata() { diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/app_text.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/app_text.rs index 9ceca148eb2..f9cd3554fc4 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/ax/app_text.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/app_text.rs @@ -91,7 +91,7 @@ pub(super) unsafe fn associated_title( Some((super::tree::AXIdentity::retained(element), title)) } -pub(super) unsafe fn element_url(element: AXUIElementRef) -> Option<String> { +pub(crate) unsafe fn element_url(element: AXUIElementRef) -> Option<String> { let value = copy_attribute(element, "AXURL")?; url(value.as_CFTypeRef()) } diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs index 95559d0d6d7..9b3befa0a4e 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs @@ -13,9 +13,11 @@ use core_foundation::{ array::CFArrayRef, base::{CFRelease, CFRetain, CFTypeID, CFTypeRef}, + data::{CFData, CFDataRef}, string::CFStringRef, }; use std::os::raw::{c_int, c_void}; +use std::time::{Duration, Instant}; // ── AXUIElement opaque type ────────────────────────────────────────────────── @@ -705,13 +707,21 @@ pub fn focused_window_id_of_pid(pid: i32) -> Option<u32> { } pub fn app_window_id_of_pid(pid: i32) -> Option<u32> { + app_window_id_of_pid_with_readiness(pid, None) +} + +pub(crate) fn app_window_id_of_pid_with_readiness( + pid: i32, + readiness_deadline: Option<std::time::Instant>, +) -> Option<u32> { unsafe { let app = AXUIElementCreateApplication(pid); if app.is_null() { return None; } let _ = AXUIElementSetMessagingTimeout(app, 2.0); - super::enablement::ensure_chromium_ax_enabled(pid, app); + super::enablement::ensure_chromium_ax_enabled(pid, app, readiness_deadline); + let _ = AXUIElementSetMessagingTimeout(app, 2.0); for attribute in ["AXFocusedWindow", "AXMainWindow"] { let selection = copy_element_attr_with_status(app, attribute); if let Some(window) = selection.value { @@ -747,19 +757,125 @@ pub fn app_window_id_of_pid(pid: i32) -> Option<u32> { } unsafe fn app_target_window_id(window: AXUIElementRef) -> Option<u32> { + app_target_window_id_with_proven_root(window, None) +} + +pub(crate) unsafe fn app_target_window_id_with_proven_root( + window: AXUIElementRef, + proven_root: Option<(i32, u32)>, +) -> Option<u32> { let _ = AXUIElementSetMessagingTimeout(window, 2.0); let (sheets, complete) = super::sheets::copy_attached_sheets(&[window]); if !complete || sheets.len() > 1 { return None; } - if let Some(id) = sheets.first().and_then(|sheet| sheet.window_id) { - if crate::windows::window_info_by_id(id).is_some() { - return Some(id); + if let Some(sheet) = sheets.first() { + if let Some(id) = sheet.window_id { + if let Some(info) = crate::windows::window_info_by_id(id) { + if proven_root.is_none_or(|(pid, _)| info.pid == pid) { + return Some(id); + } + } + } + if proven_root.is_some() { + return None; } } + if let Some((pid, id)) = proven_root { + return crate::windows::window_info_by_id(id) + .filter(|info| info.pid == pid) + .map(|_| id); + } ax_get_window_id(window).filter(|id| crate::windows::window_info_by_id(*id).is_some()) } +type CreateWithRemoteToken = unsafe extern "C" fn(CFDataRef) -> AXUIElementRef; + +fn remote_token_creator() -> Option<CreateWithRemoteToken> { + static CREATOR: std::sync::OnceLock<Option<CreateWithRemoteToken>> = std::sync::OnceLock::new(); + *CREATOR.get_or_init(|| unsafe { + let symbol = libc::dlsym( + libc::RTLD_DEFAULT, + c"_AXUIElementCreateWithRemoteToken".as_ptr(), + ); + (!symbol.is_null()) + .then(|| std::mem::transmute::<*mut c_void, CreateWithRemoteToken>(symbol)) + }) +} + +fn remote_window_token(pid: i32, element_id: u64) -> [u8; 20] { + let mut token = [0; 20]; + token[..4].copy_from_slice(&pid.to_ne_bytes()); + token[8..12].copy_from_slice(&0x636f636fi32.to_ne_bytes()); + token[12..].copy_from_slice(&element_id.to_ne_bytes()); + token +} + +unsafe fn set_remote_probe_timeout(element: AXUIElementRef, deadline: Instant) -> bool { + let remaining = deadline.saturating_duration_since(Instant::now()); + !remaining.is_zero() + && AXUIElementSetMessagingTimeout( + element, + remaining.min(Duration::from_millis(50)).as_secs_f32(), + ) == kAXErrorSuccess +} + +/// Recover one omitted AXWindow with exact process and SPI window identity. +/// The caller owns the returned retain and must restore its normal AX timeout. +pub(crate) unsafe fn copy_remote_window_for_id( + pid: i32, + window_id: u32, + caller_deadline: Option<Instant>, +) -> Option<AXUIElementRef> { + let probe_deadline = Instant::now() + Duration::from_millis(300); + let deadline = caller_deadline.map_or(probe_deadline, |limit| limit.min(probe_deadline)); + if Instant::now() >= deadline || pid <= 0 || window_id == 0 { + return None; + } + let info = crate::windows::window_info_by_id(window_id)?; + if info.pid != pid || info.layer != 0 { + return None; + } + match crate::windows::window_on_current_space_by_id(window_id) { + Some(false) => {} + Some(true) if !info.is_on_screen => {} + _ => return None, + } + let create = remote_token_creator()?; + for element_id in 0..2000 { + if Instant::now() >= deadline { + break; + } + let token = CFData::from_buffer(&remote_window_token(pid, element_id)); + let element = create(token.as_concrete_TypeRef()); + if element.is_null() { + continue; + } + let mut owner = 0; + let mut mapped_window = 0; + let exact = core_foundation::base::CFGetTypeID(element as CFTypeRef) + == AXUIElementGetTypeID() + && set_remote_probe_timeout(element, deadline) + && copy_string_attr(element, "AXRole").as_deref() == Some("AXWindow") + && set_remote_probe_timeout(element, deadline) + && AXUIElementGetPid(element, &mut owner) == kAXErrorSuccess + && owner == pid + && set_remote_probe_timeout(element, deadline) + && _AXUIElementGetWindow(element, &mut mapped_window) == kAXErrorSuccess + && mapped_window == window_id + && Instant::now() < deadline; + if exact + && crate::windows::window_info_by_id(window_id) + .is_some_and(|current| current.pid == pid && current.layer == 0) + && Instant::now() < deadline + { + return Some(element); + } + CFRelease(element as CFTypeRef); + } + None +} + /// The returned menu-bar item owns its retain. Only an explicitly selected /// item with a visible submenu establishes menu context; cached children of a /// closed menu do not. @@ -1082,14 +1198,19 @@ pub unsafe fn set_bool_attr_true(element: AXUIElementRef, attr_name: &str) -> AX AXUIElementSetAttributeValue(element, attr.as_concrete_TypeRef(), cf_true.as_CFTypeRef()) } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AccessibilityOptIn { + ManualAccessibility, + EnhancedUserInterface, + NotAccepted, +} + /// Signal to a Chromium/Electron application root that a real assistive client /// is present so it materializes its full web-content accessibility tree. /// -/// Returns `true` when an attribute write was accepted — meaning the app was -/// flipped from "tree off" to "tree building" and the caller should let the -/// tree settle before walking. Returns `false` when the app does not support -/// either attribute (native Cocoa apps such as Finder / Calculator / TextEdit), -/// in which case no settle delay is warranted. +/// Distinguishes the modern Chromium opt-in from the legacy fallback. An +/// accepted write does not prove web-content readiness; callers decide whether +/// their observation budget permits waiting. Native apps commonly reject both. /// /// `AXManualAccessibility` is the modern opt-in with no screen-reader side /// effects; `AXEnhancedUserInterface` is the legacy fallback some Electron @@ -1099,18 +1220,24 @@ pub unsafe fn set_bool_attr_true(element: AXUIElementRef, attr_name: &str) -> AX /// # Safety /// /// `app_element` must be a valid, live application `AXUIElementRef`. -pub unsafe fn enable_chromium_accessibility(app_element: AXUIElementRef) -> bool { +pub(crate) unsafe fn enable_chromium_accessibility( + app_element: AXUIElementRef, +) -> AccessibilityOptIn { let manual = set_bool_attr_true(app_element, "AXManualAccessibility"); if manual == kAXErrorSuccess { - return true; + return AccessibilityOptIn::ManualAccessibility; } if manual != kAXErrorAttributeUnsupported { // A transient error (e.g. timeout / app busy) rather than a hard // "this app has no such attribute" — don't bother with the legacy // fallback, and don't claim enablement happened. - return false; + return AccessibilityOptIn::NotAccepted; + } + if set_bool_attr_true(app_element, "AXEnhancedUserInterface") == kAXErrorSuccess { + AccessibilityOptIn::EnhancedUserInterface + } else { + AccessibilityOptIn::NotAccepted } - set_bool_attr_true(app_element, "AXEnhancedUserInterface") == kAXErrorSuccess } /// Get the CGWindowID of an AX window element. @@ -1164,6 +1291,25 @@ mod tests { use super::*; use core_foundation::{boolean::CFBoolean, number::CFNumber}; + #[test] + fn remote_window_token_matches_the_native_unpadded_spi_layout() { + let token = remote_window_token(0x12345678, 0x0123456789abcdef); + assert_eq!(token.len(), 20); + assert_eq!( + i32::from_ne_bytes(token[0..4].try_into().unwrap()), + 0x12345678 + ); + assert_eq!(token[4..8], [0, 0, 0, 0]); + assert_eq!( + i32::from_ne_bytes(token[8..12].try_into().unwrap()), + 0x636f636f + ); + assert_eq!( + u64::from_ne_bytes(token[12..20].try_into().unwrap()), + 0x0123456789abcdef + ); + } + #[test] fn stringish_value_coerces_cfstring_cfnumber_and_cfboolean() { let string = CFStr::new("Search"); diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/cache.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/cache.rs index 68b535b7522..9934c0acecc 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/ax/cache.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/cache.rs @@ -1,35 +1,15 @@ -//! Per-(pid, window_id) element cache. -//! -//! After `get_window_state`, each actionable element's AXUIElementRef pointer -//! is cached by element_index. Subsequent `click`, `type_text`, etc. look up -//! the element_index to get the raw pointer and perform AX actions on it. -//! -//! Cache is scoped per (pid, window_id) — a new `get_window_state` call -//! for the same (pid, window_id) replaces the entire entry. -//! -//! Memory contract: -//! tree::walk_element retains each actionable element before storing its ptr. -//! CachedSnapshot::drop releases those retains so we have no AX leaks. -//! -//! The locked-HashMap plumbing lives in `cua_driver_core::element_cache` — see -//! `docs/dedup-audit.md` item #3. This module owns the macOS-specific -//! `CacheKey`, `CachedSnapshot`, and the `Drop` impl that fires `CFRelease` -//! when an entry is replaced or removed. +//! Native AX snapshots owned by element-token bindings. The per-target directory +//! holds weak references; only current tokens and resolved actions keep payloads +//! alive. Walked pointers transfer their extra CFRetain into CachedSnapshot. use super::bindings::AXUIElementRef; use super::tree::AXNode; use core_foundation::base::{CFRelease, CFRetain, CFTypeRef}; use cua_driver_core::element_cache::ElementCacheCore; +use cua_driver_core::element_token::{ResolvedElement, SnapshotBinding}; +use std::sync::{Arc, Weak}; -/// An AXUIElementRef borrowed out of the cache with an extra `CFRetain`, so it -/// stays alive for the duration of an AX action even if a concurrent -/// `get_window_state` (→ [`ElementCache::update`]) replaces and drops the -/// snapshot it came from. Without this, the snapshot's `Drop` could `CFRelease` -/// the element to zero while an in-flight click was still dereferencing the raw -/// pointer — a use-after-free that trips `AXUIElementCopyActionNames` → -/// `CFGetTypeID` (`EXC_BREAKPOINT`) and crashes the daemon. The retain is taken -/// under the cache lock (see [`ElementCache::get_element_retained`]); the -/// matching `CFRelease` fires on drop. +/// An extra retain for an action that can outlive its resolved snapshot view. pub struct RetainedElement(usize); impl RetainedElement { @@ -78,7 +58,7 @@ impl Drop for CachedSnapshot { /// Global element cache. pub struct ElementCache { - core: ElementCacheCore<CacheKey, CachedSnapshot>, + core: ElementCacheCore<CacheKey, Weak<CachedSnapshot>>, } impl ElementCache { @@ -89,51 +69,101 @@ impl ElementCache { } /// Replace the snapshot for (pid, window_id) with the nodes from a fresh walk. - pub fn update(&self, pid: i32, window_id: u32, nodes: &[AXNode]) { + pub fn update(&self, pid: i32, window_id: u32, nodes: &[AXNode]) -> u32 { + self.publish(pid, window_id, Self::prepare(nodes)) + } + + pub fn prepare(nodes: &[AXNode]) -> Arc<CachedSnapshot> { let elements: Vec<usize> = nodes .iter() .filter(|n| n.element_index.is_some()) .map(|n| n.element_ptr) .collect(); - self.core - .insert(CacheKey { pid, window_id }, CachedSnapshot { elements }); + Arc::new(CachedSnapshot { elements }) } - /// Look up + `CFRetain` the element for `element_index` in (pid, window_id), - /// returning a guard that releases on drop. The retain happens **under the - /// cache lock**, so a concurrent [`update`](Self::update) (which replaces - /// the snapshot and drops its retains) cannot free the element between the - /// lookup and the retain. Hold the returned guard for the entire AX action — - /// this is what makes element actions safe when two sessions drive the same - /// `(pid, window_id)`. Returns `None` if the index isn't cached. + pub fn publish(&self, pid: i32, window_id: u32, snapshot: Arc<CachedSnapshot>) -> u32 { + self.publish_for_session(pid, window_id, snapshot, None) + } + + pub fn publish_for_session( + &self, + pid: i32, + window_id: u32, + snapshot: Arc<CachedSnapshot>, + session: Option<&str>, + ) -> u32 { + let id = cua_driver_core::element_token::global().register_snapshot_payload_for_session( + pid, + window_id, + 0..snapshot.elements.len(), + snapshot.clone(), + session, + ); + self.core + .insert(CacheKey { pid, window_id }, Arc::downgrade(&snapshot)); + id + } + + /// Number of indexed elements for (pid, window_id), or 0 if not cached. + pub fn element_count(&self, pid: i32, window_id: u32) -> usize { + self.core + .with_snapshot(&CacheKey { pid, window_id }, |s| { + s.upgrade().map_or(0, |s| s.elements.len()) + }) + .unwrap_or(0) + } + + pub fn clear_target(&self, pid: i32, window_id: u32) { + cua_driver_core::element_token::global().clear_current_target(pid, window_id); + self.core.remove(&CacheKey { pid, window_id }); + } +} + +#[derive(Clone)] +pub struct BoundElementCache { + binding: Option<SnapshotBinding>, +} + +impl BoundElementCache { + pub fn from_resolved(resolved: &ResolvedElement) -> Self { + Self { + binding: resolved.binding(), + } + } + + pub fn from_binding(binding: SnapshotBinding) -> Self { + Self { + binding: Some(binding), + } + } + + fn with_snapshot<R>( + &self, + key: &CacheKey, + read: impl FnOnce(&CachedSnapshot) -> R, + ) -> Option<R> { + self.binding + .as_ref()? + .with_payload(key.pid as i32, key.window_id as u32, read) + } + + /// Retain from this immutable snapshot, never from a newer window snapshot. pub fn get_element_retained( &self, pid: i32, window_id: u32, element_index: usize, ) -> Option<RetainedElement> { - self.core - .with_snapshot(&CacheKey { pid, window_id }, |s| { - let ptr = s.elements.get(element_index).copied()?; - if ptr != 0 { - // Safety: still inside `with_snapshot`'s lock, so the - // snapshot (and thus this CFTypeRef) is alive right now. - unsafe { CFRetain(ptr as AXUIElementRef as CFTypeRef) }; - } - Some(RetainedElement(ptr)) - }) - .flatten() - } - - /// Number of indexed elements for (pid, window_id), or 0 if not cached. - pub fn element_count(&self, pid: i32, window_id: u32) -> usize { - self.core - .with_snapshot(&CacheKey { pid, window_id }, |s| s.elements.len()) - .unwrap_or(0) - } - - pub fn clear_target(&self, pid: i32, window_id: u32) { - self.core.remove(&CacheKey { pid, window_id }); + self.with_snapshot(&CacheKey { pid, window_id }, |s| { + let ptr = s.elements.get(element_index).copied()?; + if ptr != 0 { + // The binding owns the snapshot throughout this retain. + unsafe { CFRetain(ptr as AXUIElementRef as CFTypeRef) }; + } + Some(RetainedElement(ptr)) + }) + .flatten() } } @@ -188,8 +218,8 @@ mod tests { /// a concurrent `get_window_state` replaces the snapshot and its `Drop` /// `CFRelease`s the element to zero — freeing it under the in-flight click /// (use-after-free → `EXC_BREAKPOINT` in `AXUIElementCopyActionNames`). - /// `get_element_retained` takes an extra retain under the lock so the - /// element stays alive across the replace. This asserts that accounting. + /// `get_element_retained` takes an extra retain from the bound snapshot so + /// the element outlives both snapshot replacement and the binding itself. #[test] fn retained_element_survives_concurrent_snapshot_replace() { let s = CFString::new("cua-driver-uaf-test-element-placeholder"); @@ -200,7 +230,7 @@ mod tests { // to the cache, and CachedSnapshot::drop releases that retain. unsafe { CFRetain(ptr as CFTypeRef) }; let cache = ElementCache::new(); - cache.update(1, 2, &[node_with_ptr(ptr)]); + let sid = cache.update(721_500, 2, &[node_with_ptr(ptr)]); assert_eq!( unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1, @@ -208,8 +238,12 @@ mod tests { ); // Borrow the element out for an action. - let guard = cache - .get_element_retained(1, 2, 0) + let (_, _, binding) = cua_driver_core::element_token::global() + .resolve_bound(721_500, &cua_driver_core::element_token::token_for(sid, 0)) + .unwrap(); + let bound = BoundElementCache::from_binding(binding.unwrap()); + let guard = bound + .get_element_retained(721_500, 2, 0) .expect("element is cached"); assert_eq!( unsafe { CFGetRetainCount(ptr as CFTypeRef) }, @@ -219,7 +253,8 @@ mod tests { // Concurrent get_window_state replaces the snapshot → old one dropped → // CFRelease of the cache's retain. The guard's retain must remain. - cache.update(1, 2, &[]); + drop(bound); + cache.update(721_500, 2, &[]); assert_eq!( unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1, @@ -235,12 +270,73 @@ mod tests { ); } - /// A missing index returns None without retaining anything. + #[test] + fn prepared_and_resolved_snapshots_keep_their_native_identity() { + let first = CFString::new("first-native-snapshot-object-with-a-heap-allocation"); + let second = CFString::new("second-native-snapshot-object-with-a-heap-allocation"); + let a = first.as_concrete_TypeRef() as usize; + let b = second.as_concrete_TypeRef() as usize; + let baseline = unsafe { CFGetRetainCount(a as CFTypeRef) }; + let pid = 721_501; + let window = 721_502; + unsafe { + CFRetain(a as CFTypeRef); + CFRetain(b as CFTypeRef); + } + let pending = ElementCache::prepare(&[node_with_ptr(a)]); + let cache = ElementCache::new(); + let sid_b = cache.update(pid, window, &[node_with_ptr(b)]); + let sid_a = cache.publish(pid, window, pending); + let registry = cua_driver_core::element_token::global(); + assert!(registry + .resolve(pid, &cua_driver_core::element_token::token_for(sid_b, 0)) + .is_err()); + let (_, _, binding) = registry + .resolve_bound(pid, &cua_driver_core::element_token::token_for(sid_a, 0)) + .unwrap(); + let bound = BoundElementCache::from_binding(binding.unwrap()); + cache.update(pid, window, &[]); + let element = bound.get_element_retained(pid, window, 0).unwrap(); + assert_eq!(element.as_ptr(), a); + assert!(bound.get_element_retained(pid, window + 1, 0).is_none()); + registry.clear_target(pid, window); + drop(element); + assert_eq!(unsafe { CFGetRetainCount(a as CFTypeRef) }, baseline + 1); + drop(bound); + assert_eq!(unsafe { CFGetRetainCount(a as CFTypeRef) }, baseline); + + unsafe { + CFRetain(a as CFTypeRef); + } + let cancelled = ElementCache::prepare(&[node_with_ptr(a)]); + drop(cancelled); + assert_eq!(unsafe { CFGetRetainCount(a as CFTypeRef) }, baseline); + } + + #[test] + fn clearing_unresolved_target_revokes_token_and_releases_native_payload() { + let value = CFString::new("unresolved-window-owned-native-object-heap-allocation"); + let ptr = value.as_concrete_TypeRef() as usize; + let baseline = unsafe { CFGetRetainCount(ptr as CFTypeRef) }; + let pid = 721_504; + let cache = ElementCache::new(); + unsafe { CFRetain(ptr as CFTypeRef) }; + let sid = cache.update(pid, 7, &[node_with_ptr(ptr)]); + let registry = cua_driver_core::element_token::global(); + let token = cua_driver_core::element_token::token_for(sid, 0); + assert!(registry.resolve(pid, &token).is_ok()); + cache.clear_target(pid, 7); + assert!(registry.resolve(pid, &token).is_err()); + assert_eq!(cache.element_count(pid, 7), 0); + assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, baseline); + } + #[test] fn missing_index_returns_none() { let cache = ElementCache::new(); - assert!(cache.get_element_retained(1, 2, 0).is_none()); - cache.update(1, 2, &[]); - assert!(cache.get_element_retained(1, 2, 5).is_none()); + let bound = BoundElementCache::from_resolved(&ResolvedElement::None); + assert!(bound.get_element_retained(721_503, 2, 0).is_none()); + cache.update(721_503, 2, &[]); + assert!(bound.get_element_retained(721_503, 2, 5).is_none()); } } diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs index 18a5df6c7bc..6512babe27a 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs @@ -1,31 +1,37 @@ -//! Process-lifetime Chromium/Electron accessibility enablement. -//! -//! Chromium-family apps (Arc, VS Code, Electron shells) ship their web-content -//! AX tree OFF and only build it once an assistive client asks for it. The -//! walker flips `AXManualAccessibility` (falling back to -//! `AXEnhancedUserInterface` only when the modern attribute is unsupported — -//! see [`super::bindings::enable_chromium_accessibility`]) and pays a one-time -//! settle so the asynchronously-built tree exists before it is read. -//! -//! The "already enabled" cache is keyed by the observed process lifetime -//! (pid + kernel start time), not by the numeric pid alone: pids are recycled, -//! and a relaunched Electron app must not inherit a stale "enabled" decision -//! that would skip enablement and return an empty web-content tree. +//! Chromium opt-in and readiness are distinct: accepted ManualAccessibility +//! may still expose only a shell. Only explicit AX observations wait for web +//! content, using a caller-owned deadline shared with application selection. use std::collections::HashMap; use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; -use super::bindings::{enable_chromium_accessibility, AXUIElementRef}; - -/// How long to let a freshly-enabled Chromium/Electron app build its -/// web-content AX tree before we read it. Paid at most once per process -/// lifetime. -const CHROMIUM_SETTLE_SECONDS: f64 = 0.5; +use super::bindings::{ + copy_children_with_status, copy_string_attr_with_status, enable_chromium_accessibility, + AXUIElementRef, AXUIElementSetMessagingTimeout, AccessibilityOptIn, +}; +use core_foundation::base::{CFRelease, CFTypeRef}; type ProcessStartStamp = (u64, u64); +const RETRY_BACKOFF: Duration = Duration::from_secs(30); +const MAX_WAIT_ATTEMPTS: u32 = 3; +const POLL_INTERVAL: Duration = Duration::from_millis(100); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ChromiumReadiness { + NotApplicable, + Ready, + Pending(Option<ProcessStartStamp>), +} + +#[derive(Clone, Copy, Debug)] +struct ProcessEnablement { + stamp: ProcessStartStamp, + ready: bool, + attempts: u32, + attempted_at: Option<Instant>, +} -/// Kernel start time of a process: `(pbi_start_tvsec, pbi_start_tvusec)`. -/// `None` when the process is gone or proc info is unreadable. pub(crate) fn process_start_stamp(pid: i32) -> Option<ProcessStartStamp> { // SAFETY: proc_pidinfo writes at most `size` bytes into `info` and returns // the number of bytes filled (<= size) or <= 0 on failure. @@ -46,41 +52,175 @@ pub(crate) fn process_start_stamp(pid: i32) -> Option<ProcessStartStamp> { } } -/// Process lifetimes for which enablement has already run and the settle has -/// been paid. Values are the process start stamp observed at enablement time. -fn enabled_processes() -> &'static Mutex<HashMap<i32, Option<ProcessStartStamp>>> { - static ENABLED: OnceLock<Mutex<HashMap<i32, Option<ProcessStartStamp>>>> = OnceLock::new(); +fn enabled_processes() -> &'static Mutex<HashMap<i32, ProcessEnablement>> { + static ENABLED: OnceLock<Mutex<HashMap<i32, ProcessEnablement>>> = OnceLock::new(); ENABLED.get_or_init(|| Mutex::new(HashMap::new())) } fn cached_lifetime_is_current( - cached: Option<&Option<ProcessStartStamp>>, + cached: &ProcessEnablement, observed: Option<ProcessStartStamp>, ) -> bool { - cached.is_some_and(|cached| cached.is_some() && *cached == observed) + observed == Some(cached.stamp) } -/// Flip Chromium/Electron accessibility on for `pid`'s application element and -/// wait for the web-content tree to settle — once per observed process -/// lifetime. Native Cocoa apps reject the attribute and pay no settle cost. -/// -/// # Safety -/// -/// `app_element` must be a valid application `AXUIElementRef` for `pid`. -pub unsafe fn ensure_chromium_ax_enabled(pid: i32, app_element: AXUIElementRef) { - let stamp = process_start_stamp(pid); - let already_enabled = enabled_processes() - .lock() - .map(|cache| cached_lifetime_is_current(cache.get(&pid), stamp)) - .unwrap_or(false); - if already_enabled { - return; +fn should_wait(cached: Option<ProcessEnablement>, now: Instant) -> bool { + cached.is_none_or(|cached| { + !cached.ready + && cached.attempts < MAX_WAIT_ATTEMPTS + && cached + .attempted_at + .is_none_or(|at| now.saturating_duration_since(at) >= RETRY_BACKOFF) + }) +} + +fn merge_state( + cached: Option<ProcessEnablement>, + stamp: ProcessStartStamp, + ready: bool, + attempted_at: Option<Instant>, +) -> ProcessEnablement { + let cached = cached.filter(|cached| cached.stamp == stamp); + ProcessEnablement { + stamp, + ready: ready || cached.is_some_and(|cached| cached.ready), + attempts: cached.map_or(0, |cached| cached.attempts) + u32::from(attempted_at.is_some()), + attempted_at: attempted_at.or(cached.and_then(|cached| cached.attempted_at)), } - if enable_chromium_accessibility(app_element) { - crate::permissions::panel::pump_run_loop_briefly(CHROMIUM_SETTLE_SECONDS); - if let Ok(mut cache) = enabled_processes().lock() { - cache.insert(pid, stamp); +} + +fn remember( + pid: i32, + stamp: Option<ProcessStartStamp>, + ready: bool, + attempted_at: Option<Instant>, +) { + let Some(stamp) = stamp else { return }; + if let Ok(mut cache) = enabled_processes().lock() { + // Serialize the identity check with publication: an old completion + // must not overwrite a newer lifetime that was already cached. + if process_start_stamp(pid) != Some(stamp) { + return; } + let state = merge_state(cache.get(&pid).copied(), stamp, ready, attempted_at); + cache.insert(pid, state); + } +} + +// Every borrowed root and retained child gets its own remaining AX timeout. +unsafe fn probe_web_area( + element: AXUIElementRef, + depth: usize, + visits: &mut usize, + deadline: Instant, +) -> bool { + if depth == 0 || *visits == 0 || Instant::now() >= deadline { + return false; + } + *visits -= 1; + let remaining = deadline + .saturating_duration_since(Instant::now()) + .as_secs_f32(); + let _ = AXUIElementSetMessagingTimeout(element, remaining.min(0.1).max(0.001)); + if copy_string_attr_with_status(element, "AXRole") + .value + .as_deref() + == Some("AXWebArea") + { + return true; + } + if Instant::now() >= deadline { + return false; + } + let remaining = deadline + .saturating_duration_since(Instant::now()) + .as_secs_f32(); + let _ = AXUIElementSetMessagingTimeout(element, remaining.min(0.1).max(0.001)); + let children = copy_children_with_status(element); + let mut found = false; + for child in children.elements { + if !found { + found = probe_web_area(child, depth - 1, visits, deadline); + } + CFRelease(child as CFTypeRef); + } + found +} + +unsafe fn await_web_content(app_element: AXUIElementRef, deadline: Instant) -> bool { + while Instant::now() < deadline { + let mut visits = 400; + if probe_web_area(app_element, 10, &mut visits, deadline) { + return true; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break; + } + std::thread::sleep(remaining.min(POLL_INTERVAL)); + } + false +} + +/// `app_element` must remain a valid application AX element for `pid`. +/// A missing deadline permits opt-in, but never a materialization wait. +pub(crate) unsafe fn ensure_chromium_ax_enabled( + pid: i32, + app_element: AXUIElementRef, + readiness_deadline: Option<Instant>, +) -> ChromiumReadiness { + let stamp = process_start_stamp(pid); + let cached = enabled_processes() + .lock() + .ok() + .and_then(|cache| cache.get(&pid).copied()) + .filter(|cached| cached_lifetime_is_current(cached, stamp)); + if cached.is_some_and(|cached| cached.ready) { + return ChromiumReadiness::Ready; + } + if cached.is_none() { + if let Some(deadline) = readiness_deadline { + let remaining = deadline + .saturating_duration_since(Instant::now()) + .as_secs_f32(); + let _ = AXUIElementSetMessagingTimeout(app_element, remaining.min(0.1).max(0.001)); + } + match enable_chromium_accessibility(app_element) { + AccessibilityOptIn::NotAccepted => return ChromiumReadiness::NotApplicable, + AccessibilityOptIn::EnhancedUserInterface => { + remember(pid, stamp, true, None); + return ChromiumReadiness::Ready; + } + AccessibilityOptIn::ManualAccessibility => remember(pid, stamp, false, None), + } + } + let now = Instant::now(); + if let Some(deadline) = readiness_deadline.filter(|deadline| *deadline > now) { + if should_wait(cached, now) { + if await_web_content(app_element, deadline) { + remember(pid, stamp, true, None); + return ChromiumReadiness::Ready; + } + remember(pid, stamp, false, Some(now)); + } + } + ChromiumReadiness::Pending(stamp) +} + +// A real walk can discover naturally materialized content during cooldown. +// That positive evidence also prevents a late timeout from downgrading Ready. +pub(crate) fn complete_chromium_observation( + pid: i32, + readiness: ChromiumReadiness, + web_area_seen: bool, +) -> bool { + match readiness { + ChromiumReadiness::Pending(stamp) if web_area_seen => { + remember(pid, stamp, true, None); + true + } + ChromiumReadiness::Pending(_) => false, + ChromiumReadiness::Ready | ChromiumReadiness::NotApplicable => true, } } @@ -114,20 +254,65 @@ mod tests { } #[test] - fn cache_hit_requires_the_same_readable_process_lifetime() { - let first_launch = Some((100, 10)); - let relaunched = Some((101, 20)); - - assert!(cached_lifetime_is_current( - Some(&first_launch), - first_launch + fn accepted_is_pending_and_cooldown_does_not_prevent_positive_promotion() { + let now = Instant::now(); + let pending = merge_state(None, (100, 10), false, None); + assert!(should_wait(Some(pending), now)); + let timed_out = merge_state(Some(pending), pending.stamp, false, Some(now)); + assert!(!should_wait( + Some(timed_out), + now + Duration::from_millis(100) )); + let ready = merge_state(Some(timed_out), timed_out.stamp, true, None); + assert!(ready.ready); + assert!(!should_wait(Some(ready), now + RETRY_BACKOFF)); + let late = merge_state(Some(ready), ready.stamp, false, Some(now)); assert!( - !cached_lifetime_is_current(Some(&first_launch), relaunched), - "a relaunched process must not inherit the prior enablement cache entry" + late.ready, + "a late timeout must not downgrade positive evidence" ); - assert!(!cached_lifetime_is_current(Some(&first_launch), None)); - assert!(!cached_lifetime_is_current(Some(&None), first_launch)); - assert!(!cached_lifetime_is_current(None, first_launch)); + } + + #[test] + fn no_wait_does_not_consume_attempts_and_new_lifetime_does_not_alias() { + let now = Instant::now(); + let pending = merge_state(None, (100, 10), false, None); + let again = merge_state(Some(pending), pending.stamp, false, None); + assert_eq!(again.attempts, 0); + assert!(should_wait(Some(again), now)); + assert!(!cached_lifetime_is_current(&again, None)); + assert!(!cached_lifetime_is_current(&again, Some((101, 20)))); + let old_ready = merge_state(Some(again), again.stamp, true, None); + let relaunched = merge_state(Some(old_ready), (101, 20), false, None); + assert!(!relaunched.ready); + assert_eq!(relaunched.attempts, 0); + } + + #[test] + fn a_later_walk_with_web_area_is_complete_even_without_a_cacheable_stamp() { + assert!(!complete_chromium_observation( + 0, + ChromiumReadiness::Pending(None), + false + )); + assert!(complete_chromium_observation( + 0, + ChromiumReadiness::Pending(None), + true + )); + } + + #[test] + fn stale_completion_does_not_replace_the_live_lifetime_or_its_ready_state() { + let pid = std::process::id() as i32; + let stamp = process_start_stamp(pid).expect("own process start time"); + remember(pid, Some(stamp), true, None); + let stale_stamp = (stamp.0.saturating_sub(1), stamp.1); + assert_ne!(stale_stamp, stamp); + remember(pid, Some(stale_stamp), false, Some(Instant::now())); + let cached = enabled_processes().lock().unwrap()[&pid]; + assert_eq!(cached.stamp, stamp); + assert!(cached.ready); + enabled_processes().lock().unwrap().remove(&pid); } } diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs index 8677a23d6ab..4eb9d07a6e5 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs @@ -11,12 +11,17 @@ use core_foundation::base::{CFEqual, CFRelease, CFRetain, CFTypeRef}; use cua_driver_core::background_input::{ BackgroundTargetFacts, ElementAncestry, WindowServerOwnership, }; +use std::collections::HashSet; +use std::time::{Duration, Instant}; use super::bindings::{ - ax_get_window_id, copy_ax_windows_with_status, copy_bool_attr, copy_element_attr, - copy_string_attr, focused_element_of_pid, kAXErrorSuccess, AXUIElementCreateApplication, - AXUIElementGetPid, AXUIElementRef, AXUIElementSetMessagingTimeout, + _AXUIElementGetWindow, ax_get_window_id, copy_ax_windows_with_status, copy_bool_attr, + copy_bool_attr_with_status, copy_children_with_status, copy_element_attr, + copy_remote_window_for_id, copy_string_attr, focused_element_of_pid, kAXErrorSuccess, + AXUIElementCreateApplication, AXUIElementGetPid, AXUIElementRef, + AXUIElementSetMessagingTimeout, }; +use super::tree::AXIdentity; use crate::windows::{all_windows, resolve_window_owner, WindowOwner}; /// Bounded `AXParent` ascent used when an element does not expose `AXWindow`. @@ -24,7 +29,7 @@ const MAX_ANCESTRY_DEPTH: usize = 40; /// Resolve the CGWindowID of the top-level AX window that owns `element`. /// -/// Prefers the element's `AXWindow` attribute and falls back to a bounded +/// Prefers its native window mapping, then `AXWindow` and a bounded /// `AXParent` walk. `None` means ancestry could not be proven — callers must /// treat that as "not the requested window", never as a wildcard. /// @@ -33,6 +38,12 @@ const MAX_ANCESTRY_DEPTH: usize = 40; /// `element` must be a valid `AXUIElementRef` for the duration of the call. pub unsafe fn element_window_id(element: AXUIElementRef) -> Option<u32> { let _ = AXUIElementSetMessagingTimeout(element, 0.1); + // AppKit sheet fields can expose their parent in AXWindow, while their + // native mapping identifies the actual sheet receiving keyboard input. + let mut window_id = 0; + if _AXUIElementGetWindow(element, &mut window_id) == kAXErrorSuccess && window_id != 0 { + return Some(window_id); + } if let Some(window) = copy_element_attr(element, "AXWindow") { let window_id = ax_get_window_id(window); CFRelease(window as CFTypeRef); @@ -132,8 +143,17 @@ unsafe fn is_app_menu( /// /// Caller must `CFRelease` the returned element. pub unsafe fn focused_element_in_window(pid: i32, window_id: u32) -> Option<AXUIElementRef> { - let element = focused_element_of_pid(pid)?; + let Some(element) = focused_element_of_pid(pid) else { + return focused_text_descendant(pid, window_id); + }; if element_window_id(element) == Some(window_id) { + // Some apps report an unfocused container above the actual text focus. + if copy_string_attr(element, "AXRole").as_deref() == Some("AXGroup") + && copy_bool_attr(element, "AXFocused") == Some(false) + { + CFRelease(element as CFTypeRef); + return focused_text_descendant(pid, window_id); + } Some(element) } else { CFRelease(element as CFTypeRef); @@ -141,6 +161,91 @@ pub unsafe fn focused_element_in_window(pid: i32, window_id: u32) -> Option<AXUI } } +// WKWebView can expose AXFocused on its input while the application's +// AXFocusedUIElement is empty. Never infer focus from a cached observation. +unsafe fn focused_text_descendant(pid: i32, window_id: u32) -> Option<AXUIElementRef> { + let deadline = Instant::now() + Duration::from_millis(300); + let app = AXUIElementCreateApplication(pid); + if app.is_null() { + return None; + } + let _ = AXUIElementSetMessagingTimeout(app, 0.05); + let root = copy_element_attr(app, "AXFocusedWindow"); + CFRelease(app as CFTypeRef); + let root = root?; + let identity = AXIdentity::retained(root); + CFRelease(root as CFTypeRef); + if ax_get_window_id(root) != Some(window_id) { + return None; + } + let mut pending = vec![(identity, 0)]; + let mut visited = HashSet::new(); + let mut focused: Option<AXIdentity> = None; + while let Some((identity, depth)) = pending.pop() { + if !visited.insert(identity.clone()) { + continue; + } + if visited.len() > 256 || Instant::now() >= deadline { + return None; + } + let element = identity.as_ptr() as AXUIElementRef; + let timeout = deadline + .saturating_duration_since(Instant::now()) + .as_secs_f32() + .clamp(0.001, 0.025); + let _ = AXUIElementSetMessagingTimeout(element, timeout); + let mut owner = 0; + if AXUIElementGetPid(element, &mut owner) != kAXErrorSuccess || owner <= 0 { + return None; + } + let focus = copy_bool_attr_with_status(element, "AXFocused"); + if !focus.complete { + return None; + } + if focus.value == Some(true) + && matches!( + copy_string_attr(element, "AXRole").as_deref(), + Some("AXTextField" | "AXTextArea" | "AXSearchField" | "AXComboBox") + ) + { + // A window can contain remote helper containers. Only the final + // editable destination must belong to this process and window. + if owner != pid || focused.is_some() || element_window_id(element) != Some(window_id) { + return None; + } + focused = Some(identity.clone()); + } + let children = copy_children_with_status(element); + let children_owned: Vec<_> = children + .elements + .into_iter() + .map(|child| { + let identity = AXIdentity::retained(child); + CFRelease(child as CFTypeRef); + identity + }) + .collect(); + if !children.complete + || (depth >= MAX_ANCESTRY_DEPTH && !children_owned.is_empty()) + || pending.len() + children_owned.len() + visited.len() > 256 + { + return None; + } + pending.extend(children_owned.into_iter().map(|child| (child, depth + 1))); + } + let focused = focused?; + let element = focused.as_ptr() as AXUIElementRef; + if Instant::now() >= deadline + || super::bindings::focused_window_id_of_pid(pid) != Some(window_id) + || copy_bool_attr(element, "AXFocused") != Some(true) + || element_window_id(element) != Some(window_id) + { + return None; + } + CFRetain(element as CFTypeRef); + Some(element) +} + fn keyboard_focus_matches_target(pid: i32, window_id: u32) -> bool { if super::bindings::focused_window_id_of_pid(pid) != Some(window_id) { return false; @@ -199,9 +304,13 @@ fn attached_sheet_record( }) } -/// Map fresh AXWindows, resolving a missing target only through proven sheets. +/// Map fresh AXWindows, proven sheets, and an exactly recovered remote window. /// An unmapped window or incomplete attachment discovery cannot unlock input. -unsafe fn ax_window_records(app: AXUIElementRef, target_window_id: u32) -> Vec<AxWindowRecord> { +unsafe fn ax_window_records( + app: AXUIElementRef, + pid: i32, + target_window_id: u32, +) -> Vec<AxWindowRecord> { let windows_read = copy_ax_windows_with_status(app); let windows = windows_read.elements; let mut records: Vec<_> = windows @@ -213,6 +322,7 @@ unsafe fn ax_window_records(app: AXUIElementRef, target_window_id: u32) -> Vec<A }) }) .collect(); + let all_windows_mapped = records.len() == windows.len(); if windows_read.complete && !records .iter() @@ -220,6 +330,12 @@ unsafe fn ax_window_records(app: AXUIElementRef, target_window_id: u32) -> Vec<A { let (sheets, complete) = super::sheets::copy_attached_sheets(&windows); if complete { + let all_sheets_mapped = sheets.iter().all(|sheet| { + sheet.window_id.is_some() + && sheet.root_window_id.is_some_and(|root_id| { + records.iter().any(|record| record.window_id == root_id) + }) + }); for sheet in sheets { if let Some(record) = attached_sheet_record(sheet.window_id, sheet.root_window_id, &records) @@ -227,6 +343,21 @@ unsafe fn ax_window_records(app: AXUIElementRef, target_window_id: u32) -> Vec<A records.push(record); } } + if all_windows_mapped + && all_sheets_mapped + && !records + .iter() + .any(|record| record.window_id == target_window_id) + { + if let Some(window) = copy_remote_window_for_id(pid, target_window_id, None) { + let _ = AXUIElementSetMessagingTimeout(window, 0.1); + records.push(AxWindowRecord { + window_id: target_window_id, + minimized: copy_bool_attr(window, "AXMinimized"), + }); + CFRelease(window as CFTypeRef); + } + } } } for window in windows { @@ -294,8 +425,8 @@ pub fn gather_background_facts( let _ = AXUIElementSetMessagingTimeout(app, 0.1); // Electron/Chromium apps may need per-process-lifetime enablement // before their AX windows and subtrees are materialized. - super::enablement::ensure_chromium_ax_enabled(pid, app); - let records = ax_window_records(app, window_id); + super::enablement::ensure_chromium_ax_enabled(pid, app, None); + let records = ax_window_records(app, pid, window_id); let app_hidden = copy_bool_attr(app, "AXHidden"); let element = element_ptr.map(|ptr| match element_window_id(ptr as AXUIElementRef) { Some(id) if id == window_id => ElementAncestry::ProvenDescendant, diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/launch.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/launch.rs new file mode 100644 index 00000000000..0e1cf984843 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/launch.rs @@ -0,0 +1,52 @@ +use std::time::{Duration, Instant}; + +const LAUNCH_LOOKUP_BUDGET: Duration = Duration::from_secs(1); + +pub(crate) fn lookup_deadline(readiness_deadline: Option<Instant>) -> Option<Instant> { + // Both App selection and raw AX observations construct readiness from the + // same observation start. Preserve that anchor across native consumers. + readiness_deadline? + .checked_sub(cua_driver_core::tool::AX_READINESS_BUDGET)? + .checked_add(LAUNCH_LOOKUP_BUDGET) +} + +pub(crate) fn is_still_launching(pid: i32) -> bool { + unsafe { + objc2_app_kit::NSRunningApplication::runningApplicationWithProcessIdentifier(pid) + .is_some_and(|app| !app.isFinishedLaunching()) + } +} + +pub(crate) fn retry_pause( + pid: i32, + deadline: Option<Instant>, + attempt_started: Instant, +) -> Option<Duration> { + let deadline = deadline?; + if Instant::now() >= deadline || !is_still_launching(pid) { + return None; + } + let now = Instant::now(); + let attempt_duration = now.saturating_duration_since(attempt_started); + let remaining = deadline.checked_duration_since(now)?; + let slack = remaining.checked_sub(attempt_duration)?; + (!slack.is_zero()).then_some(slack.min(Duration::from_millis(50))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lookup_keeps_the_original_observation_anchor_and_no_wait_mode() { + let start = Instant::now(); + let readiness = start + cua_driver_core::tool::AX_READINESS_BUDGET; + assert_eq!( + lookup_deadline(Some(readiness)), + Some(start + Duration::from_secs(1)) + ); + assert_eq!(lookup_deadline(None), None); + assert_eq!(retry_pause(0, None, start), None); + assert_eq!(retry_pause(0, Some(start), start), None); + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/mod.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/mod.rs index 62f9f53c368..2975f472240 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/ax/mod.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/mod.rs @@ -22,9 +22,11 @@ pub mod cache; pub(crate) mod dialogs; pub mod enablement; pub mod exact_target; +pub(crate) mod launch; pub(crate) mod menu; mod placeholder; pub(crate) mod projection; +pub(crate) mod reveal_scroll; pub mod revision; pub mod sheets; pub mod tree; diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs new file mode 100644 index 00000000000..61e568966e1 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs @@ -0,0 +1,451 @@ +//! Native background scrolling adapted from upstream #4861/#5079. Selection +//! is read-only; dispatch reveals one element without waiting or replaying. + +use core_foundation::base::{CFRelease, CFRetain, CFTypeRef}; +use std::time::{Duration, Instant}; + +use super::bindings::{ + copy_children, copy_element_attr, copy_string_attr, element_at_screen_position, + element_screen_rect, kAXErrorSuccess, perform_action, AXUIElementGetPid, AXUIElementRef, +}; + +type Rect = [f64; 4]; + +struct Element(usize); + +impl Drop for Element { + fn drop(&mut self) { + unsafe { CFRelease(self.0 as CFTypeRef) }; + } +} + +// Retained AX proxies already cross blocking workers in the element cache. +unsafe impl Send for Element {} + +impl Element { + fn ptr(&self) -> AXUIElementRef { + self.0 as AXUIElementRef + } +} + +pub(crate) struct RevealTarget { + element: Element, + pub container_role: String, +} + +impl RevealTarget { + pub fn element_ptr(&self) -> usize { + self.element.0 + } + + /// The caller holds the background mutation lease and has just gated this + /// exact element. Consume the target so no retry can follow an AX error. + pub unsafe fn dispatch(self, pid: i32, window_id: u32) -> Result<i32, &'static str> { + if !belongs_to(self.element.ptr(), pid, window_id) + || crate::windows::resolve_window_owner(pid, window_id) + != crate::windows::WindowOwner::SamePid + { + return Err("The native scroll target no longer belongs to the requested window."); + } + Ok(perform_action(self.element.ptr(), "AXScrollToVisible")) + } +} + +unsafe fn belongs_to(element: AXUIElementRef, pid: i32, window_id: u32) -> bool { + let mut owner = 0; + AXUIElementGetPid(element, &mut owner) == kAXErrorSuccess + && owner == pid + && super::exact_target::element_window_id(element) == Some(window_id) +} + +unsafe fn children(element: AXUIElementRef) -> Vec<Element> { + copy_children(element) + .into_iter() + .map(|p| Element(p as usize)) + .collect() +} + +fn valid_rect(rect: Rect) -> bool { + rect.iter().all(|v| v.is_finite()) && rect[2] > 0.0 && rect[3] > 0.0 +} + +fn intersect(a: Rect, b: Rect) -> Option<Rect> { + if !valid_rect(a) || !valid_rect(b) { + return None; + } + let left = a[0].max(b[0]); + let top = a[1].max(b[1]); + let right = (a[0] + a[2]).min(b[0] + b[2]); + let bottom = (a[1] + a[3]).min(b[1] + b[3]); + (right > left && bottom > top).then_some([left, top, right - left, bottom - top]) +} + +fn past_edge([fx, fy, fw, fh]: Rect, direction: &str, [x, y, w, h]: Rect) -> bool { + match direction { + "down" => y + h > fy + fh + 1.0 || (h <= 1.5 && y >= fy + fh - 2.5), + "up" => y < fy - 1.0 || (h <= 1.5 && y <= fy + 1.5), + "right" => x + w > fx + fw + 1.0 || (w <= 1.5 && x >= fx + fw - 2.5), + "left" => x < fx - 1.0 || (w <= 1.5 && x <= fx + 1.5), + _ => false, + } +} + +fn clips_axis(frame: Rect, direction: &str, rect: Rect) -> bool { + let opposite = match direction { + "down" => "up", + "up" => "down", + "left" => "right", + "right" => "left", + _ => return false, + }; + past_edge(frame, direction, rect) || past_edge(frame, opposite, rect) +} + +fn pick_target( + viewport: Rect, + direction: &str, + distance: f64, + nodes: &[(Rect, bool)], +) -> Option<usize> { + let [vx, vy, vw, vh] = viewport; + let mut best: Option<(usize, f64)> = None; + for (index, (rect, _)) in nodes.iter().enumerate() { + let [x, y, w, h] = *rect; + if w <= 1.5 || h <= 1.5 { + continue; + } + let (edge, goal, beyond, fits) = match direction { + "down" => (y + h, vy + vh + distance, y + h > vy + vh + 1.0, h <= vh), + "up" => (y, vy - distance, y < vy - 1.0, h <= vh), + "right" => (x + w, vx + vw + distance, x + w > vx + vw + 1.0, w <= vw), + "left" => (x, vx - distance, x < vx - 1.0, w <= vw), + _ => return None, + }; + let gap = (edge - goal).abs(); + if beyond && fits && best.is_none_or(|(_, old)| gap < old) { + best = Some((index, gap)); + } + } + if let Some((index, _)) = best { + return Some(index); + } + let vertical = matches!(direction, "up" | "down"); + let sliver = |r: &Rect| if vertical { r[3] <= 1.5 } else { r[2] <= 1.5 }; + let visible: Vec<(usize, f64)> = nodes + .iter() + .enumerate() + .filter(|(_, (r, leaf))| { + *leaf + && !sliver(r) + && r[0] >= vx + && r[1] >= vy + && r[0] + r[2] <= vx + vw + && r[1] + r[3] <= vy + vh + }) + .map(|(index, (r, _))| (index, if vertical { r[1] } else { r[0] })) + .collect(); + let forward = matches!(direction, "down" | "right"); + let visible_edge = if forward { + visible.last() + } else { + visible.first() + } + .map(|(index, _)| *index); + let mut clamped: Vec<usize> = nodes + .iter() + .enumerate() + .filter(|(index, (rect, leaf))| { + // AX can clamp an earlier row to the far edge. Respect the known + // visible document order before revealing another row. + *leaf + && sliver(rect) + && past_edge(viewport, direction, *rect) + && visible_edge.is_none_or(|edge| { + if forward { + *index > edge + } else { + *index < edge + } + }) + }) + .map(|(index, _)| index) + .collect(); + if clamped.is_empty() { + return None; + } + if matches!(direction, "up" | "left") { + clamped.reverse(); + } + let mut starts: Vec<f64> = visible.into_iter().map(|(_, start)| start).collect(); + starts.sort_by(f64::total_cmp); + starts.dedup_by(|a, b| (*a - *b).abs() < 2.0); + let mut gaps: Vec<f64> = starts.windows(2).map(|p| p[1] - p[0]).collect(); + gaps.sort_by(f64::total_cmp); + let pitch = gaps.get(gaps.len() / 2).copied().unwrap_or(40.0).max(8.0); + let half = if vertical { vh } else { vw } / 2.0; + let rows = ((distance - half) / pitch + 0.5).round().max(1.0) as usize; + Some(clamped[rows.min(clamped.len()) - 1]) +} + +unsafe fn scroll_container( + mut element: Element, + direction: &str, + deadline: Instant, +) -> Option<(Element, String)> { + let mut inner = None; + for _ in 0..40 { + if Instant::now() >= deadline { + return None; + } + let role = copy_string_attr(element.ptr(), "AXRole")?; + if matches!(role.as_str(), "AXScrollArea" | "AXWebArea") { + return Some(inner.unwrap_or((element, role))); + } + if matches!(role.as_str(), "AXWindow" | "AXApplication") { + return None; + } + if inner.is_none() { + if let Some(frame) = element_screen_rect(element.ptr()).filter(|r| valid_rect(*r)) { + let clipped = children(element.ptr()).iter().any(|child| { + element_screen_rect(child.ptr()) + .filter(|r| valid_rect(*r)) + .is_some_and(|r| clips_axis(frame, direction, r)) + }); + if clipped { + CFRetain(element.ptr() as CFTypeRef); + inner = Some((Element(element.0), role)); + } + } + } + element = Element(copy_element_attr(element.ptr(), "AXParent")? as usize); + } + None +} + +/// Select from one exact window. The supplied element remains retained by the +/// blocking caller; the returned target owns its own reference across awaits. +pub(crate) unsafe fn prepare( + pid: i32, + window_id: u32, + element: Option<usize>, + point: (f64, f64), + window: Rect, + direction: &str, + distance: f64, +) -> Option<RevealTarget> { + if !valid_rect(window) + || !distance.is_finite() + || distance <= 0.0 + || !matches!(direction, "up" | "down" | "left" | "right") + || crate::windows::resolve_window_owner(pid, window_id) + != crate::windows::WindowOwner::SamePid + { + return None; + } + let deadline = Instant::now() + Duration::from_secs(1); + let start = if let Some(ptr) = element { + CFRetain(ptr as CFTypeRef); + Element(ptr) + } else { + if !point.0.is_finite() + || !point.1.is_finite() + || point.0 < window[0] + || point.1 < window[1] + || point.0 >= window[0] + window[2] + || point.1 >= window[1] + window[3] + { + return None; + } + Element(element_at_screen_position(pid, point.0, point.1)? as usize) + }; + if !belongs_to(start.ptr(), pid, window_id) { + return None; + } + let (container, role) = scroll_container(start, direction, deadline)?; + if !belongs_to(container.ptr(), pid, window_id) { + return None; + } + let viewport = intersect(element_screen_rect(container.ptr())?, window)?; + let mut stack: Vec<(Element, usize)> = children(container.ptr()) + .into_iter() + .rev() + .map(|child| (child, 1)) + .collect(); + let mut elements = Vec::new(); + let mut nodes = Vec::new(); + let mut visited = 0; + while let Some((element, depth)) = stack.pop() { + if Instant::now() >= deadline || visited >= 4_000 { + return None; + } + visited += 1; + let rect = element_screen_rect(element.ptr()).filter(|r| valid_rect(*r)); + let behind = rect.is_some_and(|[x, y, w, h]| match direction { + "down" => y + h < viewport[1], + "up" => y > viewport[1] + viewport[3], + "right" => x + w < viewport[0], + "left" => x > viewport[0] + viewport[2], + _ => false, + }); + let mut leaf = true; + if !behind { + let descendants = children(element.ptr()); + leaf = descendants.is_empty(); + if depth >= 80 && !leaf { + return None; + } + stack.extend( + descendants + .into_iter() + .rev() + .map(|child| (child, depth + 1)), + ); + } + if let Some(rect) = rect.filter(|_| !behind) { + nodes.push((rect, leaf)); + elements.push(element); + } + } + if Instant::now() >= deadline { + return None; + } + let index = pick_target(viewport, direction, distance, &nodes)?; + let target = elements.swap_remove(index); + belongs_to(target.ptr(), pid, window_id).then_some(RevealTarget { + element: target, + container_role: role, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn directional_rows_and_container_end() { + let viewport = [0.0, 100.0, 800.0, 500.0]; + let nodes: Vec<_> = (-10..30) + .map(|i| ([10.0, 100.0 + 50.0 * i as f64, 600.0, 40.0], true)) + .collect(); + assert_eq!( + nodes[pick_target(viewport, "down", 360.0, &nodes).unwrap()].0[1], + 900.0 + ); + assert_eq!( + nodes[pick_target(viewport, "up", 120.0, &nodes).unwrap()].0[1], + 0.0 + ); + assert_eq!( + pick_target( + viewport, + "down", + 120.0, + &[([0.0, 100.0, 800.0, 3000.0], false)] + ), + None + ); + let columns: Vec<_> = (0..20) + .map(|i| ([100.0 * i as f64, 200.0, 90.0, 40.0], true)) + .collect(); + assert_eq!( + columns[pick_target(viewport, "right", 120.0, &columns).unwrap()].0[0], + 800.0 + ); + assert_eq!(pick_target(viewport, "left", 120.0, &columns), None); + } + + #[test] + fn chromium_clamped_rows_and_inner_clip() { + let viewport = [500.0, 290.0, 800.0, 600.0]; + let mut nodes: Vec<_> = (0..13) + .map(|i| ([518.0, 301.0 + 45.0 * i as f64, 600.0, 41.0], true)) + .collect(); + let first = nodes.len(); + nodes.extend((0..100).map(|_| ([518.0, 889.0, 600.0, 1.0], true))); + assert_eq!( + pick_target(viewport, "down", 360.0, &nodes), + Some(first + 1) + ); + assert_eq!( + pick_target(viewport, "down", 600.0, &nodes), + Some(first + 6) + ); + assert_eq!(pick_target(viewport, "up", 120.0, &nodes), None); + assert!(past_edge( + [0.0, 100.0, 300.0, 128.0], + "down", + [0.0, 227.0, 300.0, 1.0] + )); + assert!(!past_edge( + [0.0, 100.0, 300.0, 128.0], + "down", + [0.0, 110.0, 300.0, 16.0] + )); + } + + #[test] + fn viewport_requires_finite_overlap() { + assert_eq!( + intersect([0.0, 0.0, 1000.0, 3000.0], [10.0, 50.0, 800.0, 600.0]), + Some([10.0, 50.0, 800.0, 600.0]) + ); + assert_eq!( + intersect([0.0, 0.0, 10.0, 10.0], [20.0, 20.0, 5.0, 5.0]), + None + ); + assert_eq!( + intersect([0.0, 0.0, f64::INFINITY, 10.0], [0.0, 0.0, 5.0, 5.0]), + None + ); + } + + #[test] + fn inner_container_at_its_end_still_owns_the_scroll_axis() { + let frame = [0.0, 100.0, 300.0, 128.0]; + let clipped_above = [0.0, 100.0, 200.0, 1.0]; + assert!(!past_edge(frame, "down", clipped_above)); + assert!(clips_axis(frame, "down", clipped_above)); + assert!(clips_axis(frame, "up", clipped_above)); + let clipped_right = [299.0, 110.0, 1.0, 16.0]; + assert!(!clips_axis(frame, "down", clipped_right)); + assert!(clips_axis(frame, "left", clipped_right)); + } + + #[test] + fn wrong_edge_slivers_cannot_reverse_the_visible_document_order() { + let viewport = [144.0, 320.0, 360.0, 120.0]; + let mut nodes = vec![ + ([154.0, 439.0, 143.0, 1.0], true), + ([154.0, 320.0, 143.0, 10.0], true), + ([154.0, 344.0, 140.0, 18.0], true), + ([154.0, 376.0, 143.0, 18.0], true), + ([154.0, 408.0, 260.0, 24.0], true), + ]; + assert_eq!(pick_target(viewport, "down", 120.0, &nodes), None); + nodes.push(([154.0, 439.0, 143.0, 1.0], true)); + assert_eq!(pick_target(viewport, "down", 120.0, &nodes), Some(5)); + + let mut mirrored = vec![ + ([154.0, 320.0, 143.0, 10.0], true), + ([154.0, 344.0, 140.0, 18.0], true), + ([154.0, 320.0, 143.0, 1.0], true), + ]; + assert_eq!(pick_target(viewport, "up", 120.0, &mirrored), None); + mirrored.insert(0, ([154.0, 320.0, 143.0, 1.0], true)); + assert_eq!(pick_target(viewport, "up", 120.0, &mirrored), Some(0)); + } + + #[test] + fn unanchored_clamped_geometry_retains_its_existing_behavior() { + let viewport = [144.0, 320.0, 360.0, 120.0]; + assert_eq!( + pick_target( + viewport, + "down", + 120.0, + &[([154.0, 439.0, 143.0, 1.0], true)] + ), + Some(0) + ); + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/packages/cua-driver/rust/crates/platform-macos/src/ax/tree.rs index 573dd658192..b5349c14e3c 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/ax/tree.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/ax/tree.rs @@ -16,6 +16,7 @@ use super::window_scope::{decide_window_scope, TopLevelCandidate, WindowScope}; use core_foundation::base::{CFEqual, CFHash, CFRelease, CFRetain, CFTypeRef}; use std::collections::HashSet; use std::hash::{Hash, Hasher}; +use std::time::{Duration, Instant}; /// Default maximum depth for AX tree walks. Deep menus and complex web views /// can nest deeply; 25 covers realistic app chrome without exploding on @@ -42,8 +43,29 @@ pub const DEFAULT_MAX_ELEMENTS: usize = 2_000; /// indefinitely. const AX_MESSAGING_TIMEOUT_SECONDS: f32 = 2.0; -unsafe fn set_messaging_timeout(element: AXUIElementRef) { - let _ = AXUIElementSetMessagingTimeout(element, AX_MESSAGING_TIMEOUT_SECONDS); +#[derive(Default)] +struct ObservationWalkBudget { + started: Option<Instant>, + exhausted: bool, +} + +impl ObservationWalkBudget { + fn admit(&mut self) -> bool { + let started = self.started.get_or_insert_with(Instant::now); + self.exhausted |= started.elapsed() >= Duration::from_secs(1); + !self.exhausted + } +} + +unsafe fn set_messaging_timeout(element: AXUIElementRef, deadline: Option<Instant>) { + let timeout = deadline.map_or(AX_MESSAGING_TIMEOUT_SECONDS, |deadline| { + deadline + .saturating_duration_since(Instant::now()) + .as_secs_f32() + .min(AX_MESSAGING_TIMEOUT_SECONDS) + .max(0.001) + }); + let _ = AXUIElementSetMessagingTimeout(element, timeout); } #[derive(Debug)] @@ -222,7 +244,7 @@ pub struct TreeWalkResult { /// Bounded trace naming why `complete` is false (attribute + AX error), /// drained from the walk's thread-local recorder. Empty on complete walks. pub incomplete_notes: Vec<String>, - /// True when the walk was cut short by the node or depth cap. + /// True when the walk was cut short by a node, depth or time budget. pub truncated: bool, /// Whether the requested `window_id` actually resolved to an AX surface, /// and if not, why. `None` when no `window_id` was requested. @@ -274,7 +296,17 @@ pub fn walk_tree_bounded( max_elements: usize, max_depth: usize, ) -> TreeWalkResult { - walk_tree_with_context(pid, window_id, query, max_elements, max_depth, false) + walk_tree_with_deadline( + pid, + window_id, + query, + max_elements, + max_depth, + false, + None, + None, + None, + ) } #[derive(Clone, Copy, PartialEq, Eq)] @@ -300,6 +332,49 @@ pub(crate) fn walk_tree_with_context( max_elements: usize, max_depth: usize, app_context: bool, + readiness_deadline: Option<Instant>, +) -> TreeWalkResult { + walk_tree_with_deadline( + pid, + window_id, + query, + max_elements, + max_depth, + app_context, + readiness_deadline, + None, + Some(ObservationWalkBudget::default()), + ) +} + +pub(crate) fn walk_tree_for_recording( + pid: i32, + window_id: u32, + budget: Duration, +) -> TreeWalkResult { + walk_tree_with_deadline( + pid, + Some(window_id), + None, + DEFAULT_MAX_ELEMENTS, + DEFAULT_MAX_DEPTH, + false, + None, + Some(Instant::now() + budget), + None, + ) +} + +fn walk_tree_with_deadline( + pid: i32, + window_id: Option<u32>, + query: Option<&str>, + max_elements: usize, + max_depth: usize, + app_context: bool, + readiness_deadline: Option<Instant>, + deadline: Option<Instant>, + mut observation_budget: Option<ObservationWalkBudget>, ) -> TreeWalkResult { let mut nodes: Vec<AXNode> = Vec::new(); let mut lines: Vec<(usize, String)> = Vec::new(); // (depth, line) @@ -309,7 +384,8 @@ pub(crate) fn walk_tree_with_context( // Set to true only when walk_element actually stops early due to the cap — // avoids a false-positive when the tree naturally ends on exactly the cap. let mut truncated = false; - let mut complete = true; + let mut complete: bool; + let mut web_area_seen = false; let mut window_scope: Option<WindowScope> = None; super::bindings::reset_incomplete_notes(); @@ -329,145 +405,200 @@ pub(crate) fn walk_tree_with_context( window_scope: window_id.map(|_| WindowScope::AxUnresolved { ax_window_count: 0 }), }; } - set_messaging_timeout(app_elem); + set_messaging_timeout(app_elem, deadline); // Chromium/Electron apps (Arc, VS Code, Electron shells) ship their // web-content AX tree OFF and only build it once an assistive client // asks for it. Without this, the first walk of such an app returns an // empty/title-bar-only tree (#1616). Flip the enablement attribute, - // then — only when the flip actually took and only the first time we - // see this process lifetime — let the asynchronously-built tree settle - // before we read it. Native Cocoa apps reject the attribute, so they - // pay no settle cost. This relies on the MAX_ELEMENTS node cap to keep - // the now-materialized (potentially large) tree bounded. - super::enablement::ensure_chromium_ax_enabled(pid, app_elem); + // then let an explicit AX observation wait within its shared readiness + // deadline. Recording and action/fact readers never wait for a WebArea. + // The materialized tree remains subject to the existing traversal caps. + let readiness = + super::enablement::ensure_chromium_ax_enabled(pid, app_elem, readiness_deadline); + set_messaging_timeout(app_elem, deadline); - // Union AXChildren + AXWindows — the only way to see background windows. - // AXChildren omits windows when the app isn't frontmost (AppKit limitation). - // AXWindows returns the window list regardless of activation state. - let from_children = copy_children_with_status(app_elem); - let from_windows = copy_ax_windows_with_status(app_elem); - complete &= from_children.complete && from_windows.complete; + let launch_deadline = super::launch::lookup_deadline(readiness_deadline); + let mut remote_probe_attempted = false; + let (mut walk_these, top_level) = 'resolve: loop { + let attempt_started = Instant::now(); + complete = true; + super::bindings::reset_incomplete_notes(); + // Union AXChildren + AXWindows — the only way to see background windows. + // AXChildren omits windows when the app isn't frontmost (AppKit limitation). + // AXWindows returns the window list regardless of activation state. + let from_children = copy_children_with_status(app_elem); + let from_windows = copy_ax_windows_with_status(app_elem); + complete &= from_children.complete && from_windows.complete; - let mut top_level = from_children.elements; - for w in from_windows.elements { - // AXChildren and AXWindows can return different proxy pointers for - // the same native window. CFEqual compares their AX identity; - // pointer equality alone duplicates the whole subtree and can turn - // one exact dialog action into a false ambiguity. - if !top_level - .iter() - .any(|&e| CFEqual(e as CFTypeRef, w as CFTypeRef) != 0) - { - top_level.push(w); - } else { - // Already present — release the extra retain from copy_ax_windows. - CFRelease(w as CFTypeRef); - } - } - - // Scope: keep non-window children (menu bar) + the target window — - // but ONLY once the target window has actually been identified. When - // nothing claims the requested id, `decide_window_scope` reports why - // and walks nothing; it must never fall back to "everything that isn't - // a window", which is how issue #2237 returned menu bars as panels. - let mut walk_these: Vec<AXUIElementRef> = if let Some(wid) = window_id { - let describe = |child, complete: &mut bool| { - set_messaging_timeout(child); - let role_read = copy_string_attr_with_status(child, "AXRole"); - if role_read.complete && role_read.value.is_none() { - super::bindings::note_incomplete( - "AXRole", - "top-level element with no role value", - ); - } - *complete &= role_read.complete && role_read.value.is_some(); - let role = role_read.value.unwrap_or_default(); - let subrole_read = copy_string_attr_with_status(child, "AXSubrole"); - *complete &= subrole_read.complete; - let identifier_read = copy_string_attr_with_status(child, "AXIdentifier"); - *complete &= identifier_read.complete; - // Match AX window element → CGWindowID via private SPI. - // Only windows carry one, so skip the round-trip elsewhere. - let ax_window_id = if matches!(role.as_str(), "AXWindow" | "AXSheet") { - ax_get_window_id(child) + let mut top_level = from_children.elements; + for w in from_windows.elements { + // AXChildren and AXWindows can return different proxy pointers for + // the same native window. CFEqual compares their AX identity; + // pointer equality alone duplicates the whole subtree and can turn + // one exact dialog action into a false ambiguity. + if !top_level + .iter() + .any(|&e| CFEqual(e as CFTypeRef, w as CFTypeRef) != 0) + { + top_level.push(w); } else { - None - }; - TopLevelCandidate { - role, - subrole: subrole_read.value, - identifier: identifier_read.value, - ax_window_id, + // Already present — release the extra retain from copy_ax_windows. + CFRelease(w as CFTypeRef); } - }; - let mut candidates: Vec<TopLevelCandidate> = top_level - .iter() - .map(|&child| describe(child, &mut complete)) - .collect(); - let mut decision = decide_window_scope(&candidates, wid, || { - crate::windows::resolve_window_owner(pid, wid) - }); - if matches!(decision.scope, WindowScope::AxUnresolved { .. }) { - // AppKit can omit every window from both arrays while these - // references still resolve to the requested window. - for attribute in ["AXFocusedWindow", "AXMainWindow"] { - let selection = copy_element_attr_with_status(app_elem, attribute); - complete &= selection.complete; - let Some(window) = selection.value else { - continue; - }; - let mut owner = 0; - if AXUIElementGetPid(window, &mut owner) == kAXErrorSuccess - && owner == pid - && !top_level - .iter() - .any(|&seen| CFEqual(seen as CFTypeRef, window as CFTypeRef) != 0) - { - let candidate = describe(window, &mut complete); - if matches!(candidate.role.as_str(), "AXWindow" | "AXSheet") - && candidate.ax_window_id == Some(wid) - { - top_level.push(window); - candidates.push(candidate); - continue; - } + } + + // Scope: keep non-window children (menu bar) + the target window — + // but ONLY once the target window has actually been identified. When + // nothing claims the requested id, `decide_window_scope` reports why + // and walks nothing; it must never fall back to "everything that isn't + // a window", which is how issue #2237 returned menu bars as panels. + let walk_these: Vec<AXUIElementRef> = if let Some(wid) = window_id { + let describe = |child, complete: &mut bool, proven_window_id: Option<u32>| { + set_messaging_timeout(child, deadline); + let role_read = copy_string_attr_with_status(child, "AXRole"); + if role_read.complete && role_read.value.is_none() { + super::bindings::note_incomplete( + "AXRole", + "top-level element with no role value", + ); } - CFRelease(window as CFTypeRef); - } - decision = decide_window_scope(&candidates, wid, || { + *complete &= role_read.complete && role_read.value.is_some(); + let role = role_read.value.unwrap_or_default(); + let subrole_read = copy_string_attr_with_status(child, "AXSubrole"); + *complete &= subrole_read.complete; + let identifier_read = copy_string_attr_with_status(child, "AXIdentifier"); + *complete &= identifier_read.complete; + // Match AX window element → CGWindowID via private SPI. + // Only windows carry one, so skip the round-trip elsewhere. + let ax_window_id = if matches!(role.as_str(), "AXWindow" | "AXSheet") { + proven_window_id.or_else(|| ax_get_window_id(child)) + } else { + None + }; + TopLevelCandidate { + role, + subrole: subrole_read.value, + identifier: identifier_read.value, + ax_window_id, + } + }; + let mut candidates: Vec<TopLevelCandidate> = top_level + .iter() + .map(|&child| describe(child, &mut complete, None)) + .collect(); + let mut decision = decide_window_scope(&candidates, wid, || { crate::windows::resolve_window_owner(pid, wid) }); - } - if matches!(decision.scope, WindowScope::AxUnresolved { .. }) { - let (sheets, discovery_complete) = super::sheets::copy_attached_sheets(&top_level); - complete &= discovery_complete; - if discovery_complete { - for sheet in sheets { - let element = sheet.element.as_ptr() as AXUIElementRef; - if !top_level - .iter() - .any(|&seen| CFEqual(seen as CFTypeRef, element as CFTypeRef) != 0) + if matches!(decision.scope, WindowScope::AxUnresolved { .. }) { + // AppKit can omit every window from both arrays while these + // references still resolve to the requested window. + for attribute in ["AXFocusedWindow", "AXMainWindow"] { + let selection = copy_element_attr_with_status(app_elem, attribute); + complete &= selection.complete; + let Some(window) = selection.value else { + continue; + }; + let mut owner = 0; + if AXUIElementGetPid(window, &mut owner) == kAXErrorSuccess + && owner == pid + && !top_level + .iter() + .any(|&seen| CFEqual(seen as CFTypeRef, window as CFTypeRef) != 0) { - CFRetain(element as CFTypeRef); - top_level.push(element); - candidates.push(TopLevelCandidate::new("AXSheet", sheet.window_id)); + let candidate = describe(window, &mut complete, None); + if matches!(candidate.role.as_str(), "AXWindow" | "AXSheet") + && candidate.ax_window_id == Some(wid) + { + top_level.push(window); + candidates.push(candidate); + continue; + } } + CFRelease(window as CFTypeRef); } decision = decide_window_scope(&candidates, wid, || { crate::windows::resolve_window_owner(pid, wid) }); } - } - let walk = decision - .walk - .iter() - .map(|&index| top_level[index]) - .collect(); - window_scope = Some(decision.scope); - walk - } else { - top_level.to_vec() + if matches!(decision.scope, WindowScope::AxUnresolved { .. }) { + let (sheets, discovery_complete) = + super::sheets::copy_attached_sheets(&top_level); + complete &= discovery_complete; + if discovery_complete { + for sheet in sheets { + let element = sheet.element.as_ptr() as AXUIElementRef; + if !top_level + .iter() + .any(|&seen| CFEqual(seen as CFTypeRef, element as CFTypeRef) != 0) + { + CFRetain(element as CFTypeRef); + top_level.push(element); + candidates.push(TopLevelCandidate::new("AXSheet", sheet.window_id)); + } + } + decision = decide_window_scope(&candidates, wid, || { + crate::windows::resolve_window_owner(pid, wid) + }); + } + } + if !remote_probe_attempted + && complete + && matches!(decision.scope, WindowScope::AxUnresolved { .. }) + { + remote_probe_attempted = true; + if let Some(window) = copy_remote_window_for_id(pid, wid, deadline) { + let candidate = describe(window, &mut complete, Some(wid)); + if complete && candidate.role == "AXWindow" { + top_level.push(window); + candidates.push(candidate); + decision = decide_window_scope(&candidates, wid, || { + crate::windows::resolve_window_owner(pid, wid) + }); + } else { + CFRelease(window as CFTypeRef); + } + } + } + if matches!(decision.scope, WindowScope::AxUnresolved { .. }) { + if let Some(pause) = + super::launch::retry_pause(pid, launch_deadline, attempt_started) + { + for child in top_level { + CFRelease(child as CFTypeRef); + } + std::thread::sleep(pause); + if launch_deadline.is_some_and(|limit| Instant::now() >= limit) { + window_scope = Some(decision.scope); + complete = false; + super::bindings::note_incomplete( + "AXWindow", + "application launch lookup exhausted its retry budget", + ); + break 'resolve (Vec::new(), Vec::new()); + } + continue 'resolve; + } + if launch_deadline.is_some() && super::launch::is_still_launching(pid) { + complete = false; + super::bindings::note_incomplete( + "AXWindow", + "application is still launching; exact window is unavailable", + ); + } + } + let walk = decision + .walk + .iter() + .map(|&index| top_level[index]) + .collect(); + window_scope = Some(decision.scope); + walk + } else { + top_level.to_vec() + }; + + break (walk_these, top_level); }; let mut mode = if app_context { @@ -509,9 +640,17 @@ pub(crate) fn walk_tree_with_context( &mut complete, max_elements, max_depth, + deadline, + &mut observation_budget, + &mut web_area_seen, ); } + if !super::enablement::complete_chromium_observation(pid, readiness, web_area_seen) { + complete = false; + super::bindings::note_incomplete("Chromium accessibility", "web content is not ready"); + } + if let Some(menu) = open_menu { CFRelease(menu as CFTypeRef); } @@ -549,7 +688,14 @@ pub(crate) fn walk_tree_with_context( raw_markdown }; - if truncated_flag { + if observation_budget.is_some_and(|budget| budget.exhausted) { + tree_markdown.push_str( + "\n⚠️ AX tree truncated by the 1-second observation time budget. \ + Listed elements remain valid; missing elements were not observed.", + ); + } else if truncated_flag && deadline.is_some_and(|deadline| Instant::now() >= deadline) { + tree_markdown.push_str("\n⚠️ AX tree truncated by the recording time budget."); + } else if truncated_flag { tree_markdown.push_str(&format!( "\n⚠️ AX tree truncated by traversal limits ({max_elements} nodes, depth {max_depth}) \ (app has a very large accessibility tree — Arc, Electron, or similar). \ @@ -585,12 +731,29 @@ unsafe fn walk_element( complete: &mut bool, max_elements: usize, max_depth: usize, + deadline: Option<Instant>, + observation_budget: &mut Option<ObservationWalkBudget>, + web_area_seen: &mut bool, ) { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + super::bindings::note_incomplete("walk", "recording time budget exhausted"); + *truncated = true; + *complete = false; + return; + } let identity = AXIdentity::retained(element); if visited_identities.contains(&identity) { return; } if depth <= max_depth && *visited_count < max_elements { + if observation_budget + .as_mut() + .is_some_and(|budget| !budget.admit()) + { + super::bindings::note_incomplete("walk", "observation time budget exhausted"); + *truncated = true; + return; + } visited_identities.insert(identity); } walk_element_contents( @@ -608,6 +771,9 @@ unsafe fn walk_element( complete, max_elements, max_depth, + deadline, + observation_budget, + web_area_seen, ); } @@ -627,6 +793,9 @@ unsafe fn walk_element_contents( complete: &mut bool, max_elements: usize, max_depth: usize, + deadline: Option<Instant>, + observation_budget: &mut Option<ObservationWalkBudget>, + web_area_seen: &mut bool, ) { if depth > max_depth { super::bindings::note_incomplete("walk", "max_depth exceeded"); @@ -644,7 +813,7 @@ unsafe fn walk_element_contents( // Messaging timeouts are per AX object, not inherited from the application // element, so every descendant must be bounded before any attribute read. - set_messaging_timeout(element); + set_messaging_timeout(element, deadline); let role_read = copy_string_attr_with_status(element, "AXRole"); if role_read.complete @@ -658,6 +827,7 @@ unsafe fn walk_element_contents( } *complete &= role_read.complete && role_read.value.is_some(); let role = role_read.value.unwrap_or_else(|| "AXUnknown".into()); + *web_area_seen |= role == "AXWebArea"; let in_web_content = in_web_content || is_web_content_role(&role); @@ -684,6 +854,9 @@ unsafe fn walk_element_contents( complete, max_elements, max_depth, + deadline, + observation_budget, + web_area_seen, ); CFRelease(child as CFTypeRef); } @@ -855,6 +1028,9 @@ unsafe fn walk_element_contents( complete, max_elements, max_depth, + deadline, + observation_budget, + web_area_seen, ); CFRelease(child as CFTypeRef); } @@ -1026,6 +1202,9 @@ unsafe fn walk_element_contents( complete, max_elements, max_depth, + deadline, + observation_budget, + web_area_seen, ); CFRelease(child as CFTypeRef); } @@ -1263,6 +1442,101 @@ fn leading_indent_depth(line: &str) -> usize { #[cfg(test)] mod tests { + #[test] + fn observation_deadline_preserves_completed_prefix_and_read_status() { + use super::*; + unsafe { + let element = AXUIElementCreateApplication(std::process::id() as i32); + assert!(!element.is_null()); + for reads_ok in [true, false] { + let mut nodes = vec![AXNode { + role: "AXButton".into(), + ..Default::default() + }]; + let mut lines = vec![(0, "completed prefix".into())]; + let mut visited = 1; + let mut truncated = false; + let mut complete = reads_ok; + let mut budget = Some(ObservationWalkBudget { + started: Some(Instant::now() - Duration::from_secs(2)), + exhausted: false, + }); + super::super::bindings::reset_incomplete_notes(); + walk_element( + element, + 0, + None, + false, + WalkMode::Legacy, + &mut HashSet::new(), + &mut nodes, + &mut lines, + &mut 0, + &mut visited, + &mut truncated, + &mut complete, + 2000, + 25, + None, + &mut budget, + &mut false, + ); + assert!(truncated); + assert_eq!(complete, reads_ok); + assert_eq!(visited, 1); + assert_eq!(nodes.len(), 1); + assert_eq!(lines, vec![(0, "completed prefix".into())]); + assert!(super::super::bindings::take_incomplete_notes() + .iter() + .any(|note| note.contains("observation time budget exhausted"))); + } + CFRelease(element as CFTypeRef); + } + } + + #[test] + fn recording_deadline_stops_before_native_read_and_preserves_partial_tree() { + use super::*; + let mut nodes = vec![AXNode { + role: "AXButton".into(), + ..Default::default() + }]; + let mut lines = vec![(0, "completed prefix".into())]; + let mut visited = 1; + let mut truncated = false; + let mut complete = true; + super::super::bindings::reset_incomplete_notes(); + unsafe { + walk_element( + std::ptr::null_mut(), + 0, + None, + false, + WalkMode::Legacy, + &mut HashSet::new(), + &mut nodes, + &mut lines, + &mut 0, + &mut visited, + &mut truncated, + &mut complete, + 2000, + 25, + Some(Instant::now()), + &mut None, + &mut false, + ); + } + assert!(truncated); + assert!(!complete); + assert_eq!(visited, 1); + assert_eq!(nodes.len(), 1); + assert_eq!(lines, vec![(0, "completed prefix".into())]); + assert!(super::super::bindings::take_incomplete_notes() + .iter() + .any(|note| note.contains("recording time budget exhausted"))); + } + #[test] fn traversal_limits_do_not_hide_read_failures() { for (depth, visited) in [(26, 0), (0, 2000)] { @@ -1286,6 +1560,9 @@ mod tests { &mut complete, 2000, 25, + None, + &mut None, + &mut false, ); } assert!(truncated); diff --git a/packages/cua-driver/rust/crates/platform-macos/src/capture.rs b/packages/cua-driver/rust/crates/platform-macos/src/capture.rs index 4b84f9fb199..85542a8f266 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/capture.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/capture.rs @@ -23,6 +23,9 @@ //! //! Compatibility fallback: `screencapture -l <windowID> -x -o <file>` when //! the native path errors or returns empty bytes. +//! Known off-Space windows first try a single-window SkyLight backing-store +//! image. Current-Space and unknown windows use it only after the public chain +//! fails. Both native backends share one worker and a three-second budget. //! //! ## Display capture //! @@ -237,7 +240,7 @@ fn screenshot_window_bytes_shell(window_id: u32) -> anyhow::Result<Vec<u8>> { let capture = SecureCapturePath::new("window.png")?; let tmp_path = capture.file.to_string_lossy().into_owned(); - let output = Command::new("screencapture") + let output = Command::new("/usr/sbin/screencapture") .args([ "-l", &window_id.to_string(), @@ -366,7 +369,7 @@ const WINDOW_PLAN_CACHE_CAPACITY: usize = 32; /// walk's own 20-second bound. The shell fallback retains its prior behavior. const WINDOW_CAPTURE_NATIVE_TIMEOUT: Duration = Duration::from_secs(3); -/// Permit at most one ScreenCaptureKit worker. If an Apple completion callback +/// Permit at most one native window-capture worker. If an Apple completion callback /// never fires, the timed-out worker keeps this permit, and every later request /// goes straight to the compatibility fallback instead of leaking more /// blocked threads. @@ -413,29 +416,34 @@ where T: Send + 'static, F: FnOnce() -> anyhow::Result<T> + Send + 'static, { + if timeout.is_zero() { + anyhow::bail!("native window capture budget exhausted"); + } let permit = gate .try_acquire() - .ok_or_else(|| anyhow::anyhow!("ScreenCaptureKit capture already in flight"))?; + .ok_or_else(|| anyhow::anyhow!("native window capture already in flight"))?; let (sender, receiver) = mpsc::sync_channel(1); std::thread::Builder::new() - .name("cua-sck-window-capture".into()) + .name("cua-native-window-capture".into()) .spawn(move || { let result = work(); drop(permit); let _ = sender.send(result); }) - .map_err(|error| anyhow::anyhow!("failed to start ScreenCaptureKit worker: {error}"))?; + .map_err(|error| { + anyhow::anyhow!("failed to start native window capture worker: {error}") + })?; match receiver.recv_timeout(timeout) { Ok(result) => result, Err(mpsc::RecvTimeoutError::Timeout) => { anyhow::bail!( - "ScreenCaptureKit capture timed out after {} ms", + "native window capture timed out after {} ms", timeout.as_millis() ) } Err(mpsc::RecvTimeoutError::Disconnected) => { - anyhow::bail!("ScreenCaptureKit worker ended without a result") + anyhow::bail!("native window capture worker ended without a result") } } } @@ -849,12 +857,10 @@ fn screenshot_window_bytes_sck_inner(window_id: u32) -> anyhow::Result<Vec<u8>> /// Bound the dependency's synchronous callback wrappers without permitting an /// unbounded number of detached workers. Ownership of the gate permit and all /// ScreenCaptureKit/CF objects stays on the worker until it really returns. -fn screenshot_window_bytes_sck(window_id: u32) -> anyhow::Result<Vec<u8>> { - run_native_capture_worker( - native_capture_gate(), - WINDOW_CAPTURE_NATIVE_TIMEOUT, - move || screenshot_window_bytes_sck_inner(window_id), - ) +fn screenshot_window_bytes_sck(window_id: u32, timeout: Duration) -> anyhow::Result<Vec<u8>> { + run_native_capture_worker(native_capture_gate(), timeout, move || { + screenshot_window_bytes_sck_inner(window_id) + }) .map_err(|error| { anyhow::anyhow!( "ScreenCaptureKit capture failed for window {window_id}: {error:#}; \ @@ -866,15 +872,150 @@ fn screenshot_window_bytes_sck(window_id: u32) -> anyhow::Result<Vec<u8>> { /// Capture a window by its `window_id` (CGWindowID). /// Returns raw PNG bytes or an error. /// -/// Tries ScreenCaptureKit first; falls back to the `screencapture` CLI on -/// native error or empty output. +/// Off-Space windows try the backing store first; others prefer SCK and shell. pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> { - capture_window_with_backends(window_id, screenshot_window_bytes_sck, |wid| { - if current_window_capture_identity(wid)?.requires_display_crop { - anyhow::bail!("shell capture cannot preserve attached window {wid} geometry"); + let deadline = Instant::now() + WINDOW_CAPTURE_NATIVE_TIMEOUT; + let off_space = crate::windows::window_on_current_space_by_id(window_id) == Some(false); + capture_window_with_space_route( + window_id, + off_space, + |wid| { + capture_window_with_backends( + wid, + |wid| { + screenshot_window_bytes_sck( + wid, + deadline.saturating_duration_since(Instant::now()), + ) + }, + |wid| { + if current_window_capture_identity(wid)?.requires_display_crop { + anyhow::bail!( + "shell capture cannot preserve attached window {wid} geometry" + ); + } + screenshot_window_bytes_shell(wid) + }, + ) + }, + |wid| { + run_native_capture_worker( + native_capture_gate(), + deadline.saturating_duration_since(Instant::now()), + move || screenshot_window_bytes_skylight(wid), + ) + }, + ) +} + +fn capture_window_with_space_route<P, S>( + window_id: u32, + off_space: bool, + public: P, + skylight: S, +) -> anyhow::Result<Vec<u8>> +where + P: FnOnce(u32) -> anyhow::Result<Vec<u8>>, + S: FnOnce(u32) -> anyhow::Result<CaptureIdentityValidation>, +{ + if off_space { + let private_error = match skylight(window_id) { + Ok(CaptureIdentityValidation::Changed(_)) => { + anyhow::bail!("window {window_id} changed identity during backing-store capture") + } + Ok(CaptureIdentityValidation::Matched(bytes)) if !bytes.is_empty() => return Ok(bytes), + Ok(CaptureIdentityValidation::Matched(_)) => { + anyhow::anyhow!("SkyLight capture produced empty bytes") + } + Err(error) => error, + }; + return public(window_id).map_err(|public_error| anyhow::anyhow!( + "window {window_id} is on another Space; SkyLight capture: {private_error:#}; {public_error:#}" + )); + } + match public(window_id) { + Ok(bytes) => Ok(bytes), + Err(public_error) => match skylight(window_id) { + Ok(CaptureIdentityValidation::Matched(bytes)) if !bytes.is_empty() => Ok(bytes), + Ok(CaptureIdentityValidation::Matched(_)) => Err(anyhow::anyhow!( + "{public_error:#}; SkyLight capture produced empty bytes" + )), + Ok(CaptureIdentityValidation::Changed(_)) => Err(anyhow::anyhow!( + "{public_error:#}; window {window_id} changed identity during backing-store capture" + )), + Err(private_error) => Err(anyhow::anyhow!( + "{public_error:#}; SkyLight capture: {private_error:#}" + )), + }, + } +} + +fn screenshot_window_bytes_skylight(window_id: u32) -> anyhow::Result<CaptureIdentityValidation> { + let identity = current_window_capture_identity(window_id)?; + if identity.requires_display_crop { + anyhow::bail!("backing-store capture cannot preserve attached window {window_id} geometry"); + } + capture_then_validate_identity( + identity, + || { + let image = crate::input::skylight::hw_capture_window(window_id).ok_or_else(|| { + anyhow::anyhow!("SkyLight returned no image for window {window_id}") + })?; + core_cgimage_to_png(&image) + }, + || current_window_capture_identity(window_id), + ) +} + +fn core_cgimage_to_png(image: &core_graphics::image::CGImage) -> anyhow::Result<Vec<u8>> { + use core_graphics::base::{kCGBitmapByteOrder32Big, kCGImageAlphaPremultipliedLast}; + use core_graphics::color_space::CGColorSpace; + use core_graphics::context::CGContext; + use core_graphics::geometry::{CGPoint, CGRect, CGSize}; + + let w = checked_image_dim(image.width(), "CGImage width")?; + let h = checked_image_dim(image.height(), "CGImage height")?; + let mut context = CGContext::create_bitmap_context( + None, + w as usize, + h as usize, + 8, + w as usize * 4, + &CGColorSpace::create_device_rgb(), + kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big, + ); + context.draw_image( + CGRect::new( + &CGPoint::new(0.0, 0.0), + &CGSize::new(f64::from(w), f64::from(h)), + ), + image, + ); + let mut rgba = context.data().to_vec(); + if rgba.len() != w as usize * h as usize * 4 { + anyhow::bail!("backing-store bitmap length {} != {w}*{h}*4", rgba.len()); + } + if !unpremultiply_rgba(&mut rgba) { + anyhow::bail!("backing-store image is fully transparent (no window content)"); + } + cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h) +} + +fn unpremultiply_rgba(rgba: &mut [u8]) -> bool { + let mut visible = false; + for pixel in rgba.chunks_exact_mut(4) { + let alpha = u16::from(pixel[3]); + if alpha == 0 { + continue; } - screenshot_window_bytes_shell(wid) - }) + visible = true; + if alpha < 255 { + for channel in &mut pixel[..3] { + *channel = ((u16::from(*channel) * 255 + alpha / 2) / alpha).min(255) as u8; + } + } + } + visible } /// Capture a window by its `window_id` (CGWindowID). @@ -892,7 +1033,7 @@ pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> { let capture = SecureCapturePath::new("display.png")?; let tmp_path = capture.file.to_string_lossy().into_owned(); - let output = Command::new("screencapture") + let output = Command::new("/usr/sbin/screencapture") .args(["-x", &tmp_path]) .output()?; @@ -994,6 +1135,93 @@ mod tests { use std::rc::Rc; use std::sync::Arc; + #[test] + fn space_route_prefers_the_appropriate_backend_and_never_falls_back_after_identity_change() { + let png = vec![1, 2, 3]; + assert_eq!( + capture_window_with_space_route( + 42, + true, + |_| panic!("off-Space backing store succeeded"), + |wid| { + assert_eq!(wid, 42); + Ok(CaptureIdentityValidation::Matched(png.clone())) + }, + ) + .unwrap(), + png + ); + assert_eq!( + capture_window_with_space_route( + 42, + false, + |_| Ok(png.clone()), + |_| panic!("current-Space public capture succeeded"), + ) + .unwrap(), + png + ); + assert_eq!( + capture_window_with_space_route( + 42, + true, + |_| Ok(png.clone()), + |_| anyhow::bail!("symbol missing"), + ) + .unwrap(), + png + ); + let changed = WindowCaptureIdentity::new(99, 0, 0.0, 0.0, 800.0, 600.0); + let error = capture_window_with_space_route( + 42, + true, + |_| panic!("must not recapture a replacement window"), + |_| Ok(CaptureIdentityValidation::Changed(changed)), + ) + .err() + .expect("identity change refuses the whole request"); + assert!(error.to_string().contains("changed identity")); + } + + #[test] + fn backing_store_png_preserves_pixels_and_rejects_a_transparent_image() { + use core_graphics::base::{kCGBitmapByteOrder32Big, kCGImageAlphaPremultipliedLast}; + use core_graphics::color_space::CGColorSpace; + use core_graphics::context::CGContext; + + let mut context = CGContext::create_bitmap_context( + None, + 2, + 1, + 8, + 8, + &CGColorSpace::create_device_rgb(), + kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big, + ); + context + .data() + .copy_from_slice(&[255, 0, 0, 255, 64, 32, 0, 128]); + let image = context.create_image().unwrap(); + let png = core_cgimage_to_png(&image).unwrap(); + let pixels = image::load_from_memory(&png).unwrap().into_rgba8(); + assert_eq!(pixels.dimensions(), (2, 1)); + assert_eq!(pixels.as_raw(), &[255, 0, 0, 255, 128, 64, 0, 128]); + context.data().fill(0); + let error = core_cgimage_to_png(&context.create_image().unwrap()).unwrap_err(); + assert!(error.to_string().contains("fully transparent")); + } + + #[test] + fn exhausted_native_budget_never_starts_a_worker() { + static GATE: NativeCaptureGate = NativeCaptureGate::new(); + let error = run_native_capture_worker::<(), _>(&GATE, Duration::ZERO, || { + panic!("exhausted budget cannot dispatch native work") + }) + .unwrap_err(); + assert!(error.to_string().contains("budget exhausted")); + assert!(GATE.try_acquire().is_some()); + } + #[test] fn native_window_capture_short_circuits_shell_fallback() { let png = cua_driver_core::image_utils::encode_rgba_to_png(&[0, 0, 0, 255], 1, 1) diff --git a/packages/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs b/packages/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs index a99fe59823f..21f036745a4 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs @@ -598,16 +598,14 @@ unsafe fn run_appkit(_cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs let win_h = screen_frame.size.height; // NSScreen.backingScaleFactor is the most direct source of truth — it's // what AppKit will use for the layer's native backing surface anyway. - // Fall back to the CG estimator (pixel mode width ÷ logical bounds) when + // Fall back to the CG estimator (current-mode pixels ÷ points) when // the AppKit call returns a non-positive value, since downstream paint // math divides by this and a 0.0 would zero out the cursor. let mut backing_scale: f64 = msg_send![main_screen, backingScaleFactor]; if backing_scale.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { - use core_graphics::display::{CGDisplayBounds, CGMainDisplayID}; + use core_graphics::display::CGMainDisplayID; let display_id = CGMainDisplayID(); - let bounds = CGDisplayBounds(display_id); - backing_scale = - crate::tools::get_screen_size::get_backing_scale(display_id, bounds.size.width as i64); + backing_scale = crate::tools::get_screen_size::get_backing_scale(display_id); if backing_scale.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { backing_scale = 1.0; } diff --git a/packages/cua-driver/rust/crates/platform-macos/src/document_edit.rs b/packages/cua-driver/rust/crates/platform-macos/src/document_edit.rs new file mode 100644 index 00000000000..9501fc5cb86 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-macos/src/document_edit.rs @@ -0,0 +1,276 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Keep a document's edited state in step with accessibility text writes. +//! +//! An `AXValue` write, or an `AXSelectedText` insert, changes an `NSTextView`'s +//! text without going through its edit model: no undo entry is registered and +//! the `NSDocument` is not marked edited. TextEdit then never autosaves the +//! change, and closing or saving keeps the old content on disk (Bench v2 +//! DS-07). Key events do go through the edit model. +//! +//! After an AX write that changed the text of a document window (one with an +//! `AXDocument` URL), this sends one net-zero edit as key events (a space and +//! a backspace), which registers the change with the document without +//! changing its content, and checks that the text did not move. There is no +//! accessibility readback of the edited state: `AXEdited` on the close button +//! stays false for autosave-in-place documents even after a typed edit (VM +//! check, TextEdit, 11 Oct 2026); TextEdit's autosave writing the new text is +//! the evidence. + +use std::time::Duration; + +use crate::ax::bindings::{ + copy_element_attr, copy_string_attr, AXUIElementRef, AXUIElementSetAttributeValue, +}; + +/// What the document's edited mark showed after an AX text write. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DocumentEdited { + /// The element is not in a document window, so there is nothing to save. + NotADocument, + /// The driver sent a net-zero key edit and the text did not move. + Marked, + /// The key edit could not be sent to this window, or the text moved. A + /// save may keep the old content. + NotMarked, +} + +impl DocumentEdited { + /// A sentence for the tool result, or nothing when there is nothing to say. + pub fn note(self) -> &'static str { + match self { + DocumentEdited::NotADocument => "", + DocumentEdited::Marked => { + " A net-zero key edit (space, backspace) was sent so the document registers the \ + change and saves it; an accessibility write alone does not." + } + DocumentEdited::NotMarked => { + " Warning: the document is not marked edited, so saving or closing it may keep \ + the old content. Type the text with key events instead (click the field, then \ + type_text), or edit it in the foreground." + } + } + } + + /// The structured value, `None` when the element is not in a document. + pub fn structured(self) -> Option<bool> { + match self { + DocumentEdited::NotADocument => None, + DocumentEdited::NotMarked => Some(false), + DocumentEdited::Marked => Some(true), + } + } +} + +use crate::ax::bindings::AXValueCreate; +const AX_VALUE_CF_RANGE: i32 = 4; + +#[repr(C)] +struct CfRange { + location: isize, + length: isize, +} + +/// Whether `element` sits in a document window (its window has an +/// `AXDocument` URL). +/// +/// # Safety +/// `element` must be a live `AXUIElementRef`. +unsafe fn in_document_window(element: AXUIElementRef) -> bool { + let Some(window) = copy_element_attr(element, "AXWindow") else { + return false; + }; + let document = copy_string_attr(window, "AXDocument"); + core_foundation::base::CFRelease(window as _); + document.is_some_and(|url| !url.is_empty()) +} + +/// Whether the app's key window (its `AXFocusedWindow`) is the element's +/// window, so pid-routed key events reach this text and not another window. +/// +/// # Safety +/// `element` must be a live `AXUIElementRef`. +unsafe fn keys_reach_element_window(pid: i32, element: AXUIElementRef) -> bool { + let Some(window) = copy_element_attr(element, "AXWindow") else { + return false; + }; + let own = crate::ax::bindings::ax_get_window_id(window); + core_foundation::base::CFRelease(window as _); + own.is_some() && crate::ax::bindings::focused_window_id_of_pid(pid) == own +} + +/// After an AX text write on `element` (in `pid`) that changed its value, make +/// sure its document is marked edited. See the module docs. +/// +/// # Safety +/// `element` must be a live `AXUIElementRef` for the duration of the call. +/// `caret_to_end` puts the caret at the end first (after replacing the whole +/// value); otherwise the edit happens at the caret, which an insert leaves +/// collapsed just after the inserted text. +pub unsafe fn ensure_marked_after_ax_write( + pid: i32, + element: AXUIElementRef, + caret_to_end: bool, +) -> DocumentEdited { + if !in_document_window(element) { + return DocumentEdited::NotADocument; + } + if keys_reach_element_window(pid, element) { + return send_net_zero_edit(pid, element, caret_to_end); + } + // With several windows open, the app's key window is another document, + // and pid-routed keys go there. Make this window key inside the app + // through its focus record (no activation, no raise; the user's window + // keeps key status), send the net-zero edit, and hand key back. The + // app's AXFocusedWindow does not follow a focus record, so the proof is + // the document's own modified flag, read through the app's scripting + // interface (only when Automation is already allowed). + let Some(own) = element_window_id(element) else { + return DocumentEdited::NotMarked; + }; + let Some(path) = document_path(element) else { + return DocumentEdited::NotMarked; + }; + let Some(bundle_id) = crate::apps::bundle_id_for_pid(pid) else { + return DocumentEdited::NotMarked; + }; + // A document that already has unsaved changes autosaves its whole text, + // the accessibility write included; no keys are needed. + match crate::apple_events::document_modified(&bundle_id, &path) { + Some(true) => return DocumentEdited::Marked, + // Without the readback the edit cannot be proven; send nothing. + None => return DocumentEdited::NotMarked, + Some(false) => {} + } + let previous = crate::ax::bindings::focused_window_id_of_pid(pid); + if !crate::input::skylight::activate_without_raise(pid, own) { + return DocumentEdited::NotMarked; + } + std::thread::sleep(Duration::from_millis(120)); + let sent = send_net_zero_edit(pid, element, caret_to_end); + if let Some(previous) = previous { + crate::input::skylight::activate_without_raise(pid, previous); + } + if sent == DocumentEdited::Marked + && crate::apple_events::document_modified(&bundle_id, &path) == Some(true) + { + DocumentEdited::Marked + } else { + DocumentEdited::NotMarked + } +} + +/// The file behind the element's document window. +/// +/// # Safety +/// `element` must be a live `AXUIElementRef`. +unsafe fn document_path(element: AXUIElementRef) -> Option<std::path::PathBuf> { + let window = copy_element_attr(element, "AXWindow")?; + let url = copy_string_attr(window, "AXDocument"); + core_foundation::base::CFRelease(window as _); + crate::apple_events::file_url_to_path(&url?) +} + +/// Type a space and a backspace into `element` and check the text did not +/// move. +unsafe fn send_net_zero_edit( + pid: i32, + element: AXUIElementRef, + caret_to_end: bool, +) -> DocumentEdited { + if caret_to_end { + caret_to_end_of(element); + } + if crate::input::ax_actions::focus_element(element as usize).is_err() + || !crate::input::ax_actions::is_element_focused(pid, element as usize) + { + return DocumentEdited::NotMarked; + } + let Some(before) = copy_string_attr(element, "AXValue") else { + return DocumentEdited::NotMarked; + }; + let Some((location, 0)) = crate::tools::select_text::read_range(element) else { + return DocumentEdited::NotMarked; + }; + let mut units = before.encode_utf16().collect::<Vec<_>>(); + if location > units.len() { + return DocumentEdited::NotMarked; + } + units.insert(location, u16::from(b' ')); + let Ok(inserted) = String::from_utf16(&units) else { + return DocumentEdited::NotMarked; + }; + if crate::input::keyboard::press_key(pid, "space", &[]).is_err() + || !await_value(element, &inserted) + { + return DocumentEdited::NotMarked; + } + if crate::input::keyboard::press_key(pid, "delete", &[]).is_ok() + && await_value(element, &before) + { + DocumentEdited::Marked + } else { + DocumentEdited::NotMarked + } +} + +unsafe fn await_value(element: AXUIElementRef, expected: &str) -> bool { + let deadline = std::time::Instant::now() + Duration::from_millis(150); + loop { + if copy_string_attr(element, "AXValue").as_deref() == Some(expected) { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(5)); + } +} + +/// The CGWindowID of the window `element` sits in. +/// +/// # Safety +/// `element` must be a live `AXUIElementRef`. +unsafe fn element_window_id(element: AXUIElementRef) -> Option<u32> { + let window = copy_element_attr(element, "AXWindow")?; + let id = crate::ax::bindings::ax_get_window_id(window); + core_foundation::base::CFRelease(window as _); + id +} + +/// Collapse the selection to the end of the text. +/// +/// # Safety +/// `element` must be a live `AXUIElementRef`. +unsafe fn caret_to_end_of(element: AXUIElementRef) { + let len = copy_string_attr(element, "AXValue") + .map(|v| v.encode_utf16().count()) + .unwrap_or(0); + let range = CfRange { + location: len as isize, + length: 0, + }; + let value = AXValueCreate(AX_VALUE_CF_RANGE, &range as *const CfRange as *const _); + if !value.is_null() { + use core_foundation::{base::TCFType, string::CFString}; + let attr = CFString::new("AXSelectedTextRange"); + let _ = AXUIElementSetAttributeValue(element, attr.as_concrete_TypeRef(), value as _); + core_foundation::base::CFRelease(value as _); + } +} + +#[cfg(test)] +mod tests { + use super::DocumentEdited; + + #[test] + fn only_a_document_left_unedited_gets_a_warning() { + assert_eq!(DocumentEdited::NotADocument.note(), ""); + assert!(DocumentEdited::Marked.note().contains("net-zero key edit")); + assert!(DocumentEdited::NotMarked + .note() + .contains("may keep the old content")); + assert_eq!(DocumentEdited::NotADocument.structured(), None); + assert_eq!(DocumentEdited::Marked.structured(), Some(true)); + assert_eq!(DocumentEdited::NotMarked.structured(), Some(false)); + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/focus_guard.rs b/packages/cua-driver/rust/crates/platform-macos/src/focus_guard.rs index 70aa6062343..bb24bb44f19 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/focus_guard.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/focus_guard.rs @@ -41,15 +41,11 @@ //! //! 1. A "is target already frontmost?" check so we don't arm a useless //! self→self suppressor (matches Swift's `isTargetFrontmost` guard). -//! 2. A 50ms post-action sleep that gives any in-flight focus-grab -//! reflex time to fire and be observed by the suppressor before the -//! lease is dropped. Matches Swift's `Task.sleep(nanoseconds: 50ms)`. +//! 2. A key-focus guard sharing the target's synthetic-focus observer. //! 3. A static `origin` label for tracing — call sites pass a short //! string like `"click.AXPress"` so leaked leases / late-firing //! observers can be traced back to the caller. -use std::time::Duration; - use crate::apps; use crate::focus_steal; @@ -65,9 +61,8 @@ use crate::focus_steal; /// - `origin` — short static label for tracing, e.g. `"click.AXPress"`. /// - `f` — the action to run with suppression armed. /// -/// Returns whatever `f` returns. Drops the lease ~50ms after `f` -/// resolves so any in-flight reflex activation is observed before -/// suppression ends. +/// Returns as soon as `f` completes; key-focus protection keeps its bounded +/// observer tail without delaying the action result. /// /// If `target_pid` is already the frontmost app (no point fighting /// ourselves) or `prior_frontmost` is `None` (no app to restore to), @@ -107,15 +102,11 @@ where None }; + let _key_focus = target_pid + .filter(|_| should_arm) + .and_then(crate::key_focus_guard::protect); let result = f().await; - // Post-action settle — give the reactive observer time to fire on - // any side-effect activation before we drop the lease. Matches - // Swift's 50ms sleep in `FocusGuard.withFocusSuppressed`. - if _lease.is_some() { - tokio::time::sleep(Duration::from_millis(50)).await; - } - // _lease drops here (RAII end_suppression). result } @@ -180,7 +171,7 @@ mod tests { /// observe the dispatcher state through the public API here — /// the focus_steal module's own tests cover the lease lifecycle — /// but we exercise the codepath to make sure it compiles and the - /// 50ms sleep doesn't deadlock. + /// guard lifetime does not deadlock. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn arms_lease_when_target_differs_from_prior() { let n = with_focus_suppressed(Some(123), Some(456), "test.armed", || async { 99 }).await; diff --git a/packages/cua-driver/rust/crates/platform-macos/src/focus_steal.rs b/packages/cua-driver/rust/crates/platform-macos/src/focus_steal.rs index c1978b78ef2..b22b6f80ad3 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/focus_steal.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/focus_steal.rs @@ -532,6 +532,10 @@ fn handle_activation(dispatcher: &Arc<Dispatcher>, note: &objc2_foundation::NSNo pid as i32 }; + if crate::key_focus_guard::user_switched_recently() { + dispatcher.allow_user_activation(activated_pid); + return; + } let restore_pids = dispatcher.snapshot_matches(activated_pid); for pid in restore_pids { restore_focus(pid); diff --git a/packages/cua-driver/rust/crates/platform-macos/src/input/app_focus.rs b/packages/cua-driver/rust/crates/platform-macos/src/input/app_focus.rs index 07d05ac4cc4..ba28cddc72c 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/input/app_focus.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/input/app_focus.rs @@ -102,6 +102,20 @@ pub(crate) struct FocusLease { } impl FocusLease { + pub(crate) fn wait_until_monitored(&self, timeout: std::time::Duration) -> bool { + let deadline = std::time::Instant::now() + timeout; + while self.is_current() { + if self.process.lock().is_ok_and(|entry| entry.monitored) { + return true; + } + if std::time::Instant::now() >= deadline { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + false + } + pub(crate) fn is_current(&self) -> bool { process_stamp(self.pid) == Some(self.stamp) && registry().lock().is_ok_and(|entries| { @@ -345,6 +359,14 @@ unsafe extern "C" fn observe_event( } return event; } + let pid = match process.lock() { + Ok(entry) => entry.pid, + Err(_) => return event, + }; + let guarded = crate::key_focus_guard::observe_event(pid, event_type, event); + if guarded.is_null() { + return guarded; + } if matches!(event_type, 13 | 21) && !event.is_null() { objc2::rc::autoreleasepool(|_| { use objc2::{class, msg_send}; @@ -396,7 +418,7 @@ fn start_monitor(pid: i32, process: Arc<Mutex<ProcessFocus>>) { let context = Arc::as_ptr(&process) as *mut c_void; // Observe only activation/process notifications. No keyboard or // pointer contents are collected by the per-process focus monitor. - let tap = create(pid, 0, 1, (1 << 13) | (1 << 21), observe_event, context); + let tap = create(pid, 0, 0, (1 << 13) | (1 << 21), observe_event, context); if tap.is_null() { return; } @@ -568,7 +590,7 @@ mod tests { (Preparation::Activate, 13), (Preparation::ReturnKeyFocus, 21), ] { - let events = super::super::app_pointer::focus_events(preparation, 1234, None).unwrap(); + let events = super::super::app_pointer::focus_events(preparation, 1234).unwrap(); let event = events[0].as_ptr() as *mut c_void; assert_eq!( unsafe { observe_event(std::ptr::null_mut(), kind, event, context) }, diff --git a/packages/cua-driver/rust/crates/platform-macos/src/input/app_pointer.rs b/packages/cua-driver/rust/crates/platform-macos/src/input/app_pointer.rs index 4698ab4707c..5143401f42a 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/input/app_pointer.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/input/app_pointer.rs @@ -66,7 +66,6 @@ fn native_event( pub(super) fn focus_events( preparation: super::app_focus::Preparation, window_id: u32, - activation_point: Option<((f64, f64), (f64, f64))>, ) -> anyhow::Result<Vec<CGEvent>> { use super::app_focus::Preparation; let (kind, subtype, window_number, flags) = match preparation { @@ -86,24 +85,7 @@ pub(super) fn focus_events( CFRetain(raw as CFTypeRef); Ok(CGEvent::from_ptr(raw as _)) })?; - let mut events = vec![notification]; - if preparation == Preparation::Activate { - if let Some((point, local)) = activation_point { - for (kind, number) in [(1, 1), (2, 2)] { - events.push(native_event( - kind, - 0, - 1, - number, - point, - local, - window_id, - CGEventFlags::empty(), - )?); - } - } - } - Ok(events) + Ok(vec![notification]) } fn next_event_number() -> isize { @@ -182,38 +164,20 @@ mod tests { } #[test] - fn activation_uses_notification_then_paired_native_activation_click() { - let events = focus_events( - super::super::app_focus::Preparation::Activate, - 1234, - Some(((620.0, 350.0), (20.0, 50.0))), - ) - .unwrap(); - assert_eq!(events.len(), 3); + fn background_activation_never_adds_a_mouse_click() { + let events = focus_events(super::super::app_focus::Preparation::Activate, 1234).unwrap(); + assert_eq!(events.len(), 1); assert_eq!( unsafe { CGEventGetType(events[0].as_ptr() as *mut c_void) }, 13 ); - for (index, event) in events.iter().enumerate().skip(1) { - assert_eq!(event.get_integer_value_field(0), index as i64); - assert_eq!(event.get_integer_value_field(1), 1); - assert_eq!(event.get_integer_value_field(7), 3); - assert_eq!(event.get_integer_value_field(91), 1234); - assert_eq!(event.get_flags(), CGEventFlags::empty()); - } - assert_eq!(events[1].get_type() as u32, 1); - assert_eq!(events[2].get_type() as u32, 2); assert_eq!(events[0].get_flags().bits(), 0xc0000); } #[test] - fn focus_return_never_clicks_activation_point() { - let events = focus_events( - super::super::app_focus::Preparation::ReturnKeyFocus, - 1234, - Some(((620.0, 350.0), (20.0, 50.0))), - ) - .unwrap(); + fn background_focus_return_never_adds_a_mouse_click() { + let events = + focus_events(super::super::app_focus::Preparation::ReturnKeyFocus, 1234).unwrap(); assert_eq!(events.len(), 1); assert_eq!( unsafe { CGEventGetType(events[0].as_ptr() as *mut c_void) }, diff --git a/packages/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/packages/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs index 8d61dca575e..88fbac22e93 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs @@ -240,7 +240,7 @@ pub(super) fn with_global_modifier_keys<T>( let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; - let mut active_flags = CGEventFlags::CGEventFlagNull; + let mut active_flags = held_flags(); let mut pressed = Vec::new(); for modifier in modifiers { @@ -282,11 +282,25 @@ fn post_global_key( ) -> anyhow::Result<()> { let event = CGEvent::new_keyboard_event(source.clone(), key_code, key_down) .map_err(|_| anyhow::anyhow!("CGEvent keyboard event creation failed"))?; - event.set_flags(flags); + event.set_flags(key_event_flags(key_code, flags)); event.post(tap); Ok(()) } +fn key_event_flags(key_code: u16, modifiers: CGEventFlags) -> CGEventFlags { + // Preserve key structure without inheriting physical modifiers from the + // event source. Fn (63) is a modifier, not an intrinsically marked key. + let intrinsic = match key_code { + 123..=126 => CGEventFlags::CGEventFlagSecondaryFn | CGEventFlags::CGEventFlagNumericPad, + 64 | 71 | 79 | 80 | 96..=101 | 103 | 105..=107 | 109 | 111 | 113..=122 => { + CGEventFlags::CGEventFlagSecondaryFn + } + 65 | 67 | 69 | 75 | 76 | 78 | 81..=89 | 91 | 92 => CGEventFlags::CGEventFlagNumericPad, + _ => CGEventFlags::CGEventFlagNull, + }; + modifiers | intrinsic +} + fn release_global_modifiers( source: &CGEventSource, pressed: &[(u16, CGEventFlags)], @@ -606,10 +620,7 @@ fn post_key(pid: i32, key_code: u16, key_down: bool, flags: CGEventFlags) -> any .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; let event = CGEvent::new_keyboard_event(source, key_code, key_down) .map_err(|_| anyhow::anyhow!("CGEvent::new_keyboard_event failed"))?; - // HIDSystemState can inherit physically held modifiers. Always overwrite - // the event flags, including the empty case, so an unrelated Shift/Caps - // state cannot leak into a targeted key press. - event.set_flags(flags); + event.set_flags(key_event_flags(key_code, flags)); post_keyboard_event(pid, &event); Ok(()) } @@ -624,7 +635,7 @@ fn post_key_no_auth( .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; let event = CGEvent::new_keyboard_event(source, key_code, key_down) .map_err(|_| anyhow::anyhow!("CGEvent::new_keyboard_event failed"))?; - event.set_flags(flags); + event.set_flags(key_event_flags(key_code, flags)); let event_ptr = event.as_ptr() as *mut std::ffi::c_void; // attach_auth_message = false → IOHIDPostEvent path → NSMenu fires if !crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false) { @@ -739,6 +750,39 @@ mod tests { use super::*; use core_graphics::event::CGEventType; + #[test] + fn named_key_flags_match_native_structure_without_inheriting_modifiers() { + let source = CGEventSource::new(CGEventSourceStateID::Private).unwrap(); + let structural = CGEventFlags::CGEventFlagSecondaryFn | CGEventFlags::CGEventFlagNumericPad; + for code in 0..128 { + let down = CGEvent::new_keyboard_event(source.clone(), code, true).unwrap(); + let up = CGEvent::new_keyboard_event(source.clone(), code, false).unwrap(); + // A modifier's down-only bit is state, not key structure. + let intrinsic = down.get_flags() & up.get_flags() & structural; + for owned in [ + CGEventFlags::CGEventFlagNull, + CGEventFlags::CGEventFlagControl, + CGEventFlags::CGEventFlagShift | CGEventFlags::CGEventFlagControl, + CGEventFlags::CGEventFlagSecondaryFn, + CGEventFlags::CGEventFlagCommand | CGEventFlags::CGEventFlagAlternate, + ] { + assert_eq!( + key_event_flags(code, owned), + intrinsic | owned, + "code {code}" + ); + } + } + assert_eq!( + key_event_flags(63, flags_for_codes(&[63])), + CGEventFlags::CGEventFlagSecondaryFn + ); + assert_eq!( + key_event_flags(63, flags_for_codes(&[])), + CGEventFlags::CGEventFlagNull + ); + } + #[test] fn synthesized_line_breaks_coalesce_crlf_and_preserve_unicode() { assert_eq!( @@ -964,3 +1008,41 @@ mod tests { assert!(!is_screen_sharing_bundle_id("com.microsoft.rdc.macos")); } } + +pub struct DesktopKeyboard; +impl cua_driver_core::desktop_keyboard::DesktopKeyboard for DesktopKeyboard { + fn resolve(&self, key: &str) -> Result<u32, String> { + key_name_to_code(key) + .map(u32::from) + .map_err(|error| error.to_string()) + } + fn transition(&self, code: u32, down: bool, held: &[u32]) -> Result<(), String> { + let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) + .map_err(|_| "Could not create a keyboard event source".to_owned())?; + let event = CGEvent::new_keyboard_event(source, code as u16, down) + .map_err(|_| "Could not create a keyboard event".to_owned())?; + event.set_flags(key_event_flags(code as u16, flags_for_codes(held))); + event.post(core_graphics::event::CGEventTapLocation::HID); + Ok(()) + } +} + +fn flags_for_codes(codes: &[u32]) -> CGEventFlags { + codes + .iter() + .fold(CGEventFlags::CGEventFlagNull, |flags, code| { + flags + | match code { + 55 | 54 => CGEventFlags::CGEventFlagCommand, + 56 | 60 => CGEventFlags::CGEventFlagShift, + 58 | 61 => CGEventFlags::CGEventFlagAlternate, + 59 | 62 => CGEventFlags::CGEventFlagControl, + 63 => CGEventFlags::CGEventFlagSecondaryFn, + _ => CGEventFlags::CGEventFlagNull, + } + }) +} + +pub(super) fn held_flags() -> CGEventFlags { + flags_for_codes(&cua_driver_core::desktop_keyboard::held_codes()) +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/packages/cua-driver/rust/crates/platform-macos/src/input/mouse.rs index f442ce191fe..ef540ffc82c 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/input/mouse.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/input/mouse.rs @@ -17,6 +17,66 @@ use core_graphics::{ }; use foreign_types::ForeignType; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum MousePostMode { + Both, + SkyLightPreferred, + PublicOnly, +} + +#[derive(Clone, Copy)] +pub enum WindowClickDelivery { + Background, + Foreground, +} + +impl WindowClickDelivery { + pub fn from_foreground(foreground: bool) -> Self { + if foreground { + Self::Foreground + } else { + Self::Background + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PostRoute { + SkyLight, + PublicPid, + Both, +} + +fn post_route(mode: MousePostMode, skylight_available: bool) -> PostRoute { + match mode { + MousePostMode::Both => PostRoute::Both, + MousePostMode::SkyLightPreferred if skylight_available => PostRoute::SkyLight, + MousePostMode::SkyLightPreferred | MousePostMode::PublicOnly => PostRoute::PublicPid, + } +} + +fn dispatch_mouse_event(mode: MousePostMode, pid: i32, event: &CGEvent) { + let available = crate::input::skylight::is_available(); + #[cfg(test)] + let available = post_sink::availability().unwrap_or(available); + let route = post_route(mode, available); + #[cfg(test)] + if post_sink::record(route, event) { + return; + } + let ptr = event.as_ptr() as *mut std::ffi::c_void; + match route { + PostRoute::Both => { + crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false); + event.post_to_pid(pid as libc::pid_t); + } + PostRoute::SkyLight => { + crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false); + } + PostRoute::PublicPid => event.post_to_pid(pid as libc::pid_t), + } +} + /// Left-click at `(x, y)` screen coordinates, posted to `pid`. /// /// Window-local coordinates for backgrounded targets: if `window_local` is @@ -29,7 +89,7 @@ pub fn click_at_xy( count: usize, modifiers: &[&str], ) -> anyhow::Result<()> { - click_at_xy_inner(pid, x, y, None, None, count, modifiers) + click_at_xy_inner(pid, x, y, None, None, count, modifiers, MousePostMode::Both) } /// Screen-absolute click posted to the GLOBAL HID tap (`CGEventTapLocation::HID`), @@ -201,6 +261,7 @@ pub fn scroll_wheel_desktop( return Err(anyhow::anyhow!("CGEventCreateScrollWheelEvent2 failed")); } let event = unsafe { CGEvent::from_ptr(event_ref) }; + event.set_flags(super::keyboard::held_flags()); event.post(CGEventTapLocation::HID); std::thread::sleep(std::time::Duration::from_millis(30)); } @@ -234,9 +295,7 @@ extern "C" { fn CGAssociateMouseAndMouseCursorPosition(connected: bool) -> i32; } -/// Like `click_at_xy` but also stamps window-local `(wx, wy)` onto the event. -/// When `wid` is provided, additionally stamps Chromium routing fields -/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery. +/// Post a foreground window-targeted click once through the public pid API. // The flattened arguments mirror the native event fields used by existing callers. #[allow(clippy::too_many_arguments)] pub fn click_at_xy_with_window_local( @@ -249,7 +308,46 @@ pub fn click_at_xy_with_window_local( count: usize, modifiers: &[&str], ) -> anyhow::Result<()> { - click_at_xy_inner(pid, x, y, Some((wx, wy)), Some(wid), count, modifiers) + click_at_xy_with_window_delivery( + pid, + x, + y, + wx, + wy, + wid, + count, + modifiers, + WindowClickDelivery::Foreground, + ) +} + +#[allow(clippy::too_many_arguments)] +pub fn click_at_xy_with_window_delivery( + pid: i32, + x: f64, + y: f64, + wx: f64, + wy: f64, + wid: u32, + count: usize, + modifiers: &[&str], + delivery: WindowClickDelivery, +) -> anyhow::Result<()> { + match delivery { + WindowClickDelivery::Background => { + click_at_xy_chromium(pid, x, y, wx, wy, wid, count, modifiers) + } + WindowClickDelivery::Foreground => click_at_xy_inner( + pid, + x, + y, + Some((wx, wy)), + Some(wid), + count, + modifiers, + MousePostMode::PublicOnly, + ), + } } fn click_at_xy_inner( @@ -260,6 +358,7 @@ fn click_at_xy_inner( wid: Option<u32>, count: usize, modifiers: &[&str], + mode: MousePostMode, ) -> anyhow::Result<()> { use std::time::{SystemTime, UNIX_EPOCH}; @@ -282,7 +381,15 @@ fn click_at_xy_inner( // mouseMoved so an AppKit NSButton / NSView hit-tests the down at the // right point. Without it the synthetic mouseDown on a backgrounded // AppKit control is silently ignored. - post_mouse_moved_primer(pid, &source, point, window_local, wid, click_group_id); + post_mouse_moved_primer_with_mode( + pid, + &source, + point, + window_local, + wid, + click_group_id, + mode, + ); std::thread::sleep(std::time::Duration::from_millis(12)); for pair_index in 0..count { @@ -295,11 +402,9 @@ fn click_at_xy_inner( CGMouseButton::Left, ) .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - down.set_flags(flags); - } + down.set_flags(flags); - post_mouse_event( + post_mouse_event_with_mode( pid, &down, window_local, @@ -308,6 +413,7 @@ fn click_at_xy_inner( click_state, 0, 3, + mode, ); // 28 ms down→up gap: an NSButton's mouseDown enters a modal // tracking loop that polls for the matching mouseUp; too tight a @@ -321,11 +427,9 @@ fn click_at_xy_inner( CGMouseButton::Left, ) .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - up.set_flags(flags); - } + up.set_flags(flags); - post_mouse_event( + post_mouse_event_with_mode( pid, &up, window_local, @@ -334,6 +438,7 @@ fn click_at_xy_inner( click_state, 0, 3, + mode, ); if count > 1 { @@ -383,8 +488,7 @@ pub fn prepare_background_pixel_click(pid: i32, wid: u32) -> bool { /// - f58 = constant click-group ID across all events (gesture coalescing) /// - `CGEventSetWindowLocation` per-event (window-local point) /// -/// Uses both SkyLight `SLEventPostToPid` AND `CGEvent::post_to_pid` (belt+suspenders) -/// for AppKit / Catalyst target coverage. +/// Posts through SkyLight once, with the public API used only when the SPI is absent. // The flattened arguments mirror the native event fields used by existing callers. #[allow(clippy::too_many_arguments)] pub fn click_at_xy_chromium( @@ -435,16 +539,11 @@ pub fn click_at_xy_chromium( } set(58, click_group_id); // click-group ID (gesture coalescing) crate::input::skylight::set_window_location(ptr, local.0, local.1); - if flags != CGEventFlags::CGEventFlagNull { - event.set_flags(flags); - } + event.set_flags(flags); }; - // Belt+suspenders: SkyLight path for Chromium/Catalyst + public API for AppKit. let post = |event: &CGEvent| { - let ptr = event.as_ptr() as *mut std::ffi::c_void; - crate::input::skylight::post_to_pid(pid as libc::pid_t, ptr, false); - event.post_to_pid(pid as libc::pid_t); + dispatch_mouse_event(MousePostMode::SkyLightPreferred, pid, event); }; // Step 1: mouseMoved at target (phase=2, clickState=0). @@ -641,9 +740,7 @@ where let from_pt = CGPoint::new(from_x, from_y); let down = CGEvent::new_mouse_event(source.clone(), down_type, from_pt, cg_button) .map_err(|_| anyhow::anyhow!("drag mouseDown failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - down.set_flags(flags); - } + down.set_flags(flags); post_mouse_event( pid, &down, @@ -668,9 +765,7 @@ where let drag_pt = CGPoint::new(ix, iy); let drag = CGEvent::new_mouse_event(source.clone(), dragged_type, drag_pt, cg_button) .map_err(|_| anyhow::anyhow!("drag mouseDragged failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - drag.set_flags(flags); - } + drag.set_flags(flags); post_mouse_event(pid, &drag, il, wid, click_group_id, 1, button_number, 0); observe(ix, iy); if step_delay_ms > 0 { @@ -685,9 +780,7 @@ where let to_pt = CGPoint::new(to_x, to_y); let up = CGEvent::new_mouse_event(source.clone(), up_type, to_pt, cg_button) .map_err(|_| anyhow::anyhow!("drag mouseUp failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - up.set_flags(flags); - } + up.set_flags(flags); post_mouse_event(pid, &up, to_local, wid, click_group_id, 1, button_number, 0); if foreground_release { // A frontmost Chromium surface can consume PID-routed down/move @@ -816,9 +909,7 @@ where cg_button, ) .map_err(|_| anyhow::anyhow!("foreground drag mouseDown failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - down.set_flags(flags); - } + down.set_flags(flags); down.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1); down.post(CGEventTapLocation::HID); std::thread::sleep(std::time::Duration::from_millis(16)); @@ -830,9 +921,7 @@ where let event = CGEvent::new_mouse_event(source.clone(), dragged_type, CGPoint::new(x, y), cg_button) .map_err(|_| anyhow::anyhow!("foreground drag mouseDragged failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - event.set_flags(flags); - } + event.set_flags(flags); event.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1); post(&event); observe(x, y); @@ -843,9 +932,7 @@ where let up = CGEvent::new_mouse_event(source, up_type, CGPoint::new(to_x, to_y), cg_button) .map_err(|_| anyhow::anyhow!("foreground drag mouseUp failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - up.set_flags(flags); - } + up.set_flags(flags); up.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1); post(&up); // The foreground wrapper restores the previous app immediately after this @@ -866,68 +953,37 @@ pub enum DragButton { /// Middle-click at `(x, y)` with optional modifier keys. /// /// Posts an `OtherMouseDown` / `OtherMouseUp` pair with `CGMouseButton::Center` -/// to the target pid through the same SkyLight + public-API postBoth path the -/// left- and right-click primitives use. Window-local stamping mirrors +/// to the target pid once through SkyLight, with a public fallback when absent. +/// Window-local stamping mirrors /// `right_click_at_xy_with_window_local`. pub fn middle_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> { - middle_click_at_xy_inner(pid, x, y, None, modifiers) + right_click_at_xy_inner(pid, x, y, None, None, modifiers, CGMouseButton::Center) } -/// Like `middle_click_at_xy` but stamps the window-local `(wx, wy)` point. +#[allow(clippy::too_many_arguments)] pub fn middle_click_at_xy_with_window_local( pid: i32, x: f64, y: f64, wx: f64, wy: f64, + wid: u32, modifiers: &[&str], ) -> anyhow::Result<()> { - middle_click_at_xy_inner(pid, x, y, Some((wx, wy)), modifiers) -} - -fn middle_click_at_xy_inner( - pid: i32, - x: f64, - y: f64, - window_local: Option<(f64, f64)>, - modifiers: &[&str], -) -> anyhow::Result<()> { - let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) - .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; - let point = CGPoint::new(x, y); - let flags = parse_modifier_flags(modifiers); - - let down = CGEvent::new_mouse_event( - source.clone(), - CGEventType::OtherMouseDown, - point, + right_click_at_xy_inner( + pid, + x, + y, + Some((wx, wy)), + Some(wid), + modifiers, CGMouseButton::Center, ) - .map_err(|_| anyhow::anyhow!("middle mouse down failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - down.set_flags(flags); - } - post_mouse_event(pid, &down, window_local, None, None, 1, 2, 3); - std::thread::sleep(std::time::Duration::from_millis(16)); - - let up = CGEvent::new_mouse_event( - source, - CGEventType::OtherMouseUp, - point, - CGMouseButton::Center, - ) - .map_err(|_| anyhow::anyhow!("middle mouse up failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - up.set_flags(flags); - } - post_mouse_event(pid, &up, window_local, None, None, 1, 2, 3); - - Ok(()) } /// Right-click at `(x, y)` with optional modifier keys (no window routing). pub fn right_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow::Result<()> { - right_click_at_xy_inner(pid, x, y, None, None, modifiers) + right_click_at_xy_inner(pid, x, y, None, None, modifiers, CGMouseButton::Right) } /// Like `right_click_at_xy` but stamps `CGEventSetWindowLocation` with the @@ -948,7 +1004,15 @@ pub fn right_click_at_xy_with_window_local( wid: u32, modifiers: &[&str], ) -> anyhow::Result<()> { - right_click_at_xy_inner(pid, x, y, Some((wx, wy)), Some(wid), modifiers) + right_click_at_xy_inner( + pid, + x, + y, + Some((wx, wy)), + Some(wid), + modifiers, + CGMouseButton::Right, + ) } fn right_click_at_xy_inner( @@ -958,8 +1022,13 @@ fn right_click_at_xy_inner( window_local: Option<(f64, f64)>, wid: Option<u32>, modifiers: &[&str], + button: CGMouseButton, ) -> anyhow::Result<()> { use std::time::{SystemTime, UNIX_EPOCH}; + let (down_type, up_type, button_number) = match button { + CGMouseButton::Center => (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, 2), + _ => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, 1), + }; let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; @@ -975,35 +1044,47 @@ fn right_click_at_xy_inner( // Prime cursor-tracking state at the target so AppKit hit-tests the // right-down at the right NSView (same rationale as the left path). - post_mouse_moved_primer(pid, &source, point, window_local, wid, click_group_id); + post_mouse_moved_primer_with_mode( + pid, + &source, + point, + window_local, + wid, + click_group_id, + MousePostMode::SkyLightPreferred, + ); std::thread::sleep(std::time::Duration::from_millis(12)); - let down = CGEvent::new_mouse_event( - source.clone(), - CGEventType::RightMouseDown, - point, - CGMouseButton::Right, - ) - .map_err(|_| anyhow::anyhow!("right mouse down failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - down.set_flags(flags); - } - // button_number = 1 (right). Stamping 0 here routes the event as a left - // button-number on the receiving side even though the type is rightMouseDown. - post_mouse_event(pid, &down, window_local, wid, click_group_id, 1, 1, 3); + let down = CGEvent::new_mouse_event(source.clone(), down_type, point, button) + .map_err(|_| anyhow::anyhow!("right mouse down failed"))?; + down.set_flags(flags); + post_mouse_event_with_mode( + pid, + &down, + window_local, + wid, + click_group_id, + 1, + button_number, + 3, + MousePostMode::SkyLightPreferred, + ); std::thread::sleep(std::time::Duration::from_millis(28)); - let up = CGEvent::new_mouse_event( - source, - CGEventType::RightMouseUp, - point, - CGMouseButton::Right, - ) - .map_err(|_| anyhow::anyhow!("right mouse up failed"))?; - if flags != CGEventFlags::CGEventFlagNull { - up.set_flags(flags); - } - post_mouse_event(pid, &up, window_local, wid, click_group_id, 1, 1, 3); + let up = CGEvent::new_mouse_event(source, up_type, point, button) + .map_err(|_| anyhow::anyhow!("right mouse up failed"))?; + up.set_flags(flags); + post_mouse_event_with_mode( + pid, + &up, + window_local, + wid, + click_group_id, + 1, + button_number, + 3, + MousePostMode::SkyLightPreferred, + ); Ok(()) } @@ -1044,6 +1125,31 @@ pub(super) fn post_mouse_event( click_state: i64, button_number: i64, subtype: i64, +) { + post_mouse_event_with_mode( + pid, + event, + window_local, + wid, + click_group_id, + click_state, + button_number, + subtype, + MousePostMode::Both, + ); +} + +#[allow(clippy::too_many_arguments)] +fn post_mouse_event_with_mode( + pid: i32, + event: &CGEvent, + window_local: Option<(f64, f64)>, + wid: Option<u32>, + click_group_id: Option<i64>, + click_state: i64, + button_number: i64, + subtype: i64, + mode: MousePostMode, ) { let event_ptr = event.as_ptr() as *mut std::ffi::c_void; @@ -1070,13 +1176,7 @@ pub(super) fn post_mouse_event( // Always stamp f40 = target pid (Chromium synthetic-event filter). crate::input::skylight::set_integer_field(event_ptr, 40, pid as i64); - // SkyLight path: activity-monitor tickle → reaches Catalyst/Chromium. - // Mouse events skip the auth-message envelope (Swift: attachAuthMessage: false). - crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, false); - - // Public path: delivers to AppKit targets where SkyLight mouse drops. - // Belt+suspenders — both fire unconditionally (matches Swift `postBoth`). - event.post_to_pid(pid as libc::pid_t); + dispatch_mouse_event(mode, pid, event); } /// Post a stamped `mouseMoved` to `pid` at `point` before a down/up pair. @@ -1097,6 +1197,27 @@ fn post_mouse_moved_primer( window_local: Option<(f64, f64)>, wid: Option<u32>, click_group_id: Option<i64>, +) { + post_mouse_moved_primer_with_mode( + pid, + source, + point, + window_local, + wid, + click_group_id, + MousePostMode::Both, + ); +} + +#[allow(clippy::too_many_arguments)] +fn post_mouse_moved_primer_with_mode( + pid: i32, + source: &CGEventSource, + point: CGPoint, + window_local: Option<(f64, f64)>, + wid: Option<u32>, + click_group_id: Option<i64>, + mode: MousePostMode, ) { if let Ok(mv) = CGEvent::new_mouse_event( source.clone(), @@ -1104,7 +1225,7 @@ fn post_mouse_moved_primer( point, CGMouseButton::Left, ) { - post_mouse_event(pid, &mv, window_local, wid, click_group_id, 0, 0, 3); + post_mouse_event_with_mode(pid, &mv, window_local, wid, click_group_id, 0, 0, 3, mode); } } @@ -1241,3 +1362,143 @@ pub(super) fn parse_modifier_flags(modifiers: &[&str]) -> CGEventFlags { } flags } + +#[cfg(test)] +mod post_sink { + use super::*; + use std::cell::RefCell; + + #[derive(Clone, Debug, PartialEq)] + pub(super) struct Posted { + pub event: u32, + pub route: PostRoute, + pub routing: [i64; 4], + pub button: i64, + pub flags: u64, + } + thread_local! { + static STATE: RefCell<Option<(bool, Vec<Posted>)>> = const { RefCell::new(None) }; + } + pub(super) struct Capture; + impl Capture { + pub fn start(available: bool) -> Self { + STATE.with(|state| *state.borrow_mut() = Some((available, Vec::new()))); + Self + } + pub fn events(&self) -> Vec<Posted> { + STATE.with(|state| state.borrow().as_ref().unwrap().1.clone()) + } + } + impl Drop for Capture { + fn drop(&mut self) { + STATE.with(|state| *state.borrow_mut() = None); + } + } + pub fn availability() -> Option<bool> { + STATE.with(|state| state.borrow().as_ref().map(|s| s.0)) + } + pub fn record(route: PostRoute, event: &CGEvent) -> bool { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let Some((_, events)) = state.as_mut() else { + return false; + }; + events.push(Posted { + event: event.get_type() as u32, + route, + routing: [51, 91, 92, 58].map(|field| event.get_integer_value_field(field)), + button: event.get_integer_value_field(3), + flags: event.get_flags().bits(), + }); + true + }) + } +} + +#[cfg(test)] +mod tests { + use super::post_sink::Capture; + use super::*; + + #[test] + fn window_button_clicks_keep_one_transport_and_exact_window() { + for available in [true, false] { + for middle in [true, false] { + let capture = Capture::start(available); + let click = if middle { + middle_click_at_xy_with_window_local + } else { + right_click_at_xy_with_window_local + }; + click(1, 100.0, 100.0, 10.0, 10.0, 7, &["cmd", "shift"]).unwrap(); + let events = capture.events(); + let (down, up, button) = if middle { + (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, 2) + } else { + (CGEventType::RightMouseDown, CGEventType::RightMouseUp, 1) + }; + assert_eq!( + events.iter().map(|e| e.event).collect::<Vec<_>>(), + [CGEventType::MouseMoved as u32, down as u32, up as u32] + ); + let route = if available { + PostRoute::SkyLight + } else { + PostRoute::PublicPid + }; + assert!(events + .iter() + .all(|e| e.route == route && e.routing[..3] == [7, 7, 7])); + assert_ne!(events[0].routing[3], 0); + assert!(events.iter().all(|e| e.routing[3] == events[0].routing[3])); + let flags = + (CGEventFlags::CGEventFlagCommand | CGEventFlags::CGEventFlagShift).bits(); + assert!(events[1..] + .iter() + .all(|e| e.button == button && e.flags & 0x001e_0000 == flags)); + } + } + } + + #[test] + fn window_left_double_click_posts_each_pair_once() { + for foreground in [true, false] { + for available in [true, false] { + let capture = Capture::start(available); + click_at_xy_with_window_delivery( + 1, + 100.0, + 100.0, + 10.0, + 10.0, + 7, + 2, + &[], + WindowClickDelivery::from_foreground(foreground), + ) + .unwrap(); + let events = capture.events(); + let expected = if foreground { 5 } else { 7 }; // Background adds an off-screen primer pair. + assert_eq!(events.len(), expected); + let route = if foreground || !available { + PostRoute::PublicPid + } else { + PostRoute::SkyLight + }; + assert!(events + .iter() + .all(|e| e.route == route && e.routing[..3] == [7, 7, 7])); + assert_eq!(events[0].event, CGEventType::MouseMoved as u32); + for pair in events[1..].chunks_exact(2) { + assert_eq!( + (pair[0].event, pair[1].event), + ( + CGEventType::LeftMouseDown as u32, + CGEventType::LeftMouseUp as u32 + ) + ); + } + } + } + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/packages/cua-driver/rust/crates/platform-macos/src/input/skylight.rs index 4bfbdd2ad72..23c9b3c2749 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/input/skylight.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/input/skylight.rs @@ -492,37 +492,16 @@ impl SpaceQuery { /// or triggering Space-follow. Ported from yabai's /// `window_manager_focus_window_without_raise`. /// -/// Recipe: -/// 1. `_SLPSGetFrontProcess` → capture current front PSN. -/// 2. `SLSGetWindowOwner + SLSGetConnectionPSN` → target PSN, with -/// `GetProcessForPID(target_pid)` as an older-system fallback. -/// 3. Post 248-byte defocus record to front PSN (`bytes[0x8a] = 0x02`). -/// 4. Post 248-byte focus record to target PSN (`bytes[0x8a] = 0x01`, -/// `bytes[0x3c..0x3f]` = `target_wid` little-endian). -/// -/// Deliberately skips `SLPSSetFrontProcessWithOptions` — see the Swift -/// reference `FocusWithoutRaise.swift` for why omitting it keeps -/// Chromium's user-activation gate open. -/// -/// Returns `true` when all SPIs resolved and both posts succeeded. +/// Post a target-only synthetic focus record. The user's front app never +/// receives a deactivation notice (#4943). Global foreground and Space stay put. +/// Returns true when the target PSN resolves and the focus post succeeds. pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool { let post_fn = match post_event_record_to_fn() { Some(f) => f, None => return false, }; - let get_front = match get_front_process_fn() { - Some(f) => f, - None => return false, - }; - // 8-byte PSN buffers (two UInt32s). - let mut prev_psn = [0u8; 8]; let mut target_psn = [0u8; 8]; - let ok_prev = unsafe { get_front(prev_psn.as_mut_ptr() as *mut c_void) } == 0; - if !ok_prev { - return false; - } - if !get_process_psn_for_window(target_wid, target_pid, &mut target_psn) { return false; } @@ -537,15 +516,11 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool { buf[0x3E] = ((target_wid >> 16) & 0xFF) as u8; buf[0x3F] = ((target_wid >> 24) & 0xFF) as u8; - // Step 3: defocus previous front. - buf[0x8A] = 0x02; - let defocus_ok = unsafe { post_fn(prev_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0 }; - - // Step 4: focus target. + // Focus only the target. buf[0x8A] = 0x01; let focus_ok = unsafe { post_fn(target_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0 }; - defocus_ok && focus_ok + focus_ok } // ── NSMenu shortcut activation ──────────────────────────────────────────────── @@ -654,72 +629,25 @@ fn post_focus_preparation( window_id: u32, preparation: super::app_focus::Preparation, ) -> anyhow::Result<()> { - let point = if preparation == super::app_focus::Preparation::Activate { - window_activation_point(pid, window_id) - } else { - None - }; - let events = super::app_pointer::focus_events(preparation, window_id, point)?; + let post = post_event_record_to_fn() + .ok_or_else(|| anyhow::anyhow!("background key-window record transport unavailable"))?; + let mut target_psn = [0u8; 8]; + if !get_process_psn_for_window(window_id, pid, &mut target_psn) { + anyhow::bail!("background key-window target PSN unavailable"); + } + let events = super::app_pointer::focus_events(preparation, window_id)?; for event in &events { super::app_pointer::post(pid, event); } - Ok(()) -} - -fn window_activation_point(pid: pid_t, window_id: u32) -> Option<((f64, f64), (f64, f64))> { - use crate::ax::bindings::*; - use core_foundation::base::{CFGetTypeID, CFRelease, CFTypeRef, TCFType}; - use core_foundation::string::CFString; - unsafe { - let app = AXUIElementCreateApplication(pid); - if app.is_null() { - return None; + // AppKit activation alone leaves keyWindow unset. Address only the native + // target window; changing WindowServer's front process would take user input. + for kind in [0x01, 0x02] { + let record = make_key_window_record(window_id, kind); + if unsafe { post(target_psn.as_ptr() as *const c_void, record.as_ptr()) } != 0 { + anyhow::bail!("background key-window record was rejected"); } - let mut target = None; - for candidate in copy_ax_windows(app) { - if target.is_none() && ax_get_window_id(candidate) == Some(window_id) { - target = Some(candidate); - } else { - CFRelease(candidate as CFTypeRef); - } - } - CFRelease(app as CFTypeRef); - let window = target?; - let frame = element_screen_rect(window); - let mut value: CFTypeRef = std::ptr::null(); - let status = AXUIElementCopyAttributeValue( - window, - CFString::new("AXActivationPoint").as_concrete_TypeRef(), - &mut value, - ); - CFRelease(window as CFTypeRef); - if value.is_null() { - return None; - } - let mut point = core_graphics::geometry::CGPoint::new(0.0, 0.0); - let valid = status == kAXErrorSuccess - && CFGetTypeID(value) == AXValueGetTypeID() - && AXValueGetType(value as AXValueRef) == kAXValueCGPointType - && AXValueGetValue( - value as AXValueRef, - kAXValueCGPointType, - &mut point as *mut _ as *mut c_void, - ); - CFRelease(value); - let [x, y, width, height] = frame?; - if !valid - || ![point.x, point.y, x, y, width, height] - .iter() - .all(|v| v.is_finite()) - || point.x < x - || point.x > x + width - || point.y < y - || point.y > y + height - { - return None; - } - Some(((point.x, point.y), (point.x - x, point.y - y))) } + Ok(()) } /// Make `target_pid` and `target_wid` WindowServer-frontmost and leave them @@ -731,6 +659,7 @@ fn window_activation_point(pid: pid_t, window_id: u32) -> Option<((f64, f64), (f /// Returns `true` only when the target PSN resolved and WindowServer accepted /// `SLPSSetFrontProcessWithOptions`. pub fn set_front_process_persistently(target_pid: libc::pid_t, target_wid: u32) -> bool { + crate::window_change_detector::note_intentional_activation(); let Some(set_front) = set_front_process_fn() else { return false; }; @@ -825,6 +754,7 @@ pub fn with_foreground_assist( target_wid: u32, body: impl FnOnce() -> anyhow::Result<()>, ) -> anyhow::Result<bool> { + crate::window_change_detector::note_intentional_activation(); let set_front = match set_front_process_fn() { Some(f) => f, None => { @@ -915,6 +845,7 @@ pub fn with_foreground_hid_activation( target_wid: u32, action: impl FnOnce() -> anyhow::Result<()>, ) -> anyhow::Result<()> { + crate::window_change_detector::note_intentional_activation(); let set_front = set_front_process_fn() .ok_or_else(|| anyhow::anyhow!("foreground HID delivery is unavailable"))?; @@ -986,6 +917,7 @@ pub fn with_menu_shortcut_activation( target_wid: u32, action: impl FnOnce() -> anyhow::Result<()>, ) -> anyhow::Result<bool> { + crate::window_change_detector::note_intentional_activation(); let set_front = match set_front_process_fn() { Some(f) => f, None => { @@ -1077,3 +1009,177 @@ mod tests { )); } } + +type HwCaptureWindowListFn = unsafe extern "C" fn(c_int, *mut u32, c_int, u32) -> *const c_void; + +fn hw_capture_window_list_fn() -> Option<HwCaptureWindowListFn> { + static SYM: OnceLock<Option<HwCaptureWindowListFn>> = OnceLock::new(); + *SYM.get_or_init(|| { + find_sym(b"SLSHWCaptureWindowList\0") + .or_else(|| find_sym(b"CGSHWCaptureWindowList\0")) + .map(|p| unsafe { as_fn(p) }) + }) +} + +pub(crate) fn hw_capture_window(window_id: u32) -> Option<core_graphics::image::CGImage> { + use core_foundation::{ + array::CFArray, + base::{CFGetTypeID, CFRetain, CFTypeRef, TCFType}, + }; + use foreign_types::ForeignType; + + extern "C" { + fn CGImageGetTypeID() -> core_foundation::base::CFTypeID; + } + + let capture = hw_capture_window_list_fn()?; + let cid = c_int::try_from(main_connection_id()?).ok()?; + let mut window = window_id; + // Ignore global Space clipping and request backing-store (Retina) pixels. + let images = unsafe { capture(cid, &mut window, 1, (1 << 11) | (1 << 8)) }; + if images.is_null() { + return None; + } + let images: CFArray<CFTypeRef> = unsafe { CFArray::wrap_under_create_rule(images as _) }; + let image = *images.get(0)?; + if image.is_null() || unsafe { CFGetTypeID(image) } != unsafe { CGImageGetTypeID() } { + return None; + } + // The returned array owns the image; keep an independent retain. + unsafe { + CFRetain(image); + Some(core_graphics::image::CGImage::from_ptr(image as *mut _)) + } +} + +// ── Key-focus theft ─────────────────────────────────────────────────────────── + +/// `CGError SLPSStealKeyFocusReturningID(uint32_t cid, uint32_t options, int32_t *theft_id)` +type StealKeyFocusReturningIdFn = unsafe extern "C" fn(u32, u32, *mut i32) -> i32; + +/// `CGError SLPSReleaseKeyFocusWithID(int32_t theft_id)` +type ReleaseKeyFocusWithIdFn = unsafe extern "C" fn(i32) -> i32; + +fn steal_key_focus_fn() -> Option<StealKeyFocusReturningIdFn> { + static SYM: OnceLock<Option<StealKeyFocusReturningIdFn>> = OnceLock::new(); + *SYM.get_or_init(|| find_sym(b"SLPSStealKeyFocusReturningID\0").map(|p| unsafe { as_fn(p) })) +} + +fn release_key_focus_fn() -> Option<ReleaseKeyFocusWithIdFn> { + static SYM: OnceLock<Option<ReleaseKeyFocusWithIdFn>> = OnceLock::new(); + *SYM.get_or_init(|| find_sym(b"SLPSReleaseKeyFocusWithID\0").map(|p| unsafe { as_fn(p) })) +} + +/// `OSStatus SLPSGetKeyFocusProcess(ProcessSerialNumber *psn, uint32_t *connection)`. +/// Both are out-parameters; the second must be a valid pointer. +type GetKeyFocusProcessFn = unsafe extern "C" fn(*mut c_void, *mut c_void) -> i32; + +/// `OSStatus GetProcessPID(const ProcessSerialNumber *psn, pid_t *pid)` +type GetProcessPidFn = unsafe extern "C" fn(*const c_void, *mut pid_t) -> i32; + +fn get_key_focus_process_fn() -> Option<GetKeyFocusProcessFn> { + static SYM: OnceLock<Option<GetKeyFocusProcessFn>> = OnceLock::new(); + *SYM.get_or_init(|| find_sym(b"SLPSGetKeyFocusProcess\0").map(|p| unsafe { as_fn(p) })) +} + +fn get_process_pid_fn() -> Option<GetProcessPidFn> { + static SYM: OnceLock<Option<GetProcessPidFn>> = OnceLock::new(); + *SYM.get_or_init(|| find_sym(b"GetProcessPID\0").map(|p| unsafe { as_fn(p) })) +} + +/// The pid of the process WindowServer gives key focus to: normally the +/// frontmost app, or a non-activating panel (Spotlight-like launchers) or a +/// menu that took it. +pub fn key_focus_pid() -> Option<pid_t> { + let get = get_key_focus_process_fn()?; + let to_pid = get_process_pid_fn()?; + let mut psn = [0u8; 8]; + // Room for whatever the second out-parameter holds (a connection id). + let mut second = [0u64; 2]; + let err = unsafe { + get( + psn.as_mut_ptr() as *mut c_void, + second.as_mut_ptr() as *mut c_void, + ) + }; + if err != 0 || psn == [0u8; 8] { + return None; + } + let mut pid: pid_t = 0; + if unsafe { to_pid(psn.as_ptr() as *const c_void, &mut pid) } != 0 || pid <= 0 { + return None; + } + Some(pid) +} + +/// The pid of the process WindowServer has frontmost now. Unlike +/// `NSWorkspace.frontmostApplication`, this does not wait for the caller's +/// AppKit run loop to process an activation notice, so it sees a background +/// app's self-activation at once. +pub fn front_process_pid() -> Option<pid_t> { + let get_front = get_front_process_fn()?; + let to_pid = get_process_pid_fn()?; + let mut psn = [0u8; 8]; + if unsafe { get_front(psn.as_mut_ptr() as *mut c_void) } != 0 || psn == [0u8; 8] { + return None; + } + let mut pid: pid_t = 0; + if unsafe { to_pid(psn.as_ptr() as *const c_void, &mut pid) } != 0 || pid <= 0 { + return None; + } + Some(pid) +} + +/// Whether the key-focus theft SPIs resolve on this system. +pub fn key_focus_theft_available() -> bool { + steal_key_focus_fn().is_some() + && release_key_focus_fn().is_some() + && main_connection_id().is_some() +} + +/// Steal key focus for this process's own connection and return the theft id. +/// +/// WindowServer numbers thefts from one session-wide counter, so the id tells +/// the caller which id the next theft will get. AppKit uses the same call, +/// with the same options, when it opens a menu in an app that is not active. +pub fn steal_key_focus() -> Option<i32> { + let steal = steal_key_focus_fn()?; + let cid = main_connection_id()?; + let mut id = 0i32; + let err = unsafe { steal(cid, 0xC80, &mut id) }; + (err == 0 && id != 0).then_some(id) +} + +/// Release a key-focus theft by id, whichever connection made it. +/// Returns the CGError (0 on success; 1010 when no such theft is live). +pub fn release_key_focus(theft_id: i32) -> i32 { + match release_key_focus_fn() { + Some(release) => unsafe { release(theft_id) }, + None => -1, + } +} + +pub fn key_window_of_pid(pid: pid_t) -> Option<u32> { + use crate::ax::bindings::{ax_get_window_id, copy_element_attr, AXUIElementCreateApplication}; + let focused = unsafe { + let app = AXUIElementCreateApplication(pid); + if app.is_null() { + None + } else { + let window = copy_element_attr(app, "AXFocusedWindow"); + core_foundation::base::CFRelease(app as _); + window.and_then(|window| { + let wid = ax_get_window_id(window); + core_foundation::base::CFRelease(window as _); + wid + }) + } + }; + focused.or_else(|| { + crate::windows::visible_windows() + .into_iter() + .filter(|w| w.pid == pid && w.layer == 0 && w.is_on_screen) + .max_by_key(|w| w.z_index) + .map(|w| w.window_id) + }) +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs b/packages/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs new file mode 100644 index 00000000000..71adad1c8d5 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs @@ -0,0 +1,964 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Cua AI, Inc. +// Ported from upstream #4943, #4955, #4976, #5091 and #5129. +//! Keep the user's key window while a background target opens a menu. +//! +//! When an app that is not active opens a menu (a popup button, a `<select>`, +//! a context menu), AppKit's menu tracking calls +//! `SLPSStealKeyFocusReturningID`. WindowServer then tells the app that holds +//! key focus, normally the user's frontmost app, that it lost key focus, and +//! routes the keyboard to the menu until it closes. The user's window resigns +//! key and their typing stops landing in it, even though no app was activated +//! and no window was raised. +//! +//! While the driver acts on a background target, this module holds a +//! per-process event tap on the user's frontmost app. When that app is told it +//! lost key focus to a guarded target (a type-21 event, subtype `0x4000`, whose +//! field 73 is the thief's pid), the tap releases the theft by id with +//! `SLPSReleaseKeyFocusWithID` and drops the event. The user's app never sees +//! the loss; the menu stays open and is still operable through accessibility. +//! +//! Opening a menu also makes WindowServer order the menu's window group to the +//! front, which can lift the target's parent window above the user's window. +//! The guard restores the parent window order after the menu appears; it does +//! not establish uninterrupted visibility. Once the menu window is on screen +//! (and again when the action ends), if a target window sits above the user's +//! front window, the guard asks the front app to raise its current focused +//! window. It checks the exact visible window and leaves recent hardware +//! input alone. Key status does not move and the target's menu stays open. +//! +//! Theft ids come from one session-wide counter that only the thief learns. +//! The guard learns the target's id only after the tap has seen the theft: +//! it steals and releases key focus on the driver's own connection, which +//! takes the next id, and then releases the newest live theft below it. By +//! then the target's menu already holds the keyboard, so the round trip costs +//! the user nothing. Arming the guard never moves key focus: a steal on every +//! action used to route a user keystroke that arrived in that instant to the +//! driver, which dropped it. +//! +//! The driver's steal tells the current key-focus holder, the target whose +//! menu is open, that it lost key focus, and AppKit ends menu tracking on that +//! notice. So the guard also taps each guarded target and drops the notices +//! whose thief is the driver itself; the menu stays open through the lookup. + +use std::collections::HashMap; +use std::ffi::c_void; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use crate::input::skylight; + +/// How long a guarded target's tap hides the driver's own theft after an id +/// lookup starts. Outside that window the notice passes: the target may be the +/// frontmost app by then, and dropping a key-focus change there leaves AppKit +/// with a stale key window. +const LOOKUP_WINDOW: Duration = Duration::from_millis(500); +/// How long a target stays guarded after the action that armed it ends. +/// AppKit opens a pressed popup's menu from a delayed perform that runs just +/// after the accessibility call returns. +const TAIL: Duration = Duration::from_secs(3); +/// Older ids tried after the newest one when another app stole in between. +const ID_SLACK: i32 = 3; +/// Longest wait for the thief's menu window before releasing the theft. +const MENU_WAIT: Duration = Duration::from_millis(150); +/// CGWindowLevel of popup menus (`kCGPopUpMenuWindowLevel`). +const POPUP_MENU_LAYER: i32 = 101; + +/// Whether a normal-level window of `target` sits above the frontmost +/// normal-level window of `front` in the on-screen order. +fn target_above_front(target: i32, front: i32) -> bool { + // Every layer, without the Space queries of `visible_windows`: this runs + // while the raise is on screen, so it has to be quick. + let windows = crate::windows::visible_windows_any_layer(); + target_above_front_in(&windows, target, front) +} + +fn target_above_front_in(windows: &[crate::windows::WindowInfo], target: i32, front: i32) -> bool { + let top = |pid: i32| { + windows + .iter() + .filter(|w| w.pid == pid && w.layer == 0 && w.is_on_screen) + .filter(|w| w.bounds.width > 1.0 && w.bounds.height > 1.0) + .map(|w| w.z_index) + .max() + }; + match (top(target), top(front)) { + (Some(t), Some(f)) => t > f, + _ => false, + } +} + +/// Ask `front` to raise its current focused window if `target` was +/// lifted over it (see [`restore_front_if_raised`]). +pub fn restore_order_now(target: i32, front: i32) -> bool { + raise_front_window_if_covered(target, front) == Some(crate::ax::bindings::kAXErrorSuccess) +} + +/// Whether a normal-level window of `target` sits above `front`'s frontmost +/// normal-level window. +pub fn target_window_above_front(target: i32, front: i32) -> bool { + target_above_front(target, front) +} + +fn restore_front_if_raised(target: i32, front: i32) -> bool { + let Some(error) = raise_front_window_if_covered(target, front) else { + return false; + }; + RESTORED.fetch_add(1, Ordering::Relaxed); + tracing::debug!( + target, + front, + error, + "requested the user's focused window above a background target" + ); + true +} + +fn raise_front_window_if_covered(target: i32, front: i32) -> Option<crate::ax::bindings::AXError> { + use crate::ax::bindings::*; + use core_foundation::base::{CFType, CFTypeRef, TCFType}; + + if skylight::front_process_pid() != Some(front) + || user_switched_recently() + || !target_above_front(target, front) + { + return None; + } + unsafe { + let app = AXUIElementCreateApplication(front); + if app.is_null() { + return None; + } + let _app = CFType::wrap_under_create_rule(app as CFTypeRef); + if AXUIElementSetMessagingTimeout(app, 0.1) != kAXErrorSuccess { + return None; + } + let window = copy_element_attr(app, "AXFocusedWindow")?; + let _window = CFType::wrap_under_create_rule(window as CFTypeRef); + if AXUIElementSetMessagingTimeout(window, 0.1) != kAXErrorSuccess { + return None; + } + let mut pid = 0; + if AXUIElementGetPid(window, &mut pid) != kAXErrorSuccess || pid != front { + return None; + } + let mut wid = 0; + if _AXUIElementGetWindow(window, &mut wid) != kAXErrorSuccess || wid == 0 { + return None; + } + let windows = crate::windows::visible_windows_any_layer(); + if !windows.iter().any(|w| { + w.pid == front + && w.window_id == wid + && w.layer == 0 + && w.is_on_screen + && w.bounds.width > 1.0 + && w.bounds.height > 1.0 + }) || !target_above_front_in(&windows, target, front) + || skylight::front_process_pid() != Some(front) + || user_switched_recently() + { + return None; + } + Some(perform_action(window, "AXRaise")) + } +} + +/// Whether WindowServer's front process is the background target: it +/// activated itself (`activateIgnoringOtherApps:`) in answer to the action, +/// such as Chrome focusing its omnibox on cmd+L or GnuCash opening a register +/// on a double-click. No client can stop an app from activating itself. +fn target_self_activated(front_process: Option<i32>, target: i32, front: i32) -> bool { + target != front && front_process == Some(target) +} + +/// Undo a background target's self-activation: re-activate the user's app +/// and hand key focus back to its window. Reads WindowServer's front process +/// directly; `NSWorkspace` only learns of the activation once the daemon's run +/// loop processes the notice, which took about a second in Bench v2 CDB-G04. +/// Revert `target`'s self-activation right now, giving `front` its key +/// window back (see [`revert_self_activation`]). +pub fn revert_now(target: i32, front: i32) -> bool { + revert_self_activation(target, front) +} + +fn revert_self_activation(target: i32, front: i32) -> bool { + if !target_self_activated(skylight::front_process_pid(), target, front) { + return false; + } + // The user may have switched to the target themselves (a click on its + // window, or Cmd-Tab). Leave that alone. + if user_switched_recently() { + return false; + } + let started = Instant::now(); + // Make the user's key window frontmost and key again, the way the + // foreground tools front an exact window (a user-generated front request + // plus the make-key records). `NSRunningApplication.activate` from the + // daemon is only a request under cooperative activation and can be + // ignored, so it is the fallback. + let restored = skylight::key_window_of_pid(front) + .is_some_and(|wid| skylight::make_exact_window_key(front, wid)) + && skylight::front_process_pid() == Some(front); + if !restored { + use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication}; + unsafe { + if let Some(app) = NSRunningApplication::runningApplicationWithProcessIdentifier(front) + { + app.activateWithOptions(NSApplicationActivationOptions(0)); + } + } + } + REVERTED.fetch_add(1, Ordering::Relaxed); + tracing::debug!( + target, + front, + restored, + elapsed_ms = started.elapsed().as_millis() as u64, + "reverted a background target's self-activation" + ); + true +} + +/// A hardware mouse-down or modifier change within this long before an +/// activation means the user may have caused it. +const USER_SWITCH_WINDOW_S: f64 = 1.0; + +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn CGEventSourceSecondsSinceLastEventType(state: i32, event_type: u32) -> f64; +} + +/// Whether the user pressed a mouse button or a modifier key on the real +/// hardware just now. Events the driver posts to a pid do not count. +pub(crate) fn user_switched_recently() -> bool { + const HID_SYSTEM_STATE: i32 = 1; + const LEFT_MOUSE_DOWN: u32 = 1; + const RIGHT_MOUSE_DOWN: u32 = 3; + const FLAGS_CHANGED: u32 = 12; + [LEFT_MOUSE_DOWN, RIGHT_MOUSE_DOWN, FLAGS_CHANGED] + .into_iter() + .any(|kind| { + let since = unsafe { CGEventSourceSecondsSinceLastEventType(HID_SYSTEM_STATE, kind) }; + since.is_finite() && since < USER_SWITCH_WINDOW_S + }) +} + +/// How long the order stays watched after a guarded action ends. A target +/// can raise a window a moment after the call that caused it returns (a +/// banner, a sheet, or a menu that a renderer process opens later). +const ORDER_WATCH: Duration = Duration::from_millis(2500); +const ORDER_POLL: Duration = Duration::from_millis(20); +const MAX_RESTORES_PER_WATCH: u32 = 3; + +/// One target's watch after a guarded action. +#[derive(Clone, Copy)] +struct Watch { + until: Instant, + front: i32, + /// Also restore the window order (off when the user had put the target + /// above their window before the action). + order: bool, + /// [`crate::window_change_detector::intentional_epoch`] when armed. + epoch: u64, +} + +/// Keep restoring the user's window above `target`, and undoing the target's +/// self-activation, for a short while after the action. One watcher per +/// target; a later action extends it. +fn watch_order_after_action(target: i32, front: i32, order: bool, epoch: u64) { + static WATCHING: OnceLock<Mutex<HashMap<i32, Watch>>> = OnceLock::new(); + let watching = WATCHING.get_or_init(|| Mutex::new(HashMap::new())); + let watch = Watch { + until: Instant::now() + ORDER_WATCH, + front, + order, + epoch, + }; + { + let mut map = watching + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let running = map.contains_key(&target); + map.insert(target, watch); + if running { + return; + } + } + let spawned = std::thread::Builder::new() + .name("cua-order-watch".into()) + .spawn(move || { + // Some apps re-raise right after each restore (a Chromium popup + // that stays open); cap the restores so the user's app is not + // re-activated in a loop. + let mut restores = 0; + let mut reverts = 0; + loop { + std::thread::sleep(ORDER_POLL); + let current = watching + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&target) + .copied(); + let Some(current) = current else { return }; + // An activation the driver asked for since (bring_to_front, + // foreground delivery) is not reverted. + if reverts < MAX_RESTORES_PER_WATCH + && current.epoch == crate::window_change_detector::intentional_epoch() + && revert_self_activation(target, current.front) + { + reverts += 1; + } + if current.order + && restores < MAX_RESTORES_PER_WATCH + && restore_front_if_raised(target, current.front) + { + restores += 1; + } + let mut map = watching + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if map + .get(&target) + .is_none_or(|watch| Instant::now() >= watch.until) + { + map.remove(&target); + return; + } + } + }); + if spawned.is_err() { + watching + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&target); + } +} + +/// Wait until `pid` shows a menu-level window, or `budget` runs out. +fn wait_for_menu_window(pid: i32, budget: Duration) { + let deadline = Instant::now() + budget; + while Instant::now() < deadline { + // `visible_windows` lists layer 0 only; a menu is at layer 101. + if crate::windows::all_windows_any_layer() + .iter() + .any(|w| w.pid == pid && w.layer >= POPUP_MENU_LAYER && w.is_on_screen) + { + return; + } + std::thread::sleep(Duration::from_millis(1)); + } +} + +/// CGEvent type of the WindowServer focus notices an app receives. +const FOCUS_NOTICE_TYPE: u32 = 21; +/// Field holding the notice subtype. +const FIELD_SUBTYPE: u32 = 64; +/// Field holding the pid of the process that took key focus. +const FIELD_THIEF_PID: u32 = 73; +/// Subtype: "you lost key focus". +const SUBTYPE_LOST_KEY_FOCUS: i64 = 0x4000; + +const TAP_DISABLED_BY_TIMEOUT: u32 = 0xFFFF_FFFE; +const TAP_DISABLED_BY_USER_INPUT: u32 = 0xFFFF_FFFF; + +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn CGEventGetIntegerValueField(event: *mut c_void, field: u32) -> i64; +} + +/// What the guard did, for tool results and tests. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Stats { + /// Thefts by a guarded target that were released. + pub released: u64, + /// Thefts by a guarded target that could not be released. + pub missed: u64, +} + +struct Tap { + /// The process the tap sits on. + pid: i32, + _lease: crate::input::app_focus::FocusLease, +} + +#[derive(Default)] +struct State { + tap: Option<Tap>, + /// Taps on guarded targets that hide the driver's own id lookup from + /// them (see [`target_tap_callback`]). + target_taps: HashMap<i32, Tap>, + /// Guarded pid -> (live leases, end of the tail after the last lease, + /// intentional-activation epoch when last armed). + guarded: HashMap<i32, (usize, Instant, u64)>, + /// End of the window in which target taps drop the driver's own theft + /// notice (see [`release_target_theft`]). + lookup_until: Option<Instant>, +} + +fn state() -> &'static Mutex<State> { + static STATE: OnceLock<Mutex<State>> = OnceLock::new(); + STATE.get_or_init(|| Mutex::new(State::default())) +} + +fn lock() -> std::sync::MutexGuard<'static, State> { + state() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +static RELEASED: AtomicU64 = AtomicU64::new(0); +static MISSED: AtomicU64 = AtomicU64::new(0); +static RESTORED: AtomicU64 = AtomicU64::new(0); +static REVERTED: AtomicU64 = AtomicU64::new(0); + +/// Counters since the daemon started. +pub fn stats() -> Stats { + Stats { + released: RELEASED.load(Ordering::Relaxed), + missed: MISSED.load(Ordering::Relaxed), + } +} + +/// While alive, key-focus thefts by the guarded pid are handed back to the +/// user's frontmost app. Dropping it starts a short tail (see [`TAIL`]). +pub struct Lease { + pid: i32, + front: i32, + /// A target window already sat above the user's window when the action + /// started (the user arranged it so); then nothing is restored. + above_before: bool, + /// [`crate::window_change_detector::intentional_epoch`] when armed. If + /// the driver activated an app itself since, the target's activation is + /// not reverted. + epoch: u64, +} + +impl Drop for Lease { + fn drop(&mut self) { + if self.epoch == crate::window_change_detector::intentional_epoch() { + revert_self_activation(self.pid, self.front); + } + if !self.above_before { + restore_front_if_raised(self.pid, self.front); + } + watch_order_after_action(self.pid, self.front, !self.above_before, self.epoch); + let mut st = lock(); + if let Some((leases, until, _)) = st.guarded.get_mut(&self.pid) { + *leases = leases.saturating_sub(1); + *until = Instant::now() + TAIL; + } + drop(st); + schedule_idle_teardown(); + } +} + +/// Guard the user's key focus while the driver acts on `target_pid` in the +/// background. Returns `None` (no guard) when the target is the frontmost app, +/// when there is no other frontmost app, or when the SPIs or the event tap are +/// unavailable; the action then runs exactly as before. +pub fn protect(target_pid: i32) -> Option<Lease> { + let own = std::process::id() as i32; + if !skylight::key_focus_theft_available() { + return None; + } + // Tap the process that holds key focus now. That is the one WindowServer + // tells when a menu steals it, and it may be a non-activating panel + // rather than the frontmost app. Only guard when it is the frontmost app: + // otherwise a launcher-style panel would see the calibration theft. + let front = crate::apps::frontmost_pid()?; + let holder = skylight::key_focus_pid().unwrap_or(front); + if holder != front || front == target_pid || front == own { + return None; + } + // Register first, so an idle teardown cannot remove the tap right after + // ensure_tap. + { + let mut st = lock(); + let epoch = crate::window_change_detector::intentional_epoch(); + let entry = st + .guarded + .entry(target_pid) + .or_insert((0, Instant::now(), epoch)); + entry.0 += 1; + entry.1 = Instant::now() + TAIL; + entry.2 = epoch; + } + let lease = Lease { + pid: target_pid, + front, + above_before: target_above_front(target_pid, front), + epoch: crate::window_change_detector::intentional_epoch(), + }; + // Do not touch key focus here. This runs before every background action + // while the user may be typing, and any key event that reaches + // WindowServer while focus is away from their app is lost. + if !ensure_tap(front) { + return None; + } + // Arming this tap does not move key focus either. Without it, the + // driver's id lookup would close the target's menu. + if !ensure_target_tap(target_pid) { + tracing::warn!(target_pid, "key-focus guard: target event tap unavailable"); + } + Some(lease) +} + +/// Whether key-focus thefts by `pid` are background menu thefts to release. +/// Not once the driver itself has activated an app since the guard was armed +/// (bring_to_front, launch_app, foreground delivery): the target taking key +/// focus then is that activation, and releasing it would leave the target +/// active without a key window (0.35.4 gate: bring_to_front within the tail +/// of a refused background invoke_menu, then the foreground invoke_menu found +/// its key-window-gated item disabled). +fn is_guarded(pid: i32) -> bool { + let epoch = crate::window_change_detector::intentional_epoch(); + let st = lock(); + st.guarded.get(&pid).is_some_and(|&(leases, until, armed)| { + guard_applies(leases, until, armed, Instant::now(), epoch) + }) +} + +/// Pure part of [`is_guarded`], for tests. +fn guard_applies(leases: usize, until: Instant, armed: u64, now: Instant, epoch: u64) -> bool { + armed == epoch && (leases > 0 || now < until) +} + +/// Release the theft a guarded target just made. Only called once the tap +/// has seen that theft, so the target's menu holds the keyboard already: a +/// steal on the driver's own connection takes the next id, and the target's +/// theft is the newest live one below it. +fn release_target_theft() -> bool { + lock().lookup_until = Some(Instant::now() + LOOKUP_WINDOW); + let Some(own) = skylight::steal_key_focus() else { + return false; + }; + skylight::release_key_focus(own); + release_newest_theft_below(own, skylight::release_key_focus).is_some() +} + +/// Release the newest live theft older than `own`, trying [`ID_SLACK`] older +/// ids too in case another app stole in between. `release` returns 0 when the +/// id was live. Pure apart from `release`, for tests. +fn release_newest_theft_below(own: i32, mut release: impl FnMut(i32) -> i32) -> Option<i32> { + (1..=ID_SLACK + 1) + .map(|back| own.wrapping_sub(back)) + .find(|&id| id != 0 && release(id) == 0) +} + +/// What the tap does with one focus notice. Pure, for tests. +#[derive(Debug, PartialEq, Eq)] +enum NoticeAction { + Pass, + /// The driver's own theft while it looks up a target's id: swallow it. + DropOwn, + /// A guarded target stole key focus: release it, then swallow the notice. + ReleaseAndDrop, +} + +fn classify_notice( + event_type: u32, + subtype: i64, + thief_pid: i64, + own_pid: i32, + guarded: impl Fn(i32) -> bool, +) -> NoticeAction { + if event_type != FOCUS_NOTICE_TYPE || subtype != SUBTYPE_LOST_KEY_FOCUS { + return NoticeAction::Pass; + } + let Ok(thief) = i32::try_from(thief_pid) else { + return NoticeAction::Pass; + }; + if thief == own_pid { + NoticeAction::DropOwn + } else if guarded(thief) { + NoticeAction::ReleaseAndDrop + } else { + NoticeAction::Pass + } +} + +extern "C" fn tap_callback( + _proxy: *mut c_void, + event_type: u32, + event: *mut c_void, + _user_info: *mut c_void, +) -> *mut c_void { + if event_type == TAP_DISABLED_BY_TIMEOUT || event_type == TAP_DISABLED_BY_USER_INPUT { + return event; + } + if event.is_null() { + return event; + } + let (subtype, thief) = unsafe { + ( + CGEventGetIntegerValueField(event, FIELD_SUBTYPE), + CGEventGetIntegerValueField(event, FIELD_THIEF_PID), + ) + }; + match classify_notice( + event_type, + subtype, + thief, + std::process::id() as i32, + is_guarded, + ) { + NoticeAction::Pass => event, + NoticeAction::DropOwn => std::ptr::null_mut(), + NoticeAction::ReleaseAndDrop => { + // Swallow the notice now, so the user's window never resigns key, + // but release the theft only once the menu window is on screen. + // Released earlier, AppKit orders the menu's window group to the + // front, which raises the target's window over the user's. + let thief_pid = thief as i32; + let spawned = std::thread::Builder::new() + .name("cua-key-focus-release".into()) + .spawn(move || { + wait_for_menu_window(thief_pid, MENU_WAIT); + let front = lock().tap.as_ref().map(|tap| tap.pid); + if let Some(front) = front { + restore_front_if_raised(thief_pid, front); + } + if release_target_theft() { + RELEASED.fetch_add(1, Ordering::Relaxed); + tracing::debug!(thief_pid, "released a background key-focus theft"); + } else { + MISSED.fetch_add(1, Ordering::Relaxed); + tracing::warn!(thief_pid, "could not release a background key-focus theft"); + } + }); + if spawned.is_ok() { + std::ptr::null_mut() + } else { + event + } + } + } +} + +/// Make sure the tap sits on `front_pid`, the app that holds key focus. +fn ensure_tap(front_pid: i32) -> bool { + { + let mut st = lock(); + if st.tap.as_ref().is_some_and(|tap| tap.pid == front_pid) { + return true; + } + if let Some(old) = st.tap.take() { + stop_tap(old); + } + } + let Some(tap) = spawn_tap(front_pid) else { + tracing::warn!(front_pid, "key-focus guard: event tap unavailable"); + return false; + }; + if let Some(old) = lock().tap.replace(tap) { + stop_tap(old); + } + true +} + +/// Make sure a guarded target has a tap that hides the driver's own theft. +fn ensure_target_tap(target_pid: i32) -> bool { + if lock().target_taps.contains_key(&target_pid) { + return true; + } + let Some(tap) = spawn_tap(target_pid) else { + return false; + }; + if let Some(old) = lock().target_taps.insert(target_pid, tap) { + stop_tap(old); + } + true +} + +/// Tap on a guarded target: drop the "lost key focus" notice the driver's id +/// lookup sends it while its menu holds the keyboard. AppKit ends menu +/// tracking on that notice, which closed popups and context menus the action +/// had just opened. Every other event passes, including the driver's own +/// key-focus changes outside a lookup: the tap outlives the action by the +/// tail, and a foreground action that follows (bring_to_front, then +/// invoke_menu) needs AppKit to see them, or its menu items validate against +/// a stale key window (0.35.4 gate, harness_appkit_invoke_menu_live_path). +extern "C" fn target_tap_callback( + _proxy: *mut c_void, + event_type: u32, + event: *mut c_void, + _user_info: *mut c_void, +) -> *mut c_void { + if event_type == TAP_DISABLED_BY_TIMEOUT || event_type == TAP_DISABLED_BY_USER_INPUT { + return event; + } + if event.is_null() { + return event; + } + let (subtype, thief) = unsafe { + ( + CGEventGetIntegerValueField(event, FIELD_SUBTYPE), + CGEventGetIntegerValueField(event, FIELD_THIEF_PID), + ) + }; + let in_lookup = lock() + .lookup_until + .is_some_and(|until| Instant::now() < until); + if in_lookup && is_own_theft_notice(event_type, subtype, thief, std::process::id() as i32) { + std::ptr::null_mut() + } else { + event + } +} + +/// Whether a notice tells its receiver that the driver itself took key focus. +fn is_own_theft_notice(event_type: u32, subtype: i64, thief_pid: i64, own_pid: i32) -> bool { + classify_notice(event_type, subtype, thief_pid, own_pid, |_| false) == NoticeAction::DropOwn +} + +/// Install a focus-notice tap on `pid` on its own run-loop thread. +fn spawn_tap(pid: i32) -> Option<Tap> { + let lease = crate::input::app_focus::track(pid).ok()?; + if !lease.wait_until_monitored(Duration::from_millis(500)) { + return None; + } + Some(Tap { pid, _lease: lease }) +} +fn stop_tap(tap: Tap) { + drop(tap); +} + +pub(crate) fn observe_event(pid: i32, event_type: u32, event: *mut c_void) -> *mut c_void { + let (front, target) = { + let state = lock(); + ( + state.tap.as_ref().is_some_and(|tap| tap.pid == pid), + state.target_taps.contains_key(&pid), + ) + }; + if front { + tap_callback( + std::ptr::null_mut(), + event_type, + event, + std::ptr::null_mut(), + ) + } else if target { + target_tap_callback( + std::ptr::null_mut(), + event_type, + event, + std::ptr::null_mut(), + ) + } else { + event + } +} + +/// Remove the tap once nothing has been guarded for a while, so the driver +/// does not keep a tap on the user's app between tasks. +fn schedule_idle_teardown() { + let _ = std::thread::Builder::new() + .name("cua-key-focus-guard-idle".into()) + .spawn(|| { + std::thread::sleep(TAIL + Duration::from_secs(1)); + let mut st = lock(); + let now = Instant::now(); + st.guarded + .retain(|_, (leases, until, _)| *leases > 0 || now < *until); + let idle: Vec<i32> = st + .target_taps + .keys() + .filter(|pid| !st.guarded.contains_key(pid)) + .copied() + .collect(); + for pid in idle { + if let Some(tap) = st.target_taps.remove(&pid) { + stop_tap(tap); + } + } + if st.guarded.is_empty() { + if let Some(tap) = st.tap.take() { + stop_tap(tap); + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + const OWN: i32 = 100; + + #[test] + fn a_guarded_target_theft_is_released_and_swallowed() { + let action = classify_notice(21, 0x4000, 42, OWN, |pid| pid == 42); + assert_eq!(action, NoticeAction::ReleaseAndDrop); + } + + fn win(pid: i32, layer: i32, z_index: usize) -> crate::windows::WindowInfo { + crate::windows::WindowInfo { + window_id: z_index as u32, + pid, + app_name: String::new(), + title: String::new(), + bounds: crate::windows::WindowBounds { + x: 0.0, + y: 0.0, + width: 400.0, + height: 300.0, + }, + layer, + z_index, + is_on_screen: true, + current_space_id: None, + on_current_space: None, + space_ids: None, + } + } + + #[test] + fn only_the_targets_own_activation_is_reverted() { + // The target is WindowServer's front process: it activated itself. + assert!(super::target_self_activated(Some(7), 7, OWN)); + // The user's app is still in front, or a third app took it (not ours + // to undo), or nothing is known. + assert!(!super::target_self_activated(Some(OWN), 7, OWN)); + assert!(!super::target_self_activated(Some(9), 7, OWN)); + assert!(!super::target_self_activated(None, 7, OWN)); + // A target that is the user's app is never reverted. + assert!(!super::target_self_activated(Some(7), 7, 7)); + } + + #[test] + fn a_target_window_above_the_users_window_is_detected() { + // z_index: higher is closer to the front. + let raised = [win(7, 0, 3), win(42, 0, 5), win(7, 0, 4)]; + assert!(target_above_front_in(&raised, 42, 7)); + let below = [win(7, 0, 5), win(42, 0, 3)]; + assert!(!target_above_front_in(&below, 42, 7)); + // A menu-level window does not count; only normal windows are raised. + let menu_only = [win(7, 0, 4), win(42, 101, 9), win(42, 0, 2)]; + assert!(!target_above_front_in(&menu_only, 42, 7)); + // Without a window of either app there is nothing to restore. + assert!(!target_above_front_in(&[win(42, 0, 9)], 42, 7)); + } + + /// The target's tap hides only the driver's own id lookup: AppKit closes + /// an open menu when its app is told it lost key focus (0.35.3 release + /// gate: popup click, popup set_value and context menu all closed). + #[test] + fn the_target_tap_drops_only_the_drivers_own_theft_notice() { + assert!(is_own_theft_notice(21, 0x4000, OWN as i64, OWN)); + // Another app taking key focus still closes the menu, as it should. + assert!(!is_own_theft_notice(21, 0x4000, 42, OWN)); + // Other notices and event types pass. + assert!(!is_own_theft_notice(21, 0x2000, OWN as i64, OWN)); + assert!(!is_own_theft_notice(10, 0x4000, OWN as i64, OWN)); + } + + #[test] + fn an_intentional_activation_ends_the_guard() { + let now = Instant::now(); + let later = now + Duration::from_secs(3); + // Live lease or tail, same epoch: guarded. + assert!(guard_applies(1, now, 5, now, 5)); + assert!(guard_applies(0, later, 5, now, 5)); + // The driver activated an app since (bring_to_front in the tail): + // the target's key-focus change is that activation, not a menu. + assert!(!guard_applies(0, later, 5, now, 6)); + assert!(!guard_applies(1, later, 5, now, 6)); + // Tail over. + assert!(!guard_applies(0, now, 5, later, 5)); + } + + #[test] + fn the_drivers_own_theft_is_swallowed() { + assert_eq!( + classify_notice(21, 0x4000, OWN as i64, OWN, |_| false), + NoticeAction::DropOwn + ); + } + + #[test] + fn other_apps_and_other_notices_pass_through() { + // An app the driver is not acting on keeps normal macOS behaviour. + assert_eq!( + classify_notice(21, 0x4000, 7, OWN, |pid| pid == 42), + NoticeAction::Pass + ); + // "Key focus returned" and other subtypes are never touched. + assert_eq!( + classify_notice(21, 0x8000, 42, OWN, |_| true), + NoticeAction::Pass + ); + // Other event types are never touched. + assert_eq!( + classify_notice(13, 0x4000, 42, OWN, |_| true), + NoticeAction::Pass + ); + // A malformed pid passes. + assert_eq!( + classify_notice(21, 0x4000, i64::MAX, OWN, |_| true), + NoticeAction::Pass + ); + } + + #[test] + fn the_targets_theft_is_the_newest_live_one_below_the_drivers() { + // The target stole id 41 and the driver's lookup took 42. + let mut tried = Vec::new(); + let released = release_newest_theft_below(42, |id| { + tried.push(id); + if id == 41 { + 0 + } else { + 1010 + } + }); + assert_eq!(released, Some(41)); + assert_eq!(tried, vec![41]); + + // Another app stole in between and already let go: skip its id. + assert_eq!( + release_newest_theft_below(42, |id| if id == 40 { 0 } else { 1010 }), + Some(40) + ); + + // Nothing live within reach: nothing is released, and the search + // never goes further back than the slack allows. + let mut tried = Vec::new(); + let released = release_newest_theft_below(42, |id| { + tried.push(id); + 1010 + }); + assert_eq!(released, None); + assert_eq!(tried, vec![41, 40, 39, 38]); + + // Id 0 means "no theft" and is never released. + let mut tried = Vec::new(); + release_newest_theft_below(2, |id| { + tried.push(id); + 1010 + }); + assert!(!tried.contains(&0)); + } + + /// Arming the guard runs before every background action, while the user + /// may be typing. It must not steal key focus: a key event that reached + /// WindowServer in that instant went to the driver and was lost (0.35.2, + /// one keystroke in about 600 to 2500 during an agent run). + #[test] + fn arming_the_guard_never_steals_key_focus() { + let source = include_str!("key_focus_guard.rs"); + let body = |name: &str| { + let start = source + .find(&format!("pub fn {name}(")) + .unwrap_or_else(|| panic!("{name} not found")); + let end = source[start..].find("\n}\n").expect("end of fn") + start; + &source[start..end] + }; + let protect = body("protect"); + for call in ["steal_key_focus", "calibrate", "release_target_theft"] { + assert!(!protect.contains(call), "protect() must not call {call}"); + } + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/lib.rs b/packages/cua-driver/rust/crates/platform-macos/src/lib.rs index a75c1dc96f2..00a793602f5 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/lib.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/lib.rs @@ -25,6 +25,8 @@ pub mod focus_steal; #[cfg(target_os = "macos")] pub mod input; #[cfg(target_os = "macos")] +pub mod key_focus_guard; +#[cfg(target_os = "macos")] pub mod permissions; #[cfg(target_os = "macos")] pub mod pip; @@ -150,3 +152,7 @@ mod cursor_overlay_host_tests { assert!(!cursor_overlay_facility_available(false, false)); } } + +pub mod apple_events; + +pub mod document_edit; diff --git a/packages/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs b/packages/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs index 51e18fe6742..b29d52747a8 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs @@ -6,30 +6,10 @@ //! window-local screenshot-pixel coordinates so `click.png` is also written //! on AX-indexed clicks (not just pixel-addressed ones). //! -//! The element-bounds resolver needs the per-(pid, window_id) element cache, -//! which lives in `ToolState`. `tools::register_all` shares the active cache -//! here via `set_element_cache` at startup. - -use std::{ - collections::HashMap, - sync::{Arc, Mutex, OnceLock, Weak}, -}; +//! The element-bounds resolver uses the snapshot bound to the recorded action, +//! so a concurrent observation cannot replace the element used for its marker. use crate::ax::bindings::{element_screen_center, AXUIElementRef}; -use crate::ax::cache::ElementCache; - -static ELEMENT_CACHES: OnceLock<Mutex<HashMap<String, Weak<ElementCache>>>> = OnceLock::new(); - -pub fn set_element_cache(cache: Arc<ElementCache>) { - let runtime_scope = - cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into()); - let mut caches = ELEMENT_CACHES - .get_or_init(|| Mutex::new(HashMap::new())) - .lock() - .unwrap(); - caches.retain(|_, cache| cache.strong_count() > 0); - caches.insert(runtime_scope, Arc::downgrade(&cache)); -} /// Build `app_state.json` bytes for the turn folder. Walks the AX tree for /// (pid, window_id) and emits the same shape `get_window_state` returns @@ -40,7 +20,23 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve Some(w) => u32::try_from(w).ok()?, None => crate::windows::resolve_main_window_id(pid).ok()?, }; - let result = crate::ax::tree::walk_tree(pid, Some(resolved_wid), None); + let started = std::time::Instant::now(); + let result = crate::ax::tree::walk_tree_for_recording( + pid, + resolved_wid, + cua_driver_core::recording::RECORDING_AX_WALK_BUDGET, + ); + recording_tree_json(pid, resolved_wid, result, started) +} + +fn recording_tree_json( + pid: i32, + window_id: u32, + result: crate::ax::tree::TreeWalkResult, + started: std::time::Instant, +) -> Option<Vec<u8>> { + // Recording consumes the walk's extra native retains without publishing IDs. + let _bindings = crate::ax::cache::ElementCache::prepare(&result.nodes); let element_count = result .nodes .iter() @@ -48,9 +44,13 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve .count(); let payload = serde_json::json!({ "pid": pid, - "window_id": resolved_wid, + "window_id": window_id, "element_count": element_count, "tree_markdown": result.tree_markdown, + "truncated": result.truncated, + "read_complete": result.read_complete, + "incomplete_notes": result.incomplete_notes, + "walk_elapsed_ms": started.elapsed().as_millis(), }); serde_json::to_vec_pretty(&payload).ok() } @@ -59,15 +59,13 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve /// (pid, window_id). `element_screen_center` returns SCREEN points; convert /// by subtracting the window's screen origin and multiplying by the /// screenshot's pixels-per-point scale. -pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> { - let runtime_scope = - cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into()); - let cache = ELEMENT_CACHES - .get()? - .lock() - .unwrap() - .get(&runtime_scope)? - .upgrade()?; +pub fn element_window_local_xy( + window_id: u64, + pid: i64, + element_index: u32, + binding: &cua_driver_core::element_token::SnapshotBinding, +) -> Option<(f64, f64)> { + let cache = crate::ax::cache::BoundElementCache::from_binding(binding.clone()); let pid_i32 = i32::try_from(pid).ok()?; let window_id_u32 = u32::try_from(window_id).ok()?; // Retain so a concurrent get_window_state can't free the element between @@ -84,3 +82,83 @@ pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> let wy = (sy - frame.bounds.y) * frame.scale; Some((wx, wy)) } + +#[cfg(test)] +mod tests { + use super::recording_tree_json; + use crate::ax::tree::{AXNode, TreeWalkResult}; + use core_foundation::base::{CFGetRetainCount, CFRetain, CFTypeRef, TCFType}; + use core_foundation::string::CFString; + use std::time::Instant; + + fn tree(nodes: Vec<AXNode>, partial: bool) -> TreeWalkResult { + TreeWalkResult { + nodes, + tree_markdown: "recording fixture".into(), + complete: !partial, + read_complete: !partial, + incomplete_notes: if partial { + vec!["fixture timeout".into()] + } else { + vec![] + }, + truncated: partial, + window_scope: None, + } + } + + #[test] + fn complete_and_partial_recordings_release_actionable_retains_without_replacing_tokens() { + let pid = 9_802_601; + let window = 501; + let registry = cua_driver_core::element_token::global(); + let snapshot = registry.register_snapshot(pid, window, 1); + let token = cua_driver_core::element_token::token_for(snapshot, 0); + for partial in [false, true] { + let value = CFString::new("owned recording reference fixture with a heap allocation"); + let ptr = value.as_concrete_TypeRef() as CFTypeRef; + let baseline = unsafe { CFGetRetainCount(ptr) }; + unsafe { CFRetain(ptr) }; + let nodes = vec![AXNode { + element_index: Some(0), + element_ptr: ptr as usize, + ..Default::default() + }]; + let bytes = + recording_tree_json(pid, window, tree(nodes, partial), Instant::now()).unwrap(); + let payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(unsafe { CFGetRetainCount(ptr) }, baseline); + assert_eq!(payload["element_count"], 1); + assert_eq!(payload["truncated"], partial); + assert_eq!(payload["read_complete"], !partial); + assert_eq!(registry.resolve(pid, &token), Ok((window, 0))); + } + registry.clear_pid(pid); + } + + #[test] + fn display_nodes_do_not_transfer_an_unowned_reference() { + let value = CFString::new("unindexed recording fixture with a separate heap allocation"); + let ptr = value.as_concrete_TypeRef() as CFTypeRef; + let baseline = unsafe { CFGetRetainCount(ptr) }; + let nodes = vec![AXNode { + element_ptr: ptr as usize, + ..Default::default() + }]; + let bytes = recording_tree_json(101, 501, tree(nodes, false), Instant::now()).unwrap(); + assert_eq!(unsafe { CFGetRetainCount(ptr) }, baseline); + let payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(payload["element_count"], 0); + } + + #[test] + fn empty_recording_keeps_its_metadata() { + let bytes = recording_tree_json(101, 501, tree(vec![], false), Instant::now()).unwrap(); + let payload: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(payload["pid"], 101); + assert_eq!(payload["window_id"], 501); + assert_eq!(payload["element_count"], 0); + assert_eq!(payload["tree_markdown"], "recording fixture"); + assert_eq!(payload["read_complete"], true); + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/click.rs index 1b3aed5601c..abdcef04f74 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/click.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/click.rs @@ -17,8 +17,7 @@ use async_trait::async_trait; use cua_driver_contract::{ClickButton, ClickInput}; use cua_driver_core::{ action_record::{ - ActionEffect, ActionEvidence, ActionExecutionRecord, ActionTransport, ActualDelivery, - EvidenceKind, RequestedDelivery, + ActionEffect, ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery, }, protocol::ToolResult, tool::{Tool, ToolDef}, @@ -43,6 +42,63 @@ pub struct ClickTool { state: Arc<ToolState>, } +pub(super) fn libreoffice_pixel_refusal(bundle_id: Option<&str>) -> Option<ToolResult> { + if bundle_id != Some("org.libreoffice.script") { + return None; + } + // VCL's sendMouseEventToFrame reads NSEvent.mouseLocation rather than + // the routed event's coordinates, so it can select another Calc cell. + Some( + ToolResult::error( + "LibreOffice PID-routed pixel clicks cannot reliably reach the requested position. \ + Use an advertised accessibility action.", + ) + .with_structured(serde_json::json!({ + "code": "background_unavailable", + "effect": "refused", + "sent": false, + "reason": "pointer_reading_toolkit", + "toolkit": "libreoffice-vcl" + })), + ) +} + +#[derive(Debug, thiserror::Error)] +#[error("{action} returned AX error {status}")] +struct UnconfirmedAxAction { + action: &'static str, + status: crate::ax::bindings::AXError, +} + +fn unconfirmed_ax_result( + error: &UnconfirmedAxAction, + index: Option<usize>, + foreground: bool, +) -> ToolResult { + let mut result = super::ax_action_unconfirmed(error.action, index, error.status); + if foreground { + result.structured_content.as_mut().unwrap()["delivery_mode"] = serde_json::json!("unknown"); + } + result.with_action_record( + ActionExecutionRecord::builder( + ActionEffect::Unverifiable, + ActionTransport::MacosAxAction, + if foreground { + RequestedDelivery::Foreground + } else { + RequestedDelivery::Background + }, + ) + .actual_delivery(if foreground { + ActualDelivery::Unknown + } else { + ActualDelivery::Background + }) + .build() + .expect("attempted AX action has a valid uncertain record"), + ) +} + impl ClickTool { pub fn new(state: Arc<ToolState>) -> Self { Self { state } @@ -81,12 +137,18 @@ fn app_click_record(accessibility: bool, foreground: bool) -> ActionExecutionRec } else { ActionTransport::MacosCgEventPid }, - RequestedDelivery::Foreground, + if foreground { + RequestedDelivery::Foreground + } else { + RequestedDelivery::Background + }, ) - .actual_delivery(ActualDelivery::Foreground) - .evidence(ActionEvidence { - kind: EvidenceKind::NativeApiResult, - detail: "the app-bound click actuator completed".into(), + .actual_delivery(if !foreground { + ActualDelivery::Background + } else if accessibility { + ActualDelivery::Unknown + } else { + ActualDelivery::Foreground }) .build() .expect("app click record is valid") @@ -394,22 +456,7 @@ impl Tool for ClickTool { // screenshot width / logical screen width. This is robust even when // CGDisplayPixelsWide under-reports the backing scale (it returns the // scaled-mode point width on some Retina configs → a bogus 1.0). - let desktop_ratio = tokio::task::spawn_blocking(|| { - let logical_w = - super::get_screen_size::main_screen_size().map(|(w, _, _)| w as f64); - let shot_w = crate::capture::screenshot_display_bytes() - .ok() - .and_then(|png| crate::capture::png_dimensions(&png).ok()) - .map(|(w, _)| w as f64); - match (shot_w, logical_w) { - (Some(sw), Some(lw)) if lw > 0.0 && sw > lw => sw / lw, - _ => 1.0, - } - }) - .await - .unwrap_or(1.0); - let sx = sx_shot / desktop_ratio; - let sy = sy_shot / desktop_ratio; + let (sx, sy) = super::desktop_screenshot_point(sx_shot, sy_shot).await; let button = match input.button.unwrap_or(ClickButton::Left) { ClickButton::Left => "left", ClickButton::Right => "right", @@ -489,12 +536,14 @@ impl Tool for ClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id, _via_token) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg, false), cua_driver_core::element_token::ResolvedElement::Element { window_id: wid, element_index: idx, via_token, + .. } => (Some(idx), wid, via_token), }; let x = args @@ -555,7 +604,7 @@ impl Tool for ClickTool { // concurrent get_window_state on the same (pid, window_id) while // this click is mid-flight (use-after-free → daemon crash). The // guard lives to the end of this method, past the AX action below. - let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) { + let element_guard = match element_cache.get_element_retained(pid, wid, idx) { Some(e) => e, None => { return ToolResult::error(format!( @@ -588,6 +637,13 @@ impl Tool for ClickTool { None }; + let pixel_refusal = libreoffice_pixel_refusal(apps::bundle_id_for_pid(pid).as_deref()); + if button_str == "middle" { + if let Some(refusal) = pixel_refusal { + return refusal; + } + } + // Surface 5: button=right on the AX path → AXShowMenu (the same surface // the dedicated `right_click` tool dispatches). Threads through the // identical perform_ax_click code path with the action remapped. @@ -690,7 +746,7 @@ impl Tool for ClickTool { } else { false }; - let mut selection_pixel = if selection_candidate { + let mut selection_pixel = if selection_candidate && pixel_refusal.is_none() { if let Some((cx, cy)) = center { super::px_frame::resolve_or_refuse(wid) .await @@ -878,6 +934,9 @@ impl Tool for ClickTool { } ToolResult::text(msg).with_structured(structured) } + Ok(Err(e)) if e.is::<UnconfirmedAxAction>() => { + unconfirmed_ax_result(e.downcast_ref().unwrap(), Some(idx), foreground) + } Ok(Err(e)) => ToolResult::error(format!("AX action failed: {e}")), Err(e) => ToolResult::error(format!("Task error: {e}")), } @@ -1031,7 +1090,7 @@ impl Tool for ClickTool { let hit_test_wid = window_id.expect("guarded by window_id.is_some() above"); let ax_result = tokio::task::spawn_blocking(move || unsafe { let Some(element) = element_at_screen_position(pid, screen_x, screen_y) else { - return Ok::<bool, anyhow::Error>(false); + return Ok::<Option<i32>, anyhow::Error>(None); }; // The pid-scoped hit-test can resolve an element from a // same-process sibling overlapping the requested point. @@ -1040,21 +1099,34 @@ impl Tool for ClickTool { // (already gated for this exact window). if crate::ax::exact_target::element_window_id(element) != Some(hit_test_wid) { CFRelease(element as _); - return Ok(false); + return Ok(None); } - let delivered = if focus_only { - crate::input::ax_actions::focus_element(element as usize).is_ok() + if !focus_only + && !copy_action_names(element) + .iter() + .any(|action| action == "AXPress") + && libreoffice_pixel_refusal(apps::bundle_id_for_pid(pid).as_deref()) + .is_some() + { + CFRelease(element as _); + return Ok(None); + } + let status = if focus_only { + crate::input::ax_actions::focus_element(element as usize) + .map(|()| kAXErrorSuccess) } else { let press = core_foundation::string::CFString::new("AXPress"); - AXUIElementPerformAction(element, press.as_concrete_TypeRef()) - == kAXErrorSuccess + Ok(AXUIElementPerformAction( + element, + press.as_concrete_TypeRef(), + )) }; CFRelease(element as _); - Ok(delivered) + status.map(Some) }) .await; match ax_result { - Ok(Ok(true)) => { + Ok(Ok(Some(kAXErrorSuccess))) => { let label = if focus_only { "focused" } else { "pressed" }; return ToolResult::text(format!( "✅ PX hit-test {label} the background element via AX." @@ -1065,7 +1137,10 @@ impl Tool for ClickTool { "effect": "unverifiable" })); } - Ok(Ok(false)) if focus_only => { + Ok(Ok(Some(status))) if !focus_only => { + return super::ax_action_unconfirmed("AXPress", None, status); + } + Ok(Ok(None)) if focus_only => { return ToolResult::error( "Background PX focus is unavailable at the requested point.".to_owned(), ) @@ -1079,6 +1154,13 @@ impl Tool for ClickTool { "code": "background_unavailable" })); } + Err(error) => { + return ToolResult::computer_use_error( + "operation_failed", + format!("Background PX accessibility task failed: {error}"), + true, + ); + } _ => {} } } @@ -1087,6 +1169,11 @@ impl Tool for ClickTool { // requested foreground click without a window id still degrades to // background, matching the existing contract and result label. let fg = delivery_mode.is_foreground() && window_id.is_some(); + if let Some(refusal) = + libreoffice_pixel_refusal(apps::bundle_id_for_pid(pid).as_deref()) + { + return refusal; + } let activation_policy = pixel_activation_policy(&button_str, fg, window_id.is_some()); // Pin the overlay above the target window BEFORE animating so @@ -1200,9 +1287,9 @@ impl Tool for ClickTool { crate::input::mouse::right_click_at_xy(pid, screen_x, screen_y, &m) } "middle" => { - if let Some(_wid) = window_id { + if let Some(wid) = window_id { return crate::input::mouse::middle_click_at_xy_with_window_local( - pid, screen_x, screen_y, win_local_x, win_local_y, &m, + pid, screen_x, screen_y, win_local_x, win_local_y, wid, &m, ); } crate::input::mouse::middle_click_at_xy(pid, screen_x, screen_y, &m) @@ -1214,17 +1301,11 @@ impl Tool for ClickTool { // and Chromium-specific fields (f40, f51, f58, f91, f92) onto events // for better backgrounded-target delivery. if let Some(wid) = window_id { - if fg { - return crate::input::mouse::click_at_xy_with_window_local( - pid, screen_x, screen_y, - win_local_x, win_local_y, - wid, count, &m, - ); - } - return crate::input::mouse::click_at_xy_chromium( + return crate::input::mouse::click_at_xy_with_window_delivery( pid, screen_x, screen_y, win_local_x, win_local_y, wid, count, &m, + crate::input::mouse::WindowClickDelivery::from_foreground(fg), ); } crate::input::mouse::click_at_xy(pid, screen_x, screen_y, count, &m) @@ -1354,6 +1435,7 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool Ok(value) => value, Err(error) => return error, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (index, window) = match resolved { ResolvedElement::None => (None, window), ResolvedElement::Element { @@ -1380,10 +1462,7 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); let foreground = delivery_mode.is_foreground(); let element = if let Some(index) = index { - match state - .element_cache - .get_element_retained(pid, window_id, index) - { + match element_cache.get_element_retained(pid, window_id, index) { Some(element) => Some(element), None => { return ToolResult::error("app click element is stale; getState before clicking") @@ -1411,7 +1490,40 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool Vec::new() }; let action = app_semantic_action(&actions, &button, count, !modifiers.is_empty()); - let route = if action.is_some() { + let editable_focus = + if action.is_none() && button == "left" && count == 1 && modifiers.is_empty() { + if let Some(pointer) = element_ptr { + match tokio::task::spawn_blocking(move || unsafe { + let element = pointer as AXUIElementRef; + let role = copy_string_attr(element, "AXRole")?; + let libreoffice_text_area = role == "AXTextArea" + && apps::bundle_id_for_pid(pid).as_deref() + == Some("org.libreoffice.script"); + ((is_editable_focus_role(&role) || libreoffice_text_area) + && is_attribute_settable(element, "AXValue") + && is_attribute_settable(element, "AXFocused")) + .then_some(role) + }) + .await + { + Ok(role) => role, + Err(error) => { + return ToolResult::error(format!("click target read failed: {error}")) + } + } + } else { + None + } + } else { + None + }; + let accessibility = action.is_some() || editable_focus.is_some(); + if !foreground && !accessibility { + if let Some(refusal) = libreoffice_pixel_refusal(apps::bundle_id_for_pid(pid).as_deref()) { + return refusal; + } + } + let route = if accessibility { BackgroundAction::AxSemantic } else { BackgroundAction::WindowPointer @@ -1437,7 +1549,7 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool } else { None }; - let pointer_target = if action.is_none() { + let pointer_target = if !accessibility { let frame = match super::px_frame::resolve_or_refuse(window_id).await { Ok(frame) => frame, Err(error) => return error, @@ -1513,9 +1625,21 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool ) }; if status != kAXErrorSuccess { - anyhow::bail!("{action} failed with AX error {status}"); + return Err(UnconfirmedAxAction { action, status }.into()); } Ok(()) + } else if let Some(role) = &editable_focus { + if role == "AXTextArea" { + if !foreground { + crate::input::skylight::prepare_background_keyboard(pid, window_id)?; + } + anyhow::ensure!( + unsafe { super::select_text::read_range(element_ptr.unwrap() as AXUIElementRef) }.is_some(), + "Writer caret is unavailable before focus" + ); + } + focus_editable_control(element_ptr.unwrap() as AXUIElementRef, pid, role) + .map(|_| ()) } else { let (point, local) = pointer_target.unwrap(); let modifiers: Vec<&str> = modifiers.iter().map(String::as_str).collect(); @@ -1540,7 +1664,7 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool ) } }; - if foreground && action.is_none() { + if foreground && !accessibility { crate::input::skylight::with_foreground_hid_activation(pid, window_id, dispatch) } else if foreground { crate::input::skylight::with_foreground_assist(pid, window_id, dispatch)?; @@ -1559,9 +1683,12 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool match result { Ok(Ok(())) => ToolResult::text(format!("Click dispatched.{}", changes.result_suffix())) .with_structured(serde_json::json!({ - "path": if action.is_some() { "ax" } else { "cgevent" }, "verified": false, "effect": "unverifiable" + "path": if accessibility { "ax" } else { "cgevent" }, "verified": false, "effect": "unverifiable" })) - .with_action_record(app_click_record(action.is_some(), foreground)), + .with_action_record(app_click_record(accessibility, foreground)), + Ok(Err(error)) if error.is::<UnconfirmedAxAction>() => { + unconfirmed_ax_result(error.downcast_ref().unwrap(), index, foreground) + } Ok(Err(_)) if action.is_some() && changes.new_windows.iter().any(|window| window.pid == pid) => ToolResult::text(format!( "Click changed the app.{}", changes.result_suffix() @@ -1577,14 +1704,21 @@ pub(super) async fn invoke_app_click(state: Arc<ToolState>, args: Value) -> Tool // ── AX click implementation (blocking) ─────────────────────────────────────── -fn focus_editable_control(element: AXUIElementRef, pid: i32, role: &str) -> anyhow::Result<()> { +fn focus_editable_control(element: AXUIElementRef, pid: i32, role: &str) -> anyhow::Result<bool> { let before = unsafe { copy_string_attr(element, "AXValue") } .ok_or_else(|| anyhow::anyhow!("editable control value unavailable before focus"))?; let err = unsafe { set_bool_attr_true(element, "AXFocused") }; - anyhow::ensure!(err == kAXErrorSuccess, "AXFocused write returned {err}"); + if err != kAXErrorSuccess { + return Err(UnconfirmedAxAction { + action: "AXFocused", + status: err, + } + .into()); + } - let focused = unsafe { focused_element_of_pid(pid) } - .ok_or_else(|| anyhow::anyhow!("editable control focus could not be read back"))?; + let Some(focused) = (unsafe { focused_element_of_pid(pid) }) else { + return Ok(false); + }; let mut matches = unsafe { CFEqual(focused as CFTypeRef, element as CFTypeRef) != 0 }; // AppKit may put the first responder on the combo's embedded editor. if !matches @@ -1597,12 +1731,8 @@ fn focus_editable_control(element: AXUIElementRef, pid: i32, role: &str) -> anyh } } unsafe { CFRelease(focused as CFTypeRef) }; - anyhow::ensure!(matches, "editable control did not receive verified focus"); - anyhow::ensure!( - unsafe { copy_string_attr(element, "AXValue") }.as_deref() == Some(before.as_str()), - "editable control value changed or became unreadable during focus; observe before retrying" - ); - Ok(()) + Ok(matches + && unsafe { copy_string_attr(element, "AXValue") }.as_deref() == Some(before.as_str())) } /// Returns `(summary_text, needs_webkit_delay, suspected_noop, @@ -1649,15 +1779,19 @@ fn perform_ax_click( && unsafe { is_attribute_settable(element, "AXValue") } && unsafe { is_attribute_settable(element, "AXFocused") } { - focus_editable_control(element, pid, &role)?; + let confirmed = focus_editable_control(element, pid, &role)?; return Ok(( - format!( - "✅ Focused editable {role} [{idx}] \"{title}\"; confirmed focus on the \ + if confirmed { + format!( + "✅ Focused editable {role} [{idx}] \"{title}\"; confirmed focus on the \ control or a combo's direct text editor, with AXValue unchanged." - ), + ) + } else { + format!("Sent AXFocused to editable {role} [{idx}]. Focus/value readback did not confirm the result; observe before another action. This request was not replayed.") + }, false, false, - true, + confirmed, false, )); } @@ -1702,7 +1836,7 @@ fn perform_ax_click( &modifier_refs, )?; } else { - crate::input::mouse::click_at_xy_with_window_local( + crate::input::mouse::click_at_xy_with_window_delivery( pid, target.screen_x, target.screen_y, @@ -1711,6 +1845,7 @@ fn perform_ax_click( window_id, 1, &modifier_refs, + crate::input::mouse::WindowClickDelivery::from_foreground(foreground), )?; } // AppKit may publish a transient AXSelected transition while the @@ -1788,7 +1923,7 @@ fn perform_ax_click( &modifier_refs, )?; } else { - crate::input::mouse::click_at_xy_with_window_local( + crate::input::mouse::click_at_xy_with_window_delivery( pid, target.screen_x, target.screen_y, @@ -1797,6 +1932,7 @@ fn perform_ax_click( window_id, 1, &modifier_refs, + crate::input::mouse::WindowClickDelivery::from_foreground(foreground), )?; } return Ok(( @@ -1812,7 +1948,11 @@ fn perform_ax_click( let err = unsafe { crate::ax::bindings::perform_action(element, ax_action) }; if err != crate::ax::bindings::kAXErrorSuccess { - anyhow::bail!("AXUIElementPerformAction({ax_action}) returned {err}"); + return Err(UnconfirmedAxAction { + action: ax_action, + status: err, + } + .into()); } let mut summary = format!("✅ Performed {ax_action} on [{idx}] {role} \"{title}\"."); @@ -1937,6 +2077,67 @@ fn map_action(action: &str) -> &'static str { mod tests { use super::*; + #[test] + fn libreoffice_pixel_refusal_is_pre_dispatch_and_does_not_match_other_apps() { + let result = libreoffice_pixel_refusal(Some("org.libreoffice.script")).unwrap(); + assert_eq!(result.is_error, Some(true)); + let structured = result.structured_content.unwrap(); + assert_eq!(structured["code"], "background_unavailable"); + assert_eq!(structured["effect"], "refused"); + assert_eq!(structured["sent"], false); + assert_eq!(structured["toolkit"], "libreoffice-vcl"); + for other in [ + None, + Some("com.apple.TextEdit"), + Some("org.libreoffice.script.helper"), + ] { + assert!(libreoffice_pixel_refusal(other).is_none()); + } + } + + #[test] + fn attempted_ax_error_publishes_uncertainty_without_confirming_delivery() { + let error = UnconfirmedAxAction { + action: "AXPress", + status: -25200, + }; + for (foreground, expected_delivery) in [(false, "background"), (true, "unknown")] { + let result = unconfirmed_ax_result(&error, Some(1), foreground); + assert_ne!(result.is_error, Some(true)); + assert_eq!(result.structured_content.as_ref().unwrap()["sent"], true); + let record = result.action_record.unwrap(); + assert!(record.evidence.is_empty()); + let public = serde_json::to_value(record.public_result().unwrap()).unwrap(); + assert_eq!(public["effect"], "unverifiable"); + assert_eq!(public["route"], "accessibility"); + assert_eq!(public["delivery"]["mode"], expected_delivery); + } + } + + #[test] + fn app_click_records_the_selected_delivery_without_effect_evidence() { + for (accessibility, foreground, expected) in [ + (true, false, "background"), + (false, false, "background"), + (true, true, "unknown"), + (false, true, "foreground"), + ] { + let record = app_click_record(accessibility, foreground); + assert_eq!( + record.requested_delivery, + if foreground { + RequestedDelivery::Foreground + } else { + RequestedDelivery::Background + } + ); + assert!(record.evidence.is_empty()); + let public = serde_json::to_value(record.public_result().unwrap()).unwrap(); + assert_eq!(public["effect"], "unverifiable"); + assert_eq!(public["delivery"]["mode"], expected); + } + } + #[test] fn app_element_click_uses_the_part_inside_its_window() { let window = [195.0, 92.0, 656.0, 422.0]; @@ -1984,7 +2185,7 @@ mod tests { } #[test] - fn app_click_routes_unadvertised_text_and_non_left_gestures_to_pointer() { + fn only_single_unmodified_left_clicks_use_advertised_semantic_actions() { let press = vec!["AXPress".to_owned()]; let pick = vec!["AXPress".to_owned(), "AXPick".to_owned()]; assert_eq!(app_semantic_action(&[], "left", 1, false), None); diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs index 101c33fa450..b9c46686b25 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs @@ -107,12 +107,14 @@ impl Tool for DoubleClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { window_id: wid, element_index: idx, via_token: _, + .. } => (Some(idx), wid), }; @@ -120,7 +122,7 @@ impl Tool for DoubleClickTool { if let (Some(idx), Some(wid)) = (element_index, window_id) { // Retain out of the cache so a concurrent get_window_state can't // free the element mid-action (use-after-free → daemon crash). - let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) { + let element_guard = match element_cache.get_element_retained(pid, wid, idx) { Some(e) => e, None => { return ToolResult::error(format!( @@ -170,7 +172,7 @@ impl Tool for DoubleClickTool { .await; return match result { - Ok(Ok(msg)) => ToolResult::text(msg), + Ok(Ok(result)) => result, Ok(Err(e)) => ToolResult::error(format!("double_click failed: {e}")), Err(e) => ToolResult::error(format!("Task error: {e}")), }; @@ -244,6 +246,12 @@ impl Tool for DoubleClickTool { None }; + if let Some(refusal) = + super::click::libreoffice_pixel_refusal(crate::apps::bundle_id_for_pid(pid).as_deref()) + { + return refusal; + } + // Pin overlay above the target window before animating. if let Some(wid) = window_id { crate::cursor::overlay::send_command( @@ -265,7 +273,7 @@ impl Tool for DoubleClickTool { let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let do_click = move || -> anyhow::Result<()> { if let Some(wid) = window_id { - crate::input::mouse::click_at_xy_with_window_local( + crate::input::mouse::click_at_xy_with_window_delivery( pid, screen_x, screen_y, @@ -274,6 +282,7 @@ impl Tool for DoubleClickTool { wid, 2, &[], + crate::input::mouse::WindowClickDelivery::from_foreground(fg), ) } else { crate::input::mouse::click_at_xy(pid, screen_x, screen_y, 2, &[]) @@ -319,25 +328,25 @@ fn ax_double_click( idx: usize, cursor_key: &str, has_ax_open: bool, - allow_pointer_fallback: bool, -) -> anyhow::Result<String> { + foreground: bool, +) -> anyhow::Result<ToolResult> { let element = element_ptr as AXUIElementRef; // Try AXOpen first (Finder items, openable list rows, document cells). if has_ax_open { let err = unsafe { perform_action(element, "AXOpen") }; if err == kAXErrorSuccess { - return Ok(format!("AXOpen performed on element [{idx}].")); + return Ok(ToolResult::text(format!( + "AXOpen performed on element [{idx}]." + ))); } - if !allow_pointer_fallback { - anyhow::bail!( - "AXOpen returned {err} for element [{idx}]; background delivery will not \ - improvise a pointer fallback after choosing the semantic route" - ); - } - tracing::debug!( - "AXOpen returned {err} for element [{idx}], falling back to pixel double-click" - ); + return Ok(super::ax_action_unconfirmed("AXOpen", Some(idx), err)); + } + + if let Some(refusal) = + super::click::libreoffice_pixel_refusal(crate::apps::bundle_id_for_pid(pid).as_deref()) + { + return Ok(refusal); } // Resolve screen center and fall back to pixel double-click. @@ -366,10 +375,20 @@ fn ax_double_click( screen coordinates as window-local for element [{idx}]." ) })?; - crate::input::mouse::click_at_xy_with_window_local(pid, cx, cy, wx, wy, wid, 2, &[])?; - Ok(format!( + crate::input::mouse::click_at_xy_with_window_delivery( + pid, + cx, + cy, + wx, + wy, + wid, + 2, + &[], + crate::input::mouse::WindowClickDelivery::from_foreground(foreground), + )?; + Ok(ToolResult::text(format!( "✅ Double-clicked element [{idx}] at ({cx:.1}, {cy:.1})." - )) + ))) } #[cfg(test)] diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/drag.rs index 08b80fd26a7..6dffafc60cb 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/drag.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/drag.rs @@ -240,17 +240,13 @@ impl Tool for DragTool { // press-drag-release gesture (the explicit last resort for surfaces // that drop background CGEvents), via the same skylight assist click // uses. Requires a window_id to have a window to front. - let delivery_mode = if args.bool_or("app_context", false) { - super::DeliveryMode::Foreground - } else { - super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()) - }; + let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); if !delivery_mode.is_foreground() { return ToolResult::error( - "Background drag is unavailable on macOS; use delivery_mode:\"foreground\"." + "Background drag is unavailable on macOS; use set_value for a slider or another semantic action." .to_owned(), ) - .with_structured(serde_json::json!({ "code": "background_unavailable" })); + .with_structured(serde_json::json!({ "code": "unsupported_on_platform", "sent":false })); } // Coerce integer or float from JSON for coordinate fields. let coerce = |key: &str| -> Option<f64> { diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs index d1df99da7c1..b18790a3095 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs @@ -73,20 +73,64 @@ pub(crate) fn main_screen_size() -> Option<(i64, i64, f64)> { return None; } - let scale = get_backing_scale(display_id, w); + let scale = get_backing_scale(display_id); Some((w, h, scale)) } -/// Estimate backing scale by comparing the display's pixel mode width to its -/// logical (CoreGraphics) bounds width. -pub(crate) fn get_backing_scale(display_id: u32, logical_w: i64) -> f64 { - use core_graphics::display::CGDisplayPixelsWide; - let pixel_w = unsafe { CGDisplayPixelsWide(display_id) } as i64; - if pixel_w > 0 && logical_w > 0 { - let ratio = pixel_w as f64 / logical_w as f64; - // Round to nearest 0.5 to avoid floating point noise. - (ratio * 2.0).round() / 2.0 - } else { - 1.0 +/// Return the current display mode's backing-pixel to point ratio. +pub(crate) fn get_backing_scale(display_id: u32) -> f64 { + use core_graphics::display::CGDisplay; + + let Some(mode) = CGDisplay::new(display_id).display_mode() else { + return 1.0; + }; + backing_scale_from_widths(mode.width(), mode.pixel_width()) +} + +fn backing_scale_from_widths(point_width: u64, pixel_width: u64) -> f64 { + if point_width == 0 || pixel_width == 0 { + return 1.0; + } + let ratio = pixel_width as f64 / point_width as f64; + // Round to nearest 0.5 to avoid floating point noise. + (ratio * 2.0).round() / 2.0 +} + +#[cfg(test)] +mod tests { + use super::*; + use core_graphics::display::CGDisplay; + + #[test] + #[ignore = "requires an attached macOS display; run explicitly on a native Retina host"] + fn main_display_scale_matches_current_mode() { + let display = CGDisplay::main(); + let mode = display + .display_mode() + .expect("main display should expose its current mode"); + let point_width = mode.width(); + let pixel_width = mode.pixel_width(); + assert!(point_width > 0, "display mode should have a point width"); + assert!(pixel_width > 0, "display mode should have a pixel width"); + assert!( + pixel_width > point_width, + "native Retina validation requires more backing pixels than points; got {pixel_width}px for {point_width}pt" + ); + + let expected = ((pixel_width as f64 / point_width as f64) * 2.0).round() / 2.0; + let actual = get_backing_scale(display.id); + + assert_eq!( + actual, expected, + "backing scale should use the current mode's pixel and point widths" + ); + } + + #[test] + fn display_mode_widths_preserve_rounding_and_fallbacks() { + assert_eq!(backing_scale_from_widths(1920, 3840), 2.0); + assert_eq!(backing_scale_from_widths(1920, 2880), 1.5); + assert_eq!(backing_scale_from_widths(0, 3840), 1.0); + assert_eq!(backing_scale_from_widths(1920, 0), 1.0); } } diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs index 9f0abe7f5cb..d146d6f104a 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs @@ -81,6 +81,7 @@ fn def() -> &'static ToolDef { "app_context": { "type": "boolean", "description": "Capture the app interaction context with native compact AX projection." }, "query": { "type": "string", "description": "Case-insensitive filter for tree_markdown and structured elements. Returns matching actionable rows plus their actionable ancestors without renumbering element_index values." }, "capture_mode": cua_driver_core::capture_mode::capture_mode_schema(), + "include_accessibility": {"type":"boolean","default":true,"description":"Set false for a screenshot without an accessibility walk."}, "include_screenshot": { "type": "boolean", "description": "Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return the tree only (the cheap path when you're just re-indexing before an element ax action; saves the image tokens + screen-grab latency). screenshot_out_file still forces a capture to disk." @@ -122,6 +123,17 @@ fn def() -> &'static ToolDef { }) } +fn off_space_staleness_note( + on_current_space: Option<bool>, + chromium: bool, +) -> Option<&'static str> { + (on_current_space == Some(false) && chromium).then_some( + "This browser window is on another Space. Chromium may pause painting and \ + accessibility updates while hidden, so this screenshot and tree can show \ + an older state. Do not use this read alone to confirm an action's effect.", + ) +} + fn chromium_browser_window(pid: i32) -> bool { let identity = format!( "{} {}", @@ -274,10 +286,16 @@ impl Tool for GetWindowStateTool { .and_then(Value::as_bool) .unwrap_or(false); - // Always walk the AX tree (perception returns both tree + screenshot). - let tree_result = { + let (tree_result, pending_snapshot) = if args["include_accessibility"] == false { + (None, None) + } else { let q = query.clone(); let state = Arc::clone(&self.state); + let readiness_deadline = Some( + cua_driver_core::tool::current_ax_readiness_deadline().unwrap_or_else(|| { + std::time::Instant::now() + cua_driver_core::tool::AX_READINESS_BUDGET + }), + ); // Keep the product deadline below the public client's 25-second // deadline so callers receive a structured driver error. The AX // walker also applies a native per-element messaging timeout because @@ -296,16 +314,19 @@ impl Tool for GetWindowStateTool { max_elements, max_depth, app_context, + readiness_deadline, ) }; - if app_context { - crate::apps::with_app_observation(pid, walk) + let tree = if app_context { + crate::apps::with_app_observation(pid, readiness_deadline, walk) } else { walk() - } + }; + let snapshot = crate::ax::cache::ElementCache::prepare(&tree.nodes); + (tree, snapshot) }); match tokio::time::timeout(std::time::Duration::from_secs(20), walk_future).await { - Ok(Ok(r)) => Some(r), + Ok(Ok((tree, snapshot))) => (Some(tree), Some(snapshot)), Ok(Err(e)) => return ToolResult::error(format!("AX tree walk failed: {e}")), Err(_elapsed) => { return ToolResult::error(format!( @@ -332,21 +353,8 @@ impl Tool for GetWindowStateTool { // `window_scope` is None only when no window_id was requested, which // this tool never does — so treat that as resolved. let scope_matched = window_scope.as_ref().is_none_or(|s| s.is_matched()); - - // Update element cache — ONLY for a resolved window scope. Caching an - // unresolved scope's nodes under (pid, window_id) is what turned a - // wrong-surface snapshot into a wrong-surface *action*: a follow-up - // click(element_index=N) picked whatever the walk happened to return. - // For an unresolved scope, replace any prior entry with an empty - // snapshot so a stale index map cannot be clicked through either. - if !observation_only { - if let Some(ref r) = tree_result { - if scope_matched { - self.state.element_cache.update(pid, window_id, &r.nodes); - } else { - self.state.element_cache.update(pid, window_id, &[]); - } - } + if !observation_only && tree_result.is_some() && !scope_matched { + self.state.element_cache.clear_target(pid, window_id); } let observation_revision = @@ -583,16 +591,15 @@ impl Tool for GetWindowStateTool { // Skipped entirely for an unresolved window scope: an element_token is // a promise that index N addresses a row of THIS window, and there is // no such row to promise (issue #2237). - let elem_count_for_snapshot = tree_result - .as_ref() - .map(|r| r.nodes.iter().filter(|n| n.element_index.is_some()).count()) - .unwrap_or(0); let snapshot_id = if scope_matched && !observation_only { - Some(cua_driver_core::element_token::global().register_snapshot( - pid, - window_id, - elem_count_for_snapshot, - )) + pending_snapshot.map(|snapshot| { + self.state.element_cache.publish_for_session( + pid, + window_id, + snapshot, + args.get("_session_id").and_then(Value::as_str), + ) + }) } else { None }; @@ -823,6 +830,16 @@ impl Tool for GetWindowStateTool { if let Some(ref fp) = screenshot_file_path { structured["screenshot_file_path"] = serde_json::json!(fp); } + let on_current_space = crate::windows::window_on_current_space_by_id(window_id); + if on_current_space == Some(false) { + structured["on_current_space"] = serde_json::json!(false); + if let Some(note) = + off_space_staleness_note(on_current_space, chromium_browser_window(pid)) + { + structured["off_space_staleness"] = serde_json::json!(note); + content.push(Content::text(note)); + } + } cua_driver_core::window_inspection::mark_browser_chrome_capture_coverage( &mut structured, chromium_browser_window(pid).then_some( diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs index 149d1f18c89..66164434946 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs @@ -237,12 +237,14 @@ impl Tool for HotkeyTool { Ok(resolved) => resolved, Err(error) => return error, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { window_id, element_index, via_token: _, + .. } => (Some(element_index), window_id), }; // delivery_mode gates whether we raise: background (default) never fronts @@ -258,11 +260,7 @@ impl Tool for HotkeyTool { } let element_guard = if let (Some(index), Some(window_id)) = (element_index, window_id) { - match self - .state - .element_cache - .get_element_retained(pid, window_id, index) - { + match element_cache.get_element_retained(pid, window_id, index) { Some(guard) => Some(guard), None => { return ToolResult::error(format!( @@ -312,6 +310,42 @@ impl Tool for HotkeyTool { None }; + if !fg && px.is_none() && element_ptr.is_none() && apps::frontmost_pid() != Some(pid) { + if let (Some(wid), Some(combo)) = ( + window_id, + super::menu_equivalent::Combo::parse(&modifiers, &key), + ) { + let save = key.eq_ignore_ascii_case("s") + && modifiers.len() == 1 + && matches!( + modifiers[0].to_ascii_lowercase().as_str(), + "cmd" | "command" | "meta" | "super" + ); + let native = !crate::apps::bundle_id_for_pid(pid) + .as_deref() + .is_some_and(crate::browser::BrowserJs::supports) + && !crate::browser::electron_js::ElectronJs::is_electron(pid); + if native { + let result = tokio::task::spawn_blocking(move || { + super::menu_equivalent::find(pid, &combo) + .map(|item| super::menu_equivalent::run(pid, wid, item, save)) + }) + .await; + match result { + Ok(Some(result)) => return result, + Err(error) => { + return ToolResult::computer_use_error( + "menu_unavailable", + error.to_string(), + true, + ) + } + _ => {} + } + } + } + } + // Web-content AX nodes can acknowledge AXFocused without moving the // renderer's real first responder. That makes a direct focus write an // unsafe oracle for Chromium/WebKit hotkeys: the following PID/HID diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs index 6d01dc1b52e..ca1d59e1e44 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs @@ -77,7 +77,7 @@ unsafe fn semantic_children(parent: AXUIElementRef) -> Vec<AXUIElementRef> { out } -unsafe fn resolve_exact_prefix( +pub(super) unsafe fn resolve_exact_prefix( menu_bar: AXUIElementRef, prefix: &[String], ) -> Result<AXUIElementRef, String> { diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/list_apps.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/list_apps.rs index c3f8c3279b5..586b9a9844f 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/list_apps.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/list_apps.rs @@ -21,8 +21,9 @@ fn def() -> &'static ToolDef { - kind: `\"desktop\"` for `.app` bundles on macOS.\n\ - last_used: RFC3339 timestamp from the bundle's filesystem mtime, \ when readable; otherwise null.\n\n\ - Only apps with NSApplicationActivationPolicyRegular are included — \ - background helpers and system UI agents are filtered out. Installed \ + Standalone running entries include Regular-policy apps. Installed \ + accessory apps also report live state when their bundle path matches; \ + uninstalled background helpers stay excluded. Installed \ apps come from scanning /Applications, /Applications/Utilities, \ ~/Applications, /System/Applications, and /System/Applications/Utilities.\n\n\ Use this for \"is X installed?\" as well as \"is X running?\". For \ @@ -34,7 +35,7 @@ fn def() -> &'static ToolDef { input_schema: serde_json::json!({ "type": "object", "properties": { - "running_only": { "type": "boolean", "description": "Skip installed-app scanning when refreshing a bound app's process identity." } + "running_only": { "type": "boolean", "description": "Return only live applications, including installed accessory apps." } }, "additionalProperties": false }), @@ -57,11 +58,11 @@ impl Tool for ListAppsTool { .and_then(Value::as_bool) .unwrap_or(false); let apps = tokio::task::spawn_blocking(move || { + let mut apps = crate::apps::list_all_apps(); if running_only { - crate::apps::list_running_apps() - } else { - crate::apps::list_all_apps() + apps.retain(|app| app.running); } + apps }) .await .unwrap_or_default(); diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs index ad3a1e4cb16..fbae3a435a1 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/list_windows.rs @@ -9,16 +9,41 @@ pub struct ListWindowsTool; static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new(); -fn resolve_app_window(pid: i32) -> Option<crate::windows::WindowInfo> { +fn resolve_app_window( + pid: i32, + readiness_deadline: Option<std::time::Instant>, +) -> Option<crate::windows::WindowInfo> { if let Some(window) = - crate::ax::bindings::app_window_id_of_pid(pid).and_then(crate::windows::window_info_by_id) + crate::ax::bindings::app_window_id_of_pid_with_readiness(pid, readiness_deadline) + .and_then(crate::windows::window_info_by_id) { return Some(window); } - crate::apps::with_app_observation(pid, || { - crate::ax::bindings::app_window_id_of_pid(pid).and_then(crate::windows::window_info_by_id) - }) + if let Some(window) = crate::apps::with_app_observation(pid, readiness_deadline, || { + crate::ax::bindings::app_window_id_of_pid_with_readiness(pid, readiness_deadline) + .and_then(crate::windows::window_info_by_id) + }) { + return Some(window); + } + + let mut candidates = crate::windows::all_windows() + .into_iter() + .filter(|window| window.pid == pid && window.layer == 0); + let candidate = candidates.next()?; + if candidates.next().is_some() { + return None; + } + let window_id = unsafe { + let root = crate::ax::bindings::copy_remote_window_for_id(pid, candidate.window_id, None)?; + let selected = crate::ax::bindings::app_target_window_id_with_proven_root( + root, + Some((pid, candidate.window_id)), + ); + core_foundation::base::CFRelease(root as core_foundation::base::CFTypeRef); + selected? + }; + crate::windows::window_info_by_id(window_id).filter(|window| window.pid == pid) } fn def() -> &'static ToolDef { @@ -73,6 +98,7 @@ impl Tool for ListWindowsTool { let pid_filter: Option<i32> = args.opt_i64("pid").map(|v| v as i32); let on_screen_only = args.bool_or("on_screen_only", false); let app_context = args.bool_or("app_context", false); + let readiness_deadline = cua_driver_core::tool::current_ax_readiness_deadline(); if app_context && pid_filter.is_none() { return ToolResult::error("app_context requires pid"); } @@ -104,7 +130,9 @@ impl Tool for ListWindowsTool { let app_target = if app_context { let pid = pid_filter.expect("validated above"); - match tokio::task::spawn_blocking(move || resolve_app_window(pid)).await { + match tokio::task::spawn_blocking(move || resolve_app_window(pid, readiness_deadline)) + .await + { Ok(target) => target, Err(error) => { return ToolResult::error(format!("Could not resolve app window: {error}")) diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/menu_equivalent.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/menu_equivalent.rs new file mode 100644 index 00000000000..5b21ff55bea --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/menu_equivalent.rs @@ -0,0 +1,316 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Find the menu item a key combination stands for. +//! +//! Background key events reach a window's views but never its app's menu key +//! equivalents (see `input::keyboard::hotkey_no_auth`). A background +//! `cmd+s` to TextEdit was reported as "Pressed" and saved nothing. When a +//! combo is a menu item's key equivalent, `hotkey` acts on that menu item +//! through accessibility instead, or refuses when macOS keeps the item +//! disabled in the background. + +use core_foundation::base::{CFRelease, CFTypeRef}; + +use crate::ax::bindings::{ + copy_bool_attr, copy_children, copy_element_attr, copy_number_attr, copy_string_attr, + AXUIElementCreateApplication, AXUIElementRef, AXUIElementSetMessagingTimeout, +}; + +/// `kAXMenuItemModifier*` bits of `AXMenuItemCmdModifiers`. +const MOD_SHIFT: i64 = 1 << 0; +const MOD_OPTION: i64 = 1 << 1; +const MOD_CONTROL: i64 = 1 << 2; +const MOD_NO_COMMAND: i64 = 1 << 3; + +/// Upper bound on menu items read per lookup, so a huge dynamic menu cannot +/// stall the call. +const MAX_ITEMS: usize = 2000; + +/// A hotkey as a menu key equivalent compares it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Combo { + shift: bool, + option: bool, + control: bool, + /// The uppercase character of the base key. + character: String, +} + +impl Combo { + /// A combo with Command and one printable base key; `None` for anything + /// else (no Command, a named key such as `return`), which keeps today's + /// key-event route. + pub(super) fn parse(modifiers: &[String], key: &str) -> Option<Self> { + let mut cmd = false; + let mut combo = Combo { + shift: false, + option: false, + control: false, + character: String::new(), + }; + for modifier in modifiers { + match modifier.to_ascii_lowercase().as_str() { + "cmd" | "command" | "super" | "meta" | "win" => cmd = true, + "shift" => combo.shift = true, + "option" | "alt" => combo.option = true, + "ctrl" | "control" => combo.control = true, + _ => return None, + } + } + let mut chars = key.chars(); + let (Some(ch), None) = (chars.next(), chars.next()) else { + return None; + }; + if !cmd || ch.is_whitespace() || ch.is_control() { + return None; + } + combo.character = ch.to_uppercase().collect(); + Some(combo) + } + + /// Whether a menu item with this `AXMenuItemCmdChar` and + /// `AXMenuItemCmdModifiers` fires on this combo. + fn matches(&self, cmd_char: &str, modifiers: i64) -> bool { + modifiers & MOD_NO_COMMAND == 0 + && cmd_char.to_uppercase() == self.character + && (modifiers & MOD_SHIFT != 0) == self.shift + && (modifiers & MOD_OPTION != 0) == self.option + && (modifiers & MOD_CONTROL != 0) == self.control + } +} + +/// A menu item found for a combo. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct MenuMatch { + /// Titles from the menu bar item down to the item. + pub path: Vec<String>, + pub enabled: bool, +} + +/// Find the item in `pid`'s menu bar whose key equivalent is `combo`. An +/// enabled match wins over a disabled one with the same equivalent. The +/// Apple menu is skipped: its commands belong to the system. +pub(super) fn find(pid: i32, combo: &Combo) -> Option<MenuMatch> { + unsafe { + let app = AXUIElementCreateApplication(pid); + if app.is_null() { + return None; + } + let _ = AXUIElementSetMessagingTimeout(app, 1.0); + let bar = copy_element_attr(app, "AXMenuBar"); + CFRelease(app as CFTypeRef); + let bar = bar?; + let mut found: Option<MenuMatch> = None; + let mut budget = MAX_ITEMS; + for (index, top) in semantic_items(bar).into_iter().enumerate() { + let title = copy_string_attr(top, "AXTitle").unwrap_or_default(); + if index > 0 || title != "Apple" { + search(top, vec![title], combo, &mut found, &mut budget, 0); + } + CFRelease(top as CFTypeRef); + if found.as_ref().is_some_and(|m| m.enabled) { + break; + } + } + CFRelease(bar as CFTypeRef); + found + } +} + +/// The items under a menu bar item or submenu item (through the untitled +/// `AXMenu` container AppKit puts between them). +unsafe fn semantic_items(parent: AXUIElementRef) -> Vec<AXUIElementRef> { + let mut out = Vec::new(); + for child in copy_children(parent) { + if copy_string_attr(child, "AXRole").as_deref() == Some("AXMenu") { + out.extend(copy_children(child)); + CFRelease(child as CFTypeRef); + } else { + out.push(child); + } + } + out +} + +unsafe fn search( + parent: AXUIElementRef, + path: Vec<String>, + combo: &Combo, + found: &mut Option<MenuMatch>, + budget: &mut usize, + depth: usize, +) { + for item in semantic_items(parent) { + if *budget == 0 || found.as_ref().is_some_and(|m| m.enabled) { + CFRelease(item as CFTypeRef); + continue; + } + *budget -= 1; + let title = copy_string_attr(item, "AXTitle").unwrap_or_default(); + let cmd_char = copy_string_attr(item, "AXMenuItemCmdChar").unwrap_or_default(); + if !cmd_char.is_empty() { + let modifiers = copy_number_attr(item, "AXMenuItemCmdModifiers").unwrap_or(0.0) as i64; + if combo.matches(&cmd_char, modifiers) { + let enabled = copy_bool_attr(item, "AXEnabled") != Some(false); + if found.is_none() || enabled { + let mut item_path = path.clone(); + item_path.push(title.clone()); + *found = Some(MenuMatch { + path: item_path, + enabled, + }); + } + } + } else if depth < 2 && !title.is_empty() { + let mut sub_path = path.clone(); + sub_path.push(title); + search(item, sub_path, combo, found, budget, depth + 1); + } + CFRelease(item as CFTypeRef); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn combo(modifiers: &[&str], key: &str) -> Option<Combo> { + let modifiers: Vec<String> = modifiers.iter().map(|m| m.to_string()).collect(); + Combo::parse(&modifiers, key) + } + + #[test] + fn only_command_combos_with_one_printable_key_are_menu_candidates() { + assert!(combo(&["cmd"], "s").is_some()); + assert!(combo(&["command", "shift"], "S").is_some()); + assert!(combo(&["cmd"], ",").is_some()); + assert_eq!(combo(&["ctrl"], "a"), None); + assert_eq!(combo(&["cmd"], "return"), None); + assert_eq!(combo(&["cmd"], "up"), None); + assert_eq!(combo(&["cmd", "hyper"], "s"), None); + } + + #[test] + fn modifiers_must_match_exactly() { + let save = combo(&["cmd"], "s").unwrap(); + assert!(save.matches("S", 0)); + assert!(save.matches("s", 0)); + assert!(!save.matches("S", MOD_SHIFT), "Save As is cmd+shift+s"); + assert!(!save.matches("S", MOD_NO_COMMAND)); + assert!(!save.matches("A", 0)); + let save_as = combo(&["cmd", "shift"], "s").unwrap(); + assert!(save_as.matches("S", MOD_SHIFT)); + assert!(!save_as.matches("S", 0)); + let minimize_all = combo(&["cmd", "option"], "m").unwrap(); + assert!(minimize_all.matches("M", MOD_OPTION)); + assert!(!minimize_all.matches("M", MOD_OPTION | MOD_CONTROL)); + } +} + +pub(super) fn run( + pid: i32, + window_id: u32, + item: MenuMatch, + save: bool, +) -> cua_driver_core::protocol::ToolResult { + use crate::ax::bindings::*; + use cua_driver_core::protocol::ToolResult; + let is_save = item.path.last().is_some_and(|title| { + matches!( + title.trim().to_lowercase().as_str(), + "save" | "保存" | "存储" + ) + }); + if save && is_save { + let path = unsafe { + let app = AXUIElementCreateApplication(pid); + if app.is_null() { + return ToolResult::computer_use_error( + "window_not_found", + "The app is unavailable", + false, + ); + } + let mut url = None; + for window in copy_ax_windows(app) { + if ax_get_window_id(window) == Some(window_id) { + url = copy_string_attr(window, "AXDocument"); + } + CFRelease(window as CFTypeRef); + } + CFRelease(app as CFTypeRef); + url.as_deref() + .and_then(crate::apple_events::file_url_to_path) + }; + let Some(path) = path else { + return ToolResult::computer_use_error( + "save_unavailable", + "This window has no existing file to save. Use its Save dialog.", + false, + ); + }; + let Some(bundle) = crate::apps::bundle_id_for_pid(pid) else { + return ToolResult::computer_use_error( + "save_unavailable", + "The app has no scripting identifier", + false, + ); + }; + return match crate::apple_events::save_document(&bundle, &bundle, &path) { + Ok(_) => { + ToolResult::text("Saved through Apple Events; the app reports no unsaved changes.") + .with_structured(serde_json::json!({ + "verified":true,"path":"apple_events","effect":"confirmed", + "delivery_mode":"background" + })) + } + Err(error) => ToolResult::computer_use_error( + error.code, + error.reason, + !matches!( + error.code, + "automation_denied" | "automation_consent_required" + ), + ), + }; + } + if !item.enabled { + return ToolResult::computer_use_error( + "menu_unavailable", + "The menu command is disabled in this background app", + false, + ); + } + unsafe { + let app = AXUIElementCreateApplication(pid); + if app.is_null() { + return ToolResult::computer_use_error("app_not_running", "The app stopped", false); + } + let bar = copy_element_attr(app, "AXMenuBar"); + CFRelease(app as CFTypeRef); + let Some(bar) = bar else { + return ToolResult::computer_use_error( + "menu_unavailable", + "The app exposes no menu bar", + false, + ); + }; + let target = super::invoke_menu::resolve_exact_prefix(bar, &item.path); + CFRelease(bar as CFTypeRef); + let target = match target { + Ok(target) => target, + Err(error) => return ToolResult::computer_use_error("menu_unavailable", error, false), + }; + let _guard = crate::key_focus_guard::protect(pid); + let result = perform_action(target, "AXPress"); + CFRelease(target as CFTypeRef); + if result == kAXErrorSuccess { + ToolResult::text("Sent background menu command; observe before retrying.") + .with_structured(serde_json::json!({ + "path":"ax","verified":false,"effect":"unverifiable", + "delivery_mode":"background" + })) + } else { + super::ax_action_unconfirmed("AXPress", None, result) + } + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/mod.rs index 0f3458fb263..306015fbaea 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/mod.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/mod.rs @@ -17,7 +17,7 @@ mod pasteboard; mod press_key; mod right_click; mod scroll; -mod select_text; +pub(crate) mod select_text; mod set_value; mod set_window_frame; mod type_text; @@ -216,6 +216,58 @@ pub(crate) fn background_refusal_result( .with_structured(structured) } +pub(super) fn ax_action_unconfirmed( + action: &str, + index: Option<usize>, + status: crate::ax::bindings::AXError, +) -> cua_driver_core::protocol::ToolResult { + let warning = format!( + "{action} returned {status}; delivery is uncertain. Observe before another action; this request was not replayed." + ); + let target = index.map_or_else( + || "the hit-tested element".to_owned(), + |i| format!("element [{i}]"), + ); + cua_driver_core::protocol::ToolResult::text(format!("Sent {action} to {target}. {warning}")) + .with_structured(serde_json::json!({ + "path": "ax", "delivery_mode": "background", "sent": true, "verified": false, + "effect": "unverifiable", "ax_status": status, "warning": warning + })) +} + +#[cfg(test)] +mod ax_dispatch_tests { + #[test] + fn ax_errors_remain_unconfirmed_through_the_public_action_projection() { + use cua_driver_core::action_record::{ActionEffect, ActionExecutionRecord}; + + for (tool, action) in [ + ("click", "AXPress"), + ("double_click", "AXOpen"), + ("right_click", "AXShowMenu"), + ] { + let result = super::ax_action_unconfirmed(action, None, -25202); + assert_ne!(result.is_error, Some(true)); + let structured = result.structured_content.unwrap(); + assert_eq!(structured["sent"], true); + assert!(structured["warning"].as_str().unwrap().contains("Observe")); + let record = ActionExecutionRecord::from_legacy( + tool, + &serde_json::json!({"pid": 101, "window_id": 501, "delivery_mode": "foreground"}), + &structured, + ) + .unwrap(); + assert_eq!(record.effect, ActionEffect::Unverifiable); + assert!(record.evidence.is_empty()); + let public = record.public_result().unwrap(); + public.validate_invariants().unwrap(); + let public = serde_json::to_value(public).unwrap(); + assert_eq!(public["effect"], "unverifiable"); + assert_eq!(public["delivery"]["mode"], "background"); + } + } +} + /// Exclusive per-process ownership of one background mutation. Callers must /// keep this value alive through actuator dispatch, focus restoration, and /// target-bound verification. @@ -288,10 +340,7 @@ pub(crate) async fn acquire_background_mutation(pid: i32) -> BackgroundMutationL } } -/// Finish the post-action observation window. Embedded interactive clients -/// that already observe the target continuously may opt out through the -/// private registry argument to avoid adding a one-second acknowledgement -/// delay to every input event. Regular MCP callers retain the full observer. +/// Collect immediately available window changes. Bounded settling belongs to observations. pub(crate) async fn finish_window_observation( snapshot: crate::window_change_detector::Snapshot, args: &serde_json::Value, @@ -615,6 +664,9 @@ pub fn load_driver_config() -> DriverConfig { /// used by CoreGraphics input APIs. Retina scaled modes cannot rely on the /// nominal backing factor alone, so derive the ratio from the actual PNG. pub async fn desktop_screenshot_point(x: f64, y: f64) -> (f64, f64) { + if cua_driver_core::tool::computer_use_native_dispatch() { + return (x, y); + } let ratio = tokio::task::spawn_blocking(|| { let logical_w = get_screen_size::main_screen_size().map(|(w, _, _)| w as f64); let shot_w = crate::capture::screenshot_display_bytes() @@ -962,10 +1014,6 @@ pub fn register_all( cua_driver_core::observation_revision::revision_tokens().clear_runtime(&runtime_scope); }); } - // Share the element cache with the recording-hook layer so it can - // resolve element_index → window-local screenshot coords for click.png. - crate::recording_hooks::set_element_cache(state.element_cache.clone()); - // Drop a disconnecting session's config overrides + owned cursor on // `session_end`. The daemon fans the session id out to this hook; // recording ownership is handled separately on the core RecordingSession. @@ -995,6 +1043,10 @@ pub fn register_all( registry.retain_session_revive_hook(revive_registration); } + cua_driver_core::desktop_keyboard::register( + registry, + Arc::new(crate::input::keyboard::DesktopKeyboard), + ); registry.register(Box::new(list_apps::ListAppsTool)); registry.register(Box::new(list_windows::ListWindowsTool)); registry.register(Box::new(get_window_state::GetWindowStateTool::new( @@ -1369,3 +1421,5 @@ mod recording_start_guard_tests { let _ = rec.stop_owner(None); } } + +mod menu_equivalent; diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/paste.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/paste.rs index 2a62341cc4f..20f08cb06cf 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/paste.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/paste.rs @@ -422,7 +422,7 @@ impl Tool for PasteTool { Ok(content) => content, Err(error) => return ToolResult::error(format!("paste: {error}")), }; - let foreground = input.app_context == Some(true); + let foreground = false; let mutation = if foreground { super::acquire_background_mutation(pid).await } else { diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/perform_secondary_action.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/perform_secondary_action.rs index a88c946ed43..893ad880047 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/perform_secondary_action.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/perform_secondary_action.rs @@ -16,12 +16,12 @@ use crate::ax::bindings::{copy_action_names, kAXErrorSuccess, perform_action, AX use super::ToolState; pub struct PerformSecondaryActionTool { - state: Arc<ToolState>, + _state: Arc<ToolState>, } impl PerformSecondaryActionTool { pub fn new(state: Arc<ToolState>) -> Self { - Self { state } + Self { _state: state } } } @@ -116,6 +116,7 @@ impl Tool for PerformSecondaryActionTool { Ok(resolved) => resolved, Err(error) => return error, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let cua_driver_core::element_token::ResolvedElement::Element { window_id: Some(window_id), element_index, @@ -125,20 +126,15 @@ impl Tool for PerformSecondaryActionTool { return ToolResult::error("perform_secondary_action requires a current element_token") .with_structured(json!({ "code": "invalid_element_target" })); }; - let element = - match self - .state - .element_cache - .get_element_retained(pid, window_id, element_index) - { - Some(element) => element, - None => { - return ToolResult::error( - "element_token no longer resolves in the current AX snapshot", - ) - .with_structured(json!({ "code": "stale_element_token" })) - } - }; + let element = match element_cache.get_element_retained(pid, window_id, element_index) { + Some(element) => element, + None => { + return ToolResult::error( + "element_token no longer resolves in the current AX snapshot", + ) + .with_structured(json!({ "code": "stale_element_token" })) + } + }; let element_ptr = element.as_ptr(); let _lease = match super::gate_background_window_action( pid, diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs index 67a0e074226..8e3e45a4b67 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs @@ -285,12 +285,14 @@ impl Tool for PressKeyTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { window_id: wid, element_index: idx, via_token: _, + .. } => (Some(idx), wid), }; @@ -332,7 +334,7 @@ impl Tool for PressKeyTool { // the element before the suppressed focus below dereferences it // (use-after-free → daemon crash). Guard lives to method end. let pre_focus_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) { - match self.state.element_cache.get_element_retained(pid, wid, idx) { + match element_cache.get_element_retained(pid, wid, idx) { Some(guard) => Some(guard), None => { return ToolResult::error(format!( diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs index ee81aa4f10b..6afb22c4763 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs @@ -117,12 +117,14 @@ impl Tool for RightClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { window_id: wid, element_index: idx, via_token: _, + .. } => (Some(idx), wid), }; let x = args.opt_f64("x"); @@ -150,7 +152,7 @@ impl Tool for RightClickTool { if let (Some(idx), Some(wid)) = (element_index, window_id) { // Retain out of the cache so a concurrent get_window_state can't // free the element mid-action (use-after-free → daemon crash). - let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) { + let element_guard = match element_cache.get_element_retained(pid, wid, idx) { Some(e) => e, None => { return ToolResult::error(format!( @@ -175,11 +177,16 @@ impl Tool for RightClickTool { let ax_result = tokio::task::spawn_blocking(move || try_ax_show_menu(element_ptr, idx)).await; match ax_result { - Ok(Some(message)) => return ToolResult::text(message), + Ok(Some(result)) => return result, Ok(None) => {} Err(error) => return ToolResult::error(format!("Task error: {error}")), } + if let Some(refusal) = super::click::libreoffice_pixel_refusal( + crate::apps::bundle_id_for_pid(pid).as_deref(), + ) { + return refusal; + } if let Err(refusal_result) = mutation_lease .gate_again( wid, @@ -256,6 +263,12 @@ impl Tool for RightClickTool { None }; + if let Some(refusal) = + super::click::libreoffice_pixel_refusal(crate::apps::bundle_id_for_pid(pid).as_deref()) + { + return refusal; + } + // Pin overlay above the target window before animating. if let Some(wid) = window_id { crate::cursor::overlay::send_command( @@ -325,7 +338,7 @@ impl Tool for RightClickTool { // ── Blocking AX path ───────────────────────────────────────────────────────── -fn try_ax_show_menu(element_ptr: usize, idx: usize) -> Option<String> { +fn try_ax_show_menu(element_ptr: usize, idx: usize) -> Option<ToolResult> { let element = element_ptr as AXUIElementRef; let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default(); @@ -345,13 +358,11 @@ fn try_ax_show_menu(element_ptr: usize, idx: usize) -> Option<String> { if advertised.iter().any(|a| a == "AXShowMenu") { let err = unsafe { perform_action(element, "AXShowMenu") }; if err == kAXErrorSuccess { - return Some(format!( + return Some(ToolResult::text(format!( "Shown menu for [{idx}] {role} \"{title}\" (AXShowMenu)." - )); + ))); } - // Advertised but the action failed — fall through to the pixel path - // rather than erroring out. - tracing::debug!("AXShowMenu returned {err} for [{idx}]; falling back to pixel right-click"); + return Some(super::ax_action_unconfirmed("AXShowMenu", Some(idx), err)); } None } diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs index 33567cf00ad..6e2ee98b138 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs @@ -63,10 +63,12 @@ fn def() -> &'static ToolDef { the driver synthesizes a real mouse-wheel event (CGEventCreateScrollWheelEvent, \ at that screen point. The renderer hit-tests the wheel at the \ cursor, so the scroll lands on whatever element is under the point — exactly \ - like physically rolling the wheel over it. This is the ONLY way to scroll a \ + like physically rolling the wheel over it. Use it to scroll a \ nested `overflow:auto` region (e.g. a scrollable <div> with no tabindex): such \ regions never take keyboard focus, so the keystroke path below no-ops on them. \ - Use this for inner/nested scrollers in web views.\n\n\ + Use this for inner/nested scrollers in web views. Background Electron \ + uses one native AX reveal instead of a wheel event, without activating \ + the window. The distance is approximate; observe afterward.\n\n\ • **Keystroke path (focused region)** — when you pass NO target (just pid + \ direction): synthesizes PageDown/PageUp (by='page') or Down/Up arrows \ (by='line'); horizontal uses Left/Right arrows. Drives the focused / page \ @@ -174,13 +176,8 @@ impl Tool for ScrollTool { // background CGEvents). Only the pixel-wheel path honors it; the // keystroke path is background-by-design and untouched. let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); - if !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid) { - return ToolResult::error( - "Background scroll is unavailable for Electron/Chromium windows on macOS." - .to_owned(), - ) - .with_structured(serde_json::json!({ "code": "background_unavailable" })); - } + let electron_background = + !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid); let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e, @@ -202,12 +199,14 @@ impl Tool for ScrollTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { window_id: wid, element_index: idx, via_token: _, + .. } => (Some(idx), wid), }; @@ -218,7 +217,7 @@ impl Tool for ScrollTool { // the element before the suppressed focus below dereferences it // (use-after-free → daemon crash). Guard lives to method end. let pre_focus_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) { - self.state.element_cache.get_element_retained(pid, wid, idx) + element_cache.get_element_retained(pid, wid, idx) } else { None }; @@ -235,6 +234,10 @@ impl Tool for ScrollTool { )); } } + if electron_background { + return electron_reveal_scroll(&self.state, pid, window_id, pre_focus_guard, &args) + .await; + } let mut _mutation_lease: Option<super::BackgroundMutationLease> = None; // AppKit exposes vertical scroll-bar buttons beneath the text area's @@ -268,10 +271,7 @@ impl Tool for ScrollTool { } } } - let native_element_guard = self - .state - .element_cache - .get_element_retained(pid, wid, index); + let native_element_guard = element_cache.get_element_retained(pid, wid, index); let direction_for_ax = direction.clone(); let by_for_ax = by.clone(); let foreground = delivery_mode.is_foreground(); @@ -705,6 +705,131 @@ impl Tool for ScrollTool { } } +fn reveal_unavailable(reason: &str) -> ToolResult { + ToolResult::error(format!("Background native scroll refused: {reason}")).with_structured( + serde_json::json!({ + "code": "background_unavailable", "sent": false, "effect": "refused" + }), + ) +} + +async fn electron_reveal_scroll( + state: &ToolState, + pid: i32, + window_id: Option<u32>, + element: Option<crate::ax::cache::RetainedElement>, + args: &Value, +) -> ToolResult { + use cua_driver_core::tool_args::ArgsExt; + let Some(wid) = window_id else { + return reveal_unavailable("an exact window_id is required."); + }; + let Some(bounds) = crate::windows::window_bounds_by_id(wid) else { + return reveal_unavailable("the requested window has no live frame."); + }; + let point = if element.is_none() { + if let (Some(x), Some(y)) = (args.opt_f64("x"), args.opt_f64("y")) { + let ratio = state.resize_registry.ratio(pid, Some(wid)).unwrap_or(1.0); + let frame = match super::px_frame::resolve_or_refuse(wid).await { + Ok(frame) => frame, + Err(mut refusal) => { + refusal + .structured_content + .get_or_insert_with(|| serde_json::json!({}))["sent"] = false.into(); + return refusal; + } + }; + let (sx, sy, _, _) = frame.to_screen(x * ratio, y * ratio); + (sx, sy) + } else { + ( + bounds.x + bounds.width / 2.0, + bounds.y + bounds.height / 2.0, + ) + } + } else { + (0.0, 0.0) + }; + let window = [bounds.x, bounds.y, bounds.width, bounds.height]; + let direction = args.str_or("direction", "down"); + let step = if args.str_or("by", "line") == "page" { + WHEEL_STEP_PAGE_PX + } else { + WHEEL_STEP_LINE_PX + }; + let distance = step as f64 * args.u64_or("amount", 3).clamp(1, 50) as f64; + let lease = super::acquire_background_mutation(pid).await; + let direction_for_read = direction.clone(); + let prepared = tokio::task::spawn_blocking(move || unsafe { + crate::ax::reveal_scroll::prepare( + pid, + wid, + element.as_ref().map(|held| held.as_ptr()), + point, + window, + &direction_for_read, + distance, + ) + }) + .await; + let target = match prepared { + Ok(Some(target)) => target, + Ok(None) => return reveal_unavailable("no safe content past the selected container's edge was found; observe the window before choosing another action."), + Err(error) => return reveal_unavailable(&format!("native target selection failed: {error}")), + }; + let role = target.container_role.clone(); + let result = tokio::task::spawn_blocking(move || { + use cua_driver_core::background_input::{ + decide_background_input, BackgroundAction, BackgroundInputDecision, ExactWindowTarget, + }; + // Keep both the native reference and lease in the worker if its async + // caller is cancelled during the exact-target reads or dispatch. + let _lease = lease; + let facts = + crate::ax::exact_target::gather_background_facts(pid, wid, Some(target.element_ptr())); + if let BackgroundInputDecision::Refuse(refusal) = decide_background_input( + ExactWindowTarget { + pid, + window_id: wid, + }, + &facts, + BackgroundAction::AxSemantic, + ) { + let mut result = super::background_refusal_result(pid, wid, &refusal); + result + .structured_content + .get_or_insert_with(|| serde_json::json!({}))["sent"] = false.into(); + return Err(result); + } + unsafe { target.dispatch(pid, wid) }.map_err(reveal_unavailable) + }) + .await; + match result { + Ok(Ok(status)) => { + let warning = if status == kAXErrorSuccess { + "Effect is not verified; observe before another action.".to_owned() + } else { + format!("AX returned {status}; delivery is uncertain. Observe before another action; this request was not replayed.") + }; + ToolResult::text(format!( + "Sent one native {direction} scroll request for {role}. {warning}" + )) + .with_structured(serde_json::json!({ + "path": "ax", "sent": true, "verified": false, + "effect": "unverifiable", "ax_status": status, + "warning": warning + })) + } + Ok(Err(refusal)) => refusal, + Err(error) => ToolResult::error(format!( + "Native scroll task interrupted: {error}. Delivery is uncertain; observe before another action." + )) + .with_structured(serde_json::json!({ + "path": "ax", "sent": true, "verified": false, "effect": "unverifiable" + })), + } +} + unsafe fn scroll_native_text_area( element: AXUIElementRef, direction: &str, diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/select_text.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/select_text.rs index 42c43d68173..bfe52b4b608 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/select_text.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/select_text.rs @@ -29,12 +29,12 @@ use super::ToolState; use crate::ax::bindings::*; pub struct SelectTextTool { - state: Arc<ToolState>, + _state: Arc<ToolState>, } impl SelectTextTool { pub fn new(state: Arc<ToolState>) -> Self { - Self { state } + Self { _state: state } } } @@ -59,7 +59,7 @@ impl Tool for SelectTextTool { if input.text.is_empty() { return refusal("invalid_arguments", "text must not be empty"); } - let index = match resolve_element_args( + let resolved = match resolve_element_args( pid, None, Some(&input.element_token), @@ -67,20 +67,20 @@ impl Tool for SelectTextTool { Some(window_id), "select_text", ) { - Ok(ResolvedElement::Element { element_index, .. }) => element_index, - Ok(_) => { + Ok(resolved) => resolved, + Err(error) => return error, + }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); + let index = match resolved { + ResolvedElement::Element { element_index, .. } => element_index, + _ => { return refusal( "invalid_arguments", "An observed element token is required.", ) } - Err(error) => return error, }; - let Some(element) = self - .state - .element_cache - .get_element_retained(pid, window_id, index) - else { + let Some(element) = element_cache.get_element_retained(pid, window_id, index) else { return refusal( "stale_element", "The text element is stale. Observe the app again.", @@ -355,7 +355,7 @@ unsafe fn click_field(element: AXUIElementRef, pid: i32, window_id: u32) -> anyh ) } -unsafe fn read_range(element: AXUIElementRef) -> Option<(usize, usize)> { +pub(crate) unsafe fn read_range(element: AXUIElementRef) -> Option<(usize, usize)> { let attr = CFString::new("AXSelectedTextRange"); let mut value = std::ptr::null(); let status = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs index 3ed0125487e..496d7628af2 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs @@ -3,9 +3,9 @@ //! Two modes, determined by the element's AXRole: //! //! * **AXPopUpButton**: Find the child option whose AXTitle or AXValue matches -//! `value` (case-insensitive) and AXPress it directly. The native macOS popup -//! menu is never opened, so focus is never stolen. Safari's front-document -//! fallback is used only when the exact requested window is already focused. +//! `value` (case-insensitive) and AXPress it. Open the menu first when its +//! options are not exposed while closed. Safari's front-document fallback +//! is used only when the exact requested window is already focused. //! //! * **Everything else**: Write `AXValue` directly (sliders, steppers, native //! text fields that expose a settable AXValue). @@ -20,8 +20,8 @@ use std::sync::Arc; use crate::apps; use crate::ax::bindings::{ - copy_children, copy_number_attr, copy_string_attr, kAXErrorSuccess, perform_action, - set_number_attr, set_string_attr, AXUIElementRef, + copy_bool_attr, copy_children, copy_number_attr, copy_string_attr, kAXErrorSuccess, + perform_action, set_number_attr, set_string_attr, AXUIElementRef, }; use crate::focus_guard; use crate::window_change_detector::WindowChangeDetector; @@ -48,9 +48,9 @@ fn def() -> &'static ToolDef { "Set a value on a UI element. Two modes depending on element role:\n\ \n\ - **AXPopUpButton / select dropdown**: finds the child option whose \ - title or value matches `value` (case-insensitive) and AXPresses it \ - directly — the native macOS popup menu is never opened, so focus \ - is never stolen. Use this for HTML <select> elements in Safari or \ + title or value matches `value` (case-insensitive) and AXPresses it. \ + Opens the menu first when the requested option is not exposed \ + while closed. Use this for HTML <select> elements in Safari or \ any native NSPopUpButton.\n\ \n\ - **All other elements**: writes AXValue directly (sliders, steppers, \ @@ -120,6 +120,7 @@ impl Tool for SetValueTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => { return ToolResult::error( @@ -131,6 +132,7 @@ impl Tool for SetValueTool { window_id: Some(wid), element_index: idx, via_token: _, + .. } => (idx, wid), cua_driver_core::element_token::ResolvedElement::Element { window_id: None, .. @@ -145,19 +147,15 @@ impl Tool for SetValueTool { // Retain out of the cache so a concurrent get_window_state can't free // the element mid-action (use-after-free → daemon crash). Guard lives // to the end of this method, past the AX write below. - let element_guard = - match self - .state - .element_cache - .get_element_retained(pid, window_id, element_index) - { - Some(e) => e, - None => { - return ToolResult::error(format!( - "Element index {element_index} not found. Call get_window_state first." - )) - } - }; + let element_guard = match element_cache.get_element_retained(pid, window_id, element_index) + { + Some(e) => e, + None => { + return ToolResult::error(format!( + "Element index {element_index} not found. Call get_window_state first." + )) + } + }; let element_ptr = element_guard.as_ptr(); // set_value is an always-background semantic AX mutation. Re-prove @@ -176,6 +174,64 @@ impl Tool for SetValueTool { Err(refusal_result) => return refusal_result, }; + let rename = unsafe { + let element = element_ptr as AXUIElementRef; + let filename = copy_string_attr(element, "AXFilename"); + let role = copy_string_attr(element, "AXRole"); + let finder = crate::apps::bundle_id_for_pid(pid).as_deref() == Some("com.apple.finder"); + if finder + && role.as_deref() == Some("AXTextField") + && copy_string_attr(element, "AXIdentifier").as_deref() == Some("Name") + && copy_bool_attr(element, "AXFocused") != Some(true) + { + return ToolResult::computer_use_error( + "not_settable", + "Finder Get Info name requires the explicit rename UI", + false, + ); + } + if filename.is_some_and(|name| !name.is_empty()) + && copy_bool_attr(element, "AXFocused") != Some(true) + { + let path = crate::ax::app_text::element_url(element) + .as_deref() + .and_then(crate::apple_events::file_url_to_path); + if !finder || path.is_none() { + return ToolResult::computer_use_error( + "not_settable", + "This listed file name requires the application's rename action", + false, + ); + } + path + } else { + None + } + }; + if let Some(path) = rename { + let renamed = tokio::task::spawn_blocking(move || { + crate::apple_events::finder_rename(&path, &value) + }) + .await; + return match renamed { + Ok(Ok(path)) => { + ToolResult::text(format!("Renamed and verified on disk: {}", path.display())) + .with_structured(serde_json::json!({ + "path":"apple_events","verified":true,"effect":"confirmed", + "delivery_mode":"background" + })) + } + Ok(Err(error)) => ToolResult::computer_use_error( + error.code, + error.reason, + !matches!(error.code, "automation_denied" | "invalid_file_name"), + ), + Err(error) => { + ToolResult::computer_use_error("rename_unconfirmed", error.to_string(), true) + } + }; + } + let cursor_key = super::cursor_tools::resolve_cursor_key(&args); let center_ptr = element_ptr as usize; if let Ok(Some((screen_x, screen_y))) = tokio::task::spawn_blocking(move || unsafe { @@ -361,6 +417,11 @@ fn set_value_blocking( changed: None, }) } else { + if !matches!(role.as_str(), "AXSlider" | "AXStepper") + && !unsafe { crate::ax::bindings::is_attribute_settable(element, "AXValue") } + { + anyhow::bail!("not_settable: this control does not permit replacing AXValue"); + } // Default path: write AXValue directly. Numeric controls (AXSlider / // AXStepper) reject a CFString with -25201 and need a CFNumber; text // fields take a CFString. Try numeric first when the value parses as a @@ -392,6 +453,17 @@ fn set_value_blocking( value, numeric_target.is_some(), ); + if verified == Some(false) { + anyhow::bail!("write_not_applied: replacing AXValue did not match value readback; observe before retrying"); + } + if verified == Some(true) + && changed == Some(true) + && matches!(role.as_str(), "AXTextArea" | "AXTextField") + && unsafe { crate::document_edit::ensure_marked_after_ax_write(pid, element, true) } + == crate::document_edit::DocumentEdited::NotMarked + { + anyhow::bail!("write_not_applied: text value changed but the document could not register it for saving; observe before retrying"); + } let suffix = match (verified, changed) { (Some(true), Some(false)) => " Value already matched; write was idempotent.", (Some(true), _) => "", @@ -531,6 +603,147 @@ fn step_to_value(element: AXUIElementRef, target: f64) -> bool { // ── AXPopUpButton path ─────────────────────────────────────────────────────── +// Popup selection ported from upstream 0.35.4. +/// How long to wait for a popup's menu to publish its items after AXPress. +const POPUP_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(2500); +/// Consecutive unchanged polls that mean the menu has finished filling in. +const POPUP_STABLE_POLLS: u32 = 3; +/// How long a dismissed menu may take to close before Escape is sent. +const POPUP_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(300); +const POPUP_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); +/// AX messaging timeout while opening the menu. The target app enters its +/// menu-tracking loop inside the AXPress, so the call would otherwise block +/// for the full default timeout (~1.5 s) with the menu already open. +const POPUP_PRESS_TIMEOUT_SECONDS: f32 = 0.5; + +/// One selectable entry of a popup: the retained AX element plus the title +/// and value it reports. +struct PopupOption { + element: AXUIElementRef, + title: String, + value: String, +} + +/// Release every retained option. +fn release_options(options: &[PopupOption]) { + for option in options { + unsafe { CFRelease(option.element as _) }; + } +} + +/// The popup's options as AX exposes them: its direct children, or, when a +/// child is an `AXMenu` (AppKit `NSPopUpButton`, Chromium `<select>`), that +/// menu's `AXMenuItem` children. Menus without items yield nothing, so an +/// unopened popup reads as empty instead of as one untitled option. +fn popup_options(popup: AXUIElementRef) -> Vec<PopupOption> { + let mut options = Vec::new(); + for child in unsafe { copy_children(popup) } { + let role = unsafe { copy_string_attr(child, "AXRole") }.unwrap_or_default(); + if role == "AXMenu" { + for item in unsafe { copy_children(child) } { + options.push(describe_option(item)); + } + unsafe { CFRelease(child as _) }; + } else { + options.push(describe_option(child)); + } + } + options.retain(|option| { + let keep = !(option.title.is_empty() && option.value.is_empty()); + if !keep { + unsafe { + CFRelease(option.element as _); + } + } + keep + }); + options +} + +fn describe_option(element: AXUIElementRef) -> PopupOption { + PopupOption { + element, + title: unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default(), + value: unsafe { copy_string_attr(element, "AXValue") }.unwrap_or_default(), + } +} + +/// Index of the option whose title or value equals `value`, ignoring case and +/// surrounding whitespace. +fn matching_option(options: &[(String, String)], value: &str) -> Option<usize> { + let wanted = value.trim().to_lowercase(); + let mut matches = options + .iter() + .enumerate() + .filter(|(_, (title, option_value))| { + title.trim().to_lowercase() == wanted || option_value.trim().to_lowercase() == wanted + }); + let index = matches.next()?.0; + matches.next().is_none().then_some(index) +} + +fn option_pairs(options: &[PopupOption]) -> Vec<(String, String)> { + options + .iter() + .map(|option| (option.title.clone(), option.value.clone())) + .collect() +} + +fn describe_available(options: &[PopupOption]) -> String { + options + .iter() + .map(|option| { + let label = if option.title.is_empty() { + &option.value + } else { + &option.title + }; + format!("\"{label}\"") + }) + .collect::<Vec<_>>() + .join(", ") +} + +/// Whether the popup's menu is open right now: an `AXMenu` child that already +/// lists items. A closed AppKit popup has an empty `AXMenu` (or none), and a +/// closed Chromium popup lists only its selected item directly. +fn popup_menu_is_open(popup: AXUIElementRef) -> bool { + let mut open = false; + for child in unsafe { copy_children(popup) } { + if unsafe { copy_string_attr(child, "AXRole") }.as_deref() == Some("AXMenu") { + let items = unsafe { copy_children(child) }; + open |= !items.is_empty(); + for item in items { + unsafe { CFRelease(item as _) }; + } + } + unsafe { CFRelease(child as _) }; + } + open +} + +/// Close a menu this call opened, without choosing anything. `AXCancel` is +/// enough for AppKit; Chromium's menu ignores it, so fall back to Escape sent +/// to the app (the open menu is what receives it). +fn dismiss_popup_menu(popup: AXUIElementRef, pid: i32) { + for child in unsafe { copy_children(popup) } { + let role = unsafe { copy_string_attr(child, "AXRole") }.unwrap_or_default(); + if role == "AXMenu" { + let _ = unsafe { perform_action(child, "AXCancel") }; + } + unsafe { CFRelease(child as _) }; + } + let deadline = std::time::Instant::now() + POPUP_CLOSE_TIMEOUT; + while popup_menu_is_open(popup) { + if std::time::Instant::now() >= deadline { + let _ = crate::input::keyboard::press_key_no_auth(pid, "escape", &[]); + std::thread::sleep(POPUP_CLOSE_TIMEOUT); + return; + } + std::thread::sleep(POPUP_POLL_INTERVAL); + } +} + fn select_popup_option( element: AXUIElementRef, element_index: usize, @@ -539,82 +752,180 @@ fn select_popup_option( value: &str, element_title: &str, ) -> anyhow::Result<String> { - let children = unsafe { copy_children(element) }; + let mut options = popup_options(element); + let mut opened_menu = false; + let mut press_thread = None; - if !children.is_empty() { - // Strategy 1: AX children (native AppKit NSPopUpButton). - let value_lower = value.to_lowercase(); - let mut matched_idx: Option<usize> = None; - let mut available: Vec<String> = Vec::with_capacity(children.len()); - - for (i, &child) in children.iter().enumerate() { - let child_title = unsafe { copy_string_attr(child, "AXTitle") }.unwrap_or_default(); - let child_value = unsafe { copy_string_attr(child, "AXValue") }.unwrap_or_default(); - available.push(child_title.clone()); - if child_title.to_lowercase() == value_lower - || child_value.to_lowercase() == value_lower - { - matched_idx = Some(i); + if matching_option(&option_pairs(&options), value).is_none() { + // Safari/WebKit: no AX children while the popup is closed. Set the + // <select> through the DOM instead of opening a menu. + if options.is_empty() { + let app_name = crate::apps::get_app_name_for_pid(pid).unwrap_or_default(); + if app_name == "Safari" { + if !safari_front_document_matches( + pid, + window_id, + apps::frontmost_pid(), + crate::ax::bindings::focused_window_id_of_pid(pid), + ) { + release_options(&options); + return Err(anyhow::Error::new(SafariFrontDocumentUnavailable)); + } + release_options(&options); + return set_select_via_js(element_index, element_title, value); + } + } + // AppKit NSPopUpButton and Chromium <select> publish their items only + // while the menu is open (a closed Chromium popup lists just the + // selected one). Open it, wait for the full list, press the match, + // and close the menu again whatever happens. The menu's tracking + // steals key focus; in the background the key-focus guard armed by + // the caller hands it straight back to the user's front window. + // + // The AXPress returns only once the app's menu tracking lets it (or + // the messaging timeout fires), so run it on its own thread and read + // the items as soon as they appear: the menu, and with it the key + // focus it holds, stays open for the shortest time. + let already_open = popup_menu_is_open(element); + let closed_titles = if already_open { + Vec::new() + } else { + option_pairs(&options) + }; + release_options(&options); + opened_menu = true; + // A menu left open by an earlier click is already showing its items; + // pressing again would close it. + if !already_open { + let popup_address = element as usize; + press_thread = Some(std::thread::spawn(move || unsafe { + let popup = popup_address as AXUIElementRef; + crate::ax::bindings::AXUIElementSetMessagingTimeout( + popup, + POPUP_PRESS_TIMEOUT_SECONDS, + ); + let _ = perform_action(popup, "AXPress"); + crate::ax::bindings::AXUIElementSetMessagingTimeout(popup, 0.0); + })); + } + let deadline = std::time::Instant::now() + POPUP_OPEN_TIMEOUT; + let mut last_seen: Vec<(String, String)> = Vec::new(); + let mut stable_polls = 0; + loop { + options = popup_options(element); + let seen = option_pairs(&options); + if matching_option(&seen, value).is_some() { break; } + // The menu is open once the list differs from the closed one; it + // is complete once the list stops changing. + if !seen.is_empty() && seen != closed_titles { + stable_polls = if seen == last_seen { + stable_polls + 1 + } else { + 0 + }; + if stable_polls >= POPUP_STABLE_POLLS { + break; + } + } + last_seen = seen; + if std::time::Instant::now() >= deadline { + break; + } + release_options(&options); + std::thread::sleep(POPUP_POLL_INTERVAL); } + if options.is_empty() { + dismiss_popup_menu(element, pid); + if let Some(press) = press_thread.take() { + let _ = press.join(); + } + anyhow::bail!( + "AXPopUpButton [{element_index}] \"{element_title}\" exposed no options even \ + after its menu was opened, so nothing was selected. Click the popup, then \ + choose the option with press_key (down, return) or a pixel click on the item." + ) + } + } - let result = if let Some(i) = matched_idx { - let child = children[i]; - let opt_title = - unsafe { copy_string_attr(child, "AXTitle") }.unwrap_or_else(|| value.to_string()); - let err = unsafe { perform_action(child, "AXPress") }; + let result = match matching_option(&option_pairs(&options), value) { + Some(i) => { + let option = &options[i]; + let err = unsafe { perform_action(option.element, "AXPress") }; if err == kAXErrorSuccess { + let label = if option.title.is_empty() { + &option.value + } else { + &option.title + }; Ok(format!( - "✅ Selected '{opt_title}' in AXPopUpButton [{element_index}] \ - \"{element_title}\" via AX child AXPress." + "✅ Selected '{label}' in AXPopUpButton [{element_index}] \ + \"{element_title}\" via AX menu item AXPress{}.", + if opened_menu { + " (the popup's menu was opened for the selection and closed again)" + } else { + "" + } )) } else { - anyhow::bail!("AXPress on child option failed with error {err}") - } - } else { - let avail = available - .iter() - .map(|t| format!("\"{t}\"")) - .collect::<Vec<_>>() - .join(", "); - anyhow::bail!( - "No AX child matching '{value}' in AXPopUpButton [{element_index}] \ - \"{element_title}\". Available: [{avail}]" - ) - }; - - // Release children (copy_children retains each one). - for &child in &children { - unsafe { - CFRelease(child as _); + if opened_menu { + dismiss_popup_menu(element, pid); + } + Err(anyhow::anyhow!( + "AXPress on menu item failed with error {err}" + )) } } + None => { + if opened_menu { + dismiss_popup_menu(element, pid); + } + Err(anyhow::anyhow!( + "No option matching '{value}' in AXPopUpButton [{element_index}] \ + \"{element_title}\". Available: [{}]", + describe_available(&options) + )) + } + }; + // The opening AXPress returns once the menu is closed or its timeout + // fires; wait for it so no AX call outlives this tool call. + if let Some(press) = press_thread { + let _ = press.join(); + } + release_options(&options); + result +} - return result; +#[cfg(test)] +mod popup_option_tests { + use super::matching_option; + + fn pairs(titles: &[&str]) -> Vec<(String, String)> { + titles + .iter() + .map(|title| ((*title).to_owned(), String::new())) + .collect() } - // Strategy 2: Safari/WebKit — no AX children when popup is closed. - // Use osascript do JavaScript to set the <select> element's DOM value. - let app_name = crate::apps::get_app_name_for_pid(pid).unwrap_or_default(); - - if app_name != "Safari" { - anyhow::bail!( - "AXPopUpButton [{element_index}] '{element_title}' has no AX children and \ - target is '{app_name}' (not Safari) — no fallback available." - ) + #[test] + fn matches_title_ignoring_case_and_whitespace() { + let options = pairs(&["Open", "Duplicate", "Closed"]); + assert_eq!(matching_option(&options, "duplicate"), Some(1)); + assert_eq!(matching_option(&options, " Closed "), Some(2)); + assert_eq!(matching_option(&options, "Pending"), None); } - if !safari_front_document_matches( - pid, - window_id, - apps::frontmost_pid(), - crate::ax::bindings::focused_window_id_of_pid(pid), - ) { - return Err(anyhow::Error::new(SafariFrontDocumentUnavailable)); + #[test] + fn refuses_ambiguous_options() { + assert_eq!(matching_option(&pairs(&["Same", "same"]), "Same"), None); } - set_select_via_js(element_index, element_title, value) + #[test] + fn matches_value_when_title_differs() { + let options = vec![("Duplicate of".to_owned(), "dup".to_owned())]; + assert_eq!(matching_option(&options, "DUP"), Some(0)); + } } // ── Safari JavaScript fallback ─────────────────────────────────────────────── diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs index b721d984001..4296eb7230e 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs @@ -227,12 +227,14 @@ impl Tool for TypeTextTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { window_id: wid, element_index: idx, via_token: _, + .. } => (Some(idx), wid), }; let delay_ms = args.u64_or("delay_ms", 30); @@ -267,7 +269,7 @@ impl Tool for TypeTextTool { // the blocking type below dereferences it (use-after-free → daemon // crash). The guard lives to method end, past type_text_blocking. let element_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) { - match self.state.element_cache.get_element_retained(pid, wid, idx) { + match element_cache.get_element_retained(pid, wid, idx) { Some(e) => Some((e, idx)), None => { return ToolResult::error(format!( @@ -380,8 +382,8 @@ impl Tool for TypeTextTool { // AX value-set is silently dropped — see crate::terminal docs. // Skip the AX path entirely so the caller never sees the // "success but nothing typed" symptom. - let synthesize_only = crate::terminal::is_terminal_pid(pid) - || args.get("app_context").and_then(Value::as_bool) == Some(true); + let app_context = args.get("app_context").and_then(Value::as_bool) == Some(true); + let synthesize_only = crate::terminal::is_terminal_pid(pid); let blocking_policy = keyboard_policy.clone(); let result = focus_guard::with_focus_suppressed( @@ -390,6 +392,15 @@ impl Tool for TypeTextTool { "type_text.AXSelectedText", || async move { tokio::task::spawn_blocking(move || { + if app_context && !delivery_mode.is_foreground() { + return type_app_text( + pid, + window_id.expect("app window"), + &text_clone, + delay_ms, + &blocking_policy, + ); + } type_text_blocking( pid, &text_clone, @@ -1105,6 +1116,88 @@ fn await_typed_delivery( /// Returns `(detail, path, verified)`. `verified` is `true` only when a /// read-back positively confirmed the text; `false` means the agent must /// confirm the result from a fresh observation before deciding what to do next. +fn type_app_text( + pid: i32, + window: u32, + text: &str, + delay_ms: u64, + policy: &BackgroundKeyboardPolicy, +) -> anyhow::Result<TypeTextDelivery> { + struct Release(AXUIElementRef); + impl Drop for Release { + fn drop(&mut self) { + unsafe { + CFRelease(self.0 as _); + } + } + } + let target = unsafe { crate::ax::exact_target::focused_element_in_window(pid, window) } + .ok_or_else(|| { + anyhow::anyhow!("no_focused_text_input: no text input is focused in this window") + })?; + let _release = Release(target); + let role = unsafe { copy_string_attr(target, "AXRole") }.unwrap_or_default(); + if !matches!( + role.as_str(), + "AXTextField" | "AXTextArea" | "AXSearchField" | "AXComboBox" + ) { + anyhow::bail!("no_focused_text_input: focused role {role} is not a text input"); + } + let before = unsafe { copy_string_attr(target, "AXValue") }; + let (path, verified) = if text.is_ascii() { + if let BackgroundKeyboardPolicy::SemanticOnly(refusal) = policy { + return Ok(TypeTextDelivery::Refused(refusal.clone())); + } + crate::input::skylight::prepare_background_keyboard(pid, window)?; + crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)?; + (PATH_KEY_EVENTS, false) + } else { + if !unsafe { crate::ax::bindings::is_attribute_settable(target, "AXSelectedText") } { + anyhow::bail!("not_settable: focused input cannot accept a Unicode insertion through accessibility"); + } + let before = before.ok_or_else(|| { + anyhow::anyhow!( + "not_settable: focused input value cannot be read for Unicode verification" + ) + })?; + let (location, length) = + unsafe { super::select_text::read_range(target) }.ok_or_else(|| { + anyhow::anyhow!("no_focused_text_input: the text caret cannot be read") + })?; + let mut expected = before.encode_utf16().collect::<Vec<_>>(); + let end = location + .checked_add(length) + .filter(|end| *end <= expected.len()) + .ok_or_else(|| { + anyhow::anyhow!( + "no_focused_text_input: the text caret is outside the current value" + ) + })?; + expected.splice(location..end, text.encode_utf16()); + let expected = String::from_utf16(&expected)?; + let status = unsafe { set_string_attr(target, "AXSelectedText", text) }; + if status != kAXErrorSuccess { + anyhow::bail!("write_not_applied: Unicode insertion returned AX error {status}; observe before retrying"); + } + let after = unsafe { copy_string_attr(target, "AXValue") }; + if after.as_deref() != Some(expected.as_str()) { + anyhow::bail!("write_not_applied: Unicode insertion did not match value readback; observe before retrying"); + } + if unsafe { crate::document_edit::ensure_marked_after_ax_write(pid, target, false) } + == crate::document_edit::DocumentEdited::NotMarked + { + anyhow::bail!("write_not_applied: text readback matched but the document could not register the edit for saving; observe before retrying"); + } + (PATH_AX, true) + }; + Ok(TypeTextDelivery::Typed(TypeTextOutcome { + detail: " at the current text caret".into(), + path, + verified, + delivered_chars: verified.then_some(text.chars().count()), + })) +} + fn type_text_blocking( pid: i32, text: &str, diff --git a/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs b/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs index 342fe0ab846..78530aea59c 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/tools/type_text_chars.rs @@ -9,12 +9,12 @@ use std::sync::Arc; use super::ToolState; pub struct TypeTextCharsTool { - state: Arc<ToolState>, + _state: Arc<ToolState>, } impl TypeTextCharsTool { pub fn new(state: Arc<ToolState>) -> Self { - Self { state } + Self { _state: state } } } @@ -84,12 +84,14 @@ impl Tool for TypeTextCharsTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::ax::cache::BoundElementCache::from_resolved(&resolved); let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { window_id: wid, element_index: idx, via_token: _, + .. } => (Some(idx), wid), }; let type_chars_only = args.bool_or("type_chars_only", false); @@ -98,7 +100,7 @@ impl Tool for TypeTextCharsTool { // get_window_state can't free it during the gate/focus below // (use-after-free → daemon crash). Guard lives past the focus call. let element_guard = if let (Some(idx), Some(wid)) = (element_index, window_id) { - match self.state.element_cache.get_element_retained(pid, wid, idx) { + match element_cache.get_element_retained(pid, wid, idx) { Some(guard) => Some(guard), None => { return ToolResult::error(format!( diff --git a/packages/cua-driver/rust/crates/platform-macos/src/video_sckit.rs b/packages/cua-driver/rust/crates/platform-macos/src/video_sckit.rs index 4e534ad5c5b..d11fc8c67c0 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/video_sckit.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/video_sckit.rs @@ -19,11 +19,19 @@ //! SCStream config + SCRecordingOutput pointing at the mp4 path, //! attaches the recording output, calls `start_capture()`. //! 2. Caller stays alive while recording. -//! 3. `stop()` calls `stop_capture()` (which finalises the mp4 moov -//! atom) and returns the elapsed-time metadata. +//! 3. `stop()` calls `stop_capture()`, then waits (bounded by +//! [`FINISH_TIMEOUT`]) for SCRecordingOutput's delegate to report that +//! the mp4 is written, and returns the elapsed-time metadata. +//! +//! `stop_capture()` returning does not mean the file is complete: +//! SCRecordingOutput finishes writing asynchronously and signals it through +//! `recordingOutputDidFinishRecording:` (or `recordingOutput:didFailWithError:`). +//! Reporting the video as finalized before that callback let an immediate +//! reader see a 0-byte file for short recordings. use std::path::Path; -use std::time::Instant; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; use cua_driver_core::video::{VideoBackend, VideoBackendFactory, VideoMetadata}; @@ -32,9 +40,98 @@ use screencapturekit::prelude::{ }; use screencapturekit::recording_output::{ SCRecordingOutput, SCRecordingOutputCodec, SCRecordingOutputConfiguration, - SCRecordingOutputFileType, + SCRecordingOutputDelegate, SCRecordingOutputFileType, }; +/// Upper bound on how long `stop()` waits for SCRecordingOutput to finish +/// writing the mp4 after capture stops. Finalizing normally takes tens of +/// milliseconds; the bound keeps a lost callback from hanging `stop_recording`. +const FINISH_TIMEOUT: Duration = Duration::from_secs(15); + +/// Terminal outcome reported by the SCRecordingOutput delegate. +#[derive(Debug, Clone, PartialEq, Eq)] +enum FinishOutcome { + Finished, + Failed(String), +} + +/// Why waiting for the recording to finish did not succeed. +#[derive(Debug, Clone, PartialEq, Eq)] +enum FinishWaitError { + /// The delegate reported `recordingOutput:didFailWithError:`. + Failed(String), + /// Neither delegate callback arrived within the timeout. + TimedOut(Duration), +} + +impl std::fmt::Display for FinishWaitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Failed(error) => write!(f, "SCRecordingOutput failed to write the recording: {error}"), + Self::TimedOut(timeout) => write!( + f, + "SCRecordingOutput did not finish writing the recording within {} ms after capture stopped", + timeout.as_millis() + ), + } + } +} + +/// One-shot latch the delegate sets when the recording output reaches a +/// terminal state. The first terminal event wins; later events are ignored. +#[derive(Debug, Default)] +struct RecordingFinishSignal { + outcome: Mutex<Option<FinishOutcome>>, + changed: Condvar, +} + +impl RecordingFinishSignal { + fn settle(&self, outcome: FinishOutcome) { + let mut slot = self.outcome.lock().unwrap_or_else(|e| e.into_inner()); + if slot.is_none() { + *slot = Some(outcome); + self.changed.notify_all(); + } + } + + fn mark_finished(&self) { + self.settle(FinishOutcome::Finished); + } + + fn mark_failed(&self, error: String) { + self.settle(FinishOutcome::Failed(error)); + } + + /// Blocks until the recording output reports a terminal state or + /// `timeout` elapses. + fn wait(&self, timeout: Duration) -> Result<(), FinishWaitError> { + let slot = self.outcome.lock().unwrap_or_else(|e| e.into_inner()); + let (slot, _) = self + .changed + .wait_timeout_while(slot, timeout, |outcome| outcome.is_none()) + .unwrap_or_else(|e| e.into_inner()); + match slot.as_ref() { + Some(FinishOutcome::Finished) => Ok(()), + Some(FinishOutcome::Failed(error)) => Err(FinishWaitError::Failed(error.clone())), + None => Err(FinishWaitError::TimedOut(timeout)), + } + } +} + +/// Delegate that forwards SCRecordingOutput's terminal callbacks into a +/// [`RecordingFinishSignal`]. +struct FinishDelegate(Arc<RecordingFinishSignal>); + +impl SCRecordingOutputDelegate for FinishDelegate { + fn recording_did_fail(&self, error: String) { + self.0.mark_failed(error); + } + + fn recording_did_finish(&self) { + self.0.mark_finished(); + } +} + pub struct SckitVideoBackendFactory; impl VideoBackendFactory for SckitVideoBackendFactory { @@ -47,9 +144,10 @@ pub struct SckitVideoBackend { stream: SCStream, // SCStream's add_recording_output is non-owning — Apple's API requires // the SCRecordingOutput stay alive for the stream's lifetime, so we - // keep it parked here. Dropping it before stop_capture aborts the - // encode mid-file. + // keep it parked here. Dropping it before the finish callback aborts + // the encode mid-file and unregisters the delegate. _recording: SCRecordingOutput, + finish: Arc<RecordingFinishSignal>, output_path: std::path::PathBuf, started_at: Instant, } @@ -108,13 +206,16 @@ impl SckitVideoBackend { .with_video_codec(SCRecordingOutputCodec::H264) .with_output_file_type(SCRecordingOutputFileType::MP4); - let recording = SCRecordingOutput::new(&rec_config).ok_or_else(|| { - anyhow::anyhow!( - "SCRecordingOutput::new returned nil — macOS 15.0+ is required for \ + let finish = Arc::new(RecordingFinishSignal::default()); + let recording = + SCRecordingOutput::new_with_delegate(&rec_config, FinishDelegate(finish.clone())) + .ok_or_else(|| { + anyhow::anyhow!( + "SCRecordingOutput::new_with_delegate returned nil — macOS 15.0+ is required for \ native ScreenCaptureKit video; older macOS needs to use the ffmpeg \ backend (currently disabled on macOS)." ) - })?; + })?; let stream = SCStream::new(&filter, &config); stream @@ -135,6 +236,7 @@ impl SckitVideoBackend { Ok(Self { stream, _recording: recording, + finish, output_path: output_path.to_path_buf(), started_at: Instant::now(), }) @@ -144,20 +246,113 @@ impl SckitVideoBackend { impl VideoBackend for SckitVideoBackend { fn stop(self: Box<Self>) -> anyhow::Result<VideoMetadata> { let elapsed = self.started_at.elapsed(); - // SCStream::stop_capture finalises the mp4 moov atom synchronously - // on the recording output before returning. Errors here mean the - // file may be unplayable — surface as `finalized: false`. - let finalized = self.stream.stop_capture().is_ok(); - if !finalized { + // stop_capture() only stops the stream; SCRecordingOutput writes the + // rest of the mp4 asynchronously. Errors here mean the file may be + // unplayable — surface as `finalized: false`. + if let Err(error) = self.stream.stop_capture() { tracing::warn!( target: "recording", + %error, "SCStream::stop_capture failed; recording.mp4 may be incomplete" ); + return Ok(VideoMetadata { + path: self.output_path, + duration_ms: elapsed.as_millis() as u64, + finalized: false, + }); } + let wait_started = Instant::now(); + self.finish + .wait(FINISH_TIMEOUT) + .map_err(|error| anyhow::anyhow!("{error} ({})", self.output_path.display()))?; + tracing::debug!( + target: "recording", + path = %self.output_path.display(), + finish_wait_ms = wait_started.elapsed().as_millis() as u64, + "sckit recording finished writing" + ); Ok(VideoMetadata { path: self.output_path, duration_ms: elapsed.as_millis() as u64, - finalized, + finalized: true, }) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + + #[test] + fn finish_before_wait_returns_immediately() { + let signal = RecordingFinishSignal::default(); + signal.mark_finished(); + assert_eq!(signal.wait(Duration::from_millis(1)), Ok(())); + } + + #[test] + fn wait_blocks_until_the_delegate_reports_finish() { + let signal = Arc::new(RecordingFinishSignal::default()); + let delegate = FinishDelegate(signal.clone()); + let started = Instant::now(); + let finisher = thread::spawn(move || { + thread::sleep(Duration::from_millis(150)); + delegate.recording_did_finish(); + }); + assert_eq!(signal.wait(Duration::from_secs(10)), Ok(())); + assert!( + started.elapsed() >= Duration::from_millis(150), + "wait returned before the finish callback" + ); + finisher.join().unwrap(); + } + + #[test] + fn delegate_failure_is_reported_as_an_error() { + let signal = Arc::new(RecordingFinishSignal::default()); + FinishDelegate(signal.clone()).recording_did_fail("disk full".into()); + let error = signal.wait(Duration::from_secs(1)).unwrap_err(); + assert_eq!(error, FinishWaitError::Failed("disk full".into())); + assert!(error.to_string().contains("disk full")); + } + + #[test] + fn first_terminal_event_wins() { + let signal = RecordingFinishSignal::default(); + signal.mark_finished(); + signal.mark_failed("late".into()); + assert_eq!(signal.wait(Duration::from_millis(1)), Ok(())); + } + + #[test] + fn missing_callback_times_out_with_a_clear_error() { + let signal = RecordingFinishSignal::default(); + let timeout = Duration::from_millis(50); + let started = Instant::now(); + let error = signal.wait(timeout).unwrap_err(); + assert!(started.elapsed() >= timeout); + assert_eq!(error, FinishWaitError::TimedOut(timeout)); + assert!(error.to_string().contains("did not finish writing")); + } + + /// Live regression for D-MAC-10: a sub-second recording must be + /// non-empty the moment `stop()` reports it finalized. + #[test] + #[ignore = "requires Screen Recording permission and a display; run with --ignored on a TCC-authorized macOS host"] + fn short_recording_is_written_when_stop_returns() { + let dir = std::env::temp_dir().join(format!("cua-sck-finish-{}", std::process::id())); + let path = dir.join("recording.mp4"); + for _ in 0..5 { + let backend = SckitVideoBackendFactory + .start(&path) + .expect("start sckit recording"); + thread::sleep(Duration::from_millis(120)); + let meta = backend.stop().expect("stop sckit recording"); + assert!(meta.finalized); + let len = std::fs::metadata(&path).expect("recording exists").len(); + assert!(len > 0, "recording is empty when stop() returned"); + } + let _ = std::fs::remove_dir_all(dir); + } +} diff --git a/packages/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/packages/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs index b23de83fc42..a3ee2d28e87 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs @@ -37,8 +37,17 @@ //! idempotent so explicit-detect + later-drop is safe. use std::collections::HashSet; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; +static INTENTIONAL_EPOCH: AtomicU64 = AtomicU64::new(0); +pub fn intentional_epoch() -> u64 { + INTENTIONAL_EPOCH.load(Ordering::Acquire) +} +pub fn note_intentional_activation() { + INTENTIONAL_EPOCH.fetch_add(1, Ordering::AcqRel); +} + use crate::apps; use crate::focus_steal::{self, SuppressionLease}; use crate::windows::{self, WindowInfo}; @@ -149,7 +158,7 @@ impl Changes { /// Default poll deadline — new windows triggered by a click typically /// appear within ~200ms on macOS; 1.0s gives the wildcard suppressor /// time to fire and settle. -const DEFAULT_TIMEOUT: Duration = Duration::from_millis(1000); +const DEFAULT_TIMEOUT: Duration = Duration::ZERO; /// Default inter-poll interval. Matches Swift's 50ms. const DEFAULT_POLL_INTERVAL: Duration = Duration::from_millis(50); diff --git a/packages/cua-driver/rust/crates/platform-macos/src/windows.rs b/packages/cua-driver/rust/crates/platform-macos/src/windows.rs index 245865322f5..dc458f20b36 100644 --- a/packages/cua-driver/rust/crates/platform-macos/src/windows.rs +++ b/packages/cua-driver/rust/crates/platform-macos/src/windows.rs @@ -98,6 +98,16 @@ pub(crate) fn all_windows_any_layer() -> Vec<WindowInfo> { enumerate_windows(kCGWindowListExcludeDesktopElements, LayerFilter::AnyLayer).windows } +/// Visible stacking order must come from the on-screen query: the all-window +/// query can report a different order even for rows marked on-screen. +pub(crate) fn visible_windows_any_layer() -> Vec<WindowInfo> { + enumerate_windows( + kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements, + LayerFilter::AnyLayer, + ) + .windows +} + pub(crate) fn match_ax_window( pid: i32, title: Option<&str>, @@ -127,7 +137,7 @@ pub(crate) fn match_ax_window( .into_iter() .filter(|window| window.title.trim() == title) .collect(); - (titled.len() == 1).then_some(titled[0].window_id) + (titled.len() == 1).then(|| titled[0].window_id) } /// Supplement an explicitly requested application's visible AX dialogs only. @@ -399,6 +409,7 @@ fn get_bounds_num( None } }) + .filter(|value| value.is_finite()) .unwrap_or(0.0) } @@ -412,6 +423,12 @@ pub fn window_info_by_id(window_id: u32) -> Option<WindowInfo> { .find(|w| w.window_id == window_id) } +pub(crate) fn window_on_current_space_by_id(window_id: u32) -> Option<bool> { + let query = crate::input::skylight::SpaceQuery::new()?; + let spaces = query.window_space_ids(window_id)?; + window_on_current_space(Some(&spaces), query.current_space_for_window(window_id)) +} + /// Look up a window's bounds by its CGWindowID. /// /// Returns `None` if the window is not currently known to WindowServer @@ -483,6 +500,33 @@ pub fn resolve_main_window_id(pid: i32) -> anyhow::Result<u32> { mod tests { use super::*; + #[test] + fn window_bounds_keep_finite_values_when_core_foundation_reports_invalid_numbers() { + use core_foundation::{ + base::TCFType, dictionary::CFDictionary, number::CFNumber, string::CFString, + }; + + for value in [12.5, -50.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let dictionary = CFDictionary::from_CFType_pairs(&[( + CFString::new("X").as_CFType(), + CFNumber::from(value).as_CFType(), + )]); + let bounds = + unsafe { CFDictionary::wrap_under_get_rule(dictionary.as_concrete_TypeRef()) }; + let parsed = get_bounds_num(&bounds, "X"); + assert_eq!(parsed, if value.is_finite() { value } else { 0.0 }); + assert_eq!(get_bounds_num(&bounds, "Missing"), 0.0); + assert!(serde_json::to_value(WindowBounds { + x: parsed, + y: 0.0, + width: 100.0, + height: 100.0, + }) + .unwrap()["x"] + .is_number()); + } + } + #[test] fn cg_front_to_back_order_normalizes_to_higher_is_frontmost() { let indices: Vec<_> = (0..3) @@ -568,6 +612,25 @@ mod tests { ); } + #[test] + fn ax_window_matching_refuses_a_title_without_a_matching_candidate() { + let mut target = window(42, 800, "TextEdit"); + target.title = "First".into(); + let mut duplicate = target.clone(); + duplicate.window_id = 43; + duplicate.title = "Second".into(); + let windows = vec![target, duplicate]; + + assert_eq!( + match_ax_window(800, Some("Missing"), [9.0, 580.0, 500.0, 500.0], &windows), + None + ); + assert_eq!( + match_ax_window(800, Some("Missing"), [0.0, 580.0, 500.0, 500.0], &windows), + None + ); + } + #[test] fn ax_window_matching_refuses_invalid_or_non_unique_geometry() { let windows = vec![window(42, 800, "TextEdit")]; diff --git a/packages/cua-driver/rust/crates/platform-windows/Cargo.toml b/packages/cua-driver/rust/crates/platform-windows/Cargo.toml index 095ead75370..f0d4ed51d0e 100644 --- a/packages/cua-driver/rust/crates/platform-windows/Cargo.toml +++ b/packages/cua-driver/rust/crates/platform-windows/Cargo.toml @@ -46,6 +46,8 @@ windows = { version = "0.58", features = [ "Win32_System_LibraryLoader", "Win32_Graphics_OpenGL", "Win32_Security", + "Win32_Security_AppLocker", + "Win32_System_Pipes", "Win32_System_Com", "Win32_System_Ole", "Win32_Storage_Xps", @@ -83,3 +85,6 @@ windows = { version = "0.58", features = [ "Win32_System_WinRT_Graphics_Capture", ] } base64 = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/packages/cua-driver/rust/crates/platform-windows/src/browser_launch_token.rs b/packages/cua-driver/rust/crates/platform-windows/src/browser_launch_token.rs new file mode 100644 index 00000000000..bb4c2f4654f --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-windows/src/browser_launch_token.rs @@ -0,0 +1,513 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Pure decision logic for the token that runs a Windows isolated browser. +//! +//! Cua Driver launches an isolated Chrome or Edge only from a vendor-signed +//! installation that the token running the browser cannot modify. An +//! elevated token (an elevated administrator, the built-in Administrator, or +//! any administrator when UAC is off) can modify `Program Files`, so such a +//! Driver never runs the browser with its own token. It derives a standard +//! user token from its own token instead: the same user, logon session, and +//! desktop, with `BUILTIN\Administrators` and the other groups that +//! `runas /trustlevel:0x20000` disables set to deny-only, administrative +//! privileges removed, and Medium integrity. Every protection check is then +//! made for that derived token, and the browser runs with it. +//! +//! Verification checks the properties Windows uses for access decisions: +//! group membership, privileges, and integrity level. It does not require +//! `TokenElevation` to be false. Windows keeps reporting that flag from the +//! elevated source token, and the flag grants no access by itself. +//! +//! The Win32 adapter gathers token facts and this module decides. It has no +//! Win32 dependencies so its unit tests run on any host. + +/// Mandatory integrity level RIDs (`SECURITY_MANDATORY_*_RID`). +pub(crate) const MEDIUM_INTEGRITY_RID: u32 = 0x2000; +pub(crate) const HIGH_INTEGRITY_RID: u32 = 0x3000; + +/// How a token holds `BUILTIN\Administrators` (S-1-5-32-544). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AdministratorsMembership { + Absent, + /// Present only for deny access control entries, as in a UAC-filtered or + /// standard-user token. + DenyOnly, + /// Present and usable for allow access control entries. + Enabled, +} + +/// Security-relevant facts about one access token. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TokenFacts { + /// `TokenElevation` reported the token as elevated. Used to decide + /// whether the Driver is elevated; a derived token inherits this flag from + /// its source token, so it is not a verification criterion for one. + pub elevated: bool, + pub integrity_rid: u32, + pub administrators: AdministratorsMembership, + /// Every privilege present in the token, enabled or not, by name. + pub privileges: Vec<String>, +} + +/// The token that runs a driver-owned isolated browser. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum BrowserLaunchToken { + /// The Driver's own token. Used whenever the Driver is not elevated, so + /// that configuration behaves exactly as before. + Driver, + /// A standard-user token derived from the elevated Driver's token. + StandardUser, +} + +/// Privileges an ordinary standard-user token holds. A derived token that +/// keeps any other privilege is refused. +pub(crate) const STANDARD_USER_PRIVILEGES: &[&str] = &[ + "SeChangeNotifyPrivilege", + "SeShutdownPrivilege", + "SeUndockPrivilege", + "SeIncreaseWorkingSetPrivilege", + "SeTimeZonePrivilege", +]; + +/// Choose the browser token for a Driver with these token facts. +pub(crate) fn browser_launch_token(driver: &TokenFacts) -> BrowserLaunchToken { + if driver.elevated + || driver.administrators == AdministratorsMembership::Enabled + || driver.integrity_rid >= HIGH_INTEGRITY_RID + { + BrowserLaunchToken::StandardUser + } else { + BrowserLaunchToken::Driver + } +} + +/// Privileges present in a derived token that a standard user does not hold. +pub(crate) fn privileges_beyond_standard_user(privileges: &[String]) -> Vec<&str> { + privileges + .iter() + .map(String::as_str) + .filter(|name| !STANDARD_USER_PRIVILEGES.contains(name)) + .collect() +} + +/// Require the derived browser token to be a standard-user token that sits +/// below the Driver. Integrity ordering matters: Windows forbids a lower +/// integrity process from writing to a higher integrity process, so the +/// browser cannot tamper with the elevated Driver process. `derived.elevated` +/// is deliberately ignored (see the module documentation). +pub(crate) fn validate_standard_user_token( + driver: &TokenFacts, + derived: &TokenFacts, +) -> Result<(), String> { + if derived.administrators == AdministratorsMembership::Enabled { + return Err("the derived token still has BUILTIN\\Administrators enabled".to_owned()); + } + if derived.integrity_rid > MEDIUM_INTEGRITY_RID { + return Err(format!( + "the derived token has integrity level 0x{:04x}, above Medium", + derived.integrity_rid + )); + } + if derived.integrity_rid >= driver.integrity_rid { + return Err(format!( + "the derived token integrity level 0x{:04x} is not below the Driver's 0x{:04x}, so \ + Windows would not stop the browser from writing to the Driver process", + derived.integrity_rid, driver.integrity_rid + )); + } + let extra = privileges_beyond_standard_user(&derived.privileges); + if !extra.is_empty() { + return Err(format!( + "the derived token keeps privileges a standard user does not hold: {}", + extra.join(", ") + )); + } + Ok(()) +} + +/// Refusal for an elevated Driver that cannot produce a verified +/// standard-user token. The Driver never falls back to its elevated token. +pub(crate) fn standard_user_token_unavailable_message(reason: &str) -> String { + format!( + "Cua Driver is running elevated and could not prepare a verified standard-user token \ + for the isolated browser ({reason}); it refuses to run the browser with its elevated \ + token. Run Cua Driver from a non-elevated session to use isolated browsers" + ) +} + +const BACKSLASH: u16 = b'\\' as u16; +const QUOTE: u16 = b'"' as u16; + +/// Build a `CreateProcess` command line whose `CommandLineToArgvW` parse is +/// exactly `program` followed by `args`. Works on UTF-16 so no path is lossily +/// converted. The program is always quoted and must not contain a quote. +pub(crate) fn windows_command_line(program: &[u16], args: &[Vec<u16>]) -> Result<Vec<u16>, String> { + if program.is_empty() || program.contains("E) { + return Err("the browser executable path cannot be quoted safely".to_owned()); + } + let mut line = Vec::with_capacity(program.len() + 2); + line.push(QUOTE); + line.extend_from_slice(program); + line.push(QUOTE); + for arg in args { + if arg.contains(&0) { + return Err("a browser argument contains a NUL character".to_owned()); + } + line.push(b' ' as u16); + append_quoted_argument(&mut line, arg); + } + Ok(line) +} + +fn append_quoted_argument(line: &mut Vec<u16>, arg: &[u16]) { + let needs_quotes = arg.is_empty() + || arg + .iter() + .any(|&unit| matches!(unit, 0x20 | 0x09 | 0x0a | 0x0b) || unit == QUOTE); + if needs_quotes { + line.push(QUOTE); + } + let mut backslashes = 0usize; + for &unit in arg { + if unit == BACKSLASH { + backslashes += 1; + } else { + if unit == QUOTE { + // Double the preceding backslashes and escape the quote. + line.extend(std::iter::repeat_n(BACKSLASH, backslashes + 1)); + } + backslashes = 0; + } + line.push(unit); + } + if needs_quotes { + // Trailing backslashes before the closing quote must be doubled. + line.extend(std::iter::repeat_n(BACKSLASH, backslashes)); + line.push(QUOTE); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn facts( + elevated: bool, + integrity_rid: u32, + administrators: AdministratorsMembership, + privileges: &[&str], + ) -> TokenFacts { + TokenFacts { + elevated, + integrity_rid, + administrators, + privileges: privileges.iter().map(|name| (*name).to_owned()).collect(), + } + } + + fn elevated_admin() -> TokenFacts { + facts( + true, + HIGH_INTEGRITY_RID, + AdministratorsMembership::Enabled, + &[ + "SeChangeNotifyPrivilege", + "SeDebugPrivilege", + "SeBackupPrivilege", + "SeImpersonatePrivilege", + ], + ) + } + + fn standard_user() -> TokenFacts { + facts( + false, + MEDIUM_INTEGRITY_RID, + AdministratorsMembership::DenyOnly, + &["SeChangeNotifyPrivilege", "SeShutdownPrivilege"], + ) + } + + #[test] + fn non_elevated_driver_keeps_its_own_token() { + // A standard user, and an administrator's UAC-filtered token. + for driver in [ + facts( + false, + MEDIUM_INTEGRITY_RID, + AdministratorsMembership::Absent, + &["SeChangeNotifyPrivilege"], + ), + standard_user(), + ] { + assert_eq!(browser_launch_token(&driver), BrowserLaunchToken::Driver); + } + } + + #[test] + fn elevated_driver_uses_a_standard_user_token() { + // Elevated administrator with UAC on. + assert_eq!( + browser_launch_token(&elevated_admin()), + BrowserLaunchToken::StandardUser + ); + // Built-in Administrator or UAC off: Administrators enabled even if + // TokenElevation were to report false. + let uac_off = facts( + false, + HIGH_INTEGRITY_RID, + AdministratorsMembership::Enabled, + &[], + ); + assert_eq!( + browser_launch_token(&uac_off), + BrowserLaunchToken::StandardUser + ); + // Each signal alone is enough. + let admins_only = facts( + false, + MEDIUM_INTEGRITY_RID, + AdministratorsMembership::Enabled, + &[], + ); + assert_eq!( + browser_launch_token(&admins_only), + BrowserLaunchToken::StandardUser + ); + let high_only = facts( + false, + HIGH_INTEGRITY_RID, + AdministratorsMembership::DenyOnly, + &[], + ); + assert_eq!( + browser_launch_token(&high_only), + BrowserLaunchToken::StandardUser + ); + let system = facts(false, 0x4000, AdministratorsMembership::DenyOnly, &[]); + assert_eq!( + browser_launch_token(&system), + BrowserLaunchToken::StandardUser + ); + let elevated_only = facts( + true, + MEDIUM_INTEGRITY_RID, + AdministratorsMembership::DenyOnly, + &[], + ); + assert_eq!( + browser_launch_token(&elevated_only), + BrowserLaunchToken::StandardUser + ); + } + + #[test] + fn verified_standard_user_token_is_accepted() { + assert_eq!( + validate_standard_user_token(&elevated_admin(), &standard_user()), + Ok(()) + ); + let mut absent = standard_user(); + absent.administrators = AdministratorsMembership::Absent; + assert_eq!( + validate_standard_user_token(&elevated_admin(), &absent), + Ok(()) + ); + let all_standard = facts( + false, + MEDIUM_INTEGRITY_RID, + AdministratorsMembership::DenyOnly, + STANDARD_USER_PRIVILEGES, + ); + assert_eq!( + validate_standard_user_token(&elevated_admin(), &all_standard), + Ok(()) + ); + } + + #[test] + fn derived_token_is_judged_by_groups_not_the_inherited_elevation_flag() { + // A SAFER token derived from an elevated token keeps TokenElevation + // set; with Administrators deny-only, standard privileges, and Medium + // integrity it is still a standard-user token. + let mut inherited_flag = standard_user(); + inherited_flag.elevated = true; + assert_eq!( + validate_standard_user_token(&elevated_admin(), &inherited_flag), + Ok(()) + ); + let mut admin = standard_user(); + admin.administrators = AdministratorsMembership::Enabled; + assert!(validate_standard_user_token(&elevated_admin(), &admin) + .unwrap_err() + .contains("Administrators enabled")); + } + + #[test] + fn derived_token_must_be_medium_or_lower_and_below_the_driver() { + let mut high = standard_user(); + high.integrity_rid = HIGH_INTEGRITY_RID; + assert!(validate_standard_user_token(&elevated_admin(), &high) + .unwrap_err() + .contains("above Medium")); + let mut medium_plus = standard_user(); + medium_plus.integrity_rid = 0x2100; + assert!(validate_standard_user_token(&elevated_admin(), &medium_plus).is_err()); + // An elevated-by-group Driver at Medium integrity cannot be protected + // from a Medium browser by integrity ordering. + let medium_admin_driver = facts( + false, + MEDIUM_INTEGRITY_RID, + AdministratorsMembership::Enabled, + &[], + ); + assert!( + validate_standard_user_token(&medium_admin_driver, &standard_user()) + .unwrap_err() + .contains("not below the Driver") + ); + } + + #[test] + fn derived_token_must_not_keep_administrative_privileges() { + for privilege in [ + "SeDebugPrivilege", + "SeBackupPrivilege", + "SeRestorePrivilege", + "SeTakeOwnershipPrivilege", + "SeImpersonatePrivilege", + "SeCreateGlobalPrivilege", + "SeLoadDriverPrivilege", + ] { + let mut derived = standard_user(); + derived.privileges.push(privilege.to_owned()); + let error = validate_standard_user_token(&elevated_admin(), &derived).unwrap_err(); + assert!(error.contains(privilege), "{error}"); + } + assert_eq!( + privileges_beyond_standard_user(&elevated_admin().privileges), + vec![ + "SeDebugPrivilege", + "SeBackupPrivilege", + "SeImpersonatePrivilege" + ] + ); + } + + #[test] + fn unavailable_token_message_refuses_the_elevated_fallback() { + let message = standard_user_token_unavailable_message("SaferCreateLevel failed"); + assert!(message.contains("running elevated")); + assert!(message.contains("SaferCreateLevel failed")); + assert!(message.contains("refuses to run the browser with its elevated token")); + } + + fn wide(text: &str) -> Vec<u16> { + text.encode_utf16().collect() + } + + fn command_line(program: &str, args: &[&str]) -> Result<String, String> { + let args = args.iter().map(|arg| wide(arg)).collect::<Vec<_>>(); + windows_command_line(&wide(program), &args).map(|line| String::from_utf16(&line).unwrap()) + } + + /// Reference `CommandLineToArgvW` argument parser (post-2008 rules) used + /// to prove the quoting round-trips. + fn parse_arguments(line: &str) -> Vec<String> { + let chars = line.chars().collect::<Vec<_>>(); + let mut index = 0; + // Program name: quoted or up to whitespace; no escape processing. + if chars.first() == Some(&'"') { + index = 1; + while index < chars.len() && chars[index] != '"' { + index += 1; + } + index += 1; + } else { + while index < chars.len() && chars[index] != ' ' && chars[index] != '\t' { + index += 1; + } + } + let mut arguments = Vec::new(); + loop { + while index < chars.len() && (chars[index] == ' ' || chars[index] == '\t') { + index += 1; + } + if index >= chars.len() { + return arguments; + } + let mut current = String::new(); + let mut quoted = false; + while index < chars.len() { + let ch = chars[index]; + if ch == '\\' { + let start = index; + while index < chars.len() && chars[index] == '\\' { + index += 1; + } + let count = index - start; + if index < chars.len() && chars[index] == '"' { + current.extend(std::iter::repeat_n('\\', count / 2)); + if count % 2 == 1 { + current.push('"'); + index += 1; + } + } else { + current.extend(std::iter::repeat_n('\\', count)); + } + continue; + } + if ch == '"' { + if quoted && chars.get(index + 1) == Some(&'"') { + current.push('"'); + index += 2; + continue; + } + quoted = !quoted; + index += 1; + continue; + } + if !quoted && (ch == ' ' || ch == '\t') { + break; + } + current.push(ch); + index += 1; + } + arguments.push(current); + } + } + + #[test] + fn command_line_round_trips_browser_arguments() { + let program = r"C:\Program Files\Google\Chrome\Application\chrome.exe"; + let args = [ + "--remote-debugging-port=0", + r"--user-data-dir=C:\Users\Jane Doe\AppData\Local\CuaDriver\BrowserProfiles\isolated-1", + "", + r"trailing\", + r"spaced trailing\\", + r#"quote"inside"#, + r#"back\"slash"#, + "tab\there", + "about:blank", + ]; + let line = command_line(program, &args).expect("quotable"); + assert!(line.starts_with(&format!("\"{program}\" "))); + assert_eq!(parse_arguments(&line), args); + } + + #[test] + fn simple_arguments_stay_unquoted() { + assert_eq!( + command_line(r"C:\b.exe", &["--no-first-run", r"C:\p\q"]).unwrap(), + r#""C:\b.exe" --no-first-run C:\p\q"# + ); + } + + #[test] + fn command_line_rejects_unquotable_input() { + assert!(command_line(r#"C:\bad"name.exe"#, &[]).is_err()); + assert!(command_line("", &[]).is_err()); + let nul = vec![wide("a\0b")]; + assert!(windows_command_line(&wide(r"C:\b.exe"), &nul).is_err()); + } +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/browser_standard_user.rs b/packages/cua-driver/rust/crates/platform-windows/src/browser_standard_user.rs new file mode 100644 index 00000000000..ebd0a9e16d0 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-windows/src/browser_standard_user.rs @@ -0,0 +1,349 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Win32 side of the standard-user launch for isolated browsers. +//! +//! An elevated Driver derives a standard-user token from its own process token +//! with the SAFER "Normal User" level (the level `runas /trustlevel:0x20000` +//! uses): `BUILTIN\Administrators` and the other administrative groups become +//! deny-only and administrative privileges are removed. The Driver then lowers +//! the token to Medium integrity, removes any remaining privilege a standard +//! user does not hold, and verifies the result before use. Applications are created with that token. The Driver never falls back to its own elevated +//! token. Decision rules live in `browser_launch_token`. +//! +//! A restricted token from `CreateRestrictedToken` (with or without +//! `LUA_TOKEN`/`DISABLE_MAX_PRIVILEGE`) is deliberately not used: Chrome's +//! sandbox cannot launch its GPU and renderer processes under one. + +use crate::browser_launch_token::{ + browser_launch_token, privileges_beyond_standard_user, standard_user_token_unavailable_message, + validate_standard_user_token, AdministratorsMembership, BrowserLaunchToken, TokenFacts, + MEDIUM_INTEGRITY_RID, +}; +use cua_driver_core::browser::refusal::{BrowserRefusal, BrowserRefusalCode}; +use std::ffi::c_void; +use std::sync::{Arc, Mutex}; +use windows::core::{PCWSTR, PWSTR}; +use windows::Win32::Foundation::{CloseHandle, GetLastError, ERROR_NOT_ALL_ASSIGNED, HANDLE, LUID}; +use windows::Win32::Security::AppLocker::{ + SaferCloseLevel, SaferComputeTokenFromLevel, SaferCreateLevel, + SAFER_COMPUTE_TOKEN_FROM_LEVEL_FLAGS, SAFER_LEVELID_NORMALUSER, SAFER_LEVEL_OPEN, + SAFER_SCOPEID_USER, +}; +use windows::Win32::Security::{ + AdjustTokenPrivileges, CreateWellKnownSid, EqualSid, GetLengthSid, GetSidSubAuthority, + GetSidSubAuthorityCount, GetTokenInformation, LookupPrivilegeNameW, SetTokenInformation, + TokenElevation, TokenGroups, TokenIntegrityLevel, TokenPrivileges, WinBuiltinAdministratorsSid, + WinMediumLabelSid, LUID_AND_ATTRIBUTES, PSID, SAFER_LEVEL_HANDLE, SE_PRIVILEGE_REMOVED, + SID_AND_ATTRIBUTES, TOKEN_ACCESS_MASK, TOKEN_ASSIGN_PRIMARY, TOKEN_DUPLICATE, TOKEN_ELEVATION, + TOKEN_GROUPS, TOKEN_INFORMATION_CLASS, TOKEN_MANDATORY_LABEL, TOKEN_PRIVILEGES, TOKEN_QUERY, +}; +use windows::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; + +// `SE_GROUP_*` attribute bits from winnt.h. +const SE_GROUP_ENABLED: u32 = 0x0000_0004; +const SE_GROUP_USE_FOR_DENY_ONLY: u32 = 0x0000_0010; +const SE_GROUP_INTEGRITY: u32 = 0x0000_0020; + +/// An owned kernel handle closed on drop. +pub(crate) struct OwnedHandle(HANDLE); + +// SAFETY: a kernel handle value is process-wide and may be used and closed +// from any thread. The wrapper never hands out the handle for closing. +unsafe impl Send for OwnedHandle {} +unsafe impl Sync for OwnedHandle {} + +impl OwnedHandle { + /// Take ownership of `handle`, which is closed on drop. + pub(crate) fn new(handle: HANDLE) -> Self { + Self(handle) + } + + pub(crate) fn raw(&self) -> HANDLE { + self.0 + } + + /// Release ownership without closing the handle. + pub(crate) fn into_raw(self) -> HANDLE { + let handle = self.0; + std::mem::forget(self); + handle + } +} + +impl Drop for OwnedHandle { + fn drop(&mut self) { + if !self.0.is_invalid() { + let _ = unsafe { CloseHandle(self.0) }; + } + } +} + +/// The token that will run the isolated browser. +#[derive(Clone)] +pub(crate) enum BrowserLaunchContext { + /// The Driver is not elevated; the browser runs with the Driver's token. + Driver, + /// A verified standard-user token derived from the elevated Driver token. + StandardUser(Arc<OwnedHandle>), +} + +/// A process token cannot change after process start, so one verified +/// standard-user token serves every isolated launch. Failures are not cached. +static STANDARD_USER_TOKEN: Mutex<Option<Arc<OwnedHandle>>> = Mutex::new(None); + +fn refusal(message: impl Into<String>) -> BrowserRefusal { + BrowserRefusal::new(BrowserRefusalCode::BrowserRouteUnavailable, message) +} + +/// Decide which token runs the isolated browser and, for an elevated Driver, +/// produce the verified standard-user token. Never returns the Driver's +/// elevated token for the browser. +pub(crate) fn browser_launch_context() -> Result<BrowserLaunchContext, BrowserRefusal> { + let driver_token = current_process_token(TOKEN_QUERY) + .map_err(|error| refusal(format!("could not open the Driver process token: {error}")))?; + let driver = token_facts(driver_token.raw()).map_err(|error| { + refusal(format!( + "could not inspect the Driver process token: {error}" + )) + })?; + if browser_launch_token(&driver) == BrowserLaunchToken::Driver { + return Ok(BrowserLaunchContext::Driver); + } + let mut cached = STANDARD_USER_TOKEN + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(token) = cached.as_ref() { + return Ok(BrowserLaunchContext::StandardUser(token.clone())); + } + let token = derive_verified_standard_user_token(&driver) + .map(Arc::new) + .map_err(|reason| refusal(standard_user_token_unavailable_message(&reason)))?; + *cached = Some(token.clone()); + Ok(BrowserLaunchContext::StandardUser(token)) +} + +fn current_process_token(access: TOKEN_ACCESS_MASK) -> Result<OwnedHandle, String> { + let mut token = HANDLE::default(); + unsafe { OpenProcessToken(GetCurrentProcess(), access, &mut token) } + .map_err(|error| format!("OpenProcessToken failed: {error}"))?; + Ok(OwnedHandle(token)) +} + +/// Derive, harden, and verify the standard-user token for `driver`. +fn derive_verified_standard_user_token(driver: &TokenFacts) -> Result<OwnedHandle, String> { + let driver_token = current_process_token(TOKEN_QUERY | TOKEN_DUPLICATE | TOKEN_ASSIGN_PRIMARY)?; + let derived = safer_normal_user_token(driver_token.raw())?; + let facts = token_facts(derived.raw())?; + if facts.integrity_rid > MEDIUM_INTEGRITY_RID { + set_medium_integrity(derived.raw())?; + } + remove_privileges(derived.raw(), &facts)?; + let verified = token_facts(derived.raw())?; + validate_standard_user_token(driver, &verified).map_err(|reason| { + format!( + "{reason}; derived token: integrity=0x{:04x}, administrators={:?}, privileges=[{}]", + verified.integrity_rid, + verified.administrators, + verified.privileges.join(", ") + ) + })?; + Ok(derived) +} + +fn safer_normal_user_token(driver_token: HANDLE) -> Result<OwnedHandle, String> { + let mut level = SAFER_LEVEL_HANDLE::default(); + unsafe { + SaferCreateLevel( + SAFER_SCOPEID_USER, + SAFER_LEVELID_NORMALUSER, + SAFER_LEVEL_OPEN, + &mut level, + None, + ) + } + .map_err(|error| format!("SaferCreateLevel failed: {error}"))?; + let mut derived = HANDLE::default(); + let computed = unsafe { + SaferComputeTokenFromLevel( + level, + driver_token, + &mut derived, + SAFER_COMPUTE_TOKEN_FROM_LEVEL_FLAGS(0), + None, + ) + }; + let _ = unsafe { SaferCloseLevel(level) }; + computed.map_err(|error| format!("SaferComputeTokenFromLevel failed: {error}"))?; + Ok(OwnedHandle(derived)) +} + +fn set_medium_integrity(token: HANDLE) -> Result<(), String> { + // SECURITY_MAX_SID_SIZE is 68 bytes; use u64 storage for alignment. + let mut sid_buffer = [0u64; 12]; + let mut sid_size = std::mem::size_of_val(&sid_buffer) as u32; + let sid = PSID(sid_buffer.as_mut_ptr().cast()); + unsafe { CreateWellKnownSid(WinMediumLabelSid, PSID::default(), sid, &mut sid_size) } + .map_err(|error| format!("could not build the Medium integrity SID: {error}"))?; + let label = TOKEN_MANDATORY_LABEL { + Label: SID_AND_ATTRIBUTES { + Sid: sid, + Attributes: SE_GROUP_INTEGRITY, + }, + }; + let size = std::mem::size_of::<TOKEN_MANDATORY_LABEL>() as u32 + unsafe { GetLengthSid(sid) }; + unsafe { + SetTokenInformation( + token, + TokenIntegrityLevel, + std::ptr::from_ref(&label).cast(), + size, + ) + } + .map_err(|error| format!("could not lower the token to Medium integrity: {error}")) +} + +fn remove_privileges(token: HANDLE, facts: &TokenFacts) -> Result<(), String> { + let extra = privileges_beyond_standard_user(&facts.privileges); + if extra.is_empty() { + return Ok(()); + } + for (name, luid) in token_privileges(token)? { + if !extra.contains(&name.as_str()) { + continue; + } + let state = TOKEN_PRIVILEGES { + PrivilegeCount: 1, + Privileges: [LUID_AND_ATTRIBUTES { + Luid: luid, + Attributes: SE_PRIVILEGE_REMOVED, + }], + }; + unsafe { AdjustTokenPrivileges(token, false, Some(&state), 0, None, None) } + .map_err(|error| format!("could not remove {name}: {error}"))?; + if unsafe { GetLastError() } == ERROR_NOT_ALL_ASSIGNED { + return Err(format!( + "could not remove {name}: not all privileges assigned" + )); + } + } + Ok(()) +} + +/// Read one variable-length token information class into aligned storage. +fn token_information(token: HANDLE, class: TOKEN_INFORMATION_CLASS) -> Result<Vec<u64>, String> { + let mut needed = 0u32; + let _ = unsafe { GetTokenInformation(token, class, None, 0, &mut needed) }; + if needed == 0 { + return Err(format!("GetTokenInformation({}) reported no data", class.0)); + } + let mut buffer = vec![0u64; (needed as usize).div_ceil(8)]; + unsafe { + GetTokenInformation( + token, + class, + Some(buffer.as_mut_ptr().cast()), + needed, + &mut needed, + ) + } + .map_err(|error| format!("GetTokenInformation({}) failed: {error}", class.0))?; + Ok(buffer) +} + +fn sid_last_rid(sid: PSID) -> Option<u32> { + unsafe { + let count = GetSidSubAuthorityCount(sid); + if count.is_null() || *count == 0 { + return None; + } + let rid = GetSidSubAuthority(sid, u32::from(*count - 1)); + (!rid.is_null()).then(|| *rid) + } +} + +fn token_privileges(token: HANDLE) -> Result<Vec<(String, LUID)>, String> { + let buffer = token_information(token, TokenPrivileges)?; + let header = buffer.as_ptr().cast::<TOKEN_PRIVILEGES>(); + let count = unsafe { (*header).PrivilegeCount } as usize; + let first = unsafe { std::ptr::addr_of!((*header).Privileges).cast::<LUID_AND_ATTRIBUTES>() }; + let mut privileges = Vec::with_capacity(count); + for index in 0..count { + let luid = unsafe { (*first.add(index)).Luid }; + let mut name = [0u16; 128]; + let mut length = name.len() as u32; + let resolved = unsafe { + LookupPrivilegeNameW(PCWSTR::null(), &luid, PWSTR(name.as_mut_ptr()), &mut length) + }; + // An unresolvable privilege keeps a synthetic name, which is never on + // the standard-user allow list, so validation fails closed. + let name = match resolved { + Ok(()) => String::from_utf16_lossy(&name[..length as usize]), + Err(_) => format!("LUID({:#x}:{:#x})", luid.HighPart, luid.LowPart), + }; + privileges.push((name, luid)); + } + Ok(privileges) +} + +fn administrators_membership(token: HANDLE) -> Result<AdministratorsMembership, String> { + let mut sid_buffer = [0u64; 12]; + let mut sid_size = std::mem::size_of_val(&sid_buffer) as u32; + let administrators = PSID(sid_buffer.as_mut_ptr().cast()); + unsafe { + CreateWellKnownSid( + WinBuiltinAdministratorsSid, + PSID::default(), + administrators, + &mut sid_size, + ) + } + .map_err(|error| format!("could not build the Administrators SID: {error}"))?; + let buffer = token_information(token, TokenGroups)?; + let header = buffer.as_ptr().cast::<TOKEN_GROUPS>(); + let count = unsafe { (*header).GroupCount } as usize; + let first = unsafe { std::ptr::addr_of!((*header).Groups).cast::<SID_AND_ATTRIBUTES>() }; + for index in 0..count { + let group = unsafe { *first.add(index) }; + if unsafe { EqualSid(group.Sid, administrators) }.is_ok() { + let attributes = group.Attributes; + return Ok(if attributes & SE_GROUP_USE_FOR_DENY_ONLY != 0 { + AdministratorsMembership::DenyOnly + } else if attributes & SE_GROUP_ENABLED != 0 { + AdministratorsMembership::Enabled + } else { + // A present, disabled, allow-capable group can be re-enabled + // by the token holder, so treat it as enabled. + AdministratorsMembership::Enabled + }); + } + } + Ok(AdministratorsMembership::Absent) +} + +/// Gather the security-relevant facts of `token`. +pub(crate) fn token_facts(token: HANDLE) -> Result<TokenFacts, String> { + let mut elevation = TOKEN_ELEVATION::default(); + let mut returned = 0u32; + unsafe { + GetTokenInformation( + token, + TokenElevation, + Some(std::ptr::from_mut(&mut elevation).cast::<c_void>()), + std::mem::size_of::<TOKEN_ELEVATION>() as u32, + &mut returned, + ) + } + .map_err(|error| format!("GetTokenInformation(TokenElevation) failed: {error}"))?; + let label = token_information(token, TokenIntegrityLevel)?; + let label = label.as_ptr().cast::<TOKEN_MANDATORY_LABEL>(); + let integrity_rid = sid_last_rid(unsafe { (*label).Label.Sid }) + .ok_or_else(|| "the token integrity label has no RID".to_owned())?; + Ok(TokenFacts { + elevated: elevation.TokenIsElevated != 0, + integrity_rid, + administrators: administrators_membership(token)?, + privileges: token_privileges(token)? + .into_iter() + .map(|(name, _)| name) + .collect(), + }) +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/capture.rs b/packages/cua-driver/rust/crates/platform-windows/src/capture.rs index 1dbee847810..ed177dfdb7b 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/capture.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/capture.rs @@ -152,10 +152,28 @@ unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec<u8>, i32, i32) bail!("screen-region fallback: window has zero/negative bounds: {w}x{h}"); } - let screen_dc = GetDC(HWND(std::ptr::null_mut())); // NULL HWND → desktop DC + let screen_dc = GetDC(HWND(std::ptr::null_mut())); + if screen_dc.is_invalid() { + bail!("screen-region fallback: GetDC(NULL) returned an invalid desktop DC"); + } let mem_dc = CreateCompatibleDC(screen_dc); + if mem_dc.is_invalid() { + ReleaseDC(HWND(std::ptr::null_mut()), screen_dc); + bail!("screen-region fallback: CreateCompatibleDC returned an invalid memory DC"); + } let bitmap = CreateCompatibleBitmap(screen_dc, w, h); + if bitmap.is_invalid() { + let _ = DeleteDC(mem_dc); + ReleaseDC(HWND(std::ptr::null_mut()), screen_dc); + bail!("screen-region fallback: CreateCompatibleBitmap returned an invalid bitmap"); + } let old_bitmap = SelectObject(mem_dc, bitmap); + if old_bitmap.is_invalid() { + let _ = DeleteObject(bitmap); + let _ = DeleteDC(mem_dc); + ReleaseDC(HWND(std::ptr::null_mut()), screen_dc); + bail!("screen-region fallback: SelectObject failed to select the capture bitmap"); + } // Copy from physical screen coords into our memory DC at (0, 0). let blt_ok = BitBlt( @@ -571,6 +589,19 @@ unsafe fn screenshot_window_bytes_with_occlusion_unsafe(hwnd: u64) -> Result<(Ve /// Capture the primary display (full screen), returning raw PNG bytes. pub fn screenshot_display_bytes() -> Result<Vec<u8>> { + screenshot_display_bytes_with(|memory, screen, width, height| unsafe { + BitBlt(memory, 0, 0, width, height, screen, 0, 0, SRCCOPY) + }) +} + +fn screenshot_display_bytes_with( + copy: impl FnOnce( + windows::Win32::Graphics::Gdi::HDC, + windows::Win32::Graphics::Gdi::HDC, + i32, + i32, + ) -> windows::core::Result<()>, +) -> Result<Vec<u8>> { unsafe { use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN}; // Under Per-Monitor V2 DPI awareness, GetSystemMetrics returns @@ -583,10 +614,28 @@ pub fn screenshot_display_bytes() -> Result<Vec<u8>> { bail!("Could not get screen metrics"); } let screen_dc = GetDC(HWND::default()); + if screen_dc.is_invalid() { + bail!("GetDC(NULL) returned an invalid desktop DC"); + } let mem_dc = CreateCompatibleDC(screen_dc); + if mem_dc.is_invalid() { + ReleaseDC(HWND::default(), screen_dc); + bail!("CreateCompatibleDC returned an invalid memory DC"); + } let bitmap = CreateCompatibleBitmap(screen_dc, w, h); + if bitmap.is_invalid() { + let _ = DeleteDC(mem_dc); + ReleaseDC(HWND::default(), screen_dc); + bail!("CreateCompatibleBitmap returned an invalid bitmap"); + } let old_bitmap = SelectObject(mem_dc, bitmap); - BitBlt(mem_dc, 0, 0, w, h, screen_dc, 0, 0, SRCCOPY)?; + if old_bitmap.is_invalid() { + let _ = DeleteObject(bitmap); + let _ = DeleteDC(mem_dc); + ReleaseDC(HWND::default(), screen_dc); + bail!("SelectObject failed to select the capture bitmap"); + } + let blt_result = copy(mem_dc, screen_dc, w, h); let mut bmi = BITMAPINFO { bmiHeader: BITMAPINFOHEADER { biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32, @@ -614,6 +663,9 @@ pub fn screenshot_display_bytes() -> Result<Vec<u8>> { let _ = DeleteObject(bitmap); let _ = DeleteDC(mem_dc); ReleaseDC(HWND::default(), screen_dc); + if let Err(error) = blt_result { + bail!("desktop capture: BitBlt failed: {error}"); + } if ok == 0 { bail!("GetDIBits returned 0"); } @@ -659,3 +711,54 @@ pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result<Vec<u8> pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> { cua_driver_core::image_utils::png_dimensions(data) } + +#[cfg(test)] +mod capture_resource_tests { + use super::*; + + #[test] + fn failed_desktop_copy_releases_gdi_resources() { + const CHILD: &str = "QWEN_CUA_CAPTURE_RESOURCE_TEST"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "capture::capture_resource_tests::failed_desktop_copy_releases_gdi_resources", + "--nocapture", + ]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + use windows::Win32::System::Threading::{ + GetCurrentProcess, GetGuiResources, GR_GDIOBJECTS, + }; + let fail_copy = || { + let called = std::cell::Cell::new(false); + let error = screenshot_display_bytes_with(|_, _, _, _| { + called.set(true); + Err(windows::core::Error::from_hresult( + windows::Win32::Foundation::E_FAIL, + )) + }) + .unwrap_err(); + assert!(called.get(), "GDI allocation did not reach copy: {error}"); + assert!(error.to_string().contains("BitBlt failed")); + }; + fail_copy(); + let before = unsafe { GetGuiResources(GetCurrentProcess(), GR_GDIOBJECTS) }; + for _ in 0..3 { + fail_copy(); + } + let after = unsafe { GetGuiResources(GetCurrentProcess(), GR_GDIOBJECTS) }; + assert_eq!(after, before, "failed capture leaked GDI handles"); + } +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/input/delivery.rs b/packages/cua-driver/rust/crates/platform-windows/src/input/delivery.rs index 5066872cd7b..88bf0a67c1d 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/input/delivery.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/input/delivery.rs @@ -96,11 +96,7 @@ impl DeliveryMode { /// Parse from a tool's JSON args, reading the `delivery_mode` field. pub fn from_args(args: &Value) -> Self { - if args.get("app_context").and_then(Value::as_bool) == Some(true) { - Self::Foreground - } else { - Self::parse(args.get("delivery_mode").and_then(|v| v.as_str())) - } + Self::parse(args.get("delivery_mode").and_then(|v| v.as_str())) } pub fn is_foreground(self) -> bool { @@ -175,7 +171,7 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool { // genuine keyboard input for the focused widget. The messages can be // accepted by PostMessage while the Entry receives nothing, so refuse // instead of reporting a false background success. - return matches!(kind, Keystroke | KeyCombo | TextInput); + return matches!(kind, MouseClick | Keystroke | KeyCombo | TextInput); } // NB: WinUI3 (`WinUIDesktopWin32WindowClass`) is deliberately NOT flagged // here. It looks WPF-like, but its composition input-site does NOT consume @@ -342,6 +338,7 @@ pub fn background_unavailable_error( ); cua_driver_core::protocol::ToolResult::error(text).with_structured(serde_json::json!({ "code": "background_unavailable", + "sent": false, "target_class": class, "event_kind": kind.name(), "suggestion": "Retry this action with delivery_mode:\"foreground\".", @@ -375,23 +372,17 @@ pub fn background_unavailable_error_with_cause( "background_unavailable" }; let text = format!( - "Background delivery is not available for target window class \ - '{class}' on this event kind ({}): {cause}. Retry this action with \ - delivery_mode:\"foreground\"; Cua Driver will activate the target for \ - the action and restore the previous foreground afterward.", + "Background actuator failed for target window class '{class}' on {}: {cause}. Input may have been sent. Observe before deciding on another action.", kind.name() ); cua_driver_core::protocol::ToolResult::error(text).with_structured(serde_json::json!({ "code": code, + "sent": true, "target_class": class, "event_kind": kind.name(), "cause": cause, - "suggestion": "Retry this action with delivery_mode:\"foreground\".", - "escalation": { - "recommended": "foreground", - "reason": "background input could not be delivered by the coordinate \ - actuator; retry this action with delivery_mode:\"foreground\".", - }, + "suggestion": "Input may have been sent. Observe before retrying.", + })) } @@ -515,7 +506,8 @@ mod tests { _ => panic!("expected text content"), }; assert!(text.contains("occluded"), "result text lost cause: {text}"); - assert!(text.contains("Retry this action with delivery_mode:\"foreground\"")); + assert!(!text.contains("Retry this action")); + assert_eq!(structured["sent"], true); assert!(!text.contains("bring_to_front")); assert!(!structured["suggestion"] .as_str() diff --git a/packages/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs b/packages/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs index ca8a80e40f1..db909a86a1f 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs @@ -471,9 +471,12 @@ pub fn send_key_synthesized_after_focus( events.push(key_input(*mvk, true)); } - with_confirmed_foreground(target, "key delivery", focus, || unsafe { + with_confirmed_foreground(target, "key delivery", focus, |_, _| unsafe { let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32); if sent as usize != events.len() { + let mut releases = vec![key_input(key_vk, true)]; + releases.extend(mod_vks.iter().rev().map(|vk| key_input(*vk, true))); + let _ = SendInput(&releases, std::mem::size_of::<INPUT>() as i32); bail!( "SendInput inserted only {sent} of {} events. Likely cause: \ the daemon is not at UIAccess integrity, so SetForegroundWindow \ @@ -515,51 +518,35 @@ pub fn send_text_synthesized_after_focus( if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) { bail!(msg); } - // Build the key-event sequence. Printable codepoints go through as Unicode - // packets (KEYEVENTF_UNICODE, wVk=0, wScan=code unit), but line breaks are - // mapped to a real VK_RETURN keystroke — mirroring the background path - // (`post_enter_keystroke`) — because terminals and rich editors honour an - // Enter key event, not a raw `\r`/`\n` Unicode packet. `\r\n` collapses to - // a single Return (the `\r` emits the Enter; the following `\n` is silent). - let return_vk = windows::Win32::UI::Input::KeyboardAndMouse::VK_RETURN; - let mut events: Vec<INPUT> = Vec::with_capacity(text.len() * 2); - let mut prev_was_cr = false; - for ch in text.chars() { - match ch { - '\n' if prev_was_cr => { - prev_was_cr = false; - } - '\n' | '\r' => { - events.push(key_input(return_vk, false)); - events.push(key_input(return_vk, true)); - prev_was_cr = ch == '\r'; - } - _ => { - prev_was_cr = false; - let mut buf = [0u16; 2]; - for unit in ch.encode_utf16(&mut buf) { - events.push(unicode_key_input(*unit, false)); - events.push(unicode_key_input(*unit, true)); + let units = text_units(text); + with_confirmed_foreground( + target, + "text delivery", + focus, + |attachment, foreground_target| unsafe { + let attachment = attachment.expect("attached input queue"); + for (index, unit) in units.iter().enumerate() { + super::user_input::checkpoint()?; + let current = GetForegroundWindow(); + if !crate::win32::foreground_matches_target_or_owned_window( + foreground_target, + current.0 as usize as u64, + ) { + bail!("foreground_lost: target changed after {index} UTF-16 units; remaining text was not sent; observe before retrying"); } + match attachment.send_and_await_read(&unit_events(unit), Duration::from_secs(2)) { + UnitRead::Read => {}, + UnitRead::Short(sent) => { + let releases = [unit_events(unit)[1], sentinel_key_input(true)]; + let _ = SendInput(&releases, std::mem::size_of::<INPUT>() as i32); + bail!("text_delivery_stalled: only {sent} events for unit {index} were inserted; no replay"); + }, + UnitRead::Stalled => bail!("text_delivery_stalled: target did not read unit {index}; no replay; observe before retrying"), } - } - } - if events.is_empty() { - return Ok(()); - } - - with_confirmed_foreground(target, "text delivery", focus, || unsafe { - let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32); - if sent as usize != events.len() { - bail!( - "SendInput inserted only {sent} of {} key events. Likely cause: \ - the daemon is not at UIAccess integrity, so SetForegroundWindow \ - was rejected and the events landed on the wrong window.", - events.len() - ); - } - Ok(()) - }) + } + Ok(()) + }, + ) } fn wait_for_exact_foreground(target: HWND, timeout: Duration) -> bool { @@ -581,12 +568,21 @@ fn wait_for_exact_foreground(target: HWND, timeout: Duration) -> bool { /// external Win32/UIA state instead of inferred from API return values or a /// fixed settle delay. The prior foreground is restored on every body/focus /// result after activation succeeds. +static FOREGROUND_TRANSACTION: std::sync::Mutex<()> = std::sync::Mutex::new(()); + pub(crate) fn with_confirmed_foreground<T>( target: HWND, operation: &str, focus: impl FnOnce() -> Result<()>, - body: impl FnOnce() -> Result<T>, + body: impl FnOnce( + Option<&InputQueueAttachment>, + crate::win32::windows::ForegroundTarget, + ) -> Result<T>, ) -> Result<T> { + let _foreground = FOREGROUND_TRANSACTION + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _user_input = super::user_input::Guard::start()?; let previous = unsafe { GetForegroundWindow() }; let foreground_target = crate::win32::capture_foreground_target(target.0 as usize as u64) .ok_or_else(|| { @@ -606,7 +602,9 @@ pub(crate) fn with_confirmed_foreground<T>( ); } + let mut attachment = None; let result = (|| { + super::user_input::checkpoint()?; focus()?; let deadline = Instant::now() + Duration::from_millis(250); let actual = loop { @@ -634,17 +632,27 @@ pub(crate) fn with_confirmed_foreground<T>( actual.0 ); } - body() + super::user_input::checkpoint()?; + attachment = InputQueueAttachment::attach(actual); + if attachment.is_none() { + bail!("foreground_unavailable: target input queue could not be attached for delivery confirmation; no input was sent"); + } + body(attachment.as_ref(), foreground_target) })(); - // Give the target message loop a bounded opportunity to consume the - // inserted sequence before restoring the user's prior foreground. - if result.is_ok() { - sleep(Duration::from_millis(40)); + let drained = attachment + .as_ref() + .is_none_or(|attachment| attachment.wait_for_drain(Duration::from_secs(2))); + drop(attachment); + if !drained { + bail!("text_delivery_stalled: target input queue did not drain; remaining input was not sent; observe before retrying"); } let restored = crate::win32::restore_input_foreground(previous.0 as usize as u64, foreground_target); - result.and_then(|value| restored.map(|()| value)) + result.and_then(|value| { + super::user_input::checkpoint()?; + restored.map(|()| value) + }) } /// Build a single Unicode keyboard INPUT struct for one UTF-16 code unit, @@ -728,7 +736,9 @@ fn is_extended(vk: VIRTUAL_KEY) -> bool { use windows::Win32::UI::Input::KeyboardAndMouse::*; matches!( vk, - VK_DELETE + VK_LWIN + | VK_RWIN + | VK_DELETE | VK_INSERT | VK_HOME | VK_END @@ -740,7 +750,6 @@ fn is_extended(vk: VIRTUAL_KEY) -> bool { | VK_RIGHT | VK_RCONTROL | VK_RMENU - | VK_RWIN | VK_NUMLOCK | VK_SNAPSHOT ) @@ -807,3 +816,180 @@ fn key_name_to_vk(key: &str) -> Result<VIRTUAL_KEY> { }; Ok(vk) } + +/// Unassigned virtual key used as a harmless input-drain sentinel (the same +/// "mask key" convention AutoHotkey uses): applications do not bind it and it +/// produces no character. +const DRAIN_SENTINEL_VK: VIRTUAL_KEY = VIRTUAL_KEY(0xE8); + +enum UnitRead { + Read, + Short(u32), + Stalled, +} + +/// The caller's thread attached to the foreground thread's input queue for +/// the duration of one foreground input transaction; detaches on drop. +pub(crate) struct InputQueueAttachment { + own_thread: u32, + target_thread: u32, +} + +impl InputQueueAttachment { + fn attach(foreground: HWND) -> Option<Self> { + use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; + let target_thread = unsafe { GetWindowThreadProcessId(foreground, None) }; + let own_thread = unsafe { GetCurrentThreadId() }; + if target_thread == 0 || target_thread == own_thread { + return None; + } + unsafe { AttachThreadInput(own_thread, target_thread, true) } + .as_bool() + .then_some(Self { + own_thread, + target_thread, + }) + } + + /// Block until the foreground thread has retrieved all input inserted so + /// far, bounded by `timeout`. A sentinel key press is appended behind the + /// caller's events; while attached, `GetKeyState` reflects the shared + /// queue's synchronous key state, whose toggle bit flips only when the + /// target thread reads the sentinel key-down. + fn wait_for_drain(&self, timeout: Duration) -> bool { + use windows::Win32::UI::Input::KeyboardAndMouse::GetKeyState; + let toggled = || unsafe { GetKeyState(DRAIN_SENTINEL_VK.0 as i32) } & 1; + let before = toggled(); + let sentinel = [sentinel_key_input(false), sentinel_key_input(true)]; + let sent = unsafe { SendInput(&sentinel, std::mem::size_of::<INPUT>() as i32) }; + if sent as usize != sentinel.len() { + let _ = unsafe { + SendInput( + &[sentinel_key_input(true)], + std::mem::size_of::<INPUT>() as i32, + ) + }; + sleep(Duration::from_millis(40)); + return false; + } + let deadline = Instant::now() + timeout; + loop { + if toggled() != before { + return true; + } + if Instant::now() >= deadline { + tracing::warn!( + "foreground input drain: target thread did not consume the sentinel within {} ms", + timeout.as_millis() + ); + return false; + } + sleep(Duration::from_millis(5)); + } + } + + /// Insert `events`, then yield until the sentinel key-up is read or `timeout`. + fn send_and_await_read(&self, events: &[INPUT], timeout: Duration) -> UnitRead { + use windows::Win32::UI::Input::KeyboardAndMouse::GetKeyState; + let before = unsafe { GetKeyState(DRAIN_SENTINEL_VK.0 as i32) } & 1; + let sent = unsafe { SendInput(events, std::mem::size_of::<INPUT>() as i32) }; + if (sent as usize) < events.len() { + return UnitRead::Short(sent); + } + let deadline = Instant::now() + timeout; + loop { + let state = unsafe { GetKeyState(DRAIN_SENTINEL_VK.0 as i32) }; + if (state & 1) != before && state >= 0 { + return UnitRead::Read; + } + if Instant::now() >= deadline { + return UnitRead::Stalled; + } + std::thread::yield_now(); + } + } +} + +impl Drop for InputQueueAttachment { + fn drop(&mut self) { + use windows::Win32::System::Threading::AttachThreadInput; + let _ = unsafe { AttachThreadInput(self.own_thread, self.target_thread, false) }; + } +} + +fn sentinel_key_input(up: bool) -> INPUT { + INPUT { + r#type: INPUT_KEYBOARD, + Anonymous: INPUT_0 { + ki: KEYBDINPUT { + wVk: DRAIN_SENTINEL_VK, + wScan: 0, + dwFlags: if up { + KEYEVENTF_KEYUP + } else { + KEYBD_EVENT_FLAGS(0) + }, + time: 0, + dwExtraInfo: 0, + }, + }, + } +} + +enum TextUnit { + Packet(u16), + Return, +} + +/// One readable VK_PACKET, or one Return. A supplementary scalar is two Packets (high, low). +fn text_units(text: &str) -> Vec<TextUnit> { + let mut units = Vec::with_capacity(text.len()); + let mut prev_was_cr = false; + for ch in text.chars() { + match ch { + '\n' if prev_was_cr => { + prev_was_cr = false; + } + '\n' | '\r' => { + units.push(TextUnit::Return); + prev_was_cr = ch == '\r'; + } + _ => { + prev_was_cr = false; + let mut buf = [0u16; 2]; + for unit in ch.encode_utf16(&mut buf) { + units.push(TextUnit::Packet(*unit)); + } + } + } + } + units +} + +/// The unit's key events followed by the drain sentinel, inserted by one SendInput. +fn unit_events(unit: &TextUnit) -> [INPUT; 4] { + use windows::Win32::UI::Input::KeyboardAndMouse::VK_RETURN; + let (d, u) = match unit { + TextUnit::Packet(c) => (unicode_key_input(*c, false), unicode_key_input(*c, true)), + TextUnit::Return => (key_input(VK_RETURN, false), key_input(VK_RETURN, true)), + }; + [d, u, sentinel_key_input(false), sentinel_key_input(true)] +} + +pub struct DesktopKeyboard; +impl cua_driver_core::desktop_keyboard::DesktopKeyboard for DesktopKeyboard { + fn resolve(&self, key: &str) -> Result<u32, String> { + key_name_to_vk(key) + .map(|key| u32::from(key.0)) + .map_err(|error| error.to_string()) + } + fn transition(&self, code: u32, down: bool, _held: &[u32]) -> Result<(), String> { + let event = key_input(VIRTUAL_KEY(code as u16), !down); + let sent = unsafe { SendInput(&[event], std::mem::size_of::<INPUT>() as i32) }; + if sent == 1 { + Ok(()) + } else { + Err("SendInput did not accept the key transition".into()) + } + } +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/packages/cua-driver/rust/crates/platform-windows/src/input/mod.rs index a573f971f1a..e68de031bf2 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/input/mod.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/input/mod.rs @@ -13,6 +13,7 @@ pub mod delivery; pub mod inject; pub mod keyboard; pub mod mouse; +mod user_input; pub(crate) use inject::{force_foreground_assisted, force_foreground_attached}; pub use inject::{ diff --git a/packages/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/packages/cua-driver/rust/crates/platform-windows/src/input/mouse.rs index e1b6d531e34..8a2159e1305 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/input/mouse.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/input/mouse.rs @@ -463,7 +463,66 @@ pub fn send_click_synthesized_active_mods( button: &str, modifiers: &[&str], ) -> Result<()> { - send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, true) + super::keyboard::with_confirmed_foreground( + HWND(target as *mut _), + "click delivery", + || Ok(()), + |_, owner| { + let (downs, ups) = super::keyboard::modifier_hold_inputs(modifiers); + super::user_input::checkpoint()?; + unsafe { + if !downs.is_empty() + && SendInput(&downs, std::mem::size_of::<INPUT>() as i32) as usize + != downs.len() + { + let _ = SendInput(&ups, std::mem::size_of::<INPUT>() as i32); + bail!("SendInput inserted partial modifiers; observe before retrying"); + } + } + let result = (|| { + for index in 0..count.max(1) { + super::user_input::checkpoint()?; + if !crate::win32::foreground_matches_target_or_owned_window(owner, unsafe { + GetForegroundWindow().0 as u64 + }) { + bail!("foreground_lost: target changed; remaining clicks were not sent"); + } + let (down, up) = match button { + "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), + "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), + _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), + }; + move_cursor_desktop(sx, sy)?; + let events = [down, up].map(|flag| INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dwFlags: flag, + ..Default::default() + }, + }, + }); + let sent = unsafe { SendInput(&events, std::mem::size_of::<INPUT>() as i32) }; + if sent != 2 { + unsafe { + let _ = SendInput(&events[1..], std::mem::size_of::<INPUT>() as i32); + } + bail!("SendInput inserted partial click; observe before retrying"); + } + if index + 1 < count { + sleep(Duration::from_millis(80)); + } + } + Ok(()) + })(); + unsafe { + if !ups.is_empty() { + let _ = SendInput(&ups, std::mem::size_of::<INPUT>() as i32); + } + } + result + }, + ) } fn send_click_synthesized_mods_impl( @@ -727,167 +786,69 @@ pub fn send_drag_synthesized( steps: usize, button: &str, ) -> Result<()> { - let target = HWND(target as *mut _); - if target.0.is_null() { - bail!("invalid target hwnd"); - } - if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) { - bail!(msg); - } - - let (down_flag, up_flag) = match button { - "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), - "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), - _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), - }; - - let (vd_x, vd_y, vd_w, vd_h) = unsafe { - ( - GetSystemMetrics(SM_XVIRTUALSCREEN), - GetSystemMetrics(SM_YVIRTUALSCREEN), - GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1), - GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1), - ) - }; - // Same VIRTUALDESK normalization as `send_click_synthesized`; see - // `crate::virtualdesk` for the math + the cross-platform unit tests. - let norm = |sx: i32, sy: i32| -> (i32, i32) { - crate::virtualdesk::to_virtualdesk_absolute(sx, sy, vd_x, vd_y, vd_w, vd_h) - }; - let make_input = |dx: i32, dy: i32, flags| INPUT { - r#type: INPUT_MOUSE, - Anonymous: INPUT_0 { - mi: MOUSEINPUT { - dx, - dy, - mouseData: 0, - dwFlags: flags | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, - time: 0, - dwExtraInfo: 0, - }, + super::keyboard::with_confirmed_foreground( + HWND(target as *mut _), + "drag delivery", + || Ok(()), + |_, owner| { + let (down, up) = match button { + "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), + "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), + _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), + }; + let event = |flag| INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dwFlags: flag, + ..Default::default() + }, + }, + }; + super::user_input::checkpoint()?; + move_cursor_desktop(sx_from, sy_from)?; + unsafe { + if SendInput(&[event(down)], std::mem::size_of::<INPUT>() as i32) != 1 { + bail!("SendInput drag press was not accepted"); + } + } + let result = (|| { + let steps = steps.max(1); + for step in 1..=steps { + super::user_input::checkpoint()?; + if !crate::win32::foreground_matches_target_or_owned_window(owner, unsafe { + GetForegroundWindow().0 as u64 + }) { + bail!("foreground_lost: drag stopped; observe before retrying"); + } + let fraction = step as f64 / steps as f64; + move_cursor_desktop( + sx_from + ((sx_to - sx_from) as f64 * fraction).round() as i32, + sy_from + ((sy_to - sy_from) as f64 * fraction).round() as i32, + )?; + if unsafe { + SendInput( + &[event(MOUSEEVENTF_MOVE)], + std::mem::size_of::<INPUT>() as i32, + ) + } != 1 + { + bail!("SendInput drag motion was not accepted"); + } + sleep(Duration::from_millis(duration_ms / steps as u64)); + } + Ok(()) + })(); + let released = unsafe { SendInput(&[event(up)], std::mem::size_of::<INPUT>() as i32) }; + result.and_then(|()| { + if released == 1 { + Ok(()) + } else { + anyhow::bail!("SendInput drag release was not accepted") + } + }) }, - }; - - let steps = steps.max(1); - let step_delay_ms = if steps > 1 { - duration_ms / steps as u64 - } else { - 0 - }; - - unsafe { - let prev_fg = GetForegroundWindow(); - let mut prev_cursor = POINT::default(); - let _ = GetCursorPos(&mut prev_cursor); - - // Lock-free z-order raise (no focus steal) so the coordinate-routed drag - // lands on the target — same technique as send_click_synthesized. - // SetForegroundWindow is lock-denied without UIAccess and isn't needed - // for pointer input; NoActivateGuard keeps the press from activating it. - let _noact = crate::input::NoActivateGuard::arm(target); - // Capture whether the target was ALREADY always-on-top so we only demote - // below if WE promoted it (else we'd strip a legitimate topmost window). - let was_topmost = (GetWindowLongPtrW(target, GWL_EXSTYLE) as u32) & WS_EX_TOPMOST.0 != 0; - let _ = SetWindowPos( - target, - HWND_TOPMOST, - 0, - 0, - 0, - 0, - SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE, - ); - - // 1. Move + press at the start of the drag. - let (nfx, nfy) = norm(sx_from, sy_from); - let _ = SetCursorPos(sx_from, sy_from); - let prelude = [ - make_input(nfx, nfy, MOUSEEVENTF_MOVE), - make_input(nfx, nfy, down_flag), - ]; - let sent = SendInput(&prelude, std::mem::size_of::<INPUT>() as i32); - if sent as usize != prelude.len() { - if !was_topmost { - let _ = SetWindowPos( - target, - HWND_NOTOPMOST, - 0, - 0, - 0, - 0, - SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE, - ); - } - if !prev_fg.0.is_null() && prev_fg != target { - let _ = SetWindowPos( - prev_fg, - HWND_TOP, - 0, - 0, - 0, - 0, - SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE, - ); - } - let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); - bail!( - "SendInput drag-prelude inserted {sent}/{} events", - prelude.len() - ); - } - - // 2. Interpolate the path. SetCursorPos + MOUSEEVENTF_MOVE in lockstep - // so both the visible cursor and the system input queue track the - // same path — WPF's drag-handler watches GetKeyState during each - // move event. - for i in 1..=steps { - let t = i as f64 / steps as f64; - let x = sx_from + ((sx_to - sx_from) as f64 * t).round() as i32; - let y = sy_from + ((sy_to - sy_from) as f64 * t).round() as i32; - let (nx, ny) = norm(x, y); - let _ = SetCursorPos(x, y); - let mv = [make_input(nx, ny, MOUSEEVENTF_MOVE)]; - let _ = SendInput(&mv, std::mem::size_of::<INPUT>() as i32); - if step_delay_ms > 0 { - sleep(Duration::from_millis(step_delay_ms)); - } - } - - // 3. Release at the end. - let (ntx, nty) = norm(sx_to, sy_to); - let release = [make_input(ntx, nty, up_flag)]; - let _ = SendInput(&release, std::mem::size_of::<INPUT>() as i32); - - // Brief settle, then restore z-order (demote target, restack user's - // window — no activation) and the cursor. - sleep(Duration::from_millis(40)); - if !was_topmost { - let _ = SetWindowPos( - target, - HWND_NOTOPMOST, - 0, - 0, - 0, - 0, - SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE, - ); - } - if !prev_fg.0.is_null() && prev_fg != target { - let _ = SetWindowPos( - prev_fg, - HWND_TOP, - 0, - 0, - 0, - 0, - SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE, - ); - } - let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); - drop(_noact); - } - - Ok(()) + ) } /// Standard wheel notch delta. A `mouseData` value of `±WHEEL_DELTA` is one @@ -920,6 +881,7 @@ fn wheel_mouse_data(ticks: i32) -> u32 { /// delivery follows the cursor, so positioning the cursor is sufficient. The /// cursor is restored to its previous position afterward. pub fn send_wheel_synthesized(sx: i32, sy: i32, ticks: i32, horizontal: bool) -> Result<()> { + super::user_input::checkpoint()?; let flag = if horizontal { MOUSEEVENTF_HWHEEL } else { @@ -975,8 +937,32 @@ pub fn foreground_window() -> Result<u64> { /// Move the real OS pointer to a desktop coordinate. pub fn move_cursor_desktop(x: i32, y: i32) -> Result<()> { - unsafe { SetCursorPos(x, y) } - .map_err(|error| anyhow::anyhow!("SetCursorPos({x}, {y}) failed: {error}")) + let (dx, dy) = unsafe { + crate::virtualdesk::to_virtualdesk_absolute( + x, + y, + GetSystemMetrics(SM_XVIRTUALSCREEN), + GetSystemMetrics(SM_YVIRTUALSCREEN), + GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1), + GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1), + ) + }; + let input = INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx, + dy, + dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + ..Default::default() + }, + }, + }; + if unsafe { SendInput(&[input], std::mem::size_of::<INPUT>() as i32) } == 1 { + Ok(()) + } else { + bail!("SendInput pointer move was not accepted") + } } /// Classify a `WM_NCHITTEST` result into the non-client regions where a diff --git a/packages/cua-driver/rust/crates/platform-windows/src/input/user_input.rs b/packages/cua-driver/rust/crates/platform-windows/src/input/user_input.rs new file mode 100644 index 00000000000..557ae20f3f3 --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-windows/src/input/user_input.rs @@ -0,0 +1,126 @@ +//! Content-free physical input detection. Hooks always forward every event. +use anyhow::{anyhow, bail, Result}; +use std::cell::RefCell; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::time::Duration; +use windows::Win32::Foundation::{LPARAM, LRESULT, WPARAM}; +use windows::Win32::UI::WindowsAndMessaging::*; +#[derive(Default)] +struct State { + stopped: AtomicBool, + detected: AtomicBool, +} +thread_local! { + static OBSERVING: RefCell<Option<Arc<State>>> = const { RefCell::new(None) }; + static ACTIVE: RefCell<Option<Arc<State>>> = const { RefCell::new(None) }; +} +unsafe extern "system" fn keyboard(code: i32, w: WPARAM, l: LPARAM) -> LRESULT { + if code >= 0 + && !(*(l.0 as *const KBDLLHOOKSTRUCT)) + .flags + .contains(LLKHF_INJECTED) + { + mark(); + } + CallNextHookEx(None, code, w, l) +} +unsafe extern "system" fn mouse(code: i32, w: WPARAM, l: LPARAM) -> LRESULT { + if code >= 0 && (*(l.0 as *const MSLLHOOKSTRUCT)).flags & 1 == 0 { + mark(); + } + CallNextHookEx(None, code, w, l) +} +fn mark() { + OBSERVING.with(|state| { + if let Some(state) = state.borrow().as_ref() { + state.detected.store(true, Ordering::Release); + } + }); +} +pub(super) struct Guard { + state: Arc<State>, + previous: Option<Arc<State>>, + worker: Option<std::thread::JoinHandle<()>>, +} +impl Guard { + pub(super) fn start() -> Result<Self> { + let state = Arc::new(State::default()); + let worker_state = state.clone(); + let (ready, started) = std::sync::mpsc::sync_channel(1); + let worker = std::thread::Builder::new() + .name("cua-user-input".into()) + .spawn(move || unsafe { + OBSERVING.with(|state| { + state.replace(Some(worker_state.clone())); + }); + let keyboard = match SetWindowsHookExW(WH_KEYBOARD_LL, Some(keyboard), None, 0) { + Ok(hook) => hook, + Err(error) => { + let _ = ready.send(Err(error.to_string())); + return; + } + }; + let mouse = match SetWindowsHookExW(WH_MOUSE_LL, Some(mouse), None, 0) { + Ok(hook) => hook, + Err(error) => { + let _ = UnhookWindowsHookEx(keyboard); + let _ = ready.send(Err(error.to_string())); + return; + } + }; + let _ = ready.send(Ok(())); + let mut message = MSG::default(); + while !worker_state.stopped.load(Ordering::Acquire) { + while PeekMessageW(&mut message, None, 0, 0, PM_REMOVE).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + std::thread::sleep(Duration::from_millis(1)); + } + let _ = UnhookWindowsHookEx(mouse); + let _ = UnhookWindowsHookEx(keyboard); + OBSERVING.with(|state| { + state.take(); + }); + })?; + match started.recv_timeout(Duration::from_secs(1)) { + Ok(Ok(())) => {} + failure => { + state.stopped.store(true, Ordering::Release); + let _ = worker.join(); + bail!("foreground_unavailable: physical-input observer could not start; no input was sent: {failure:?}"); + } + } + let previous = ACTIVE.with(|active| active.replace(Some(state.clone()))); + Ok(Self { + state, + previous, + worker: Some(worker), + }) + } +} +impl Drop for Guard { + fn drop(&mut self) { + ACTIVE.with(|active| { + active.replace(self.previous.take()); + }); + self.state.stopped.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} +pub(super) fn checkpoint() -> Result<()> { + if ACTIVE.with(|active| { + active + .borrow() + .as_ref() + .is_some_and(|state| state.detected.load(Ordering::Acquire)) + }) { + return Err(anyhow!("user_input_detected: physical input arrived during foreground delivery; input stopped; observe before retrying")); + } + Ok(()) +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/lib.rs b/packages/cua-driver/rust/crates/platform-windows/src/lib.rs index c9f0af43419..b11459357b2 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/lib.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/lib.rs @@ -100,3 +100,10 @@ pub fn register_tools_with_cursor_and_provider( } tools::build_registry_with_provider(compat, provider) } + +#[cfg(any(windows, test))] +mod browser_launch_token; +#[cfg(windows)] +mod browser_standard_user; +#[cfg(windows)] +pub mod standard_user_launch; diff --git a/packages/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs b/packages/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs index aa29319b4e4..1d28719e619 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs @@ -6,15 +6,6 @@ //! window-local screenshot-pixel coordinates so `click.png` is also written //! on UIA/MSAA-indexed clicks (not just pixel-addressed ones). -#[cfg(target_os = "windows")] -use std::{ - collections::HashMap, - sync::{Arc, Mutex, OnceLock, Weak}, -}; - -#[cfg(target_os = "windows")] -use crate::uia::ElementCache; - use cua_driver_core::recording::ScreenshotCapture; #[cfg(target_os = "windows")] @@ -28,21 +19,6 @@ use windows::Win32::UI::WindowsAndMessaging::{ GetLastActivePopup, GetWindowThreadProcessId, IsWindow, IsWindowVisible, }; -#[cfg(target_os = "windows")] -static ELEMENT_CACHES: OnceLock<Mutex<HashMap<String, Weak<ElementCache>>>> = OnceLock::new(); - -#[cfg(target_os = "windows")] -pub fn set_element_cache(cache: Arc<ElementCache>) { - let runtime_scope = - cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into()); - let mut caches = ELEMENT_CACHES - .get_or_init(|| Mutex::new(HashMap::new())) - .lock() - .unwrap(); - caches.retain(|_, cache| cache.strong_count() > 0); - caches.insert(runtime_scope, Arc::downgrade(&cache)); -} - /// Resolve the window whose application evidence should be captured. Keep a /// live explicit HWND so occluded/background turns capture the exact target. /// When an action closes a modal HWND, fall back to another top-level window @@ -97,7 +73,46 @@ pub fn screenshot_for_recording(window_id: Option<u64>, pid: Option<i64>) -> Scr pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> { let pid = u32::try_from(pid?).ok()?; let hwnd = resolve_window_for_recording(window_id, Some(pid.into()))?; - let result = crate::uia::walk_tree(hwnd, None); + recording_tree_json(pid, hwnd) +} + +#[cfg(target_os = "windows")] +struct RecordingApartment; + +#[cfg(target_os = "windows")] +impl RecordingApartment { + fn new() -> Option<Self> { + use windows::Win32::System::Com::{CoInitializeEx, COINIT_MULTITHREADED}; + if unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }.is_err() { + return None; + } + Some(Self) + } +} + +#[cfg(target_os = "windows")] +impl Drop for RecordingApartment { + fn drop(&mut self) { + unsafe { windows::Win32::System::Com::CoUninitialize() }; + } +} + +#[cfg(target_os = "windows")] +struct RecordingTree(crate::uia::UiaTreeResult); + +#[cfg(target_os = "windows")] +impl Drop for RecordingTree { + fn drop(&mut self) { + crate::uia::settle_node_bindings(&mut self.0.nodes, self.0.backend, false); + } +} + +#[cfg(target_os = "windows")] +fn recording_tree_json(pid: u32, hwnd: u64) -> Option<Vec<u8>> { + // Declaration order keeps the apartment alive until raw node references drop. + let _apartment = RecordingApartment::new()?; + let tree = RecordingTree(unsafe { crate::uia::walk_tree_for_recording(hwnd) }); + let result = &tree.0; let element_count = result .nodes .iter() @@ -113,15 +128,13 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve } #[cfg(target_os = "windows")] -pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> { - let runtime_scope = - cua_driver_core::tool::current_dispatch_runtime_scope().unwrap_or_else(|| "legacy".into()); - let cache = ELEMENT_CACHES - .get()? - .lock() - .unwrap() - .get(&runtime_scope)? - .upgrade()?; +pub fn element_window_local_xy( + window_id: u64, + pid: i64, + element_index: u32, + binding: &cua_driver_core::element_token::SnapshotBinding, +) -> Option<(f64, f64)> { + let cache = crate::uia::cache::BoundElementCache::from_binding(binding.clone()); let pid_u32 = u32::try_from(pid).ok()?; let (sx, sy) = cache.get_element_center(pid_u32, window_id, element_index as usize)?; // The cached center is in SCREEN coords. Convert to window-local pixel @@ -149,6 +162,172 @@ pub fn element_window_local_xy( _window_id: u64, _pid: i64, _element_index: u32, + _binding: &cua_driver_core::element_token::SnapshotBinding, ) -> Option<(f64, f64)> { None } + +#[cfg(all(test, target_os = "windows"))] +mod recording_com_tests { + use super::*; + use crate::uia::{UiaBackend, UiaNode, UiaTreeResult}; + use std::ffi::c_void; + use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }; + use windows::core::{IUnknown_Vtbl, GUID, HRESULT}; + use windows::Win32::Foundation::{CO_E_NOTINITIALIZED, E_NOINTERFACE}; + use windows::Win32::System::Com::{ + CoGetApartmentType, CoInitializeEx, CoUninitialize, APTTYPE, APTTYPEQUALIFIER, + APTTYPEQUALIFIER_IMPLICIT_MTA, APTTYPEQUALIFIER_NONE, COINIT_APARTMENTTHREADED, + COINIT_MULTITHREADED, + }; + + fn apartment() -> windows::core::Result<(APTTYPE, APTTYPEQUALIFIER)> { + let mut kind = APTTYPE::default(); + let mut qualifier = APTTYPEQUALIFIER::default(); + unsafe { CoGetApartmentType(&mut kind, &mut qualifier)? }; + Ok((kind, qualifier)) + } + + fn assert_uninitialized() { + // Another UIA worker may keep the process MTA alive. That gives this + // thread an implicit apartment, without a CoInitializeEx reference. + match apartment() { + Err(error) => assert_eq!(error.code(), CO_E_NOTINITIALIZED), + Ok((_, qualifier)) => assert_eq!(qualifier, APTTYPEQUALIFIER_IMPLICIT_MTA), + } + } + + #[test] + fn recording_worker_balances_com_and_preserves_other_apartments() { + std::thread::spawn(|| unsafe { + assert_uninitialized(); + // The unchanged legacy entry demonstrates the old initialization + // lifetime. HWND 0 exercises native failure without opening a GUI. + let mut legacy = crate::uia::walk_tree(0, None); + crate::uia::settle_node_bindings(&mut legacy.nodes, legacy.backend, false); + assert_eq!(apartment().unwrap().1, APTTYPEQUALIFIER_NONE); + drop(legacy); + CoUninitialize(); + assert_uninitialized(); + + assert!(recording_tree_json(0, 0).is_some()); + assert_uninitialized(); + + CoInitializeEx(None, COINIT_MULTITHREADED).ok().unwrap(); + let previous = apartment().unwrap(); + assert!(recording_tree_json(0, 0).is_some()); + assert_eq!(apartment().unwrap(), previous); // S_FALSE was paired once. + CoUninitialize(); + assert_uninitialized(); + + CoInitializeEx(None, COINIT_APARTMENTTHREADED).ok().unwrap(); + let previous = apartment().unwrap(); + assert!(recording_tree_json(0, 0).is_none()); // Changed mode is rejected. + assert_eq!(apartment().unwrap(), previous); + CoUninitialize(); + assert_uninitialized(); + }) + .join() + .unwrap(); + } + + #[repr(C)] + struct ReleaseProbe { + vtable: *const IUnknown_Vtbl, + releases: Arc<AtomicUsize>, + live_apartment: Arc<AtomicBool>, + } + + unsafe extern "system" fn query( + _: *mut c_void, + _: *const GUID, + out: *mut *mut c_void, + ) -> HRESULT { + *out = std::ptr::null_mut(); + E_NOINTERFACE + } + unsafe extern "system" fn add_ref(_: *mut c_void) -> u32 { + 2 + } + unsafe extern "system" fn release(ptr: *mut c_void) -> u32 { + let probe = Box::from_raw(ptr.cast::<ReleaseProbe>()); + probe.live_apartment.store( + apartment().is_ok_and(|(_, qualifier)| qualifier == APTTYPEQUALIFIER_NONE), + Ordering::SeqCst, + ); + probe.releases.fetch_add(1, Ordering::SeqCst); + 0 + } + static VTABLE: IUnknown_Vtbl = IUnknown_Vtbl { + QueryInterface: query, + AddRef: add_ref, + Release: release, + }; + + fn node(ptr: usize, actionable: bool) -> UiaNode { + UiaNode { + element_index: actionable.then_some(0), + control_type: "probe".into(), + name: None, + value: None, + automation_id: None, + help_text: None, + actions: Vec::new(), + runtime_id: None, + enabled: None, + selected: None, + element_ptr: ptr, + center_x: 0, + center_y: 0, + rect: None, + msaa_role: None, + depth: 0, + parent_element_index: None, + in_web_content: false, + } + } + + #[test] + fn recording_releases_uia_and_msaa_bindings_before_apartment_on_unwind() { + std::thread::spawn(|| { + for backend in [UiaBackend::Uia, UiaBackend::Msaa] { + let releases = Arc::new(AtomicUsize::new(0)); + let live_apartment = Arc::new(AtomicBool::new(false)); + let nodes = [false, true] + .into_iter() + .map(|actionable| { + // Only IUnknown::Release is called by the real settlement + // path; this owned fake has no UIA provider or desktop data. + let ptr = Box::into_raw(Box::new(ReleaseProbe { + vtable: &VTABLE, + releases: releases.clone(), + live_apartment: live_apartment.clone(), + })); + node(ptr as usize, actionable) + }) + .collect(); + let unwound = std::panic::catch_unwind(|| { + let _apartment = RecordingApartment::new().unwrap(); + let _tree = RecordingTree(UiaTreeResult { + nodes, + backend, + tree_markdown: String::new(), + complete: true, + truncated: false, + incomplete_notes: Vec::new(), + }); + panic!("controlled serialization unwind"); + }); + assert!(unwound.is_err()); + assert_eq!(releases.load(Ordering::SeqCst), 2); + assert!(live_apartment.load(Ordering::SeqCst)); + assert_uninitialized(); + } + }) + .join() + .unwrap(); + } +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/standard_user_launch.rs b/packages/cua-driver/rust/crates/platform-windows/src/standard_user_launch.rs new file mode 100644 index 00000000000..c02021d033d --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-windows/src/standard_user_launch.rs @@ -0,0 +1,360 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! Standard-user app launches for an elevated Windows Driver (#3607). +//! +//! A process inherits its creator's primary token. An elevated Driver (the +//! default autostart daemon runs at `RunLevel=Highest`) would therefore start +//! every `launch_app` target as an elevated administrator: the user's ordinary +//! tools could not close or automate the app, and it would behave differently +//! from the same app started from Explorer. An elevated Driver instead +//! launches apps with the verified standard-user token that already runs +//! isolated browsers (#4234): the same user and desktop, `BUILTIN\Administrators` +//! deny-only, and Medium integrity. The Driver keeps full control, because a +//! higher-integrity process may send input to and automate a lower one. +//! +//! Direct executable launches use `CreateProcessAsUserW` with that token. +//! `ShellExecuteExW` (PATH and App Paths lookup, file associations, +//! `shell:AppsFolder` registrations, URLs) always creates processes with its +//! caller's primary token, so shell launches run in a short-lived helper that +//! already holds the standard-user token: this Driver executable, started with +//! [`SHELL_LAUNCH_HELPER_FLAG`]. The helper performs one `ShellExecuteExW` call, +//! reports the launched process ID or the shell error on its standard output, +//! and exits. A process that embeds the Driver runtime without the Driver's +//! entry point (for example an SDK host) cannot run the helper, so an elevated +//! host of that kind refuses shell launches instead of starting them elevated. + +use std::ffi::{c_void, OsStr, OsString}; +use std::io::Read as _; +use std::os::windows::ffi::OsStrExt; +use std::os::windows::io::FromRawHandle as _; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use windows::core::{PCWSTR, PWSTR}; +use windows::Win32::Foundation::{ + SetHandleInformation, HANDLE, HANDLE_FLAG_INHERIT, WAIT_OBJECT_0, WAIT_TIMEOUT, +}; +use windows::Win32::System::Pipes::CreatePipe; +use windows::Win32::System::Threading::{ + CreateProcessAsUserW, DeleteProcThreadAttributeList, GetExitCodeProcess, + InitializeProcThreadAttributeList, TerminateProcess, UpdateProcThreadAttribute, + WaitForSingleObject, CREATE_NO_WINDOW, EXTENDED_STARTUPINFO_PRESENT, + LPPROC_THREAD_ATTRIBUTE_LIST, PROCESS_CREATION_FLAGS, PROCESS_INFORMATION, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, STARTF_USESHOWWINDOW, STARTF_USESTDHANDLES, STARTUPINFOEXW, + STARTUPINFOW, +}; + +use crate::browser_launch_token::windows_command_line; +use crate::browser_standard_user::{browser_launch_context, BrowserLaunchContext, OwnedHandle}; + +/// First argument that turns this Driver executable into the shell launch +/// helper. Never a public command. +pub const SHELL_LAUNCH_HELPER_FLAG: &str = "--cua-driver-standard-user-shell-launch"; + +/// Set once the Driver's own entry point has checked for the helper flag, +/// which proves this executable can serve as the helper. +static SHELL_LAUNCH_HELPER_AVAILABLE: AtomicBool = AtomicBool::new(false); + +/// The token that runs apps started by `launch_app`. +#[derive(Clone)] +pub(crate) enum AppLaunchToken { + /// The Driver is not elevated; apps start with the Driver's own token. + Driver, + /// A verified standard-user token derived from the elevated Driver token. + StandardUser(Arc<OwnedHandle>), +} + +/// Choose the launch token. An elevated Driver that cannot derive a verified +/// standard-user token refuses rather than starting the app elevated. +pub(crate) fn app_launch_token() -> Result<AppLaunchToken, String> { + match browser_launch_context() { + Ok(BrowserLaunchContext::Driver) => Ok(AppLaunchToken::Driver), + Ok(BrowserLaunchContext::StandardUser(token)) => Ok(AppLaunchToken::StandardUser(token)), + Err(refusal) => Err(format!( + "The Driver is elevated and could not prepare the standard-user token it uses to \ + launch apps, so no app was started: {}", + refusal.message + )), + } +} + +fn wide(value: &OsStr) -> Vec<u16> { + value.encode_wide().collect() +} + +fn nul_terminated(mut value: Vec<u16>) -> Vec<u16> { + value.push(0); + value +} + +/// Create `application` with `command_line` under `token`, showing its first +/// window with `n_show`. The child inherits no handles. +pub(crate) fn create_process( + token: &OwnedHandle, + application: &str, + command_line: &str, + n_show: i32, +) -> Result<u32, String> { + let application = nul_terminated(wide(OsStr::new(application))); + let mut command_line = nul_terminated(wide(OsStr::new(command_line))); + let startup = STARTUPINFOW { + cb: std::mem::size_of::<STARTUPINFOW>() as u32, + dwFlags: STARTF_USESHOWWINDOW, + wShowWindow: n_show as u16, + ..Default::default() + }; + let mut process = PROCESS_INFORMATION::default(); + unsafe { + CreateProcessAsUserW( + token.raw(), + PCWSTR(application.as_ptr()), + PWSTR(command_line.as_mut_ptr()), + None, + None, + false, + PROCESS_CREATION_FLAGS(0), + None, + PCWSTR::null(), + &startup, + &mut process, + ) + } + .map_err(|error| format!("CreateProcessAsUserW failed: {error}"))?; + drop(OwnedHandle::new(process.hThread)); + drop(OwnedHandle::new(process.hProcess)); + Ok(process.dwProcessId) +} + +/// Run `ShellExecuteExW` for `file` and `parameters` in a helper that holds +/// `token`. Returns the launched process ID, or 0 when the shell handed the +/// request to an existing process (for example a URL opened in a running +/// browser). The helper is terminated if it does not finish within `timeout`. +pub(crate) fn shell_execute( + token: &OwnedHandle, + file: &str, + parameters: &str, + n_show: i32, + timeout: Duration, +) -> Result<u32, String> { + if !SHELL_LAUNCH_HELPER_AVAILABLE.load(Ordering::Relaxed) { + return Err( + "The Driver is elevated and this process cannot start its standard-user shell launch \ + helper, so no app was started. Launch the app by executable path with \ + start_minimized, or run this host without elevation." + .to_owned(), + ); + } + let helper = std::env::current_exe() + .map_err(|error| format!("could not locate the Driver executable: {error}"))?; + let program = wide(helper.as_os_str()); + let args = [ + wide(OsStr::new(SHELL_LAUNCH_HELPER_FLAG)), + wide(OsStr::new(&n_show.to_string())), + wide(OsStr::new(file)), + wide(OsStr::new(parameters)), + ]; + let mut command_line = nul_terminated(windows_command_line(&program, &args)?); + let application = nul_terminated(program); + + // Only the write end of this pipe crosses into the helper. + let mut read = HANDLE::default(); + let mut write = HANDLE::default(); + unsafe { CreatePipe(&mut read, &mut write, None, 0) } + .map_err(|error| format!("could not create the helper result pipe: {error}"))?; + let read = OwnedHandle::new(read); + let write = OwnedHandle::new(write); + unsafe { SetHandleInformation(write.raw(), HANDLE_FLAG_INHERIT.0, HANDLE_FLAG_INHERIT) } + .map_err(|error| format!("could not share the helper result pipe: {error}"))?; + + let mut attribute_size = 0usize; + let _ = unsafe { + InitializeProcThreadAttributeList( + LPPROC_THREAD_ATTRIBUTE_LIST(std::ptr::null_mut()), + 1, + 0, + &mut attribute_size, + ) + }; + let mut attribute_storage = vec![0u64; attribute_size.div_ceil(8)]; + let attributes = LPPROC_THREAD_ATTRIBUTE_LIST(attribute_storage.as_mut_ptr().cast()); + unsafe { InitializeProcThreadAttributeList(attributes, 1, 0, &mut attribute_size) } + .map_err(|error| format!("could not prepare the helper handle list: {error}"))?; + let inherited = [write.raw()]; + let updated = unsafe { + UpdateProcThreadAttribute( + attributes, + 0, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST as usize, + Some(inherited.as_ptr().cast::<c_void>()), + std::mem::size_of_val(&inherited), + None, + None, + ) + }; + let startup = STARTUPINFOEXW { + StartupInfo: STARTUPINFOW { + cb: std::mem::size_of::<STARTUPINFOEXW>() as u32, + dwFlags: STARTF_USESTDHANDLES, + hStdOutput: write.raw(), + hStdError: write.raw(), + ..Default::default() + }, + lpAttributeList: attributes, + }; + let mut process = PROCESS_INFORMATION::default(); + let created = updated.and_then(|()| unsafe { + CreateProcessAsUserW( + token.raw(), + PCWSTR(application.as_ptr()), + PWSTR(command_line.as_mut_ptr()), + None, + None, + true, + EXTENDED_STARTUPINFO_PRESENT | CREATE_NO_WINDOW, + None, + PCWSTR::null(), + &startup.StartupInfo, + &mut process, + ) + }); + unsafe { DeleteProcThreadAttributeList(attributes) }; + created.map_err(|error| format!("could not start the standard-user launch helper: {error}"))?; + drop(write); + drop(OwnedHandle::new(process.hThread)); + let helper = OwnedHandle::new(process.hProcess); + + let timeout_ms = u32::try_from(timeout.as_millis()).unwrap_or(u32::MAX); + let waited = unsafe { WaitForSingleObject(helper.raw(), timeout_ms) }; + if waited == WAIT_TIMEOUT { + let _ = unsafe { TerminateProcess(helper.raw(), 1) }; + return Err(format!( + "the standard-user launch helper did not finish within {}s; a blocking shell dialog \ + likely appeared on the session desktop", + timeout.as_secs() + )); + } + if waited != WAIT_OBJECT_0 { + return Err(format!( + "could not wait for the standard-user launch helper: {}", + std::io::Error::last_os_error() + )); + } + let mut exit_code = 0u32; + let _ = unsafe { GetExitCodeProcess(helper.raw(), &mut exit_code) }; + + // The helper has exited, so its end of the pipe is closed and the read + // ends at the helper's last write. + let mut output = String::new(); + let mut pipe = unsafe { std::fs::File::from_raw_handle(read.into_raw().0) }; + let _ = pipe.read_to_string(&mut output); + parse_helper_output(&output, exit_code) +} + +/// Interpret the helper's single result line. +fn parse_helper_output(output: &str, exit_code: u32) -> Result<u32, String> { + let line = output.lines().next().unwrap_or("").trim(); + if let Some(pid) = line.strip_prefix("ok ") { + return pid + .trim() + .parse::<u32>() + .map_err(|_| format!("the standard-user launch helper reported {line:?}")); + } + if let Some(message) = line.strip_prefix("error ") { + return Err(message.trim().to_owned()); + } + Err(format!( + "the standard-user launch helper exited with code {exit_code:#x} without a result" + )) +} + +/// Entry point for the helper. The Driver's `main` calls this first; it +/// returns `None` for every ordinary invocation. +pub fn run_shell_launch_helper_if_requested() -> Option<i32> { + SHELL_LAUNCH_HELPER_AVAILABLE.store(true, Ordering::Relaxed); + let mut args = std::env::args_os().skip(1); + if args.next().as_deref() != Some(OsStr::new(SHELL_LAUNCH_HELPER_FLAG)) { + return None; + } + let rest: Vec<OsString> = args.collect(); + let result = match rest.as_slice() { + [n_show, file, parameters] => match n_show.to_str().and_then(|v| v.parse::<i32>().ok()) { + Some(n_show) => helper_shell_execute(file, parameters, n_show), + None => Err("the launch helper received an invalid show state".to_owned()), + }, + _ => Err("the launch helper received malformed arguments".to_owned()), + }; + match result { + Ok(pid) => { + println!("ok {pid}"); + Some(0) + } + Err(message) => { + println!("error {message}"); + Some(1) + } + } +} + +fn helper_shell_execute(file: &OsStr, parameters: &OsStr, n_show: i32) -> Result<u32, String> { + use windows::Win32::Foundation::CloseHandle; + use windows::Win32::System::Com::{ + CoInitializeEx, COINIT, COINIT_APARTMENTTHREADED, COINIT_DISABLE_OLE1DDE, + }; + use windows::Win32::System::Threading::GetProcessId; + use windows::Win32::UI::Shell::{ + ShellExecuteExW, SEE_MASK_FLAG_NO_UI, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW, + }; + + // Shell verbs may be implemented by COM handlers. + let _ = unsafe { + CoInitializeEx( + None, + COINIT(COINIT_APARTMENTTHREADED.0 | COINIT_DISABLE_OLE1DDE.0), + ) + }; + let verb = nul_terminated(wide(OsStr::new("open"))); + let file = nul_terminated(wide(file)); + let parameters_wide = nul_terminated(wide(parameters)); + let mut info = SHELLEXECUTEINFOW { + cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32, + fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI, + lpVerb: PCWSTR(verb.as_ptr()), + lpFile: PCWSTR(file.as_ptr()), + lpParameters: if parameters.is_empty() { + PCWSTR::null() + } else { + PCWSTR(parameters_wide.as_ptr()) + }, + nShow: n_show, + ..Default::default() + }; + unsafe { ShellExecuteExW(&mut info) }.map_err(|error| error.to_string())?; + if info.hProcess.is_invalid() { + return Ok(0); + } + let pid = unsafe { GetProcessId(info.hProcess) }; + let _ = unsafe { CloseHandle(info.hProcess) }; + Ok(pid) +} + +#[cfg(test)] +mod tests { + use super::parse_helper_output; + + #[test] + fn helper_output_reports_the_process_id_or_the_shell_error() { + assert_eq!(parse_helper_output("ok 4242\r\n", 0), Ok(4242)); + assert_eq!(parse_helper_output("ok 0\n", 0), Ok(0)); + assert_eq!( + parse_helper_output("error The system cannot find the file specified.\n", 1), + Err("The system cannot find the file specified.".to_owned()) + ); + assert_eq!( + parse_helper_output("", 0xc000_0005), + Err( + "the standard-user launch helper exited with code 0xc0000005 without a result" + .to_owned() + ) + ); + } +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/packages/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index dafb42d760d..1bc84b1a30a 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -50,7 +50,7 @@ fn pin_overlay_above(key: &str, hwnd: u64) { /// window. MSAA-walked elements are skipped (no ScrollItemPattern) and keep the /// existing failure. fn resolve_onscreen_point_with_scroll( - element_cache: &crate::uia::cache::ElementCache, + element_cache: &crate::uia::cache::BoundElementCache, pid: u32, hwnd: u64, idx: usize, @@ -447,6 +447,18 @@ impl Tool for ExactPidWindowTargetGuard { if let Err(result) = exact_window_ownership_result(pid, window_id, owner_pid) { return result; } + if matches!( + self.inner.def().name.as_str(), + "click" | "double_click" | "right_click" + ) && crate::input::delivery::DeliveryMode::from_args(&args) + == crate::input::delivery::DeliveryMode::Background + && crate::input::delivery::is_tk_target_window(window_id) + { + return crate::input::delivery::background_unavailable_error( + window_id, + crate::input::delivery::EventKind::MouseClick, + ); + } } } self.inner.invoke(args).await @@ -548,7 +560,7 @@ pub fn load_driver_config() -> DriverConfig { } pub struct ResizeRegistry { - ratios: std::sync::Mutex<std::collections::HashMap<u32, f64>>, + ratios: std::sync::Mutex<std::collections::HashMap<(u32, u64), f64>>, } impl ResizeRegistry { @@ -557,14 +569,14 @@ impl ResizeRegistry { ratios: std::sync::Mutex::new(Default::default()), } } - pub fn set_ratio(&self, pid: u32, ratio: f64) { - self.ratios.lock().unwrap().insert(pid, ratio); + pub fn set_ratio(&self, pid: u32, window: u64, ratio: f64) { + self.ratios.lock().unwrap().insert((pid, window), ratio); } - pub fn clear_ratio(&self, pid: u32) { - self.ratios.lock().unwrap().remove(&pid); + pub fn clear_ratio(&self, pid: u32, window: u64) { + self.ratios.lock().unwrap().remove(&(pid, window)); } - pub fn ratio(&self, pid: u32) -> Option<f64> { - self.ratios.lock().unwrap().get(&pid).copied() + pub fn ratio(&self, pid: u32, window: u64) -> Option<f64> { + self.ratios.lock().unwrap().get(&(pid, window)).copied() } } @@ -1119,6 +1131,33 @@ mod list_windows_z_index_tests { // ── get_window_state ───────────────────────────────────────────────────────── +struct PendingUiaTree(crate::uia::UiaTreeResult); + +impl std::ops::Deref for PendingUiaTree { + type Target = crate::uia::UiaTreeResult; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for PendingUiaTree { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl PendingUiaTree { + fn settle(&mut self, transferred: bool) { + crate::uia::settle_node_bindings(&mut self.0.nodes, self.0.backend, transferred); + } +} + +impl Drop for PendingUiaTree { + fn drop(&mut self) { + self.settle(false); + } +} + pub struct GetWindowStateTool { state: Arc<ToolState>, } @@ -1186,6 +1225,7 @@ impl Tool for GetWindowStateTool { "window_id":{"type":"integer","description":"HWND of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array."}, "app_context":{"type":"boolean","description":"Use the compact app observation projection."}, "capture_mode": cua_driver_core::capture_mode::capture_mode_schema(), + "include_accessibility":{"type":"boolean","default":true,"description":"Set false for a screenshot without a UIA walk."}, "include_screenshot":{"type":"boolean","description":"Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return tree only (the cheap path for re-indexing before an element ax action)."}, "screenshot_out_file":{"type":"string","description":"When set, write the PNG to this file path instead of embedding base64 in the response. The structured output will contain `screenshot_file_path` instead."}, "query":{"type":"string","description":"Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count."}, @@ -1406,8 +1446,12 @@ impl Tool for GetWindowStateTool { let (progress_tx, mut progress_rx) = tokio::sync::mpsc::unbounded_channel(); let collect_runtime_ids = revision_request_for_capture.is_some(); + let include_accessibility = args["include_accessibility"] != false; let mut tree_task = tokio::task::spawn_blocking(move || { - crate::uia::walk_tree_bounded_with_progress( + if !include_accessibility { + return None; + } + Some(PendingUiaTree(crate::uia::walk_tree_bounded_with_progress( hwnd, tree_query.as_deref(), max_elements, @@ -1416,13 +1460,13 @@ impl Tool for GetWindowStateTool { &mut |node, line| { let _ = progress_tx.send((node.depth, node, line)); }, - ) + ))) }); let tree_deadline = tokio::time::sleep(std::time::Duration::from_secs(4)); tokio::pin!(tree_deadline); let mut partial_nodes = Vec::new(); let mut partial_lines = Vec::new(); - let (mut tree_result, uia_worker_restart_required) = loop { + let (tree_result, uia_worker_restart_required) = loop { tokio::select! { joined = &mut tree_task => { let tree = joined.map_err(|error| anyhow::anyhow!("UIA worker task failed: {error}")); @@ -1442,7 +1486,7 @@ impl Tool for GetWindowStateTool { query.as_deref(), "uia_provider_timeout", ); - break (Ok(partial), true); + break (Ok(Some(PendingUiaTree(partial))), true); } } }; @@ -1455,7 +1499,7 @@ impl Tool for GetWindowStateTool { let observation_revision = match ( revision_request_for_capture.as_ref(), observation_session, - tree_result.as_ref().ok(), + tree_result.as_ref().ok().and_then(Option::as_ref), ) { (Some(request), Some(session), Some(tree)) => match observation_revisions.observe( session, @@ -1468,9 +1512,6 @@ impl Tool for GetWindowStateTool { ) { Ok(revision) => Some(revision), Err(error) => { - if let Ok(tree) = tree_result.as_mut() { - crate::uia::settle_node_bindings(&mut tree.nodes, tree.backend, false); - } return ToolResult::error(error); } }, @@ -1478,7 +1519,7 @@ impl Tool for GetWindowStateTool { }; let result = tree_result.map(|tree| { ( - Some(tree), + tree, screenshot_opt, screenshot_err, observation_revision, @@ -1548,34 +1589,25 @@ impl Tool for GetWindowStateTool { // IAccessible and click must dispatch through MSAA. let bindings_transferred = !observation_only && (tr.complete || (!is_msaa && tr.read_complete())); - if bindings_transferred { - if is_msaa { - state.element_cache.update_msaa(pid, hwnd, &tr.nodes); + let snapshot_id = bindings_transferred.then(|| { + let kind = if is_msaa { + crate::uia::cache::SnapshotKind::Msaa } else { - state.element_cache.update(pid, hwnd, &tr.nodes); - } - } - crate::uia::settle_node_bindings( - &mut tr.nodes, - tr.backend, - bindings_transferred, - ); + crate::uia::cache::SnapshotKind::Uia + }; + state.element_cache.update_for_session( + pid, + hwnd, + &tr.nodes, + kind, + args.get("_session_id").and_then(Value::as_str), + ) + }); + tr.settle(bindings_transferred); structured["element_count"] = json!(count); structured["elements_complete"] = json!(tr.complete); structured["element_bindings_retained"] = json!(bindings_transferred); - // Surface 6: register a snapshot in the global token - // registry. Windows uses u64 HWND but the registry - // stores u32 — truncate (HWND fits in 32-bit on - // every supported edition; the upper 32 bits are - // zero in user-space). - let snapshot_id = bindings_transferred.then(|| { - cua_driver_core::element_token::global().register_snapshot( - pid as i32, - hwnd as u32, - count, - ) - }); if observation_revision.is_some() && !observation_only { state.watch_target(pid, hwnd); } @@ -1793,10 +1825,12 @@ impl Tool for GetWindowStateTool { if !observation_only { if let Some(ow) = orig_w { if w > 0 { - state.resize_registry.set_ratio(pid, ow as f64 / w as f64); + state + .resize_registry + .set_ratio(pid, hwnd, ow as f64 / w as f64); } } else { - state.resize_registry.clear_ratio(pid); + state.resize_registry.clear_ratio(pid, hwnd); } } // base64 is embedded only when no out_file was given (vision @@ -2623,6 +2657,10 @@ impl Tool for LaunchAppTool { // Legacy ShellExecuteExW path — unchanged behavior for plain // Win32 apps and for callers passing an explicit `path` / // `launch_path`. + let launch_token = match crate::standard_user_launch::app_launch_token() { + Ok(token) => token, + Err(message) => return ToolResult::error(message), + }; let urls_clone = urls.clone(); let target_for_shell = target_file_opt.clone(); let extra_for_shell = extra_joined.clone(); @@ -2666,39 +2704,64 @@ impl Tool for LaunchAppTool { }); let args_w = to_wide(&extra_for_shell); + use crate::standard_user_launch::AppLaunchToken; let pid = if direct_minimized_exe { let target = target_for_shell .as_deref() .expect("checked executable path"); - let mut command_line = to_wide(&if extra_for_shell.is_empty() { + let command_line_text = if extra_for_shell.is_empty() { format!(r#""{target}""#) } else { format!(r#""{target}" {extra_for_shell}"#) - }); - let startup = STARTUPINFOW { - cb: std::mem::size_of::<STARTUPINFOW>() as u32, - dwFlags: STARTF_USESHOWWINDOW, - wShowWindow: n_show_for_shell as u16, - ..Default::default() }; - let mut process = PROCESS_INFORMATION::default(); - unsafe { - CreateProcessW( - PCWSTR(file_w.as_ptr()), - PWSTR(command_line.as_mut_ptr()), - None, - None, - false, - PROCESS_CREATION_FLAGS(0), - None, - PCWSTR::null(), - &startup, - &mut process, - )?; - let _ = CloseHandle(process.hThread); - let _ = CloseHandle(process.hProcess); + if let AppLaunchToken::StandardUser(token) = &launch_token { + crate::standard_user_launch::create_process( + token, + target, + &command_line_text, + n_show_for_shell, + ) + .map_err(anyhow::Error::msg)? + } else { + let mut command_line = to_wide(&command_line_text); + let startup = STARTUPINFOW { + cb: std::mem::size_of::<STARTUPINFOW>() as u32, + dwFlags: STARTF_USESHOWWINDOW, + wShowWindow: n_show_for_shell as u16, + ..Default::default() + }; + let mut process = PROCESS_INFORMATION::default(); + unsafe { + CreateProcessW( + PCWSTR(file_w.as_ptr()), + PWSTR(command_line.as_mut_ptr()), + None, + None, + false, + PROCESS_CREATION_FLAGS(0), + None, + PCWSTR::null(), + &startup, + &mut process, + )?; + let _ = CloseHandle(process.hThread); + let _ = CloseHandle(process.hProcess); + } + process.dwProcessId } - process.dwProcessId + } else if let AppLaunchToken::StandardUser(token) = &launch_token { + let file = target_for_shell + .as_deref() + .or_else(|| urls_clone.first().map(String::as_str)) + .unwrap_or(""); + crate::standard_user_launch::shell_execute( + token, + file, + &extra_for_shell, + n_show_for_shell, + std::time::Duration::from_secs(10), + ) + .map_err(anyhow::Error::msg)? } else { let mut info = SHELLEXECUTEINFOW { cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32, @@ -2735,6 +2798,16 @@ impl Tool for LaunchAppTool { // `lpFile`, so only their remaining URLs belong here. let first_unopened_url = first_unopened_shell_url_index(target_for_shell.is_some()); for url in &urls_clone[first_unopened_url.min(urls_clone.len())..] { + if let AppLaunchToken::StandardUser(token) = &launch_token { + let _ = crate::standard_user_launch::shell_execute( + token, + url, + "", + n_show_for_shell, + std::time::Duration::from_secs(10), + ); + continue; + } let file = to_wide(url); let mut url_info = SHELLEXECUTEINFOW { cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32, @@ -3400,6 +3473,7 @@ impl Tool for ClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let elem_idx = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -3443,6 +3517,7 @@ impl Tool for ClickTool { ) .with_structured(json!({ "code": "background_unavailable", + "sent": false, "effect": "refused", "escalation": { "recommended": "foreground", @@ -3506,10 +3581,8 @@ impl Tool for ClickTool { // `accDoDefaultAction` here because LO's MSAA impl applies // the change asynchronously and returns S_OK either way, // so the visible behavior is the same as a center click. - if let Some((SnapshotKind::Msaa, role)) = self - .state - .element_cache - .get_element_kind_and_role(pid, hwnd, idx) + if let Some((SnapshotKind::Msaa, role)) = + element_cache.get_element_kind_and_role(pid, hwnd, idx) { const ROLE_BUTTONDROPDOWN: i32 = 0x38; const ROLE_BUTTONMENU: i32 = 0x39; @@ -3526,7 +3599,7 @@ impl Tool for ClickTool { ) ); let (tx, ty) = if want_expand && is_dropdown_role { - match self.state.element_cache.get_element_rect(pid, hwnd, idx) { + match element_cache.get_element_rect(pid, hwnd, idx) { Some((_l, t, r, b)) => { // Right-edge of the SplitButton — the dropdown // arrow half. -4 puts the click safely inside @@ -3549,7 +3622,7 @@ impl Tool for ClickTool { Use action:\"invoke\" or omit the action arg." )); } else { - match self.state.element_cache.get_element_center(pid, hwnd, idx) { + match element_cache.get_element_center(pid, hwnd, idx) { Some(v) => v, None => { return ToolResult::error(format!( @@ -3569,10 +3642,6 @@ impl Tool for ClickTool { }, ); let btn_fg = button.clone(); - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let mods_owned = modifiers.clone(); let activate = delivery == DeliveryMode::Foreground; let send_result = tokio::task::spawn_blocking(move || { @@ -3588,9 +3657,6 @@ impl Tool for ClickTool { } }) .await; - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } let half = if want_expand { "dropdown" } else { "press" }; return match send_result { Ok(Ok(())) => ToolResult::text(format!( @@ -3605,7 +3671,7 @@ impl Tool for ClickTool { } // UIA path: get cached center (no COM call needed — captured at walk time). - let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) { + let (cx, cy) = match element_cache.get_element_center(pid, hwnd, idx) { Some(v) => v, None => { return ToolResult::error(format!( @@ -3638,7 +3704,6 @@ impl Tool for ClickTool { // WPF/WinUI menus and tree nodes whose visual click target is // transient or scroll-adjusted. if action_req.as_deref() == Some("expand") { - let state = self.state.clone(); let expand = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { use windows::core::Interface; use windows::Win32::UI::Accessibility::{ @@ -3646,8 +3711,7 @@ impl Tool for ClickTool { UIA_ExpandCollapsePatternId, }; - let retained = state - .element_cache + let retained = element_cache .get_element_retained(pid, hwnd, idx) .ok_or_else(|| { anyhow::anyhow!("element [{idx}] is not in the UIA cache") @@ -3697,8 +3761,7 @@ impl Tool for ClickTool { // stale rectangle is still inside the outer HWND. Ask the item // to scroll itself into view before trusting that rectangle. let (cx, cy) = if !modifiers.is_empty() { - self.state - .element_cache + element_cache .get_element_retained(pid, hwnd, idx) .and_then(|retained| { retained.is_uia().then(|| unsafe { @@ -3714,7 +3777,7 @@ impl Tool for ClickTool { (cx, cy) }; let (cx, cy) = match resolve_onscreen_point_with_scroll( - &self.state.element_cache, + &element_cache, pid, hwnd, idx, @@ -3727,10 +3790,6 @@ impl Tool for ClickTool { }; let btn_fg = btn.clone(); let mods_owned = modifiers.clone(); - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let send_result = tokio::task::spawn_blocking(move || { let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect(); crate::input::send_click_synthesized_active_mods( @@ -3738,9 +3797,6 @@ impl Tool for ClickTool { ) }) .await; - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } return match send_result { Ok(Ok(())) => ToolResult::text(format!( "✅ Performed SendInput click on [{idx}] at screen ({cx},{cy}) (delivery_mode:foreground)." @@ -3764,7 +3820,8 @@ impl Tool for ClickTool { && (count > 1 || btn == "right" || btn == "middle") { if let Some(r) = - winui3_background_gesture(&self.state, pid, hwnd, Some(idx), count, &btn).await + winui3_background_gesture(&element_cache, pid, hwnd, Some(idx), count, &btn) + .await { return r; } @@ -3807,7 +3864,6 @@ impl Tool for ClickTool { // different patterns; keep PostMessage for now) // - count > 1 (double-click semantics aren't an Invoke // concept — PostMessage produces the actual WM_LBUTTONDBLCLK) - let state_clone = self.state.clone(); let use_uia_invoke = (btn == "left" || btn == "middle") && count == 1; let result = tokio::task::spawn_blocking(move || -> Result<String, ClickDispatchError> { // Direct Chromium UIA Invoke can return S_OK without firing a @@ -3818,7 +3874,7 @@ impl Tool for ClickTool { && crate::input::is_chromium_target_window(hwnd) { let (cx, cy) = resolve_onscreen_point_with_scroll( - &state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking", + &element_cache, pid, hwnd, idx, cx, cy, "clicking", ) .map_err(ClickDispatchError::Refusal)?; return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) @@ -3838,7 +3894,7 @@ impl Tool for ClickTool { // is mid-flight (use-after-free → daemon crash). The guard // lives to the end of this `if let` block, past every UIA // pattern dispatch below; its Release fires when it drops. - if let Some(element_guard) = state_clone.element_cache.get_element_retained(pid, hwnd, idx) { + if let Some(element_guard) = element_cache.get_element_retained(pid, hwnd, idx) { let ptr = element_guard.as_ptr(); use windows::Win32::UI::Accessibility::{ IUIAutomationElement, IUIAutomationInvokePattern, @@ -3929,7 +3985,7 @@ impl Tool for ClickTool { ) { let (cx, cy) = resolve_onscreen_point_with_scroll( - &state_clone.element_cache, + &element_cache, pid, hwnd, idx, @@ -3999,7 +4055,7 @@ impl Tool for ClickTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, hwnd) { px *= ratio; py *= ratio; } @@ -4049,10 +4105,6 @@ impl Tool for ClickTool { // background-safe and would deliver the click without the // foreground swap the caller asked for). if delivery == DeliveryMode::Foreground { - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let mods_owned = modifiers.clone(); let send_result = tokio::task::spawn_blocking(move || { let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect(); @@ -4061,9 +4113,6 @@ impl Tool for ClickTool { ) }) .await; - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } return match send_result { Ok(Ok(())) => { let click_word = match count { @@ -4089,7 +4138,7 @@ impl Tool for ClickTool { && (count > 1 || btn == "right" || btn == "middle") { if let Some(r) = - winui3_background_gesture(&self.state, pid, hwnd, None, count, &btn).await + winui3_background_gesture(&element_cache, pid, hwnd, None, count, &btn).await { return r; } @@ -4154,8 +4203,8 @@ impl Tool for ClickTool { ) }) .await - .unwrap_or(false); - if invoked { + .unwrap_or(crate::uia::windows_enum::PointInvokeOutcome::Timeout); + if invoked == crate::uia::windows_enum::PointInvokeOutcome::Invoked { return ToolResult::text(format!( "✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}." )) @@ -4163,6 +4212,14 @@ impl Tool for ClickTool { json!({ "path": "ax", "verified": false, "effect": "unverifiable" }), ); } + if !matches!( + invoked, + crate::uia::windows_enum::PointInvokeOutcome::Miss + | crate::uia::windows_enum::PointInvokeOutcome::Unavailable + ) { + return ToolResult::error("UIA point action is busy or its effect is unconfirmed; no second click was sent; observe before retrying") + .with_structured(json!({"code":"uia_input_unconfirmed","sent":true})); + } } // UIA did not land. Known dropped surfaces other than direct @@ -4313,7 +4370,7 @@ pub struct TypeTextTool { } fn wait_for_cached_element_keyboard_focus( - state: &ToolState, + element_cache: &crate::uia::cache::BoundElementCache, pid: u32, hwnd: u64, element_index: usize, @@ -4321,11 +4378,7 @@ fn wait_for_cached_element_keyboard_focus( ) -> bool { let deadline = std::time::Instant::now() + timeout; loop { - if state - .element_cache - .element_has_keyboard_focus(pid, hwnd, element_index) - == Some(true) - { + if element_cache.element_has_keyboard_focus(pid, hwnd, element_index) == Some(true) { return true; } if std::time::Instant::now() >= deadline { @@ -4341,7 +4394,7 @@ fn wait_for_cached_element_keyboard_focus( /// bounded fallback. Both routes require `CurrentHasKeyboardFocus` read-back /// before any keyboard input is allowed to leave the driver. fn focus_cached_element_for_foreground( - state: &ToolState, + element_cache: &crate::uia::cache::BoundElementCache, pid: u32, hwnd: u64, element_index: usize, @@ -4353,10 +4406,10 @@ fn focus_cached_element_for_foreground( // which would make the enclosing verify-before-SendInput transaction // fail closed. The foreground route has already activated the exact HWND, // so focus the cached element directly and verify it below. - let set_focus = state.element_cache.focus_element(pid, hwnd, element_index); + let set_focus = element_cache.focus_element(pid, hwnd, element_index); if set_focus.is_ok() && wait_for_cached_element_keyboard_focus( - state, + &element_cache, pid, hwnd, element_index, @@ -4369,7 +4422,7 @@ fn focus_cached_element_for_foreground( if let Some((x, y)) = click_point { crate::input::send_click_synthesized_active_mods(hwnd, x, y, 1, "left", &[])?; if wait_for_cached_element_keyboard_focus( - state, + &element_cache, pid, hwnd, element_index, @@ -4510,6 +4563,7 @@ impl Tool for TypeTextTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let elem_idx: Option<u64> = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index as u64) @@ -4600,11 +4654,7 @@ impl Tool for TypeTextTool { } }; if let Some(idx) = elem_idx { - if let Some((cx, cy)) = - self.state - .element_cache - .get_element_center(pid, hwnd, idx as usize) - { + if let Some((cx, cy)) = element_cache.get_element_center(pid, hwnd, idx as usize) { pin_overlay_above(&cursor_key, hwnd); overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; self.state @@ -4655,21 +4705,16 @@ impl Tool for TypeTextTool { // activation/focus/input transaction. The focus itself happens // only after exact top-level foreground is confirmed. let focus_target = if let Some(idx) = elem_idx { - let (cx, cy) = - match self - .state - .element_cache - .get_element_center(pid, hwnd, idx as usize) - { - Some(center) => center, - None => { - return ToolResult::error(format!( + let (cx, cy) = match element_cache.get_element_center(pid, hwnd, idx as usize) { + Some(center) => center, + None => { + return ToolResult::error(format!( "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first." )) - } - }; + } + }; let (cx, cy) = match resolve_onscreen_point_with_scroll( - &self.state.element_cache, + &element_cache, pid, hwnd, idx as usize, @@ -4685,11 +4730,16 @@ impl Tool for TypeTextTool { None }; let text_fg = text.clone(); - let state = self.state.clone(); let r = tokio::task::spawn_blocking(move || { crate::input::send_text_synthesized_after_focus(hwnd, &text_fg, || { if let Some((idx, point)) = focus_target { - focus_cached_element_for_foreground(&state, pid, hwnd, idx, Some(point))?; + focus_cached_element_for_foreground( + &element_cache, + pid, + hwnd, + idx, + Some(point), + )?; } Ok(()) }) @@ -4722,11 +4772,7 @@ impl Tool for TypeTextTool { // Only when an element_index is supplied (we have its cached center); // the focused-element path has no resolvable position to point at. if let Some(idx) = elem_idx { - if let Some((cx, cy)) = - self.state - .element_cache - .get_element_center(pid, hwnd, idx as usize) - { + if let Some((cx, cy)) = element_cache.get_element_center(pid, hwnd, idx as usize) { overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; crate::overlay::send_command( cursor_key.clone(), @@ -4754,14 +4800,14 @@ impl Tool for TypeTextTool { // (most legacy Win32 EDITs consume WM_CHAR fine without focus steal). if let Some(idx) = elem_idx { let idx = idx as usize; - let state = self.state.clone(); let text_for_uia = text.clone(); + let cache_for_write = element_cache.clone(); let set_result = tokio::task::spawn_blocking(move || { - // Retain the element under the cache lock so a concurrent + // Retain the element from its bound snapshot so a concurrent // get_window_state snapshot-replace on the same (pid, hwnd) // can't Release it to zero while this SetValue is in flight. // The guard is held for the whole closure. - let element_guard = state.element_cache.get_element_retained(pid, hwnd, idx)?; + let element_guard = cache_for_write.get_element_retained(pid, hwnd, idx)?; let ptr = element_guard.as_ptr(); use windows::core::{Interface, BSTR}; use windows::Win32::UI::Accessibility::{ @@ -4803,9 +4849,8 @@ impl Tool for TypeTextTool { // (works regardless of which window is foreground), proving the // a11y write actually took rather than trusting SetValue's // return alone. - let state_rb = self.state.clone(); let verify = tokio::task::spawn_blocking(move || { - let value = read_cached_element_value(&state_rb, pid, hwnd, idx); + let value = read_cached_element_value(&element_cache, pid, hwnd, idx); classify_value_write_readback(value.as_deref(), &before, &expected) }) .await @@ -4899,14 +4944,13 @@ impl Tool for TypeTextTool { let text_for_post = text.clone(); let verify_pid = pid; let verify_idx = elem_idx.map(|i| i as usize); - let state_rb = self.state.clone(); let result = tokio::task::spawn_blocking(move || { // Prefer a focus-independent read of the *specific* cached element // when we have its index; only fall back to the (flaky, focus- // dependent) system focused element when typing into "whatever is // focused" with no element_index. let read = |idx: Option<usize>| match idx { - Some(i) => read_cached_element_value(&state_rb, verify_pid, hwnd, i), + Some(i) => read_cached_element_value(&element_cache, verify_pid, hwnd, i), None => read_focused_value_uia(verify_pid), }; let before = read(verify_idx); @@ -5093,13 +5137,18 @@ fn classify_value_write_readback( /// `None` if the index isn't cached or the element exposes no readable text /// pattern. Mirrors the cache-retain + `mem::forget` discipline of the /// ValuePattern.SetValue path so the cached COM ref isn't released. -fn read_cached_element_value(state: &ToolState, pid: u32, hwnd: u64, idx: usize) -> Option<String> { +fn read_cached_element_value( + element_cache: &crate::uia::cache::BoundElementCache, + pid: u32, + hwnd: u64, + idx: usize, +) -> Option<String> { use windows::core::Interface; use windows::Win32::UI::Accessibility::{ IUIAutomationElement, IUIAutomationTextPattern, IUIAutomationValuePattern, UIA_TextPatternId, UIA_ValuePatternId, }; - let guard = state.element_cache.get_element_retained(pid, hwnd, idx)?; + let guard = element_cache.get_element_retained(pid, hwnd, idx)?; let ptr = guard.as_ptr(); let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) }; let val = unsafe { @@ -5269,6 +5318,7 @@ impl Tool for PressKeyTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let elem_idx: Option<u64> = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index as u64) @@ -5379,17 +5429,13 @@ impl Tool for PressKeyTool { None }; if let Some(idx) = elem_idx.filter(|_| background_webview_focus) { - let Some((cx, cy)) = - self.state - .element_cache - .get_element_center(pid, hwnd, idx as usize) - else { + let Some((cx, cy)) = element_cache.get_element_center(pid, hwnd, idx as usize) else { return ToolResult::error(format!( "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first." )); }; let (cx, cy) = match resolve_onscreen_point_with_scroll( - &self.state.element_cache, + &element_cache, pid, hwnd, idx as usize, @@ -5401,7 +5447,7 @@ impl Tool for PressKeyTool { Err(result) => return result, }; let (mut px, mut py) = screen_to_bitmap(hwnd, cx, cy); - if let Some(ratio) = self.state.resize_registry.ratio(pid) { + if let Some(ratio) = self.state.resize_registry.ratio(pid, hwnd) { px = (px as f64 / ratio).round() as i32; py = (py as f64 / ratio).round() as i32; } @@ -5426,10 +5472,10 @@ impl Tool for PressKeyTool { return error; } } else if let Some(idx) = elem_idx.filter(|_| delivery != DeliveryMode::Foreground) { - let state = self.state.clone(); + let cache_for_focus = element_cache.clone(); let focused = tokio::task::spawn_blocking(move || { crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || { - state.element_cache.focus_element(pid, hwnd, idx as usize) + cache_for_focus.focus_element(pid, hwnd, idx as usize) }) }) .await; @@ -5452,18 +5498,14 @@ impl Tool for PressKeyTool { // goes via the plain background post path below. if !px_focus && delivery == DeliveryMode::Foreground { let focus_target = elem_idx.map(|idx| { - let point = self - .state - .element_cache - .get_element_center(pid, hwnd, idx as usize); + let point = element_cache.get_element_center(pid, hwnd, idx as usize); (idx as usize, point) }); - let state = self.state.clone(); let send_result = tokio::task::spawn_blocking(move || { let m: Vec<&str> = mods.iter().map(String::as_str).collect(); crate::input::send_key_synthesized_after_focus(hwnd, &key, &m, || { if let Some((idx, point)) = focus_target { - focus_cached_element_for_foreground(&state, pid, hwnd, idx, point)?; + focus_cached_element_for_foreground(&element_cache, pid, hwnd, idx, point)?; } Ok(()) }) @@ -5683,6 +5725,7 @@ impl Tool for HotkeyTool { Ok(resolved) => resolved, Err(error) => return error, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let elem_idx = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -5789,10 +5832,10 @@ impl Tool for HotkeyTool { None }; if let Some(idx) = elem_idx.filter(|_| delivery == DeliveryMode::Background) { - let state = self.state.clone(); + let cache_for_focus = element_cache.clone(); let focused = tokio::task::spawn_blocking(move || { crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || { - state.element_cache.focus_element(pid, hwnd, idx) + cache_for_focus.focus_element(pid, hwnd, idx) }) }) .await; @@ -5887,16 +5930,15 @@ impl Tool for HotkeyTool { // chord even though the renderer control is focused. let use_send_input = delivery == DeliveryMode::Foreground; let focus_target = elem_idx.map(|idx| { - let point = self.state.element_cache.get_element_center(pid, hwnd, idx); + let point = element_cache.get_element_center(pid, hwnd, idx); (idx, point) }); - let state = self.state.clone(); let result = tokio::task::spawn_blocking(move || { let m: Vec<&str> = mods.iter().map(String::as_str).collect(); if use_send_input { crate::input::send_key_synthesized_after_focus(hwnd, &key, &m, || { if let Some((idx, point)) = focus_target { - focus_cached_element_for_foreground(&state, pid, hwnd, idx, point)?; + focus_cached_element_for_foreground(&element_cache, pid, hwnd, idx, point)?; } Ok(()) }) @@ -5924,7 +5966,7 @@ impl Tool for HotkeyTool { // ── perform_secondary_action ───────────────────────────────────────────────── pub struct PerformSecondaryActionTool { - state: Arc<ToolState>, + _state: Arc<ToolState>, } static PERFORM_SECONDARY_ACTION_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new(); @@ -5980,6 +6022,7 @@ impl Tool for PerformSecondaryActionTool { Ok(resolved) => resolved, Err(error) => return error, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let cua_driver_core::element_token::ResolvedElement::Element { window_id: Some(window_id), element_index, @@ -5990,20 +6033,15 @@ impl Tool for PerformSecondaryActionTool { .with_structured(json!({ "code": "invalid_element_target" })); }; let hwnd = window_id as u64; - let advertised = - match self - .state - .element_cache - .get_element_actions(pid, hwnd, element_index) - { - Some(actions) => actions, - None => { - return ToolResult::error( - "element_token no longer resolves in the current UIA snapshot", - ) - .with_structured(json!({ "code": "stale_element_token" })) - } - }; + let advertised = match element_cache.get_element_actions(pid, hwnd, element_index) { + Some(actions) => actions, + None => { + return ToolResult::error( + "element_token no longer resolves in the current UIA snapshot", + ) + .with_structured(json!({ "code": "stale_element_token" })) + } + }; let mut matches = advertised .iter() .filter(|action| action.as_str() == requested); @@ -6028,18 +6066,13 @@ impl Tool for PerformSecondaryActionTool { } use crate::uia::cache::SnapshotKind; - let (kind, _) = - match self - .state - .element_cache - .get_element_kind_and_role(pid, hwnd, element_index) - { - Some(kind) => kind, - None => { - return ToolResult::error("element is stale") - .with_structured(json!({ "code": "stale_element_token" })) - } - }; + let (kind, _) = match element_cache.get_element_kind_and_role(pid, hwnd, element_index) { + Some(kind) => kind, + None => { + return ToolResult::error("element is stale") + .with_structured(json!({ "code": "stale_element_token" })) + } + }; if kind == SnapshotKind::Msaa && action != "invoke" { return ToolResult::error(format!( "MSAA supports only its native default invoke action; '{action}' has no stable semantic dispatch" @@ -6070,7 +6103,6 @@ impl Tool for PerformSecondaryActionTool { .collect::<std::collections::HashSet<_>>(); let _no_activate = crate::input::NoActivateGuard::arm(windows::Win32::Foundation::HWND(hwnd as *mut _)); - let state = self.state.clone(); let action_for_dispatch = action.clone(); let mut dispatch = tokio::task::spawn_blocking(move || -> anyhow::Result<String> { use windows::core::{Interface, VARIANT}; @@ -6081,8 +6113,7 @@ impl Tool for PerformSecondaryActionTool { UIA_SelectionItemPatternId, UIA_TogglePatternId, }; - let retained = state - .element_cache + let retained = element_cache .get_element_retained(pid, hwnd, element_index) .ok_or_else(|| anyhow::anyhow!("element is stale"))?; match kind { @@ -6222,7 +6253,7 @@ mod perform_secondary_action_schema_tests { #[test] fn runtime_schema_stays_platform_owned() { let tool = PerformSecondaryActionTool { - state: ToolState::new(), + _state: ToolState::new(), }; let def = tool.def(); assert_eq!(def.name, "perform_secondary_action"); @@ -6239,7 +6270,7 @@ mod perform_secondary_action_schema_tests { // ── set_value ───────────────────────────────────────────────────────────────── pub struct SetValueTool { - state: Arc<ToolState>, + _state: Arc<ToolState>, } static SET_VALUE_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new(); @@ -6308,6 +6339,7 @@ impl Tool for SetValueTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let (hwnd, idx) = match resolved { cua_driver_core::element_token::ResolvedElement::Element { window_id: Some(wid), @@ -6348,7 +6380,7 @@ impl Tool for SetValueTool { // value, so a value write gets the same visual feedback as a click — // the viewer can see *where* the agent is acting. No-op when the // overlay is disabled or the element has no cached center. - if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx) { + if let Some((cx, cy)) = element_cache.get_element_center(pid, hwnd, idx) { pin_overlay_above(&cursor_key, hwnd); overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; crate::overlay::send_command( @@ -6360,14 +6392,12 @@ impl Tool for SetValueTool { ); } - let state = self.state.clone(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> { - // Retain the element under the cache lock so a concurrent + // Retain the element from its bound snapshot so a concurrent // get_window_state snapshot-replace on the same (pid, hwnd) can't // Release it to zero while this Value/RangeValue SetValue is in // flight. The guard is held for the whole closure. - let element_guard = state - .element_cache + let element_guard = element_cache .get_element_retained(pid, hwnd, idx) .ok_or_else(|| anyhow::anyhow!("Element {idx} not in cache."))?; let ptr = element_guard.as_ptr(); @@ -6377,6 +6407,20 @@ impl Tool for SetValueTool { }; let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) }; + let combo = unsafe { elem.CurrentNativeWindowHandle() } + .ok() + .and_then(|native| crate::win32::combo::combo_for_window(native.0 as isize)); + if let Some(combo) = combo { + let result = crate::win32::combo::set_value(combo, &value).and_then(|write| { + if crate::win32::combo::write_verified(&write, &value) { + Ok("Win32Combo".to_owned()) + } else { + anyhow::bail!("write_not_applied: combo selection readback did not match") + } + }); + std::mem::forget(elem); + return result; + } // Try ValuePattern first (text inputs, editable combos, etc). // The SetValue is shielded by the EnableWindow bypass: a // Chromium/Electron (or XAML) SetValue handler self-foregrounds via @@ -6439,7 +6483,7 @@ impl Tool for SetValueTool { // ── scroll ──────────────────────────────────────────────────────────────────── pub struct ScrollTool { - state: Arc<ToolState>, + _state: Arc<ToolState>, } static SCROLL_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new(); @@ -6568,6 +6612,7 @@ impl Tool for ScrollTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let elem_idx: Option<u64> = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index as u64) @@ -6632,11 +6677,9 @@ impl Tool for ScrollTool { } else { None }; - let state = self.state.clone(); let direction_for_uia = direction.clone(); let uia_result = tokio::task::spawn_blocking(move || { - let retained = state - .element_cache + let retained = element_cache .get_element_retained(pid, hwnd, idx as usize) .ok_or_else(|| anyhow::anyhow!("element [{idx}] is not in the UIA cache"))?; if !retained.is_uia() { @@ -6771,7 +6814,7 @@ impl Tool for ScrollTool { windows::Win32::Foundation::HWND(hwnd as *mut _), "scroll delivery", || Ok(()), - || crate::input::send_wheel_synthesized(cx, cy, ticks, horizontal), + |_, _| crate::input::send_wheel_synthesized(cx, cy, ticks, horizontal), ) }) .await; @@ -6918,13 +6961,13 @@ async fn chromium_click_short_circuit( /// `spawn_blocking`. Returns `Some(Ok)` on success, `Some(Err)` if Invoke /// failed, `None` if the element isn't cached or has no InvokePattern. fn winui3_uia_multi_invoke( - state: &Arc<ToolState>, + element_cache: &crate::uia::cache::BoundElementCache, pid: u32, hwnd: u64, idx: usize, count: usize, ) -> Option<anyhow::Result<()>> { - let guard = state.element_cache.get_element_retained(pid, hwnd, idx)?; + let guard = element_cache.get_element_retained(pid, hwnd, idx)?; let ptr = guard.as_ptr(); use windows::core::Interface; use windows::Win32::UI::Accessibility::{ @@ -6979,7 +7022,7 @@ fn winui3_uia_multi_invoke( /// Returns `None` for non-WinUI3 targets (caller falls through to its normal /// routing). async fn winui3_background_gesture( - state: &Arc<ToolState>, + element_cache: &crate::uia::cache::BoundElementCache, pid: u32, hwnd: u64, idx: Option<usize>, @@ -6995,9 +7038,9 @@ async fn winui3_background_gesture( } if count >= 2 && button == "left" { if let Some(idx) = idx { - let st = state.clone(); + let element_cache = element_cache.clone(); let uia = tokio::task::spawn_blocking(move || { - winui3_uia_multi_invoke(&st, pid, hwnd, idx, count) + winui3_uia_multi_invoke(&element_cache, pid, hwnd, idx, count) }) .await .ok() @@ -7089,6 +7132,7 @@ impl Tool for DoubleClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let elem_idx = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -7147,7 +7191,7 @@ impl Tool for DoubleClickTool { }; if let Some(idx) = elem_idx { - let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) { + let (cx, cy) = match element_cache.get_element_center(pid, hwnd, idx) { Some(v) => v, None => { return ToolResult::error(format!( @@ -7156,7 +7200,7 @@ impl Tool for DoubleClickTool { } }; let (cx, cy) = match resolve_onscreen_point_with_scroll( - &self.state.element_cache, + &element_cache, pid, hwnd, idx, @@ -7186,7 +7230,7 @@ impl Tool for DoubleClickTool { // click isn't expressible → structured error. if delivery == DeliveryMode::Background { if let Some(r) = - winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 2, "left").await + winui3_background_gesture(&element_cache, pid, hwnd, Some(idx), 2, "left").await { return r; } @@ -7217,17 +7261,10 @@ impl Tool for DoubleClickTool { } // delivery_mode:"foreground" — route through SendInput at the cached coords. if delivery == DeliveryMode::Foreground { - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let send_result = tokio::task::spawn_blocking(move || { crate::input::send_click_synthesized_active_mods(hwnd, cx, cy, 2, "left", &[]) }) .await; - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } return match send_result { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent double-click via SendInput on [{idx}] at screen ({cx},{cy}) (delivery_mode:foreground)." @@ -7274,7 +7311,7 @@ impl Tool for DoubleClickTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, hwnd) { px *= ratio; py *= ratio; } @@ -7294,7 +7331,7 @@ impl Tool for DoubleClickTool { // error (caller retries with delivery_mode:foreground or uses element_index). if delivery == DeliveryMode::Background { if let Some(r) = - winui3_background_gesture(&self.state, pid, hwnd, None, 2, "left").await + winui3_background_gesture(&element_cache, pid, hwnd, None, 2, "left").await { return r; } @@ -7322,10 +7359,6 @@ impl Tool for DoubleClickTool { } // delivery_mode:"foreground" — SendInput at screen coords with FG swap. if delivery == DeliveryMode::Foreground { - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let send_result = tokio::task::spawn_blocking(move || { crate::input::send_click_synthesized_active_mods( hwnd, @@ -7337,9 +7370,6 @@ impl Tool for DoubleClickTool { ) }) .await; - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } return match send_result { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent double-click via SendInput to pid {pid} at screen ({sx_i},{sy_i}) (delivery_mode:foreground)." @@ -7447,6 +7477,7 @@ impl Tool for RightClickTool { Ok(r) => r, Err(e) => return e, }; + let element_cache = crate::uia::cache::BoundElementCache::from_resolved(&resolved); let elem_idx = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => { Some(*element_index) @@ -7505,7 +7536,7 @@ impl Tool for RightClickTool { }; if let Some(idx) = elem_idx { - let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) { + let (cx, cy) = match element_cache.get_element_center(pid, hwnd, idx) { Some(v) => v, None => { return ToolResult::error(format!( @@ -7514,7 +7545,7 @@ impl Tool for RightClickTool { } }; let (cx, cy) = match resolve_onscreen_point_with_scroll( - &self.state.element_cache, + &element_cache, pid, hwnd, idx, @@ -7541,7 +7572,8 @@ impl Tool for RightClickTool { // Return the structured error (retry with delivery_mode:foreground). if delivery == DeliveryMode::Background { if let Some(r) = - winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 1, "right").await + winui3_background_gesture(&element_cache, pid, hwnd, Some(idx), 1, "right") + .await { return r; } @@ -7569,17 +7601,10 @@ impl Tool for RightClickTool { }; } if delivery == DeliveryMode::Foreground { - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let send_result = tokio::task::spawn_blocking(move || { crate::input::send_click_synthesized_active_mods(hwnd, cx, cy, 1, "right", &[]) }) .await; - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } return match send_result { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent right-click via SendInput on [{idx}] at screen ({cx},{cy}) (delivery_mode:foreground)." @@ -7625,7 +7650,7 @@ impl Tool for RightClickTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, hwnd) { px *= ratio; py *= ratio; } @@ -7644,7 +7669,7 @@ impl Tool for RightClickTool { // ignored, pen-barrel injector steals foreground) → structured error. if delivery == DeliveryMode::Background { if let Some(r) = - winui3_background_gesture(&self.state, pid, hwnd, None, 1, "right").await + winui3_background_gesture(&element_cache, pid, hwnd, None, 1, "right").await { return r; } @@ -7671,10 +7696,6 @@ impl Tool for RightClickTool { }; } if delivery == DeliveryMode::Foreground { - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let send_result = tokio::task::spawn_blocking(move || { crate::input::send_click_synthesized_active_mods( hwnd, @@ -7686,9 +7707,6 @@ impl Tool for RightClickTool { ) }) .await; - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } return match send_result { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent right-click via SendInput to pid {pid} at screen ({sx_i},{sy_i}) (delivery_mode:foreground)." @@ -7850,6 +7868,24 @@ impl Tool for DragTool { }; let hwnd_opt = args.opt_u64("window_id"); + let hwnd = match hwnd_opt { + Some(h) => h, + None => { + let windows = + tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))) + .await + .unwrap_or_default(); + match windows.first() { + Some(w) => w.hwnd, + None => { + return ToolResult::error(format!( + "No windows found for pid {pid}. Provide window_id." + )) + } + } + } + }; + let duration_ms = args.u64_or("duration_ms", 500); let steps = args.u64_or("steps", 20) as usize; let button = args.str_or("button", "left"); @@ -7871,31 +7907,13 @@ impl Tool for DragTool { )) } } - } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + } else if let Some(ratio) = self.state.resize_registry.ratio(pid, hwnd) { from_x *= ratio; from_y *= ratio; to_x *= ratio; to_y *= ratio; } - let hwnd = match hwnd_opt { - Some(h) => h, - None => { - let windows = - tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))) - .await - .unwrap_or_default(); - match windows.first() { - Some(w) => w.hwnd, - None => { - return ToolResult::error(format!( - "No windows found for pid {pid}. Provide window_id." - )) - } - } - } - }; - // Compute screen-coord endpoints. Same correction as the click // tools — bitmap pixels are anchored to the DWM-frame top-left, // not the client area top-left (see `bitmap_to_screen` doc). @@ -7911,8 +7929,17 @@ impl Tool for DragTool { // (a WinUI3 Slider can also be set with the set_value tool, which uses // UIA RangeValuePattern and holds the contract). if delivery == DeliveryMode::Background { - if let Some(r) = - winui3_background_gesture(&self.state, pid, hwnd, None, 1, &button).await + if let Some(r) = winui3_background_gesture( + &crate::uia::cache::BoundElementCache::from_resolved( + &cua_driver_core::element_token::ResolvedElement::None, + ), + pid, + hwnd, + None, + 1, + &button, + ) + .await { return r; } @@ -7980,10 +8007,6 @@ impl Tool for DragTool { if delivery == DeliveryMode::Foreground { let btn_fg = button.clone(); pin_overlay_above(&cursor_key, hwnd); - let foreground_target = crate::win32::capture_foreground_target(hwnd); - let prev_fg_addr = unsafe { - windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize - }; let send_result = tokio::task::spawn_blocking(move || { crate::input::mouse::send_drag_synthesized( hwnd, @@ -8004,9 +8027,6 @@ impl Tool for DragTool { steps, ); let (send_result, ()) = tokio::join!(send_result, visual_drag); - if let Err(error) = restore_after_input(prev_fg_addr, foreground_target).await { - return ToolResult::error(error); - } let button_suffix = if button == "left" { String::new() } else { @@ -8859,11 +8879,7 @@ impl Tool for SetConfigTool { // legacy per-field shape; documented as intentional. description: "Write a setting into the persistent driver config. Values take \ effect immediately.\n\n\ - Two input shapes:\n\ - - **Swift-compatible** (preferred): `{\"key\": \"max_image_dimension\", \"value\": 1568}` \ - — single dotted-path leaf write.\n\ - - **Legacy per-field** (Rust-only): `{\"max_image_dimension\": 0}` \ - — bulk write of named fields.\n\n\ + Pass named configuration fields, for example max_image_dimension.\n\n\ Known keys:\n\ - `capture_mode` (string: `vision` | `ax` | `som`) — DEPRECATED and ignored; \ `get_window_state` always returns both the UIA tree and a screenshot. Still \ @@ -8876,9 +8892,9 @@ impl Tool for SetConfigTool { "key":{"type":"string","description":"Dotted snake_case path to a leaf config field (Swift-compatible shape). Pair with `value`."}, "value":{"description":"New value for `key`. JSON type depends on the key."}, "capture_mode":{"type":"string","enum":["ax","vision"],"description":"DEPRECATED and ignored — get_window_state always returns both the UIA tree and a screenshot. Still accepted/persisted for back-compat but has no effect. (\"som\"/\"screenshot\" still decode as deprecated aliases.)"}, - "max_image_dimension":{"type":"integer","description":"Legacy per-field shape."}, - "experimental_pip":{"type":"boolean","description":"Legacy per-field shape. Enables PiP preview (applies next restart)."}, - "experimental_pip_geometry":{"type":"string","description":"Legacy per-field shape. PiP window size + optional position."} + "max_image_dimension":{"type":"integer","description":"Configuration field."}, + "experimental_pip":{"type":"boolean","description":"Configuration field. Enables PiP preview (applies next restart)."}, + "experimental_pip_geometry":{"type":"string","description":"Configuration field. PiP window size + optional position."} },"additionalProperties":false}), read_only: false, destructive: false, idempotent: true, open_world: false, }) @@ -8951,7 +8967,7 @@ impl Tool for SetConfigTool { )), } } - // Legacy per-field shape. + // Configuration field. if let Some(mode) = args.get("capture_mode").and_then(|v| v.as_str()) { cfg.capture_mode = mode.to_owned(); if let Err(e) = @@ -9174,7 +9190,7 @@ impl Tool for ZoomTool { let ratio = self .state .resize_registry - .ratio(raw_pid as u32) + .ratio(raw_pid as u32, hwnd) .unwrap_or(1.0); let (nx1, ny1, nx2, ny2) = (x1 * ratio, y1 * ratio, x2 * ratio, y2 * ratio); @@ -10299,10 +10315,6 @@ pub fn build_registry_with_provider( }, )) }; - // Share the element cache with the recording-hook layer so it can - // resolve element_index → window-local screenshot coords for click.png. - crate::recording_hooks::set_element_cache(state.element_cache.clone()); - // Drop a session's owned cursor on `session_end` (explicit end_session, the // CLI `session end` verb, or the daemon idle-TTL sweep). The session id IS // the cursor key (caller-declared `session`), so this prunes the metadata @@ -10346,6 +10358,10 @@ pub fn build_registry_with_provider( cua_driver_core::observation_revision::revision_tokens().clear_runtime(&runtime_scope); }); } + cua_driver_core::desktop_keyboard::register( + &mut r, + Arc::new(crate::input::keyboard::DesktopKeyboard), + ); r.register(Box::new(ListAppsTool)); r.register(Box::new(ListWindowsTool)); r.register(Box::new(GetWindowStateTool { @@ -10412,19 +10428,19 @@ pub fn build_registry_with_provider( )); r.register(pid_window_guarded( SetValueTool { - state: state.clone(), + _state: state.clone(), }, &pid_window_candidates, )); r.register(pid_window_guarded( PerformSecondaryActionTool { - state: state.clone(), + _state: state.clone(), }, &pid_window_candidates, )); r.register(pid_window_guarded( ScrollTool { - state: state.clone(), + _state: state.clone(), }, &pid_window_candidates, )); diff --git a/packages/cua-driver/rust/crates/platform-windows/src/uia/cache.rs b/packages/cua-driver/rust/crates/platform-windows/src/uia/cache.rs index 5e58c1880bb..b815ddd7f68 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/uia/cache.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/uia/cache.rs @@ -1,36 +1,15 @@ -//! Accessibility element cache for Windows. -//! -//! Mirrors the macOS ElementCache design: per-(pid, window_id) snapshots of -//! retained COM pointers (either `IUIAutomationElement` for the UIA primary -//! path, or `IAccessible` for the MSAA fallback used on SAL/VCL targets). -//! -//! Memory contract: `UiaNode::element_ptr` is a raw COM vtable pointer with -//! an extra AddRef from `clone()+forget()` in the walker. `CachedSnapshot::Drop` -//! calls `Release()` via the kind-appropriate vtable to balance. -//! -//! The locked-HashMap plumbing lives in `cua_driver_core::element_cache` — see -//! `docs/dedup-audit.md` item #3. This module owns the Windows-specific -//! `CacheKey`, `CachedSnapshot`, and the `Drop` impl that fires COM `Release` -//! when an entry is replaced or removed. +//! Immutable UIA/MSAA snapshots owned by element-token bindings. The latest +//! target directory contains weak references, so invalidation releases native +//! pointers when the final in-flight action drops its bound snapshot. use super::UiaNode; use cua_driver_core::element_cache::ElementCacheCore; +use cua_driver_core::element_token::{ResolvedElement, SnapshotBinding}; +use std::sync::{Arc, Weak}; use windows::core::Interface; use windows::Win32::UI::Accessibility::{IAccessible, IUIAutomationElement}; -/// A cached element pointer borrowed out of the cache with an extra COM -/// `AddRef`, so it stays alive for the duration of a UIA/MSAA action even if a -/// concurrent `get_window_state` (→ [`ElementCache::update`]) replaces and -/// drops the snapshot it came from. Without this, the snapshot's `Drop` could -/// `Release` the element to zero while an in-flight click / SetValue was still -/// dereferencing the raw COM vtable pointer — a use-after-free. This is the -/// Windows analogue of the macOS AX-element crash fixed in #1796 -/// (`EXC_BREAKPOINT` in `AXUIElementCopyActionNames`); here the same shape -/// would fault inside `IUIAutomation*::Invoke` / `ValuePattern::SetValue`. -/// -/// The `AddRef` is taken **under the cache lock** (see -/// [`ElementCache::get_element_retained`]); the matching `Release` fires on -/// drop, via the `kind`-appropriate interface — mirroring `CachedSnapshot::drop`. +/// An extra COM reference for an action that can outlive its snapshot view. pub struct RetainedElement { ptr: usize, kind: SnapshotKind, @@ -140,7 +119,7 @@ impl Drop for CachedSnapshot { } pub struct ElementCache { - core: ElementCacheCore<CacheKey, CachedSnapshot>, + core: ElementCacheCore<CacheKey, Weak<CachedSnapshot>>, } impl ElementCache { @@ -152,18 +131,25 @@ impl ElementCache { /// Update the snapshot for (pid, hwnd) with the actionable elements /// from a UIA walk. Mirrors `update_msaa` for the MSAA path. - pub fn update(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) { - self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Uia); + pub fn update(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) -> u32 { + self.update_for_session(pid, hwnd, nodes, SnapshotKind::Uia, None) } /// Same as `update` but tags the snapshot as MSAA so Drop releases the /// pointers as `IAccessible` and the click tool routes through the /// MSAA dispatch path. - pub fn update_msaa(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) { - self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Msaa); + pub fn update_msaa(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) -> u32 { + self.update_for_session(pid, hwnd, nodes, SnapshotKind::Msaa, None) } - fn update_with_kind(&self, pid: u32, hwnd: u64, nodes: &[UiaNode], kind: SnapshotKind) { + pub fn update_for_session( + &self, + pid: u32, + hwnd: u64, + nodes: &[UiaNode], + kind: SnapshotKind, + session: Option<&str>, + ) -> u32 { let actionable: Vec<&UiaNode> = nodes.iter().filter(|n| n.element_index.is_some()).collect(); let elements: Vec<usize> = actionable.iter().map(|n| n.element_ptr).collect(); @@ -174,65 +160,115 @@ impl ElementCache { let rects: Vec<Option<(i32, i32, i32, i32)>> = actionable.iter().map(|n| n.rect).collect(); let msaa_roles: Vec<Option<i32>> = actionable.iter().map(|n| n.msaa_role).collect(); let actions = actionable.iter().map(|node| node.actions.clone()).collect(); - self.core.insert( - CacheKey { pid, hwnd }, - CachedSnapshot { - kind, - elements, - centers, - rects, - msaa_roles, - actions, - }, - ); + let snapshot = Arc::new(CachedSnapshot { + kind, + elements, + centers, + rects, + msaa_roles, + actions, + }); + self.publish_for_session(pid, hwnd, snapshot, session) } - /// Look up + COM-`AddRef` the element for `element_index` in (pid, hwnd), - /// returning a guard that `Release`s on drop. The `AddRef` happens **under - /// the cache lock**, so a concurrent [`update`](Self::update) (which - /// replaces the snapshot and `Release`s its pointers in - /// `CachedSnapshot::drop`) cannot free the element between the lookup and - /// the `AddRef`. Hold the returned guard for the entire UIA/MSAA action — - /// this is what makes element actions safe when two sessions drive the same - /// `(pid, hwnd)`. Returns `None` if the index isn't cached. + #[cfg(test)] + fn publish(&self, pid: u32, hwnd: u64, snapshot: Arc<CachedSnapshot>) -> u32 { + self.publish_for_session(pid, hwnd, snapshot, None) + } + + fn publish_for_session( + &self, + pid: u32, + hwnd: u64, + snapshot: Arc<CachedSnapshot>, + session: Option<&str>, + ) -> u32 { + let id = cua_driver_core::element_token::global().register_snapshot_payload_for_session( + pid as i32, + hwnd as u32, + 0..snapshot.elements.len(), + snapshot.clone(), + session, + ); + self.core + .insert(CacheKey { pid, hwnd }, Arc::downgrade(&snapshot)); + id + } + + pub fn clear_target(&self, pid: u32, hwnd: u64) { + cua_driver_core::element_token::global().clear_current_target(pid as i32, hwnd as u32); + self.core.remove(&CacheKey { pid, hwnd }); + } + + pub fn element_count(&self, pid: u32, hwnd: u64) -> usize { + self.core + .with_snapshot(&CacheKey { pid, hwnd }, |s| { + s.upgrade().map_or(0, |s| s.elements.len()) + }) + .unwrap_or(0) + } +} + +#[derive(Clone)] +pub struct BoundElementCache { + binding: Option<SnapshotBinding>, +} + +impl BoundElementCache { + pub fn from_resolved(resolved: &ResolvedElement) -> Self { + Self { + binding: resolved.binding(), + } + } + + pub fn from_binding(binding: SnapshotBinding) -> Self { + Self { + binding: Some(binding), + } + } + + fn with_snapshot<R>( + &self, + key: &CacheKey, + read: impl FnOnce(&CachedSnapshot) -> R, + ) -> Option<R> { + self.binding + .as_ref()? + .with_payload(key.pid as i32, key.hwnd as u32, read) + } + + /// Retain from the bound snapshot, including its original interface kind. pub fn get_element_retained( &self, pid: u32, hwnd: u64, element_index: usize, ) -> Option<RetainedElement> { - self.core - .with_snapshot(&CacheKey { pid, hwnd }, |s| { - let ptr = s.elements.get(element_index).copied()?; - if ptr != 0 { - // Safety: still inside `with_snapshot`'s lock — the same - // `Mutex` that `insert` (the snapshot replace) takes — so - // the snapshot and thus this COM object is alive right now. - // Bump the refcount with the kind-appropriate vtable, using - // the walker's clone()+forget() idiom: `dup` is the extra - // ref the guard owns; the borrowed `iface` is forgotten so - // it doesn't Release the cache's own ref. - unsafe { - match s.kind { - SnapshotKind::Uia => { - let iface: IUIAutomationElement = - IUIAutomationElement::from_raw(ptr as *mut _); - let dup = iface.clone(); // AddRef → +1 - std::mem::forget(iface); // don't Release cache's ref - std::mem::forget(dup); // guard owns the +1 - } - SnapshotKind::Msaa => { - let iface: IAccessible = IAccessible::from_raw(ptr as *mut _); - let dup = iface.clone(); - std::mem::forget(iface); - std::mem::forget(dup); - } + self.with_snapshot(&CacheKey { pid, hwnd }, |s| { + let ptr = s.elements.get(element_index).copied()?; + if ptr != 0 { + // The immutable binding owns the original COM reference. + unsafe { + match s.kind { + SnapshotKind::Uia => { + let iface: IUIAutomationElement = + IUIAutomationElement::from_raw(ptr as *mut _); + let dup = iface.clone(); // AddRef → +1 + std::mem::forget(iface); // don't Release cache's ref + std::mem::forget(dup); // guard owns the +1 + } + SnapshotKind::Msaa => { + let iface: IAccessible = IAccessible::from_raw(ptr as *mut _); + let dup = iface.clone(); + std::mem::forget(iface); + std::mem::forget(dup); } } } - Some(RetainedElement { ptr, kind: s.kind }) - }) - .flatten() + } + Some(RetainedElement { ptr, kind: s.kind }) + }) + .flatten() } pub fn get_element_center( @@ -241,11 +277,10 @@ impl ElementCache { hwnd: u64, element_index: usize, ) -> Option<(i32, i32)> { - self.core - .with_snapshot(&CacheKey { pid, hwnd }, |s| { - s.centers.get(element_index).copied() - }) - .flatten() + self.with_snapshot(&CacheKey { pid, hwnd }, |s| { + s.centers.get(element_index).copied() + }) + .flatten() } /// Focus a cached UIA element without activating its top-level window. @@ -294,11 +329,10 @@ impl ElementCache { hwnd: u64, element_index: usize, ) -> Option<(i32, i32, i32, i32)> { - self.core - .with_snapshot(&CacheKey { pid, hwnd }, |s| { - s.rects.get(element_index).copied().flatten() - }) - .flatten() + self.with_snapshot(&CacheKey { pid, hwnd }, |s| { + s.rects.get(element_index).copied().flatten() + }) + .flatten() } /// Kind + MSAA role for the element. `(Msaa, Some(0x38))` identifies a @@ -310,12 +344,11 @@ impl ElementCache { hwnd: u64, element_index: usize, ) -> Option<(SnapshotKind, Option<i32>)> { - self.core - .with_snapshot(&CacheKey { pid, hwnd }, |s| { - let role = s.msaa_roles.get(element_index).copied().flatten(); - Some((s.kind, role)) - }) - .flatten() + self.with_snapshot(&CacheKey { pid, hwnd }, |s| { + let role = s.msaa_roles.get(element_index).copied().flatten(); + Some((s.kind, role)) + }) + .flatten() } pub fn get_element_actions( @@ -324,21 +357,10 @@ impl ElementCache { hwnd: u64, element_index: usize, ) -> Option<Vec<String>> { - self.core - .with_snapshot(&CacheKey { pid, hwnd }, |snapshot| { - snapshot.actions.get(element_index).cloned() - }) - .flatten() - } - - pub fn clear_target(&self, pid: u32, hwnd: u64) { - self.core.remove(&CacheKey { pid, hwnd }); - } - - pub fn element_count(&self, pid: u32, hwnd: u64) -> usize { - self.core - .with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.len()) - .unwrap_or(0) + self.with_snapshot(&CacheKey { pid, hwnd }, |snapshot| { + snapshot.actions.get(element_index).cloned() + }) + .flatten() } } diff --git a/packages/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs b/packages/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs index 5981d79967f..94703b088b5 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/uia/cache_uaf_repro.rs @@ -35,9 +35,9 @@ //! releases → B dereferences) with channels instead of relying on luck, so //! BEFORE deterministically trips and AFTER deterministically survives. -use super::{CacheKey, CachedSnapshot, ElementCache, SnapshotKind}; +use super::{BoundElementCache, CacheKey, CachedSnapshot, ElementCache, SnapshotKind}; use std::ffi::c_void; -use std::sync::atomic::{AtomicIsize, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicIsize, AtomicU32, AtomicUsize, Ordering}; use std::sync::mpsc; use std::sync::Arc; use std::thread; @@ -139,7 +139,9 @@ fn snapshot_with(ptrs: Vec<usize>) -> CachedSnapshot { fn old_get_element_ptr(cache: &ElementCache, pid: u32, hwnd: u64, idx: usize) -> Option<usize> { cache .core - .with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.get(idx).copied()) + .with_snapshot(&CacheKey { pid, hwnd }, |s| { + s.upgrade()?.elements.get(idx).copied() + }) .flatten() } @@ -153,7 +155,7 @@ unsafe fn touch_vtable(ptr: usize) { drop(dup); // Release — reads the vtable again } -const PID: u32 = 4242; +static NEXT_PID: AtomicU32 = AtomicU32::new(721_520); const HWND: u64 = 0x1234; /// Forced interleave: B looks up the element, THEN A replaces+releases the @@ -163,15 +165,10 @@ const HWND: u64 = 0x1234; fn run_forced_interleave(use_retained: bool, poison_on_zero: bool) -> usize { let uaf_hits: &'static AtomicUsize = Box::leak(Box::new(AtomicUsize::new(0))); let cache = Arc::new(ElementCache::new()); + let pid = NEXT_PID.fetch_add(1, Ordering::Relaxed); let ptr = make_fake(uaf_hits, poison_on_zero); - cache.core.insert( - CacheKey { - pid: PID, - hwnd: HWND, - }, - snapshot_with(vec![ptr]), - ); + let sid = cache.publish(pid, HWND, Arc::new(snapshot_with(vec![ptr]))); let (b_looked_up_tx, b_looked_up_rx) = mpsc::channel::<()>(); let (a_replaced_tx, a_replaced_rx) = mpsc::channel::<()>(); @@ -179,17 +176,27 @@ fn run_forced_interleave(use_retained: bool, poison_on_zero: bool) -> usize { let cache_b = cache.clone(); let worker_b = thread::spawn(move || { if use_retained { - // FIXED path: AddRef under the lock; guard pins the object alive. - let guard = cache_b - .get_element_retained(PID, HWND, 0) + // FIXED path: bind the original snapshot, then retain its object. + let binding = cua_driver_core::element_token::global() + .resolve_bound( + pid as i32, + &cua_driver_core::element_token::token_for(sid, 0), + ) + .unwrap() + .2 + .unwrap(); + let bound = BoundElementCache::from_binding(binding); + let guard = bound + .get_element_retained(pid, HWND, 0) .expect("element present"); + drop(bound); b_looked_up_tx.send(()).unwrap(); a_replaced_rx.recv().unwrap(); // A has now replaced + released unsafe { touch_vtable(guard.as_ptr()) }; // safe: guard holds +1 drop(guard); } else { // PRE-FIX path: bare pointer, no AddRef. - let raw = old_get_element_ptr(&cache_b, PID, HWND, 0).expect("element present"); + let raw = old_get_element_ptr(&cache_b, pid, HWND, 0).expect("element present"); b_looked_up_tx.send(()).unwrap(); a_replaced_rx.recv().unwrap(); // A has now replaced + released → freed unsafe { touch_vtable(raw) }; // USE-AFTER-FREE @@ -201,18 +208,13 @@ fn run_forced_interleave(use_retained: bool, poison_on_zero: bool) -> usize { b_looked_up_rx.recv().unwrap(); // get_window_state on the same (pid, hwnd): replace the snapshot. The // old snapshot's Drop fires COM Release on `ptr`. - cache_a.core.insert( - CacheKey { - pid: PID, - hwnd: HWND, - }, - snapshot_with(vec![]), - ); + cache_a.publish(pid, HWND, Arc::new(snapshot_with(vec![]))); a_replaced_tx.send(()).unwrap(); }); worker_a.join().unwrap(); worker_b.join().unwrap(); + cua_driver_core::element_token::global().clear_pid(pid as i32); uaf_hits.load(Ordering::SeqCst) } @@ -244,16 +246,12 @@ fn prefix_path_commits_use_after_free() { fn fixed_path_stress_no_uaf() { let uaf_hits: &'static AtomicUsize = Box::leak(Box::new(AtomicUsize::new(0))); let cache = Arc::new(ElementCache::new()); + let pid = NEXT_PID.fetch_add(1, Ordering::Relaxed); // Seed a snapshot of several elements. let seed: Vec<usize> = (0..8).map(|_| make_fake(uaf_hits, false)).collect(); - cache.core.insert( - CacheKey { - pid: PID, - hwnd: HWND, - }, - snapshot_with(seed), - ); + let sid = cache.publish(pid, HWND, Arc::new(snapshot_with(seed))); + let latest = Arc::new(AtomicU32::new(sid)); const ITERS: usize = 4000; let mut handles = Vec::new(); @@ -261,27 +259,32 @@ fn fixed_path_stress_no_uaf() { // Replacer threads: continuously run `update` (snapshot replace → Release). for _ in 0..3 { let cache_r = cache.clone(); + let latest = latest.clone(); handles.push(thread::spawn(move || { for _ in 0..ITERS { let fresh: Vec<usize> = (0..8).map(|_| make_fake(uaf_hits, false)).collect(); - cache_r.core.insert( - CacheKey { - pid: PID, - hwnd: HWND, - }, - snapshot_with(fresh), - ); + let sid = cache_r.publish(pid, HWND, Arc::new(snapshot_with(fresh))); + latest.store(sid, Ordering::Release); } })); } // Actor threads: lookup-retain-deref-release, the click/type/set_value path. for _ in 0..3 { - let cache_c = cache.clone(); + let latest = latest.clone(); handles.push(thread::spawn(move || { for i in 0..ITERS { - if let Some(guard) = cache_c.get_element_retained(PID, HWND, i % 8) { - unsafe { touch_vtable(guard.as_ptr()) }; + let token = cua_driver_core::element_token::token_for( + latest.load(Ordering::Acquire), + i % 8, + ); + if let Ok((_, _, Some(binding))) = + cua_driver_core::element_token::global().resolve_bound(pid as i32, &token) + { + let bound = BoundElementCache::from_binding(binding); + if let Some(guard) = bound.get_element_retained(pid, HWND, i % 8) { + unsafe { touch_vtable(guard.as_ptr()) }; + } } } })); @@ -290,6 +293,7 @@ fn fixed_path_stress_no_uaf() { for h in handles { h.join().unwrap(); } + cua_driver_core::element_token::global().clear_pid(pid as i32); assert_eq!( uaf_hits.load(Ordering::SeqCst), 0, @@ -299,6 +303,98 @@ fn fixed_path_stress_no_uaf() { // ---- Hard-crash demonstrations (run manually, never in the suite) ----------- +#[test] +fn target_clear_revokes_token_without_releasing_an_in_flight_com_binding() { + let hits = Box::leak(Box::new(AtomicUsize::new(0))); + let ptr = make_fake(hits, false); + let pid = NEXT_PID.fetch_add(1, Ordering::Relaxed); + let cache = ElementCache::new(); + let sid = cache.publish(pid, HWND, Arc::new(snapshot_with(vec![ptr]))); + let registry = cua_driver_core::element_token::global(); + let token = cua_driver_core::element_token::token_for(sid, 0); + let binding = registry + .resolve_bound(pid as i32, &token) + .unwrap() + .2 + .unwrap(); + cache.clear_target(pid, HWND); + assert!(registry.resolve(pid as i32, &token).is_err()); + assert_eq!(cache.element_count(pid, HWND), 0); + assert_eq!( + unsafe { (*(ptr as *const FakeObj)).refcount.load(Ordering::SeqCst) }, + 1 + ); + drop(binding); + assert_eq!( + unsafe { (*(ptr as *const FakeObj)).refcount.load(Ordering::SeqCst) }, + 0 + ); + assert_eq!(hits.load(Ordering::SeqCst), 0); +} + +#[test] +fn resolved_snapshot_keeps_pointer_kind_geometry_and_actions_together() { + let hits = Box::leak(Box::new(AtomicUsize::new(0))); + let first = make_fake(hits, false); + let second = make_fake(hits, false); + let pid = NEXT_PID.fetch_add(1, Ordering::Relaxed); + let cache = ElementCache::new(); + let mut a = snapshot_with(vec![first]); + a.centers[0] = (10, 20); + a.rects[0] = Some((1, 2, 19, 38)); + a.actions[0] = vec!["first-action".into()]; + let sid = cache.publish(pid, HWND, Arc::new(a)); + let registry = cua_driver_core::element_token::global(); + let token = cua_driver_core::element_token::token_for(sid, 0); + let binding = registry + .resolve_bound(pid as i32, &token) + .unwrap() + .2 + .unwrap(); + let bound = BoundElementCache::from_binding(binding); + let mut b = snapshot_with(vec![second]); + b.kind = SnapshotKind::Msaa; + b.centers[0] = (90, 100); + b.msaa_roles[0] = Some(0x38); + b.actions[0] = vec!["replacement-action".into()]; + cache.publish(pid, HWND, Arc::new(b)); + assert!(registry.resolve(pid as i32, &token).is_err()); + let retained = bound.get_element_retained(pid, HWND, 0).unwrap(); + assert_eq!(retained.as_ptr(), first); + assert!(retained.is_uia()); + assert_eq!(bound.get_element_center(pid, HWND, 0), Some((10, 20))); + assert_eq!(bound.get_element_rect(pid, HWND, 0), Some((1, 2, 19, 38))); + assert!(matches!( + bound.get_element_kind_and_role(pid, HWND, 0), + Some((SnapshotKind::Uia, None)) + )); + assert_eq!( + bound.get_element_actions(pid, HWND, 0), + Some(vec!["first-action".into()]) + ); + assert!(bound.get_element_center(pid, HWND + 1, 0).is_none()); + registry.clear_pid(pid as i32); + drop(retained); + assert_eq!( + unsafe { (*(first as *const FakeObj)).refcount.load(Ordering::SeqCst) }, + 1 + ); + drop(bound); + assert_eq!( + unsafe { (*(first as *const FakeObj)).refcount.load(Ordering::SeqCst) }, + 0 + ); + assert_eq!( + unsafe { + (*(second as *const FakeObj)) + .refcount + .load(Ordering::SeqCst) + }, + 0 + ); + assert_eq!(hits.load(Ordering::SeqCst), 0); +} + /// BEFORE (real fault): the pre-fix path takes a genuine /// STATUS_ACCESS_VIOLATION. Poison-on-zero makes the otherwise heap-reuse- /// dependent UAF deterministic. Run with: diff --git a/packages/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/packages/cua-driver/rust/crates/platform-windows/src/uia/mod.rs index f6f59dd7a5c..57628b38adb 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/uia/mod.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/uia/mod.rs @@ -407,10 +407,40 @@ unsafe fn walk_tree_unsafe( max_elements: usize, max_depth: usize, collect_runtime_ids: bool, - mut progress: Option<&mut dyn FnMut(UiaNode, String)>, + progress: Option<&mut dyn FnMut(UiaNode, String)>, ) -> UiaTreeResult { let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + walk_tree_initialized( + hwnd, + query, + max_elements, + max_depth, + collect_runtime_ids, + progress, + ) +} +/// The recording caller owns COM initialization through serialization and +/// release of every retained node binding. +pub(crate) unsafe fn walk_tree_for_recording(hwnd: u64) -> UiaTreeResult { + walk_tree_initialized( + hwnd, + None, + DEFAULT_MAX_TOTAL_ELEMENTS, + DEFAULT_MAX_DEPTH, + false, + None, + ) +} + +unsafe fn walk_tree_initialized( + hwnd: u64, + query: Option<&str>, + max_elements: usize, + max_depth: usize, + collect_runtime_ids: bool, + mut progress: Option<&mut dyn FnMut(UiaNode, String)>, +) -> UiaTreeResult { let automation: IUIAutomation = match CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) { Ok(a) => a, diff --git a/packages/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs b/packages/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs index 17abc8fa7e4..dd12b768891 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs @@ -432,14 +432,53 @@ fn is_coord_independent_action(elem: &IUIAutomationElement) -> bool { ) } -pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool { - run_uia_with_deadline_cancelable( +fn point_pattern_available<T>( + result: windows::core::Result<T>, +) -> Result<bool, PointInvokeOutcome> { + match result { + Ok(_) => Ok(true), + // GetCurrentPattern returns a null interface for an unsupported pattern; + // windows-rs 0.58 represents that successful null result as Error::empty(). + Err(error) + if error.code().is_ok() + || error.code() == windows::Win32::Foundation::E_NOINTERFACE + || error.code().0 as u32 + == windows::Win32::UI::Accessibility::UIA_E_NOTSUPPORTED => + { + Ok(false) + } + Err(_) => Err(PointInvokeOutcome::Unavailable), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PointInvokeOutcome { + Invoked, + Miss, + Busy, + Timeout, + Unavailable, + Unconfirmed, +} + +impl PointInvokeOutcome { + fn from_worker(result: Result<Self, UiaDeadlineError>) -> Self { + match result { + Ok(outcome) => outcome, + Err(UiaDeadlineError::Busy) => Self::Busy, + Err(UiaDeadlineError::Timeout) => Self::Timeout, + Err(UiaDeadlineError::Unavailable) => Self::Unconfirmed, + } + } +} + +pub(crate) fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> PointInvokeOutcome { + PointInvokeOutcome::from_worker(run_uia_with_deadline_cancelable( "window hit-test invoke", SUBTREE_OP_TIMEOUT, - "PostMessage click delivery", + "an unknown-effect error without another input attempt", move |cancelled| try_invoke_in_window_at_point_unbounded(hwnd, sx, sy, &cancelled), - ) - .unwrap_or(false) + )) } fn try_invoke_in_window_at_point_unbounded( @@ -447,48 +486,52 @@ fn try_invoke_in_window_at_point_unbounded( sx: i32, sy: i32, cancelled: &AtomicBool, -) -> bool { +) -> PointInvokeOutcome { + use PointInvokeOutcome::{Invoked, Miss, Timeout, Unavailable, Unconfirmed}; // Keep this first so COM interfaces drop before CoUninitialize. let _com = ComInit::new(); if hwnd == 0 { - return false; + return Unavailable; } let uia = match get_uia() { Some(u) => u, - None => return false, + None => return Unavailable, }; unsafe { let root = match uia.ElementFromHandle(HWND(hwnd as *mut _)) { Ok(r) => r, Err(e) => { tracing::debug!(target: "click", "ElementFromHandle(0x{hwnd:x}) failed: {e}"); - return false; + return Unavailable; } }; let cond = match uia.CreateTrueCondition() { Ok(c) => c, Err(e) => { tracing::debug!(target: "click", "CreateTrueCondition failed: {e}"); - return false; + return Unavailable; } }; let arr = match root.FindAll(TreeScope_Subtree, &cond) { Ok(a) => a, Err(e) => { tracing::debug!(target: "click", "FindAll(Subtree) on 0x{hwnd:x} failed: {e}"); - return false; + return Unavailable; } }; - let n = arr.Length().unwrap_or(0); + let n = match arr.Length() { + Ok(n) => n, + Err(_) => return Unavailable, + }; let mut best: Option<(IUIAutomationElement, i64)> = None; for i in 0..n { let elem = match arr.GetElement(i) { Ok(e) => e, - Err(_) => continue, + Err(_) => return Unavailable, }; let rect = match elem.CurrentBoundingRectangle() { Ok(r) => r, - Err(_) => continue, + Err(_) => return Unavailable, }; if sx < rect.left || sx > rect.right || sy < rect.top || sy > rect.bottom { continue; @@ -498,10 +541,18 @@ fn try_invoke_in_window_at_point_unbounded( // Invoke does nothing on them, only Expand opens the submenu. // (See FreeCAD finding 2026-05-21: clicking File menu via Invoke // returned ✅ but the menu never opened.) - let has_invoke = elem.GetCurrentPattern(UIA_InvokePatternId).is_ok(); - let has_expand = elem - .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId) - .is_ok(); + let has_invoke = + match point_pattern_available(elem.GetCurrentPattern(UIA_InvokePatternId)) { + Ok(available) => available, + Err(outcome) => return outcome, + }; + let has_expand = + match point_pattern_available(elem.GetCurrentPattern( + windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId, + )) { + Ok(available) => available, + Err(outcome) => return outcome, + }; if !has_invoke && !has_expand { continue; } @@ -537,12 +588,12 @@ fn try_invoke_in_window_at_point_unbounded( target: "click", "no Invoke/ExpandCollapse descendant of 0x{hwnd:x} contains screen ({sx},{sy}) (scanned {n} elems)" ); - return false; + return Miss; } }; if cancelled.load(Ordering::Acquire) { tracing::debug!(target: "click", "UIA hit-test invoke cancelled before activation"); - return false; + return Timeout; } // Pattern preference for menu items: when both Invoke AND // ExpandCollapse are advertised, the element is almost always a @@ -550,22 +601,25 @@ fn try_invoke_in_window_at_point_unbounded( // submenu" — Invoke would be a no-op. Prefer ExpandCollapse.Expand // in that case. Pure-Invoke leaves (buttons, links, etc.) go // through Invoke as before. - let winner_has_expand = winner - .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId) - .is_ok(); - let winner_has_invoke = winner.GetCurrentPattern(UIA_InvokePatternId).is_ok(); + let winner_has_expand = match point_pattern_available( + winner + .GetCurrentPattern(windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId), + ) { + Ok(available) => available, + Err(outcome) => return outcome, + }; if cancelled.load(Ordering::Acquire) { - return false; + return Timeout; } // UWP foreground-steal bypass: gate the entire activation block on // `is_xaml_host_hwnd(hwnd)`. For non-XAML hosts the closure is a // straight passthrough. crate::uia::fg_bypass::run_with_uwp_bypass(hwnd, || { if cancelled.load(Ordering::Acquire) { - return false; + return Timeout; } - if winner_has_expand && winner_has_invoke { - // Try Expand first, fall back to Invoke if Expand fails. + if winner_has_expand { + // A failed activation may already have taken effect. Do not replay it. if let Ok(pat) = winner.GetCurrentPattern( windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId, ) { @@ -573,45 +627,29 @@ fn try_invoke_in_window_at_point_unbounded( .cast::<windows::Win32::UI::Accessibility::IUIAutomationExpandCollapsePattern>() { if cancelled.load(Ordering::Acquire) { - return false; - } - if ec.Expand().is_ok() { - return true; + return Timeout; } + return if ec.Expand().is_ok() { Invoked } else { Unconfirmed }; } } - // Expand failed — fall through to Invoke as best-effort. - } else if winner_has_expand && !winner_has_invoke { - if let Ok(pat) = winner.GetCurrentPattern( - windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId, - ) { - if let Ok(ec) = pat - .cast::<windows::Win32::UI::Accessibility::IUIAutomationExpandCollapsePattern>() - { - if cancelled.load(Ordering::Acquire) { - return false; - } - return ec.Expand().is_ok(); - } - } - return false; + return Unavailable; } let pattern = match winner.GetCurrentPattern(UIA_InvokePatternId) { Ok(p) => p, - Err(_) => return false, + Err(_) => return Unavailable, }; let inv: IUIAutomationInvokePattern = match pattern.cast() { Ok(i) => i, - Err(_) => return false, + Err(_) => return Unavailable, }; if cancelled.load(Ordering::Acquire) { - return false; + return Timeout; } match inv.Invoke() { - Ok(()) => true, + Ok(()) => Invoked, Err(e) => { tracing::debug!(target: "click", "UIA Invoke (windowed) at ({sx},{sy}) failed: {e}"); - false + Unconfirmed } } }) @@ -1112,6 +1150,14 @@ mod tests { assert_eq!(select_window_rect(false, Some(dwm), Some(win32)), Some(dwm)); } + #[test] + fn a_worker_without_a_result_cannot_prove_input_was_not_sent() { + assert_eq!( + PointInvokeOutcome::from_worker(Err(UiaDeadlineError::Unavailable)), + PointInvokeOutcome::Unconfirmed, + ); + } + #[test] fn accelerator_deadline_errors_preserve_the_failure_class() { assert!(accelerator_deadline_error(UiaDeadlineError::Timeout) diff --git a/packages/cua-driver/rust/crates/platform-windows/src/uia_worker.rs b/packages/cua-driver/rust/crates/platform-windows/src/uia_worker.rs index 52732235ac4..3c6c903478b 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/uia_worker.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/uia_worker.rs @@ -317,7 +317,17 @@ async fn call_worker(name: &str, args: Value) -> anyhow::Result<ToolResult> { .await?; client.flush().await?; let mut line = String::new(); - let read = BufReader::new(client).read_line(&mut line).await?; + let mut reader = BufReader::new(client); + let read = tokio::time::timeout( + std::time::Duration::from_secs(20), + reader.read_line(&mut line), + ) + .await + .map_err(|_| { + anyhow::anyhow!( + "UIAccess worker response timed out; input may have been sent; observe before retrying" + ) + })??; if read == 0 { anyhow::bail!("UIAccess worker closed before returning {name}"); } @@ -327,11 +337,9 @@ async fn call_worker(name: &str, args: Value) -> anyhow::Result<ToolResult> { }) } -/// Returns None only inside the authenticated UIAccess worker itself. Every -/// caller in the ordinary runtime receives either the worker result or a hard -/// availability error; foreground input never falls back to medium integrity. +/// UIAccess is an explicit optional route. Once selected, failure never falls back. pub(crate) async fn forward_required(name: &str, args: Value) -> Option<ToolResult> { - if is_worker_process() { + if is_worker_process() || std::env::var("CUA_DRIVER_USE_UIACCESS").as_deref() != Ok("1") { return None; } Some(call_worker(name, args).await.unwrap_or_else(|error| { diff --git a/packages/cua-driver/rust/crates/platform-windows/src/win32/combo.rs b/packages/cua-driver/rust/crates/platform-windows/src/win32/combo.rs new file mode 100644 index 00000000000..c84242253ba --- /dev/null +++ b/packages/cua-driver/rust/crates/platform-windows/src/win32/combo.rs @@ -0,0 +1,361 @@ +// Portions adapted from trycua/cua 0.35.4, Copyright (c) 2025 Cua AI. MIT license. +//! `set_value` on a native Win32 combo box (the `COMBOBOX` class, which +//! WinForms and MFC combo boxes subclass). +//! +//! A UIA `ValuePattern.SetValue` on such a combo only rewrites the window +//! text (`WM_SETTEXT`): the control shows the new value but the app is never +//! told, so `SelectedIndexChanged` / `TextChanged` (WinForms) or the dialog's +//! `CBN_SELCHANGE` handler never run. This module makes the change the way the +//! control itself reports a user's choice, without focusing or raising the +//! window: +//! +//! * a listed option is selected with `CB_SETCURSEL`, then the parent gets the +//! `WM_COMMAND` notifications a real selection sends (`CBN_SELENDOK`, +//! `CBN_SELCHANGE`); +//! * an editable combo's text is replaced through its edit child with +//! `EM_SETSEL` + `EM_REPLACESEL`, which the edit treats as typing, so the +//! combo forwards `CBN_EDITUPDATE` / `CBN_EDITCHANGE` itself. +//! +//! The string messages (`CB_FINDSTRINGEXACT`, `CB_GETLBTEXT`, +//! `EM_REPLACESEL`, `WM_GETTEXT`) are marshalled across processes by the +//! system for these control classes. + +use windows::core::PCWSTR; +use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; +use windows::Win32::UI::WindowsAndMessaging::{ + FindWindowExW, GetClassNameW, GetDlgCtrlID, GetParent, GetWindowLongW, SendMessageTimeoutW, + GWL_STYLE, SMTO_ABORTIFHUNG, +}; + +const CB_ERR: isize = -1; +const CB_GETCOUNT: u32 = 0x0146; +const CB_GETCURSEL: u32 = 0x0147; +const CB_GETLBTEXT: u32 = 0x0148; +const CB_GETLBTEXTLEN: u32 = 0x0149; +const CB_SETCURSEL: u32 = 0x014E; +const CB_FINDSTRINGEXACT: u32 = 0x0158; +const CBN_SELCHANGE: u16 = 1; +const CBN_EDITCHANGE: u16 = 5; +const CBN_EDITUPDATE: u16 = 6; +const CBN_SELENDOK: u16 = 9; +const CBS_TYPE_MASK: i32 = 0x3; +const CBS_DROPDOWNLIST: i32 = 0x3; +const EM_SETSEL: u32 = 0x00B1; +const EM_REPLACESEL: u32 = 0x00C2; +const WM_COMMAND: u32 = 0x0111; +const WM_GETTEXT: u32 = 0x000D; +const WM_GETTEXTLENGTH: u32 = 0x000E; +const WM_SETTEXT: u32 = 0x000C; + +/// Per-message timeout. A notification runs the app's change handler before +/// it returns, so this is the budget for that handler too. +const SEND_TIMEOUT_MS: u32 = 3_000; +/// Options listed in a "no such option" refusal. +const MAX_LISTED_OPTIONS: usize = 50; + +/// A native combo box found behind a UIA element. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Combo { + pub hwnd: isize, + /// `CBS_DROPDOWN` / `CBS_SIMPLE` (has an edit field), as opposed to + /// `CBS_DROPDOWNLIST`. + pub editable: bool, +} + +/// What [`set_value`] did. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ComboWrite { + /// The value was applied. `selected` is the option index chosen, if the + /// value names an option. + Applied { + selected: Option<usize>, + /// The control already held this value; nothing was sent. + unchanged: bool, + /// Text and selection read back from the control afterwards. + readback_text: Option<String>, + readback_index: Option<usize>, + }, + /// A drop-down list has no option with this text; nothing was changed. + NoSuchOption { options: Vec<String> }, +} + +/// `true` for the Win32 combo box class and its subclasses (WinForms +/// `WindowsForms10.COMBOBOX.*`), not `ComboBoxEx32` (whose notifications +/// differ) or the drop-down's `ComboLBox` list. +pub fn is_combo_class(class: &str) -> bool { + let class = class.to_ascii_lowercase(); + class.contains("combobox") && !class.contains("comboboxex") && !class.contains("combolbox") +} + +/// `true` for the `CBS_*` type bits of an editable combo (simple or drop-down). +pub fn style_is_editable(style: i32) -> bool { + style & CBS_TYPE_MASK != CBS_DROPDOWNLIST +} + +/// `WM_COMMAND` wParam for a control notification: `MAKEWPARAM(id, code)`. +pub fn command_wparam(control_id: i32, code: u16) -> usize { + (control_id as u32 as usize & 0xFFFF) | ((code as usize) << 16) +} + +/// Whether the write took: a listed option must be the selected index, and +/// an editable combo must read back the text (case-insensitively when the +/// option it matched is spelled differently). +pub fn write_verified(write: &ComboWrite, value: &str) -> bool { + match write { + ComboWrite::Applied { + selected, + readback_text, + readback_index, + .. + } => match selected { + Some(index) => *readback_index == Some(*index), + None => readback_text.as_deref() == Some(value), + }, + ComboWrite::NoSuchOption { .. } => false, + } +} + +fn hwnd(raw: isize) -> HWND { + HWND(raw as *mut _) +} + +fn class_name(raw: isize) -> String { + let mut buf = [0u16; 256]; + let len = unsafe { GetClassNameW(hwnd(raw), &mut buf) }; + String::from_utf16_lossy(&buf[..len.max(0) as usize]) +} + +fn send(raw: isize, msg: u32, wparam: usize, lparam: isize) -> Option<isize> { + let mut result = 0usize; + let ok = unsafe { + SendMessageTimeoutW( + hwnd(raw), + msg, + WPARAM(wparam), + LPARAM(lparam), + SMTO_ABORTIFHUNG, + SEND_TIMEOUT_MS, + Some(&mut result), + ) + }; + (ok.0 != 0).then_some(result as isize) +} + +fn wide(text: &str) -> Vec<u16> { + text.encode_utf16().chain(std::iter::once(0)).collect() +} + +fn window_text(raw: isize) -> Option<String> { + let len = send(raw, WM_GETTEXTLENGTH, 0, 0)?.max(0) as usize; + let mut buf = vec![0u16; len + 1]; + let copied = send(raw, WM_GETTEXT, buf.len(), buf.as_mut_ptr() as isize)?.max(0) as usize; + Some(String::from_utf16_lossy(&buf[..copied.min(len)])) +} + +fn option_text(combo: isize, index: usize) -> Option<String> { + let len = send(combo, CB_GETLBTEXTLEN, index, 0)?; + if len < 0 { + return None; + } + let mut buf = vec![0u16; len as usize + 1]; + let copied = send(combo, CB_GETLBTEXT, index, buf.as_mut_ptr() as isize)?; + if copied < 0 { + return None; + } + Some(String::from_utf16_lossy( + &buf[..(copied as usize).min(len as usize)], + )) +} + +fn selected_index(combo: isize) -> Option<usize> { + send(combo, CB_GETCURSEL, 0, 0).and_then(|index| usize::try_from(index).ok()) +} + +fn options(combo: isize) -> Vec<String> { + let count = send(combo, CB_GETCOUNT, 0, 0).unwrap_or(0).max(0) as usize; + (0..count.min(MAX_LISTED_OPTIONS)) + .filter_map(|index| option_text(combo, index)) + .collect() +} + +fn edit_child(combo: isize) -> Option<isize> { + let class = wide("Edit"); + unsafe { + FindWindowExW( + hwnd(combo), + HWND::default(), + PCWSTR(class.as_ptr()), + PCWSTR::null(), + ) + } + .ok() + .filter(|edit| !edit.is_invalid()) + .map(|edit| edit.0 as isize) +} + +/// Send a combo notification to the combo's parent, as the control does. +fn notify_parent(combo: isize, code: u16) -> bool { + let Ok(parent) = (unsafe { GetParent(hwnd(combo)) }) else { + return false; + }; + let id = unsafe { GetDlgCtrlID(hwnd(combo)) }; + send( + parent.0 as isize, + WM_COMMAND, + command_wparam(id, code), + combo, + ) + .is_some() +} + +/// The native combo box behind a UIA element's window handle: the combo +/// itself, or the combo that owns this edit field. +pub fn combo_for_window(raw: isize) -> Option<Combo> { + if raw == 0 { + return None; + } + let combo = if is_combo_class(&class_name(raw)) { + raw + } else if class_name(raw).eq_ignore_ascii_case("Edit") { + let parent = unsafe { GetParent(hwnd(raw)) }.ok()?.0 as isize; + if parent == 0 || !is_combo_class(&class_name(parent)) { + return None; + } + parent + } else { + return None; + }; + let style = unsafe { GetWindowLongW(hwnd(combo), GWL_STYLE) }; + Some(Combo { + hwnd: combo, + editable: style_is_editable(style), + }) +} + +/// Select `value` in the combo (or type it into an editable one) and tell +/// the app, as described in the module docs. +pub fn set_value(combo: Combo, value: &str) -> anyhow::Result<ComboWrite> { + let text = wide(value); + let found = send( + combo.hwnd, + CB_FINDSTRINGEXACT, + usize::MAX, + text.as_ptr() as isize, + ) + .ok_or_else(|| anyhow::anyhow!("the combo box did not answer CB_FINDSTRINGEXACT"))?; + let selected = (found != CB_ERR).then_some(found as usize); + let before = selected_index(combo.hwnd); + + if !combo.editable { + let Some(index) = selected else { + return Ok(ComboWrite::NoSuchOption { + options: options(combo.hwnd), + }); + }; + let unchanged = before == Some(index); + if !unchanged { + send(combo.hwnd, CB_SETCURSEL, index, 0) + .ok_or_else(|| anyhow::anyhow!("the combo box did not answer CB_SETCURSEL"))?; + notify_parent(combo.hwnd, CBN_SELENDOK); + notify_parent(combo.hwnd, CBN_SELCHANGE); + } + return Ok(ComboWrite::Applied { + selected, + unchanged, + readback_text: selected_index(combo.hwnd).and_then(|i| option_text(combo.hwnd, i)), + readback_index: selected_index(combo.hwnd), + }); + } + + let edit = edit_child(combo.hwnd); + let current = edit.or(Some(combo.hwnd)).and_then(window_text); + let unchanged = current.as_deref() == Some(value) && (selected.is_none() || before == selected); + if !unchanged { + match edit { + // Replacing the selection is how the edit sees typing: it sends + // EN_UPDATE / EN_CHANGE and the combo forwards them to the app. + Some(edit) => { + send(edit, EM_SETSEL, 0, -1); + send(edit, EM_REPLACESEL, 1, text.as_ptr() as isize) + .ok_or_else(|| anyhow::anyhow!("the combo's edit field did not answer"))?; + } + None => { + send(combo.hwnd, WM_SETTEXT, 0, text.as_ptr() as isize) + .ok_or_else(|| anyhow::anyhow!("the combo box did not answer WM_SETTEXT"))?; + notify_parent(combo.hwnd, CBN_EDITUPDATE); + notify_parent(combo.hwnd, CBN_EDITCHANGE); + } + } + if let Some(index) = selected { + send(combo.hwnd, CB_SETCURSEL, index, 0); + notify_parent(combo.hwnd, CBN_SELENDOK); + notify_parent(combo.hwnd, CBN_SELCHANGE); + } + } + Ok(ComboWrite::Applied { + selected, + unchanged, + readback_text: edit.or(Some(combo.hwnd)).and_then(window_text), + readback_index: selected_index(combo.hwnd), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn combo_classes_include_subclasses_but_not_ex_or_list() { + assert!(is_combo_class("ComboBox")); + assert!(is_combo_class( + "WindowsForms10.COMBOBOX.app.0.29c74c7_r10_ad1" + )); + assert!(!is_combo_class("ComboBoxEx32")); + assert!(!is_combo_class("ComboLBox")); + assert!(!is_combo_class("Edit")); + } + + #[test] + fn style_bits_tell_editable_from_drop_down_list() { + // Styles observed on WinForms combos (WS_* bits plus CBS_*). + assert!(!style_is_editable(0x5601_0243)); + assert!(style_is_editable(0x5601_0242)); + assert!(style_is_editable(0x1)); // CBS_SIMPLE + } + + #[test] + fn command_wparam_packs_id_and_code() { + assert_eq!(command_wparam(66056, 1), (66056 & 0xFFFF) | (1 << 16)); + assert_eq!(command_wparam(1001, 9), 1001 | (9 << 16)); + } + + #[test] + fn verification_needs_the_selected_index_or_the_text() { + let applied = |selected, text: Option<&str>, index| ComboWrite::Applied { + selected, + unchanged: false, + readback_text: text.map(str::to_owned), + readback_index: index, + }; + assert!(write_verified( + &applied(Some(2), Some("Blue"), Some(2)), + "blue" + )); + assert!(!write_verified( + &applied(Some(2), Some("Red"), Some(0)), + "Blue" + )); + assert!(write_verified( + &applied(None, Some("Custom"), None), + "Custom" + )); + assert!(!write_verified( + &applied(None, Some("Small"), Some(0)), + "Custom" + )); + assert!(!write_verified( + &ComboWrite::NoSuchOption { options: vec![] }, + "x" + )); + } +} diff --git a/packages/cua-driver/rust/crates/platform-windows/src/win32/installed_apps.rs b/packages/cua-driver/rust/crates/platform-windows/src/win32/installed_apps.rs index ba2b496bc09..1eb5aa9fcc1 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/win32/installed_apps.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/win32/installed_apps.rs @@ -25,7 +25,7 @@ use windows::Win32::System::Com::{ CoCreateInstance, CoInitializeEx, CoUninitialize, IPersistFile, CLSCTX_INPROC_SERVER, COINIT_APARTMENTTHREADED, STGM, }; -use windows::Win32::UI::Shell::{IShellLinkW, ShellLink, SLGP_RAWPATH}; +use windows::Win32::UI::Shell::{IShellLinkW, ShellLink}; /// Parsed metadata for an installed Windows application. #[derive(Debug, Clone)] @@ -211,7 +211,7 @@ unsafe fn read_lnk_target(lnk_path: &Path) -> windows::core::Result<(String, Str persist.Load(PCWSTR(wide_path.as_ptr()), STGM(0))?; let mut path_buf = vec![0u16; MAX_PATH as usize + 1]; - shell_link.GetPath(&mut path_buf, std::ptr::null_mut(), SLGP_RAWPATH.0 as u32)?; + shell_link.GetPath(&mut path_buf, std::ptr::null_mut(), 0)?; // INFOTIPSIZE (1024) is the documented upper bound for the LNK // arguments field; allocate one extra slot for the NUL terminator. @@ -655,6 +655,69 @@ mod tests { use super::*; use std::sync::atomic::AtomicUsize; + #[test] + fn shortcut_environment_target_resolves_without_changing_arguments() { + unsafe { CoInitializeEx(None, COINIT_APARTMENTTHREADED) } + .ok() + .expect("initialize COM for shortcut resolution"); + struct Apartment; + impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + let _apartment = Apartment; + let directory = tempfile::tempdir().expect("create shortcut fixture directory"); + let link_path = directory.path().join("Environment target.lnk"); + let link_path_wide = to_wide_nul(link_path.to_str().expect("fixture path is UTF-8")); + let target = to_wide_nul(r"%windir%\system32\cmd.exe"); + let expected = PathBuf::from(std::env::var_os("windir").expect("Windows directory")) + .join("system32") + .join("cmd.exe") + .canonicalize() + .expect("Windows command processor exists"); + + for arguments in ["", r#"/d /c echo "argument with spaces""#] { + let link: IShellLinkW = unsafe { + CoCreateInstance(&ShellLink, None, CLSCTX_INPROC_SERVER).expect("create shell link") + }; + let arguments_wide = to_wide_nul(arguments); + let persist: IPersistFile = link.cast().expect("shell link persistence"); + unsafe { + link.SetPath(PCWSTR(target.as_ptr())) + .expect("set environment target"); + link.SetArguments(PCWSTR(arguments_wide.as_ptr())) + .expect("set arguments"); + persist + .Save(PCWSTR(link_path_wide.as_ptr()), true) + .expect("save shortcut"); + let mut raw = vec![0u16; MAX_PATH as usize + 1]; + link.GetPath( + &mut raw, + std::ptr::null_mut(), + windows::Win32::UI::Shell::SLGP_RAWPATH.0 as u32, + ) + .expect("inspect raw target"); + assert!( + decode_wstr(&raw).starts_with('%'), + "fixture must preserve its environment token" + ); + } + + let app = resolve_lnk(&link_path).expect("discover environment shortcut"); + assert_eq!( + PathBuf::from(&app.bundle_id) + .canonicalize() + .expect("resolved executable exists"), + expected + ); + assert!(!app.launch_path.contains("%windir%")); + if !arguments.is_empty() { + assert!(app.launch_path.ends_with(&format!(" {arguments}"))); + } + } + } + #[test] fn wedged_uwp_scan_has_bounded_worker_growth_and_recovers() { let gate = Arc::new(UwpScanSingleFlight::new(40)); diff --git a/packages/cua-driver/rust/crates/platform-windows/src/win32/mod.rs b/packages/cua-driver/rust/crates/platform-windows/src/win32/mod.rs index 00e0c5404a4..dcefffc2dbc 100644 --- a/packages/cua-driver/rust/crates/platform-windows/src/win32/mod.rs +++ b/packages/cua-driver/rust/crates/platform-windows/src/win32/mod.rs @@ -12,3 +12,5 @@ pub(crate) use windows::{ visible_owned_windows, window_owner_pid, ForegroundTarget, }; pub use windows::{list_windows, resolve_uwp_host_window, WindowInfo}; + +pub mod combo; diff --git a/packages/cua-driver/scripts/_install-common.psm1 b/packages/cua-driver/scripts/_install-common.psm1 index 63ca27919aa..ae192abad25 100644 --- a/packages/cua-driver/scripts/_install-common.psm1 +++ b/packages/cua-driver/scripts/_install-common.psm1 @@ -1,13 +1,10 @@ -# _install-common.psm1 - shared helpers for install.ps1 + install-local.ps1. +# _install-common.psm1 - daemon cleanup helpers for install.ps1. # -# Both scripts import this module to avoid drift in the daemon-kill logic. -# -# * install-local.ps1 runs from a checked-out repo, so it imports the -# module from disk via $PSScriptRoot/_install-common.psm1. # * install.ps1 is fetched via `irm | iex` and has no file on disk # during execution. It uses Import-CuaDriverInstallModule (below) # which prefers the on-disk copy when available (dev / CI) and # falls back to fetching the .psm1 from GitHub raw. +# * install-local.ps1 manages its separate -local process identity. # # Keep this module narrow on purpose - it's loaded over the network in # the production install path, so every additional line is paid for in @@ -18,6 +15,36 @@ Set-StrictMode -Version Latest +# A tree kill of an updater ancestor also terminates this installer. +# A parent PID created after its child has been reused and ends the walk. +function Get-CuaDriverAncestorProcessIds { + [CmdletBinding()] + param() + $ids = @([int]$PID) + try { + $child = Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" -ErrorAction Stop + if (-not $child) { throw "current process is unavailable" } + while ($child) { + $parentId = [int]$child.ParentProcessId + if ($parentId -le 0 -or $ids -contains $parentId) { break } + $parent = Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentId" -ErrorAction Stop + if (-not $parent) { break } + if ($parent.CreationDate -and $child.CreationDate -and $parent.CreationDate -gt $child.CreationDate) { break } + $ids += $parentId + $child = $parent + } + } catch { + throw "Cannot inspect the installer's process ancestry; refusing daemon cleanup: $($_.Exception.Message)" + } + return $ids +} + +function Get-CuaDriverProcesses { + param([int[]]$ExcludeIds = @()) + return @(Get-Process -Name "qwen-cua-driver","qwen-cua-driver-uia" -ErrorAction SilentlyContinue | + Where-Object { $ExcludeIds -notcontains $_.Id }) +} + # Best-effort kill of any running Qwen driver and UIA worker processes # so the next `qwen-cua-driver autostart kick` / `qwen-cua-driver mcp` starts the # FRESH binary, not whatever's still in memory. Without this the @@ -30,26 +57,38 @@ Set-StrictMode -Version Latest # Scheduler runs as SYSTEM so it can kill High-IL processes that # a Medium-IL shell can't. /End on an instance the current user # registered does NOT need admin. -# 2. taskkill /F /IM - Medium-IL backstop for any process that -# wasn't task-attached. -# 3. Returns the surviving process list so callers can warn the user +# Skip this operation when a driver ancestor might own the task. +# 2. taskkill /F /T /PID and Stop-Process - stop non-ancestor drivers. +# 3. Returns only non-ancestor survivors so callers can warn the user # (these are the processes a Medium-IL shell genuinely can't reach # - High-IL daemons whose parent wasn't `cua-driver-serve`). function Stop-CuaDriverDaemons { [CmdletBinding()] param() + $ancestorIds = @(Get-CuaDriverAncestorProcessIds) + $kept = @(Get-CuaDriverProcesses | Where-Object { $ancestorIds -contains $_.Id }) $prevEAP = $ErrorActionPreference $ErrorActionPreference = 'Continue' try { - & schtasks.exe /End /TN "qwen-cua-driver-serve" 2>$null | Out-Null - Start-Sleep -Milliseconds 200 - & taskkill.exe /F /IM "qwen-cua-driver.exe" /T 2>$null | Out-Null - & taskkill.exe /F /IM "qwen-cua-driver-uia.exe" /T 2>$null | Out-Null + if ($kept.Count -eq 0) { + & schtasks.exe /End /TN "qwen-cua-driver-serve" 2>$null | Out-Null + Start-Sleep -Milliseconds 200 + } + foreach ($proc in (Get-CuaDriverProcesses -ExcludeIds $ancestorIds)) { + & taskkill.exe /F /T /PID $proc.Id 2>$null | Out-Null + } + foreach ($proc in (Get-CuaDriverProcesses -ExcludeIds $ancestorIds)) { + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + } } finally { $ErrorActionPreference = $prevEAP } + if ($kept.Count -gt 0) { + $keptIds = ($kept | ForEach-Object { $_.Id }) -join ', ' + Write-Host "Leaving the Qwen driver process(es) that launched this installer running (pid: $keptIds)." -ForegroundColor DarkGray + } Start-Sleep -Milliseconds 200 - return @(Get-Process -Name "qwen-cua-driver","qwen-cua-driver-uia" -ErrorAction SilentlyContinue) + return (Get-CuaDriverProcesses -ExcludeIds $ancestorIds) } # Probe whether `\\.\pipe\qwen-cua-driver` is currently accepting connections. diff --git a/packages/cua-driver/scripts/_install-local-rust.sh b/packages/cua-driver/scripts/_install-local-rust.sh index a173463f7e3..cf2eb3bbfbd 100755 --- a/packages/cua-driver/scripts/_install-local-rust.sh +++ b/packages/cua-driver/scripts/_install-local-rust.sh @@ -313,27 +313,6 @@ if [ -d "$SOURCE_SKILLS" ]; then echo "${GREEN}staged skill pack at $STAGED_SKILLS${NORMAL}" fi -# Keep an already-installed GNOME helper aligned with the source-built driver. -# Installing the helper is still opt-in. Once present, however, leaving old -# compositor artwork behind after install-local creates a misleading -# cross-platform mismatch. -if [ "$OS" = "Linux" ]; then - SOURCE_WAYLAND_HELPER="$REPO_ROOT/../wayland-helper" - if [ -d "$SOURCE_WAYLAND_HELPER/winrects@cua" ]; then - STAGED_WAYLAND_HELPER="$VERSIONED_DIR/wayland-helper" - mkdir -p "$STAGED_WAYLAND_HELPER" - cp -R "$SOURCE_WAYLAND_HELPER/." "$STAGED_WAYLAND_HELPER/" - - INSTALLED_WAYLAND_HELPER="${XDG_DATA_HOME:-$HOME/.local/share}/gnome-shell/extensions/winrects@cua" - if [ -d "$INSTALLED_WAYLAND_HELPER" ]; then - cp "$SOURCE_WAYLAND_HELPER/winrects@cua/metadata.json" \ - "$SOURCE_WAYLAND_HELPER/winrects@cua/extension.js" \ - "$INSTALLED_WAYLAND_HELPER/" - echo "${GREEN}updated installed GNOME helper; reload the GNOME session to activate it${NORMAL}" - fi - fi -fi - # Atomically point `current` at the new versioned release dir. # # Previous version used `ln -s … current.new` + `mv -Tf current.new current` diff --git a/packages/cua-driver/scripts/_install-rust.sh b/packages/cua-driver/scripts/_install-rust.sh index 7915ca5ea6f..7450e006c54 100644 --- a/packages/cua-driver/scripts/_install-rust.sh +++ b/packages/cua-driver/scripts/_install-rust.sh @@ -145,8 +145,8 @@ CHANNEL_EXPLICIT=0 # See packages/cua-driver/rust/scripts/CuaDriverBundle/Contents/Info.plist and # the matching docs on `qwen-cua-driver mcp`'s auto-relaunch behavior. # Identical to the prior Qwen driver's QwenCuaDriver.app + com.qwencode.cua-driver -# pair — the Rust port replaces the Swift install at this path, -# preserving TCC grants (they're keyed on bundle id, which we share). +# pair. Permission continuity also depends on the designated signing +# requirement, which is checked before reporting an upgrade's status. APP_NAME="QwenCuaDriver.app" APP_DEST="/Applications/$APP_NAME" @@ -178,6 +178,27 @@ TMP_DIR=$(mktemp -d) log() { printf '==> %s\n' "$*"; } err() { printf 'error: %s\n' "$*" >&2; } +macos_designated_requirement() { + codesign -d -r- "$1" 2>/dev/null \ + | sed -n -e 's/^designated => //p' -e 's/^# designated => //p' +} + +verify_macos_release_app() { + local app="$1" + local bundle_id + if ! codesign --verify --deep --strict "$app"; then + err "macOS release app failed signature verification: $app" + return 1 + fi + bundle_id=$(/usr/libexec/PlistBuddy -c 'Print :CFBundleIdentifier' \ + "$app/Contents/Info.plist" 2>/dev/null || true) + if [[ "$bundle_id" != "com.qwencode.cua-driver" \ + || ! -x "$app/Contents/MacOS/$BINARY_NAME" ]]; then + err "macOS release app has an unexpected bundle identity or missing executable: $app" + return 1 + fi +} + # --- Concurrent-install lockfile --------------------------------------- # # A second install kicked off while a first is still running can race @@ -209,6 +230,38 @@ LOCK_POLL_INTERVAL_SECONDS=1 LOCK_STALE_AFTER_SECONDS=600 LOCK_HELD=0 +MACOS_APP_SWAP_STARTED=0 +MACOS_APP_HAD_PREVIOUS=0 +MACOS_APP_INSTALL_COMMITTED=0 +MACOS_APP_BACKUP="" +MACOS_DAEMONS_STOPPED=0 + +restore_macos_app_backup_on_exit() { + [[ "$MACOS_APP_SWAP_STARTED" == "1" ]] || return 0 + if [[ "$MACOS_APP_INSTALL_COMMITTED" == "1" ]]; then + rm -rf "$MACOS_APP_BACKUP" \ + || err "could not remove the old macOS install backup: $MACOS_APP_BACKUP" + MACOS_APP_SWAP_STARTED=0 + return 0 + fi + if [[ "$MACOS_APP_HAD_PREVIOUS" == "1" ]]; then + # A failed rename leaves the original path in place. Never remove it + # unless its backup exists; a failed restore must keep the backup. + if [[ -e "$MACOS_APP_BACKUP" || -L "$MACOS_APP_BACKUP" ]]; then + if rm -rf "$APP_DEST" && mv "$MACOS_APP_BACKUP" "$APP_DEST"; then + printf 'warning: restored the previous app after an incomplete installation\n' >&2 + MACOS_APP_SWAP_STARTED=0 + else + err "could not restore the previous app; recoverable backup remains at $MACOS_APP_BACKUP" + fi + fi + else + rm -rf "$APP_DEST" || err "could not remove the incomplete app: $APP_DEST" + MACOS_APP_SWAP_STARTED=0 + fi + return 0 +} + release_install_lock() { if (( LOCK_HELD == 1 )); then rm -rf "$LOCK_DIR" 2>/dev/null || true @@ -220,6 +273,7 @@ release_install_lock() { # clobbers the other. INT/TERM also re-raise via $? so the user-visible # exit code reflects the signal. cleanup_on_exit() { + restore_macos_app_backup_on_exit rm -rf "$TMP_DIR" 2>/dev/null || true release_install_lock } @@ -402,12 +456,8 @@ prune_old_releases() { # under the home. Every step is best-effort + idempotent; a machine with no # prior local install is a clean no-op. # -# TCC is preserved deliberately: we do NOT `tccutil reset` here. The bundle at -# /Applications/QwenCuaDriver.app is shared (bundle id com.qwencode.cua-driver) and the -# subsequent release `ditto` re-points the binary in place; grants keyed on the -# bundle id survive (macOS may re-prompt once on the cdhash change, same as any -# upgrade). Churning the signing identity would gratuitously invalidate -# cert-pinned grants, so we leave it alone. +# Permission records are left unchanged. Bundle identity alone does not prove +# signing-requirement compatibility or that previous grants will remain usable. cleanup_prior_local_install() { local releases_dir="$HOME_DIR/packages/releases" local tcc_marker="$HOME_DIR/.tcc-signing-identity" @@ -431,7 +481,9 @@ cleanup_prior_local_install() { # Stop the local daemon BEFORE we yank its binary out from under it, # mirroring the post-swap stop both installers already do. Best-effort. - stop_cua_driver_daemons + if [[ "$MACOS_DAEMONS_STOPPED" != "1" ]]; then + stop_cua_driver_daemons + fi # Remove the `*-local-*` release dirs. The release install stages into its # own `<version>-<target>` dir and swaps `current` to it, so deleting the @@ -523,7 +575,7 @@ done # asset — see the recovery at the download step below. # # ~~~ BAKED_VERSION: must match the release version in this change ~~~ -CUA_DRIVER_RS_BAKED_VERSION="0.20.11" +CUA_DRIVER_RS_BAKED_VERSION="0.26.0" # ~~~ END_BAKED_VERSION ~~~ # Run API requests with an optional token. Keep the header construction here @@ -840,7 +892,6 @@ case "$LABEL" in *) SRC="$TMP_DIR/$BINARY_NAME" SRC_THEME="$TMP_DIR/cua-cursor-theme" - SRC_WAYLAND_HELPER="$TMP_DIR/wayland-helper" SRC_APP="" ;; esac @@ -877,10 +928,10 @@ fi # --- Install ------------------------------------------------------------ -# Before staging the new release, sweep any prior install-local build that -# shares this home so the release install ends up authoritative (see the -# function definition above for the conservative marker-gated logic). -cleanup_prior_local_install +# Keep macOS local-build state until its replacement has been validated. +if [[ "$OS" != "Darwin" ]]; then + cleanup_prior_local_install +fi mkdir -p "$BIN_DIR" @@ -945,8 +996,7 @@ fi # anchor for both TCC attribution (cdhash + bundle id) and # LaunchServices' `open -a QwenCuaDriver` discovery — symlinking the # .app from /Applications to a versioned dir under $HOME_DIR breaks -# both. The asymmetry is deliberate; rollback on macOS = reinstall -# an older release tag. +# both. A sibling backup protects the previous bundle during replacement. # # Linux: drop the binary into the per-version dir under # $HOME_DIR/packages/releases/<version>-<target>/ and swap the @@ -971,17 +1021,19 @@ if [[ "$OS" == "Darwin" && -n "$SRC_APP" && -d "$SRC_APP" ]]; then err " Without the .app bundle, \`qwen-cua-driver mcp\` from an IDE terminal will not auto-relaunch into a TCC-correct daemon." exit 1 fi - # The Rust port and the legacy Swift driver both live at - # /Applications/QwenCuaDriver.app with bundle id `com.qwencode.cua-driver` — - # bundle-id-identical so TCC grants survive the upgrade. When we - # detect a prior Swift bundle at the install path we log it for - # transparency, but no `tccutil reset` is needed; grants transfer - # automatically because they're keyed on bundle id. macOS may - # surface a one-time re-prompt on first action because the cdhash - # of the new binary doesn't match the old one — that's a TCC - # cdhash-pairing detail, not a grant loss. + if ! command -v codesign >/dev/null 2>&1; then + err "codesign is required to verify the macOS release app; the installed app was not changed" + exit 1 + fi + verify_macos_release_app "$SRC_APP" + STAGED_REQUIREMENT="$(macos_designated_requirement "$SRC_APP")" + if [[ -z "$STAGED_REQUIREMENT" ]]; then + err "could not read the staged signing requirement; the installed app was not changed" + exit 1 + fi REPLACED_SWIFT=0 - if [[ -e "$APP_DEST" ]]; then + REQUIREMENT_COMPATIBILITY="unknown" + if [[ -e "$APP_DEST" || -L "$APP_DEST" ]]; then PREV_BUNDLE_ID=$(/usr/libexec/PlistBuddy -c 'Print :CFBundleIdentifier' "$APP_DEST/Contents/Info.plist" 2>/dev/null || true) PREV_BUNDLE_VERSION=$(/usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' "$APP_DEST/Contents/Info.plist" 2>/dev/null || true) if [[ "$PREV_BUNDLE_ID" == "com.qwencode.cua-driver" ]] && [[ -n "$PREV_BUNDLE_VERSION" ]]; then @@ -990,21 +1042,54 @@ if [[ "$OS" == "Darwin" && -n "$SRC_APP" && -d "$SRC_APP" ]]; then elif [[ -n "$PREV_BUNDLE_ID" ]]; then log "replacing existing $APP_DEST (bundle id $PREV_BUNDLE_ID)" else - log "removing existing $APP_DEST" + log "replacing existing $APP_DEST" fi - rm -rf "$APP_DEST" + if [[ "$PREV_BUNDLE_ID" == "com.qwencode.cua-driver" ]] \ + && codesign --verify --deep --strict "$APP_DEST" >/dev/null 2>&1; then + PREVIOUS_REQUIREMENT="$(macos_designated_requirement "$APP_DEST" || true)" + if [[ -n "$PREVIOUS_REQUIREMENT" ]]; then + if codesign --verify --deep --strict -R "=$PREVIOUS_REQUIREMENT" "$SRC_APP" >/dev/null 2>&1; then + REQUIREMENT_COMPATIBILITY="compatible" + elif [[ "$?" == "3" ]]; then + REQUIREMENT_COMPATIBILITY="incompatible" + fi + fi + fi + fi + MACOS_APP_BACKUP="${APP_DEST}.install-backup.$$" + if [[ -e "$MACOS_APP_BACKUP" || -L "$MACOS_APP_BACKUP" ]]; then + err "macOS backup path already exists: $MACOS_APP_BACKUP" + exit 1 + fi + stop_cua_driver_daemons + show_cua_driver_daemon_survivors + MACOS_DAEMONS_STOPPED=1 + if [[ -e "$APP_DEST" || -L "$APP_DEST" ]]; then + MACOS_APP_HAD_PREVIOUS=1 + fi + MACOS_APP_SWAP_STARTED=1 + if [[ "$MACOS_APP_HAD_PREVIOUS" == "1" ]]; then + mv "$APP_DEST" "$MACOS_APP_BACKUP" fi log "installing $APP_DEST" # `ditto` preserves the bundle's metadata + nested symlinks the way # Apple's installer would. `cp -R` works but doesn't preserve as # much, and ditto is always present on macOS. ditto "$SRC_APP" "$APP_DEST" + verify_macos_release_app "$APP_DEST" + INSTALLED_REQUIREMENT="$(macos_designated_requirement "$APP_DEST")" + if [[ "$INSTALLED_REQUIREMENT" != "$STAGED_REQUIREMENT" ]]; then + err "the copied app did not preserve its verified signing requirement" + exit 1 + fi APP_BINARY="$APP_DEST/Contents/MacOS/$BINARY_NAME" if [[ ! -x "$APP_BINARY" ]]; then err "binary missing at $APP_BINARY (refusing to create broken symlink)" exit 1 fi ln -sf "$APP_BINARY" "$BIN_LINK" + MACOS_APP_INSTALL_COMMITTED=1 + restore_macos_app_backup_on_exit log "symlinked $BIN_LINK -> $APP_BINARY" else # Linux: versioned-dirs + atomic `current` symlink swap. @@ -1029,18 +1114,6 @@ else if [[ "$THEME_AVAILABLE" == "1" ]]; then install -m 0755 "$SRC_THEME" "$VERSIONED_DIR/cua-cursor-theme" fi - if [[ -d "${SRC_WAYLAND_HELPER:-}" ]]; then - mkdir -p "$VERSIONED_DIR/wayland-helper" - cp -R "$SRC_WAYLAND_HELPER/." "$VERSIONED_DIR/wayland-helper/" - - INSTALLED_WAYLAND_HELPER="${XDG_DATA_HOME:-$HOME/.local/share}/gnome-shell/extensions/winrects@cua" - if [[ -d "$INSTALLED_WAYLAND_HELPER" ]]; then - cp "$SRC_WAYLAND_HELPER/winrects@cua/metadata.json" \ - "$SRC_WAYLAND_HELPER/winrects@cua/extension.js" \ - "$INSTALLED_WAYLAND_HELPER/" - log "updated installed GNOME helper; reload the GNOME session to activate it" - fi - fi log "installed $VERSIONED_DIR/$BINARY_NAME (version $VERSION, target $TARGET)" # `ln -sfn` would replace an existing dir-symlink in place but is @@ -1075,6 +1148,10 @@ else prune_old_releases "$RELEASES_DIR" "$CURRENT_LINK" "$TARGET" "$KEEP_VERSIONS" fi +if [[ "$OS" == "Darwin" ]]; then + cleanup_prior_local_install +fi + # --- Sweep the legacy ~/.cua-driver-rs home ----------------------------- # # This release installer used to default HOME_DIR to ~/.cua-driver-rs (the @@ -1110,8 +1187,10 @@ fi # LaunchAgent / systemd user unit / manual `serve` shell keeps serving # pre-upgrade behaviour until logout, which is what surfaces to users # as "the bug I just fixed is still there". -stop_cua_driver_daemons -show_cua_driver_daemon_survivors +if [[ "$MACOS_DAEMONS_STOPPED" != "1" ]]; then + stop_cua_driver_daemons + show_cua_driver_daemon_survivors +fi # Agent skill pack: NOT auto-linked. The install script never touches # ~/.claude/skills/, ~/.agents/skills/, etc. Run `qwen-cua-driver skills @@ -1165,11 +1244,14 @@ echo "" if [[ "${REPLACED_SWIFT:-0}" == "1" ]]; then echo "Upgraded the qwen-cua-driver bundle that was previously at $APP_DEST." - echo "TCC grants (Accessibility, Screen Recording) are keyed on the bundle id" - echo "(com.qwencode.cua-driver) — which is preserved — so they transfer to the new" - echo "binary automatically. macOS may surface a one-time re-grant prompt on" - echo "first action because the new binary's cdhash doesn't match the old" - echo "one's; approve once and the grants persist." + if [[ "$REQUIREMENT_COMPATIBILITY" == "compatible" ]]; then + echo "The replacement satisfies the previous signing requirement." + elif [[ "$REQUIREMENT_COMPATIBILITY" == "incompatible" ]]; then + echo "The signing requirement changed; macOS may require authorization again." + else + echo "Compatibility with the previous signing requirement could not be established." + fi + echo "Existing Accessibility and Screen Recording permission records were left unchanged." echo "" fi diff --git a/packages/cua-driver/scripts/build-node-runtime.mjs b/packages/cua-driver/scripts/build-node-runtime.mjs index 088b345639b..4c562453f7b 100644 --- a/packages/cua-driver/scripts/build-node-runtime.mjs +++ b/packages/cua-driver/scripts/build-node-runtime.mjs @@ -24,17 +24,14 @@ import { writeFileSync, } from "node:fs" import { tmpdir } from "node:os" +import { resolveUbrn } from "./resolve-ubrn.mjs" import { basename, dirname, join, resolve } from "node:path" import { fileURLToPath } from "node:url" const scriptDirectory = dirname(fileURLToPath(import.meta.url)) const driverRoot = resolve(scriptDirectory, "..") const typescriptRoot = join(driverRoot, "typescript") -const upstreamRoot = join( - typescriptRoot, - "node_modules", - "uniffi-bindgen-react-native", -) +const upstreamRoot = resolveUbrn(typescriptRoot) const outputIndex = process.argv.indexOf("--output") if (outputIndex < 0 || !process.argv[outputIndex + 1]) { throw new Error("usage: build-node-runtime.mjs --output <path> [--target <triple>]") @@ -66,7 +63,7 @@ const manifest = JSON.parse(readFileSync(join(typescriptRoot, "package.json"), " const expectedVersion = manifest.devDependencies?.["uniffi-bindgen-react-native"] const upstreamManifestPath = join(upstreamRoot, "package.json") if (!existsSync(upstreamManifestPath)) { - throw new Error("missing pinned UBRN source; run npm ci in packages/cua-driver/typescript") + throw new Error("missing pinned UBRN source; run corepack pnpm install at the repository root") } const actualVersion = JSON.parse(readFileSync(upstreamManifestPath, "utf8")).version if (actualVersion !== expectedVersion) { @@ -170,9 +167,12 @@ try { if (result.error) throw result.error if (result.status !== 0) throw new Error(`cargo exited with status ${result.status}`) + const cargoTargetDirectory = process.env.CARGO_TARGET_DIR + ? resolve(process.env.CARGO_TARGET_DIR) + : join(runtimeRoot, "napi", "target") const targetDirectory = target - ? join(runtimeRoot, "napi", "target", target, "release") - : join(runtimeRoot, "napi", "target", "release") + ? join(cargoTargetDirectory, target, "release") + : join(cargoTargetDirectory, "release") const candidates = [ "libuniffi_runtime_napi.dylib", "libuniffi_runtime_napi.so", diff --git a/packages/cua-driver/scripts/generate-uniffi-bindings.mjs b/packages/cua-driver/scripts/generate-uniffi-bindings.mjs index 42076ec27b1..25113db85a0 100644 --- a/packages/cua-driver/scripts/generate-uniffi-bindings.mjs +++ b/packages/cua-driver/scripts/generate-uniffi-bindings.mjs @@ -14,6 +14,7 @@ import { writeFileSync, } from "node:fs" import { tmpdir } from "node:os" +import { resolveUbrn } from "./resolve-ubrn.mjs" import { dirname, isAbsolute, join, resolve } from "node:path" import { fileURLToPath } from "node:url" @@ -174,15 +175,9 @@ try { ) const typescriptOutput = join(temporaryRoot, "typescript") - const ubrn = join( - typescriptRoot, - "node_modules", - "uniffi-bindgen-react-native", - "bin", - "cli.cjs", - ) + const ubrn = join(resolveUbrn(typescriptRoot), "bin", "cli.cjs") if (!existsSync(ubrn)) { - throw new Error("missing pinned UBRN generator; run npm ci in packages/cua-driver/typescript") + throw new Error("missing pinned UBRN generator; run corepack pnpm install at the repository root") } run( process.execPath, diff --git a/packages/cua-driver/scripts/install-local.ps1 b/packages/cua-driver/scripts/install-local.ps1 index 67c8031d6d5..8409869db08 100644 --- a/packages/cua-driver/scripts/install-local.ps1 +++ b/packages/cua-driver/scripts/install-local.ps1 @@ -309,10 +309,13 @@ Write-Host " exe: $(Join-Path $VisibleBinDir $BinaryName)" Write-Host " source: $installedBinary" Write-Host "" +$AutoStartRegistered = $false + if ($AutoStart) { Write-Step "registering Scheduled Task 'qwen-cua-driver-local-serve'" try { Register-CuaDriverAutostart -InstalledBinary (Join-Path $VisibleBinDir $BinaryName) + $AutoStartRegistered = $true Write-Host " Registered. qwen-cua-driver-local serve auto-starts at every interactive logon." -ForegroundColor Green } catch { @@ -339,6 +342,7 @@ if ($AutoStart) { Write-Step "found existing 'qwen-cua-driver-local-serve' task - re-registering against fresh binary" try { Register-CuaDriverAutostart -InstalledBinary (Join-Path $VisibleBinDir $BinaryName) + $AutoStartRegistered = $true Write-Host " Re-registered. Task action now uses this build's hidden-console wrapper." -ForegroundColor Green } catch { @@ -388,8 +392,8 @@ if (-not (Test-Path -LiteralPath $releaseBinary -PathType Leaf)) { } # Windows-specific autostart hint (kept inline; per-shell natural location). -if ($AutoStart) { - # Default branch: autostart was enabled (either by default or explicitly). +if ($AutoStartRegistered) { + # Registration succeeded for this installation. # Surface the management subcommands so the user knows how to inspect / # disable later without digging through Task Scheduler. Write-Host "" @@ -399,9 +403,8 @@ if ($AutoStart) { Write-Host " qwen-cua-driver-local autostart kick (start now without re-logging)" -ForegroundColor Cyan Write-Host "" } else { - # Opt-out branch (-NoAutoStart or -AutoStart:`$false`). Write-Host "" - Write-Host "Auto-start at logon (NOT enabled - re-run without -NoAutoStart to register, or:):" -ForegroundColor Cyan + Write-Host "Auto-start for this installation was not confirmed. To register:" -ForegroundColor Cyan Write-Host " qwen-cua-driver-local autostart enable (register Scheduled Task at RunLevel=Highest)" -ForegroundColor Cyan Write-Host " qwen-cua-driver-local autostart kick (start now without re-logging)" -ForegroundColor Cyan Write-Host " qwen-cua-driver-local autostart status (inspect)" -ForegroundColor Cyan diff --git a/packages/cua-driver/scripts/install.ps1 b/packages/cua-driver/scripts/install.ps1 index 70de2486dca..1c40a781764 100644 --- a/packages/cua-driver/scripts/install.ps1 +++ b/packages/cua-driver/scripts/install.ps1 @@ -94,6 +94,11 @@ param( # `-NoAutoStart` is the explicit opt-out and takes precedence over # the default-true `-AutoStart`. if ($NoAutoStart) { $AutoStart = $false } +$AutoStartRequested = [bool]$AutoStart -and + (Test-Path variable:PSBoundParameters) -and + $PSBoundParameters.ContainsKey('AutoStart') +$SkipIsolatedAutostart = [bool]($env:CUA_DRIVER_RS_INSTALL_DIR -or $env:CUA_DRIVER_RS_HOME) -and + -not $AutoStartRequested Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" @@ -118,7 +123,7 @@ $ThemeBinaryName = "cua-cursor-theme.exe" # where the baked line hasn't been updated yet. # # ~~~ BAKED_VERSION: must match the release version in this change ~~~ -$Script:CuaDriverRsBakedVersion = "0.20.11" +$Script:CuaDriverRsBakedVersion = "0.26.0" # ~~~ END_BAKED_VERSION ~~~ $CursorThemeRequiredFrom = [version]"0.12.7" $LegacyUiaRequiredFrom = [version]"0.2.8" @@ -538,8 +543,8 @@ function Test-IsElevated { } # Stop-CuaDriverDaemons + Show-CuaDriverDaemonSurvivors live in the -# sibling _install-common.psm1 module so install-local.ps1 and this -# script share the daemon-cleanup logic. Two load paths: +# sibling _install-common.psm1 module, shared by regular and legacy +# cleanup. Two load paths: # * checked-out tree: the .psm1 sits next to install.ps1 on disk; # Import-Module from $PSScriptRoot works directly. # * `irm | iex` install: no file on disk, $PSScriptRoot is empty. @@ -1209,23 +1214,6 @@ Write-Step "cua-driver-rs installer (Windows)" Write-Step " install dir : $VisibleBinDir" Write-Step " package home: $HomeDir" -function Get-AncestorProcessIds { - # PIDs of this process and its ancestors, so cleanup never kills the caller. - # `qwen-cua-driver update --apply` runs the installer as a child of - # qwen-cua-driver.exe. - $ids = @() - $seen = @{} - $currentPid = $PID - while ($currentPid -and -not $seen.ContainsKey([int]$currentPid)) { - $seen[[int]$currentPid] = $true - $ids += $currentPid - $proc = Get-CimInstance Win32_Process -Filter "ProcessId=$currentPid" -ErrorAction SilentlyContinue - if (-not $proc -or -not $proc.ParentProcessId) { break } - $currentPid = $proc.ParentProcessId - } - return $ids -} - function Remove-LegacyInstall { # Best-effort cleanup of v0.2.13-and-earlier install paths. Runs before # any new install when default paths are in use (so users who override @@ -1248,55 +1236,9 @@ function Remove-LegacyInstall { Write-Step "detected legacy Qwen install layout; migrating to Qwen\qwen-cua-driver" - # 1. End the running daemon. Order matters: - # - # a. `schtasks /End` first — Task Scheduler runs as SYSTEM and can - # terminate elevated (RunLevel=Highest, High IL) processes that a - # Medium-IL Stop-Process from the user's shell cannot. This is - # the case any time the legacy daemon was spawned via the - # AtLogon trigger of the v0.2.13 autostart task on a non-RID-500 - # admin account (e.g. cuademo). Without this, Step 4 below fails - # with "Access to the path 'cua-driver.exe' is denied" because - # the legacy binary is held open by an unkillable elevated - # process. Discovered during the cuademo v0.2.13 → v0.2.14 - # migration dogfood. - # b. taskkill /F /IM as a backstop for any qwen-cua-driver process that - # wasn't task-attached (manual `qwen-cua-driver serve`, legacy uia - # worker, etc.). taskkill is more permissive than Stop-Process - # for cross-IL termination. - # c. Stop-Process last — catches anything taskkill missed. - $prevEAP = $ErrorActionPreference - $ErrorActionPreference = 'Continue' - # Never kill the process tree we are running inside: when the installer is - # launched by `qwen-cua-driver update --apply`, our own parent IS a - # qwen-cua-driver.exe, and an unfiltered kill terminates the update mid-flight - # (the script dies right after the step banner above). Both cleanup passes - # below skip these. - $ancestorPids = @(Get-AncestorProcessIds) - try { - # Ends the running task instance. Returns non-zero when the task - # isn't running or doesn't exist, both of which we swallow. - & schtasks.exe /End /TN "qwen-cua-driver-serve" 2>$null | Out-Null - Start-Sleep -Milliseconds 250 - # Force-kill via taskkill — handles High-IL processes that - # Stop-Process can't touch from a Medium-IL caller. - $selfFilters = @() - foreach ($ancestorPid in $ancestorPids) { - $selfFilters += '/FI' - $selfFilters += "PID ne $ancestorPid" - } - & taskkill.exe /F /IM "qwen-cua-driver.exe" /T @selfFilters 2>$null | Out-Null - & taskkill.exe /F /IM "qwen-cua-driver-uia.exe" /T @selfFilters 2>$null | Out-Null - } finally { - $ErrorActionPreference = $prevEAP - } - $procs = Get-Process -Name "qwen-cua-driver","qwen-cua-driver-uia" -ErrorAction SilentlyContinue - if ($procs) { - foreach ($p in $procs) { - if ($ancestorPids -contains $p.Id) { continue } - try { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue } catch {} - } - } + # The shared cleanup protects the updater and this installer in both + # legacy migration and the normal post-install restart path. + $null = Stop-CuaDriverDaemons Start-Sleep -Milliseconds 500 # 2. Unregister the autostart Scheduled Task if present. Idempotent — @@ -1387,8 +1329,8 @@ Write-Step " target : $target" $Script:CuaDriverRsSelectedChannel = Resolve-SelectedChannel $version = Resolve-Version $versionedDir = Join-Path $ReleasesDir "$version-$target" -$requiresSignedUia = [version]$version -ge $SignedUiaRequiredFrom -$requiresLegacyUia = [version]$version -ge $LegacyUiaRequiredFrom -and -not $requiresSignedUia +$requiresSignedUia = [version]$version -ge $SignedUiaRequiredFrom -and ([version]$version -lt [version]"0.26.0" -or $env:CUA_DRIVER_USE_UIACCESS -eq "1") +$requiresLegacyUia = [version]$version -ge $LegacyUiaRequiredFrom -and [version]$version -lt $SignedUiaRequiredFrom # If the per-version dir is already populated, skip the download — the # binary is immutable per version, so a repeat install is a no-op apart @@ -1422,8 +1364,8 @@ if (-not $skipDownload) { if ($asset.Version -ne $version) { $version = $asset.Version $versionedDir = Join-Path $ReleasesDir "$version-$target" - $requiresSignedUia = [version]$version -ge $SignedUiaRequiredFrom - $requiresLegacyUia = [version]$version -ge $LegacyUiaRequiredFrom -and -not $requiresSignedUia + $requiresSignedUia = [version]$version -ge $SignedUiaRequiredFrom -and ([version]$version -lt [version]"0.26.0" -or $env:CUA_DRIVER_USE_UIACCESS -eq "1") + $requiresLegacyUia = [version]$version -ge $LegacyUiaRequiredFrom -and [version]$version -lt $SignedUiaRequiredFrom $uiaSecureDir = Join-Path $env:ProgramFiles "Qwen\CuaDriver\$version" $uiaSecurePath = Join-Path $uiaSecureDir 'qwen-cua-driver-uia.exe' } @@ -1448,7 +1390,7 @@ if (-not $skipDownload) { } Write-Step "installed $versionedDir\$BinaryName (version $version, target $target)" } - if (-not $requiresSignedUia) { + if ($requiresLegacyUia) { foreach ($legacyUiaName in @('qwen-cua-driver-uia.exe', 'cua-driver-uia.exe')) { $legacyUiaStage = Join-Path $stageDir $legacyUiaName if (Test-Path -LiteralPath $legacyUiaStage) { @@ -1459,9 +1401,9 @@ if (-not $skipDownload) { } } if ($requiresSignedUia) { - # The UIAccess worker is required for foreground acquisition and - # input delivery. Windows only grants uiAccess to a signed binary - # installed under a secure system path. + # Historical releases require UIAccess. Current releases use it + # only after explicit opt-in; Windows still requires a trusted + # signature and the secure Program Files path. $uiaStage = Join-Path $stageDir 'qwen-cua-driver-uia.exe' if (-not (Test-Path -LiteralPath $uiaStage)) { throw "release archive is missing required qwen-cua-driver-uia.exe" @@ -1705,20 +1647,22 @@ else { # there" because the user's in-memory code is pre-fix. Best-effort: # High-IL daemons from the RunLevel=Highest autostart task survive a # Medium-IL kill and get reported via Show-CuaDriverDaemonSurvivors. -Write-Host "" +if (-not $SkipIsolatedAutostart) { + Write-Host "" Write-Host "Stopping any previous qwen-cua-driver processes (best-effort; High-IL needs admin)..." -ForegroundColor Cyan -# Repair- handles the wedged-daemon case: detect stale (process alive -# but pipe dead), prompt the user, and on consent self-elevate via -# UAC to kill the High-IL pids + restart the scheduled task. On UAC -# cancel / failure it falls back to printing the manual recovery -# instructions, same as the previous behavior. -$null = Repair-CuaDriverStaleDaemon + $null = Repair-CuaDriverStaleDaemon +} -if ($AutoStart) { +$AutoStartRegistered = $false +if ($SkipIsolatedAutostart) { + Write-Host "Isolated install: leaving the shared daemon and autostart task unchanged." -ForegroundColor Yellow + Write-Host " Pass -AutoStart explicitly to point the shared task at this installation." +} elseif ($AutoStart) { Write-Host "" Write-Host "Registering auto-start (qwen-cua-driver autostart enable)..." -ForegroundColor Cyan try { Register-CuaDriverAutostart -InstalledBinary $installedBinary + $AutoStartRegistered = $true Write-Host " qwen-cua-driver serve will auto-start at every interactive logon (RunLevel=Highest)." -ForegroundColor Green } catch { @@ -1746,6 +1690,7 @@ if ($AutoStart) { Write-Host "Existing 'qwen-cua-driver-serve' autostart task detected - re-registering against the fresh binary..." -ForegroundColor Cyan try { Register-CuaDriverAutostart -InstalledBinary $installedBinary + $AutoStartRegistered = $true Write-Host " Re-registered. Task action now uses this build's hidden-console wrapper." -ForegroundColor Green } catch { @@ -1774,13 +1719,15 @@ catch { # Windows-specific autostart hint (kept inline; OS-natural location). Write-Host "" -if ($AutoStart) { +if ($SkipIsolatedAutostart) { + Write-Host "Auto-start: unchanged for this isolated installation." -ForegroundColor Cyan +} elseif ($AutoStartRegistered) { Write-Host "Auto-start: 'qwen-cua-driver-serve' is registered at RunLevel=Highest." -ForegroundColor Cyan Write-Host " qwen-cua-driver autostart status (inspect)" -ForegroundColor Cyan Write-Host " qwen-cua-driver autostart disable (remove)" -ForegroundColor Cyan Write-Host " qwen-cua-driver autostart kick (start now without re-logging)" -ForegroundColor Cyan } else { - Write-Host "Auto-start at logon (NOT enabled - re-run without -NoAutoStart to register, or:):" -ForegroundColor Cyan + Write-Host "Auto-start for this installation was not confirmed. To register:" -ForegroundColor Cyan Write-Host " qwen-cua-driver autostart enable (Scheduled Task at RunLevel=Highest)" -ForegroundColor Cyan Write-Host " qwen-cua-driver autostart kick (start now without re-logging)" -ForegroundColor Cyan Write-Host " qwen-cua-driver autostart status (inspect)" -ForegroundColor Cyan diff --git a/packages/cua-driver/scripts/install.sh b/packages/cua-driver/scripts/install.sh index 3dfde5b6379..c53083ad466 100755 --- a/packages/cua-driver/scripts/install.sh +++ b/packages/cua-driver/scripts/install.sh @@ -16,8 +16,8 @@ # --backend=swift retired Swift backend (no-op; accepted for compat) # # Env overrides: -# CUA_DRIVER_RS_VERSION=0.20.11 pin a specific Rust release tag -# CUA_DRIVER_VERSION=0.20.11 legacy alias for CUA_DRIVER_RS_VERSION +# CUA_DRIVER_RS_VERSION=0.26.0 pin a specific Rust release tag +# CUA_DRIVER_VERSION=0.26.0 legacy alias for CUA_DRIVER_RS_VERSION # CUA_DRIVER_RS_INSTALL_DIR=PATH same as --bin-dir # CUA_DRIVER_BIN_DIR=PATH legacy alias for --bin-dir # CUA_DRIVER_NO_MODIFY_PATH=1 same as --no-modify-path diff --git a/packages/cua-driver/scripts/resolve-ubrn.mjs b/packages/cua-driver/scripts/resolve-ubrn.mjs new file mode 100644 index 00000000000..6db1880c96e --- /dev/null +++ b/packages/cua-driver/scripts/resolve-ubrn.mjs @@ -0,0 +1,21 @@ +import { createRequire } from 'node:module'; +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +export function resolveUbrn(typescriptRoot) { + const require = createRequire(join(typescriptRoot, 'package.json')); + let root = dirname(require.resolve('uniffi-bindgen-react-native')); + const expected = JSON.parse(readFileSync(join(typescriptRoot, 'package.json'), 'utf8')).devDependencies['uniffi-bindgen-react-native']; + while (dirname(root) !== root) { + const manifest = join(root, 'package.json'); + if (existsSync(manifest)) { + const metadata = JSON.parse(readFileSync(manifest, 'utf8')); + if (metadata.name === 'uniffi-bindgen-react-native') { + if (metadata.version !== expected) throw new Error(`UBRN version mismatch: expected ${expected}, found ${metadata.version}`); + return root; + } + } + root = dirname(root); + } + throw new Error('Pinned UBRN source not found; run corepack pnpm install at the repository root.'); +} diff --git a/packages/cua-driver/scripts/tests/install-windows-regression.ps1 b/packages/cua-driver/scripts/tests/install-windows-regression.ps1 index e28d2a9727e..4e15fa81a31 100644 --- a/packages/cua-driver/scripts/tests/install-windows-regression.ps1 +++ b/packages/cua-driver/scripts/tests/install-windows-regression.ps1 @@ -1,11 +1,20 @@ [CmdletBinding()] param( - [string]$InstallerPath = (Join-Path (Split-Path -Parent $PSScriptRoot) "install.ps1") + [string]$InstallerPath = (Join-Path (Split-Path -Parent $PSScriptRoot) "install.ps1"), + [switch]$UnitOnly ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" +# Legacy detection/cleanup tests must never rewrite the real user's PATH. +class InstallerTestEnvironment { + static [string] GetEnvironmentVariable([string]$name, [string]$target) { return '' } + static [void] SetEnvironmentVariable([string]$name, [string]$value, [string]$target) { + throw 'Persistent environment writes are forbidden in installer unit tests' + } +} + function Assert-True { param( [Parameter(Mandatory = $true)][bool]$Condition, @@ -26,9 +35,14 @@ function Import-InstallerFunction { }, $true) | Select-Object -First 1 if (-not $definition) { throw "function $Name not found in $InstallerPath" } $scriptDefinition = $definition.Extent.Text -replace "^function\s+$([regex]::Escape($Name))", "function script:$Name" + if ($Name -eq 'Remove-LegacyInstall') { + $scriptDefinition = $scriptDefinition.Replace('[Environment]::', '[InstallerTestEnvironment]::') + } . ([scriptblock]::Create($scriptDefinition)) } +& (Join-Path $PSScriptRoot 'uninstall-windows-regression.ps1') -UninstallerPath (Join-Path (Split-Path -Parent $InstallerPath) 'uninstall.ps1') + $tokens = $null $parseErrors = $null $ast = [System.Management.Automation.Language.Parser]::ParseFile( @@ -39,8 +53,14 @@ if ($parseErrors.Count -ne 0) { throw "install.ps1 parse errors: $($parseErrors -join '; ')" } -Import-InstallerFunction -Ast $ast -Name "Get-AncestorProcessIds" Import-InstallerFunction -Ast $ast -Name "Remove-LegacyInstall" +$modulePath = Join-Path (Split-Path -Parent $InstallerPath) '_install-common.psm1' +$moduleAst = [System.Management.Automation.Language.Parser]::ParseFile( + $modulePath, [ref]$tokens, [ref]$parseErrors) +if ($parseErrors.Count -ne 0) { throw "module parse errors: $($parseErrors -join '; ')" } +foreach ($name in @('Get-CuaDriverAncestorProcessIds', 'Get-CuaDriverProcesses', 'Stop-CuaDriverDaemons', 'Stop-CuaDriverDaemonsWithHealth')) { + Import-InstallerFunction -Ast $moduleAst -Name $name +} $script:Steps = @() $script:TaskkillCalls = @() @@ -48,25 +68,30 @@ $script:ScheduledTaskCalls = @() $script:StoppedProcessIds = @() $script:Processes = @() $script:ParentByPid = @{} +$script:CreatedByPid = @{} +$script:QueryThrows = $false function Write-Step { param($Message); $script:Steps += [string]$Message } function Start-Sleep { [CmdletBinding()] param([int]$Milliseconds, [int]$Seconds) } function taskkill.exe { $script:TaskkillCalls += ,@($args); $global:LASTEXITCODE = 0 } function schtasks.exe { $script:ScheduledTaskCalls += ,@($args); $global:LASTEXITCODE = 0 } +function Test-CuaDriverPipeAlive { return $false } function Get-CimInstance { [CmdletBinding()] param([string]$ClassName, [string]$Filter) + if ($script:QueryThrows) { throw 'controlled provider failure' } $processId = [int]($Filter -replace '^ProcessId=', '') if (-not $script:ParentByPid.ContainsKey($processId)) { return $null } [pscustomobject]@{ ProcessId = $processId ParentProcessId = $script:ParentByPid[$processId] + CreationDate = $script:CreatedByPid[$processId] } } function Get-Process { [CmdletBinding()] param([string[]]$Name) - return $script:Processes + return @($script:Processes | Where-Object { $Name -contains $_.ProcessName }) } function Stop-Process { [CmdletBinding()] @@ -120,7 +145,7 @@ try { "$marker was not detected as a legacy marker" } - # Model update --apply: PowerShell is a child of cua-driver.exe, which in + # Model update --apply: PowerShell is a child of qwen-cua-driver.exe, which in # turn has another ancestor. Both must be excluded from taskkill and the # Stop-Process backstop, while an unrelated daemon must still be stopped. $HomeDir = Join-Path $root "process-tree\modern-home" @@ -137,25 +162,59 @@ try { $script:ParentByPid[4100] = 4200 $script:ParentByPid[4200] = 0 $script:Processes = @( - [pscustomobject]@{ Id = 4100; ProcessName = "cua-driver" }, - [pscustomobject]@{ Id = 4300; ProcessName = "cua-driver" } + [pscustomobject]@{ Id = 4100; ProcessName = "qwen-cua-driver" }, + [pscustomobject]@{ Id = 4300; ProcessName = "qwen-cua-driver" } ) - $ancestors = @(Get-AncestorProcessIds) + $ancestors = @(Get-CuaDriverAncestorProcessIds) Assert-True (($ancestors -join ',') -eq "$PID,4100,4200") "ancestor discovery did not traverse the complete synthetic chain" Remove-LegacyInstall - Assert-True ($script:TaskkillCalls.Count -eq 2) "expected taskkill calls for driver and UIA processes" + Assert-True ($script:TaskkillCalls.Count -eq 1) "expected only the unrelated driver PID to be targeted" foreach ($call in $script:TaskkillCalls) { $commandLine = $call -join ' ' - foreach ($protectedPid in @($PID, 4100, 4200)) { - Assert-True ($commandLine -match "PID ne $protectedPid(?: |$)") ` - "taskkill did not exclude protected ancestor PID $protectedPid" - } + Assert-True ($commandLine -eq '/F /T /PID 4300') "cleanup did not target only the unrelated driver PID" } + Assert-True (-not (@($script:ScheduledTaskCalls | Where-Object { $_ -contains '/End' }).Count)) ` + "task-wide termination can kill the updater ancestor" Assert-True (-not ($script:StoppedProcessIds -contains 4100)) "Stop-Process targeted the cua-driver launcher" Assert-True ($script:StoppedProcessIds -contains 4300) "unrelated cua-driver daemon was not stopped" + $script:Processes = @([pscustomobject]@{ Id = 4100; ProcessName = 'qwen-cua-driver' }) + $health = Stop-CuaDriverDaemonsWithHealth + Assert-True (@($health.Survivors).Count -eq 0 -and -not $health.Stale) ` + "the updater ancestor was reported as a stale daemon" + + $script:Processes += [pscustomobject]@{ Id = 4300; ProcessName = 'qwen-cua-driver' } + $health = Stop-CuaDriverDaemonsWithHealth + Assert-True (@($health.Survivors).Count -eq 1 -and $health.Survivors.Id -eq 4300 -and $health.Stale) ` + "a surviving unrelated daemon was not reported as stale" + + $script:CreatedByPid[[int]$PID] = [datetime]'2026-01-02' + $script:CreatedByPid[4100] = [datetime]'2026-01-03' + Assert-True ((@(Get-CuaDriverAncestorProcessIds) -join ',') -eq "$PID") ` + "a reused parent PID was treated as a live ancestor" + $script:CreatedByPid[4100] = [datetime]'2026-01-01' + $script:ParentByPid[4200] = 4100 + Assert-True ((@(Get-CuaDriverAncestorProcessIds) -join ',') -eq "$PID,4100,4200") ` + "a cycle did not terminate ancestor discovery" + $script:ParentByPid.Remove(4100) + Assert-True ((@(Get-CuaDriverAncestorProcessIds) -join ',') -eq "$PID") ` + "an exited parent was included as an ancestor" + + foreach ($failure in @('provider', 'current-process-missing')) { + $script:TaskkillCalls = @() + $script:ScheduledTaskCalls = @() + $script:StoppedProcessIds = @() + $script:QueryThrows = ($failure -eq 'provider') + if ($failure -eq 'current-process-missing') { $script:ParentByPid.Remove([int]$PID) } + $refused = $false + try { $null = Stop-CuaDriverDaemons } catch { $refused = $true } + Assert-True $refused "$failure did not refuse cleanup" + Assert-True ($script:TaskkillCalls.Count -eq 0 -and $script:ScheduledTaskCalls.Count -eq 0 -and $script:StoppedProcessIds.Count -eq 0) ` + "$failure attempted termination without known ancestors" + } + Write-Host "Windows installer legacy/update regression checks passed." } finally { @@ -163,15 +222,17 @@ finally { $env:CUA_DRIVER_RS_HOME = $savedHome } +if ($UnitOnly) { return } + $rustWorkspace = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) "rust" Push-Location $rustWorkspace try { - & cargo test -p cua-driver --bin cua-driver 'autostart::tests::' --locked -- --nocapture + & cargo test -p cua-driver --bin qwen-cua-driver 'autostart::tests::' --locked -- --nocapture if ($LASTEXITCODE -ne 0) { throw "Windows autostart path tests failed with exit $LASTEXITCODE" } - & cargo build -p cua-driver --bin cua-driver --locked + & cargo build -p cua-driver --bin qwen-cua-driver --locked if ($LASTEXITCODE -ne 0) { throw "Windows autostart task probe build failed with exit $LASTEXITCODE" } @@ -190,14 +251,14 @@ $taskReleaseV1 = Join-Path $taskPackages "releases\probe-v1" $taskReleaseV2 = Join-Path $taskPackages "releases\probe-v2" $taskCurrent = Join-Path $taskPackages "current" $taskVisible = Join-Path $taskProbeRoot "bin" -$taskVisibleExe = Join-Path $taskVisible "cua-driver.exe" +$taskVisibleExe = Join-Path $taskVisible "qwen-cua-driver.exe" $taskMarker = Join-Path $taskProbeRoot "probe-v2.txt" $realSchtasks = Join-Path $env:SystemRoot "System32\schtasks.exe" try { New-Item -ItemType Directory -Force -Path $taskReleaseV1, $taskReleaseV2 | Out-Null - Copy-Item -LiteralPath (Join-Path $rustWorkspace "target\debug\cua-driver.exe") ` - -Destination (Join-Path $taskReleaseV1 "cua-driver.exe") + Copy-Item -LiteralPath (Join-Path $rustWorkspace "target\debug\qwen-cua-driver.exe") ` + -Destination (Join-Path $taskReleaseV1 "qwen-cua-driver.exe") $markerLiteral = $taskMarker.Replace('\', '\\').Replace('"', '\"') $probeSourcePath = Join-Path $taskProbeRoot "probe-v2.rs" @@ -207,7 +268,7 @@ fn main() { } "@ Set-Content -LiteralPath $probeSourcePath -Value $probeSource -Encoding UTF8 - & rustc $probeSourcePath -o (Join-Path $taskReleaseV2 "cua-driver.exe") + & rustc $probeSourcePath -o (Join-Path $taskReleaseV2 "qwen-cua-driver.exe") if ($LASTEXITCODE -ne 0) { throw "probe-v2 build failed with exit $LASTEXITCODE" } diff --git a/packages/cua-driver/scripts/tests/test_install_local.py b/packages/cua-driver/scripts/tests/test_install_local.py index c10d28a8f79..08befe30f28 100644 --- a/packages/cua-driver/scripts/tests/test_install_local.py +++ b/packages/cua-driver/scripts/tests/test_install_local.py @@ -49,11 +49,6 @@ def test_installer_stages_binary_from_custom_cargo_target( shutil.copy2(INSTALL_LOCAL, scripts_dir / INSTALL_LOCAL.name) shutil.copy2(LOCAL_SIGNING, scripts_dir / LOCAL_SIGNING.name) - wayland_helper = fixture_root / "wayland-helper/winrects@cua" - wayland_helper.mkdir(parents=True) - (wayland_helper / "metadata.json").write_text('{"version":5}\n', encoding="utf-8") - (wayland_helper / "extension.js").write_text("// semantic cursor v5\n", encoding="utf-8") - stale_binary = rust_dir / "target/release/qwen-cua-driver" _write_executable(stale_binary, "printf 'stale workspace target\\n'") @@ -90,10 +85,6 @@ esac local_home = tmp_path / "local-home" user_home = tmp_path / "home" - installed_helper = user_home / ".local/share/gnome-shell/extensions/winrects@cua" - installed_helper.mkdir(parents=True) - (installed_helper / "metadata.json").write_text('{"version":4}\n', encoding="utf-8") - (installed_helper / "extension.js").write_text("// legacy cursor\n", encoding="utf-8") install_bin = tmp_path / "install-bin" env = os.environ.copy() env.pop("SUDO_USER", None) @@ -127,11 +118,6 @@ esac assert ( local_home / "packages/current/cua-cursor-theme" ).read_text() == "fresh cursor theme compiler\n" - assert ( - local_home / "packages/current/wayland-helper/winrects@cua/metadata.json" - ).read_text() == '{"version":5}\n' - assert (installed_helper / "metadata.json").read_text() == '{"version":5}\n' - assert (installed_helper / "extension.js").read_text() == "// semantic cursor v5\n" def _linux_fixture(tmp_path: Path) -> tuple[Path, Path, dict[str, str]]: diff --git a/packages/cua-driver/scripts/tests/test_install_macos_recovery.py b/packages/cua-driver/scripts/tests/test_install_macos_recovery.py new file mode 100644 index 00000000000..b2460356544 --- /dev/null +++ b/packages/cua-driver/scripts/tests/test_install_macos_recovery.py @@ -0,0 +1,130 @@ +from pathlib import Path +import os +import shlex +import subprocess +import tempfile +import unittest + + +SOURCE = (Path(__file__).resolve().parents[1] / '_install-rust.sh').read_text() + + +def function(name): + start = SOURCE.index(f'{name}() {{\n') + return SOURCE[start:SOURCE.index('\n}\n', start) + 3] + + +class MacOSInstallRecoveryTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + + def run_install(self, mode, previous=True): + root = Path(self.temp.name) / mode + app = root / 'Applications/QwenCuaDriver.app' + stage = root / 'staging/QwenCuaDriver.app' + bin_dir = root / 'bin' + bin_dir.mkdir(parents=True) + for command in ('rm', 'mv', 'ln', 'cp', 'sed'): + system = '/usr/bin' if command == 'sed' else '/bin' + (bin_dir / command).symlink_to(f'{system}/{command}') + stage_binary = stage / 'Contents/MacOS/qwen-cua-driver' + stage_binary.parent.mkdir(parents=True) + stage_binary.write_bytes(b'new release') + stage_binary.chmod(0o755) + app.parent.mkdir() + old_binary = app / 'Contents/MacOS/qwen-cua-driver' + link = bin_dir / 'qwen-cua-driver' + if previous: + old_binary.parent.mkdir(parents=True) + old_binary.write_bytes(b'previous release') + old_binary.chmod(0o755) + link.symlink_to(old_binary) + start = SOURCE.index('LOCK_HELD=0\n') + lifecycle = SOURCE[start:SOURCE.index('acquire_install_lock() {', start)] + start = SOURCE.index('if [[ "$OS" == "Darwin" && -n "$SRC_APP" && -d "$SRC_APP" ]]; then') + branch = SOURCE[start:SOURCE.index('\nelse\n # Linux: versioned-dirs', start)] + '\nfi\n' + self.assertEqual(branch.count('! -w "/Applications"'), 1) + branch = branch.replace('! -w "/Applications"', '! -w "$TEST_ROOT/Applications"') + helpers = '\n'.join(function(name) for name in ( + 'macos_designated_requirement', 'verify_macos_release_app', + )) + script = '\n'.join([ + 'set -euo pipefail', + f'TEST_ROOT={shlex.quote(str(root))}', + f'APP_DEST={shlex.quote(str(app))}', + f'SRC_APP={shlex.quote(str(stage))}', + f'TMP_DIR={shlex.quote(str(stage.parent))}', + f'BIN_LINK={shlex.quote(str(link))}', + 'OS=Darwin', 'BINARY_NAME=qwen-cua-driver', + 'log() { :; }', 'err() { printf "%s\\n" "$*" >&2; }', + 'stop_cua_driver_daemons() { :; }', + 'show_cua_driver_daemon_survivors() { :; }', + '''fixture_plist() { printf 'com.qwencode.cua-driver\\n'; } +codesign() { + local app="${@: -1}" + if [[ "$1" == "-d" ]]; then + printf 'designated => fixture-requirement\\n' + elif [[ "$TEST_MODE" == "stage-invalid" && "$app" == "$SRC_APP" ]]; then + return 1 + elif [[ "$TEST_MODE" == "copy-invalid" && "$app" == "$APP_DEST" ]]; then + return 1 + fi +} +ditto() { + if [[ "$TEST_MODE" == "copy-before" ]]; then return 73; fi + command cp -R "$1" "$2" + if [[ "$TEST_MODE" == "copy-partial" || "$TEST_MODE" == "restore-fails" ]]; then + command rm "$2/Contents/MacOS/qwen-cua-driver" + return 73 + fi +} +mv() { + if [[ "$TEST_MODE" == "restore-fails" && "$1" == "$APP_DEST".install-backup.* ]]; then + return 74 + fi + command mv "$@" +}''', + helpers, lifecycle, branch, + ]).replace('/usr/libexec/PlistBuddy', 'fixture_plist') + result = subprocess.run( + ['/bin/bash', '-c', script], + env={'PATH': str(bin_dir), 'TEST_MODE': mode}, + text=True, capture_output=True, timeout=5, + ) + return result, app, old_binary, link + + def test_copy_failures_restore_the_working_old_bundle(self): + for mode in ('copy-before', 'copy-partial', 'copy-invalid', 'stage-invalid'): + with self.subTest(mode=mode): + result, app, binary, link = self.run_install(mode) + self.assertNotEqual(result.returncode, 0, result.stderr) + self.assertEqual(binary.read_bytes(), b'previous release') + self.assertEqual(link.read_bytes(), b'previous release') + self.assertEqual(list(app.parent.glob('*.install-backup.*')), []) + + def test_failed_restore_preserves_a_recoverable_backup(self): + result, app, _, _ = self.run_install('restore-fails') + self.assertEqual(result.returncode, 73, result.stderr) + backups = list(app.parent.glob('*.install-backup.*')) + self.assertEqual(len(backups), 1) + self.assertEqual((backups[0] / 'Contents/MacOS/qwen-cua-driver').read_bytes(), b'previous release') + self.assertIn(str(backups[0]), result.stderr) + + def test_first_install_failure_removes_the_partial_bundle(self): + result, app, _, link = self.run_install('copy-partial', previous=False) + self.assertEqual(result.returncode, 73, result.stderr) + self.assertFalse(app.exists()) + self.assertFalse(link.is_symlink()) + + def test_success_installs_the_new_bundle_and_removes_backup(self): + result, app, binary, link = self.run_install('success') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(binary.read_bytes(), b'new release') + self.assertEqual(link.read_bytes(), b'new release') + self.assertTrue(os.access(link, os.X_OK)) + self.assertEqual(list(app.parent.glob('*.install-backup.*')), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/packages/cua-driver/scripts/tests/test_uninstall_claude_ownership.py b/packages/cua-driver/scripts/tests/test_uninstall_claude_ownership.py new file mode 100644 index 00000000000..ecf64510e84 --- /dev/null +++ b/packages/cua-driver/scripts/tests/test_uninstall_claude_ownership.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess + +import pytest + + +SCRIPT = Path(__file__).resolve().parents[1] / 'uninstall.sh' +SOURCE = SCRIPT.read_text(encoding='utf-8') + + +def _function(name: str) -> str: + start = SOURCE.index(f'{name}() {{\n') + return SOURCE[start:SOURCE.index('\n}\n', start) + 3] + + +def _executable(path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('#!/bin/sh\nexit 0\n', encoding='utf-8') + path.chmod(0o755) + return path + + +def _clean(config: Path, *payloads: Path, path: str | None = None): + env = os.environ.copy() + env['CLAUDE_JSON'] = str(config) + if path is not None: + env['PATH'] = path + # Only extract the production helper. Never source or execute uninstall.sh. + return subprocess.run( + ['/bin/bash', '-c', 'set -euo pipefail\n' + + _function('claude_release_config') + '\nclaude_release_config "$@"', + 'ownership-fixture', *(str(p) for p in payloads)], + env=env, text=True, capture_output=True, check=False, + ) + + +def test_removes_only_live_absolute_owned_commands(tmp_path: Path) -> None: + payload = tmp_path / 'release/packages' + owned = _executable(payload / 'releases/v1/qwen-cua-driver') + launcher = tmp_path / 'bin/qwen-cua-driver' + launcher.parent.mkdir() + launcher.symlink_to(owned) + foreign = _executable(tmp_path / 'foreign/server') + prefix = _executable(tmp_path / 'release/packages-other/server') + local = _executable(tmp_path / 'local/packages/qwen-cua-driver-local') + custom = _executable(tmp_path / 'release/custom-server') + dangling = tmp_path / 'dangling-qwen-cua-driver-rs' + dangling.symlink_to(payload / 'missing') + nonexec = payload / 'readme' + nonexec.write_text('not executable', encoding='utf-8') + escape = payload / 'escape' + escape.symlink_to(foreign) + servers = { + 'owned-any-name': {'command': str(owned)}, + 'canonical-launcher': {'command': str(launcher)}, + 'qwen-cua-driver-rs': {'command': str(foreign)}, + 'arg-substring': {'command': str(foreign), 'args': [str(owned)]}, + 'legacy-arg': {'command': str(foreign), 'args': ['qwen-cua-driver-rs']}, + 'prefix': {'command': str(prefix)}, + 'relative': {'command': 'qwen-cua-driver-rs'}, + 'relative-owned': {'command': os.path.relpath(owned)}, + 'dangling': {'command': str(dangling)}, + 'nonexistent-parent': {'command': str(payload / 'missing/../releases/v1/qwen-cua-driver')}, + 'local': {'command': str(local)}, + 'custom-home': {'command': str(custom)}, + 'escape': {'command': str(escape)}, + 'non-executable': {'command': str(nonexec)}, + 'directory': {'command': str(payload)}, + 'url': {'url': 'http://localhost:1234'}, + } + config = tmp_path / 'claude.json' + data = {'mcpServers': servers, 'projects': {'fixture': { + 'mcpServers': {'owned-project': {'command': str(owned)}, + 'foreign-project': {'command': str(foreign)}}}}, 'keep': '原样'} + original = json.dumps(data, ensure_ascii=False).encode('utf-8') + b'\n\n' + config.write_bytes(original) + result = _clean(config, payload) + assert result.returncode == 0, result.stderr + actual = json.loads(config.read_bytes()) + assert set(actual['mcpServers']) == set(servers) - {'owned-any-name', 'canonical-launcher'} + assert actual['projects']['fixture']['mcpServers'] == {'foreign-project': {'command': str(foreign)}} + assert actual['keep'] == data['keep'] + backups = list(tmp_path.glob('claude.json.bak-cua-driver-rs-uninstall-*')) + assert len(backups) == 1 and backups[0].read_bytes() == original + assert owned.exists() and foreign.exists() and launcher.is_symlink() + + +@pytest.mark.parametrize('kind', ['final-symlink', 'missing', 'file']) +def test_unremoved_payload_targets_never_grant_ownership(tmp_path: Path, kind: str) -> None: + target = tmp_path / 'other-install' + command = _executable(target / 'server') + payload = tmp_path / 'payload' + if kind == 'final-symlink': + payload.symlink_to(target, target_is_directory=True) + elif kind == 'file': + payload.write_text('file', encoding='utf-8') + config = tmp_path / 'claude.json' + original = json.dumps({'mcpServers': {'qwen-cua-driver-rs': {'command': str(command)}}}).encode() + config.write_bytes(original) + result = _clean(config, payload) + assert result.returncode == 0, result.stderr + assert config.read_bytes() == original + assert not list(tmp_path.glob('*.bak-*')) + + +def test_parent_symlink_is_traversed_but_dotdot_uses_original_path(tmp_path: Path) -> None: + release = tmp_path / 'real/release' + command = _executable(release / 'packages/server') + alias = tmp_path / 'alias' + alias.symlink_to(release, target_is_directory=True) + outside = tmp_path / 'outside/nested' + outside.mkdir(parents=True) + (release / 'packages/jump').symlink_to(outside, target_is_directory=True) + # Lexical collapse would wrongly resolve to the owned server. Native path + # resolution follows jump before .., so it refers to the external server. + foreign = _executable(tmp_path / 'outside/server') + tricky = str(release / 'packages/jump/../server') + config = tmp_path / 'claude.json' + config.write_text(json.dumps({'mcpServers': { + 'owned': {'command': str(command)}, 'external-dotdot': {'command': tricky}, + }})) + result = _clean(config, alias / 'packages') + assert result.returncode == 0, result.stderr + assert set(json.loads(config.read_text())['mcpServers']) == {'external-dotdot'} + assert foreign.exists() + + +@pytest.mark.parametrize('content', ['{', '[]', 'null']) +def test_invalid_config_fails_without_modification(tmp_path: Path, content: str) -> None: + config = tmp_path / 'claude.json' + config.write_text(content) + result = _clean(config, tmp_path / 'payload') + assert result.returncode != 0 + assert config.read_text() == content + assert not list(tmp_path.glob('*.bak-*')) + + +def test_missing_python_refuses_existing_config_and_missing_config_is_noop(tmp_path: Path) -> None: + config = tmp_path / 'claude.json' + assert _clean(config, path=str(tmp_path)).returncode == 0 + config.write_text('{}') + result = _clean(config, path=str(tmp_path)) + assert result.returncode != 0 and 'python3 is required' in result.stderr + assert config.read_text() == '{}' + + +def test_exact_payload_selection_and_physical_parent_freezing(tmp_path: Path) -> None: + # Execute only the production plan + its two helpers against explicit paths. + start = SOURCE.index(' # Select executable-bearing removals once;') + plan = SOURCE[start:SOURCE.index(' # --- CLI symlink ---', start)] + assert 'CLAUDE_JSON="$HOME/.claude.json"' in plan + assert 'rm ' not in plan + real = tmp_path / 'release' + (real / 'packages').mkdir(parents=True) + alias = tmp_path / 'alias' + alias.symlink_to(real, target_is_directory=True) + legacy = tmp_path / 'legacy' + (legacy / 'packages').mkdir(parents=True) + app = tmp_path / 'app' + app.mkdir() + legacy_app = tmp_path / 'legacy-app' + legacy_app.mkdir() + for os_name, marker, purge, expected in [ + ('Linux', '1', '0', ['', '', str(real / 'packages'), str(legacy / 'packages')]), + ('Darwin', '0', '0', ['', str(legacy_app), '', str(legacy / 'packages')]), + ('Darwin', '1', '1', [str(app), str(legacy_app), str(alias), str(legacy)]), + ]: + env = os.environ.copy() + env.update(OS=os_name, RUST_INSTALL_PRESENT=marker, PURGE_DATA=purge, + HOME_DIR=str(alias), PACKAGES_DIR=str(alias / 'packages'), + LEGACY_HOME_DIR=str(legacy), APP_BUNDLE=str(app), + LEGACY_APP_BUNDLE=str(legacy_app)) + # The plan's fixed config path is replaced only in this extracted fixture. + fixture_plan = plan.replace('CLAUDE_JSON="$HOME/.claude.json"', + 'CLAUDE_JSON="$FIXTURE_CONFIG"') + env['FIXTURE_CONFIG'] = str(tmp_path / 'absent-config') + result = subprocess.run(['/bin/bash', '-c', 'set -euo pipefail\n' + + _function('claude_release_config') + _function('prepare_payload_operand') + + fixture_plan + '\nprintf "%s\\n" "$APP_PAYLOAD" "$LEGACY_APP_PAYLOAD" "$HOME_PAYLOAD" "$LEGACY_HOME_PAYLOAD"'], + env=env, text=True, capture_output=True, check=False) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == expected + assert real.exists() and app.exists() + + +def test_runtime_order_and_bash_parseability() -> None: + rust = SOURCE[:SOURCE.index('\nUSER_BIN_LINK=', SOURCE.index(' # --- Closing message ---'))] + proof = rust.index(' claude_release_config "$APP_PAYLOAD"') + assert proof < rust.index(' # --- CLI symlink ---') + assert 'claude mcp remove' not in rust + for operand in ('"$PAYLOAD"', '"$HOME_PAYLOAD"', '"$LEGACY_HOME_PAYLOAD"'): + assert f'rm -rf -- {operand}' in rust + result = subprocess.run(['/bin/bash', '-n', str(SCRIPT)], capture_output=True, text=True) + assert result.returncode == 0, result.stderr diff --git a/packages/cua-driver/scripts/tests/uninstall-windows-regression.ps1 b/packages/cua-driver/scripts/tests/uninstall-windows-regression.ps1 new file mode 100644 index 00000000000..f981495643c --- /dev/null +++ b/packages/cua-driver/scripts/tests/uninstall-windows-regression.ps1 @@ -0,0 +1,84 @@ +[CmdletBinding()] +param( + [string]$UninstallerPath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'uninstall.ps1') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$body = [IO.File]::ReadAllText($UninstallerPath) +$tokens = $null +$parseErrors = $null +$ast = [Management.Automation.Language.Parser]::ParseInput($body, [ref]$tokens, [ref]$parseErrors) +if ($parseErrors.Count) { throw ($parseErrors | Out-String) } +# Close the native identity query as well as every mutator below, so this +# whole-script regression never needs admin or opens an elevation prompt. +$elevation = @($ast.FindAll({ param($node) + $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Test-IsElevated' +}, $true)) +if ($elevation.Count -ne 1) { throw 'Expected one native elevation query' } +$extent = $elevation[0].Extent +$body = $body.Remove($extent.StartOffset, $extent.EndOffset - $extent.StartOffset).Insert( + $extent.StartOffset, 'function Test-IsElevated { return $true }') +$FixtureRoot = Join-Path ([IO.Path]::GetTempPath()) ('qwen-uninstall-regression-' + [Guid]::NewGuid().ToString('N')) +$environmentNames = @('CUA_DRIVER_RS_INSTALL_DIR','CUA_DRIVER_RS_HOME','CUA_DRIVER_RS_UNINSTALL_FORCE','CUA_DRIVER_RS_UNINSTALL_PURGE','USERPROFILE','LOCALAPPDATA','APPDATA','PATH') +$previousEnvironment = @{} +foreach ($name in $environmentNames) { $previousEnvironment[$name] = [Environment]::GetEnvironmentVariable($name, 'Process') } +try { +$cases=@('foreign','owned-direct','owned-wrapper','owned-pwsh-wrapper','owned-quoted-apostrophe-wrapper','prefix-collision','unknown-command','mixed-exec','mixed-action','wrapper-extra-command','query-failure','malformed-xml','owned-skill','default') +$all=[Collections.Generic.List[object]]::new() +foreach($case in $cases){ + & { + param($case,$body,$FixtureRoot,$all) + $script:Trace=[Collections.Generic.List[object]]::new();$script:Forbidden=0;$script:Case=$case + $env:CUA_DRIVER_RS_INSTALL_DIR=Join-Path $FixtureRoot 'synthetic-selected\bin';$env:CUA_DRIVER_RS_HOME=Join-Path $FixtureRoot 'synthetic-selected\home' + if($case -eq 'owned-quoted-apostrophe-wrapper'){$env:CUA_DRIVER_RS_HOME=Join-Path $FixtureRoot "synthetic-selected\home'quoted"} + $env:CUA_DRIVER_RS_UNINSTALL_FORCE='1';$env:CUA_DRIVER_RS_UNINSTALL_PURGE='';$env:USERPROFILE=Join-Path $FixtureRoot 'synthetic-default-user';$env:LOCALAPPDATA=Join-Path $env:USERPROFILE 'AppData\Local';$env:APPDATA=Join-Path $env:USERPROFILE 'AppData\Roaming';$env:PATH='' + if($case -eq 'default'){$env:CUA_DRIVER_RS_HOME='';$env:CUA_DRIVER_RS_INSTALL_DIR=''} + $script:SelectedHome=if($env:CUA_DRIVER_RS_HOME){$env:CUA_DRIVER_RS_HOME}else{Join-Path $env:USERPROFILE '.cua-driver'} + $script:Bin=if($env:CUA_DRIVER_RS_INSTALL_DIR){$env:CUA_DRIVER_RS_INSTALL_DIR}else{Join-Path $env:LOCALAPPDATA 'Programs\Qwen\qwen-cua-driver\bin'} + $script:LegacyBin=Join-Path $env:LOCALAPPDATA 'Programs\Qwen\qwen-cua-driver-rs\bin';$script:LegacyHome=Join-Path $env:USERPROFILE '.cua-driver-rs';$script:Skill=Join-Path $env:USERPROFILE '.claude\skills\cua-driver' + $script:OwnExe=Join-Path $script:SelectedHome 'packages\releases\test-version\qwen-cua-driver.exe';$foreign='C:\Program Files\Cua\shared\qwen-cua-driver.exe' + function x([string]$s){[Security.SecurityElement]::Escape($s)} + $command=$foreign;$arguments='serve';$extra='' + switch($case){ + 'owned-direct'{$command=$script:OwnExe} + {$_ -in @('owned-wrapper','owned-pwsh-wrapper','owned-quoted-apostrophe-wrapper','wrapper-extra-command')} {$command='C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe';if($case -eq 'owned-pwsh-wrapper'){$command='C:\Program Files\PowerShell\7\pwsh.exe'};$arguments='-NoProfile -WindowStyle Hidden -NonInteractive -Command "Start-Process -FilePath '''+$script:OwnExe.Replace("'","''")+''' -ArgumentList ''serve'' -WindowStyle Hidden -WorkingDirectory '''+$script:Bin+'''"';if($case -eq 'wrapper-extra-command'){$arguments+='; exit 0'}} + 'prefix-collision'{$command=$script:SelectedHome+'\packages-foreign\qwen-cua-driver.exe'} + 'unknown-command'{$command=''} + 'mixed-exec'{$command=$script:OwnExe;$extra='<Exec><Command>'+ (x $foreign)+'</Command><Arguments>serve</Arguments></Exec>'} + 'mixed-action'{$command=$script:OwnExe;$extra='<ComHandler><ClassId>{00000000-0000-0000-0000-000000000001}</ClassId><Data>unrelated-handler</Data></ComHandler>'} + } + $script:TaskXML='<Task xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task"><Actions Context="Author"><Exec><Command>'+(x $command)+'</Command><Arguments>'+(x $arguments)+'</Arguments></Exec>'+$extra+'</Actions></Task>' + function schtasks.exe {$script:Trace.Add(@{call='schtasks';args=@($args)});$global:LASTEXITCODE=0;if($args -contains '/Query'){if($args -contains '/XML'){if($script:Case -eq 'query-failure'){$global:LASTEXITCODE=1;return};if($script:Case -eq 'malformed-xml'){'<bad';return};$script:TaskXML}else{'synthetic existing task'}}} + function Get-Process {param([string[]]$Name,[string]$ErrorAction) $script:Trace.Add(@{call='Get-Process';name=$Name});@([pscustomobject]@{Id=701;Path='C:\Program Files\Cua\shared\qwen-cua-driver.exe'},[pscustomobject]@{Id=702;Path=$script:OwnExe},[pscustomobject]@{Id=703;Path=(Join-Path $script:Bin 'qwen-cua-driver-uia.exe')},[pscustomobject]@{Id=704;Path=($script:SelectedHome+'\packages-foreign\qwen-cua-driver.exe')},[pscustomobject]@{Id=705;Path=$null})} + function Stop-Process {param([int]$Id,[switch]$Force,[string]$ErrorAction)$script:Trace.Add(@{call='Stop-Process';id=$Id})} + function Remove-Item {param([string]$LiteralPath,[switch]$Force,[switch]$Recurse,[string]$ErrorAction)$script:Trace.Add(@{call='Remove-Item';path=$LiteralPath})} + function Test-Path {param([string]$LiteralPath) return ($LiteralPath.StartsWith($script:SelectedHome,[StringComparison]::OrdinalIgnoreCase) -or $LiteralPath -eq $script:Bin -or $LiteralPath -eq $script:LegacyBin -or $LiteralPath -eq (Split-Path $script:LegacyBin) -or $LiteralPath -eq (Split-Path (Split-Path $script:LegacyBin)) -or $LiteralPath.StartsWith($script:LegacyHome,[StringComparison]::OrdinalIgnoreCase) -or $LiteralPath -eq $script:Skill)} + function Get-Item {param([string]$LiteralPath,[switch]$Force,[string]$ErrorAction)$target=Join-Path $script:LegacyHome 'skills\computer-use';if($script:Case -eq 'owned-skill'){$target=Join-Path $script:SelectedHome 'skills\computer-use'};[pscustomobject]@{Attributes=[IO.FileAttributes]::ReparsePoint;Target=@($target)}} + function Get-ChildItem {param([string]$LiteralPath,[switch]$Force,[string]$ErrorAction)@()} + function Start-Sleep {param([int]$Milliseconds)} + function Start-Process {$script:Forbidden++;throw 'forbidden external launch'} + function Set-Content {$script:Forbidden++;throw 'forbidden content write'} + function New-Item {$script:Forbidden++;throw 'forbidden item creation'} + function Move-Item {$script:Forbidden++;throw 'forbidden move'} + function Copy-Item {$script:Forbidden++;throw 'forbidden copy'} + function Invoke-WebRequest {$script:Forbidden++;throw 'forbidden network'} + function Invoke-RestMethod {$script:Forbidden++;throw 'forbidden network'} + function Read-Host {$script:Forbidden++;throw 'forbidden prompt'} + . ([scriptblock]::Create($body)) *> $null + $stopped=@($script:Trace|Where-Object{$_.call -eq 'Stop-Process'}|ForEach-Object{$_.id}) + $deleted=@($script:Trace|Where-Object{$_.call -eq 'schtasks' -and $_.args -contains '/Delete'}).Count -gt 0 + $legacy=@($script:Trace|Where-Object{$_.call -eq 'Remove-Item' -and ($_.path.StartsWith($script:LegacyHome) -or $_.path -eq $script:LegacyBin)}).Count -gt 0 + $skill=@($script:Trace|Where-Object{$_.call -eq 'Remove-Item' -and $_.path -eq $script:Skill}).Count -gt 0 + $expectedTask=$case -in @('owned-direct','owned-wrapper','owned-pwsh-wrapper','owned-quoted-apostrophe-wrapper','default') + $expectedStops=if($case -eq 'default'){@(701,702,703,704,705)}else{@(702,703)} + $result=@{case=$case;trace=@($script:Trace.ToArray());taskXML=$script:TaskXML;taskDelete=$deleted;stopped=$stopped;legacyCleanup=$legacy;skillCleanup=$skill;forbidden=$script:Forbidden;pass=($deleted -eq $expectedTask -and ($stopped -join ',') -eq ($expectedStops -join ',') -and $legacy -eq ($case -eq 'default') -and $skill -eq ($case -in @('owned-skill','default')) -and $script:Forbidden -eq 0)} + $all.Add($result) + } $case $body $FixtureRoot $all +} + $failed = @($all | Where-Object { -not $_.pass }) + if ($failed.Count) { throw ('Uninstall isolation regression: ' + ($failed | ConvertTo-Json -Depth 10)) } + Write-Host ("Windows whole-script uninstall isolation: {0} cases passed." -f $all.Count) +} finally { + foreach ($name in $environmentNames) { [Environment]::SetEnvironmentVariable($name, $previousEnvironment[$name], 'Process') } +} diff --git a/packages/cua-driver/scripts/uninstall.ps1 b/packages/cua-driver/scripts/uninstall.ps1 index 88e6ee89fd5..017418bf6a2 100644 --- a/packages/cua-driver/scripts/uninstall.ps1 +++ b/packages/cua-driver/scripts/uninstall.ps1 @@ -91,7 +91,83 @@ $Purge = $env:CUA_DRIVER_RS_UNINSTALL_PURGE -match '^(1|true|yes|on)$' # ---------- Elevation pre-check ------------------------------------------- +if ($env:CUA_DRIVER_RS_INSTALL_DIR) { + $VisibleBinDir = $env:CUA_DRIVER_RS_INSTALL_DIR +} else { + # New layout (v0.2.14+). Path rename rationale: see install.ps1. + $VisibleBinDir = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver\bin" +} + +# Legacy bin dir from v0.2.13 and earlier. We also clean these up so a +# fresh uninstall after upgrading leaves nothing behind. Empty-vendor-dir +# The legacy Qwen namespace is pruned if no other apps live under it. +$LegacyVisibleBinDir = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver-rs\bin" +$LegacyVendorDir = Join-Path $env:LOCALAPPDATA "Programs\Qwen" + +if ($env:CUA_DRIVER_RS_HOME) { + $HomeDir = $env:CUA_DRIVER_RS_HOME +} else { + $HomeDir = Join-Path $env:USERPROFILE ".cua-driver" +} + +# Legacy package home from v0.2.13 and earlier. +$LegacyHomeDir = Join-Path $env:USERPROFILE ".cua-driver-rs" + +$PackagesDir = Join-Path $HomeDir "packages" +$CurrentDir = Join-Path $PackagesDir "current" + +$IsolatedInstall = [bool]($env:CUA_DRIVER_RS_INSTALL_DIR -or $env:CUA_DRIVER_RS_HOME) + +function Test-SelectedExecutable([string]$path) { + if (-not $path) { return $false } + try { + $executable = [IO.Path]::GetFullPath($path.Trim('"')) + foreach ($root in @($PackagesDir, $VisibleBinDir)) { + $prefix = [IO.Path]::GetFullPath($root).TrimEnd('\') + '\' + if ($executable.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { return $true } + } + } catch {} + return $false +} + +function Get-SelectedProcesses { + $processes = @(Get-Process -Name "qwen-cua-driver","qwen-cua-driver-uia" -ErrorAction SilentlyContinue) + if (-not $IsolatedInstall) { return $processes } + return @($processes | Where-Object { + try { Test-SelectedExecutable $_.Path } catch { $false } + }) +} + $AutoStartTask = "qwen-cua-driver-serve" +$OwnsAutoStartTask = -not $IsolatedInstall +if ($IsolatedInstall) { + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + $taskXml = & schtasks.exe /Query /TN $AutoStartTask /XML 2>$null + if ($LASTEXITCODE -eq 0 -and $taskXml) { + $task = [xml]($taskXml -join "`n") + $actions = @($task.Task.Actions.Exec) + # Recognize the exact hidden PowerShell wrapper registered by this installer. + $wrapper = '^-NoProfile -WindowStyle Hidden -NonInteractive -Command "Start-Process -FilePath ''(?<exe>(?:[^'']|'''')+)'' -ArgumentList ''serve'' -WindowStyle Hidden -WorkingDirectory ''(?:[^'']|'''')*''"$' + $otherActions = @($task.Task.Actions.ChildNodes | Where-Object { + $_.NodeType -eq [Xml.XmlNodeType]::Element -and $_.LocalName -ne 'Exec' + }) + $OwnsAutoStartTask = $actions.Count -gt 0 -and $otherActions.Count -eq 0 + foreach ($action in $actions) { + $owned = Test-SelectedExecutable ([string]$action.Command) + if (-not $owned -and [IO.Path]::GetFileName([string]$action.Command) -in @('powershell.exe','pwsh.exe') -and [string]$action.Arguments -match $wrapper) { + $owned = Test-SelectedExecutable $Matches.exe.Replace("''", "'") + } + if (-not $owned) { $OwnsAutoStartTask = $false } + } + } + } catch { + $OwnsAutoStartTask = $false + } finally { + $ErrorActionPreference = $previousPreference + } +} function Test-IsElevated { ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) @@ -108,11 +184,11 @@ function Test-NeedsElevation { $ErrorActionPreference = 'Continue' try { & schtasks.exe /Query /TN $AutoStartTask 2>$null | Out-Null - $hasTask = ($LASTEXITCODE -eq 0) + $hasTask = $OwnsAutoStartTask -and ($LASTEXITCODE -eq 0) } finally { $ErrorActionPreference = $prevEAP } - $hasDaemon = @(Get-Process -Name "qwen-cua-driver" -ErrorAction SilentlyContinue).Count -gt 0 + $hasDaemon = @(Get-SelectedProcesses).Count -gt 0 return ($hasTask -or $hasDaemon) } @@ -150,34 +226,6 @@ if (-not (Test-IsElevated) -and (Test-NeedsElevation)) { } } -# ---------- Path resolution (mirrors install.ps1) ------------------------- - -if ($env:CUA_DRIVER_RS_INSTALL_DIR) { - $VisibleBinDir = $env:CUA_DRIVER_RS_INSTALL_DIR -} else { - # New layout (v0.2.14+). Path rename rationale: see install.ps1. - $VisibleBinDir = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver\bin" -} - -# Legacy bin dir from v0.2.13 and earlier. We also clean these up so a -# fresh uninstall after upgrading leaves nothing behind. Empty-vendor-dir -# The legacy Qwen namespace is pruned if no other apps live under it. -$LegacyVisibleBinDir = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver-rs\bin" -$LegacyVendorDir = Join-Path $env:LOCALAPPDATA "Programs\Qwen" - -if ($env:CUA_DRIVER_RS_HOME) { - $HomeDir = $env:CUA_DRIVER_RS_HOME -} else { - $HomeDir = Join-Path $env:USERPROFILE ".cua-driver" -} - -# Legacy package home from v0.2.13 and earlier. -$LegacyHomeDir = Join-Path $env:USERPROFILE ".cua-driver-rs" - -$PackagesDir = Join-Path $HomeDir "packages" -$CurrentDir = Join-Path $PackagesDir "current" -# $AutoStartTask hoisted to the elevation pre-check block above. - # Skill junctions — mirrors the AGENTS list in # packages/cua-driver/rust/crates/cua-driver/src/skills.rs (the verb that # creates them) so we remove from the same paths. Both the current @@ -233,10 +281,9 @@ function Test-IsQwenSkillReparsePoint([string]$path) { if (-not (Test-IsReparsePoint $path)) { return $false } try { $item = Get-Item -LiteralPath $path -Force -ErrorAction Stop - $ownedRoots = @( - (Join-Path $HomeDir "skills"), - (Join-Path $LegacyHomeDir "skills") - ) | ForEach-Object { [System.IO.Path]::GetFullPath($_).TrimEnd('\') } + $ownedRoots = @((Join-Path $HomeDir "skills")) + if (-not $IsolatedInstall) { $ownedRoots += (Join-Path $LegacyHomeDir "skills") } + $ownedRoots = $ownedRoots | ForEach-Object { [System.IO.Path]::GetFullPath($_).TrimEnd('\') } foreach ($target in @($item.Target)) { if (-not $target) { continue } $resolvedTarget = if ([System.IO.Path]::IsPathRooted($target)) { @@ -298,7 +345,7 @@ try { } finally { $ErrorActionPreference = $prevEAP } -if ($taskExitCode -eq 0 -and $taskQuery) { +if ($OwnsAutoStartTask -and $taskExitCode -eq 0 -and $taskQuery) { if (Confirm-Remove "scheduled task '$AutoStartTask' (autostart at logon)") { $ErrorActionPreference = 'Continue' try { @@ -324,7 +371,7 @@ if ($taskExitCode -eq 0 -and $taskQuery) { # pin the directory junction's target open and make Remove-Item # fail with "in use". Stop them up front so subsequent deletes # aren't racy. -$running = @(Get-Process -Name "qwen-cua-driver" -ErrorAction SilentlyContinue) +$running = @(Get-SelectedProcesses) if ($running.Count -gt 0) { Write-Step "stopping $($running.Count) running qwen-cua-driver.exe process(es)" foreach ($p in $running) { @@ -421,49 +468,51 @@ if (Test-Path -LiteralPath $HomeDir) { # sweep these so a fresh uninstall after upgrading via install.ps1 # leaves nothing behind. Skip silently when the legacy paths don't # exist — common case post-v0.2.14. -if (Test-Path -LiteralPath $LegacyVisibleBinDir) { - if (Test-IsReparsePoint $LegacyVisibleBinDir) { - Remove-Item -LiteralPath $LegacyVisibleBinDir -Force -Recurse -ErrorAction SilentlyContinue - Write-Step "removed legacy junction $LegacyVisibleBinDir" - } else { - Remove-Item -LiteralPath $LegacyVisibleBinDir -Force -Recurse -ErrorAction SilentlyContinue - Write-Step "removed legacy directory $LegacyVisibleBinDir" - } -} -# Empty cua-driver-rs parent (under trycua\) -$legacyParent = Split-Path -Parent $LegacyVisibleBinDir -if ((Test-Path -LiteralPath $legacyParent) -and -not (Get-ChildItem -LiteralPath $legacyParent -Force -ErrorAction SilentlyContinue)) { - Remove-Item -LiteralPath $legacyParent -Force -ErrorAction SilentlyContinue - Write-Step "removed empty legacy parent $legacyParent" -} -# Empty Qwen vendor dir -if ((Test-Path -LiteralPath $LegacyVendorDir) -and -not (Get-ChildItem -LiteralPath $LegacyVendorDir -Force -ErrorAction SilentlyContinue)) { - Remove-Item -LiteralPath $LegacyVendorDir -Force -ErrorAction SilentlyContinue - Write-Step "removed empty legacy vendor dir $LegacyVendorDir" -} -# Legacy package home. Preserve telemetry state on a normal uninstall so a -# future runtime can migrate the same pseudonymous identity to ~/.cua-driver. -if (Test-Path -LiteralPath $LegacyHomeDir) { - if ($Purge) { - Remove-Item -LiteralPath $LegacyHomeDir -Force -Recurse -ErrorAction SilentlyContinue - if (Test-Path -LiteralPath $LegacyHomeDir) { - Write-WarningStep "$LegacyHomeDir was not fully removed — some files may still be locked." +if (-not $IsolatedInstall) { + if (Test-Path -LiteralPath $LegacyVisibleBinDir) { + if (Test-IsReparsePoint $LegacyVisibleBinDir) { + Remove-Item -LiteralPath $LegacyVisibleBinDir -Force -Recurse -ErrorAction SilentlyContinue + Write-Step "removed legacy junction $LegacyVisibleBinDir" } else { - Write-Step "purged legacy package home $LegacyHomeDir" + Remove-Item -LiteralPath $LegacyVisibleBinDir -Force -Recurse -ErrorAction SilentlyContinue + Write-Step "removed legacy directory $LegacyVisibleBinDir" } - } else { - foreach ($legacyRuntimePath in @( - (Join-Path $LegacyHomeDir "packages"), - (Join-Path $LegacyHomeDir "skills"), - (Join-Path $LegacyHomeDir ".tcc-signing-identity"), - (Join-Path $LegacyHomeDir "serve.out.log"), - (Join-Path $LegacyHomeDir "serve.err.log") - )) { - if (Test-Path -LiteralPath $legacyRuntimePath) { - Remove-Item -LiteralPath $legacyRuntimePath -Force -Recurse -ErrorAction SilentlyContinue + } + # Empty cua-driver-rs parent (under trycua\) + $legacyParent = Split-Path -Parent $LegacyVisibleBinDir + if ((Test-Path -LiteralPath $legacyParent) -and -not (Get-ChildItem -LiteralPath $legacyParent -Force -ErrorAction SilentlyContinue)) { + Remove-Item -LiteralPath $legacyParent -Force -ErrorAction SilentlyContinue + Write-Step "removed empty legacy parent $legacyParent" + } + # Empty Qwen vendor dir + if ((Test-Path -LiteralPath $LegacyVendorDir) -and -not (Get-ChildItem -LiteralPath $LegacyVendorDir -Force -ErrorAction SilentlyContinue)) { + Remove-Item -LiteralPath $LegacyVendorDir -Force -ErrorAction SilentlyContinue + Write-Step "removed empty legacy vendor dir $LegacyVendorDir" + } + # Legacy package home. Preserve telemetry state on a normal uninstall so a + # future runtime can migrate the same pseudonymous identity to ~/.cua-driver. + if (Test-Path -LiteralPath $LegacyHomeDir) { + if ($Purge) { + Remove-Item -LiteralPath $LegacyHomeDir -Force -Recurse -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath $LegacyHomeDir) { + Write-WarningStep "$LegacyHomeDir was not fully removed — some files may still be locked." + } else { + Write-Step "purged legacy package home $LegacyHomeDir" } + } else { + foreach ($legacyRuntimePath in @( + (Join-Path $LegacyHomeDir "packages"), + (Join-Path $LegacyHomeDir "skills"), + (Join-Path $LegacyHomeDir ".tcc-signing-identity"), + (Join-Path $LegacyHomeDir "serve.out.log"), + (Join-Path $LegacyHomeDir "serve.err.log") + )) { + if (Test-Path -LiteralPath $legacyRuntimePath) { + Remove-Item -LiteralPath $legacyRuntimePath -Force -Recurse -ErrorAction SilentlyContinue + } + } + Write-Step "removed legacy runtime payloads and preserved legacy telemetry state" } - Write-Step "removed legacy runtime payloads and preserved legacy telemetry state" } } diff --git a/packages/cua-driver/scripts/uninstall.sh b/packages/cua-driver/scripts/uninstall.sh index fa56e26d873..71bf87b3241 100755 --- a/packages/cua-driver/scripts/uninstall.sh +++ b/packages/cua-driver/scripts/uninstall.sh @@ -48,8 +48,8 @@ # (~/.cua-driver/packages/, legacy ~/.cua-driver-rs/, QwenCuaDriverRs.app, # the LaunchAgent/systemd unit, or current Rust telemetry state). # -# Also scrubs Claude MCP registrations in ~/.claude.json that match -# the active backend. +# Also scrubs Claude MCP registrations in ~/.claude.json whose executable +# belongs to the release payload removed by this invocation. # # Revokes TCC grants on macOS by default (Accessibility + Screen Recording) # so the next install prompts cleanly under the new signing identity. Pass @@ -170,6 +170,169 @@ resolve_link() { esac } +# Prove command ownership while the selected payloads still exist. Keep the +# heredoc outside command substitution for macOS Bash 3.2. +claude_release_config() { + if [[ ! -e "$CLAUDE_JSON" && ! -L "$CLAUDE_JSON" ]]; then + return 0 + fi + if ! command -v python3 >/dev/null 2>&1; then + printf 'error: python3 is required to safely inspect Claude MCP ownership before uninstalling a release with %s; install python3 and retry. No release files were removed.\n' "$CLAUDE_JSON" >&2 + return 1 + fi + python3 - "$CLAUDE_JSON" "$@" <<'PYCLAUDE' +import json +import os +import stat +import sys +import tempfile + + +def clean(): + path, *payloads = sys.argv[1:] + try: + with open(path, "rb") as handle: + original = handle.read() + data = json.loads(original.decode("utf-8")) + if not isinstance(data, dict): + raise ValueError("top-level value must be a JSON object") + except Exception as exc: + raise ValueError(f"could not read Claude config {path}: {exc}") from exc + + # These exact operands also drive rm below. A final symlink is only + # unlinked; symlinks in its parent directories are traversed by rm. + roots = set() + for payload in payloads: + if not payload: + continue + try: + before = os.lstat(payload) + except FileNotFoundError: + continue + if not stat.S_ISDIR(before.st_mode): + continue + resolved = os.path.realpath(payload) + after = os.stat(resolved) + if resolved == os.path.sep or not os.path.samestat(before, after): + raise ValueError(f"unsafe release removal operand: {payload}") + roots.add(resolved) + + def owned(server): + if not isinstance(server, dict): + return False + command = server.get("command") + if not isinstance(command, str) or not os.path.isabs(command): + return False + try: + # stat ORIGINAL spelling first: missing/../file cannot be repaired + # into an executable, nor can symlink/.. be collapsed lexically. + before = os.stat(command) + if not stat.S_ISREG(before.st_mode) or not os.access(command, os.X_OK): + return False + resolved = os.path.realpath(command) + if not os.path.samestat(before, os.stat(resolved)): + return False + return any(os.path.commonpath((resolved, root)) == root for root in roots) + except (OSError, ValueError): + return False + + removed, kept, local = [], [], [] + + def scrub(servers, scope): + if not isinstance(servers, dict): + return + for name in list(servers): + server = servers[name] + if owned(server): + del servers[name] + removed.append(f"{scope}:{name}") + else: + command = server.get("command") if isinstance(server, dict) else None + basename = os.path.basename(command) if isinstance(command, str) else "" + if basename == "qwen-cua-driver-local": + local.append(f"{scope}:{name}") + elif name in ("qwen-cua-driver", "qwen-cua-driver-rs", "cua-computer-use") or basename == "qwen-cua-driver": + kept.append(f"{scope}:{name}") + + scrub(data.get("mcpServers"), "user") + projects = data.get("projects") + if isinstance(projects, dict): + for project_name, project in projects.items(): + if isinstance(project, dict): + scrub(project.get("mcpServers"), f"project:{project_name}") + + if removed: + directory = os.path.dirname(path) or "." + fd, backup = tempfile.mkstemp( + prefix=os.path.basename(path) + ".bak-cua-driver-rs-uninstall-", dir=directory + ) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(original) + except Exception: + os.unlink(backup) + raise + fd, temporary = tempfile.mkstemp(prefix=".claude.json.", suffix=".tmp", dir=directory) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(data, handle, indent=2, ensure_ascii=False) + handle.write("\n") + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + print("==> removed Claude MCP registration(s): " + ", ".join(removed)) + print(f"==> backed up Claude config to {backup}") + if local: + print("==> preserved local Claude MCP registration(s): " + ", ".join(local) + + "; remove qwen-cua-driver-local with packages/cua-driver/scripts/uninstall-local.sh from the source checkout") + if kept: + print("==> preserved unproven Claude MCP registration(s): " + ", ".join(kept) + + "; inspect the exact entry before removing it manually") + + +try: + clean() +except Exception as exc: + print(f"error: Claude MCP cleanup failed: {exc}", file=sys.stderr) + raise SystemExit(1) +PYCLAUDE +} + +# Resolve parent aliases once so proof and removal use the same paths. +prepare_payload_operand() { + local payload="$1" parent + if [[ -z "$payload" ]]; then printf -v "$2" '%s' ""; return 0; fi + case "$payload" in + */|.|..|./*|../*|*/./*|*/../*|*/.|*/..) + printf 'error: unsafe release removal operand: %s\n' "$payload" >&2 + return 1 ;; + esac + if [[ -d "$payload" && ! -L "$payload" ]]; then + if [[ ! -r "$payload" || ! -x "$payload" ]]; then + printf 'error: inaccessible release removal operand: %s\n' "$payload" >&2 + return 1 + fi + # A sentinel protects trailing newlines from command substitution. + payload="$(CDPATH= cd -- "$payload" && pwd -P && printf '.')" || return 1 + payload="${payload%$'\n.'}" + if [[ "$payload" == "/" ]]; then + printf 'error: unsafe release removal operand: /\n' >&2 + return 1 + fi + elif [[ -L "$payload" ]]; then + case "$payload" in + */*) parent="${payload%/*}"; [[ -n "$parent" ]] || parent="/" ;; + *) parent="." ;; + esac + parent="$(CDPATH= cd -- "$parent" && pwd -P && printf '.')" || return 1 + parent="${parent%$'\n.'}" + payload="$parent/${payload##*/}" + fi + # printf -v avoids another lossy command substitution at the call site. + printf -v "$2" '%s' "$payload" +} + # ---------------------------------------------------------------------- # Rust uninstall branch (default on Linux + macOS). # ---------------------------------------------------------------------- @@ -220,6 +383,26 @@ if [[ "$USE_RUST_BACKEND" == "1" ]]; then RUST_INSTALL_PRESENT=1 fi + # Select executable-bearing removals once; proof and deletion consume the + # same operands. Empty slots mean this invocation will not remove that tree. + APP_PAYLOAD="" LEGACY_APP_PAYLOAD="" HOME_PAYLOAD="" LEGACY_HOME_PAYLOAD="" + if [[ "$OS" == "Darwin" ]]; then + if [[ -d "$LEGACY_APP_BUNDLE" ]]; then LEGACY_APP_PAYLOAD="$LEGACY_APP_BUNDLE"; fi + if [[ -d "$APP_BUNDLE" && "$RUST_INSTALL_PRESENT" == "1" ]]; then APP_PAYLOAD="$APP_BUNDLE"; fi + fi + if [[ -d "$HOME_DIR" && ( "$RUST_INSTALL_PRESENT" == "1" || "$PURGE_DATA" == "1" ) ]]; then + if [[ "$PURGE_DATA" == "1" ]]; then HOME_PAYLOAD="$HOME_DIR"; else HOME_PAYLOAD="$PACKAGES_DIR"; fi + fi + if [[ -d "$LEGACY_HOME_DIR" ]]; then + if [[ "$PURGE_DATA" == "1" ]]; then LEGACY_HOME_PAYLOAD="$LEGACY_HOME_DIR"; else LEGACY_HOME_PAYLOAD="$LEGACY_HOME_DIR/packages"; fi + fi + prepare_payload_operand "$APP_PAYLOAD" APP_PAYLOAD || exit 1 + prepare_payload_operand "$LEGACY_APP_PAYLOAD" LEGACY_APP_PAYLOAD || exit 1 + prepare_payload_operand "$HOME_PAYLOAD" HOME_PAYLOAD || exit 1 + prepare_payload_operand "$LEGACY_HOME_PAYLOAD" LEGACY_HOME_PAYLOAD || exit 1 + CLAUDE_JSON="$HOME/.claude.json" + claude_release_config "$APP_PAYLOAD" "$LEGACY_APP_PAYLOAD" "$HOME_PAYLOAD" "$LEGACY_HOME_PAYLOAD" || exit 1 + # --- CLI symlink --- # Only remove ~/.local/bin/qwen-cua-driver when it resolves into a # cua-driver-rs install. Pre-rename installs at @@ -309,40 +492,14 @@ if [[ "$USE_RUST_BACKEND" == "1" ]]; then # take — otherwise it fails with -10814 and the grant silently survives. maybe_reset_tcc - # --- .app bundle (macOS only) --- - # Legacy /Applications/QwenCuaDriverRs.app is unambiguously Rust and - # always removed when present. /Applications/QwenCuaDriver.app is the - # current canonical Rust path BUT also where the Swift driver - # lives (bundle id `com.qwencode.cua-driver`), so we only remove - # it when $RUST_INSTALL_PRESENT — protects a Swift-only Mac from - # losing its bundle if `uninstall.sh --experimental-rust` is run - # by mistake. - if [[ "$OS" == "Darwin" ]]; then - if [[ -d "$LEGACY_APP_BUNDLE" ]]; then - SUDO="" - if [[ ! -w "$(dirname "$LEGACY_APP_BUNDLE")" ]]; then - SUDO="sudo" - fi - $SUDO rm -rf "$LEGACY_APP_BUNDLE" - log "removed $LEGACY_APP_BUNDLE" - else - log "no app bundle at $LEGACY_APP_BUNDLE (skipping)" - fi - if [[ -d "$APP_BUNDLE" ]]; then - if [[ "$RUST_INSTALL_PRESENT" == "1" ]]; then - SUDO="" - if [[ ! -w "$(dirname "$APP_BUNDLE")" ]]; then - SUDO="sudo" - fi - $SUDO rm -rf "$APP_BUNDLE" - log "removed $APP_BUNDLE" - else - log "$APP_BUNDLE exists but no Rust marker on disk (~/.cua-driver/packages/, ~/.cua-driver-rs/, QwenCuaDriverRs.app, current/legacy LaunchAgent or systemd unit); leaving it (looks like a Swift-only install)" - fi - else - log "no app bundle at $APP_BUNDLE (skipping)" - fi - fi + # --- Selected app bundles (macOS only) --- + for PAYLOAD in "$LEGACY_APP_PAYLOAD" "$APP_PAYLOAD"; do + [[ -n "$PAYLOAD" ]] || continue + SUDO="" + if [[ ! -w "$(dirname "$PAYLOAD")" ]]; then SUDO="sudo"; fi + $SUDO rm -rf -- "$PAYLOAD" + log "removed $PAYLOAD" + done # --- Package home --- # A normal uninstall deliberately keeps the pseudonymous installation ID, @@ -351,41 +508,26 @@ if [[ "$USE_RUST_BACKEND" == "1" ]]; then # any events while disabled. `--purge` is the explicit identity reset. # All removal remains gated on the Rust marker so a mistaken invocation # cannot damage a Swift-only Mac's shared ~/.cua-driver state. - if [[ -d "$HOME_DIR" ]]; then - if [[ "$RUST_INSTALL_PRESENT" == "1" || "$PURGE_DATA" == "1" ]]; then - if [[ "$PURGE_DATA" == "1" ]]; then - rm -rf "$HOME_DIR" - log "purged $HOME_DIR (including telemetry identity and preference)" - else - # Remove only installer/runtime-owned payloads. Unknown files - # and all telemetry state remain untouched. - rm -rf "$HOME_DIR/packages" "$HOME_DIR/skills" - rm -f \ - "$HOME_DIR/.tcc-signing-identity" \ - "$HOME_DIR/serve.out.log" \ - "$HOME_DIR/serve.err.log" - log "removed runtime payloads from $HOME_DIR" - log "preserved telemetry identity, preference, and registration markers" - fi + if [[ -n "$HOME_PAYLOAD" ]]; then + rm -rf -- "$HOME_PAYLOAD" + if [[ "$PURGE_DATA" == "1" ]]; then + log "purged $HOME_DIR (including telemetry identity and preference)" else - log "$HOME_DIR exists but no Rust marker on disk; leaving it (looks like a Swift-only / shared config dir)" + rm -rf -- "$HOME_DIR/skills" + rm -f -- "$HOME_DIR/.tcc-signing-identity" "$HOME_DIR/serve.out.log" "$HOME_DIR/serve.err.log" + log "removed runtime payloads from $HOME_DIR" + log "preserved telemetry identity, preference, and registration markers" fi else - log "no package home at $HOME_DIR (skipping)" + log "no selected release package home at $HOME_DIR (skipping)" fi - # Preserve legacy telemetry state during a normal uninstall so the - # runtime's existing one-shot migration can carry the same identity into - # ~/.cua-driver on reinstall. - if [[ -d "$LEGACY_HOME_DIR" ]]; then + if [[ -n "$LEGACY_HOME_PAYLOAD" ]]; then + rm -rf -- "$LEGACY_HOME_PAYLOAD" if [[ "$PURGE_DATA" == "1" ]]; then - rm -rf "$LEGACY_HOME_DIR" log "purged legacy package home $LEGACY_HOME_DIR" else - rm -rf "$LEGACY_HOME_DIR/packages" "$LEGACY_HOME_DIR/skills" - rm -f \ - "$LEGACY_HOME_DIR/.tcc-signing-identity" \ - "$LEGACY_HOME_DIR/serve.out.log" \ - "$LEGACY_HOME_DIR/serve.err.log" + rm -rf -- "$LEGACY_HOME_DIR/skills" + rm -f -- "$LEGACY_HOME_DIR/.tcc-signing-identity" "$LEGACY_HOME_DIR/serve.out.log" "$LEGACY_HOME_DIR/serve.err.log" log "removed legacy runtime payloads and preserved legacy telemetry state" fi fi @@ -449,137 +591,6 @@ if [[ "$USE_RUST_BACKEND" == "1" ]]; then log "no Rust install marker; leaving agent skill symlinks untouched" fi - # --- Claude Code MCP registrations --- - # Same scrub shape as the Swift branch, keyed on the cua-driver-rs - # binary name + the per-platform install paths. Unrelated MCP - # servers are left alone. - CLAUDE_JSON="$HOME/.claude.json" - if [[ -f "$CLAUDE_JSON" ]] && command -v python3 >/dev/null 2>&1; then - PY_OUTPUT="$( - CLAUDE_JSON="$CLAUDE_JSON" HOME_DIR="$HOME_DIR" RUST_INSTALL_PRESENT="$RUST_INSTALL_PRESENT" python3 <<'PY' -import json -import os -import shutil -import sys -import tempfile -import time - -path = os.environ["CLAUDE_JSON"] -home_dir = os.environ.get("HOME_DIR", "") -rust_install_present = os.environ.get("RUST_INSTALL_PRESENT", "0") == "1" - -try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) -except Exception as exc: - print(f"could not read Claude config {path}: {exc}", file=sys.stderr) - raise SystemExit(0) - -removed = [] - -def text_parts(value): - if isinstance(value, str): - return [value] - if isinstance(value, list): - return [item for item in value if isinstance(item, str)] - return [] - -def invokes_cua_driver_rs(server): - if not isinstance(server, dict): - return False - parts = [] - parts.extend(text_parts(server.get("command"))) - parts.extend(text_parts(server.get("args"))) - joined = " ".join(parts) - # Match the Rust-port-specific anchors: bundle name, package home, - # explicit ".cua-driver-rs" segment. Plain "qwen-cua-driver" alone is - # ambiguous (the Swift binary uses the same filename). The shared - # /Applications/QwenCuaDriver.app path is ALSO ambiguous (Rust took - # over the Swift bundle id) — only count it as Rust when a Rust - # install marker is on disk; otherwise it is almost certainly a - # Swift registration we should not scrub. - if home_dir and home_dir in joined: - return True - if "QwenCuaDriverRs.app" in joined or ".cua-driver-rs" in joined or "qwen-cua-driver-rs" in joined: - return True - if rust_install_present and "/Applications/QwenCuaDriver.app" in joined: - return True - return False - -def should_remove(name, server): - return name in {"qwen-cua-driver-rs"} or invokes_cua_driver_rs(server) - -def scrub_servers(servers, scope): - if not isinstance(servers, dict): - return - for name in list(servers.keys()): - if should_remove(name, servers[name]): - del servers[name] - removed.append(f"{scope}:{name}") - -scrub_servers(data.get("mcpServers"), "user") - -projects = data.get("projects") -if isinstance(projects, dict): - for project in projects.values(): - if isinstance(project, dict): - scrub_servers(project.get("mcpServers"), "project") - -if not removed: - raise SystemExit(0) - -backup = f"{path}.bak-cua-driver-rs-uninstall-{int(time.time())}" -shutil.copy2(path, backup) - -directory = os.path.dirname(path) or "." -fd, tmp_path = tempfile.mkstemp( - prefix=".claude.json.", - suffix=".tmp", - dir=directory, - text=True, -) -try: - with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - f.write("\n") - os.replace(tmp_path, path) -except Exception: - try: - os.unlink(tmp_path) - except OSError: - pass - raise - -print(f"removed Claude MCP registration(s): {', '.join(removed)}") -print(f"backed up Claude config to {backup}") -PY - )" - if [[ -n "$PY_OUTPUT" ]]; then - while IFS= read -r line; do - log "$line" - done <<< "$PY_OUTPUT" - else - log "no Claude MCP registrations for cua-driver-rs found in $CLAUDE_JSON" - fi - else - log "no Claude config cleanup via python3 (missing $CLAUDE_JSON or python3)" - fi - - # Best-effort CLI cleanup. `claude mcp remove` only touches the - # active project / user scopes — fine to run; it's a no-op when the - # entries were already scrubbed above. - if command -v claude >/dev/null 2>&1; then - for SERVER in cua-driver-rs; do - for SCOPE in local project user; do - if claude mcp remove "$SERVER" -s "$SCOPE" >/dev/null 2>&1; then - log "removed Claude MCP server $SERVER from $SCOPE scope" - fi - done - done - else - log "claude CLI not found (skipping Claude MCP CLI cleanup)" - fi - # --- Closing message --- # TCC grants were already revoked above, before the app was removed, so # the reset could still resolve the bundle id through LaunchServices. diff --git a/packages/cua-driver/scripts/verify-cua-sdk-package.mjs b/packages/cua-driver/scripts/verify-cua-sdk-package.mjs index a1ffccb4c27..c0822a63e3a 100644 --- a/packages/cua-driver/scripts/verify-cua-sdk-package.mjs +++ b/packages/cua-driver/scripts/verify-cua-sdk-package.mjs @@ -17,7 +17,7 @@ import { fileURLToPath } from "node:url" const scriptDirectory = dirname(fileURLToPath(import.meta.url)) const driverRoot = resolve(scriptDirectory, "..") const packageRoot = join(driverRoot, "typescript") -const skillResources = ["SKILL.md"] + function valueAfter(flag) { const index = process.argv.indexOf(flag) @@ -34,6 +34,8 @@ function optionalValueAfter(flag) { const nativeDirectory = resolve(valueAfter("--native-dir")) const outputDirectory = resolve(valueAfter("--output-dir")) +const browserTarballArgument = optionalValueAfter("--browser-tarball") +const browserTarball = browserTarballArgument ? resolve(browserTarballArgument) : undefined const releaseDirectoryArgument = optionalValueAfter("--release-dir") const releaseDirectory = releaseDirectoryArgument ? resolve(releaseDirectoryArgument) @@ -168,13 +170,18 @@ for (const required of [ "computer-use/index.js", "computer-use/app.js", "computer-use/index.d.ts", - ...skillResources.map((resource) => `computer-use/${resource}`), + "computer-use/runtime.js", + "computer-use/runtime.d.ts", + "computer-use/documentation.js", + "computer-use/browser.js", + "computer-use/interface.json", "scripts/install-native.mjs", ]) { if (!packedPaths.includes(required)) { throw new Error(`packed SDK is missing ${required}`) } } +if (packedPaths.some((path) => path.endsWith("SKILL.md"))) throw new Error("SDK must use runtime documentation, not a bundled skill"); const bundledNative = packedPaths.find((path) => /\.(?:dll|dylib|node|so)$/u.test(path), ) @@ -224,6 +231,7 @@ try { [ "install", tarball, + ...(browserTarball ? [browserTarball] : []), "--no-audit", "--no-fund", "--ignore-scripts=false", @@ -231,13 +239,6 @@ try { ], { cwd: consumer, env: environment }, ) - for (const resource of skillResources) { - const canonical = readFileSync(join(driverRoot, "..", "core", "src", "skills", "bundled", "computer-use", resource)) - const installed = readFileSync(join(consumer, "node_modules", "@qwen-code", "cua-sdk", "computer-use", resource)) - if (!canonical.equals(installed)) { - throw new Error(`installed Computer Use skill differs from canonical resource: ${resource}`) - } - } const dependencyTree = run("npm", ["ls", "--all", "--json"], { cwd: consumer, env: environment, @@ -265,14 +266,18 @@ try { if (!Array.isArray(listing.tools) || listing.tools.length === 0) { throw new Error("native SDK returned no tools"); } - const revisionSupported = await computer.supportsObservationRevision(); - if (typeof revisionSupported !== "boolean") { - throw new Error("ComputerUse capability probe returned an invalid result"); + let safeRefusal = false; + try { + await computer.appCall("click", { scope: "desktop", x: 1, y: 1 }); + } catch (error) { + if (error.code !== "screenshot_required" || error.sent !== false) throw error; + safeRefusal = true; } + if (!safeRefusal || listing.tools.length !== 19) throw new Error("Computer Use session/manifest contract mismatch"); process.stdout.write(JSON.stringify({ package: "@qwen-code/cua-sdk", toolCount: listing.tools.length, - revisionSupported, + safeRefusal, }) + "\\n"); } finally { await computer.close(); diff --git a/packages/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js b/packages/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js index 82e1cf17c6c..d24fe85b204 100644 --- a/packages/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js +++ b/packages/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js @@ -8,15 +8,13 @@ const fs = require('fs'); const http = require('http'); const path = require('path'); const sentinelMode = process.env.CUA_E2E_SENTINEL === '1'; -const nativeWayland = process.platform === 'linux' && Boolean(process.env.WAYLAND_DISPLAY); -const customCuaCompositor = process.env.CUA_E2E_WAYLAND_SESSION === 'cua-compositor'; const fixtureJournalUrl = process.env.CUA_E2E_FIXTURE_JOURNAL_URL || ''; const sentinelJournalPath = process.env.CUA_E2E_SENTINEL_JOURNAL || ''; if (process.env.CUA_E2E_USER_DATA_DIR) { app.setPath('userData', process.env.CUA_E2E_USER_DATA_DIR); } -if (process.platform === 'linux' && process.env.WAYLAND_DISPLAY) { - app.commandLine.appendSwitch('ozone-platform', 'wayland'); +if (process.platform === 'linux') { + app.commandLine.appendSwitch('ozone-platform', 'x11'); app.commandLine.appendSwitch('enable-features', 'UseOzonePlatform'); } @@ -78,15 +76,11 @@ function createWindow() { title: fixedTitle, // Map the normal harness immediately. Xvfb/Openbox can enumerate a // deferred BrowserWindow while never painting it into the root desktop. - // The sentinel normally stays hidden until it has maximized and claimed - // focus. cua-compositor has no window-policy transition to perform, so map - // it at construction time; a synchronous show() after DOMContentLoaded can - // otherwise stall Chromium before the renderer paints or schedules timers. - show: !sentinelMode || customCuaCompositor, + show: !sentinelMode, // A floating-level macOS window is omitted by cua-driver's deliberate // layer-0 top-level window contract. Foreground + maximized is sufficient // for occlusion there and lets an unexpected target raise remain visible. - alwaysOnTop: sentinelMode && process.platform !== 'darwin' && !nativeWayland, + alwaysOnTop: sentinelMode && process.platform !== 'darwin', autoHideMenuBar: true, webPreferences: { nodeIntegration: false, @@ -117,14 +111,7 @@ function createWindow() { })); mainWindow.webContents.on('did-create-window', child => { child.once('ready-to-show', () => { - // Mutter may activate a newly mapped native-Wayland child even when - // Electron requests showInactive(), which makes a background AX action - // appear to steal focus. The scenario's behavioral oracle is child - // creation, so keep the child hidden on native Wayland and preserve the - // visible non-activating window contract on the other backends. - if (!nativeWayland || customCuaCompositor) { - child.showInactive(); - } + child.showInactive(); }); }); @@ -138,9 +125,7 @@ function createWindow() { mainWindow.setTitle(fixedTitle); if (sentinelMode) { // Drive the clock from the main process but require the renderer to - // handle each probe before it can journal a heartbeat. Native - // Wayland fullscreen surfaces can suspend renderer-owned interval - // timers even while renderer IPC remains healthy. + // handle each probe before it can journal a heartbeat. if (!sentinelHeartbeatTimer) { sentinelHeartbeatTimer = setInterval(() => { if (mainWindow && !mainWindow.isDestroyed()) { @@ -149,25 +134,12 @@ function createWindow() { }, 100); sentinelHeartbeatTimer.unref(); } - if (process.platform !== 'darwin' && !nativeWayland) { + if (process.platform !== 'darwin') { mainWindow.setAlwaysOnTop(true); } - if (nativeWayland && !customCuaCompositor) { - mainWindow.setFullScreen(true); - } else if (!customCuaCompositor) { - mainWindow.maximize(); - } - // The minimal nested cua-compositor intentionally has no fullscreen - // policy implementation. Requesting fullscreen leaves Chromium - // waiting on a configure transition and stops the heartbeat oracle. - // Its 1280x900 sentinel already covers the smaller fixture at origin. - // cua-compositor mapped and focused this toplevel at construction - // time. Avoid a second synchronous show/configure/focus transition - // after the renderer has emitted its ready event. - if (!customCuaCompositor) { - mainWindow.show(); - mainWindow.focus(); - } + mainWindow.maximize(); + mainWindow.show(); + mainWindow.focus(); } else { // Xvfb/Openbox can keep a showInactive window inspectable through // AT-SPI while never mapping it onto the captured root desktop. diff --git a/packages/cua-driver/tests/fixtures/build/linux.sh b/packages/cua-driver/tests/fixtures/build/linux.sh index 252ba601875..f9b62d84f28 100755 --- a/packages/cua-driver/tests/fixtures/build/linux.sh +++ b/packages/cua-driver/tests/fixtures/build/linux.sh @@ -59,8 +59,7 @@ if [[ "$SKIP" != "gtk3" && "$ONLY" == *",gtk3,"* ]]; then cat > "$GTK3_STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER' #!/usr/bin/env bash -# X11 remains the default for the canonical Xvfb lane. A native Wayland lane -# exports GDK_BACKEND=wayland before launching this repo-owned fixture. +# Linux fixtures use X11, including when the surrounding session uses XWayland. export GDK_BACKEND="${GDK_BACKEND:-x11}" exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@" LAUNCHER diff --git a/packages/cua-driver/typescript/README.md b/packages/cua-driver/typescript/README.md index cd4a80ddb24..e5b76b26932 100644 --- a/packages/cua-driver/typescript/README.md +++ b/packages/cua-driver/typescript/README.md @@ -5,7 +5,7 @@ Rust-backed TypeScript/Node SDK for Cua Driver client applications. The package root exposes the complete typed driver SDK. The `@qwen-code/cua-sdk/computer-use` subpath exposes the smaller high-level API for application discovery, revision observations, element-token actions, and state -verification. Both entrypoints ship in this one npm package. +verification. Both entrypoints ship in this one npm package. The `/runtime` entrypoint supplies global `cua` to Node REPL, including explicit desktop input and the repository browser SDK facade. SDK, browser SDK and native release use version 0.26.0; Node REPL ships the same version. ## Product boundary @@ -72,8 +72,7 @@ try { SDK operations are asynchronous and require a native library matching the host OS and architecture. The package postinstall downloads the SDK library and Node runtime from the exact same-version `qwen-cua-driver` GitHub Release and -verifies the archive against that release's `checksums.txt`. It caches the native library and Node adapter. On Windows it also stages the -UIAccess worker under Program Files and requires a valid trusted signature. It +verifies the archive against that release's `checksums.txt`. It caches the native library and Node adapter. Windows installs use the normal input path by default. Explicit `CUA_DRIVER_USE_UIACCESS=1` stages the optional worker under Program Files and requires a valid trusted signature. It does not install the driver application or daemon. The macOS payload requires macOS 13 or newer. Desktop calls return a typed `ToolResult` with text, diff --git a/packages/cua-driver/typescript/computer-use/README.md b/packages/cua-driver/typescript/computer-use/README.md index 9656f35930c..b910ef33b0d 100644 --- a/packages/cua-driver/typescript/computer-use/README.md +++ b/packages/cua-driver/typescript/computer-use/README.md @@ -1,260 +1,29 @@ -# @qwen-code/cua-sdk/computer-use +# Computer Use runtime -Computer Use API included in `@qwen-code/cua-sdk`. It uses the typed native -SDK and works in ordinary Node.js or a persistent Node REPL. - -## Platform workflows - -`await computer.getPlatform()` returns `macos`, `windows`, or `linux` from the -connected driver's inventory. It does not infer the target from the CLI or Node -host and does not capture the desktop. Missing or invalid platform metadata -raises `driver_platform_unavailable`; update the driver and SDK before continuing. - -The single [Computer Use Skill](./SKILL.md) contains the complete App workflow -for all three platforms. After changing connections, query the platform again. - -## App workflow - -On macOS, Linux and Windows, bind an application by name, identifier or installation path. The -handle resolves its current native window or owned dialog and owns targeting internally: +Register `@qwen-code/node-repl-mcp` as an MCP server. Its `js` tool preloads the global `cua` object and displays the current platform's documentation. Use `cua.rewriteDocumentation()` after context compaction. ```js -import { ComputerUse } from "@qwen-code/cua-sdk/computer-use"; - -const computer = await ComputerUse.create(); -const app = await computer.getApp("Microsoft Excel"); -console.log((await app.getState()).text); -// Use an ID from the returned state. -await app.click(37); -await app.typeText("hello"); -console.log((await app.getState()).text); -await computer.close(); +var app = await cua.getApp('TextEdit'); // full AX text, no image +await app.click(3); +await app.typeText('Hello'); +await app.getAXState(); // driver-rendered diff ``` -`getApp()` binds identity without launching. `getState()` can open a discovered -stopped app through the native background launcher; actions never restart an app. -Ambiguous names or identifiers require a unique installation path from -the caller. Public `listApps()` returns only `id`, `displayName` and -`isRunning`; internal process and window addressing stays on the app handle. -Native selection uses focused/main AX windows and sheets on macOS, native -active/stacking order and transient ownership on X11, and foreground/Z-order -plus owned popup relationships on Windows. Ambiguous or unavailable targets -fail without dispatching input. Wayland requires compositor support for window -discovery and activation; an unsupported desktop reports the limitation. +`getAXState`, `getScreenshot` and `getAXStateAndScreenshot` return strings and display their output automatically. `{emit:false}` suppresses display. `{forceFull:true,maxTextChars:20000}` requests a larger complete AX view. Screenshot-only calls skip accessibility walking. Coordinates use the latest screenshot of that window. Actions return void; observe their result in the same JavaScript call. A dispatched action is never automatically replayed. -App methods accept short observed IDs or screenshot coordinates, and do not -accept process IDs, window IDs, opaque tokens or delivery options. -`getState()` returns `{ app, window, mode, text, screenshot? }`. Native accessibility projections preserve controls, meaningful state and text -and render compact full/diff/no-change output. macOS additionally removes -redundant layout/text structure and includes immediate menu-bar items. Its -selected open menu supplies nested and disabled commands. -Normal actions do not emit another full tree or image. +The native driver owns application/window resolution, short IDs, snapshot cursors, rendering, text budgets, delivery and errors. Each app operation uses one SDK call. IDs are scoped to the current app window and authenticated session. Errors preserve `code`, `reason`, `sent` and `next` across direct MCP and JavaScript. An unconfirmed dispatch displays an advisory before returning. -App text defaults to at most 12,000 characters; `maxTextChars` (minimum 512) -adjusts the limit. Warnings and truncation notices appear before whole captured -rows. To see more text, call `app.getState({ disableDiff: true, maxTextChars: 24000 })`. -The App retains current short-ID action bindings even when text is truncated. -A traversal-limited capture can return no-change when its captured state is -identical; changed bounded state returns full. Read failures require using only -IDs from the latest observation. Omitted rows do not prove absence. +The direct `qwen-cua-driver mcp` entry point exposes the same app operations and parameters, using snake_case names. `computer-use/interface.json` is generated from the Rust contract with `cargo run -p cua-driver-contract --bin cua-contract-gen -- computer-use`; the same source supplies MCP descriptions and runtime documentation. Set the host environment `CUA_DRIVER_EXTRA_TOOLS=1` to also list advanced native tools. The underlying typed SDK remains available from `@qwen-code/cua-sdk` for native integrations. -Call `getState()` after a dialog, sheet or menu opens or closes before acting -on its IDs. A process/window/session change invalidates prior IDs. Every App -observation captures a current screenshot internally so a later AX-only -diff/no-change does not discard the coordinate frame. The default return omits -that image; use `getState({ includeScreenshot: true })` when the caller needs to -inspect it. +`ComputerUse.create()` owns an embedded native runtime and its authenticated session. Standalone daemon clients use the low-level `CuaDriver.connect(socketPath)` transport. The compatibility `ComputerUse.connect()` factory cannot bind a trusted facade session to an ordinary `serve` daemon. Close the owning transport when finished. Node REPL supplies cancellation barriers and waits for native cleanup after dispatch. -Native code selects semantic or synthesized input after checking the target. -App input makes one guarded activation of the exact target, dispatches once and -restores the prior app. Failed, partial, unverifiable and cancelled -possible-dispatch actions are never replayed. -Errors request fresh observation before another action; they do not ask the -model to choose a delivery mode. Argument errors detected before native dispatch -retain their specific correction; uncertain post-dispatch failures retain the -cautious observe-before-retry message. +Linux supports X11 applications, including X11 apps under XWayland. Inherit DISPLAY, XAUTHORITY and the desktop accessibility session bus. Native Wayland targets are unsupported. -## macOS text operations -`app.paste(text, { format?, signal? })` pastes once into the current app window. -`format` defaults to `text`; `md` and `html` supply formatted content, and the -receiving app chooses which supplied format it accepts. The clipboard is restored -only while the transaction still owns it, preserving newer external clipboard -changes. -The App method activates the exact window only for the Command-V dispatch and -restores the previous foreground app. The exact-window `computer.paste(...)` -method retains PID-addressed background delivery. +## Explicit desktop and browser interfaces -`app.selectText(element, text, { prefix?, suffix?, selection?, signal? })` uses a -current short element ID and selects one exact, case-sensitive text match. Prefix -and suffix are optional immediately adjacent context; no match or multiple matches -fail. `selection` defaults to `text`; `cursor_before` and `cursor_after` place the -insertion point at that boundary. The element must support writable text selection. +`cua.desktop.getScreenshot()` displays the primary display. Desktop point actions use pixels from that session's latest image; changed display geometry invalidates it. `click({x,y},{key:'shift'})` holds modifiers around one click. `keyDown/keyUp` are session-owned desktop operations; use try/finally and release keys before app input or ordinary text/chords. Normal reset, session end and runtime shutdown release held keys; abrupt process termination cannot promise release. -```js -await app.selectText(37, "draft", { prefix: "Status: " }); -console.log((await app.getState()).text); -// After confirming the intended selection: -await app.paste("ready"); -console.log((await app.getState()).text); -``` +`cua.getBrowser('Chrome')` displays that browser's documentation and tabs. `getTab(id,{browser:'Chrome'})` or an unambiguous `{url}` displays its DOM snapshot; `createBrowserTab('Chrome',url)` opens and displays a tab. These return the existing browser-use SDK objects without changing their methods or behavior. Browser extension setup remains required. The browser SDK is a same-version npm dependency, separate from the native driver. -Both methods return the native action effect. An error, cancellation or completed -dispatch does not establish what changed; observe before deciding whether to retry. -Neither method accepts delivery options. Exact-window callers can use -`computer.paste({ pid, windowId, text, format? })` or -`computer.selectText({ pid, windowId, elementToken, text, prefix?, suffix?, selection? })`. -These operations are macOS-only and reject other driver platforms before mutation. - -## Exact-window SDK compatibility - -The lower-level `ComputerUse` methods remain available to programmatic clients. -The bundled model Skill uses the App workflow on all platforms. The rest of this document describes the existing exact-window contract. - -## Observation revisions - -Observation uses the driver's versioned -`accessibility.observation_revision.v1` capability. `ComputerUse` keeps one -revision cursor per exact `pid` and `windowId`. The first successful observation -is full; later calls automatically request a validated `diff` / `no_change` -response from the last successful observation for that surface. Different -windows are isolated, and reconnecting clears every cursor before observation -resumes. - -Pass `disableDiff: true` only when a fresh complete tree is required. It maps to -the native force-full flag for that call alone, and a successful response -becomes the next automatic base. The legacy `forceFull` spelling remains a -compatibility alias; passing both names is rejected. If a base is stale, the -native driver returns a full resync and the wrapper adopts the replacement -revision. The wrapper never computes a second semantic diff. - -Successful native accessibility reads bounded by traversal limits retain a separate -baseline. Identical captured state returns `no_change`; changed bounded state -returns full, since nodes outside the budget cannot be reported as deleted. -`diagnostics.captureComplete` remains false and `captureTruncated` is true. -Read failures invalidate the baseline and receive one automatic retry, including -failures mixed with truncation. `captureReadComplete` distinguishes them from pure -budget truncation, which does not retry. Current snapshot tokens remain available in -`elements`; when `stableElementIds` is false, use only the latest observation's -tokens. `captureIncompleteDetails` explains the capture limitation. - -Drivers that do not advertise the capability keep the legacy full-snapshot -behavior; observations then report `diagnostics.revisionSupported: false`. - -Revision and lineage identifiers are internal to `ComputerUse` and are not -returned on `WindowObservation`. Normal callers receive the current `mode`, an -optional `resyncReason`, AX text/elements, and an optional screenshot. Protocol -metrics live under `diagnostics`; the raw native response is not exposed. -`context` preserves native `backgroundInput`, `degraded`, `degradedReason`, -`escalation`, `windowBounds`, `screenshotScale`, `screenshotFrameValid`, and -`screenshotError` when available. Consult this context before deciding whether -pixel input or an explicit foreground request is appropriate. Coordinates for -SDK actions remain screenshot pixels; `windowBounds` describes screen points. -The native revision's `capture_complete` flag takes precedence over a legacy -root-level flag. - -Treat a full response as the current captured AX state, subject to capture and -text limits. Apply later diffs to that state; a no-change response leaves it -intact within the captured scope. `elements` remains the current captured -actionable list for retained full, diff, and no-change responses. -While the same stable lineage is retained, tokens for unchanged elements remain -current across all three modes; only removed or replaced element tokens become -invalid. - -Text is capped at 12,000 characters by default. `maxTextChars` -(minimum 512) controls this output budget independently of the native -`maxElements` and `maxDepth` capture limits. `diagnostics.textTruncated` and -`textChars` describe the returned text; capture warnings and truncation notices -appear first. Rows are never cut in half. The full captured element array stays -in `elements`, so filter it for the controls or text you need before printing. -For more full text, request `disableDiff: true` with a larger `maxTextChars`. -An omitted row does not prove absence, and the character budget is not a token -count. Do not repeatedly print the entire element array. - -Screenshot capture is independent from the observation revision mode. -`includeScreenshot: true` requests the image; `disableDiff: true` requests a -fresh complete AX tree. Combine them only when both outputs are independently -needed. - -## Delivery defaults - -Actions that support input delivery resolve their mode in this order: - -1. the action's explicit `deliveryMode`; -2. `QWEN_CUA_SDK_DEFAULT_DELIVERY_MODE` (`background` or `foreground`); -3. the connected platform's default: Linux permits guarded focus preparation; - Windows and the macOS exact-window compatibility API retain `background`. - -On Linux, ordinary callers omit delivery options. The facade passes the existing -`foreground` permission ceiling internally, allowing the native driver to choose -semantic or target-addressed input and prepare exact-window focus when needed. -This uses the connected platform, not the Node host OS. Explicit overrides remain -available to legacy programmatic clients. - -The resolved value is passed to the typed driver for every supported action. -Use the environment default when a whole isolated process, such as a test -worker, should consistently use foreground delivery. A per-action value still -overrides it. Invalid environment values fail facade creation, and the public -JavaScript option remains camelCase: `delivery_mode` is rejected instead of -being silently ignored. - -## Exact-window usage - -Programmatic clients can still select an exact window from `listWindows()`. -Use application-specific criteria to identify it before dispatching input. - -```js -import { ComputerUse } from "@qwen-code/cua-sdk/computer-use"; - -const computer = await ComputerUse.create(); // configured in-process runtime + trusted session -try { - const windows = await computer.listWindows(); - const target = windows.find((window) => window.title === "Task document"); - if (!target) throw new Error("Open the task document first"); - - const first = await computer.observeWindow({ - pid: target.pid, - windowId: target.window_id, - }); - // ... deliver first.text downstream, act on element tokens ... - await computer.click({ pid: target.pid, elementToken: first.elements[0].element_token }); - - const second = await computer.observeWindow({ - pid: target.pid, - windowId: target.window_id, - }); - console.log(second.mode); // "diff" | "no_change" | "full" - - await computer.drag({ - pid: target.pid, - windowId: target.window_id, - fromX: 100, - fromY: 100, - toX: 300, - toY: 100, - deliveryMode: "foreground", // explicit last resort when background drag is unavailable - }); - - const complete = await computer.observeWindow({ - pid: target.pid, - windowId: target.window_id, - disableDiff: true, - }); - console.log(complete.mode); // "full" -} finally { - await computer.close(); -} -``` - -`ComputerUse.connect({ socketPath })` instead binds the same trusted-session -surface to a caller-selected daemon. In-process use inherits the host process's -platform accessibility permissions; daemon use inherits the selected daemon's -identity and permissions. - -## Tests - -- `npm test` — hermetic facade tests against a fake driver handle and Skill packaging checks. -- `npm run test:e2e` — standalone high-level wrapper run against a real target; - set `COMPUTER_USE_PID` and `COMPUTER_USE_WINDOW`. It uses an isolated - configured runtime by default; set `COMPUTER_USE_SOCKET` only when testing a - specific compatible daemon. Unset target variables skip the suite. +Direct MCP uses the same operation names with `scope:'desktop'` instead of `app`. The default manifest contains 19 operations. Internal start/end lifecycle calls remain callable without enlarging the default inventory. diff --git a/packages/cua-driver/typescript/computer-use/app.js b/packages/cua-driver/typescript/computer-use/app.js index da2bc8f46d8..86293b9876b 100644 --- a/packages/cua-driver/typescript/computer-use/app.js +++ b/packages/cua-driver/typescript/computer-use/app.js @@ -1,293 +1,17 @@ -import { ComputerUseError } from "./index.js"; -import { posix, win32 } from "node:path"; - -const appQueues = new WeakMap(); - -const MANAGED_OPTIONS = new Set([ - "pid", "windowId", "window_id", "elementToken", "element_token", - "deliveryMode", "delivery_mode", "foreground", "background", "appContext", -]); - -function optionsForApp(options = {}) { - for (const key of Object.keys(options)) { - if (MANAGED_OPTIONS.has(key)) { - throw new ComputerUseError(`${key} is managed by the app handle`, { - code: "app_option_managed", - }); - } - } - return options; -} - -function appPath(value) { - if (typeof value !== "string") return undefined; - if (value.startsWith("/")) return posix.normalize(value).replace(/\/$/, ""); - if (win32.isAbsolute(value) && !value.startsWith("/")) { - return win32.normalize(value).replace(/\\$/, "").toLocaleLowerCase(); - } -} - -function hasWindowsLaunchCommand(app) { - return typeof app.launch_path === "string" && app.launch_path && - typeof app.bundle_id === "string" && win32.isAbsolute(app.bundle_id) && - !app.bundle_id.startsWith("/") && app.launch_path !== app.bundle_id; -} - -export function appIdentity(app) { - // A Windows launcher can include case-sensitive arguments. It is an opaque - // identity, not a filesystem path to normalize or a command to reconstruct. - if (hasWindowsLaunchCommand(app)) { - return app.launch_path; - } - return appPath(app.launch_path) || app.bundle_id || app.name; -} - -export function resolveApp(apps, selector, { allowStopped = false } = {}) { - if (typeof selector !== "string" || !selector.trim()) { - throw new ComputerUseError("getApp requires an application name, identifier or path"); - } - const path = appPath(selector); - const needle = selector.trim().toLocaleLowerCase(); - const matches = apps.filter((app) => appIdentity(app) === selector || (path - ? [hasWindowsLaunchCommand(app) ? undefined : app.launch_path, app.bundle_id].some((value) => appPath(value) === path) - : [app.name, app.bundle_id].some((value) => typeof value === "string" && value.toLocaleLowerCase() === needle))); - const running = matches.filter((app) => - app.running !== false && Number.isSafeInteger(app.pid) && app.pid > 0); - const candidates = allowStopped ? matches : running; - if (candidates.length !== 1) { - throw new ComputerUseError( - candidates.length > 1 - ? `Application ${JSON.stringify(selector)} is ambiguous; use a unique application path.` - : `No running app matched ${JSON.stringify(selector)}. Open the app, then observe again.`, - { code: candidates.length > 1 ? "app_ambiguous" : "app_not_running" }, - ); - } - return candidates[0]; -} - -function currentWindow(windows, { allowNone = false } = {}) { - const selected = windows.filter((window) => window.is_app_target === true && - Number.isSafeInteger(window.window_id ?? window.windowId)); - if (allowNone && selected.length === 0 && windows.every((window) => !window.title?.trim())) return undefined; - if (selected.length !== 1) { - throw new ComputerUseError("The native app window could not be determined. Open or select its window, then observe again.", { - code: "app_window_unavailable", - }); - } - return selected[0]; -} - +/** Typed arguments and same-application serialization; all target state lives in the driver. */ export class ComputerUseApp { #computer; - #listApps; - #launch; - #identity; - #pid; - #window; - #observation; - #elements = new Map(); - #generation; - - constructor(computer, app, launch, listApps) { - this.#launch = launch; + #selector; + #tail = Promise.resolve(); + constructor(computer, selector, name) { this.#computer = computer; - this.#listApps = listApps; - this.#identity = appIdentity(app); - this.#pid = app.pid; - this.#generation = computer.connectionGeneration; - Object.defineProperty(this, "name", { value: app.name || this.#identity, enumerable: true }); + this.#selector = selector; + this.name = name; } - - #serial(operation) { - const next = (appQueues.get(this.#computer) ?? Promise.resolve()).then(operation); - appQueues.set(this.#computer, next.catch(() => undefined)); - return next; + call(name, args = {}, options = {}) { + const pending = this.#tail.catch(() => {}).then(() => + this.#computer.appCall(name, { ...args, app: this.#selector }, options)); + this.#tail = pending; + return pending; } - - #invalidate() { - this.#elements.clear(); - this.#observation = undefined; - this.#window = undefined; - } - - async #target(signal, { launch = false, allowNoWindow = false } = {}) { - let app; - try { - app = resolveApp(await this.#listApps({ signal, runningOnly: true }), this.#identity); - } catch (error) { - if (!launch || error.code !== "app_not_running") throw error; - this.#invalidate(); - await this.#launch(signal); - app = resolveApp(await this.#listApps({ signal, runningOnly: true }), this.#identity); - } - if (app.pid !== this.#pid) this.#invalidate(); - this.#pid = app.pid; - if (this.#generation !== this.#computer.connectionGeneration) { - this.#invalidate(); - this.#generation = this.#computer.connectionGeneration; - } - const windows = await this.#computer.listWindows({ pid: this.#pid, onScreenOnly: false, appContext: true, signal }); - if (this.#generation !== this.#computer.connectionGeneration) { - this.#invalidate(); - this.#generation = this.#computer.connectionGeneration; - } - const window = currentWindow(windows, { allowNone: allowNoWindow }); - if (!window) return { pid: this.#pid }; - return { window, pid: window.pid ?? this.#pid, windowId: window.window_id ?? window.windowId, key: `${window.pid ?? this.#pid}:${window.window_id ?? window.windowId}` }; - } - - async #observe(options = {}, resolved) { - optionsForApp(options); - if (options.includeScreenshot !== undefined && typeof options.includeScreenshot !== "boolean") { - throw new ComputerUseError("includeScreenshot must be a boolean"); - } - const exposeScreenshot = options.includeScreenshot === true; - const target = resolved ?? await this.#target(options.signal, { launch: true, allowNoWindow: true }); - if (!target.window) { - this.#invalidate(); - this.#generation = this.#computer.connectionGeneration; - return { app: this.name, window: "", mode: "full", text: "No open application window." }; - } - const changed = this.#window !== target.key; - if (changed) this.#invalidate(); - let state; - try { - state = await this.#computer.observeWindow({ - ...options, - includeScreenshot: true, - pid: target.pid, - windowId: target.windowId, - appContext: true, - ...(changed ? { disableDiff: true } : {}), - }); - } catch (error) { - this.#invalidate(); - throw error; - } - this.#window = target.key; - this.#observation = state; - this.#generation = this.#computer.connectionGeneration; - this.#elements = new Map(state.elements.flatMap((element) => { - const id = element.element_id ?? (state.diagnostics.revisionSupported ? undefined : element.element_index); - return Number.isSafeInteger(id) && typeof element.element_token === "string" - ? [[id, element]] : []; - })); - return { - app: this.name, - window: target.window.title ?? "", - mode: state.mode, - text: state.text, - ...(exposeScreenshot && state.screenshot ? { screenshot: state.screenshot } : {}), - }; - } - - getState(options = {}) { - return this.#serial(async () => { - try { - return await this.#observe(options); - } catch (error) { - throw this.#publicError(error, true); - } - }); - } - - #address(point, target) { - if (Number.isSafeInteger(point) && point >= 0) { - const element = this.#elements.get(point); - if (!element) { - throw new ComputerUseError(`Element ${point} is not actionable in the current observation. Call app.getState().`, { - code: "app_element_unavailable", - }); - } - return { pid: target.pid, windowId: target.windowId, elementToken: element.element_token }; - } - optionsForApp(point); - if (!point || !Number.isFinite(point.x) || !Number.isFinite(point.y)) { - throw new ComputerUseError("Provide a short element ID or screenshot coordinates { x, y }"); - } - if (!this.#observation?.screenshot?.images.length || this.#observation.context.screenshotFrameValid === false) { - throw new ComputerUseError("Coordinates require a current screenshot. Call app.getState({ includeScreenshot: true }).", { - code: "app_screenshot_required", - }); - } - return { pid: target.pid, windowId: target.windowId, x: point.x, y: point.y }; - } - - #publicError(error, observing = false) { - if (typeof error?.code === "string" && error.code.startsWith("app_")) return error; - const operation = error?.details?.operation; - if (error instanceof ComputerUseError && - (operation?.dispatched === false || - (operation === undefined && error.details === undefined))) { - return error; - } - const refusalReason = !observing && error?.details?.effect === "refused" - && error?.code === "off_space_or_ax_unresolved" - ? "the target window could not be verified on the current desktop" : undefined; - return new ComputerUseError( - observing ? "The app state could not be read. Check that its window is available, then call app.getState() again." : - (refusalReason ? `Native input refused (${error.code}): ${refusalReason}. ` : "") + - "The app action could not be completed or confirmed. It may already have affected the app. " + - "Call app.getState() before deciding whether to retry.", - { code: error?.code, details: { operation: error?.details?.operation } }, - ); - } - - #act(method, point, options = {}, elementRequired = false) { - optionsForApp(options); - return this.#serial(async () => { - try { - const target = await this.#target(options.signal); - if (this.#window !== target.key || !this.#observation) { - this.#invalidate(); - if (point !== undefined || elementRequired || method === "drag") { - throw new ComputerUseError("The app's window or session changed. Call app.getState() before using an element or coordinates.", { - code: "app_observation_required", - }); - } - await this.#observe({ signal: options.signal }, target); - } - if (elementRequired && !Number.isSafeInteger(point)) { - throw new ComputerUseError("This action requires a short element ID", { code: "app_element_required" }); - } - let address = point === undefined - ? { pid: target.pid, windowId: target.windowId } - : this.#address(point, target); - if (method === "drag") { - if (![options.fromX, options.fromY, options.toX, options.toY].every(Number.isFinite)) { - throw new ComputerUseError( - "drag requires flat, finite fromX, fromY, toX and toY coordinates", - ); - } - this.#address({ x: options.fromX, y: options.fromY }, target); - this.#address({ x: options.toX, y: options.toY }, target); - } - const semantic = ["setValue", "performSecondaryAction", "paste", "selectText"].includes(method); - address = { ...options, ...address, ...(semantic ? {} : { deliveryMode: "foreground" }) }; - if (["click", "doubleClick", "rightClick", "drag", "scroll", "typeText", "paste"].includes(method)) address.appContext = true; - const result = await this.#computer[method](address); - const nativeEffects = ["confirmed", "partial", "unverifiable", "suspected_noop", "refused"]; - const nativeEffect = result.action?.effect; - return { effect: result.effect ?? - (Number.isInteger(nativeEffect) ? nativeEffects[nativeEffect] : nativeEffect) ?? "unverifiable" }; - } catch (error) { - throw this.#publicError(error); - } - }); - } - - click(point, options) { return this.#act("click", point, options); } - doubleClick(point, options) { return this.#act("doubleClick", point, options); } - rightClick(point, options) { return this.#act("rightClick", point, options); } - scroll(point, options) { return this.#act("scroll", point, options); } - drag(options) { return this.#act("drag", undefined, options); } - setValue(element, value, options) { return this.#act("setValue", element, { ...options, value }, true); } - performSecondaryAction(element, action, options) { - return this.#act("performSecondaryAction", element, { ...options, action }, true); - } - typeText(text, options) { return this.#act("typeText", undefined, { ...options, text }); } - paste(text, options) { return this.#act("paste", undefined, { ...options, text }); } - selectText(element, text, options) { return this.#act("selectText", element, { ...options, text }, true); } - pressKey(key, options) { return this.#act("pressKey", undefined, { ...options, key }); } - hotkey(keys, options) { return this.#act("hotkey", undefined, { ...options, keys }); } } diff --git a/packages/cua-driver/typescript/computer-use/browser.js b/packages/cua-driver/typescript/computer-use/browser.js new file mode 100644 index 00000000000..ba7828a306b --- /dev/null +++ b/packages/cua-driver/typescript/computer-use/browser.js @@ -0,0 +1,56 @@ +export function createBrowserFacade(runtime, load = () => import('@qwen-code/browser-use')) { + let module; + let agent; + let closed = false; + const browsers = new Map(); + async function connected() { + if (closed) throw new Error('Computer Use browser session is closed.'); + const loaded = module ?? await load(); + if (closed) throw new Error('Computer Use browser session is closed.'); + module = loaded; + return (agent ??= await module.setupBrowserRuntime()); + } + async function getBrowser(selector) { + const current = await connected(); + const inventory = await current.browsers.list(); + const id = typeof selector === 'string' ? selector : selector?.id; + const matches = inventory.filter((browser) => id === undefined || browser.id === id || browser.name === id); + if (matches.length !== 1) { + throw new Error(`Select one exact browser ID or name. Available browsers: ${JSON.stringify(inventory)}`); + } + const info = matches[0]; + if (!browsers.has(info.id)) { + const browser = await current.browsers.get(info.id); + runtime.write(await browser.documentation()); + runtime.write(await browser.tabs.list()); + browsers.set(info.id, browser); + } + return browsers.get(info.id); + } + async function show(tab) { + runtime.write(await tab.playwright.domSnapshot()); + return tab; + } + return { + getBrowser, + async getTab(reference, { browser: selector } = {}) { + const browser = await getBrowser(selector); + if (typeof reference === 'string') return show(await browser.tabs.get(reference)); + if (typeof reference?.url !== 'string') throw new TypeError('getTab expects a tab ID or {url}.'); + const matches = (await browser.tabs.list()).filter((tab) => tab.url === reference.url); + if (matches.length !== 1) throw new Error(`Expected one tab with URL ${reference.url}; found ${matches.length}. Select its tab ID.`); + return show(await browser.tabs.get(matches[0].id)); + }, + async createBrowserTab(selector, url) { + const browser = await getBrowser(selector); + const tab = await browser.tabs.new(); + if (url !== undefined) await tab.goto(url); + return show(tab); + }, + async close() { + closed = true; + browsers.clear(); + if (module) await module.closeBrowserRuntime(); + }, + }; +} diff --git a/packages/cua-driver/typescript/computer-use/documentation.js b/packages/cua-driver/typescript/computer-use/documentation.js new file mode 100644 index 00000000000..8129bc81fc7 --- /dev/null +++ b/packages/cua-driver/typescript/computer-use/documentation.js @@ -0,0 +1,29 @@ +import manifest from './interface.json' with { type: 'json' }; + +const platformGuidance = { + darwin: 'macOS: use application names, bundle IDs or absolute application paths.', + win32: 'Windows: use an exact application name or executable path. Input delivery is managed by the driver.', + linux: 'Linux: only X11 applications are supported. The server must inherit DISPLAY, XAUTHORITY and desktop accessibility bus variables.', +}; + +export function documentation(platform = process.platform) { + return [ + 'Computer Use JavaScript runtime', + 'Global cua is already loaded. Reuse handles and bindings across calls; use assignment, var or block scope to avoid redeclaring const/let.', + 'Use await for every operation. Calls finish with a terminal result; timeout_ms defaults to 30000. Timeout may leave external effects; observe before retrying. js_reset discards the session.', + 'After context compaction, call cua.rewriteDocumentation() before continuing.', + platformGuidance[platform] ?? 'This platform has no native Computer Use backend.', + 'Start with var app = await cua.getApp("Application"); it displays full AX state without a screenshot. cua.getState() displays application and window inventory.', + 'app.getAXState(options?) returns and automatically displays AX text. Subsequent observations use diff; forceFull:true requests a full tree.', + 'app.getScreenshot(options?) displays the current window image. app.getAXStateAndScreenshot(options?) displays both AX and image. Observation methods return strings; emit:false suppresses automatic output.', + 'Use the short numeric element IDs from the latest observation of this window. Coordinates {x,y} are window screenshot pixels and require a current screenshot.', + 'Actions: await app.click(idOrPoint), doubleClick(idOrPoint), rightClick(idOrPoint), scroll(idOrPoint,{direction,amount}), drag({fromX,fromY,toX,toY}), setValue(id,value), typeText(text), pressKey(key), hotkey(keys). Actions return void. Confirm with an observation in the same call.', + 'For an explicit desktop task, await cua.desktop.getScreenshot() then use cua.desktop.click({x,y}), doubleClick, rightClick, scroll({x,y},{direction,amount}) or drag({fromX,fromY,toX,toY}). Coordinates are pixels in the latest primary-display image. Desktop input operates the visible foreground desktop and can move the real pointer.', + 'Desktop keyboard: typeText(text), pressKey(key), hotkey(keys). To hold a key use try { await cua.desktop.keyDown("shift"); await cua.desktop.click({x,y}); } finally { await cua.desktop.keyUp("shift"); }. Held keys belong to this session; use keyDown/keyUp for chords while holding keys. Release them before text, normal chords, explicit click modifiers or app input. Session end/reset releases owned keys.', + 'Browser: var browser = await cua.getBrowser("Chrome") selects an exact browser ID or name and displays its documentation and tabs. var tab = await cua.getTab(tabId,{browser:"Chrome"}) or cua.getTab({url},{browser:"Chrome"}) selects one tab and displays its DOM snapshot. cua.createBrowserTab("Chrome",url) opens a tab and displays its state. These return the repository browser-use SDK handles unchanged: use browser.tabs/user and tab.playwright/cua/dom_cua. Read the selected browser documentation before acting. Browser selection requires the existing Qwen browser extension connection.', + ...manifest.tools.map((tool) => `${['getState', 'getApp'].includes(tool.method) || tool.method.startsWith('desktop.') ? 'cua' : 'app'}.${tool.method}: ${tool.description}`), + 'Do not wrap observations in nodeRepl.write or emitImage: they display themselves. For other output use nodeRepl.write(value) and await nodeRepl.emitImage(bytesOrDataUrl). Plain expressions produce no output.', + 'Imports use await import(); Node modules run with the server permissions. nodeRepl.cwd, homeDir, tmpDir and signal are available. A timeout stops JS continuation; actions already dispatched are never automatically replayed.', + 'Once the requested result is visible, stop and report it.', + ].join('\n'); +} diff --git a/packages/cua-driver/typescript/computer-use/index.d.ts b/packages/cua-driver/typescript/computer-use/index.d.ts index c5e370d52eb..875fd0cefad 100644 --- a/packages/cua-driver/typescript/computer-use/index.d.ts +++ b/packages/cua-driver/typescript/computer-use/index.d.ts @@ -1,378 +1,38 @@ -import type { - ActionResult as NativeActionResult, - SessionOutput as NativeSessionOutput, - VerifyStateOutput as NativeVerifyStateOutput, -} from "../dist/index.js"; - -export type JsonPrimitive = string | number | boolean | null; -export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; -export type JsonObject = { [key: string]: JsonValue }; - -export interface ComputerUseOptions { +import type { SessionOutput } from '../dist/index.js'; +export interface CallOptions { signal?: AbortSignal } +export interface ComputerUseOptions extends CallOptions { session?: string; - /** Opt into a finite session lifetime. Omit both TTL fields for owner-lifetime persistence. */ sessionTtlSeconds?: number; - /** Opt into a finite idle lifetime. Omit both TTL fields for owner-lifetime persistence. */ idleTtlSeconds?: number; - signal?: AbortSignal; } - -export interface CallOptions { - signal?: AbortSignal; -} - -export type AppPoint = number | { x: number; y: number }; - -export interface AppObservationOptions extends CallOptions { - disableDiff?: boolean; - /** Returned text budget. Default 12,000; minimum 512. */ - maxTextChars?: number; - /** Expose the screenshot captured with this App observation. */ - includeScreenshot?: boolean; -} - -export interface ComputerUseApplication { - id: string; - displayName: string; - isRunning: boolean; -} - -export interface AppObservation { - app: string; - window: string; - mode: "full" | "diff" | "no_change"; +export interface ComputerUseConnectOptions extends ComputerUseOptions { socketPath?: string } +export interface DriverObservation { text: string; - screenshot?: ComputerUseScreenshot; + structured?: Record<string, unknown>; + images: Array<{ mimeType: string; dataBase64: string }>; } - -export interface AppActionResult { - effect: ActionEffect; -} - -export interface PasteOptions extends CallOptions { - format?: "text" | "md" | "html"; -} - -export interface TextSelectionOptions extends CallOptions { - prefix?: string; - suffix?: string; - selection?: "text" | "cursor_before" | "cursor_after"; -} - -export interface ComputerUseApp { - readonly name: string; - getState(options?: AppObservationOptions): Promise<AppObservation>; - click(point: AppPoint, options?: CallOptions & { button?: "left" | "right" | "middle"; count?: number }): Promise<AppActionResult>; - doubleClick(point: AppPoint, options?: CallOptions): Promise<AppActionResult>; - rightClick(point: AppPoint, options?: CallOptions & { modifier?: string[] }): Promise<AppActionResult>; - scroll(point: AppPoint, options: CallOptions & { direction: "up" | "down" | "left" | "right"; by?: "line" | "page"; amount?: number }): Promise<AppActionResult>; - drag(options: CallOptions & { fromX: number; fromY: number; toX: number; toY: number; durationMs?: number; steps?: number; button?: "left" | "right" | "middle"; modifier?: string[] }): Promise<AppActionResult>; - setValue(element: number, value: string, options?: CallOptions): Promise<AppActionResult>; - performSecondaryAction(element: number, action: string, options?: CallOptions): Promise<AppActionResult>; - typeText(text: string, options?: CallOptions & { delayMs?: number }): Promise<AppActionResult>; - /** macOS only. Paste once and restore the clipboard unless another writer changed it. */ - paste(text: string, options?: PasteOptions): Promise<AppActionResult>; - /** macOS only. Select a unique match in an observed text element. */ - selectText(element: number, text: string, options?: TextSelectionOptions): Promise<AppActionResult>; - pressKey(key: string, options?: CallOptions & { modifiers?: string[] }): Promise<AppActionResult>; - hotkey(keys: string[], options?: CallOptions): Promise<AppActionResult>; -} - -export interface DeliveryOptions { - /** Legacy override. Omit on Linux to let native input prepare exact-window focus. */ - deliveryMode?: "background" | "foreground"; -} - -export interface ComputerUseConnectOptions extends ComputerUseOptions { - socketPath?: string; -} - -export interface WindowRef { - pid: number; - windowId: number; -} - -export interface ElementRef { - pid: number; - elementToken: string; - windowId?: number; - x?: never; - y?: never; -} - -export interface CoordinateRef extends WindowRef { - x: number; - y: number; - elementToken?: never; -} - -export type PointOrElementRef = CoordinateRef | ElementRef; -export type ExactActionRef = (WindowRef & { elementToken?: never }) | ElementRef; - -export interface ComputerUseElement { - [key: string]: unknown; - element_index?: number; - element_id?: number; - element_token?: string; - role?: string; - label?: string; - automation_id?: string; - value?: JsonValue; - enabled?: boolean; - actions?: string[]; - selected?: boolean; -} - -export interface ComputerUseScreenshot { - width?: number; - height?: number; - mimeType?: string; - filePath?: string; - images: Array<{ - mimeType: string; - dataBase64: string; - }>; -} - -export interface ComputerUseObservationDiagnostics { - revisionSupported: boolean; - stableElementIds: boolean; - captureComplete?: boolean; - captureReadComplete?: boolean; - captureTruncated: boolean; - captureIncompleteDetails: string[]; - textTruncated: boolean; - textChars: number; - serializerVersion?: string; - projectionVersion?: string; - selectedBytes?: number; - fullBytes?: number; - estimatedTokens?: number; - serializerDurationUs?: number; - cacheEstimateBytes?: number; -} - -export interface ObserveWindowOptions extends WindowRef, CallOptions { - disableDiff?: boolean; - /** @deprecated Use disableDiff. */ - forceFull?: boolean; - includeScreenshot?: boolean; - screenshotOutFile?: string; - maxElements?: number; - maxDepth?: number; - /** Returned text budget, independent of capture limits. Default 12,000; minimum 512. */ - maxTextChars?: number; -} - -export interface WindowObservation { - pid: number; - windowId: number; - mode: "full" | "diff" | "no_change"; - resyncReason?: string; - text: string; - elements: ComputerUseElement[]; - screenshot?: ComputerUseScreenshot; - context: { - backgroundInput?: JsonObject; - degraded?: boolean; - degradedReason?: string; - escalation?: JsonObject; - windowBounds?: { x: number; y: number; width: number; height: number }; - screenshotScale?: number; - screenshotFrameValid?: boolean; - screenshotError?: JsonObject; - }; - diagnostics: ComputerUseObservationDiagnostics; -} - -export interface VerifyStateOptions extends WindowRef, CallOptions { - expect: JsonObject[]; - timeoutMs?: number; - stableSamples?: number; - includeScreenshot?: boolean; -} - -export type ClickOptions = PointOrElementRef & - CallOptions & - DeliveryOptions & { - button?: "left" | "right" | "middle"; - count?: number; - }; - -export type RightClickOptions = PointOrElementRef & - CallOptions & - DeliveryOptions & { - modifier?: string[]; - }; - -export interface DragOptions extends WindowRef, CallOptions, DeliveryOptions { - fromX: number; - fromY: number; - toX: number; - toY: number; - durationMs?: number; - steps?: number; - button?: "left" | "right" | "middle"; - modifier?: string[]; -} - -export type ScrollOptions = PointOrElementRef & - CallOptions & - DeliveryOptions & { - direction: "up" | "down" | "left" | "right"; - by?: "line" | "page"; - amount?: number; - }; - -export interface ElementValueOptions extends ElementRef, CallOptions { - value: string; -} - -export type TextOptions = ExactActionRef & - CallOptions & - DeliveryOptions & { - text: string; - delayMs?: number; - }; - -export type KeyOptions = ExactActionRef & - CallOptions & - DeliveryOptions & { - key: string; - modifiers?: string[]; - }; - -export type HotkeyOptions = ExactActionRef & - CallOptions & - DeliveryOptions & { - keys: string[]; - }; - -export interface SecondaryActionOptions extends ElementRef, CallOptions { - action: string; -} - -export type ActionEffect = "confirmed" | "partial" | "unverifiable" | "suspected_noop" | "refused"; -export type ActionRoute = - | "accessibility" - | "synthetic_events" - | "global_input" - | "system_api" - | "dom" - | "trusted_input"; - -export interface ComputerUseOperationResult { - id: string; - state: "accepted" | "dispatched" | "committed" | "completed"; - dispatched: boolean; - committed: boolean; - cancellationRequested: boolean; -} - -export interface ComputerUseActionResult { - /** Native action message, including new-window notices when available. */ - text?: string; - effect: ActionEffect; - route: ActionRoute; - delivery?: { - mode: "background" | "foreground" | "not_applicable" | "unknown"; - delivered_count?: number; - }; - evidence?: Array<{ kind: "value_readback" | "window_change" }>; - escalation?: { - target: "pixel" | "foreground" | "page" | "session"; - reason: - | "route_unavailable" - | "delivery_failed" - | "effect_unconfirmed" - | "suspected_noop" - | "permission_required"; - }; - /** Terminal lifecycle evidence for the one native action dispatch. */ - operation: ComputerUseOperationResult; - /** The generated UniFFI record returned alongside the JSON projection. */ - action?: NativeActionResult; -} - -export type VerificationStatus = "satisfied" | "unsatisfied" | "unknown"; -export interface ComputerUseVerificationResult { - status: VerificationStatus; - stable: boolean; - elapsed_ms: number; - samples: number; - predicates: Array<{ - index: number; - status: VerificationStatus; - unknown_reason?: - | "invalid_predicate" - | "unsupported_predicate" - | "untrusted_source" - | "multi_match" - | "target_missing" - | "observation_unavailable" - | "stability_unproven" - | null; - observed_json?: string | null; - }>; - /** The generated UniFFI record returned alongside the JSON projection. */ - verification?: NativeVerifyStateOutput; -} - -export interface ActAndVerifyOptions { - action: () => Promise<ComputerUseActionResult>; - verify: (action: ComputerUseActionResult) => Promise<ComputerUseVerificationResult>; -} - -export interface ActAndVerifyResult { - action: ComputerUseActionResult; - verification: ComputerUseVerificationResult; -} - export class ComputerUseError extends Error { readonly code?: string; + readonly reason?: string; + readonly sent?: boolean; + readonly next?: string; readonly details?: unknown; } - +/** Internal typed session transport. Use createCua for the model-facing API. */ +export class ComputerUseApp { + readonly name: string; + call(name: string, args?: Record<string, unknown>, options?: CallOptions & { readOnly?: boolean }): Promise<DriverObservation>; +} export class ComputerUse { private constructor(); - static create(options?: ComputerUseOptions): Promise<ComputerUse>; static connect(options?: ComputerUseConnectOptions): Promise<ComputerUse>; - readonly connectionGeneration: number; - - supportsObservationRevision(): Promise<boolean>; - getPlatform(options?: CallOptions): Promise<"macos" | "windows" | "linux">; - sessionInfo(options?: CallOptions): Promise<NativeSessionOutput>; - reconnect(options?: CallOptions): Promise<{ - connectionGeneration: number; - operation?: ComputerUseOperationResult; - }>; - listApps(options?: CallOptions): Promise<ComputerUseApplication[]>; - getApp(selector: string, options?: CallOptions): Promise<ComputerUseApp>; - listWindows( - options?: CallOptions & { - pid?: number; - onScreenOnly?: boolean; - }, - ): Promise<JsonObject[]>; - getWindow(options: WindowRef & CallOptions): Promise<JsonObject>; - observeWindow(options: ObserveWindowOptions): Promise<WindowObservation>; - verifyState(options: VerifyStateOptions): Promise<ComputerUseVerificationResult>; - click(options: ClickOptions): Promise<ComputerUseActionResult>; - doubleClick( - options: PointOrElementRef & CallOptions & DeliveryOptions, - ): Promise<ComputerUseActionResult>; - rightClick(options: RightClickOptions): Promise<ComputerUseActionResult>; - drag(options: DragOptions): Promise<ComputerUseActionResult>; - scroll(options: ScrollOptions): Promise<ComputerUseActionResult>; - setValue(options: ElementValueOptions): Promise<ComputerUseActionResult>; - typeText(options: TextOptions): Promise<ComputerUseActionResult>; - paste(options: WindowRef & PasteOptions & { text: string }): Promise<ComputerUseActionResult>; - selectText(options: WindowRef & ElementRef & TextSelectionOptions & { text: string }): Promise<ComputerUseActionResult>; - pressKey(options: KeyOptions): Promise<ComputerUseActionResult>; - hotkey(options: HotkeyOptions): Promise<ComputerUseActionResult>; - performSecondaryAction(options: SecondaryActionOptions): Promise<ComputerUseActionResult>; - actAndVerify(options: ActAndVerifyOptions): Promise<ActAndVerifyResult>; + getPlatform(options?: CallOptions): Promise<'macos' | 'windows' | 'linux'>; + sessionInfo(options?: CallOptions): Promise<SessionOutput>; + reconnect(options?: CallOptions): Promise<{ connectionGeneration: number }>; + appCall(name: string, args?: Record<string, unknown>, options?: CallOptions & { readOnly?: boolean }): Promise<DriverObservation>; + getState(options?: CallOptions): Promise<DriverObservation>; + getApp(selector: string, options?: CallOptions): Promise<{ app: ComputerUseApp; initial: DriverObservation }>; close(): Promise<void>; } diff --git a/packages/cua-driver/typescript/computer-use/index.js b/packages/cua-driver/typescript/computer-use/index.js index 25ab8a40f47..0e3dded7f16 100644 --- a/packages/cua-driver/typescript/computer-use/index.js +++ b/packages/cua-driver/typescript/computer-use/index.js @@ -1,156 +1,34 @@ -/** - * Standalone Computer Use facade over the typed @qwen-code/cua-sdk API. - * The facade owns revision cursors; CuaDriver owns native identity, - * authorization, transport, and cleanup. - */ -import { randomUUID } from "node:crypto"; -import { ComputerUseApp, appIdentity, resolveApp } from "./app.js"; +/** Thin session transport for the driver-owned Computer Use interface. */ +import { randomUUID } from 'node:crypto'; +import { ComputerUseApp } from './app.js'; +import manifest from './interface.json' with { type: 'json' }; -const OBSERVATION_REVISION_CAPABILITY = "accessibility.observation_revision.v1"; -const ACCESSIBILITY_SERIALIZER_VERSION = "accessibility-render-v1"; -const ACCESSIBILITY_PROJECTION_VERSION = "full-tree-v1"; const DEFAULT_EXPLICIT_SESSION_TTL_SECONDS = 60 * 60; const DEFAULT_EXPLICIT_IDLE_TTL_SECONDS = 5 * 60; -const DEFAULT_DELIVERY_MODE_ENV = "QWEN_CUA_SDK_DEFAULT_DELIVERY_MODE"; -const DEFAULT_MAX_TEXT_CHARS = 12_000; -const RECONNECTABLE_SESSION_CODES = new Set([ - "authorization_context_expired", - "session_unavailable", -]); - -async function loadCuaDriver() { - return import("@qwen-code/cua-sdk"); -} +const RECONNECTABLE_SESSION_CODES = new Set(['authorization_context_expired', 'session_unavailable']); +async function loadCuaDriver() { return import('@qwen-code/cua-sdk'); } export class ComputerUseError extends Error { constructor(message, info = {}) { super(message); - this.name = "ComputerUseError"; + this.name = 'ComputerUseError'; this.code = info.code; + this.reason = info.reason; + this.sent = info.sent; + this.next = info.next; this.details = info.details; } } -function publicDeliveryModeGuidance(value) { - if (typeof value === "string") return value.replace(/\bdelivery_mode\b/g, "deliveryMode"); - if (Array.isArray(value)) return value.map(publicDeliveryModeGuidance); - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value).map(([key, entry]) => [key, publicDeliveryModeGuidance(entry)]), - ); - } - return value; -} - function unwrapToolResult(tool, result, operation) { - let structured; - if (typeof result.structuredJson === "string" && result.structuredJson !== "") { - try { - structured = JSON.parse(result.structuredJson); - } catch { - structured = undefined; - } - } + const structured = result.structuredJson ? JSON.parse(result.structuredJson) : undefined; if (result.isError) { - const publicStructured = publicDeliveryModeGuidance(structured); - const code = - (structured && typeof structured.code === "string" && structured.code) || - (structured && typeof structured.refusal?.code === "string" && structured.refusal.code) || - result.errorCode || - undefined; - throw new ComputerUseError(publicDeliveryModeGuidance(result.text) || `${tool} failed`, { - code, - details: - publicStructured && typeof publicStructured === "object" - ? { ...publicStructured, operation } - : { result: publicStructured, operation }, + throw new ComputerUseError(result.text || `${tool} failed`, { + ...structured, code: structured?.code ?? result.errorCode, + details: { ...structured, operation }, }); } - return { - text: result.text, - structured, - images: result.images ?? [], - action: result.action, - verification: result.verification, - degraded: result.degraded === true, - rawJson: result.rawJson, - operation, - }; -} - -function actionResult(result) { - const value = result.structured ?? { text: result.text }; - return { - ...value, - ...(result.text ? { text: result.text } : {}), - ...(result.action === undefined ? {} : { action: result.action }), - operation: result.operation, - }; -} - -function compactApp(app) { - const displayName = typeof app?.name === "string" ? app.name : ""; - const id = typeof app?.bundle_id === "string" && app.bundle_id !== "" - ? app.bundle_id : displayName || "unknown"; - return { id, displayName, isRunning: app?.running === true }; -} - -function verificationResult(result) { - const value = result.structured ?? { status: "unknown", stable: false }; - return result.verification === undefined - ? value - : { ...value, verification: result.verification }; -} - -function captureStatus(structured) { - const revision = structured?.observation_revision; - const details = - revision?.capture_incomplete_details ?? structured?.capture_incomplete_details; - const incompleteDetails = Array.isArray(details) - ? [...new Set(details.filter((detail) => typeof detail === "string"))] - : []; - const isBudgetDetail = (detail) => - /^(walk: max_(elements truncated|depth exceeded)|max_(elements|depth)_reached)$/.test(detail); - return { - complete: revision?.capture_complete ?? structured?.capture_complete, - readComplete: - revision?.capture_read_complete ?? - structured?.capture_read_complete ?? - (incompleteDetails.some((detail) => !isBudgetDetail(detail)) ? false : undefined), - truncated: - revision?.capture_truncated ?? - structured?.capture_truncated ?? - incompleteDetails.some(isBudgetDetail), - incompleteDetails, - }; -} - -function observationText(treeText, capture, maxTextChars, appContext) { - let warning = ""; - if (capture.complete === false) { - warning = - capture.truncated && capture.readComplete !== false - ? "Accessibility capture is incomplete (traversal limit); this view covers captured nodes only.\n" - : `Accessibility capture is incomplete; use current snapshot ${appContext ? "IDs" : "tokens"} only. Retry after the UI settles or use a screenshot.\n`; - if (capture.incompleteDetails.length) { - warning += `Capture details: ${capture.incompleteDetails.join("; ").slice(0, 180)}\n`; - } - } - const lines = treeText.split("\n"); - if (warning.length + treeText.length <= maxTextChars) { - return { text: warning + treeText, truncated: false }; - } - const notice = appContext - ? "Text truncated; call app.getState with disableDiff:true and a larger maxTextChars.\n" - : "Text truncated; inspect current .elements, or request disableDiff:true with a larger maxTextChars.\n"; - const selected = []; - let length = warning.length + notice.length; - for (const line of lines) { - if (length + line.length + 1 > maxTextChars) break; - selected.push(line); - length += line.length + 1; - } - return { text: warning + notice + selected.join("\n"), truncated: true }; + return { text: result.text ?? '', structured, images: result.images ?? [], operation }; } function requirePositiveInteger(name, value, { allowZero = false } = {}) { @@ -240,15 +118,6 @@ function createTrustedSessionAsync(sdk, owner, options) { return sdk.createTrustedSessionAsync(owner, options); } -function requirePid(value) { - return requireIntegerRange("pid", value, 1, 0xffffffff); -} - -function requireStringList(name, value) { - if (!Array.isArray(value)) throw new ComputerUseError(`${name} must be an array`); - return value.map((entry, index) => requireNonEmptyString(`${name}[${index}]`, entry)); -} - function requireNonEmptyString(name, value) { if (typeof value !== "string" || value.length === 0) { throw new ComputerUseError(`${name} must be a non-empty string`); @@ -256,50 +125,6 @@ function requireNonEmptyString(name, value) { return value; } -function observationDisablesDiff(options) { - const hasDisableDiff = options && Object.hasOwn(options, "disableDiff"); - const hasForceFull = options && Object.hasOwn(options, "forceFull"); - if (hasDisableDiff && hasForceFull) { - throw new ComputerUseError("disableDiff cannot be combined with forceFull", { - code: "observation_option_conflict", - }); - } - const name = hasDisableDiff ? "disableDiff" : "forceFull"; - const value = options?.[name]; - if (value !== undefined && typeof value !== "boolean") { - throw new ComputerUseError(`${name} must be a boolean`); - } - return value === true; -} - -function defaultDeliveryMode(environment) { - const value = environment?.[DEFAULT_DELIVERY_MODE_ENV]; - if (value === undefined) return undefined; - if (typeof value !== "string") { - throw new ComputerUseError( - `${DEFAULT_DELIVERY_MODE_ENV} must be background or foreground`, - ); - } - const normalized = value.toLowerCase(); - if (normalized !== "background" && normalized !== "foreground") { - throw new ComputerUseError( - `${DEFAULT_DELIVERY_MODE_ENV} must be background or foreground`, - ); - } - return normalized; -} - -function optionalWindowId(value) { - return value === undefined ? undefined : BigInt(requirePositiveInteger("windowId", value)); -} - -function exactWindow(pid, windowId) { - return { - pid: requirePid(pid), - windowId: BigInt(requirePositiveInteger("windowId", windowId)), - }; -} - function sessionOptions(sdk, options) { const finiteLifetimeRequested = options.sessionTtlSeconds !== undefined || options.idleTtlSeconds !== undefined; @@ -403,12 +228,7 @@ export class ComputerUse { #sessionFactory; #reconnectPromise; #connectionGeneration = 1; - #revisionCursors = new Map(); - #observationQueues = new Map(); #closed = false; - #revisionSupport; - #defaultDeliveryMode; - #connectedPlatform; #apps = new Map(); /** Internal injection seam for hermetic tests. Use create/connect in applications. */ @@ -420,10 +240,9 @@ export class ComputerUse { ownsSession = false, publicSession, sessionFactory, - environment = {}, } = {}, ) { - if (!driver || typeof driver.getWindowState !== "function") { + if (!driver || typeof driver.callTool !== "function") { throw new ComputerUseError("ComputerUse requires a typed driver session"); } this.#driver = driver; @@ -432,7 +251,6 @@ export class ComputerUse { this.#ownsSession = ownsSession; this.#publicSession = publicSession; this.#sessionFactory = sessionFactory; - this.#defaultDeliveryMode = defaultDeliveryMode(environment); } /** Create a same-process configured runtime and one bound standard session. */ @@ -466,7 +284,7 @@ export class ComputerUse { } } - /** Connect to a caller-selected daemon and bind a transport-owned session. */ + /** Compatibility factory; ordinary serve daemons reject trusted session binding. */ static async connect(options = {}) { const signal = options.signal; requireDispatchableSignal("createTrustedSession", signal); @@ -532,7 +350,10 @@ export class ComputerUse { // Do not hand caller cancellation to UniFFI after dispatch. UniFFI drops // the Rust future immediately, while platform work may continue. Awaiting // the native terminal result is what prevents ambiguous late actions. - const value = await awaitNativeTerminal(call.call(driver, input), signal); + const pending = method === "callTool" + ? call.call(driver, input.name, JSON.stringify(input.args)) + : call.call(driver, input); + const value = await awaitNativeTerminal(pending, signal); const refusedEffect = this.#sdk.ActionEffect?.Refused ?? 4; const refused = value?.action?.effect === refusedEffect || value?.action?.effect === "refused"; @@ -597,7 +418,6 @@ export class ComputerUse { async getPlatform(options = {}) { this.#requireOpen(); requireDispatchableSignal("getPlatform", options.signal); - this.#connectedPlatform = undefined; if (typeof this.#owner?.listToolsJson !== "function") { throw new ComputerUseError("the connected driver does not report its platform", { code: "driver_platform_unavailable", @@ -617,7 +437,6 @@ export class ComputerUse { code: "driver_platform_unavailable", }); } - this.#connectedPlatform = platform; return platform; } @@ -682,10 +501,7 @@ export class ComputerUse { this.#publicSession = replacementPublicSession; } this.#connectionGeneration += 1; - this.#revisionSupport = undefined; - this.#connectedPlatform = undefined; - this.#revisionCursors.clear(); - lifecycle.operation.state = "committed"; + lifecycle.operation.state = "committed"; lifecycle.operation.committed = true; lifecycle.operation.state = "completed"; return { @@ -707,658 +523,29 @@ export class ComputerUse { return operation; } - #clickButton(value) { - if (value === undefined) return undefined; - const normalized = requireNonEmptyString("button", value).toLowerCase(); - const values = { - left: this.#sdk.ClickButton?.Left ?? "left", - right: this.#sdk.ClickButton?.Right ?? "right", - middle: this.#sdk.ClickButton?.Middle ?? "middle", - }; - if (!(normalized in values)) throw new ComputerUseError(`unsupported button: ${value}`); - return values[normalized]; + async appCall(name, args = {}, { signal } = {}) { + const readOnly = manifest.tools.find((tool) => tool.name === name)?.read_only === true; + return this.#invoke('callTool', { name, args }, { signal, readOnly }); } - #deliveryMode(value) { - if (value === undefined) return undefined; - const normalized = requireNonEmptyString("deliveryMode", value).toLowerCase(); - const values = { - background: this.#sdk.DeliveryMode?.Background ?? "background", - foreground: this.#sdk.DeliveryMode?.Foreground ?? "foreground", - }; - if (!(normalized in values)) { - throw new ComputerUseError(`unsupported deliveryMode: ${value}`); - } - return values[normalized]; - } - - async #actionDeliveryMode(options) { - if (options && Object.hasOwn(options, "delivery_mode")) { - throw new ComputerUseError("delivery_mode is not supported; use deliveryMode"); - } - const value = options?.deliveryMode === undefined ? this.#defaultDeliveryMode : options.deliveryMode; - if (value !== undefined) return this.#deliveryMode(value); - const platform = this.#connectedPlatform ?? await this.getPlatform({ signal: options?.signal }); - // Pass the possible focus change through native authorization. The Linux - // driver still chooses semantic input before guarded global input. - return this.#deliveryMode(platform === "linux" ? "foreground" : "background"); - } - - #scrollDirection(value) { - const normalized = requireNonEmptyString("direction", value).toLowerCase(); - const values = { - up: this.#sdk.ScrollDirection?.Up ?? "up", - down: this.#sdk.ScrollDirection?.Down ?? "down", - left: this.#sdk.ScrollDirection?.Left ?? "left", - right: this.#sdk.ScrollDirection?.Right ?? "right", - }; - if (!(normalized in values)) throw new ComputerUseError(`unsupported direction: ${value}`); - return values[normalized]; - } - - #scrollBy(value) { - if (value === undefined) return undefined; - const normalized = requireNonEmptyString("by", value).toLowerCase(); - const values = { - line: this.#sdk.ScrollBy?.Line ?? "line", - page: this.#sdk.ScrollBy?.Page ?? "page", - }; - if (!(normalized in values)) throw new ComputerUseError(`unsupported scroll unit: ${value}`); - return values[normalized]; - } - - #windowAddress(options, { coordinates = true, tokenRequired = false } = {}) { - const { pid, windowId, elementToken, x, y } = options ?? {}; - const input = { - pid: requirePid(pid), - windowId: optionalWindowId(windowId), - }; - if (elementToken !== undefined) { - input.elementToken = requireNonEmptyString("elementToken", elementToken); - if (x !== undefined || y !== undefined) { - throw new ComputerUseError("elementToken cannot be combined with x/y"); - } - return input; - } - if (tokenRequired) throw new ComputerUseError("elementToken is required"); - if (!coordinates) { - if (input.windowId === undefined) { - throw new ComputerUseError("provide elementToken or windowId"); - } - return input; - } - if (x === undefined || y === undefined) { - throw new ComputerUseError("provide elementToken or both x and y"); - } - if (input.windowId === undefined) { - throw new ComputerUseError("windowId is required for window-local coordinates"); - } - if (!Number.isFinite(x) || !Number.isFinite(y)) { - throw new ComputerUseError("x and y must be finite numbers"); - } - input.x = x; - input.y = y; - return input; - } - - async #supportsObservationRevision(options) { - this.#requireOpen(); - if (this.#revisionSupport === undefined) { - let advertised = false; - const owner = this.#owner; - const listTools = owner?.listToolsJson; - if (typeof listTools === "function") { - const lifecycle = beginOperation("listToolsJson", options.signal); - try { - if (lifecycle.operation.cancellationRequested) { - cancelledBeforeDispatch("listToolsJson", lifecycle.operation); - } - lifecycle.operation.state = "dispatched"; - lifecycle.operation.dispatched = true; - const listing = JSON.parse( - await awaitNativeTerminal(listTools.call(owner), options.signal), - ); - lifecycle.operation.state = "completed"; - const tools = Array.isArray(listing?.tools) ? listing.tools : []; - const entry = tools.find((tool) => tool?.name === "get_window_state"); - advertised = - Array.isArray(entry?.capabilities) && - entry.capabilities.includes(OBSERVATION_REVISION_CAPABILITY); - } catch (error) { - if (error?.code === "call_cancelled") { - throw error; - } - advertised = false; - } finally { - if (lifecycle.operation.state !== "completed") { - lifecycle.operation.state = "completed"; - } - lifecycle.release(); - } - } - this.#revisionSupport = advertised; - } - return this.#revisionSupport; - } - - async supportsObservationRevision() { - return this.#supportsObservationRevision({}); - } - - async #listAppsDetailed(options = {}) { - const { structured } = await this.#invoke( - "listApps", - options.runningOnly ? { runningOnly: true } : {}, - { - readOnly: true, - signal: options.signal, - }, - ); - return structured?.apps ?? structured ?? []; - } - - async listApps(options = {}) { - try { - await this.getPlatform({ signal: options.signal }); - } catch (error) { - if (error?.code !== "driver_platform_unavailable") throw error; - } - const apps = await this.#listAppsDetailed(options); - return apps.map(compactApp); + async getState(options = {}) { + return this.appCall('get_state', {}, { ...options, readOnly: true }); } async getApp(selector, options = {}) { - const app = resolveApp(await this.#listAppsDetailed(options), selector, { allowStopped: true }); - const identity = appIdentity(app); - if (!this.#apps.has(identity)) { - this.#apps.set(identity, new ComputerUseApp( - this, - app, - async (signal) => { - const platform = await this.getPlatform({ signal }); - const input = platform === "windows" && app.launch_path - ? { name: app.name, launchPath: app.launch_path } - : { name: app.launch_path || app.bundle_id || app.name }; - return this.#invoke("launchApp", input, { signal }); - }, - (listOptions) => this.#listAppsDetailed(listOptions), - )); + const initial = await this.appCall('get_app', { app: selector }, options); + const id = initial.structured.app; + if (!this.#apps.has(id)) { + this.#apps.set(id, new ComputerUseApp(this, id, initial.structured.name)); } - return this.#apps.get(identity); - } - - async listWindows({ pid, onScreenOnly, appContext, signal } = {}) { - const input = {}; - if (pid !== undefined) input.pid = requirePid(pid); - if (onScreenOnly !== undefined) input.onScreenOnly = Boolean(onScreenOnly); - if (appContext) input.appContext = true; - const { structured } = await this.#invoke("listWindows", input, { - readOnly: true, - signal, - }); - return structured?.windows ?? structured ?? []; - } - - async getWindow({ pid, windowId, signal }) { - const target = exactWindow(pid, windowId); - const windows = await this.listWindows({ pid: target.pid, signal }); - const found = windows.find( - (window) => String(window.window_id ?? window.windowId) === String(target.windowId), - ); - if (!found) { - throw new ComputerUseError(`window ${windowId} for pid ${pid} was not found`, { - code: "window_not_found", - }); - } - return found; - } - - async observeWindow(options) { - const cursorField = [ - "baseRevisionId", - "revisionId", - "lineageId", - "observationRevision", - ].find((field) => options && Object.hasOwn(options, field)); - if (cursorField) { - throw new ComputerUseError(`${cursorField} is managed by ComputerUse`, { - code: "revision_cursor_managed", - }); - } - const target = exactWindow(options?.pid, options?.windowId); - const surface = `${target.pid}:${target.windowId}`; - const cursorKey = `${surface}${options?.appContext ? ":app" : ""}`; - const previous = this.#observationQueues.get(surface); - const queued = (async () => { - if (previous) { - await previous.catch(() => undefined); - } - return this.#observeWindow(options ?? {}, target, cursorKey); - })(); - this.#observationQueues.set(surface, queued); - try { - return await queued; - } finally { - if (this.#observationQueues.get(surface) === queued) { - this.#observationQueues.delete(surface); - } - } - } - - async #observeWindow(options, target, surface) { - const { - pid, - windowId, - includeScreenshot = false, - screenshotOutFile, - maxElements, - maxDepth, - maxTextChars = DEFAULT_MAX_TEXT_CHARS, - signal, - } = options; - const disableDiff = observationDisablesDiff(options); - requireIntegerRange("maxTextChars", maxTextChars, 512, Number.MAX_SAFE_INTEGER); - const input = { - pid: target.pid, - windowId: target.windowId, - includeScreenshot, - }; - if (options.appContext) input.appContext = true; - const projectionVersion = options.appContext ? "app-tree-v1" : ACCESSIBILITY_PROJECTION_VERSION; - if (screenshotOutFile !== undefined) input.screenshotOutFile = screenshotOutFile; - if (maxElements !== undefined) { - input.maxElements = requirePositiveInteger("maxElements", maxElements); - } - if (maxDepth !== undefined) input.maxDepth = requirePositiveInteger("maxDepth", maxDepth); - if (await this.#supportsObservationRevision({ signal })) { - const cursor = this.#revisionCursors.get(surface); - input.observationRevision = { - version: 1, - serializerVersion: ACCESSIBILITY_SERIALIZER_VERSION, - projectionVersion, - }; - if (disableDiff) { - input.observationRevision.forceFull = true; - } else if (cursor?.generation === this.#connectionGeneration) { - input.observationRevision.baseRevisionId = cursor.revisionId; - } - } - let activeInput = input; - let observed; - let retriedIncompleteCapture = false; - while (true) { - let observedGeneration; - observed = await this.#invoke("getWindowState", activeInput, { - readOnly: true, - signal, - afterReconnect: () => { - if (!activeInput.observationRevision) return activeInput; - return { - ...activeInput, - observationRevision: { - ...activeInput.observationRevision, - baseRevisionId: undefined, - forceFull: true, - }, - }; - }, - onDispatch: (generation) => { - observedGeneration = generation; - const revision = activeInput.observationRevision; - const cursor = this.#revisionCursors.get(surface); - if ( - revision?.baseRevisionId !== undefined && - cursor?.generation !== generation - ) { - revision.baseRevisionId = undefined; - revision.forceFull = true; - } - }, - }); - const envelope = observed.structured?.observation_revision; - const capture = captureStatus(observed.structured); - if (observedGeneration === this.#connectionGeneration) { - const revisionId = envelope?.revision_id; - const mode = envelope?.mode; - if ( - typeof revisionId === "string" && - revisionId.length > 0 && - envelope?.stable_element_ids === true && - ["full", "diff", "no_change"].includes(mode) - ) { - this.#revisionCursors.set(surface, { - generation: observedGeneration, - revisionId, - }); - } else { - this.#revisionCursors.delete(surface); - } - } - if ( - !retriedIncompleteCapture && - !capture.incompleteDetails.some((detail) => - detail === "walk_deadline_reached" || detail === "element_bounds_timeout") && - capture.complete === false && - (!capture.truncated || capture.readComplete === false) && - envelope?.stable_element_ids !== true && - envelope?.resync_reason === "capture_incomplete" && - activeInput.observationRevision - ) { - activeInput = { - ...activeInput, - observationRevision: { - version: 1, - serializerVersion: ACCESSIBILITY_SERIALIZER_VERSION, - projectionVersion, - }, - }; - retriedIncompleteCapture = true; - continue; - } - break; - } - const { text, structured, images } = observed; - const envelope = structured?.observation_revision; - const capture = captureStatus(structured); - const treeText = structured?.tree_markdown ?? text ?? ""; - const publicText = observationText(treeText, capture, maxTextChars, options.appContext); - return { - pid, - windowId, - mode: envelope?.mode ?? "full", - resyncReason: envelope?.resync_reason ?? undefined, - text: publicText.text, - elements: capture.complete === false - ? (structured?.elements ?? []).filter((element) => element.element_token) - : (structured?.elements ?? []), - screenshot: - structured?.screenshot_width !== undefined || structured?.screenshot_file_path - ? { - width: structured?.screenshot_width, - height: structured?.screenshot_height, - mimeType: structured?.screenshot_mime_type, - filePath: structured?.screenshot_file_path, - images, - } - : undefined, - context: { - backgroundInput: structured?.background_input, - degraded: structured?.degraded, - degradedReason: structured?.degraded_reason, - escalation: structured?.escalation, - windowBounds: structured?.window_bounds, - screenshotScale: structured?.screenshot_scale, - screenshotFrameValid: structured?.screenshot_frame_valid, - screenshotError: structured?.screenshot_error, - }, - diagnostics: { - revisionSupported: Boolean(envelope), - stableElementIds: envelope?.stable_element_ids === true, - captureComplete: capture.complete, - captureReadComplete: capture.readComplete, - captureTruncated: capture.truncated, - captureIncompleteDetails: capture.incompleteDetails, - textTruncated: publicText.truncated, - textChars: publicText.text.length, - serializerVersion: envelope?.serializer_version, - projectionVersion: envelope?.projection_version, - selectedBytes: envelope?.selected_bytes, - fullBytes: envelope?.full_bytes, - estimatedTokens: envelope?.estimated_tokens, - serializerDurationUs: envelope?.serializer_duration_us, - cacheEstimateBytes: envelope?.cache_estimate_bytes, - }, - }; - } - - async verifyState(options) { - const { pid, windowId, expect, timeoutMs, stableSamples, includeScreenshot, signal } = - options ?? {}; - const target = exactWindow(pid, windowId); - if (!Array.isArray(expect) || expect.length === 0) { - throw new ComputerUseError("expect must contain at least one predicate"); - } - const input = { - pid: BigInt(target.pid), - windowId: target.windowId, - expect, - }; - if (timeoutMs !== undefined) { - input.timeoutMs = BigInt(requireIntegerRange("timeoutMs", timeoutMs, 0, 10000)); - } - if (stableSamples !== undefined) { - input.stableSamples = BigInt(requireIntegerRange("stableSamples", stableSamples, 1, 5)); - } - if (includeScreenshot !== undefined) input.includeScreenshot = includeScreenshot; - const result = await this.#invoke("verifyState", input, { - readOnly: true, - signal, - }); - return verificationResult(result); - } - - async click(options) { - const input = this.#windowAddress(options); - if (options?.appContext) input.appContext = true; - const { button, count, signal } = options ?? {}; - if (button !== undefined) input.button = this.#clickButton(button); - if (count !== undefined) input.count = requireIntegerRange("count", count, 1, 3); - input.deliveryMode = await this.#actionDeliveryMode(options); - return actionResult(await this.#invoke("windowClick", input, { signal })); - } - - async doubleClick(options) { - const input = this.#windowAddress(options); - if (options?.appContext) input.appContext = true; - input.deliveryMode = await this.#actionDeliveryMode(options); - return actionResult( - await this.#invoke("doubleClick", input, { - signal: options?.signal, - }), - ); - } - - async rightClick(options) { - const input = this.#windowAddress(options); - if (options?.appContext) input.appContext = true; - if (options?.modifier !== undefined) { - input.modifier = requireStringList("modifier", options.modifier); - } - input.deliveryMode = await this.#actionDeliveryMode(options); - return actionResult( - await this.#invoke("rightClick", input, { - signal: options?.signal, - }), - ); - } - - async drag(options) { - const { - pid, - windowId, - fromX, - fromY, - toX, - toY, - durationMs, - steps, - button, - modifier, - signal, - } = options ?? {}; - const target = exactWindow(pid, windowId); - for (const [name, value] of Object.entries({ fromX, fromY, toX, toY })) { - if (!Number.isFinite(value)) throw new ComputerUseError(`${name} must be a finite number`); - } - const input = { - fromX, - fromY, - toX, - toY, - pid: target.pid, - windowId: target.windowId, - }; - if (options.appContext) input.appContext = true; - if (durationMs !== undefined) { - input.durationMs = BigInt(requireIntegerRange("durationMs", durationMs, 0, 10000)); - } - if (steps !== undefined) { - input.steps = BigInt(requireIntegerRange("steps", steps, 1, 200)); - } - input.deliveryMode = await this.#actionDeliveryMode(options); - if (button !== undefined) input.button = this.#clickButton(button); - if (modifier !== undefined) input.modifier = requireStringList("modifier", modifier); - return actionResult(await this.#invoke("windowDrag", input, { signal })); - } - - async scroll(options) { - const input = this.#windowAddress(options); - if (options?.appContext) input.appContext = true; - input.direction = this.#scrollDirection(options?.direction); - if (options?.amount !== undefined) { - input.amount = BigInt(requireIntegerRange("amount", options.amount, 1, 50)); - } - if (options?.by !== undefined) input.by = this.#scrollBy(options.by); - input.deliveryMode = await this.#actionDeliveryMode(options); - return actionResult( - await this.#invoke("windowScroll", input, { - signal: options?.signal, - }), - ); - } - - async setValue(options) { - const input = this.#windowAddress(options, { - coordinates: false, - tokenRequired: true, - }); - input.value = typeof options?.value === "string" ? options.value : undefined; - if (input.value === undefined) throw new ComputerUseError("value must be a string"); - return actionResult( - await this.#invoke("setValue", input, { - signal: options?.signal, - }), - ); - } - - async typeText(options) { - const input = this.#windowAddress(options, { coordinates: false }); - if (typeof options?.text !== "string") throw new ComputerUseError("text must be a string"); - if (options?.appContext === true) input.appContext = true; - input.text = options.text; - if (options.delayMs !== undefined) { - input.delayMs = BigInt( - requireIntegerRange("delayMs", options.delayMs, 0, Number.MAX_SAFE_INTEGER), - ); - } - input.deliveryMode = await this.#actionDeliveryMode(options); - return actionResult( - await this.#invoke("windowTypeText", input, { - signal: options.signal, - }), - ); - } - - async paste(options) { - const input = exactWindow(options?.pid, options?.windowId); - if (typeof options?.text !== "string") throw new ComputerUseError("text must be a string"); - const format = options.format ?? "text"; - const formats = { text: "Text", md: "Md", html: "Html" }; - if (typeof format !== "string" || !Object.hasOwn(formats, format)) throw new ComputerUseError("format must be text, md, or html"); - input.text = options.text; - input.format = this.#sdk.PasteFormat?.[formats[format]] ?? format; - if (options?.appContext === true) input.appContext = true; - if (await this.getPlatform({ signal: options.signal }) !== "macos") { - throw new ComputerUseError("paste is supported only by the macOS driver", { code: "unsupported_platform" }); - } - return actionResult(await this.#invoke("paste", input, { signal: options.signal })); - } - - async selectText(options) { - const input = this.#windowAddress(options, { coordinates: false, tokenRequired: true }); - Object.assign(input, exactWindow(options?.pid, options?.windowId)); - input.text = requireNonEmptyString("text", options?.text); - for (const field of ["prefix", "suffix"]) { - if (options[field] !== undefined) { - if (typeof options[field] !== "string") throw new ComputerUseError(`${field} must be a string`); - input[field] = options[field]; - } - } - const selection = options.selection ?? "text"; - const selections = { text: "Text", cursor_before: "CursorBefore", cursor_after: "CursorAfter" }; - if (typeof selection !== "string" || !Object.hasOwn(selections, selection)) throw new ComputerUseError("selection must be text, cursor_before, or cursor_after"); - input.selection = this.#sdk.TextSelection?.[selections[selection]] ?? selection; - if (await this.getPlatform({ signal: options.signal }) !== "macos") { - throw new ComputerUseError("selectText is supported only by the macOS driver", { code: "unsupported_platform" }); - } - return actionResult(await this.#invoke("selectText", input, { signal: options.signal })); - } - - async pressKey(options) { - const input = this.#windowAddress(options, { coordinates: false }); - input.key = requireNonEmptyString("key", options?.key); - if (options?.modifiers !== undefined) { - input.modifiers = requireStringList("modifiers", options.modifiers); - } - input.deliveryMode = await this.#actionDeliveryMode(options); - return actionResult( - await this.#invoke("windowPressKey", input, { - signal: options?.signal, - }), - ); - } - - async hotkey(options) { - const input = this.#windowAddress(options, { coordinates: false }); - if (!Array.isArray(options?.keys) || options.keys.length < 2) { - throw new ComputerUseError("keys must list modifiers plus one key"); - } - input.keys = requireStringList("keys", options.keys); - input.deliveryMode = await this.#actionDeliveryMode(options); - return actionResult( - await this.#invoke("windowHotkey", input, { - signal: options?.signal, - }), - ); - } - - async performSecondaryAction(options) { - const input = this.#windowAddress(options, { - coordinates: false, - tokenRequired: true, - }); - input.action = requireNonEmptyString("action", options?.action); - return actionResult( - await this.#invoke("performSecondaryAction", input, { - signal: options?.signal, - }), - ); - } - - async actAndVerify({ action, verify } = {}) { - if (typeof action !== "function" || typeof verify !== "function") { - throw new ComputerUseError("actAndVerify requires action and verify functions"); - } - const actionOutcome = await action(); - const verification = await verify(actionOutcome); - const admissibleEffect = ["confirmed", "partial", "unverifiable"].includes( - actionOutcome?.effect, - ); - const verified = verification?.status === "satisfied" && verification?.stable === true; - if (!admissibleEffect || !verified) { - throw new ComputerUseError("the action postcondition was not stably satisfied", { - code: "postcondition_not_satisfied", - details: { action: actionOutcome, verification }, - }); - } - return { action: actionOutcome, verification }; + return { app: this.#apps.get(id), initial }; } /** Close the bound session, then the owning runtime/client handle. */ async close() { if (this.#closed) return; this.#closed = true; - this.#revisionCursors.clear(); + this.#apps.clear(); let failure; if (this.#ownsSession && typeof this.#driver?.endSession === "function") { try { diff --git a/packages/cua-driver/typescript/computer-use/interface.json b/packages/cua-driver/typescript/computer-use/interface.json new file mode 100644 index 00000000000..ed588c4e292 --- /dev/null +++ b/packages/cua-driver/typescript/computer-use/interface.json @@ -0,0 +1,721 @@ +{ + "tools": [ + { + "description": "List applications and their windows. Use an exact app selector for subsequent calls.", + "input_schema": { + "additionalProperties": false, + "properties": { + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "method": "getState", + "name": "get_state", + "read_only": true + }, + { + "description": "Resolve an application and return its full AX state without an image. A stopped, uniquely identified application may be opened for observation; actions never relaunch it.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path from get_state/get_app.", + "minLength": 1, + "type": "string" + }, + "max_depth": { + "minimum": 1, + "type": "integer" + }, + "max_elements": { + "minimum": 1, + "type": "integer" + }, + "max_text_chars": { + "default": 12000, + "maximum": 1000000, + "minimum": 512, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "app" + ], + "type": "object" + }, + "method": "getApp", + "name": "get_app", + "read_only": false + }, + { + "description": "Observe AX text without capturing pixels. Uses a diff after the first observation; force_full requests a complete view. Short IDs belong to this app window's current observation.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path from get_state/get_app.", + "minLength": 1, + "type": "string" + }, + "force_full": { + "default": false, + "type": "boolean" + }, + "max_depth": { + "minimum": 1, + "type": "integer" + }, + "max_elements": { + "minimum": 1, + "type": "integer" + }, + "max_text_chars": { + "default": 12000, + "maximum": 1000000, + "minimum": 512, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "app" + ], + "type": "object" + }, + "method": "getAXState", + "name": "get_ax_state", + "read_only": true + }, + { + "description": "Observe the current application window screenshot without an AX walk. Coordinates in actions use this image's pixels.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "method": "getScreenshot", + "name": "get_screenshot", + "read_only": true + }, + { + "description": "Observe AX text and the current window image together. Uses an AX diff unless force_full is set.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path from get_state/get_app.", + "minLength": 1, + "type": "string" + }, + "force_full": { + "default": false, + "type": "boolean" + }, + "max_depth": { + "minimum": 1, + "type": "integer" + }, + "max_elements": { + "minimum": 1, + "type": "integer" + }, + "max_text_chars": { + "default": 12000, + "maximum": 1000000, + "minimum": 512, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "app" + ], + "type": "object" + }, + "method": "getAXStateAndScreenshot", + "name": "get_ax_state_and_screenshot", + "read_only": true + }, + { + "description": "Click a current short element ID or screenshot point. Optional key holds modifiers during the click. Observe the result before retrying an unconfirmed action.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "button": { + "default": "left", + "enum": [ + "left", + "middle", + "right" + ], + "type": "string" + }, + "count": { + "default": 1, + "maximum": 3, + "minimum": 1, + "type": "integer" + }, + "element": { + "description": "Short ID from the latest observation of this app window.", + "minimum": 0, + "type": "integer" + }, + "key": { + "description": "Modifier keys held during the click, such as [\"shift\"] or [\"ctrl\",\"shift\"].", + "items": { + "type": "string" + }, + "type": "array" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "x": { + "description": "Window screenshot pixel x. Requires a current screenshot.", + "type": "number" + }, + "y": { + "description": "Window screenshot pixel y. Requires a current screenshot.", + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "method": "click", + "name": "click", + "read_only": false + }, + { + "description": "Double-click a current short element ID or screenshot point, then observe the result.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "element": { + "minimum": 0, + "type": "integer" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "method": "doubleClick", + "name": "double_click", + "read_only": false + }, + { + "description": "Right-click a current short element ID or screenshot point, then observe the result.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "element": { + "minimum": 0, + "type": "integer" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [], + "type": "object" + }, + "method": "rightClick", + "name": "right_click", + "read_only": false + }, + { + "description": "Scroll at a current element or screenshot point. Observe the result before sending another scroll.", + "input_schema": { + "additionalProperties": false, + "properties": { + "amount": { + "default": 3, + "maximum": 50, + "minimum": 1, + "type": "integer" + }, + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "direction": { + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + "element": { + "description": "Short ID from the latest observation of this app window.", + "minimum": 0, + "type": "integer" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "x": { + "description": "Window screenshot pixel x. Requires a current screenshot.", + "type": "number" + }, + "y": { + "description": "Window screenshot pixel y. Requires a current screenshot.", + "type": "number" + } + }, + "required": [ + "direction" + ], + "type": "object" + }, + "method": "scroll", + "name": "scroll", + "read_only": false + }, + { + "description": "Drag between current window screenshot points. Unsupported background gestures refuse before sending input; use semantic value operations when available.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "from_x": { + "type": "number" + }, + "from_y": { + "type": "number" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "to_x": { + "type": "number" + }, + "to_y": { + "type": "number" + } + }, + "required": [ + "from_x", + "from_y", + "to_x", + "to_y" + ], + "type": "object" + }, + "method": "drag", + "name": "drag", + "read_only": false + }, + { + "description": "Insert text at the current text caret. This does not replace the whole field; use set_value for replacement.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "type": "object" + }, + "method": "typeText", + "name": "type_text", + "read_only": false + }, + { + "description": "Replace the entire value of the current short element ID and read back the result. Refuses a non-settable element or a write that was not applied.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path from get_state/get_app.", + "minLength": 1, + "type": "string" + }, + "element": { + "minimum": 0, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "app", + "element", + "value" + ], + "type": "object" + }, + "method": "setValue", + "name": "set_value", + "read_only": false + }, + { + "description": "Press a key in the selected app. Optional modifiers form a chord. Observe before retrying an unconfirmed dispatch.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "key": { + "type": "string" + }, + "modifiers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "key" + ], + "type": "object" + }, + "method": "pressKey", + "name": "press_key", + "read_only": false + }, + { + "description": "Press a modifier/key chord in the selected app.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop.", + "minLength": 1, + "type": "string" + }, + "keys": { + "items": { + "type": "string" + }, + "minItems": 1, + "type": "array" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "keys" + ], + "type": "object" + }, + "method": "hotkey", + "name": "hotkey", + "read_only": false + }, + { + "description": "Hold a desktop key for this session. Release with key_up in finally. Session end releases owned keys. While keys are held, use key_down/key_up for chords and desktop pointer actions without explicit modifiers.", + "input_schema": { + "additionalProperties": false, + "properties": { + "key": { + "minLength": 1, + "type": "string" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "scope", + "key" + ], + "type": "object" + }, + "method": "desktop.keyDown", + "name": "key_down", + "read_only": false + }, + { + "description": "Release a desktop key held by this session. Another session cannot release it.", + "input_schema": { + "additionalProperties": false, + "properties": { + "key": { + "minLength": 1, + "type": "string" + }, + "scope": { + "description": "Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app.", + "enum": [ + "desktop" + ], + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "scope", + "key" + ], + "type": "object" + }, + "method": "desktop.keyUp", + "name": "key_up", + "read_only": false + }, + { + "description": "Paste text at the caret using a temporary clipboard value, then restore the previous clipboard even if the operation fails.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path from get_state/get_app.", + "minLength": 1, + "type": "string" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": [ + "app", + "text" + ], + "type": "object" + }, + "method": "paste", + "name": "paste", + "read_only": false + }, + { + "description": "Select one unambiguous text match inside a current short element ID. Does not change the field value.", + "input_schema": { + "additionalProperties": false, + "properties": { + "app": { + "description": "Exact application name, identifier or absolute path from get_state/get_app.", + "minLength": 1, + "type": "string" + }, + "element": { + "minimum": 0, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": [ + "app", + "element", + "text" + ], + "type": "object" + }, + "method": "selectText", + "name": "select_text", + "read_only": false + }, + { + "description": "Perform an explicitly named secondary accessibility action on a current short element ID.", + "input_schema": { + "additionalProperties": false, + "properties": { + "action": { + "type": "string" + }, + "app": { + "description": "Exact application name, identifier or absolute path from get_state/get_app.", + "minLength": 1, + "type": "string" + }, + "element": { + "minimum": 0, + "type": "integer" + }, + "session": { + "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session.", + "type": "string" + } + }, + "required": [ + "app", + "element", + "action" + ], + "type": "object" + }, + "method": "performSecondaryAction", + "name": "perform_secondary_action", + "read_only": false + } + ], + "version": 1 +} diff --git a/packages/cua-driver/typescript/computer-use/runtime.d.ts b/packages/cua-driver/typescript/computer-use/runtime.d.ts new file mode 100644 index 00000000000..6eb45fb459d --- /dev/null +++ b/packages/cua-driver/typescript/computer-use/runtime.d.ts @@ -0,0 +1,57 @@ +import type { ComputerUse, CallOptions } from './index.js'; +import type { Browser, BrowserTab } from '@qwen-code/browser-use'; +export interface RuntimeOutput { + write(value: unknown): void; + emitImage(value: string): void | Promise<void>; + readonly signal?: AbortSignal; +} +export interface ObservationOptions extends CallOptions { + emit?: boolean; + forceFull?: boolean; + maxTextChars?: number; + maxElements?: number; + maxDepth?: number; +} +export type AppPoint = number | { x: number; y: number }; +export interface RuntimeApp { + readonly name: string; + getAXState(options?: ObservationOptions): Promise<string>; + getScreenshot(options?: CallOptions & { emit?: boolean }): Promise<string>; + getAXStateAndScreenshot(options?: ObservationOptions): Promise<string>; + click(point: AppPoint, options?: CallOptions & { key?: string | string[]; button?: 'left' | 'right' | 'middle'; count?: number }): Promise<void>; + doubleClick(point: AppPoint, options?: CallOptions): Promise<void>; + rightClick(point: AppPoint, options?: CallOptions): Promise<void>; + scroll(point: AppPoint, options: CallOptions & { direction: 'up' | 'down' | 'left' | 'right'; amount?: number }): Promise<void>; + drag(options: CallOptions & { fromX: number; fromY: number; toX: number; toY: number }): Promise<void>; + setValue(element: number, value: string, options?: CallOptions): Promise<void>; + performSecondaryAction(element: number, action: string, options?: CallOptions): Promise<void>; + typeText(text: string, options?: CallOptions): Promise<void>; + paste(text: string, options?: CallOptions): Promise<void>; + selectText(element: number, text: string, options?: CallOptions): Promise<void>; + pressKey(key: string, options?: CallOptions & { modifiers?: string[] }): Promise<void>; + hotkey(keys: string[], options?: CallOptions): Promise<void>; +} +export interface RuntimeDesktop { + getScreenshot(options?: CallOptions & { emit?: boolean }): Promise<string>; + click(point: { x: number; y: number }, options?: CallOptions & { key?: string | string[]; button?: 'left' | 'right' | 'middle'; count?: number }): Promise<void>; + doubleClick(point: { x: number; y: number }, options?: CallOptions): Promise<void>; + rightClick(point: { x: number; y: number }, options?: CallOptions): Promise<void>; + scroll(point: { x: number; y: number }, options: CallOptions & { direction: 'up' | 'down' | 'left' | 'right'; amount?: number }): Promise<void>; + drag(options: CallOptions & { fromX: number; fromY: number; toX: number; toY: number }): Promise<void>; + typeText(text: string, options?: CallOptions): Promise<void>; + pressKey(key: string, options?: CallOptions & { modifiers?: string[] }): Promise<void>; + hotkey(keys: string[], options?: CallOptions): Promise<void>; + keyDown(key: string, options?: CallOptions): Promise<void>; + keyUp(key: string, options?: CallOptions): Promise<void>; +} +export interface CuaRuntime { + readonly desktop: RuntimeDesktop; + getBrowser(selector?: string | { id: string }): Promise<Browser>; + getTab(reference: string | { url: string }, options?: { browser?: string | { id: string } }): Promise<BrowserTab>; + createBrowserTab(browser?: string | { id: string }, url?: string): Promise<BrowserTab>; + [Symbol.asyncDispose](): Promise<void>; + rewriteDocumentation(): string; + getState(options?: CallOptions & { emit?: boolean }): Promise<string>; + getApp(selector: string): Promise<RuntimeApp>; +} +export function createCua(output: RuntimeOutput, options?: { computer?: ComputerUse }): CuaRuntime; diff --git a/packages/cua-driver/typescript/computer-use/runtime.js b/packages/cua-driver/typescript/computer-use/runtime.js new file mode 100644 index 00000000000..ed909471284 --- /dev/null +++ b/packages/cua-driver/typescript/computer-use/runtime.js @@ -0,0 +1,117 @@ +import { ComputerUse } from './index.js'; +import { documentation } from './documentation.js'; +import { createBrowserFacade } from './browser.js'; + +export function createCua(runtime, { computer, browserModule } = {}) { + const browser = createBrowserFacade(runtime, browserModule && (() => Promise.resolve(browserModule))); + let closed = false; + let connection = computer ? Promise.resolve(computer) : undefined; + const handles = new WeakMap(); + const options = (value = {}) => ({ signal: runtime.signal, ...value }); + const parameters = (input = {}) => Object.fromEntries( + Object.entries(input).filter(([key]) => key !== 'signal' && key !== 'emit') + .map(([key, value]) => [key.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`), value]), + ); + const point = (value) => typeof value === 'number' ? { element: value } : value; + async function connected() { + if (closed) throw new Error('Computer Use session is closed.'); + if (!connection) { + connection = ComputerUse.create(options()).catch((error) => { + connection = undefined; + throw error; + }); + } + return connection; + } + async function display(result, emit = true) { + if (emit) { + if (result.text) runtime.write(result.text); + for (const image of result.images) { + await runtime.emitImage(`data:${image.mimeType};base64,${image.dataBase64}`); + } + } + return result.text; + } + function handle(app) { + if (handles.has(app)) return handles.get(app); + async function observe(name, input = {}) { + const result = await app.call(name, parameters(input), { ...options(input), readOnly: true }); + return display(result, input.emit !== false); + } + async function act(name, args, input = {}) { + await display(await app.call(name, { ...parameters(input), ...args }, options(input))); + } + const wrapped = Object.freeze({ + name: app.name, + getAXState: (input) => observe('get_ax_state', input), + getScreenshot: (input) => observe('get_screenshot', input), + getAXStateAndScreenshot: (input) => observe('get_ax_state_and_screenshot', input), + click: (target, input) => act('click', point(target), input), + doubleClick: (target, input) => act('double_click', point(target), input), + rightClick: (target, input) => act('right_click', point(target), input), + scroll: (target, input) => act('scroll', point(target), input), + drag: (input) => act('drag', {}, input), + performSecondaryAction: (element, action, input) => act('perform_secondary_action', { element, action }, input), + setValue: (element, value, input) => act('set_value', { element, value }, input), + typeText: (text, input) => act('type_text', { text }, input), + paste: (text, input) => act('paste', { text }, input), + selectText: (element, text, input) => act('select_text', { element, text }, input), + pressKey: (key, input) => act('press_key', { key }, input), + hotkey: (keys, input) => act('hotkey', { keys }, input), + }); + handles.set(app, wrapped); + return wrapped; + } + let desktopTail = Promise.resolve(); + function desktopCall(name, args, input = {}) { + const next = desktopTail.catch(() => {}).then(async () => { + const result = await (await connected()).appCall(name, { ...parameters(input), ...args, scope: 'desktop' }, options(input)); + const text = await display(result, input.emit !== false); + return name === 'get_screenshot' ? text : undefined; + }); + desktopTail = next; + return next; + } + const desktop = Object.freeze({ + getScreenshot: (input) => desktopCall('get_screenshot', {}, input), + click: (target, input) => desktopCall('click', target, input), + doubleClick: (target, input) => desktopCall('double_click', target, input), + rightClick: (target, input) => desktopCall('right_click', target, input), + scroll: (target, input) => desktopCall('scroll', target, input), + drag: (input) => desktopCall('drag', {}, input), + typeText: (text, input) => desktopCall('type_text', { text }, input), + pressKey: (key, input) => desktopCall('press_key', { key }, input), + hotkey: (keys, input) => desktopCall('hotkey', { keys }, input), + keyDown: (key, input) => desktopCall('key_down', { key }, input), + keyUp: (key, input) => desktopCall('key_up', { key }, input), + }); + return Object.freeze({ + desktop, + getBrowser: browser.getBrowser, + getTab: browser.getTab, + createBrowserTab: browser.createBrowserTab, + async [Symbol.asyncDispose]() { + if (closed) return; + closed = true; + try { + await desktopTail.catch(() => {}); + if (connection) await (await connection.catch(() => undefined))?.close(); + } finally { + await browser.close(); + } + }, + rewriteDocumentation() { + const text = documentation(); + runtime.write(text); + return text; + }, + async getState({ emit = true, ...input } = {}) { + return display(await (await connected()).getState(options(input)), emit); + }, + async getApp(selector) { + const { app, initial } = await (await connected()).getApp(selector, options()); + await display(initial); + return handle(app); + }, + }); +} diff --git a/packages/cua-driver/typescript/computer-use/test/app-lifecycle.test.mjs b/packages/cua-driver/typescript/computer-use/test/app-lifecycle.test.mjs deleted file mode 100644 index 9c296a569e6..00000000000 --- a/packages/cua-driver/typescript/computer-use/test/app-lifecycle.test.mjs +++ /dev/null @@ -1,174 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { ComputerUse } from "../index.js"; - -function result(structured, extra = {}) { - return { text: "", images: [], structuredJson: JSON.stringify(structured), isError: false, ...extra }; -} - -const appRecord = { name: "Fixture", bundle_id: "org.example.fixture", launch_path: "/Owned/A/Fixture.app", pid: 42, running: true }; - -function session(name, { apps = [{ ...appRecord }], expireWindows = () => false, click, launch } = {}) { - const mutations = []; - const observations = []; - const launches = []; - return { - mutations, - observations, - launches, - async listToolsJson() { return JSON.stringify({ platform: "macos", tools: [] }); }, - async listApps() { - return result({ apps }); - }, - async launchApp(input) { - launches.push(input); - return launch ? launch(input) : result({}); - }, - async listWindows(input) { - observations.push({ method: "listWindows", input }); - if (expireWindows()) { - return result({ code: "authorization_context_expired" }, { isError: true }); - } - return result({ windows: [{ window_id: input.pid + 100, title: `Document ${input.pid}`, z_index: 3, is_on_screen: true, is_app_target: true }] }); - }, - async getWindowState(input) { - observations.push({ method: "getWindowState", input }); - return result({ - tree_markdown: '[37] Button "Save"', - elements: [{ element_index: 2, element_id: 37, element_token: `rv1:${name}:91`, role: "AXButton" }], - }); - }, - async windowClick(input) { - mutations.push(input); - return click ? click(input) : result({ effect: "confirmed" }); - }, - }; -} - -test("an explicit session replacement invalidates app element bindings", async () => { - const first = session("first"); - const second = session("second"); - const computer = new ComputerUse(first, { ownsSession: true, sessionFactory: async () => second }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await computer.reconnect(); - await assert.rejects(app.click(37), { code: "app_observation_required" }); - assert.equal(second.mutations.length, 0); - await app.getState(); - await app.click(37); - assert.equal(second.mutations[0].elementToken, "rv1:second:91"); -}); - -test("distinct paths sharing a bundle ID retain distinct app handles and targets", async () => { - const other = { ...appRecord, launch_path: "/Owned/B/Fixture.app", pid: 84 }; - const driver = session("first", { apps: [appRecord, other] }); - const computer = new ComputerUse(driver); - const first = await computer.getApp(appRecord.launch_path); - const second = await computer.getApp(other.launch_path); - assert.notStrictEqual(first, second); - assert.equal(first, await computer.getApp("/Owned/A/../A/Fixture.app/")); - assert.equal((await first.getState()).window, "Document 42"); - assert.equal((await second.getState()).window, "Document 84"); - assert.deepEqual(driver.observations.filter((call) => call.method === "getWindowState").map((call) => call.input.pid), [42, 84]); - assert.ok(driver.observations.every((call) => call.input.appContext === true)); -}); - -test("a path-bound app cannot migrate to another installation after its process exits", async () => { - const apps = [{ ...appRecord }, { ...appRecord, launch_path: "/Owned/B/Fixture.app", pid: 84 }]; - const driver = session("first", { apps }); - const computer = new ComputerUse(driver); - const app = await computer.getApp(appRecord.launch_path); - apps.splice(0, 1); - await assert.rejects(app.getState(), { code: "app_not_running" }); - assert.equal(driver.observations.length, 0); - assert.equal(driver.mutations.length, 0); -}); - -test("a restarted process at the same app path invalidates old element bindings", async () => { - const apps = [{ ...appRecord }]; - const driver = session("first", { apps }); - const computer = new ComputerUse(driver); - const app = await computer.getApp(appRecord.launch_path); - await app.getState(); - apps[0].pid = 84; - await assert.rejects(app.click(37), { code: "app_observation_required" }); - assert.equal(driver.mutations.length, 0); - await app.getState(); - await app.click(37); - assert.equal(driver.mutations[0].pid, 84); - assert.equal(driver.mutations[0].windowId, 184n); -}); - -test("getApp only binds a stopped app; getState launches its canonical path and refreshes the PID", async () => { - const apps = [{ ...appRecord, running: false, pid: null }]; - const driver = session("first", { apps, launch: () => { - apps[0] = { ...appRecord, pid: 84 }; - return result({ pid: 999 }); - } }); - const computer = new ComputerUse(driver); - const app = await computer.getApp("/Owned/A/../A/Fixture.app/"); - assert.equal(driver.launches.length, 0); - assert.equal(driver.observations.length, 0); - const state = await app.getState(); - assert.deepEqual(driver.launches, [{ name: appRecord.launch_path }]); - assert.equal(state.window, "Document 84"); - assert.ok(driver.observations.every((call) => call.input.pid === 84)); - await app.getState(); - assert.equal(driver.launches.length, 1); -}); - -test("app actions never launch an app that exited after observation", async () => { - const apps = [{ ...appRecord }]; - const driver = session("first", { apps }); - const computer = new ComputerUse(driver); - const app = await computer.getApp(appRecord.launch_path); - await app.getState(); - apps[0] = { ...appRecord, running: false, pid: null }; - for (const act of [() => app.click(37), () => app.pressKey("Return")]) { - await assert.rejects(act(), { code: "app_not_running" }); - } - assert.equal(driver.launches.length, 0); - assert.equal(driver.mutations.length, 0); - assert.equal(driver.observations.length, 2); -}); - -test("running and stopped installations with the same name remain ambiguous", async () => { - const apps = [appRecord, { ...appRecord, launch_path: "/Owned/B/Fixture.app", running: false, pid: null }]; - const driver = session("first", { apps }); - const computer = new ComputerUse(driver); - for (const selector of [appRecord.name, appRecord.bundle_id]) { - await assert.rejects(computer.getApp(selector), { code: "app_ambiguous" }); - } - assert.equal(driver.launches.length, 0); - assert.equal(driver.observations.length, 0); -}); - -test("a session replacement during window revalidation must not dispatch an old element token", async () => { - let expire = false; - const first = session("first", { expireWindows: () => expire }); - const second = session("second"); - const computer = new ComputerUse(first, { ownsSession: true, sessionFactory: async () => second }); - const app = await computer.getApp("Fixture"); - await app.getState(); - expire = true; - await assert.rejects(app.click(37), { code: "app_observation_required" }); - assert.equal(second.mutations.length, 0); -}); - -test("cancelling a dispatched app mutation waits for the native result and never replays", async () => { - let begin; - let finish; - const started = new Promise((resolve) => { begin = resolve; }); - const terminal = new Promise((resolve) => { finish = resolve; }); - const driver = session("first", { click: () => { begin(); return terminal; } }); - const computer = new ComputerUse(driver); - const app = await computer.getApp("Fixture"); - await app.getState(); - const controller = new AbortController(); - const pending = app.click(37, { signal: controller.signal }); - await started; - controller.abort(); - finish(result({ effect: "partial" })); - assert.deepEqual(await pending, { effect: "partial" }); - assert.equal(driver.mutations.length, 1); -}); diff --git a/packages/cua-driver/typescript/computer-use/test/app-oriented-baseline.test.mjs b/packages/cua-driver/typescript/computer-use/test/app-oriented-baseline.test.mjs deleted file mode 100644 index 2631e2a48b9..00000000000 --- a/packages/cua-driver/typescript/computer-use/test/app-oriented-baseline.test.mjs +++ /dev/null @@ -1,102 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; - -const { ComputerUse } = await import( - process.env.CUA_BASELINE_MODULE ?? "../index.js" -); - -const token = "rv1:l_baseline_observation:71"; -const nativeText = `Window: "Disposable fixture"\n0 AXWindow enabled=true frame=(0,0,640,480)\n Static text: Keep \"value=7\" verbatim\n 71 AXTextField element_token=${token} value="draft" enabled=true frame=(10,20,100,24)`; - -function result(structured, extra = {}) { - return { - text: "", - images: [], - structuredJson: JSON.stringify(structured), - rawJson: "{}", - isError: false, - ...extra, - }; -} - -function fixture(action) { - const calls = []; - const driver = { - async listToolsJson() { - return JSON.stringify({ platform: "macos", tools: [] }); - }, - async getWindowState(input) { - calls.push({ method: "getWindowState", input }); - return result({ - tree_markdown: nativeText, - elements: [{ element_index: 71, element_token: token, role: "AXTextField" }], - }); - }, - async windowClick(input) { - calls.push({ method: "windowClick", input }); - return action ? action(input) : result({ status: "ok" }); - }, - }; - const computer = new ComputerUse(driver, { - sdk: { - ClickButton: { Left: "left", Right: "right", Middle: "middle" }, - DeliveryMode: { Background: "background", Foreground: "foreground" }, - }, - }); - return { computer, calls }; -} - -test("requested model API exposes an application handle entry point", () => { - const { computer } = fixture(); - assert.equal(typeof computer.getApp, "function"); -}); - -test("legacy observation requires an explicit native process/window target", async () => { - const { computer, calls } = fixture(); - await assert.rejects(computer.observeWindow({}), /pid/i); - assert.equal(calls.length, 0); - await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.deepEqual( - { pid: calls[0].input.pid, windowId: calls[0].input.windowId }, - { pid: 42, windowId: 7n }, - ); -}); - -test("legacy observation passes native AX text and opaque tokens through", async () => { - const { computer } = fixture(); - const observed = await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.equal(observed.text, nativeText); - assert.equal(observed.elements[0].element_token, token); -}); - -test("legacy actions expose token identity and explicit delivery selection", async () => { - const { computer, calls } = fixture(); - await computer.click({ pid: 42, elementToken: token, deliveryMode: "foreground" }); - assert.equal(calls.length, 1); - assert.equal(calls[0].input.pid, 42); - assert.equal(calls[0].input.elementToken, token); - assert.equal(calls[0].input.deliveryMode, "foreground"); -}); - -for (const effect of ["partial", "unverifiable"]) { - test(`legacy action never replays a ${effect} terminal outcome`, async () => { - const { computer, calls } = fixture(() => result({}, { action: { effect } })); - await computer.click({ pid: 42, elementToken: token }); - assert.equal(calls.filter((call) => call.method === "windowClick").length, 1); - }); -} - -test("legacy action waits for its terminal result after cancellation and does not replay", async () => { - let finish; - let started; - const dispatched = new Promise((resolve) => { started = resolve; }); - const terminal = new Promise((resolve) => { finish = resolve; }); - const { computer, calls } = fixture(() => { started(); return terminal; }); - const controller = new AbortController(); - const pending = computer.click({ pid: 42, elementToken: token, signal: controller.signal }); - await dispatched; - controller.abort(); - finish(result({}, { action: { effect: "unverifiable" } })); - await pending; - assert.equal(calls.filter((call) => call.method === "windowClick").length, 1); -}); diff --git a/packages/cua-driver/typescript/computer-use/test/app.test.mjs b/packages/cua-driver/typescript/computer-use/test/app.test.mjs deleted file mode 100644 index c77c8cd674d..00000000000 --- a/packages/cua-driver/typescript/computer-use/test/app.test.mjs +++ /dev/null @@ -1,628 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { ComputerUse } from "../index.js"; -import { appIdentity, resolveApp } from "../app.js"; - -function result(structured, extra = {}) { - return { text: "", structuredJson: JSON.stringify(structured), images: [], isError: false, ...extra }; -} - -const appRecord = { name: "Fixture", bundle_id: "org.example.fixture", launch_path: "/Applications/Fixture.app", pid: 42, running: true }; -const document = { window_id: 7, title: "Document", z_index: 2, is_on_screen: true, is_app_target: true }; -const dialog = { window_id: 9, title: "Save", z_index: 10, is_on_screen: true, is_app_target: false }; -const compactState = '[37] TextField "Name" value="draft"\n[38] StaticText "Keep frame=1,2 and <AXButton> verbatim"'; - -function fixture({ platform = "macos", apps = [{ ...appRecord }], windows = [{ ...document }], observe, action, launch } = {}) { - const calls = []; - let revision = 0; - const driver = { - async listToolsJson() { - return JSON.stringify({ platform, tools: [{ name: "get_window_state", capabilities: ["accessibility.observation_revision.v1"] }] }); - }, - async listApps(input) { calls.push({ method: "listApps", input }); return result({ apps }); }, - async launchApp(input) { calls.push({ method: "launchApp", input }); await launch?.(input); return result({}); }, - async listWindows(input) { calls.push({ method: "listWindows", input }); return result({ windows }); }, - async getWindowState(input) { - calls.push({ method: "getWindowState", input }); - revision += 1; - const token = `rv1:window_${input.windowId}:25`; - const state = { - tree_markdown: compactState, - elements: [{ element_index: 3, element_id: 37, element_token: token, role: "AXTextField" }], - observation_revision: { mode: "full", revision_id: `r${revision}`, stable_element_ids: true, capture_complete: true }, - background_input: { routes: ["accessibility", "pid_keyboard", "window_pointer"].map((route) => ({ route, status: "available" })) }, - ...(input.includeScreenshot ? { screenshot_width: 100, screenshot_height: 80, screenshot_frame_valid: true } : {}), - }; - return result(observe ? await observe(input, state, revision) : state, { - images: input.includeScreenshot ? [{ mimeType: "image/png", dataBase64: "fixture" }] : [], - }); - }, - }; - for (const method of ["windowClick", "doubleClick", "rightClick", "windowPressKey", "windowTypeText", "windowHotkey", "windowDrag", "windowScroll", "setValue", "performSecondaryAction"]) { - driver[method] = async (input) => { - calls.push({ method, input }); - return action ? action(method, input) : result({ effect: "confirmed" }); - }; - } - return { computer: new ComputerUse(driver), calls, windows, apps }; -} - -test("app aliases bind the same handle and hide OS addressing from state", async () => { - const { computer, calls } = fixture(); - const app = await computer.getApp("Fixture"); - assert.equal(app, await computer.getApp("org.example.fixture")); - assert.equal(app, await computer.getApp("/Applications/Fixture.app")); - const state = await app.getState(); - assert.deepEqual(Object.keys(state).sort(), ["app", "mode", "text", "window"]); - assert.equal(state.text, compactState); - assert.doesNotMatch(state.text, /element_token=|enabled=true|frame=1,2,30,40/); - assert.equal(calls.find((call) => call.method === "listWindows").input.appContext, true); - const observation = calls.find((call) => call.method === "getWindowState").input; - assert.equal(observation.appContext, true); - assert.equal(observation.includeScreenshot, true); - assert.equal(observation.observationRevision.projectionVersion, "app-tree-v1"); - await app.click(37); - assert.equal(calls.at(-1).input.elementToken, "rv1:window_7:25"); - assert.equal(calls.at(-1).input.windowId, 7n); - assert.equal(calls.at(-1).input.elementIndex, undefined); - assert.equal(calls.at(-1).input.appContext, true); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - await app.typeText("owned text"); - assert.equal(calls.at(-1).input.appContext, true); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - assert.equal(calls.at(-1).input.text, "owned text"); - for (const method of ["doubleClick", "rightClick"]) { - await app[method](37); - assert.equal(calls.at(-1).input.appContext, true); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - assert.equal(calls.at(-1).input.elementToken, "rv1:window_7:25"); - } -}); - -test("app resolution rejects ambiguous names instead of selecting the first process", async () => { - const { computer } = fixture({ apps: [appRecord, { ...appRecord, pid: 84, bundle_id: "org.other.fixture", launch_path: "/Applications/OtherFixture.app" }] }); - await assert.rejects(computer.getApp("Fixture"), { code: "app_ambiguous" }); - assert.equal((await computer.getApp("org.other.fixture")).name, "Fixture"); -}); - -test("Windows launcher identity round-trips and preserves shortcut arguments on launch", async () => { - const exe = "C:\\Program Files\\Fixture\\fixture.exe"; - const command = `"${exe}" --profile-directory="CaseSensitive Profile"`; - const apps = [{ ...appRecord, bundle_id: exe, launch_path: command, running: false, pid: 0 }]; - const { computer, calls } = fixture({ platform: "windows", apps, launch: () => { - apps[0] = { ...apps[0], running: true, pid: 42 }; - } }); - const app = await computer.getApp("Fixture"); - assert.equal((await app.getState()).window, "Document"); - assert.equal(calls.find(call => call.method === "launchApp").input.launchPath, command); - assert.equal(app, await computer.getApp(exe)); - await app.click(37); - assert.equal(calls.at(-1).method, "windowClick"); -}); - -test("Windows launcher arguments are opaque while executable aliases remain case insensitive", () => { - const exe = "C:\\Fixture\\fixture.exe"; - const app = { ...appRecord, bundle_id: exe, launch_path: `${exe} --profile=CaseSensitive` }; - assert.equal(resolveApp([app], appIdentity(app)), app); - assert.equal(resolveApp([app], exe.toLowerCase()), app); - assert.throws(() => resolveApp([app], app.launch_path.toLowerCase()), { code: "app_not_running" }); -}); - -test("application identity preserves POSIX syntax and case on every Node host", () => { - const app = { ...appRecord, launch_path: "/Nonexistent/Fixture.app/Contents/MacOS/Fixture" }; - const identity = appIdentity(app); - assert.equal(identity, app.launch_path); - assert.equal(appIdentity({ ...app, launch_path: identity }), identity); - assert.equal(resolveApp([app], identity), app); - assert.throws(() => resolveApp([app], identity.toLowerCase()), { code: "app_not_running" }); -}); - -test("App scroll carries its context without changing exact-window foreground scroll", async () => { - const { computer, calls } = fixture({ platform: "linux" }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await app.scroll({ x: 2, y: 3 }, { direction: "down", amount: 1 }); - assert.equal(calls.at(-1).input.appContext, true); - await computer.scroll({ pid: 42, windowId: 7, x: 2, y: 3, direction: "down", amount: 1, deliveryMode: "foreground" }); - assert.equal(calls.at(-1).input.appContext, undefined); -}); - -test("macOS app discovery exposes only stable application identity", async () => { - const { computer } = fixture({ - apps: [{ - ...appRecord, - active: true, - kind: "desktop", - last_used: "2026-09-13T00:00:00Z", - windows: [{ window_id: 7, title: "Document" }], - }], - }); - assert.deepEqual(await computer.listApps(), [{ - id: "org.example.fixture", - displayName: "Fixture", - isRunning: true, - }]); - assert.equal((await computer.getApp("Fixture")).name, "Fixture"); -}); - -test("native app target changes invalidate prior element IDs", async () => { - const { computer, calls, windows } = fixture(); - const app = await computer.getApp("Fixture"); - await app.getState(); - windows[0].is_app_target = false; - windows.push({ ...dialog, z_index: -1, is_app_target: true }); - await assert.rejects(app.click(37), { code: "app_observation_required" }); - assert.equal(calls.filter((call) => call.method === "windowClick").length, 0); - assert.equal((await app.getState()).window, "Save"); - await app.click(37); - assert.equal(calls.at(-1).input.elementToken, "rv1:window_9:25"); - windows.pop(); - windows[0].is_app_target = true; - assert.equal((await app.getState()).window, "Document"); - assert.equal(calls.at(-1).input.observationRevision.forceFull, true); -}); - -test("missing or multiple native app targets cannot silently pick a window", async () => { - for (const is_app_target of [false, true]) { - const { computer } = fixture({ windows: [{ ...document, is_app_target }, { ...dialog, is_app_target }] }); - const app = await computer.getApp("Fixture"); - await assert.rejects(app.getState(), { code: "app_window_unavailable" }); - } -}); - -test("native target selection ignores z order and on-screen ordering", async () => { - const { computer } = fixture({ windows: [dialog, { ...document, z_index: null, is_on_screen: false }] }); - assert.equal((await (await computer.getApp("Fixture")).getState()).window, "Document"); -}); - -test("app input activates the exact target once without exposing a mode choice", async () => { - const { computer, calls, windows } = fixture(); - const app = await computer.getApp("Fixture"); - await app.getState(); - await app.pressKey("Return"); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - windows.push(dialog); - await app.getState({ includeScreenshot: true }); - await app.pressKey("Return"); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - await app.drag({ fromX: 1, fromY: 2, toX: 20, toY: 25 }); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - assert.equal(calls.at(-1).input.appContext, true); - assert.equal(calls.filter((call) => call.method === "windowPressKey").length, 2); - assert.equal(calls.filter((call) => call.method === "windowDrag").length, 1); - assert.equal(calls.filter((call) => call.method === "getWindowState").length, 2); -}); - -test("app observations retain a current screenshot without exposing it by default", async () => { - const { computer, calls } = fixture({ observe: (_input, state, count) => ({ - ...state, - tree_markdown: count === 1 ? state.tree_markdown : "No accessibility changes.", - observation_revision: { ...state.observation_revision, mode: count === 1 ? "full" : "no_change" }, - }) }); - const app = await computer.getApp("Fixture"); - const full = await app.getState(); - assert.equal(full.screenshot, undefined); - const hidden = await app.getState({ includeScreenshot: false }); - assert.equal(hidden.screenshot, undefined); - await app.click(37); - await app.click({ x: 1, y: 2 }); - const unchanged = await app.getState(); - assert.equal(unchanged.mode, "no_change"); - assert.equal(unchanged.screenshot, undefined); - await app.click({ x: 2, y: 3 }); - const visible = await app.getState({ includeScreenshot: true }); - assert.equal(visible.screenshot.images[0].dataBase64, "fixture"); - assert.deepEqual( - calls.filter((call) => call.method === "getWindowState") - .map((call) => call.input.includeScreenshot), - [true, true, true, true], - ); -}); - -test("coordinates reject a screenshot frame that native marked invalid", async () => { - const { computer, calls } = fixture({ observe: (_input, state) => ({ - ...state, - screenshot_frame_valid: false, - }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await assert.rejects(app.click({ x: 1, y: 2 }), { code: "app_screenshot_required" }); - assert.equal(calls.filter((call) => call.method === "windowClick").length, 0); -}); - -test("missing capability metadata does not make the facade choose another input route", async () => { - const { computer, calls } = fixture({ observe: (_input, state) => ({ ...state, background_input: undefined }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await app.pressKey("Return"); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - assert.equal(calls.filter((call) => call.method === "windowPressKey").length, 1); -}); - -test("incomplete AX retains only current issued action bindings and preserves its warning", async () => { - const { computer, calls } = fixture({ observe: (_input, state, count) => ({ - ...state, - tree_markdown: '[0] StaticText "Hint"\n[1] Button "Save"\n[2] Button "Cancel"', - elements: [ - { element_id: 1, element_index: 0, element_token: `s${count}:0` }, - { element_id: 2, element_index: 1, element_token: `s${count}:1` }, - { element_id: 0 }, - ], - observation_revision: { - ...state.observation_revision, - capture_complete: false, - capture_read_complete: false, - stable_element_ids: false, - resync_reason: "capture_incomplete", - }, - }) }); - const app = await computer.getApp("Fixture"); - const state = await app.getState(); - assert.match(state.text, /incomplete/); - assert.match(state.text, /StaticText/); - assert.match(state.text, /current snapshot IDs/); - assert.doesNotMatch(state.text, /element_token=|tokens|element actions are unavailable/); - assert.equal(calls.filter((call) => call.method === "getWindowState").length, 2); - await app.click(1); - assert.equal(calls.at(-1).input.elementToken, "s2:0"); - await app.click(2); - assert.equal(calls.at(-1).input.elementToken, "s2:1"); - await assert.rejects(app.click(0), { code: "app_element_unavailable" }); - assert.equal(calls.filter((call) => call.method === "windowClick").length, 2); -}); - -test("old incomplete revision responses cannot bind display IDs through action indices", async () => { - const { computer, calls } = fixture({ observe: (_input, state, count) => ({ - ...state, - tree_markdown: '[0] StaticText "Hint"\n[1] Button "Save"\n[2] Button "Cancel"', - elements: [ - { element_index: 0, label: "Save", element_token: `s${count}:0` }, - { element_index: 1, label: "Cancel", element_token: `s${count}:1` }, - ], - observation_revision: { - ...state.observation_revision, - capture_complete: false, - stable_element_ids: false, - resync_reason: "capture_incomplete", - }, - }) }); - const app = await computer.getApp("Fixture"); - const state = await app.getState(); - assert.match(state.text, /\[1\] Button "Save"/); - for (const id of [0, 1, 2]) { - await assert.rejects(app.click(id), { code: "app_element_unavailable" }); - } - assert.equal(calls.filter((call) => call.method === "windowClick").length, 0); - const legacy = await computer.observeWindow({ pid: 42, windowId: 7 }); - await computer.click({ pid: 42, windowId: 7, elementToken: legacy.elements[0].element_token }); - assert.equal(calls.at(-1).input.elementToken, "s4:0"); -}); - -test("bounded app observations cap whole rows, reuse current IDs, and allow a larger full rendering", async () => { - const tree = compactState + "\n" + Array.from({ length: 800 }, (_, i) => `[${i + 100}] StaticText "Long fixture row ${i}"`).join("\n"); - for (const maxTextChars of [undefined, 512]) { - const { computer, calls } = fixture({ observe: (input, state, count) => ({ - ...state, - tree_markdown: count === 1 || input.observationRevision.forceFull ? tree : "No accessibility changes.", - observation_revision: { - ...state.observation_revision, - mode: count === 1 || input.observationRevision.forceFull ? "full" : "no_change", - capture_complete: false, - capture_read_complete: true, - capture_truncated: true, - capture_incomplete_details: ["walk: max_elements truncated"], - }, - }) }); - const app = await computer.getApp("Fixture"); - const full = await app.getState({ maxTextChars }); - assert.equal(full.mode, "full"); - assert.ok(full.text.length <= (maxTextChars ?? 12_000)); - assert.match(full.text, /^Accessibility capture is incomplete \(traversal limit\)/); - assert.match(full.text, /Text truncated; call app.getState/); - assert.doesNotMatch(full.text, /\.elements|element_token=|element actions are unavailable/); - assert.ok(tree.split("\n").includes(full.text.split("\n").at(-1))); - assert.match(full.text, /\[37\] TextField/); - const unchanged = await app.getState({ maxTextChars }); - assert.equal(unchanged.mode, "no_change"); - assert.match(unchanged.text, /No accessibility changes\.$/); - const observations = calls.filter((call) => call.method === "getWindowState"); - assert.equal(observations.length, 2); - assert.equal(observations[1].input.observationRevision.baseRevisionId, "r1"); - await app.click(37); - assert.equal(calls.at(-1).input.elementToken, "rv1:window_7:25"); - const expanded = await app.getState({ disableDiff: true, maxTextChars: 100_000 }); - assert.equal(expanded.mode, "full"); - assert.ok(expanded.text.endsWith(tree)); - assert.doesNotMatch(expanded.text, /Text truncated/); - } -}); - -test("the app advances native revision cursors and refreshes mappings even on no-change", async () => { - const { computer, calls } = fixture({ observe: (_input, state, count) => ({ - ...state, - tree_markdown: count === 1 ? state.tree_markdown : "No accessibility changes.", - observation_revision: { ...state.observation_revision, mode: count === 1 ? "full" : "no_change" }, - }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - const state = await app.getState(); - assert.equal(state.text, "No accessibility changes."); - assert.equal(calls.at(-1).input.observationRevision.baseRevisionId, "r1"); - await app.click(37); -}); - -test("app and exact-window observations serialize one native cache while retaining separate cursors", async () => { - const started = Promise.withResolvers(); - const release = Promise.withResolvers(); - const revisions = new Map(); - let active = 0; - let maximumActive = 0; - const { computer, calls } = fixture({ observe: async (input, state) => { - const appContext = input.appContext === true; - const projection = appContext ? "app" : "legacy"; - const stream = `${input.windowId}:${projection}`; - const revision = (revisions.get(stream) ?? 0) + 1; - revisions.set(stream, revision); - const sameWindow = input.windowId === 7n; - if (sameWindow) maximumActive = Math.max(maximumActive, ++active); - try { - if (sameWindow && appContext && revision === 1) { - started.resolve(); - await release.promise; - } - return { - ...state, - tree_markdown: revision === 1 ? state.tree_markdown : "No accessibility changes.", - observation_revision: { - ...state.observation_revision, - mode: revision === 1 ? "full" : "no_change", - revision_id: `${projection}-r${revision}`, - }, - }; - } finally { - if (sameWindow) active -= 1; - } - } }); - const app = await computer.getApp("Fixture"); - const appFull = app.getState(); - await started.promise; - const legacyFull = computer.observeWindow({ pid: 42, windowId: 7 }); - try { - // A different window can reach the native driver while this window is held. - // It also drains earlier unblocked observation work without a timed sleep. - await computer.observeWindow({ pid: 42, windowId: 8 }); - } finally { - release.resolve(); - } - const full = await Promise.all([appFull, legacyFull]); - assert.deepEqual(full.map((state) => state.mode), ["full", "full"]); - assert.equal(maximumActive, 1); - - const unchanged = await Promise.all([ - app.getState(), - computer.observeWindow({ pid: 42, windowId: 7 }), - ]); - assert.deepEqual(unchanged.map((state) => state.mode), ["no_change", "no_change"]); - assert.equal(maximumActive, 1); - const observations = calls.filter((call) => call.method === "getWindowState" && call.input.windowId === 7n); - const appInputs = observations.filter((call) => call.input.appContext === true).map((call) => call.input.observationRevision); - const legacyInputs = observations.filter((call) => call.input.appContext !== true).map((call) => call.input.observationRevision); - assert.deepEqual(appInputs.map((revision) => revision.baseRevisionId), [undefined, "app-r1"]); - assert.deepEqual(legacyInputs.map((revision) => revision.baseRevisionId), [undefined, "legacy-r1"]); - assert.ok(appInputs.every((revision) => revision.projectionVersion === "app-tree-v1")); - assert.ok(legacyInputs.every((revision) => revision.projectionVersion !== "app-tree-v1")); -}); - -test("app handle refresh forwards runningOnly to the native app query", async () => { - const { computer, calls } = fixture(); - const app = await computer.getApp("Fixture"); - await app.getState(); - await app.pressKey("Return"); - await app.getState(); - assert.deepEqual(calls.filter((call) => call.method === "listApps").map((call) => call.input), [ - {}, - { runningOnly: true }, - { runningOnly: true }, - { runningOnly: true }, - ]); -}); - -test("app API rejects manually supplied targeting and routing options", async () => { - const { computer, calls } = fixture(); - const app = await computer.getApp("Fixture"); - await app.getState(); - for (const key of ["deliveryMode", "delivery_mode", "foreground", "background", "pid", "windowId", "elementToken", "appContext"]) { - assert.throws(() => app.click(37, { [key]: "untrusted" }), { code: "app_option_managed" }); - } - assert.equal(calls.filter((call) => call.method === "windowClick").length, 0); -}); - -test("app API preserves actionable validation errors before dispatch", async () => { - const { computer, calls } = fixture(); - const app = await computer.getApp("Fixture"); - await app.getState(); - await assert.rejects(app.click({ elementIndex: 29 }), /short element ID or screenshot coordinates/); - await assert.rejects(app.hotkey("Meta", "r"), /keys must list modifiers plus one key/); - await assert.rejects(app.pressKey({ key: "ArrowDown" }), /key must be a non-empty string/); - await assert.rejects( - app.drag({ from: { x: 1, y: 2 }, to: { x: 20, y: 25 } }), - /drag requires flat, finite fromX, fromY, toX and toY coordinates/, - ); - const mutations = [ - "windowClick", - "windowHotkey", - "windowPressKey", - "windowDrag", - ]; - assert.equal(calls.filter((call) => mutations.includes(call.method)).length, 0); -}); - -test("an app input refusal is not replayed after its automatic foreground attempt", async () => { - const { computer, calls } = fixture({ action: () => - result({ code: "input_unavailable", effect: "refused" }, { isError: true }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await assert.rejects(app.pressKey("Return"), { code: "input_unavailable" }); - assert.deepEqual(calls.filter((call) => call.method === "windowPressKey").map((call) => call.input.deliveryMode), ["foreground"]); -}); - -test("getState returns a compact state when a running app has no windows", async () => { - const { computer, calls } = fixture({ windows: [] }); - const app = await computer.getApp("Fixture"); - assert.deepEqual(await app.getState(), { app: "Fixture", window: "", mode: "full", text: "No open application window." }); - await assert.rejects(app.pressKey("Return"), { code: "app_window_unavailable" }); - assert.equal(calls.filter((call) => call.method === "getWindowState").length, 0); -}); - -test("getState ignores a native placeholder after the last app window closes", async () => { - const { computer } = fixture({ windows: [{ window_id: 8, title: "", is_app_target: false }] }); - const app = await computer.getApp("Fixture"); - assert.equal((await app.getState()).text, "No open application window."); -}); - -test("getState keeps a selected untitled app window", async () => { - const { computer } = fixture({ windows: [{ ...document, title: "" }] }); - const app = await computer.getApp("Fixture"); - assert.equal((await app.getState()).text, compactState); -}); - -test("an exact-target refusal remains readable for an app semantic action", async () => { - const code = "off_space_or_ax_unresolved"; - const { computer, calls } = fixture({ action: () => result({ - code, effect: "refused", reason: "private native target details", - escalation: { recommended: "foreground" }, - }, { isError: true }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await assert.rejects(app.setValue(37, "ready"), (error) => { - assert.match(error.message, new RegExp(`Native input refused \\(${code}\\)`)); - assert.match(error.message, /may already have affected the app/); - assert.match(error.message, /Call app.getState\(\)/); - assert.doesNotMatch(error.message, /private native|foreground/); - assert.equal(error.details.operation.dispatched, true); - assert.equal(error.details.operation.committed, false); - return error.code === code; - }); - assert.equal(calls.filter((call) => call.method === "setValue").length, 1); -}); - -for (const effect of [undefined, "partial", "unverifiable", "suspected_noop"]) { - test(`a known code with effect ${effect} does not claim refusal`, async () => { - const { computer, calls } = fixture({ action: () => result({ - code: "off_space_or_ax_unresolved", effect, - }, { isError: true }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await assert.rejects(app.setValue(37, "ready"), (error) => { - assert.doesNotMatch(error.message, /Native input refused/); - assert.match(error.message, /may already have affected the app/); - return true; - }); - assert.equal(calls.filter((call) => call.method === "setValue").length, 1); - }); -} - -test("unknown refusal details are not interpolated into the public error", async () => { - const { computer } = fixture({ action: () => result({ - code: "unknown_private_code", effect: "refused", reason: "private native target details", - }, { isError: true }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await assert.rejects(app.pressKey("Return"), (error) => { - assert.doesNotMatch(error.message, /unknown_private_code|private native|Native input refused/); - return error.code === "unknown_private_code"; - }); -}); - -for (const effect of ["partial", "unverifiable", "suspected_noop"]) { - test(`a ${effect} action is not replayed`, async () => { - const { computer, calls } = fixture({ action: () => result({ effect }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - assert.deepEqual(await app.click(37), { effect }); - assert.equal(calls.filter((call) => call.method === "windowClick").length, 1); - }); -} - -test("an action error with no pre-actuator proof is never replayed or exposed as a mode choice", async () => { - const { computer, calls } = fixture({ action: () => result({ code: "background_unavailable" }, { - isError: true, text: "Try delivery_mode foreground with pid 42 and window_id 7", - }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - await assert.rejects(app.click(37), (error) => { - assert.match(error.message, /may already have affected the app/); - assert.doesNotMatch(error.message, /delivery|foreground|pid|window_id/); - return error.code === "background_unavailable"; - }); - assert.equal(calls.filter((call) => call.method === "windowClick").length, 1); -}); - -test("native compact diffs pass through without rewriting literal content", async () => { - const text = '~ [37] TextField "Name" value="frame=1,2 <AXTextField>"\nRemoved IDs: [38]'; - const { computer } = fixture({ observe: (_input, state, count) => ({ - ...state, - tree_markdown: count === 1 ? state.tree_markdown : text, - observation_revision: { ...state.observation_revision, mode: count === 1 ? "full" : "diff" }, - }) }); - const app = await computer.getApp("Fixture"); - await app.getState(); - const state = await app.getState(); - assert.equal(state.mode, "diff"); - assert.equal(state.text, text); -}); - -for (const platform of ["macos", "linux", "windows"]) { - test(`${platform} App handles follow native modal targets and refuse old IDs`, async () => { - const record = platform === "windows" - ? { ...appRecord, bundle_id: undefined, launch_path: "C:\\Program Files\\Fixture\\fixture.exe" } - : platform === "linux" ? { ...appRecord, bundle_id: "fixture.desktop", launch_path: "/usr/bin/fixture" } : appRecord; - const { computer, calls, windows } = fixture({ platform, apps: [record] }); - const app = await computer.getApp(record.launch_path); - assert.equal(app, await computer.getApp(record.name)); - if (platform === "windows") assert.equal(app, await computer.getApp("c:/program files/fixture/FIXTURE.EXE")); - assert.deepEqual(Object.keys((await computer.listApps())[0]).sort(), ["displayName", "id", "isRunning"]); - await app.getState(); - await app.click(37); - assert.equal(calls.at(-1).input.deliveryMode, "foreground"); - windows[0].is_app_target = false; - windows.push({ ...dialog, is_app_target: true }); - await assert.rejects(app.click(37), { code: "app_observation_required" }); - assert.equal(calls.filter((call) => call.method === "windowClick").length, 1); - assert.equal((await app.getState()).window, "Save"); - await app.click(37); - assert.equal(calls.at(-1).input.windowId, 9n); - assert.equal(calls.at(-1).input.elementToken, "rv1:window_9:25"); - windows.pop(); - windows[0].is_app_target = true; - await assert.rejects(app.click(37), { code: "app_observation_required" }); - assert.equal((await app.getState()).window, "Document"); - if (platform !== "macos") { - await assert.rejects(app.paste("text"), { code: "unsupported_platform" }); - await assert.rejects(app.selectText(37, "draft"), { code: "unsupported_platform" }); - } - }); -} - -test("App operations on one connection serialize across app handles", async () => { - const { ComputerUseApp } = await import("../app.js"); - let release; - let firstDispatched; - const dispatched = new Promise((resolve) => { firstDispatched = resolve; }); - const { computer, apps, calls } = fixture({ action: async () => { - if (release) return result({ effect: "confirmed" }); - firstDispatched(); - await new Promise((resolve) => { release = resolve; }); - return result({ effect: "confirmed" }); - } }); - const a = await computer.getApp("Fixture"); - const b = new ComputerUseApp(computer, appRecord, () => {}, async () => apps); - const first = a.pressKey("Tab"); - await dispatched; - const second = b.pressKey("Tab"); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(calls.filter((call) => call.method === "windowPressKey").length, 1); - release(); - await Promise.all([first, second]); - assert.equal(calls.filter((call) => call.method === "windowPressKey").length, 2); -}); diff --git a/packages/cua-driver/typescript/computer-use/test/computer-use.test.mjs b/packages/cua-driver/typescript/computer-use/test/computer-use.test.mjs index 1ec3b111572..1f426ffc2de 100644 --- a/packages/cua-driver/typescript/computer-use/test/computer-use.test.mjs +++ b/packages/cua-driver/typescript/computer-use/test/computer-use.test.mjs @@ -1,2302 +1,87 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { ComputerUse } from '../index.js'; +const result = (structured = {}, extra = {}) => ({ text: '', structuredJson: JSON.stringify(structured), images: [], ...extra }); +function deferred() { let resolve; const promise = new Promise((r) => { resolve = r; }); return { promise, resolve }; } +function transport(callTool, options) { return new ComputerUse({ callTool }, options); } -import { ComputerUse, ComputerUseError } from "../index.js"; - -const REVISION_CAPABILITY = "accessibility.observation_revision.v1"; -const METHOD_NAMES = [ - "listApps", - "listWindows", - "getSession", - "getWindowState", - "verifyState", - "windowClick", - "doubleClick", - "rightClick", - "windowDrag", - "windowScroll", - "setValue", - "windowTypeText", - "windowPressKey", - "windowHotkey", - "performSecondaryAction", -]; - -const fakeSdk = { - ActionEffect: { Refused: 4 }, - ClickButton: { Left: "left", Right: "right", Middle: "middle" }, - DeliveryMode: { Background: "background", Foreground: "foreground" }, - ScrollDirection: { Up: "up", Down: "down", Left: "left", Right: "right" }, - ScrollBy: { Line: "line", Page: "page" }, -}; - -function toolResult({ - text = "", - structured, - isError = false, - errorCode, - action, - verification, - degraded = false, -} = {}) { - return { - text, - images: [], - structuredJson: structured === undefined ? undefined : JSON.stringify(structured), - isError, - errorCode, - action, - verification, - degraded, - rawJson: "{}", - }; -} - -function fakeDriver({ revisionCapability = true, results = {}, platform = "windows" } = {}) { +test('one app action is one native call with no SDK discovery or target state', async () => { const calls = []; - const asyncOptions = []; - const driver = { - calls, - asyncOptions, - callToolCalls: 0, - async callTool() { - this.callToolCalls += 1; - throw new Error("the wrapper must not call callTool"); - }, - async listToolsJson() { - return JSON.stringify({ - platform, - tools: [ - { - name: "get_window_state", - capabilities: revisionCapability - ? ["accessibility.tree", REVISION_CAPABILITY] - : ["accessibility.tree"], - }, - ], - }); - }, - endSessionCalls: [], - async endSession(input) { - this.endSessionCalls.push(input); - return { active: false }; - }, - shutdownCalls: 0, - async shutdown() { - this.shutdownCalls += 1; - }, - destroyCalls: 0, - uniffiDestroy() { - this.destroyCalls += 1; - }, - }; - for (const method of METHOD_NAMES) { - driver[method] = async (input, options) => { - calls.push({ method, input }); - asyncOptions.push({ method, options }); - const handler = results[method]; - if (typeof handler === "function") return handler(input, options); - if (handler) return handler; - return toolResult({ structured: {} }); - }; - } - return driver; -} - -test("observeWindow starts full and automatically advances its surface cursor", async () => { - const results = [ - toolResult({ - text: "full text", - structured: { - tree_markdown: "FULL", - elements: [ - { - element_index: 0, - element_token: "rv1:l_a:1", - automation_id: "txtRootDirectory", - }, - ], - observation_revision: { - capability: REVISION_CAPABILITY, - version: 1, - mode: "full", - lineage_id: "l_a", - revision_id: "l_a:r1", - resync_reason: "missing_base", - stable_element_ids: true, - selected_bytes: 1234, - full_bytes: 1234, - }, - }, - }), - toolResult({ - text: "no changes", - structured: { - tree_markdown: "NO CHANGE", - elements: [{ element_index: 0, element_token: "rv1:l_a:1" }], - observation_revision: { - capability: REVISION_CAPABILITY, - version: 1, - serializer_version: "accessibility-render-v1", - projection_version: "full-tree-v1", - mode: "no_change", - lineage_id: "l_a", - revision_id: "l_a:r2", - base_revision_id: "l_a:r1", - stable_element_ids: true, - selected_bytes: 32, - full_bytes: 1234, - estimated_tokens: 8, - serializer_duration_us: 47, - cache_estimate_bytes: 4096, - }, - }, - }), - ]; - const driver = fakeDriver({ - results: { - getWindowState: () => results.shift(), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const first = await computer.observeWindow({ pid: 42, windowId: 7 }); - const second = await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.deepEqual(driver.calls[0], { - method: "getWindowState", - input: { - pid: 42, - windowId: 7n, - includeScreenshot: false, - observationRevision: { - version: 1, - serializerVersion: "accessibility-render-v1", - projectionVersion: "full-tree-v1", - }, - }, - }); - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, "l_a:r1"); - assert.equal(first.mode, "full"); - assert.equal(first.elements[0].automation_id, "txtRootDirectory"); - assert.equal(second.mode, "no_change"); - assert.equal(Object.hasOwn(second, "revisionId"), false); - assert.equal(Object.hasOwn(second, "lineageId"), false); - assert.equal(Object.hasOwn(second, "baseRevisionId"), false); - assert.equal(Object.hasOwn(second, "structured"), false); - assert.equal(driver.callToolCalls, 0); - assert.equal(second.diagnostics.revisionSupported, true); - assert.equal(second.diagnostics.serializerVersion, "accessibility-render-v1"); - assert.equal(second.diagnostics.projectionVersion, "full-tree-v1"); - assert.equal(second.diagnostics.stableElementIds, true); - assert.equal(second.diagnostics.selectedBytes, 32); - assert.equal(second.diagnostics.fullBytes, 1234); - assert.equal(second.diagnostics.estimatedTokens, 8); - assert.equal(second.diagnostics.serializerDurationUs, 47); - assert.equal(second.diagnostics.cacheEstimateBytes, 4096); - assert.equal(second.text, "NO CHANGE"); + const computer = transport((name, args) => { calls.push([name, JSON.parse(args)]); return result({ app: '/Fixture.app', name: 'Fixture' }); }); + const { app } = await computer.getApp('Fixture'); + await app.call('click', { element: 3 }); + assert.deepEqual(calls, [['get_app', { app: 'Fixture' }], ['click', { element: 3, app: '/Fixture.app' }]]); }); -test("forceFull is one-shot and its successful result replaces the cursor", async () => { - const results = [ - toolResult({ - structured: { - tree_markdown: "FIRST", - observation_revision: { - mode: "full", - lineage_id: "l_a", - revision_id: "l_a:r1", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "FORCED", - observation_revision: { - mode: "full", - lineage_id: "l_a", - revision_id: "l_a:r2", - resync_reason: "requested", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "NO CHANGE", - observation_revision: { - mode: "no_change", - lineage_id: "l_a", - revision_id: "l_a:r3", - base_revision_id: "l_a:r2", - stable_element_ids: true, - }, - }, - }), - ]; - const driver = fakeDriver({ - results: { - getWindowState: () => results.shift(), - }, +test('same canonical app serializes calls while another app can progress', async () => { + const gate = deferred(); const calls = []; + const computer = transport((name, raw) => { + const args = JSON.parse(raw); calls.push([name, args.app]); + if (name === 'get_app') return result({ app: args.app === 'Alias' ? 'A' : args.app, name: args.app }); + return args.app === 'A' && name === 'type_text' ? gate.promise : result(); }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - const forced = await computer.observeWindow({ - pid: 42, - windowId: 7, - forceFull: true, - }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.deepEqual(driver.calls[1].input.observationRevision, { - version: 1, - serializerVersion: "accessibility-render-v1", - projectionVersion: "full-tree-v1", - forceFull: true, - }); - assert.deepEqual(driver.calls[2].input.observationRevision, { - version: 1, - serializerVersion: "accessibility-render-v1", - projectionVersion: "full-tree-v1", - baseRevisionId: "l_a:r2", - }); - assert.equal(forced.resyncReason, "requested"); - assert.equal(Object.hasOwn(forced, "baseRevisionId"), false); + const { app: a } = await computer.getApp('A'); + const { app: alias } = await computer.getApp('Alias'); + const { app: b } = await computer.getApp('B'); + assert.equal(a, alias); + const first = a.call('type_text', { text: 'first' }); + const second = alias.call('press_key', { key: 'Enter' }); + await b.call('click', { element: 1 }); + assert(!calls.some(([name]) => name === 'press_key')); + gate.resolve(result()); await Promise.all([first, second]); + assert.equal(calls.at(-1)[0], 'press_key'); }); -test("disableDiff maps to the one-shot native full request", async () => { - const driver = fakeDriver({ - results: { - getWindowState: toolResult({ - structured: { - tree_markdown: "FULL", - observation_revision: { - mode: "full", - lineage_id: "l_a", - revision_id: "l_a:r1", - stable_element_ids: true, - }, - }, - }), - }, +test('driver errors preserve code, reason, sent and next without rewriting or replay', async () => { + let calls = 0; + const error = { code: 'write_not_applied', reason: 'Value readback differs', sent: true, next: 'Observe the field.' }; + const computer = transport(() => { calls++; return result(error, { text: 'original driver explanation', isError: true }); }); + await assert.rejects(computer.appCall('set_value', { app: 'Fixture', element: 1, value: 'new' }), (actual) => { + assert.equal(actual.message, 'original driver explanation'); + for (const key of Object.keys(error)) assert.equal(actual[key], error[key]); + return true; }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await computer.observeWindow({ - pid: 42, - windowId: 7, - disableDiff: true, - }); - - assert.equal(driver.calls[0].input.observationRevision.forceFull, true); - await assert.rejects( - computer.observeWindow({ - pid: 42, - windowId: 7, - disableDiff: true, - forceFull: true, - }), - (error) => - error instanceof ComputerUseError && - error.code === "observation_option_conflict", - ); - assert.equal(driver.calls.length, 1); + assert.equal(calls, 1); }); -test("macOS element completeness does not discard actionable elements", async () => { - const driver = fakeDriver({ - results: { - getWindowState: toolResult({ - structured: { - tree_markdown: "FULL element_token=rv1:l_mac:1", - elements_complete: false, - elements: [ - { - element_index: 0, - element_token: "rv1:l_mac:1", - role: "AXButton", - label: "Run", - }, - ], - observation_revision: { - mode: "full", - lineage_id: "l_mac", - revision_id: "l_mac:r1", - stable_element_ids: true, - capture_complete: true, - }, - }, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - const observation = await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.equal(observation.elements[0].element_token, "rv1:l_mac:1"); - assert.doesNotMatch(observation.text, /capture is incomplete/i); +test('cancellation before dispatch sends nothing', async () => { + let calls = 0; + const computer = transport(() => { calls++; return result(); }); + await assert.rejects(computer.appCall('click', { app: 'Fixture', element: 1 }, { signal: AbortSignal.abort() }), { code: 'call_cancelled' }); + assert.equal(calls, 0); }); -test("an incomplete capture retries once without disabling diffs", async () => { - const incomplete = () => - toolResult({ - structured: { - tree_markdown: "FULL tokenless controls", - elements: [ - { - element_index: 0, - role: "Button", - label: "Retry", - actions: ["invoke"], - }, - ], - observation_revision: { - capture_complete: false, - mode: "full", - lineage_id: "l_transient", - revision_id: "l_transient:r0", - resync_reason: "capture_incomplete", - stable_element_ids: false, - }, - }, - }); - const results = [ - toolResult({ - structured: { - tree_markdown: "FULL element_token=rv1:l_old:1", - elements: [{ element_index: 0, element_token: "rv1:l_old:1" }], - capture_complete: true, - observation_revision: { - mode: "full", - lineage_id: "l_old", - revision_id: "l_old:r1", - stable_element_ids: true, - }, - }, - }), - incomplete(), - incomplete(), - ]; - const driver = fakeDriver({ - results: { getWindowState: () => results.shift() }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - const observation = await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.equal( - driver.calls[1].input.observationRevision.baseRevisionId, - "l_old:r1", - ); - assert.equal( - driver.calls[2].input.observationRevision.baseRevisionId, - undefined, - ); - assert.equal(driver.calls[2].input.observationRevision.forceFull, undefined); - assert.equal(driver.calls.length, 3); - assert.equal(observation.mode, "full"); - assert.equal(observation.resyncReason, "capture_incomplete"); - assert.equal(observation.diagnostics.captureComplete, false); - assert.deepEqual(observation.elements, []); - assert.match(observation.text, /capture is incomplete/i); - assert.match(observation.text, /Retry after the UI settles/); +test('post-dispatch cancellation waits for the native terminal result without replay', async () => { + const gate = deferred(); const controller = new AbortController(); let barrier; let calls = 0; let done = false; + controller.signal.waitUntil = (promise) => { barrier = promise; return promise; }; + const computer = transport(() => { calls++; return gate.promise; }); + const pending = computer.appCall('click', { app: 'Fixture', element: 1 }, { signal: controller.signal }).then((value) => { done = true; return value; }); + controller.abort(); await new Promise(setImmediate); + assert.equal(done, false); assert.equal(barrier, gate.promise); + gate.resolve(result()); await pending; + assert.equal(calls, 1); }); -for (const nested of [false, true]) { - test(`budget captures preserve current tokens without retry (${nested ? "nested" : "top-level"})`, async () => { - let count = 0; - const capture = { - capture_complete: false, - capture_incomplete_details: ["walk: max_elements truncated"], - }; - const driver = fakeDriver({ - results: { - getWindowState: () => - toolResult({ - structured: { - ...(!nested && capture), - tree_markdown: "FULL", - elements: [{ element_token: "rv1:bounded:0", label: "Address bar" }], - observation_revision: { - ...(nested && capture), - mode: count++ === 0 ? "full" : "no_change", - stable_element_ids: true, - revision_id: `bounded:r${count}`, - }, - }, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const first = await computer.observeWindow({ pid: 42, windowId: 7 }); - const second = await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.equal(driver.calls.length, 2); - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, "bounded:r1"); - assert.equal(first.diagnostics.captureComplete, false); - assert.equal(first.diagnostics.captureReadComplete, undefined); - assert.equal(first.diagnostics.captureTruncated, true); - assert.deepEqual( - first.diagnostics.captureIncompleteDetails, - capture.capture_incomplete_details, - ); - assert.match(first.text, /^Accessibility capture is incomplete \(traversal limit\)/); - assert.equal(second.mode, "no_change"); - assert.deepEqual(second.elements, first.elements); +test('expired mutations are never replayed; an expired read can bind one replacement session', async () => { + let reads = 0; let mutations = 0; let replacements = 0; + const expired = () => result({ code: 'authorization_context_expired', sent: false }, { isError: true }); + const replacement = { callTool() { reads++; return result({}, { text: 'new session state' }); } }; + const computer = new ComputerUse({ callTool(name) { if (name === 'click') mutations++; else reads++; return expired(); } }, { + ownsSession: true, sessionFactory() { replacements++; return replacement; }, }); -} - -for (const [readComplete, details] of [ - [undefined, ["AXTitle: ax_error -25204", "walk: max_elements truncated"]], - [false, ["walk: max_elements truncated"]], - [undefined, ["provider_unresponsive", "max_elements_reached"]], -]) { - test(`mixed read failure and truncation retries once (read flag ${readComplete})`, async () => { - let reads = 0; - const driver = fakeDriver({ - results: { - getWindowState: () => - toolResult({ - structured: { - tree_markdown: "Captured controls", - capture_complete: false, - capture_read_complete: ++reads === 1 ? readComplete : true, - capture_truncated: true, - capture_incomplete_details: - reads === 1 ? details : ["walk: max_elements truncated"], - elements: [{ element_token: reads === 1 ? "s1:0" : "rv1:bounded:0" }], - observation_revision: { - mode: "full", - revision_id: reads === 1 ? "transient:r0" : "bounded:r1", - stable_element_ids: reads > 1, - resync_reason: reads === 1 ? "capture_incomplete" : "missing_base", - }, - }, - }), - }, - }); - const observation = await new ComputerUse(driver, { sdk: fakeSdk }).observeWindow({ - pid: 42, - windowId: 7, - }); - assert.equal(reads, 2); - assert.equal(observation.diagnostics.captureReadComplete, true); - assert.equal(observation.diagnostics.captureComplete, false); - assert.equal(observation.elements[0].element_token, "rv1:bounded:0"); - }); -} - -for (const reason of ["walk_deadline_reached", "element_bounds_timeout"]) { - for (const nested of [false, true]) { - test(`deadline capture returns its prefix without another walk (${reason}, nested ${nested})`, async () => { - const complete = (id) => toolResult({ structured: { - tree_markdown: "Ready", - elements: [{ element_token: `rv1:${id}:0` }], - capture_complete: true, - observation_revision: { - mode: "full", lineage_id: id, revision_id: `${id}:r1`, stable_element_ids: true, - }, - } }); - const capture = { - capture_complete: false, - capture_truncated: true, - capture_incomplete_details: [reason, "max_depth_reached"], - }; - const results = [complete("old"), toolResult({ structured: { - ...(nested ? {} : capture), - tree_markdown: "Completed prefix", - elements: [{ element_index: 0, element_token: "s00000001:0" }], - observation_revision: { - ...(nested ? capture : {}), - mode: "full", revision_id: "transient:r0", stable_element_ids: false, - resync_reason: "capture_incomplete", - }, - } }), complete("recovered")]; - const driver = fakeDriver({ results: { getWindowState: () => results.shift() } }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - const partial = await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.equal(driver.calls.length, 2); - assert.match(partial.text, /Completed prefix/); - assert.equal(partial.elements[0].element_token, "s00000001:0"); - assert.equal(partial.diagnostics.captureReadComplete, false); - assert.equal(partial.diagnostics.stableElementIds, false); - const recovered = await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.equal(driver.calls.length, 3); - assert.equal(driver.calls[2].input.observationRevision.baseRevisionId, undefined); - assert.equal(recovered.diagnostics.stableElementIds, true); - }); - } -} - -for (const detail of ["max_elements_reached", "max_depth_reached"]) { - test(`legacy Linux budget capture is not retried: ${detail}`, async () => { - const driver = fakeDriver({ results: { - getWindowState: toolResult({ structured: { - tree_markdown: "[637] Button", - elements: [{ element_index: 637, element_token: "s00000001:637" }], - capture_complete: false, capture_truncated: true, - capture_incomplete_details: [detail], - observation_revision: { mode: "full", stable_element_ids: false, - revision_id: "transient:r1", resync_reason: "capture_incomplete" }, - } }), - } }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const observation = await computer.observeWindow({ pid: 42, windowId: 7, maxElements: 1 }); - assert.equal(driver.calls.length, 1); - assert.equal(observation.elements[0].element_token, "s00000001:637"); - assert.match(observation.text, /traversal limit/); - }); -} - -test("legacy incomplete snapshots keep issued tokens and invalidate the stable cursor", async () => { - let count = 0; - const driver = fakeDriver({ - results: { - getWindowState: () => - toolResult({ - structured: { - tree_markdown: "FULL", - elements: [{ element_token: `s${++count}:0`, label: "Address bar" }], - observation_revision: { - mode: "full", - capture_complete: false, - capture_incomplete_details: ["walk: max_depth exceeded"], - stable_element_ids: false, - revision_id: `transient:r${count}`, - resync_reason: "capture_incomplete", - }, - }, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const first = await computer.observeWindow({ pid: 42, windowId: 7 }); - const second = await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.equal(driver.calls.length, 2); - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, undefined); - assert.equal(first.elements[0].element_token, "s1:0"); - assert.equal(second.elements[0].element_token, "s2:0"); + await assert.rejects(computer.appCall('click', { app: 'Fixture', element: 1 }), { code: 'authorization_context_expired' }); + assert.equal(replacements, 0); assert.equal(mutations, 1); + const observed = await computer.getState(); + assert.equal(observed.text, 'new session state'); assert.equal(replacements, 1); assert.equal(reads, 2); }); -test("text budgets preserve whole rows and full elements, and can be expanded", async () => { - const rows = Array.from( - { length: 500 }, - (_, index) => - ` [${index}] <AXButton> "Control ${index}" element_token=rv1:lineage:${index}`, - ); - const elements = rows.map((_, index) => ({ element_token: `rv1:lineage:${index}` })); - const driver = fakeDriver({ - results: { - getWindowState: toolResult({ - structured: { - tree_markdown: rows.join("\n"), - elements, - capture_complete: false, - capture_truncated: true, - }, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - for (const maxTextChars of [undefined, 512]) { - const observation = await computer.observeWindow({ - pid: 42, - windowId: 7, - maxTextChars, - }); - assert.ok(observation.text.length <= (maxTextChars ?? 12_000)); - assert.match(observation.text, /^Accessibility capture is incomplete/); - assert.match(observation.text, /Text truncated/); - assert.equal(observation.diagnostics.textTruncated, true); - assert.equal(observation.diagnostics.textChars, observation.text.length); - assert.ok(rows.includes(observation.text.split("\n").at(-1))); - assert.deepEqual(observation.elements, elements); - } - const expanded = await computer.observeWindow({ - pid: 42, - windowId: 7, - disableDiff: true, - maxTextChars: 100_000, - }); - assert.equal(expanded.diagnostics.textTruncated, false); - assert.match(expanded.text, /Control 499/); - assert.equal(driver.calls.at(-1).input.observationRevision.forceFull, true); - assert.equal(Object.hasOwn(driver.calls.at(-1).input, "maxTextChars"), false); - await assert.rejects( - computer.observeWindow({ pid: 42, windowId: 7, maxTextChars: 511 }), - /maxTextChars/, - ); -}); - -test("native revision completeness takes precedence and retries only once", async () => { - let snapshots = 0; - const driver = fakeDriver({ - results: { - getWindowState: () => - toolResult({ - structured: { - capture_complete: true, - elements: [{ element_token: `s${++snapshots}:0` }], - observation_revision: { - capture_complete: false, - mode: "full", - resync_reason: "capture_incomplete", - stable_element_ids: false, - }, - }, - }), - }, - }); - const observation = await new ComputerUse(driver, { - sdk: fakeSdk, - }).observeWindow({ pid: 42, windowId: 7 }); - assert.equal(driver.calls.length, 2); - assert.equal(observation.diagnostics.captureComplete, false); - assert.deepEqual(observation.elements, [{ element_token: "s2:0" }]); - assert.match(observation.text, /capture is incomplete/i); -}); - -test("observations preserve input and screenshot context from the native payload", async () => { - const fields = { - background_input: { supported: false }, - degraded: true, - degraded_reason: "ax_window_unresolved", - escalation: { recommended: "foreground" }, - window_bounds: { x: 10, y: 20, width: 300, height: 200 }, - screenshot_scale: 2, - screenshot_frame_valid: false, - screenshot_error: { code: "invalid_frame" }, - }; - const driver = fakeDriver({ - results: { getWindowState: () => toolResult({ structured: fields }) }, - }); - const observation = await new ComputerUse(driver, { - sdk: fakeSdk, - }).observeWindow({ pid: 42, windowId: 7 }); - assert.deepEqual(observation.context, { - backgroundInput: fields.background_input, - degraded: true, - degradedReason: fields.degraded_reason, - escalation: fields.escalation, - windowBounds: fields.window_bounds, - screenshotScale: 2, - screenshotFrameValid: false, - screenshotError: fields.screenshot_error, - }); -}); - -test("the incomplete-capture retry restores the automatic cursor", async () => { - const results = [ - toolResult({ - structured: { - tree_markdown: "OLD", - capture_complete: true, - observation_revision: { - mode: "full", - lineage_id: "l_old", - revision_id: "l_old:r1", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "INCOMPLETE", - capture_complete: false, - observation_revision: { - mode: "full", - lineage_id: "l_transient", - revision_id: "l_transient:r0", - resync_reason: "capture_incomplete", - stable_element_ids: false, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "RECOVERED element_token=rv1:l_new:1", - elements: [{ element_index: 0, element_token: "rv1:l_new:1" }], - capture_complete: true, - observation_revision: { - mode: "full", - lineage_id: "l_new", - revision_id: "l_new:r1", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "NO CHANGE", - elements: [{ element_index: 0, element_token: "rv1:l_new:1" }], - capture_complete: true, - observation_revision: { - mode: "no_change", - lineage_id: "l_new", - revision_id: "l_new:r2", - base_revision_id: "l_new:r1", - stable_element_ids: true, - }, - }, - }), - ]; - const driver = fakeDriver({ - results: { getWindowState: () => results.shift() }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - const recovered = await computer.observeWindow({ pid: 42, windowId: 7 }); - const unchanged = await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.equal(recovered.text, "RECOVERED element_token=rv1:l_new:1"); - assert.equal(recovered.elements[0].element_token, "rv1:l_new:1"); - assert.equal( - driver.calls[3].input.observationRevision.baseRevisionId, - "l_new:r1", - ); - assert.equal(unchanged.mode, "no_change"); -}); - -test("drivers without revision capability retain legacy full observations", async () => { - const driver = fakeDriver({ - revisionCapability: false, - results: { - getWindowState: toolResult({ - text: "legacy text", - structured: { tree_markdown: "LEGACY", elements: [] }, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const observation = await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.equal("observationRevision" in driver.calls[0].input, false); - assert.equal(observation.diagnostics.revisionSupported, false); - assert.equal(observation.mode, "full"); - assert.equal(observation.text, "LEGACY"); -}); - -test("manual revision cursor fields are rejected before driver dispatch", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - for (const [field, value] of [ - ["baseRevisionId", "l_manual:r1"], - ["revisionId", "l_manual:r2"], - ["lineageId", "l_manual"], - ["observationRevision", { baseRevisionId: "l_manual:r1" }], - ]) { - await assert.rejects( - computer.observeWindow({ pid: 42, windowId: 7, [field]: value }), - (error) => - error instanceof ComputerUseError && error.code === "revision_cursor_managed", - ); - } - assert.equal(driver.calls.length, 0); -}); - -test("revision cursors are isolated by exact window surface", async () => { - const revisions = new Map(); - const driver = fakeDriver({ - results: { - getWindowState: (input) => { - const windowId = Number(input.windowId); - const revision = (revisions.get(windowId) ?? 0) + 1; - revisions.set(windowId, revision); - const lineage = `l_${windowId}`; - return toolResult({ - structured: { - tree_markdown: revision === 1 ? "FULL" : "NO CHANGE", - observation_revision: { - mode: revision === 1 ? "full" : "no_change", - lineage_id: lineage, - revision_id: `${lineage}:r${revision}`, - base_revision_id: revision === 1 ? undefined : `${lineage}:r1`, - stable_element_ids: true, - }, - }, - }); - }, - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - await computer.observeWindow({ pid: 42, windowId: 8 }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - await computer.observeWindow({ pid: 42, windowId: 8 }); - - assert.equal(driver.calls[0].input.observationRevision.baseRevisionId, undefined); - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, undefined); - assert.equal(driver.calls[2].input.observationRevision.baseRevisionId, "l_7:r1"); - assert.equal(driver.calls[3].input.observationRevision.baseRevisionId, "l_8:r1"); -}); - -test("a native full resync replaces an invalid cursor", async () => { - const results = [ - toolResult({ - structured: { - tree_markdown: "FIRST", - observation_revision: { - mode: "full", - lineage_id: "l_old", - revision_id: "l_old:r1", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "RESYNC", - observation_revision: { - mode: "full", - lineage_id: "l_new", - revision_id: "l_new:r1", - resync_reason: "base_evicted", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "NO CHANGE", - observation_revision: { - mode: "no_change", - lineage_id: "l_new", - revision_id: "l_new:r2", - base_revision_id: "l_new:r1", - stable_element_ids: true, - }, - }, - }), - ]; - const driver = fakeDriver({ - results: { getWindowState: () => results.shift() }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - const resynced = await computer.observeWindow({ pid: 42, windowId: 7 }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, "l_old:r1"); - assert.equal(resynced.mode, "full"); - assert.equal(resynced.resyncReason, "base_evicted"); - assert.equal(driver.calls[2].input.observationRevision.baseRevisionId, "l_new:r1"); -}); - -test("an unretained full exposes current snapshot tokens and clears the cursor", async () => { - const results = [ - toolResult({ - structured: { - tree_markdown: "FULL element_token=rv1:l_old:1", - elements: [{ element_index: 0, element_token: "rv1:l_old:1" }], - observation_revision: { - mode: "full", - lineage_id: "l_old", - revision_id: "l_old:r1", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "FULL element_token=s00000001:0", - elements: [{ element_index: 0, element_token: "s00000001:0" }], - observation_revision: { - mode: "full", - lineage_id: "l_transient", - revision_id: "l_transient:r1", - resync_reason: "identity_unavailable", - stable_element_ids: false, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "RECOVERED element_token=rv1:l_new:1", - elements: [{ element_index: 0, element_token: "rv1:l_new:1" }], - observation_revision: { - mode: "full", - lineage_id: "l_new", - revision_id: "l_new:r1", - stable_element_ids: true, - }, - }, - }), - toolResult({ - structured: { - tree_markdown: "NO CHANGE", - elements: [{ element_index: 0, element_token: "rv1:l_new:1" }], - observation_revision: { - mode: "no_change", - lineage_id: "l_new", - revision_id: "l_new:r2", - base_revision_id: "l_new:r1", - stable_element_ids: true, - }, - }, - }), - ]; - const driver = fakeDriver({ - results: { getWindowState: () => results.shift() }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - const transient = await computer.observeWindow({ pid: 42, windowId: 7 }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, "l_old:r1"); - assert.equal(transient.mode, "full"); - assert.equal(transient.resyncReason, "identity_unavailable"); - assert.equal(transient.diagnostics.stableElementIds, false); - assert.match(transient.text, /element_token=s00000001:0/); - assert.equal(transient.elements[0].element_token, "s00000001:0"); - assert.equal(driver.calls[2].input.observationRevision.baseRevisionId, undefined); - assert.equal(driver.calls[2].input.observationRevision.forceFull, undefined); - assert.equal(driver.calls[3].input.observationRevision.baseRevisionId, "l_new:r1"); -}); - -test("concurrent observations on one surface advance in dispatch order", async () => { - let finishFirst; - let observations = 0; - const driver = fakeDriver({ - results: { - getWindowState: () => { - observations += 1; - if (observations === 1) { - return new Promise((resolve) => { - finishFirst = () => - resolve( - toolResult({ - structured: { - tree_markdown: "FULL", - observation_revision: { - mode: "full", - lineage_id: "l_a", - revision_id: "l_a:r1", - stable_element_ids: true, - }, - }, - }), - ); - }); - } - return toolResult({ - structured: { - tree_markdown: "NO CHANGE", - observation_revision: { - mode: "no_change", - lineage_id: "l_a", - revision_id: "l_a:r2", - base_revision_id: "l_a:r1", - stable_element_ids: true, - }, - }, - }); - }, - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - const first = computer.observeWindow({ pid: 42, windowId: 7 }); - await new Promise((resolve) => setImmediate(resolve)); - const second = computer.observeWindow({ pid: 42, windowId: 7 }); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(driver.calls.length, 1); - - finishFirst(); - await Promise.all([first, second]); - assert.equal(driver.calls.length, 2); - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, "l_a:r1"); -}); - -test("a failed observation preserves the last successful cursor", async () => { - const results = [ - toolResult({ - structured: { - tree_markdown: "FULL", - observation_revision: { - mode: "full", - lineage_id: "l_a", - revision_id: "l_a:r1", - stable_element_ids: true, - }, - }, - }), - toolResult({ - text: "capture failed", - structured: { code: "capture_failed" }, - isError: true, - }), - toolResult({ - structured: { - tree_markdown: "NO CHANGE", - observation_revision: { - mode: "no_change", - lineage_id: "l_a", - revision_id: "l_a:r2", - base_revision_id: "l_a:r1", - stable_element_ids: true, - }, - }, - }), - ]; - const driver = fakeDriver({ - results: { getWindowState: () => results.shift() }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - await assert.rejects(computer.observeWindow({ pid: 42, windowId: 7 }), /capture failed/); - await computer.observeWindow({ pid: 42, windowId: 7 }); - - assert.equal(driver.calls[1].input.observationRevision.baseRevisionId, "l_a:r1"); - assert.equal(driver.calls[2].input.observationRevision.baseRevisionId, "l_a:r1"); -}); - -test("typed discovery methods expose apps, windows, and exact-window lookup", async () => { - const driver = fakeDriver({ - results: { - listApps: toolResult({ - structured: { apps: [{ pid: 42, name: "Harness" }] }, - }), - listWindows: toolResult({ - structured: { windows: [{ pid: 42, window_id: 7, title: "Harness" }] }, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - assert.equal((await computer.listApps())[0].displayName, "Harness"); - assert.equal((await computer.listWindows({ pid: 42 }))[0].window_id, 7); - assert.equal((await computer.getWindow({ pid: 42, windowId: 7 })).title, "Harness"); -}); - -test("structured actions retain native new-window notices alongside action evidence", async () => { - const text = 'Pressed cmd+n.\n\n🪟 Action opened new window(s): Harness ("Untitled").'; - const structured = { effect: "unverifiable", route: "global_input" }; - const action = { effect: 2, route: 2 }; - const driver = fakeDriver({ - results: { - windowHotkey: toolResult({ text, structured, action }), - windowPressKey: toolResult({ structured }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const result = await computer.hotkey({ pid: 42, windowId: 7, keys: ["cmd", "n"] }); - assert.equal(result.text, text); - assert.equal(result.effect, structured.effect); - assert.equal(result.route, structured.route); - assert.deepEqual(result.action, action); - assert.equal(result.operation.state, "completed"); - assert.equal(result.operation.dispatched, true); - const silent = await computer.pressKey({ pid: 42, windowId: 7, key: "Enter" }); - assert.equal(Object.hasOwn(silent, "text"), false); +test('close tears down both handles after an endSession failure and is idempotent', async () => { + const calls = []; + const session = { callTool() { return result(); }, endSession() { calls.push('end'); throw Error('end failure'); }, async closeAsync() { calls.push('close'); }, uniffiDestroy() { calls.push('session destroy'); } }; + const owner = { shutdown() { calls.push('shutdown'); }, uniffiDestroy() { calls.push('owner destroy'); } }; + const computer = new ComputerUse(session, { owner, ownsSession: true }); + await assert.rejects(computer.close(), /end failure/); await computer.close(); -}); - -test("all core actions use named typed SDK methods", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await computer.click({ - pid: 42, - elementToken: "rv1:l_a:1", - count: 2, - deliveryMode: "foreground", - }); - await computer.doubleClick({ - pid: 42, - windowId: 7, - x: 10, - y: 20, - deliveryMode: "foreground", - }); - await computer.rightClick({ - pid: 42, - elementToken: "rv1:l_a:2", - modifier: ["shift"], - deliveryMode: "foreground", - }); - await computer.drag({ - pid: 42, - windowId: 7, - fromX: 1, - fromY: 2, - toX: 3, - toY: 4, - durationMs: 0, - steps: 10, - deliveryMode: "foreground", - modifier: ["shift"], - }); - await computer.scroll({ - pid: 42, - elementToken: "rv1:l_a:3", - direction: "down", - by: "line", - amount: 2, - deliveryMode: "foreground", - }); - await computer.setValue({ pid: 42, elementToken: "rv1:l_a:4", value: "hi" }); - await computer.typeText({ - pid: 42, - elementToken: "rv1:l_a:5", - text: "abc", - delayMs: 5, - deliveryMode: "foreground", - }); - await computer.pressKey({ - pid: 42, - windowId: 7, - key: "Enter", - modifiers: ["shift"], - deliveryMode: "foreground", - }); - await computer.hotkey({ - pid: 42, - windowId: 7, - keys: ["cmd", "a"], - deliveryMode: "foreground", - }); - - assert.deepEqual( - driver.calls.map((call) => call.method), - [ - "windowClick", - "doubleClick", - "rightClick", - "windowDrag", - "windowScroll", - "setValue", - "windowTypeText", - "windowPressKey", - "windowHotkey", - ], - ); - assert.deepEqual(driver.calls[0].input, { - pid: 42, - windowId: undefined, - elementToken: "rv1:l_a:1", - count: 2, - deliveryMode: "foreground", - }); - assert.equal(driver.calls[1].input.deliveryMode, "foreground"); - assert.equal(driver.calls[2].input.deliveryMode, "foreground"); - assert.equal(driver.calls[3].input.pid, 42); - assert.equal(driver.calls[3].input.windowId, 7n); - assert.equal(driver.calls[3].input.durationMs, 0n); - assert.equal(driver.calls[3].input.steps, 10n); - assert.equal(driver.calls[3].input.deliveryMode, "foreground"); - assert.deepEqual(driver.calls[3].input.modifier, ["shift"]); - assert.equal(driver.calls[4].input.amount, 2n); - assert.equal(driver.calls[4].input.deliveryMode, "foreground"); - assert.equal(driver.calls[6].input.delayMs, 5n); - assert.equal(driver.calls[6].input.deliveryMode, "foreground"); - assert.equal(driver.calls[7].input.deliveryMode, "foreground"); - assert.equal(driver.calls[8].input.deliveryMode, "foreground"); - assert.equal(driver.callToolCalls, 0); -}); - -test("the environment delivery default applies to every supported action", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { - sdk: fakeSdk, - environment: { QWEN_CUA_SDK_DEFAULT_DELIVERY_MODE: "foreground" }, - }); - - await computer.click({ pid: 42, windowId: 7, x: 10, y: 20 }); - await computer.doubleClick({ pid: 42, windowId: 7, x: 10, y: 20 }); - await computer.rightClick({ pid: 42, windowId: 7, x: 10, y: 20 }); - await computer.drag({ - pid: 42, - windowId: 7, - fromX: 1, - fromY: 2, - toX: 3, - toY: 4, - }); - await computer.scroll({ - pid: 42, - windowId: 7, - x: 10, - y: 20, - direction: "down", - }); - await computer.typeText({ pid: 42, windowId: 7, text: "abc" }); - await computer.pressKey({ pid: 42, windowId: 7, key: "Enter" }); - await computer.hotkey({ pid: 42, windowId: 7, keys: ["ctrl", "a"] }); - - assert.deepEqual( - driver.calls.map((call) => call.input.deliveryMode), - Array(8).fill("foreground"), - ); -}); - -test("an explicit delivery mode overrides the environment default", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { - sdk: fakeSdk, - environment: { QWEN_CUA_SDK_DEFAULT_DELIVERY_MODE: "foreground" }, - }); - - await computer.click({ - pid: 42, - windowId: 7, - x: 10, - y: 20, - deliveryMode: "background", - }); - - assert.equal(driver.calls[0].input.deliveryMode, "background"); -}); - -test("an unset environment default preserves Windows background delivery", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { sdk: fakeSdk, environment: {} }); - - await computer.click({ pid: 42, windowId: 7, x: 10, y: 20 }); - - assert.equal(driver.calls[0].input.deliveryMode, "background"); -}); - -test("Linux defaults authorize native focus preparation for every input action", async () => { - const driver = fakeDriver({ platform: "linux" }); - let inventories = 0; - const owner = { listToolsJson() { inventories += 1; return '{"platform":"linux"}'; } }; - const computer = new ComputerUse(driver, { sdk: fakeSdk, owner }); - const target = { pid: 42, windowId: 7 }; - const element = { ...target, elementToken: "rv1:l_a:1" }; - await computer.click(element); - await computer.doubleClick(element); - await computer.rightClick(element); - await computer.drag({ ...target, fromX: 1, fromY: 2, toX: 3, toY: 4 }); - await computer.scroll({ ...element, direction: "down" }); - await computer.typeText({ ...element, text: "once" }); - await computer.pressKey({ ...target, key: "Enter" }); - await computer.hotkey({ ...target, keys: ["ctrl", "a"] }); - assert.equal(inventories, 1); - assert.equal(driver.calls.length, 8); - assert.ok(driver.calls.every(({ input }) => input.deliveryMode === "foreground")); - assert.ok(driver.calls.every(({ input }) => input.pid === 42 && input.windowId === 7n)); - assert.equal(driver.calls[0].input.elementToken, element.elementToken); -}); - -test("Linux automatic delivery never replays an uncertain mutation error", async () => { - const driver = fakeDriver({ platform: "linux", results: { - windowTypeText: toolResult({ isError: true, structured: { - code: "verification_failed", effect: "unverifiable", - } }), - } }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await assert.rejects(computer.typeText({ pid: 42, windowId: 7, text: "once" }), { - code: "verification_failed", - }); - assert.equal(driver.calls.length, 1); -}); - -test("refreshing the connected platform refreshes the input default", async () => { - const driver = fakeDriver(); - let platform = "linux"; - const owner = { listToolsJson: () => JSON.stringify({ platform }) }; - const computer = new ComputerUse(driver, { sdk: fakeSdk, owner }); - await computer.pressKey({ pid: 42, windowId: 7, key: "Tab" }); - platform = "windows"; - await computer.getPlatform(); - await computer.pressKey({ pid: 42, windowId: 7, key: "Tab" }); - assert.deepEqual(driver.calls.map(({ input }) => input.deliveryMode), ["foreground", "background"]); -}); - -test("an invalid environment delivery default fails before dispatch", async () => { - const driver = fakeDriver(); - - assert.throws( - () => - new ComputerUse(driver, { - sdk: fakeSdk, - environment: { QWEN_CUA_SDK_DEFAULT_DELIVERY_MODE: "automatic" }, - }), - /QWEN_CUA_SDK_DEFAULT_DELIVERY_MODE must be background or foreground/, - ); - assert.equal(driver.calls.length, 0); -}); - -test("snake_case delivery_mode is rejected instead of silently ignored", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { sdk: fakeSdk, environment: {} }); - - await assert.rejects( - computer.click({ - pid: 42, - windowId: 7, - x: 10, - y: 20, - delivery_mode: "foreground", - }), - /delivery_mode is not supported; use deliveryMode/, - ); - assert.equal(driver.calls.length, 0); -}); - -test("native delivery guidance is surfaced with the public camelCase name", async () => { - const driver = fakeDriver({ - results: { - windowClick: toolResult({ - text: 'Retry this action with delivery_mode:"foreground".', - structured: { - code: "background_unavailable", - suggestion: 'Retry with delivery_mode:"foreground".', - }, - isError: true, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk, environment: {} }); - - await assert.rejects( - computer.click({ pid: 42, windowId: 7, x: 10, y: 20 }), - (error) => - error instanceof ComputerUseError && - error.message.includes('deliveryMode:"foreground"') && - !error.message.includes("delivery_mode") && - error.details.suggestion.includes('deliveryMode:"foreground"') && - !error.details.suggestion.includes("delivery_mode"), - ); -}); - -test("verifyState converts public numeric options to the generated u64 ABI", async () => { - const driver = fakeDriver({ - results: { - verifyState: toolResult({ structured: { status: "satisfied" } }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await computer.verifyState({ - pid: 42, - windowId: 7, - expect: [{ window: { titleContains: "Harness" } }], - timeoutMs: 0, - stableSamples: 1, - }); - assert.deepEqual(driver.calls[0], { - method: "verifyState", - input: { - pid: 42n, - windowId: 7n, - expect: [{ window: { titleContains: "Harness" } }], - timeoutMs: 0n, - stableSamples: 1n, - }, - }); -}); - -test("secondary action is typed and token-only", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await computer.performSecondaryAction({ - pid: 42, - elementToken: "rv1:l_a:6", - action: "Expand", - }); - assert.deepEqual(driver.calls[0], { - method: "performSecondaryAction", - input: { - pid: 42, - windowId: undefined, - elementToken: "rv1:l_a:6", - action: "Expand", - }, - }); -}); - -test("driver refusals retain their closed code without wrapper retry", async () => { - const driver = fakeDriver({ - results: { - windowClick: toolResult({ - text: "element_token is stale", - structured: { code: "stale_element_token" }, - isError: true, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await assert.rejects( - computer.click({ pid: 42, elementToken: "rv1:l_old:1" }), - (error) => error instanceof ComputerUseError && error.code === "stale_element_token", - ); - assert.equal(driver.calls.length, 1); -}); - -test("post-dispatch cancellation waits for the native terminal result", async () => { - let finishNative; - const driver = fakeDriver({ - results: { - listApps: () => - new Promise((resolve) => { - finishNative = resolve; - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const controller = new AbortController(); - let registeredTerminal; - Object.defineProperty(controller.signal, "waitUntil", { - value: (promise) => { - registeredTerminal = promise; - return promise; - }, - }); - let settled = false; - const read = computer.listApps({ signal: controller.signal }).finally(() => { - settled = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - - assert.equal(registeredTerminal instanceof Promise, true); - controller.abort(new Error("stop")); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(settled, false); - finishNative(toolResult({ structured: { apps: [{ pid: 42 }] } })); - assert.deepEqual(await read, [{ id: "unknown", displayName: "", isRunning: false }]); - assert.equal(driver.asyncOptions[0].options, undefined); -}); - -test("capability cancellation waits for discovery then prevents observation dispatch", async () => { - const driver = fakeDriver(); - let finishListing; - driver.listToolsJson = () => - new Promise((resolve) => { - finishListing = resolve; - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const controller = new AbortController(); - let registeredTerminal; - Object.defineProperty(controller.signal, "waitUntil", { - value: (promise) => { - registeredTerminal = promise; - return promise; - }, - }); - let settled = false; - const observation = computer - .observeWindow({ - pid: 42, - windowId: 7, - signal: controller.signal, - }) - .finally(() => { - settled = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - - assert.equal(registeredTerminal instanceof Promise, true); - controller.abort(); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(settled, false); - finishListing(JSON.stringify({ tools: [] })); - await assert.rejects( - observation, - (error) => error instanceof ComputerUseError && error.code === "call_cancelled", - ); - assert.equal(driver.calls.length, 0); -}); - -test("pre-dispatch cancellation performs no native action", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const controller = new AbortController(); - controller.abort(); - await assert.rejects( - computer.click({ - pid: 42, - windowId: 7, - x: 10, - y: 20, - signal: controller.signal, - }), - (error) => - error instanceof ComputerUseError && - error.code === "call_cancelled" && - error.details.operation.dispatched === false && - error.details.operation.committed === false, - ); - assert.equal(driver.calls.length, 0); -}); - -test("a refused native action is dispatched but never reported as committed", async () => { - const driver = fakeDriver({ - results: { - windowClick: toolResult({ - text: "foreground target was unavailable", - structured: { effect: "refused", route: "global_input" }, - isError: true, - errorCode: "foreground_unavailable", - action: { effect: 4, route: 2 }, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - await assert.rejects( - computer.click({ pid: 42, windowId: 7, x: 10, y: 20 }), - (error) => - error instanceof ComputerUseError && - error.code === "foreground_unavailable" && - error.details.operation.dispatched === true && - error.details.operation.committed === false, - ); - assert.equal(driver.calls.length, 1); -}); - -test("post-dispatch cancellation returns the committed action result once", async () => { - let finishAction; - const driver = fakeDriver({ - results: { - windowClick: () => - new Promise((resolve) => { - finishAction = resolve; - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - const controller = new AbortController(); - let settled = false; - const action = computer - .click({ - pid: 42, - windowId: 7, - x: 10, - y: 20, - signal: controller.signal, - }) - .finally(() => { - settled = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - - controller.abort(); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(settled, false); - finishAction( - toolResult({ - structured: { effect: "confirmed", route: "trusted_input" }, - action: { effect: "confirmed", route: "trusted_input" }, - }), - ); - const result = await action; - assert.equal(result.operation.state, "completed"); - assert.equal(result.operation.dispatched, true); - assert.equal(result.operation.committed, true); - assert.equal(result.operation.cancellationRequested, true); - assert.equal(driver.calls.length, 1); - assert.equal(driver.asyncOptions[0].options, undefined); -}); - -test("typed action and verification records survive the facade and fail closed", async () => { - const nativeAction = { effect: 0, route: 0 }; - const nativeVerification = { - status: 0, - stable: true, - elapsedMs: 1n, - samples: 2n, - predicates: [], - }; - const driver = fakeDriver({ - results: { - windowClick: toolResult({ - structured: { effect: "confirmed", route: "accessibility" }, - action: nativeAction, - }), - verifyState: toolResult({ - structured: { - status: "satisfied", - stable: true, - elapsed_ms: 1, - samples: 2, - predicates: [], - }, - verification: nativeVerification, - }), - }, - }); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - - const action = await computer.click({ pid: 42, windowId: 7, x: 1, y: 2 }); - const verification = await computer.verifyState({ - pid: 42, - windowId: 7, - expect: [{ element: { token: "rv1:l_a:1", selected: true } }], - }); - assert.equal(action.action, nativeAction); - assert.equal(verification.verification, nativeVerification); - - const succeeded = await computer.actAndVerify({ - action: () => computer.click({ pid: 42, windowId: 7, x: 1, y: 2 }), - verify: () => - computer.verifyState({ - pid: 42, - windowId: 7, - expect: [{ element: { token: "rv1:l_a:1", selected: true } }], - }), - }); - assert.equal(succeeded.action.effect, "confirmed"); - assert.equal(succeeded.verification.status, "satisfied"); - - driver.windowClick = async () => - toolResult({ - structured: { effect: "suspected_noop", route: "accessibility" }, - action: { effect: 3, route: 0 }, - }); - driver.verifyState = async () => - toolResult({ - structured: { - status: "unknown", - stable: false, - elapsed_ms: 1, - samples: 1, - predicates: [], - }, - verification: { ...nativeVerification, status: 2, stable: false }, - }); - await assert.rejects( - computer.actAndVerify({ - action: () => computer.click({ pid: 42, windowId: 7, x: 1, y: 2 }), - verify: () => - computer.verifyState({ - pid: 42, - windowId: 7, - expect: [{ element: { token: "rv1:l_a:1", selected: true } }], - }), - }), - (error) => error instanceof ComputerUseError && error.code === "postcondition_not_satisfied", - ); -}); - -test("an observation retried after reconnect forces full and advances the new cursor", async () => { - let expiredObservations = 0; - const expired = fakeDriver({ - results: { - getWindowState: () => { - expiredObservations += 1; - if (expiredObservations === 1) { - return toolResult({ - structured: { - tree_markdown: "OLD FULL", - observation_revision: { - mode: "full", - lineage_id: "l_old", - revision_id: "l_old:r1", - stable_element_ids: true, - }, - }, - }); - } - return toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }); - }, - }, - }); - expired.close = () => {}; - let replacementObservations = 0; - const replacement = fakeDriver({ - results: { - getWindowState: () => { - replacementObservations += 1; - return toolResult({ - structured: { - tree_markdown: replacementObservations === 1 ? "NEW FULL" : "NO CHANGE", - observation_revision: { - mode: replacementObservations === 1 ? "full" : "no_change", - lineage_id: "l_new", - revision_id: `l_new:r${replacementObservations}`, - base_revision_id: replacementObservations === 1 ? undefined : "l_new:r1", - stable_element_ids: true, - }, - }, - }); - }, - }, - }); - const computer = new ComputerUse(expired, { - owner: expired, - sdk: fakeSdk, - ownsSession: true, - sessionFactory: () => replacement, - }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - const retried = await computer.observeWindow({ pid: 42, windowId: 7 }); - await computer.observeWindow({ pid: 42, windowId: 7 }); - - const replacementInputs = replacement.calls - .filter((entry) => entry.method === "getWindowState") - .map((entry) => entry.input.observationRevision); - assert.equal(retried.mode, "full"); - assert.equal(replacementInputs[0].baseRevisionId, undefined); - assert.equal(replacementInputs[0].forceFull, true); - assert.equal(replacementInputs[1].baseRevisionId, "l_new:r1"); - assert.equal(replacementInputs[1].forceFull, undefined); -}); - -test("reconnect clears the old cursor so the next observation is full", async () => { - const expired = fakeDriver({ - results: { - listApps: toolResult({ - text: "session is not available to this transport", - structured: { - code: "session_unavailable", - }, - isError: true, - }), - getWindowState: toolResult({ - structured: { - tree_markdown: "OLD FULL", - observation_revision: { - mode: "full", - lineage_id: "l_old", - revision_id: "l_old:r1", - stable_element_ids: true, - }, - }, - }), - }, - }); - expired.closeCalls = 0; - expired.close = function () { - this.closeCalls += 1; - }; - const sessionOutput = { - session: "persistent-session", - implicit: false, - state: 0, - clientKind: 1, - transport: 0, - cursorVisible: false, - recordingActive: false, - idleSeconds: 0n, - expiresInSeconds: 3600n, - }; - const replacement = fakeDriver({ - results: { - listApps: toolResult({ - structured: { apps: [{ pid: 42, name: "Harness" }] }, - }), - getSession: sessionOutput, - getWindowState: toolResult({ - structured: { - tree_markdown: "FULL", - observation_revision: { - mode: "full", - lineage_id: "l_new", - revision_id: "l_new:r1", - stable_element_ids: true, - }, - }, - }), - }, - }); - const owner = fakeDriver(); - let factoryCalls = 0; - let replacementPublicSession; - const computer = new ComputerUse(expired, { - owner, - sdk: fakeSdk, - ownsSession: true, - publicSession: "persistent-session", - sessionFactory: (publicSession) => { - factoryCalls += 1; - replacementPublicSession = publicSession; - return replacement; - }, - }); - - await computer.observeWindow({ pid: 42, windowId: 7 }); - assert.equal((await computer.listApps())[0].displayName, "Harness"); - assert.equal(factoryCalls, 1); - assert.equal(expired.closeCalls, 1); - assert.equal(expired.destroyCalls, 1); - assert.equal(computer.connectionGeneration, 2); - assert.equal(typeof replacementPublicSession, "string"); - assert.notEqual(replacementPublicSession, "persistent-session"); - assert.equal(await computer.sessionInfo(), sessionOutput); - - const observation = await computer.observeWindow({ pid: 42, windowId: 7 }); - const observeInput = replacement.calls.find((entry) => entry.method === "getWindowState").input; - assert.equal(observeInput.observationRevision.baseRevisionId, undefined); - assert.equal(observation.mode, "full"); -}); - -test("explicit reconnect registers session creation as a cancellation barrier", async () => { - const previous = fakeDriver(); - previous.close = () => {}; - const replacement = fakeDriver(); - let finishReplacement; - const computer = new ComputerUse(previous, { - sdk: fakeSdk, - ownsSession: true, - sessionFactory: () => - new Promise((resolve) => { - finishReplacement = resolve; - }), - }); - const controller = new AbortController(); - let registeredTerminal; - Object.defineProperty(controller.signal, "waitUntil", { - value: (promise) => { - registeredTerminal = promise; - return promise; - }, - }); - - const reconnect = computer.reconnect({ signal: controller.signal }); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(registeredTerminal instanceof Promise, true); - controller.abort(); - finishReplacement(replacement); - - const result = await reconnect; - assert.equal(result.operation.committed, true); - assert.equal(result.operation.cancellationRequested, true); - assert.equal(computer.connectionGeneration, 2); -}); - -test("stale observations cannot restore an old generation cursor", async () => { - let resolveStaleObservation; - const expired = fakeDriver({ - results: { - listApps: toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }), - getWindowState: () => - new Promise((resolve) => { - resolveStaleObservation = resolve; - }), - }, - }); - expired.close = () => {}; - const replacement = fakeDriver({ - results: { - listApps: toolResult({ structured: { apps: [] } }), - getWindowState: toolResult({ - structured: { - tree_markdown: "FULL", - observation_revision: { - mode: "full", - lineage_id: "l_new", - revision_id: "l_new:r1", - stable_element_ids: true, - }, - }, - }), - }, - }); - const computer = new ComputerUse(expired, { - owner: expired, - sdk: fakeSdk, - ownsSession: true, - sessionFactory: () => replacement, - }); - - const staleObservation = computer.observeWindow({ pid: 42, windowId: 7 }); - await new Promise((resolve) => setImmediate(resolve)); - await computer.listApps(); - resolveStaleObservation( - toolResult({ - structured: { - tree_markdown: "STALE", - observation_revision: { - mode: "diff", - lineage_id: "l_old", - revision_id: "l_old:r2", - base_revision_id: "l_old:r1", - stable_element_ids: true, - }, - }, - }), - ); - await staleObservation; - - await computer.observeWindow({ pid: 42, windowId: 7 }); - const replacementInput = replacement.calls.find( - (entry) => entry.method === "getWindowState", - ).input; - assert.equal(replacementInput.observationRevision.baseRevisionId, undefined); -}); - -test("concurrent expired reads share one replacement session", async () => { - const expiredResult = toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }); - let releaseLateExpiration; - const lateExpiration = new Promise((resolve) => { - releaseLateExpiration = resolve; - }); - let expiredCalls = 0; - const expired = fakeDriver({ - results: { - listApps: () => { - expiredCalls += 1; - return expiredCalls === 3 ? lateExpiration : expiredResult; - }, - }, - }); - expired.closeCalls = 0; - expired.close = function () { - this.closeCalls += 1; - }; - const replacement = fakeDriver({ - results: { - listApps: toolResult({ - structured: { apps: [{ pid: 42, name: "Harness" }] }, - }), - }, - }); - let resolveFactory; - const replacementPromise = new Promise((resolve) => { - resolveFactory = resolve; - }); - let factoryCalls = 0; - const computer = new ComputerUse(expired, { - sdk: fakeSdk, - ownsSession: true, - sessionFactory: () => { - factoryCalls += 1; - return replacementPromise; - }, - }); - - const first = computer.listApps(); - const second = computer.listApps(); - const late = computer.listApps(); - await new Promise((resolve) => setImmediate(resolve)); - resolveFactory(replacement); - const firstResults = await Promise.all([first, second]); - releaseLateExpiration(expiredResult); - const lateResult = await late; - - assert.deepEqual( - [...firstResults, lateResult].map((apps) => apps[0].displayName), - ["Harness", "Harness", "Harness"], - ); - assert.equal(factoryCalls, 1); - assert.equal(expired.closeCalls, 1); - assert.equal(expired.destroyCalls, 1); - assert.equal(computer.connectionGeneration, 2); -}); - -test("automatic reconnect drains async teardown and binding before redispatch", async () => { - const expired = fakeDriver({ - results: { - listApps: toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }), - }, - }); - let syncCloseCalls = 0; - let asyncCloseOptions; - expired.close = () => { - syncCloseCalls += 1; - throw new Error("synchronous close must not run during reconnect"); - }; - expired.closeAsync = async (options) => { - asyncCloseOptions = options; - }; - const replacement = fakeDriver({ - results: { - listApps: toolResult({ structured: { apps: [{ pid: 42 }] } }), - }, - }); - let factoryCalls = 0; - const computer = new ComputerUse(expired, { - sdk: fakeSdk, - ownsSession: true, - sessionFactory: async () => { - factoryCalls += 1; - return replacement; - }, - }); - - assert.deepEqual(await computer.listApps(), [{ id: "unknown", displayName: "", isRunning: false }]); - assert.equal(syncCloseCalls, 0); - assert.equal(asyncCloseOptions, undefined); - assert.equal(factoryCalls, 1); - assert.equal(replacement.calls.length, 1); -}); - -test("caller cancellation waits for replacement binding then prevents redispatch", async () => { - const expired = fakeDriver({ - results: { - listApps: toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }), - }, - }); - expired.close = () => { - throw new Error("synchronous close must not run during reconnect"); - }; - expired.closeAsync = async () => {}; - const replacement = fakeDriver(); - let finishFactory; - const computer = new ComputerUse(expired, { - sdk: fakeSdk, - ownsSession: true, - sessionFactory: () => - new Promise((resolve) => { - finishFactory = () => resolve(replacement); - }), - }); - const controller = new AbortController(); - let settled = false; - const read = computer.listApps({ signal: controller.signal }).finally(() => { - settled = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - - controller.abort(new Error("stop reconnect")); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(settled, false); - finishFactory(); - await assert.rejects( - read, - (error) => error instanceof ComputerUseError && error.code === "call_cancelled", - ); - assert.equal(replacement.calls.length, 0); - assert.equal(computer.connectionGeneration, 2); -}); - -test("a later call retries session creation after automatic reconnect fails", async () => { - const expired = fakeDriver({ - results: { - listApps: toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }), - }, - }); - expired.close = () => {}; - const replacement = fakeDriver({ - results: { - listApps: toolResult({ - structured: { apps: [{ pid: 42, name: "Harness" }] }, - }), - }, - }); - let factoryCalls = 0; - const computer = new ComputerUse(expired, { - sdk: fakeSdk, - ownsSession: true, - sessionFactory: () => { - factoryCalls += 1; - if (factoryCalls === 1) throw new Error("transient bind failure"); - return replacement; - }, - }); - - await assert.rejects( - computer.listApps(), - (error) => error instanceof ComputerUseError && error.code === "reconnect_failed", - ); - assert.deepEqual(await computer.listApps(), [{ id: "Harness", displayName: "Harness", isRunning: false }]); - assert.equal(factoryCalls, 2); - assert.equal(computer.connectionGeneration, 2); -}); - -test("close during reconnect destroys the unused replacement", async () => { - const expired = fakeDriver({ - results: { - listApps: toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }), - }, - }); - expired.closeCalls = 0; - expired.close = function () { - this.closeCalls += 1; - }; - const replacement = fakeDriver(); - replacement.closeCalls = 0; - replacement.close = function () { - this.closeCalls += 1; - }; - let resolveFactory; - const computer = new ComputerUse(expired, { - owner: fakeDriver(), - sdk: fakeSdk, - ownsSession: true, - publicSession: "persistent-session", - sessionFactory: () => - new Promise((resolve) => { - resolveFactory = resolve; - }), - }); - - const read = computer.listApps(); - await new Promise((resolve) => setImmediate(resolve)); - await computer.close(); - resolveFactory(replacement); - - await assert.rejects(read, /closed/); - assert.equal(replacement.closeCalls, 1); - assert.equal(replacement.destroyCalls, 1); - assert.equal(computer.connectionGeneration, 1); -}); - -test("an expired state-changing action is never replayed automatically", async () => { - const driver = fakeDriver({ - results: { - windowClick: toolResult({ - text: "authorization context expired", - structured: { - status: "refused", - refusal: { code: "authorization_context_expired" }, - }, - isError: true, - }), - }, - }); - let factoryCalls = 0; - const computer = new ComputerUse(driver, { - sdk: fakeSdk, - ownsSession: true, - sessionFactory: () => { - factoryCalls += 1; - return fakeDriver(); - }, - }); - await assert.rejects( - computer.click({ pid: 42, windowId: 7, x: 1, y: 2 }), - (error) => error instanceof ComputerUseError && error.code === "authorization_context_expired", - ); - assert.equal(factoryCalls, 0); - assert.equal(driver.calls.length, 1); -}); - -test("local validation rejects ambiguous or malformed targets before dispatch", async () => { - const driver = fakeDriver(); - const computer = new ComputerUse(driver, { sdk: fakeSdk }); - await assert.rejects(computer.observeWindow({ pid: 0, windowId: 7 })); - await assert.rejects(computer.click({ pid: 42, windowId: 7, elementToken: "token", x: 1, y: 2 })); - await assert.rejects(computer.click({ pid: 42, x: 1, y: 2 })); - await assert.rejects(computer.scroll({ pid: 42, windowId: 7 })); - await assert.rejects(computer.setValue({ pid: 42, value: "x" })); - await assert.rejects(computer.typeText({ pid: 42, text: "x" })); - await assert.rejects(computer.pressKey({ pid: 42, key: "Enter" })); - await assert.rejects(computer.hotkey({ pid: 42, windowId: 7, keys: ["cmd"] })); - await assert.rejects( - computer.drag({ - pid: 42, - windowId: 7, - fromX: 1, - fromY: 2, - toX: 3, - toY: 4, - deliveryMode: "automatic", - }), - ); - assert.equal(driver.calls.length, 0); -}); - -test("closeAsync is awaited without a detachable cancellation signal", async () => { - const session = fakeDriver(); - session.closeAsyncCalls = 0; - session.closeAsync = async function (options) { - assert.equal(options, undefined); - this.closeAsyncCalls += 1; - }; - const owner = fakeDriver(); - const computer = new ComputerUse(session, { - owner, - sdk: fakeSdk, - ownsSession: true, - publicSession: "async-close-session", - }); - - await computer.close(); - assert.equal(session.closeAsyncCalls, 1); - assert.equal(session.destroyCalls, 1); - assert.equal(owner.shutdownCalls, 1); -}); - -test("close is idempotent, releases owned handles, and blocks later calls", async () => { - const session = fakeDriver(); - session.closeCalls = 0; - session.close = function () { - this.closeCalls += 1; - }; - const owner = fakeDriver(); - const computer = new ComputerUse(session, { - owner, - sdk: fakeSdk, - ownsSession: true, - publicSession: "owned-session", - }); - await computer.close(); - await computer.close(); - assert.deepEqual(session.endSessionCalls, [{ session: "owned-session" }]); - assert.equal(session.closeCalls, 1); - assert.equal(session.destroyCalls, 1); - assert.equal(owner.shutdownCalls, 1); - assert.equal(owner.destroyCalls, 1); - await assert.rejects(computer.listApps(), (error) => /closed/.test(error.message)); -}); - -test("close still tears down every owned handle when endSession fails", async () => { - const session = fakeDriver(); - session.endSession = async () => { - throw new Error("end-session-failed"); - }; - session.closeCalls = 0; - session.close = function () { - this.closeCalls += 1; - }; - const owner = fakeDriver(); - const computer = new ComputerUse(session, { - owner, - sdk: fakeSdk, - ownsSession: true, - publicSession: "owned-session", - }); - - await assert.rejects(computer.close(), /end-session-failed/); - assert.equal(session.closeCalls, 1); - assert.equal(session.destroyCalls, 1); - assert.equal(owner.shutdownCalls, 1); - assert.equal(owner.destroyCalls, 1); - await assert.rejects(computer.listApps(), /closed/); + assert.deepEqual(calls, ['end', 'close', 'session destroy', 'shutdown', 'owner destroy']); + await assert.rejects(computer.getState(), /closed/); }); diff --git a/packages/cua-driver/typescript/computer-use/test/daemon-integration.test.mjs b/packages/cua-driver/typescript/computer-use/test/daemon-integration.test.mjs index ff8acedb3b2..1a9a4643030 100644 --- a/packages/cua-driver/typescript/computer-use/test/daemon-integration.test.mjs +++ b/packages/cua-driver/typescript/computer-use/test/daemon-integration.test.mjs @@ -19,17 +19,13 @@ import { test } from "node:test"; import { ComputerUse } from "../index.js"; const socketPath = process.env.COMPUTER_USE_SOCKET; -const pid = Number(process.env.COMPUTER_USE_PID ?? ""); -const windowId = Number(process.env.COMPUTER_USE_WINDOW ?? ""); -const configured = - Number.isInteger(pid) && pid > 0 && Number.isInteger(windowId) && windowId > 0; test("default authorization follows the ComputerUse owner lifetime", async () => { const computer = await ComputerUse.create({ session: `computer-use-owner-lifetime-${process.pid}`, }); try { - await computer.listApps(); + await computer.getState(); const session = await computer.sessionInfo(); assert.ok( session.expiresInSeconds > 24n * 60n * 60n, @@ -47,7 +43,7 @@ test("finite authorization remains an explicit opt-in", async () => { idleTtlSeconds: 30, }); try { - await computer.listApps(); + await computer.getState(); const session = await computer.sessionInfo(); assert.ok(session.expiresInSeconds <= 30n); } finally { @@ -55,42 +51,16 @@ test("finite authorization remains an explicit opt-in", async () => { } }); -test( - "wrapper drives revision v1 against a live native target", - { skip: !configured && "set COMPUTER_USE_PID/WINDOW to run" }, - async () => { - const options = { session: "computer-use-integration" }; - const computer = socketPath - ? await ComputerUse.connect({ ...options, socketPath }) - : await ComputerUse.create(options); - try { - assert.equal(await computer.supportsObservationRevision(), true); - - const first = await computer.observeWindow({ pid, windowId }); - assert.equal(first.mode, "full"); - assert.equal(first.diagnostics.revisionSupported, true); - assert.ok(first.text.length > 0); - - const second = await computer.observeWindow({ pid, windowId }); - assert.equal(second.diagnostics.revisionSupported, true); - assert.ok( - ["no_change", "diff"].includes(second.mode), - `unexpected mode ${second.mode}`, - ); - assert.ok( - second.diagnostics.selectedBytes < second.diagnostics.fullBytes, - "a validated diff/no_change payload must be smaller than the current full tree", - ); - - const forced = await computer.observeWindow({ - pid, - windowId, - disableDiff: true, - }); - assert.equal(forced.mode, "full"); - assert.equal(forced.resyncReason, "requested"); - } finally { - await computer.close(); - } - }, -); +test('runtime observes a live native app without SDK window resolution', { skip: !process.env.COMPUTER_USE_APP && 'set COMPUTER_USE_APP to run' }, async () => { + const computer = socketPath ? await ComputerUse.connect({ socketPath }) : await ComputerUse.create(); + try { + const { app, initial } = await computer.getApp(process.env.COMPUTER_USE_APP); + assert.equal(initial.structured.mode, 'full'); + assert.equal(initial.images.length, 0); + const second = await app.call('get_ax_state'); + assert.ok(['diff','no_change','full'].includes(second.structured.mode)); + const full = await app.call('get_ax_state', { force_full: true }); + assert.equal(full.structured.mode, 'full'); + assert.equal(full.structured.elements, undefined); + } finally { await computer.close(); } +}); diff --git a/packages/cua-driver/typescript/computer-use/test/platform.test.mjs b/packages/cua-driver/typescript/computer-use/test/platform.test.mjs index fbf48014a45..c2c525a5249 100644 --- a/packages/cua-driver/typescript/computer-use/test/platform.test.mjs +++ b/packages/cua-driver/typescript/computer-use/test/platform.test.mjs @@ -5,7 +5,7 @@ import { ComputerUse, ComputerUseError } from "../index.js"; function fixture(listToolsJson) { const owner = { listToolsJson }; const session = { - getWindowState() { throw new Error("platform lookup must not observe the desktop"); }, + callTool() { throw new Error("platform lookup must not observe the desktop"); }, listToolsJson() { throw new Error("platform lookup must use the owner"); }, }; return { computer: new ComputerUse(session, { owner }), owner }; diff --git a/packages/cua-driver/typescript/computer-use/test/runtime.test.mjs b/packages/cua-driver/typescript/computer-use/test/runtime.test.mjs new file mode 100644 index 00000000000..68a8431154b --- /dev/null +++ b/packages/cua-driver/typescript/computer-use/test/runtime.test.mjs @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { ComputerUse } from '../index.js'; +import { createCua } from '../runtime.js'; +import { documentation } from '../documentation.js'; +function fixture() { + const calls = []; const written = []; const images = []; + const computer = new ComputerUse({ callTool(name, raw) { + const args = JSON.parse(raw); calls.push([name, args]); + return { text: name.startsWith('get_') ? 'driver-rendered state [1]' : 'Sent but not confirmed.', + structuredJson: JSON.stringify({ app: 'org.fixture', name: 'Fixture' }), + images: name.includes('screenshot') ? [{ mimeType: 'image/png', dataBase64: 'AQ==' }] : [], + }; + } }); + return { cua: createCua({ write: (x) => written.push(x), emitImage: (x) => images.push(x) }, { computer }), calls, written, images }; +} +test('getApp emits its one native full AX result; observations forward driver text unchanged', async () => { + const f = fixture(); const app = await f.cua.getApp('Fixture'); + assert.deepEqual(f.calls, [['get_app', { app: 'Fixture' }]]); + assert.deepEqual(f.written, ['driver-rendered state [1]']); assert.equal(f.images.length, 0); + assert.equal(await app.getAXState({ forceFull: true, maxTextChars: 20000, emit: false }), 'driver-rendered state [1]'); + assert.deepEqual(f.calls.at(-1), ['get_ax_state', { app: 'org.fixture', force_full: true, max_text_chars: 20000 }]); + assert.equal(f.written.length, 1); +}); +test('screenshot-only is a distinct native operation and actual MIME is preserved', async () => { + const f = fixture(); const app = await f.cua.getApp('Fixture'); + await app.getScreenshot(); + assert.equal(f.calls.at(-1)[0], 'get_screenshot'); + assert.deepEqual(f.images, ['data:image/png;base64,AQ==']); + await app.getAXStateAndScreenshot({ emit: false }); assert.equal(f.images.length, 1); +}); +test('actions return void, preserve advisories and send once with click modifiers', async () => { + const f = fixture(); const app = await f.cua.getApp('Fixture'); + assert.equal(await app.click(1, { key: ['shift'] }), undefined); + assert.deepEqual(f.calls.at(-1), ['click', { app: 'org.fixture', element: 1, key: ['shift'] }]); + assert.equal(f.written.at(-1), 'Sent but not confirmed.'); + assert.equal(f.calls.length, 2); +}); +test('documentation is platform-specific and includes generated tool descriptions', () => { + assert.match(documentation('linux'), /only X11/); + assert.match(documentation('darwin'), /macOS/); + assert.match(documentation('darwin'), /getAXStateAndScreenshot/); + assert.match(documentation('darwin'), /cua\.desktop\.keyDown/); + assert.doesNotMatch(documentation('darwin'), /app\.desktop\./); +}); + +test('desktop calls use explicit scope, serialize, return void and dispose the owning session', async () => { + const f = fixture(); + await f.cua.desktop.getScreenshot(); + assert.deepEqual(f.calls[0], ['get_screenshot', { scope: 'desktop' }]); + assert.equal(await f.cua.desktop.click({ x: 15, y: 32 }, { key: 'ctrl' }), undefined); + assert.deepEqual(f.calls[1], ['click', { key: 'ctrl', x: 15, y: 32, scope: 'desktop' }]); + await f.cua.desktop.keyDown('shift'); + await f.cua.desktop.keyUp('shift'); + assert.deepEqual(f.calls.slice(-2), [['key_down', { key: 'shift', scope: 'desktop' }], ['key_up', { key: 'shift', scope: 'desktop' }]]); + await f.cua[Symbol.asyncDispose](); + await assert.rejects(f.cua.desktop.keyDown('shift'), /closed/); +}); + +test('browser facade returns repository handles unchanged, requires unique URLs, and cleans up', async () => { + const written = []; const navigated = []; let closed = 0; + const tab = { id: 'tab-1', playwright: { domSnapshot: async () => 'DOM' }, goto: async (url) => navigated.push(url) }; + const browser = { browserId: 'chrome-1', documentation: async () => 'BROWSER DOC', tabs: { + list: async () => [{ id: 'tab-1', url: 'https://example.com' }, { id: 'tab-2', url: 'https://example.com' }], + get: async (id) => { assert.equal(id, 'tab-1'); return tab; }, new: async () => tab, + } }; + const browserModule = { setupBrowserRuntime: async () => ({ browsers: { + list: async () => [{ id: 'chrome-1', name: 'Chrome' }], get: async () => browser, + } }), closeBrowserRuntime: async () => { closed++; } }; + const cua = createCua({ write: (value) => written.push(value) }, { browserModule }); + assert.equal(await cua.getBrowser('Chrome'), browser); + assert.equal(await cua.getBrowser({ id: 'chrome-1' }), browser); + assert.equal(written.filter((text) => text === 'BROWSER DOC').length, 1); + await assert.rejects(cua.getTab({ url: 'https://example.com' }), /found 2/); + assert.equal(await cua.getTab('tab-1', { browser: 'Chrome' }), tab); + assert.equal(written.at(-1), 'DOM'); + assert.equal(await cua.createBrowserTab('Chrome', 'https://example.org'), tab); + assert.deepEqual(navigated, ['https://example.org']); + await cua[Symbol.asyncDispose](); assert.equal(closed, 1); + await assert.rejects(cua.getBrowser('Chrome'), /closed/); +}); + +test('disposing during a lazy browser import prevents a late browser connection', async () => { + let resolveModule; let connected = 0; + const pending = new Promise((resolve) => { resolveModule = resolve; }); + const cua = createCua({ write() {} }, { browserModule: pending }); + const selection = cua.getBrowser('Chrome'); + await cua[Symbol.asyncDispose](); + resolveModule({ setupBrowserRuntime() { connected++; throw new Error('must not connect'); } }); + await assert.rejects(selection, /closed/); + assert.equal(connected, 0); +}); diff --git a/packages/cua-driver/typescript/computer-use/test/text-operations.test.mjs b/packages/cua-driver/typescript/computer-use/test/text-operations.test.mjs deleted file mode 100644 index 1e8384b066a..00000000000 --- a/packages/cua-driver/typescript/computer-use/test/text-operations.test.mjs +++ /dev/null @@ -1,224 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { ComputerUse } from "../index.js"; - -const target = { pid: 42, windowId: 7 }; -const elementTarget = { ...target, elementToken: "rv1:window7:retained", text: " target_😀 " }; -const numericSdk = { - PasteFormat: { Text: 0, Md: 1, Html: 2 }, - TextSelection: { Text: 0, CursorBefore: 1, CursorAfter: 2 }, -}; - -function result(structured, extra = {}) { - return { text: "", structuredJson: JSON.stringify(structured), images: [], isError: false, ...extra }; -} - -function fixture({ sdk = {}, platform = "macos", action, reconnectable = false } = {}) { - const calls = []; - const apps = [{ name: "Text Fixture", bundle_id: "org.example.text-fixture", launch_path: "/Applications/TextFixture.app", pid: 42, running: true }]; - const windows = [{ window_id: 7, title: "Owned document", is_app_target: true }]; - let revision = 0; - let reconnects = 0; - const owner = { - async listToolsJson() { - calls.push({ method: "listToolsJson" }); - return JSON.stringify({ platform, tools: [{ name: "get_window_state", capabilities: ["accessibility.observation_revision.v1"] }] }); - }, - }; - const driver = { - async listApps(input) { calls.push({ method: "listApps", input }); return result({ apps }); }, - async listWindows(input) { calls.push({ method: "listWindows", input }); return result({ windows }); }, - async getWindowState(input) { - calls.push({ method: "getWindowState", input }); - revision += 1; - return result({ - tree_markdown: '[37] TextArea " target_😀 "', - elements: [{ element_id: 37, element_index: 4, element_token: `rv1:window${input.windowId}:retained`, role: "AXTextArea" }], - observation_revision: { mode: "full", lineage_id: `window${input.windowId}`, revision_id: `r${revision}`, stable_element_ids: true, capture_complete: true }, - }); - }, - }; - for (const method of ["paste", "selectText"]) { - driver[method] = async (input) => { - calls.push({ method, input }); - return action ? action(method, input) : result({ effect: "confirmed" }); - }; - } - const computer = new ComputerUse(driver, { - owner, - sdk, - ...(reconnectable ? { ownsSession: true, sessionFactory: () => { reconnects += 1; return driver; } } : {}), - }); - return { - computer, calls, windows, apps, - mutations: () => calls.filter(({ method }) => ["paste", "selectText"].includes(method)), - reconnects: () => reconnects, - }; -} - -for (const sdk of [{}, numericSdk]) { - test(`exact-window text methods preserve content and serialize ${sdk === numericSdk ? "numeric" : "fallback"} enums`, async () => { - const { computer, mutations } = fixture({ sdk }); - for (const [index, format] of ["text", "md", "html"].entries()) { - const output = await computer.paste({ ...target, text: " 前_😀\n后 ", format }); - assert.equal(output.effect, "confirmed"); - assert.deepEqual(mutations().at(-1).input, { - pid: 42, windowId: 7n, text: " 前_😀\n后 ", format: sdk === numericSdk ? index : format, - }); - } - for (const [index, selection] of ["text", "cursor_before", "cursor_after"].entries()) { - await computer.selectText({ ...elementTarget, prefix: "前 ", suffix: " 后", selection }); - assert.deepEqual(mutations().at(-1).input, { - pid: 42, windowId: 7n, elementToken: elementTarget.elementToken, - text: elementTarget.text, prefix: "前 ", suffix: " 后", - selection: sdk === numericSdk ? index : selection, - }); - } - await computer.paste({ ...target, text: "" }); - assert.equal(mutations().at(-1).input.format, sdk === numericSdk ? 0 : "text"); - await computer.selectText({ ...elementTarget, prefix: "", suffix: "" }); - assert.equal(mutations().at(-1).input.selection, sdk === numericSdk ? 0 : "text"); - assert.equal(mutations().at(-1).input.prefix, ""); - assert.equal(mutations().at(-1).input.suffix, ""); - }); -} - -test("app text methods bind observed IDs and route paste through app context", async () => { - const { computer, mutations } = fixture({ sdk: numericSdk }); - const app = await computer.getApp("Text Fixture"); - await app.getState(); - assert.deepEqual(await app.selectText(37, " target_😀 ", { prefix: "前", suffix: "后", selection: "cursor_after" }), { effect: "confirmed" }); - await app.paste("<b>new</b>", { format: "html" }); - assert.deepEqual(mutations().map(({ input }) => input), [ - { pid: 42, windowId: 7n, elementToken: "rv1:window7:retained", text: " target_😀 ", prefix: "前", suffix: "后", selection: 2 }, - { pid: 42, windowId: 7n, text: "<b>new</b>", format: 2, appContext: true }, - ]); -}); - -test("selectText refuses a stale app window before dispatch and accepts the refreshed token", async () => { - const { computer, windows, mutations } = fixture(); - const app = await computer.getApp("Text Fixture"); - await app.getState(); - windows[0] = { window_id: 9, title: "New dialog", is_app_target: true }; - await assert.rejects(app.selectText(37, "target"), { code: "app_observation_required" }); - assert.equal(mutations().length, 0); - await app.getState(); - await app.selectText(37, "target"); - assert.equal(mutations()[0].input.windowId, 9n); - assert.equal(mutations()[0].input.elementToken, "rv1:window9:retained"); -}); - -test("selectText invalidates observed IDs after the owning process changes", async () => { - const { computer, apps, mutations } = fixture(); - const app = await computer.getApp("Text Fixture"); - await app.getState(); - apps[0].pid = 84; - await assert.rejects(app.selectText(37, "target"), { code: "app_observation_required" }); - assert.equal(mutations().length, 0); -}); - -test("app queue finishes selection before dispatching a concurrent paste", async () => { - const started = Promise.withResolvers(); - const release = Promise.withResolvers(); - const { computer, mutations } = fixture({ - action: async (method) => { - if (method === "selectText") { - started.resolve(); - await release.promise; - } - return result({ effect: "confirmed" }); - }, - }); - const app = await computer.getApp("Text Fixture"); - await app.getState(); - const selection = app.selectText(37, "target"); - const paste = app.paste("replacement"); - await started.promise; - assert.deepEqual(mutations().map(({ method }) => method), ["selectText"]); - release.resolve(); - await Promise.all([selection, paste]); - assert.deepEqual(mutations().map(({ method }) => method), ["selectText", "paste"]); -}); - -test("invalid exact-window text inputs cannot dispatch or inspect platform metadata", async () => { - const { computer, calls } = fixture(); - const invalid = [ - () => computer.paste({ text: "x" }), - () => computer.paste({ ...target, text: 1 }), - () => computer.paste({ ...target, text: "x", format: "rtf" }), - () => computer.paste({ ...target, text: "x", format: ["text"] }), - () => computer.selectText({ ...target, text: "x" }), - () => computer.selectText({ ...elementTarget, windowId: undefined }), - () => computer.selectText({ ...elementTarget, text: "" }), - () => computer.selectText({ ...elementTarget, text: 1 }), - () => computer.selectText({ ...elementTarget, prefix: 1 }), - () => computer.selectText({ ...elementTarget, suffix: [] }), - () => computer.selectText({ ...elementTarget, selection: "all" }), - () => computer.selectText({ ...elementTarget, selection: ["text"] }), - () => computer.selectText({ ...elementTarget, x: 2, y: 3 }), - ]; - for (const operation of invalid) { - await assert.rejects(operation); - } - assert.deepEqual(calls, []); -}); - -test("app text methods reject managed targeting/routing options and unavailable IDs", async () => { - const { computer, mutations } = fixture(); - const app = await computer.getApp("Text Fixture"); - await assert.rejects(app.selectText(37, "target"), { code: "app_observation_required" }); - await app.getState(); - for (const key of ["pid", "windowId", "elementToken", "deliveryMode", "delivery_mode", "foreground", "background", "appContext"]) { - assert.throws(() => app.paste("x", { [key]: "not accepted" }), { code: "app_option_managed" }); - assert.throws(() => app.selectText(37, "target", { [key]: "not accepted" }), { code: "app_option_managed" }); - } - await assert.rejects(app.selectText(999, "target"), { code: "app_element_unavailable" }); - await assert.rejects(app.selectText({ x: 1, y: 2 }, "target"), { code: "app_element_required" }); - assert.equal(mutations().length, 0); -}); - -for (const platform of ["windows", "linux"]) { - test(`text methods refuse a ${platform} driver without native mutation`, async () => { - const { computer, mutations, calls } = fixture({ platform }); - await assert.rejects(computer.paste({ ...target, text: "x" }), { code: "unsupported_platform" }); - await assert.rejects(computer.selectText(elementTarget), { code: "unsupported_platform" }); - assert.equal(mutations().length, 0); - assert.equal(calls.filter(({ method }) => method === "getWindowState").length, 0); - }); -} - -for (const method of ["paste", "selectText"]) { - test(`failed ${method} mutations never reconnect or replay`, async () => { - const { computer, mutations, reconnects } = fixture({ - reconnectable: true, - action: () => result({}, { isError: true, errorCode: "session_unavailable", text: "could not confirm the text operation" }), - }); - const app = await computer.getApp("Text Fixture"); - await app.getState(); - const operation = method === "paste" ? app.paste("new") : app.selectText(37, "target"); - await assert.rejects(operation, { code: "session_unavailable" }); - assert.equal(mutations().length, 1); - assert.equal(reconnects(), 0); - await app.getState(); - assert.equal(mutations().length, 1); - }); -} - -test("unconfirmed text effects stay visible without retrying the mutation", async () => { - const { computer, mutations } = fixture({ - action: () => result({}, { action: { effect: 2 } }), - }); - const app = await computer.getApp("Text Fixture"); - await app.getState(); - assert.deepEqual(await app.selectText(37, "target"), { effect: "unverifiable" }); - assert.deepEqual(await app.paste("new"), { effect: "unverifiable" }); - assert.equal(mutations().length, 2); -}); - -test("pre-cancelled text operations dispatch no platform query or mutation", async () => { - const { computer, calls } = fixture(); - const signal = AbortSignal.abort(); - await assert.rejects(computer.paste({ ...target, text: "new", signal }), { code: "call_cancelled" }); - await assert.rejects(computer.selectText({ ...elementTarget, signal }), { code: "call_cancelled" }); - assert.deepEqual(calls, []); -}); diff --git a/packages/cua-driver/typescript/computer-use/test/types.test.ts b/packages/cua-driver/typescript/computer-use/test/types.test.ts index ef4a6b2345d..aa529db7d18 100644 --- a/packages/cua-driver/typescript/computer-use/test/types.test.ts +++ b/packages/cua-driver/typescript/computer-use/test/types.test.ts @@ -1,134 +1,20 @@ -import type { - ActAndVerifyResult, - ComputerUse, - ComputerUseActionResult, - ComputerUseVerificationResult, - ObserveWindowOptions, -} from "../index.js"; - -const invalidObservationOptions: ObserveWindowOptions = { - pid: 42, - windowId: 7, - // @ts-expect-error revision cursors are owned by ComputerUse - baseRevisionId: "l_manual:r1", -}; -void invalidObservationOptions; - -const invalidRevisionId: ObserveWindowOptions = { - pid: 42, - windowId: 7, - // @ts-expect-error revision cursors are owned by ComputerUse - revisionId: "l_manual:r2", -}; -void invalidRevisionId; - -const invalidObservationRevision: ObserveWindowOptions = { - pid: 42, - windowId: 7, - // @ts-expect-error the native revision request is internal to ComputerUse - observationRevision: { baseRevisionId: "l_manual:r1" }, -}; -void invalidObservationRevision; - -const invalidLineageId: ObserveWindowOptions = { - pid: 42, - windowId: 7, - // @ts-expect-error lineage state is owned by ComputerUse - lineageId: "l_manual", -}; -void invalidLineageId; - -export async function exerciseComputerUseTypes( - computer: ComputerUse, - signal: AbortSignal, -): Promise<ActAndVerifyResult> { - await computer.listApps({ signal }); - const app = await computer.getApp("org.example.fixture", { signal }); - const appState = await app.getState({ includeScreenshot: true, maxTextChars: 12_000, signal }); - appState.screenshot?.images.at(0)?.dataBase64; - await app.click(37, { signal }); - await app.setValue(37, "draft"); - await app.pressKey("Return", { modifiers: ["shift"] }); - await app.hotkey(["super", "s"]); - await app.scroll(37, { direction: "down" }); - const platform: "macos" | "windows" | "linux" = await computer.getPlatform({ signal }); - void platform; - await app.paste("plain"); - await app.paste("**formatted**", { format: "md", signal }); - await app.paste("<b>formatted</b>", { format: "html" }); - await app.selectText(37, "draft", { prefix: "Status: ", suffix: ".", signal }); - await app.selectText(37, "draft", { selection: "cursor_before" }); - await app.selectText(37, "draft", { selection: "cursor_after" }); - const textAction: ComputerUseActionResult = await computer.paste({ pid: 42, windowId: 7, text: "new", format: "text", signal }); - void textAction; - await computer.selectText({ pid: 42, windowId: 7, elementToken: "rv1:l_a:1", text: "draft", selection: "text", signal }); - // @ts-expect-error only the documented clipboard formats are accepted - await app.paste("text", { format: "rtf" }); - // @ts-expect-error app targeting stays internal - await app.paste("text", { pid: 42 }); - // @ts-expect-error app text operations do not expose delivery routing - await app.selectText(37, "draft", { deliveryMode: "foreground" }); - // @ts-expect-error public text selection uses selection, not native naming - await app.selectText(37, "draft", { selection_type: "text" }); - // @ts-expect-error selection requires a current short numeric element ID - await app.selectText("rv1:l_a:1", "draft"); - // @ts-expect-error the exact-window operation requires a window ID - await computer.selectText({ pid: 42, elementToken: "rv1:l_a:1", text: "draft" }); - // @ts-expect-error paste uses a native route without a delivery option - await computer.paste({ pid: 42, windowId: 7, text: "new", deliveryMode: "foreground" }); - // @ts-expect-error app routing is internal - await app.click(37, { deliveryMode: "foreground" }); - // @ts-expect-error app observations do not expose native identities - appState.pid; - // @ts-expect-error opaque tokens are not app action targets - await app.click({ elementToken: "private" }); - const observation = await computer.observeWindow({ - pid: 42, - windowId: 7, - disableDiff: false, - includeScreenshot: true, - maxTextChars: 12_000, - signal, - }); - observation.screenshot?.images.at(0)?.dataBase64; - observation.elements.at(0)?.automation_id; - observation.diagnostics.selectedBytes; - observation.diagnostics.captureComplete; - observation.diagnostics.captureReadComplete; - observation.diagnostics.captureTruncated; - observation.diagnostics.captureIncompleteDetails.at(0); - observation.diagnostics.textTruncated; - observation.diagnostics.textChars; - // @ts-expect-error revision identifiers are not public observation fields - observation.revisionId; - // @ts-expect-error base revision identifiers are not public observation fields - observation.baseRevisionId; - // @ts-expect-error lineage identifiers are not public observation fields - observation.lineageId; - // @ts-expect-error raw native payloads are not public observation fields - observation.structured; - const result = await computer.actAndVerify({ - action: () => - computer.click({ - pid: 42, - windowId: 7, - x: 10, - y: 20, - deliveryMode: "foreground", - signal, - }), - verify: (_action: ComputerUseActionResult) => - computer.verifyState({ - pid: 42, - windowId: 7, - expect: [{ element: { token: "rv1:l_a:1", selected: true } }], - signal, - }), - }); - const verification: ComputerUseVerificationResult = result.verification; - verification.verification?.predicates.at(0); - result.action.action?.evidence?.at(0); - (await computer.sessionInfo()).expiresInSeconds; - computer.connectionGeneration; - return result; +import { createCua, type RuntimeOutput } from '../runtime.js'; +import { ComputerUse } from '../index.js'; +async function useRuntime(output: RuntimeOutput) { + const computer = await ComputerUse.create(); + const cua = createCua(output, { computer }); + const app = await cua.getApp('Editor'); + const text: string = await app.getAXState({ forceFull: true, maxTextChars: 20000, emit: false }); + const action: void = await app.click(1, { key: ['shift'] }); + await app.setValue(2, text); + await app.typeText('你好'); + await app.getScreenshot(); + await app.drag({ fromX: 1, fromY: 2, toX: 3, toY: 4 }); + // @ts-expect-error Native targeting belongs to the driver. + await app.click(1, { pid: 1 }); + // @ts-expect-error App state is rendered text. + text.elements; + await computer.close(); + return action; } +void useRuntime; diff --git a/packages/cua-driver/typescript/package-lock.json b/packages/cua-driver/typescript/package-lock.json deleted file mode 100644 index b51e1bb8a8b..00000000000 --- a/packages/cua-driver/typescript/package-lock.json +++ /dev/null @@ -1,301 +0,0 @@ -{ - "name": "@qwen-code/cua-sdk", - "version": "0.20.11", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@qwen-code/cua-sdk", - "version": "0.20.11", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "@ubjs/core": "0.31.0-3", - "tar": "7.5.22" - }, - "devDependencies": { - "@types/node": "^22.0.0", - "electron": "43.2.0", - "typescript": "^5.7.0", - "uniffi-bindgen-react-native": "0.31.0-3" - }, - "engines": { - "node": ">=22" - } - }, - "node_modules/@electron-internal/extract-zip": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@electron-internal/extract-zip/-/extract-zip-1.0.4.tgz", - "integrity": "sha512-Zr1Vs7E9tpCNhZHDAbFVXc2gEVCG9RqPDjrno5+bdgB6LRAuvgyMHJut4NCVyYwtAieapMzc3fiQ3CSTi75ARg==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=22.12.0" - } - }, - "node_modules/@electron/get": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@electron/get/-/get-5.0.0.tgz", - "integrity": "sha512-pjoBpru1KdEtcExBnuHAP1cAc/5faoedw0hzJkL3o4/IJp7HNF1+fbrdxT3gMYRX2oJfvnA/WXeCTVQpYYxyJA==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.1.1", - "env-paths": "^3.0.0", - "graceful-fs": "^4.2.11", - "progress": "^2.0.3", - "semver": "^7.6.3", - "sumchecker": "^3.0.1" - }, - "engines": { - "node": ">=22.12.0" - }, - "optionalDependencies": { - "undici": "^7.24.4" - } - }, - "node_modules/@isaacs/fs-minipass": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", - "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", - "license": "ISC", - "dependencies": { - "minipass": "^7.0.4" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@types/node": { - "version": "22.20.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", - "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/@ubjs/core": { - "version": "0.31.0-3", - "resolved": "https://registry.npmjs.org/@ubjs/core/-/core-0.31.0-3.tgz", - "integrity": "sha512-39XrJgUZ2VVb561sSnkXPhczNoeBsNiSRArecsV0JE7CJq69ajFkcn9/tBAUS2NpgHkLIDU+z6Ks2+1wXnboxg==", - "license": "MPL-2.0" - }, - "node_modules/chownr": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", - "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/electron": { - "version": "43.2.0", - "resolved": "https://registry.npmjs.org/electron/-/electron-43.2.0.tgz", - "integrity": "sha512-80zvrgG7ZRXD+tD0IyLvrnN9n+veSxadMRsMaC9wKKP3iUbtC7rGM8+dVuCmOb0Rrwwv8ESW4awnUZh9Hbp1fA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@electron-internal/extract-zip": "^1.0.1", - "@electron/get": "^5.0.0", - "@types/node": "^24.9.0" - }, - "bin": { - "electron": "cli.js", - "install-electron": "install.js" - }, - "engines": { - "node": ">= 22.12.0" - } - }, - "node_modules/electron/node_modules/@types/node": { - "version": "24.13.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", - "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.18.0" - } - }, - "node_modules/electron/node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, - "license": "MIT" - }, - "node_modules/env-paths": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", - "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/minizlib": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", - "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", - "license": "MIT", - "dependencies": { - "minipass": "^7.1.2" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sumchecker": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz", - "integrity": "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "debug": "^4.1.0" - }, - "engines": { - "node": ">= 8.0" - } - }, - "node_modules/tar": { - "version": "7.5.22", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", - "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/fs-minipass": "^4.0.0", - "chownr": "^3.0.0", - "minipass": "^7.1.2", - "minizlib": "^3.1.0", - "yallist": "^5.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/uniffi-bindgen-react-native": { - "version": "0.31.0-3", - "resolved": "https://registry.npmjs.org/uniffi-bindgen-react-native/-/uniffi-bindgen-react-native-0.31.0-3.tgz", - "integrity": "sha512-br7giBJRr/j00rdMXhOZGMGuDWMAVUcxC9O3lco0KRqzEAxUz00+gegPW5tKamvKbrnj7NB682XUCYrcagtxFA==", - "dev": true, - "license": "MPL-2.0", - "bin": { - "ubrn": "bin/cli.cjs", - "uniffi-bindgen-react-native": "bin/cli.cjs" - } - }, - "node_modules/yallist": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", - "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" - } - } - } -} diff --git a/packages/cua-driver/typescript/package.json b/packages/cua-driver/typescript/package.json index c3b54c7db59..5c1ce42d043 100644 --- a/packages/cua-driver/typescript/package.json +++ b/packages/cua-driver/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@qwen-code/cua-sdk", - "version": "0.20.11", + "version": "0.26.0", "description": "Typed CUA Driver SDK and Computer Use API for Node.js", "type": "module", "license": "MIT", @@ -26,6 +26,10 @@ "./computer-use": { "types": "./computer-use/index.d.ts", "import": "./computer-use/index.js" + }, + "./runtime": { + "types": "./computer-use/runtime.d.ts", + "import": "./computer-use/runtime.js" } }, "files": [ @@ -42,11 +46,15 @@ "computer-use/app.js", "computer-use/index.d.ts", "computer-use/README.md", - "computer-use/SKILL.md", "scripts/install-native.mjs", "LICENSE.md", "NOTICE.md", - "README.md" + "README.md", + "computer-use/runtime.js", + "computer-use/runtime.d.ts", + "computer-use/documentation.js", + "computer-use/interface.json", + "computer-use/browser.js" ], "engines": { "node": ">=22" @@ -57,10 +65,10 @@ "scripts": { "build": "tsc -p tsconfig.json", "postinstall": "node scripts/install-native.mjs", - "prepack": "npm run build && node scripts/stage-computer-use-skill.mjs", + "prepack": "npm run build", "typecheck": "tsc -p tsconfig.json --noEmit", "typecheck:computer-use": "tsc -p computer-use/test/tsconfig.json", - "test": "npm run build && node scripts/stage-computer-use-skill.mjs && npm run typecheck:computer-use && node --test test/*.test.mjs computer-use/test/computer-use.test.mjs computer-use/test/platform.test.mjs computer-use/test/text-operations.test.mjs computer-use/test/app.test.mjs computer-use/test/app-lifecycle.test.mjs computer-use/test/app-oriented-baseline.test.mjs", + "test": "npm run build && npm run typecheck:computer-use && node --test test/*.test.mjs computer-use/test/computer-use.test.mjs computer-use/test/platform.test.mjs computer-use/test/runtime.test.mjs", "test:e2e": "node --test computer-use/test/daemon-integration.test.mjs", "generate:uniffi": "node ../scripts/generate-uniffi-bindings.mjs", "generate:uniffi:check": "node ../scripts/generate-uniffi-bindings.mjs --check", @@ -68,12 +76,12 @@ }, "devDependencies": { "@types/node": "^22.0.0", - "electron": "43.2.0", "typescript": "^5.7.0", "uniffi-bindgen-react-native": "0.31.0-3" }, "dependencies": { "@ubjs/core": "0.31.0-3", - "tar": "7.5.22" + "tar": "7.5.22", + "@qwen-code/browser-use": "0.26.0" } } diff --git a/packages/cua-driver/typescript/scripts/install-native.mjs b/packages/cua-driver/typescript/scripts/install-native.mjs index 1dfc44508a8..a5fb62c4d9d 100644 --- a/packages/cua-driver/typescript/scripts/install-native.mjs +++ b/packages/cua-driver/typescript/scripts/install-native.mjs @@ -196,9 +196,11 @@ export async function ensureNativePayload({ } = await import(NATIVE_ASSETS_MODULE.href) version ??= cuaSdkVersion() const target = nativeTarget(platform, arch, version) + const uiAccess = platform === "win32" && env.CUA_DRIVER_USE_UIACCESS === "1" + if (uiAccess) target.companions = ["qwen-cua-driver-uia.exe"] if (env[CUA_SDK_NATIVE_DIR_ENV]) { const directory = resolveNativeDirectory(env, platform, arch, version) - if (platform === "win32") { + if (uiAccess) { await installUiAccessWorker( join(directory, "qwen-cua-driver-uia.exe"), version, @@ -210,7 +212,7 @@ export async function ensureNativePayload({ const destination = cachedNativeDirectory(target, version, env, platform) if (hasCompletedNativePayload(destination, target)) { - if (platform === "win32") { + if (uiAccess) { await installUiAccessWorker( join(destination, "qwen-cua-driver-uia.exe"), version, @@ -265,7 +267,7 @@ export async function ensureNativePayload({ source: archiveResult.url, version, }) - if (platform === "win32") { + if (uiAccess) { await installUiAccessWorker( join(destination, "qwen-cua-driver-uia.exe"), version, diff --git a/packages/cua-driver/typescript/scripts/stage-computer-use-skill.mjs b/packages/cua-driver/typescript/scripts/stage-computer-use-skill.mjs deleted file mode 100644 index 78ad30e6981..00000000000 --- a/packages/cua-driver/typescript/scripts/stage-computer-use-skill.mjs +++ /dev/null @@ -1,38 +0,0 @@ -/** - * @license - * Copyright 2025 Qwen - * SPDX-License-Identifier: Apache-2.0 - */ - -import { copyFileSync, existsSync, mkdirSync, rmSync } from "node:fs" -import path from "node:path" -import { fileURLToPath } from "node:url" - -const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") -const sourceRoot = path.resolve( - packageRoot, - "..", - "..", - "core", - "src", - "skills", - "bundled", - "computer-use", -) -const resources = ["SKILL.md"] - -for (const resource of resources) { - const source = path.join(sourceRoot, resource) - if (!existsSync(source)) { - throw new Error(`canonical Computer Use skill resource not found: ${source}`) - } -} -for (const resource of resources) { - const destination = path.join(packageRoot, "computer-use", resource) - mkdirSync(path.dirname(destination), { recursive: true }) - copyFileSync(path.join(sourceRoot, resource), destination) -} - -for (const legacy of ["macos.md", "windows-linux.md"]) { - rmSync(path.join(packageRoot, "computer-use", "references", legacy), { force: true }) -} diff --git a/packages/cua-driver/typescript/src/native-assets.ts b/packages/cua-driver/typescript/src/native-assets.ts index 8e7c073f05c..e26ffa39d08 100644 --- a/packages/cua-driver/typescript/src/native-assets.ts +++ b/packages/cua-driver/typescript/src/native-assets.ts @@ -72,7 +72,7 @@ export function nativeTarget( return { archive: `cua-driver-rs-${version}-windows-${releaseArch}-binary.zip`, cacheKey: `windows-${releaseArch}`, - companions: ["qwen-cua-driver-uia.exe"], + companions: [], library: "cua_driver_sdk.dll", runtime, } diff --git a/packages/cua-driver/typescript/test/native-assets.test.mjs b/packages/cua-driver/typescript/test/native-assets.test.mjs index 7ec78836eeb..8ada421f8c5 100644 --- a/packages/cua-driver/typescript/test/native-assets.test.mjs +++ b/packages/cua-driver/typescript/test/native-assets.test.mjs @@ -36,9 +36,7 @@ test("release target mapping uses Qwen binary archives", () => { nativeTarget("win32", "arm64", "1.2.3").archive, "cua-driver-rs-1.2.3-windows-arm64-binary.zip", ) - assert.deepEqual(nativeTarget("win32", "x64", "1.2.3").companions, [ - "qwen-cua-driver-uia.exe", - ]) + assert.deepEqual(nativeTarget("win32", "x64", "1.2.3").companions, []) assert.throws(() => nativeTarget("freebsd", "x64", "1.2.3")) }) @@ -213,3 +211,18 @@ test("UIAccess worker uses the versioned secure Windows path", () => { ) assert.throws(() => uiAccessWorkerPath("1.2.3", {}), /ProgramFiles/u) }) + + +test("default Windows native installation needs no UIAccess worker or Program Files", async () => { + const directory = mkdtempSync(join(tmpdir(), "cua-sdk-windows-ordinary-")); + try { + const target = nativeTarget("win32", "x64", "1.2.3"); + writeFileSync(join(directory, target.library), "library fixture"); + writeFileSync(join(directory, target.runtime), "runtime fixture"); + const result = await ensureNativePayload({ + env: { QWEN_CUA_SDK_NATIVE_DIR: directory }, platform: "win32", arch: "x64", version: "1.2.3", + fetchImpl: () => { throw new Error("explicit native payload must not download"); }, + }); + assert.equal(result, directory); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/packages/cua-driver/typescript/test/skill-package.test.mjs b/packages/cua-driver/typescript/test/skill-package.test.mjs deleted file mode 100644 index f03ec453002..00000000000 --- a/packages/cua-driver/typescript/test/skill-package.test.mjs +++ /dev/null @@ -1,68 +0,0 @@ -import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { test } from "node:test"; - -const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const canonicalRoot = path.resolve(packageRoot, "../../core/src/skills/bundled/computer-use"); -const resources = ["SKILL.md"]; - -test("SDK skill is self-contained and npm excludes obsolete platform workflows", () => { - for (const resource of resources) { - assert.equal( - readFileSync(path.join(packageRoot, "computer-use", resource), "utf8"), - readFileSync(path.join(canonicalRoot, resource), "utf8"), - resource, - ); - } - const packed = spawnSync("npm", ["pack", "--dry-run", "--ignore-scripts", "--json"], { - cwd: packageRoot, - encoding: "utf8", - shell: process.platform === "win32", - }); - assert.equal(packed.status, 0, packed.stderr); - const files = new Set(JSON.parse(packed.stdout)[0].files.map(({ path }) => path)); - for (const resource of resources) { - assert.ok(files.has(`computer-use/${resource}`), resource); - } - assert.ok(![...files].some(file => /^computer-use\/references\//.test(file))); -}); - -test("staging refreshes the entrypoint, removes old workflows and fails before copying missing input", () => { - const root = mkdtempSync(path.join(tmpdir(), "cua-skill-stage-")); - try { - const sdk = path.join(root, "packages/cua-driver/typescript"); - const canonical = path.join(root, "packages/core/src/skills/bundled/computer-use"); - mkdirSync(path.join(sdk, "scripts"), { recursive: true }); - mkdirSync(path.join(canonical, "references"), { recursive: true }); - const script = path.join(sdk, "scripts/stage-computer-use-skill.mjs"); - copyFileSync(path.join(packageRoot, "scripts/stage-computer-use-skill.mjs"), script); - for (const resource of resources) { - writeFileSync(path.join(canonical, resource), resource); - } - const stage = () => spawnSync(process.execPath, [script], { encoding: "utf8" }); - assert.equal(stage().status, 0); - for (const resource of resources) { - assert.equal(readFileSync(path.join(sdk, "computer-use", resource), "utf8"), resource); - } - const destination = path.join(sdk, "computer-use/SKILL.md"); - mkdirSync(path.join(sdk, "computer-use/references"), { recursive: true }); - const legacy = path.join(sdk, "computer-use/references/macos.md"); - writeFileSync(legacy, "obsolete"); - writeFileSync(destination, "stale"); - assert.equal(stage().status, 0); - assert.equal(readFileSync(destination, "utf8"), "SKILL.md"); - assert.equal(existsSync(legacy), false); - rmSync(path.join(canonical, "SKILL.md")); - const failed = stage(); - assert.notEqual(failed.status, 0); - assert.match(failed.stderr, /canonical Computer Use skill resource not found/); - assert.equal(readFileSync(path.join(sdk, "computer-use/SKILL.md"), "utf8"), "SKILL.md"); - assert.ok(existsSync(destination)); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); diff --git a/packages/cua-driver/wayland-helper/README.md b/packages/cua-driver/wayland-helper/README.md deleted file mode 100644 index 8e9c3735df4..00000000000 --- a/packages/cua-driver/wayland-helper/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# cua WinRects — GNOME Shell helper extension (Wayland) - -A small GNOME Shell extension that lets cua-driver get **pixel coordinates**, -activate an exact target window, capture the compositor stage, and draw the -**agent cursor** on GNOME Mutter Wayland. A normal Wayland client cannot do -these things globally. - -It exposes `org.cua.WinRects` on the session bus: - -- `GetVersion() -> uint` — a browser-sensitive API version. cua-driver only - accepts it after resolving the immutable D-Bus owner and proving the owner is - the current user's system-installed `gnome-shell` process. -- `GetRects() -> json` — every window's frame geometry and surface-buffer - origin. cua-driver combines the buffer origin with AT-SPI - `CoordType::Window` per-widget coords: `screen = origin + window_xy`. This is - the GNOME analogue of the X11 `_GTK_FRAME_EXTENTS` reconstruction (AT-SPI's - `CoordType::Screen` is `(0,0)` for every widget on Mutter). Keeping the frame - and buffer origins separate accounts for GTK client-side shadows. -- `Activate(id) -> bool` — activate one Shell stable-sequence window and report - whether the request was accepted. cua-driver verifies focus through a second - `GetRects` snapshot before sending focus-bound portal/libei input, preventing - input from leaking into whichever application happened to be focused. -- `Capture() -> png_base64` — capture the compositor stage through Shell's - screenshot API. cua-driver crops it with the same authoritative geometry. -- `MoveCursor(x,y)` / `ClickPulse(x,y)` / `HideCursor()` — position and hide - the agent cursor as a Clutter actor on the compositor stage. -- `SetCursorState(action,delivery,target,active)` — render the same 12 semantic - action states as the cross-platform `cua.default` cursor theme. Delivery and - target context appears as host-owned chips in the session badge rather than - pointer-relative theme artwork. This contract requires helper v8. -- `SetCursorColor(fill_color)` — apply the stable per-session fill selected by - cua-driver. The helper validates the `#RRGGBB` value, updates the matching - glow, and keeps a white pointer outline. - -It runs in the shell's privileged context, so **no xdg-desktop-portal grant** is -needed (unlike libei/RemoteDesktop). - -## Install - -``` -~/.cua-driver/packages/current/wayland-helper/install.sh -# From a source checkout, use ./install.sh in this directory. -# then log out/in once (GNOME loads extensions only at session startup) -gnome-extensions info winrects@cua # -> State: ACTIVE -``` - -cua-driver auto-detects it at runtime (`wayland::shell_helper`). AX operations -still work when it is absent, but pixel geometry, the Shell cursor, and safe -foreground portal input are unavailable. cua-driver refuses focus-bound input -instead of injecting into an unverified target. - -The semantic cursor requires helper v8. When an older helper is still loaded, -cua-driver does not draw its legacy cursor. Re-run the helper installer, then -reload the GNOME session so the new compositor-owned artwork becomes active. - -Browser setup and consent are held to a stricter boundary: helper API v4 or -newer must be served by the verified GNOME Shell owner. The driver addresses -that owner's unique D-Bus name, so another same-session process cannot replace -the public name between verification and an activation request. One exact -target is activated only for the bounded operation, then the previously -focused Shell window is restored and verified. - -wlroots compositors such as Sway and labwc do not need it: cua-driver uses -foreign-toplevel activation, virtual-pointer input, and layer-shell there. - -KDE Plasma Wayland needs an equivalent target-addressable KWin activation -adapter; it is not yet provided. Portal reachability alone is insufficient -because RemoteDesktop/libei input is global to the compositor focus. diff --git a/packages/cua-driver/wayland-helper/install.sh b/packages/cua-driver/wayland-helper/install.sh deleted file mode 100755 index e4a61ff5e4c..00000000000 --- a/packages/cua-driver/wayland-helper/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Install the cua WinRects GNOME Shell extension — supplies window screen -# geometry (for AT-SPI coordinate reconstruction) and renders the agent cursor -# on GNOME Mutter Wayland, where a normal client can do neither. Best-effort: -# cua-driver works without it (just no screen coords / no cursor on Mutter). -set -euo pipefail -UUID="winrects@cua" -SRC="$(cd "$(dirname "$0")" && pwd)/$UUID" -DEST="${XDG_DATA_HOME:-$HOME/.local/share}/gnome-shell/extensions/$UUID" -mkdir -p "$DEST" -cp -f "$SRC/metadata.json" "$SRC/extension.js" "$DEST/" -# Add to the enabled set (preserves existing). -cur=$(gsettings get org.gnome.shell enabled-extensions 2>/dev/null || echo "@as []") -python3 - "$cur" "$UUID" <<'PY' -import sys, ast -try: l = ast.literal_eval(sys.argv[1]) -except Exception: l = [] -if sys.argv[2] not in l: l.append(sys.argv[2]) -import subprocess -subprocess.run(["gsettings","set","org.gnome.shell","enabled-extensions",str(l)]) -print("enabled-extensions ->", l) -PY -echo "Installed $UUID to $DEST." -echo "GNOME Shell scans extensions only at startup, so log out/in (or restart the" -echo "session) ONCE to load it. After that: gnome-extensions info $UUID should show State: ACTIVE." diff --git a/packages/cua-driver/wayland-helper/winrects@cua/extension.js b/packages/cua-driver/wayland-helper/winrects@cua/extension.js deleted file mode 100644 index 743acc2f5cc..00000000000 --- a/packages/cua-driver/wayland-helper/winrects@cua/extension.js +++ /dev/null @@ -1,1053 +0,0 @@ -import Gio from 'gi://Gio'; -import GLib from 'gi://GLib'; -import Shell from 'gi://Shell'; -import St from 'gi://St'; -import Clutter from 'gi://Clutter'; -import Cairo from 'cairo'; -import * as Main from 'resource:///org/gnome/shell/ui/main.js'; -import { Extension } from 'resource:///org/gnome/shell/extensions/extension.js'; - -Gio._promisify(Shell.Screenshot.prototype, 'screenshot'); - -const IFACE = `<node><interface name="org.cua.WinRects"> -<method name="GetVersion"><arg type="u" direction="out" name="version"/></method> -<method name="GetRects"><arg type="s" direction="out" name="json"/></method> -<method name="Capture"><arg type="s" direction="out" name="png_base64"/></method> -<method name="Activate"><arg type="u" direction="in" name="id"/><arg type="b" direction="out" name="activated"/></method> -<method name="MoveCursor"><arg type="i" direction="in" name="x"/><arg type="i" direction="in" name="y"/></method> -<method name="ClickPulse"><arg type="i" direction="in" name="x"/><arg type="i" direction="in" name="y"/></method> -<method name="SetCursorColor"><arg type="s" direction="in" name="fill_color"/></method> -<method name="SetCursorState"><arg type="s" direction="in" name="action"/><arg type="s" direction="in" name="delivery"/><arg type="s" direction="in" name="target"/><arg type="b" direction="in" name="active"/></method> -<method name="SetSessionLabel"><arg type="s" direction="in" name="label"/></method> -<method name="HideCursor"></method> -</interface></node>`; - -// GNOME cannot host the Rust renderer directly, so this Shell actor mirrors -// the embedded cua.default vector theme and semantic state vocabulary. -const CANVAS_SIZE = 128; -const DISPLAY_SIZE = 42; -const ACTOR_SIZE = 112; -const ACTOR_CENTER = ACTOR_SIZE / 2; -const SCALE = DISPLAY_SIZE / CANVAS_SIZE; -const FLOAT_DURATION = 4.0; -const GLOW_SURFACE_SCALE = 3; -const GLOW_PADDING = 24; -const BADGE_Y_OFFSET = 29; -const BADGE_HOLD_SECONDS = 2.0; -const BADGE_FADE_SECONDS = 0.4; -const BADGE_CHIP_SIZE = 18; -const ACTIONS = new Set([ - 'idle', - 'observe', - 'click', - 'drag', - 'scroll', - 'text', - 'key', - 'navigate', - 'app', - 'transfer', - 'record', - 'system', -]); -const ONE_SHOT_ACTIONS = new Set(['click', 'key', 'navigate', 'app', 'system']); -const ACTION_DURATIONS = { - idle: 4.0, - click: 0.67, - observe: 1.6, - drag: 1.6, - scroll: 1.6, - text: 1.6, - key: 1.6, - navigate: 1.6, - app: 1.6, - transfer: 1.6, - record: 1.6, - system: 1.6, -}; - -function nowSeconds() { - return GLib.get_monotonic_time() / 1_000_000; -} - -function mixChannel(base, accent, weight) { - return Math.round(base * (1 - weight) + accent * weight); -} - -function badgeStyle(fillColor) { - const start = fillColor.map((channel, index) => mixChannel([94, 151, 178][index], channel, 0.66)); - const end = fillColor.map((channel, index) => mixChannel([13, 27, 38][index], channel, 0.26)); - // The rim carries session identity now that the orb is gone, matching - // paint_session_badge in the Rust renderer. - const rim = fillColor.map((channel) => mixChannel(255, channel, 0.55)); - return [ - 'spacing: 7px', - 'padding: 6px 10px', - `background-gradient-start: rgba(${start[0]}, ${start[1]}, ${start[2]}, 0.93)`, - `background-gradient-end: rgba(${end[0]}, ${end[1]}, ${end[2]}, 0.96)`, - 'background-gradient-direction: horizontal', - `border: 1px solid rgba(${rim[0]}, ${rim[1]}, ${rim[2]}, 0.75)`, - 'border-radius: 14px', - 'box-shadow: 0 2px 9px rgba(0, 0, 0, 0.30)', - ].join(';'); -} - -function easeInOut(value) { - const t = Math.max(0, Math.min(1, value)); - return t * t * (3 - 2 * t); -} - -function triangleWave(value) { - const t = ((value % 1) + 1) % 1; - return t < 0.5 ? t * 2 : (1 - t) * 2; -} - -function setInk(cr, alpha = 1) { - cr.setSourceRGBA(1, 1, 1, alpha); -} - -function setPaper(cr, alpha = 1) { - cr.setSourceRGBA(1, 1, 1, alpha); -} - -function setFill(cr, color, alpha = 1) { - cr.setSourceRGBA(color[0] / 255, color[1] / 255, color[2] / 255, alpha); -} - -function glowPath(cr, width, alpha, glowColor) { - if (!glowColor) return; - const layers = 12; - const outerExpansion = 13; - const innerExpansion = 2.5; - const maxOpacity = 0.17; - let accumulatedOpacity = 0; - for (let layer = 0; layer < layers; layer++) { - const progress = (layer + 1) / layers; - const expansion = outerExpansion + (innerExpansion - outerExpansion) * progress; - const targetOpacity = maxOpacity * Math.max(0, Math.min(1, alpha)) * Math.pow(progress, 1.6); - const layerOpacity = - (targetOpacity - accumulatedOpacity) / Math.max(0.001, 1 - accumulatedOpacity); - accumulatedOpacity = targetOpacity; - cr.setLineWidth(width + expansion); - cr.setLineCap(Cairo.LineCap.ROUND); - cr.setLineJoin(Cairo.LineJoin.ROUND); - setFill(cr, glowColor, layerOpacity); - cr.strokePreserve(); - } -} - -function strokePath(cr, width, alpha = 1, glowColor = null) { - glowPath(cr, width, alpha, glowColor); - cr.setLineWidth(glowColor ? width + 1.5 : width); - cr.setLineCap(Cairo.LineCap.ROUND); - cr.setLineJoin(Cairo.LineJoin.ROUND); - setInk(cr, alpha); - if (!glowColor) { - cr.stroke(); - return; - } - cr.strokePreserve(); - cr.setLineWidth(Math.max(1.5, width - 1)); - setFill(cr, glowColor, alpha); - cr.stroke(); -} - -function fillPath(cr, glowWidth, alpha = 1, glowColor = null) { - glowPath(cr, glowWidth, alpha, glowColor); - cr.setLineWidth(3); - cr.setLineCap(Cairo.LineCap.ROUND); - cr.setLineJoin(Cairo.LineJoin.ROUND); - setInk(cr, alpha); - cr.strokePreserve(); - setFill(cr, glowColor, alpha); - cr.fill(); -} - -function linePath(cr, points, width = 4, alpha = 1, glowColor = null) { - if (points.length === 0) return; - cr.moveTo(points[0][0], points[0][1]); - for (let i = 1; i < points.length; i++) cr.lineTo(points[i][0], points[i][1]); - strokePath(cr, width, alpha, glowColor); -} - -function roundedRect(cr, x, y, width, height, radius) { - const r = Math.max(0, Math.min(radius, width / 2, height / 2)); - const k = 0.5522848; - cr.moveTo(x + r, y); - cr.lineTo(x + width - r, y); - cr.curveTo(x + width - r + r * k, y, x + width, y + r - r * k, x + width, y + r); - cr.lineTo(x + width, y + height - r); - cr.curveTo( - x + width, - y + height - r + r * k, - x + width - r + r * k, - y + height, - x + width - r, - y + height - ); - cr.lineTo(x + r, y + height); - cr.curveTo(x + r - r * k, y + height, x, y + height - r + r * k, x, y + height - r); - cr.lineTo(x, y + r); - cr.curveTo(x, y + r - r * k, x + r - r * k, y, x + r, y); - cr.closePath(); -} - -function traceCursorBody(cr) { - cr.moveTo(55, 30); - cr.curveTo(48, 28, 42, 33, 43, 41); - cr.lineTo(64, 98); - cr.curveTo(67, 106, 73, 106, 77, 99); - cr.lineTo(86, 79); - cr.curveTo(88, 75, 91, 72, 95, 70); - cr.lineTo(108, 63); - cr.curveTo(115, 59, 114, 53, 107, 50); - cr.closePath(); -} - -function drawCursorGlowShape(cr, fillColor) { - const layers = 36; - const outerWidth = 44; - const innerWidth = 7; - const maxOpacity = 0.34; - let accumulatedOpacity = 0; - - for (let layer = 0; layer < layers; layer++) { - const progress = (layer + 1) / layers; - const width = outerWidth + (innerWidth - outerWidth) * progress; - const targetOpacity = maxOpacity * Math.pow(progress, 1.65); - const layerOpacity = - (targetOpacity - accumulatedOpacity) / Math.max(0.001, 1 - accumulatedOpacity); - accumulatedOpacity = targetOpacity; - if (layerOpacity <= 0) continue; - - traceCursorBody(cr); - cr.setLineWidth(width); - cr.setLineCap(Cairo.LineCap.ROUND); - cr.setLineJoin(Cairo.LineJoin.ROUND); - setFill(cr, fillColor, layerOpacity); - cr.stroke(); - } -} - -function createGlowSurface(fillColor) { - const surface = new Cairo.ImageSurface( - Cairo.Format.ARGB32, - (CANVAS_SIZE + GLOW_PADDING * 2) * GLOW_SURFACE_SCALE, - (CANVAS_SIZE + GLOW_PADDING * 2) * GLOW_SURFACE_SCALE - ); - const cr = new Cairo.Context(surface); - cr.scale(GLOW_SURFACE_SCALE, GLOW_SURFACE_SCALE); - cr.translate(GLOW_PADDING, GLOW_PADDING); - drawCursorGlowShape(cr, fillColor); - cr.$dispose(); - return surface; -} - -function sharedFloatMotion(progress) { - const angle = progress * Math.PI * 2; - return { - dx: Math.sin(angle) * 5, - dy: Math.cos(angle) * 6 - 5, - rotation: Math.cos(angle) * ((2.5 * Math.PI) / 180), - scale: 1, - }; -} - -function cursorBodyMotion(progress, action) { - let dx = 0; - let dy = 0; - let rotation = 0; - let scale = 1; - - if (action === 'click') { - if (progress < 0.35) scale = 1 - easeInOut(progress / 0.35) * 0.07; - else if (progress < 0.6) scale = 0.93 + easeInOut((progress - 0.35) / 0.25) * 0.1; - else scale = 1.03 - easeInOut((progress - 0.6) / 0.4) * 0.03; - } else if (action === 'drag') { - const held = easeInOut(triangleWave(progress)); - dx = held * 7; - dy = held * 3; - } - - return { dx, dy, rotation, scale }; -} - -function applyCursorBodyMotion(cr, motion) { - cr.translate(64 + motion.dx, 64 + motion.dy); - cr.rotate(motion.rotation); - cr.scale(motion.scale, motion.scale); - cr.translate(-64, -64); -} - -function drawCursorGlow(cr, progress, action, glowSurface) { - const motion = cursorBodyMotion(progress, action); - cr.save(); - applyCursorBodyMotion(cr, motion); - cr.translate(-GLOW_PADDING, -GLOW_PADDING); - cr.scale(1 / GLOW_SURFACE_SCALE, 1 / GLOW_SURFACE_SCALE); - cr.setSourceSurface(glowSurface, 0, 0); - cr.paint(); - cr.restore(); -} - -function drawCursorBody(cr, progress, action, fillColor) { - const motion = cursorBodyMotion(progress, action); - cr.save(); - applyCursorBodyMotion(cr, motion); - traceCursorBody(cr); - setFill(cr, fillColor); - cr.fillPreserve(); - cr.setLineWidth(5); - cr.setLineCap(Cairo.LineCap.ROUND); - cr.setLineJoin(Cairo.LineJoin.ROUND); - setPaper(cr); - cr.stroke(); - cr.restore(); -} - -function drawActionCue(cr, action, progress, fillColor) { - const wave = triangleWave(progress); - const strokeCue = (width, alpha = 1) => strokePath(cr, width, alpha, fillColor); - const lineCue = (points, width = 4, alpha = 1) => linePath(cr, points, width, alpha, fillColor); - cr.save(); - switch (action) { - case 'observe': { - const opacity = Math.min(1, progress * 7); - cr.translate(8, -10); - cr.moveTo(38, 28); - cr.curveTo(27, 29, 20, 38, 20, 49); - strokeCue(4, opacity); - cr.moveTo(42, 19); - cr.curveTo(23, 19, 11, 33, 11, 51); - strokeCue(4, opacity); - break; - } - case 'click': { - const cueProgress = Math.max(0, Math.min(1, progress / 0.65)); - const opacity = Math.max( - 0, - Math.min(1, Math.min(1 - Math.pow(cueProgress, 1.35), cueProgress * 5)) - ); - const cueScale = 1.1 + easeInOut(cueProgress) * 0.4; - cr.translate(35, 28); - cr.scale(cueScale, cueScale); - cr.translate(-25, -25); - lineCue( - [ - [35, 20], - [34, 11], - ], - 4, - opacity - ); - lineCue( - [ - [27, 25], - [19, 19], - ], - 4, - opacity - ); - lineCue( - [ - [25, 34], - [15, 34], - ], - 4, - opacity - ); - break; - } - case 'drag': { - const offset = easeInOut(wave) * 7; - cr.translate(offset, offset * 0.43); - lineCue( - [ - [28, 38], - [16, 35], - ], - 4, - 0.2 + wave * 0.8 - ); - lineCue( - [ - [26, 48], - [12, 45], - ], - 4, - 0.2 + wave * 0.8 - ); - break; - } - case 'scroll': { - cr.translate(-5, 4 - wave * 8); - lineCue( - [ - [23, 31], - [31, 22], - [39, 31], - ], - 4, - 0.42 + wave * 0.58 - ); - lineCue( - [ - [23, 49], - [31, 58], - [39, 49], - ], - 4, - 0.42 + wave * 0.58 - ); - break; - } - case 'text': { - const opacity = progress < 0.34 || progress > 0.64 ? 1 : 0.18; - cr.translate(-4, 0); - lineCue( - [ - [31, 22], - [31, 58], - ], - 4, - opacity - ); - lineCue( - [ - [24, 22], - [38, 22], - ], - 4, - opacity - ); - lineCue( - [ - [24, 58], - [38, 58], - ], - 4, - opacity - ); - break; - } - case 'key': { - const bounce = Math.sin(progress * Math.PI * 2) * (1 - progress) * 3; - cr.translate(-9, bounce); - roundedRect(cr, 14, 25, 28, 28, 6); - strokeCue(3.5); - lineCue( - [ - [23, 32], - [23, 46], - [23, 39], - [33, 32], - [24, 39], - [34, 46], - ], - 3.5 - ); - break; - } - case 'navigate': { - cr.translate(-10 + easeInOut(progress) * 9, 0); - lineCue( - [ - [15, 29], - [25, 40], - [15, 51], - ], - 4, - 0.2 + wave * 0.8 - ); - lineCue( - [ - [29, 29], - [39, 40], - [29, 51], - ], - 4, - 0.2 + wave * 0.8 - ); - break; - } - case 'app': { - const s = 0.2 + easeInOut(Math.min(1, progress * 2)) * 0.8; - cr.translate(21, 39); - cr.scale(s, s); - cr.translate(-26, -39); - for (const [x, y] of [ - [13, 26], - [29, 26], - [13, 42], - [29, 42], - ]) { - roundedRect(cr, x, y, 10, 10, 2); - strokeCue(3.5); - } - break; - } - case 'transfer': - cr.translate(-9, 6 - wave * 12); - lineCue( - [ - [22, 50], - [22, 20], - [14, 28], - [22, 20], - [30, 28], - ], - 4, - 0.38 + wave * 0.62 - ); - lineCue( - [ - [37, 28], - [37, 58], - [29, 50], - [37, 58], - [45, 50], - ], - 4, - 0.38 + wave * 0.62 - ); - break; - case 'record': - cr.translate(-12, 0); - cr.arc(29, 39, 17, 0, Math.PI * 2); - strokeCue(4); - cr.arc(29, 39, 3.6 + wave * 2.1, 0, Math.PI * 2); - fillPath(cr, (3.6 + wave * 2.1) * 1.25, 0.42 + wave * 0.58, fillColor); - break; - case 'system': { - cr.translate(15, 39); - cr.rotate(((easeInOut(progress) * 68 - 18) * Math.PI) / 180); - cr.translate(-29, -39); - for (const radius of [12, 4]) { - cr.arc(29, 39, radius, 0, Math.PI * 2); - strokeCue(3.5); - } - for (const points of [ - [ - [29, 20], - [29, 25], - ], - [ - [29, 53], - [29, 58], - ], - [ - [10, 39], - [15, 39], - ], - [ - [43, 39], - [48, 39], - ], - [ - [16, 26], - [20, 30], - ], - [ - [38, 48], - [42, 52], - ], - [ - [16, 52], - [20, 48], - ], - [ - [38, 30], - [42, 26], - ], - ]) - lineCue(points, 3.5); - break; - } - default: - break; - } - cr.restore(); -} - -function drawBadgeChip(cr, glyph, filled, fillColor) { - roundedRect(cr, 0.5, 0.5, 17, 17, 5); - if (filled) { - cr.setSourceRGBA(fillColor[0] / 255, fillColor[1] / 255, fillColor[2] / 255, 0.86); - cr.fillPreserve(); - } else { - cr.setSourceRGBA(1, 1, 1, 0.09); - cr.fillPreserve(); - } - cr.setSourceRGBA(1, 1, 1, filled ? 0.72 : 0.42); - cr.setLineWidth(1); - cr.stroke(); - cr.setSourceRGBA(1, 1, 1, 0.95); - cr.setLineWidth(1.35); - cr.setLineCap(Cairo.LineCap.ROUND); - cr.setLineJoin(Cairo.LineJoin.ROUND); - - const stroke = () => cr.stroke(); - const line = (points) => { - cr.moveTo(points[0][0], points[0][1]); - for (const [x, y] of points.slice(1)) cr.lineTo(x, y); - stroke(); - }; - switch (glyph) { - case 'background': - roundedRect(cr, 4, 4, 7, 7, 1.8); - stroke(); - roundedRect(cr, 6.4, 6.4, 7, 7, 1.8); - stroke(); - break; - case 'foreground': - roundedRect(cr, 4, 5, 10, 9, 1.8); - stroke(); - line([ - [5, 7.6], - [13, 7.6], - ]); - break; - case 'ax': - line([ - [9, 5], - [9, 9], - [5, 13], - ]); - line([ - [9, 9], - [13, 13], - ]); - for (const [x, y] of [ - [9, 5], - [5, 13], - [13, 13], - ]) { - cr.arc(x, y, 1.35, 0, Math.PI * 2); - cr.fill(); - } - break; - case 'pixel': - line([ - [4, 7], - [4, 4], - [7, 4], - ]); - line([ - [11, 4], - [14, 4], - [14, 7], - ]); - line([ - [14, 11], - [14, 14], - [11, 14], - ]); - line([ - [7, 14], - [4, 14], - [4, 11], - ]); - break; - case 'browser': - cr.arc(9, 9, 5, 0, Math.PI * 2); - stroke(); - line([ - [4, 9], - [14, 9], - ]); - line([ - [9, 4], - [7, 9], - [9, 14], - [11, 9], - [9, 4], - ]); - break; - case 'desktop': - roundedRect(cr, 4, 4, 10, 7.6, 1.3); - stroke(); - line([ - [9, 11.6], - [9, 14], - [6, 14], - [12, 14], - ]); - break; - default: - break; - } -} - -export default class WinRectsExtension extends Extension { - enable() { - this._impl = Gio.DBusExportedObject.wrapJSObject(IFACE, this); - this._impl.export(Gio.DBus.session, '/org/cua/WinRects'); - this._nameId = Gio.bus_own_name( - Gio.BusType.SESSION, - 'org.cua.WinRects', - Gio.BusNameOwnerFlags.REPLACE, - null, - null, - null - ); - this._action = 'idle'; - this._delivery = ''; - this._target = ''; - this._actionStarted = nowSeconds(); - this._endingAt = null; - this._fillColor = [94, 192, 232]; - this._fillColorCss = '#5ec0e8'; - this._glowSurface = createGlowSurface(this._fillColor); - this._cursor = new St.DrawingArea({ - width: ACTOR_SIZE, - height: ACTOR_SIZE, - visible: false, - reactive: false, - can_focus: false, - }); - this._cursor.connect('repaint', (area) => { - const cr = area.get_context(); - const duration = ACTION_DURATIONS[this._action] ?? 1.6; - const elapsed = nowSeconds() - this._actionStarted; - const progress = (elapsed % duration) / duration; - const floatProgress = (elapsed % FLOAT_DURATION) / FLOAT_DURATION; - cr.save(); - cr.translate(ACTOR_CENTER, ACTOR_CENTER); - cr.scale(SCALE, SCALE); - cr.translate(-CANVAS_SIZE / 2, -CANVAS_SIZE / 2); - applyCursorBodyMotion(cr, sharedFloatMotion(floatProgress)); - drawCursorGlow(cr, progress, this._action, this._glowSurface); - drawActionCue(cr, this._action, progress, this._fillColor); - drawCursorBody(cr, progress, this._action, this._fillColor); - cr.restore(); - cr.$dispose(); - }); - this._cursor.set_pivot_point(0.5, 0.5); - Main.layoutManager.addTopChrome(this._cursor); - this._badgeLabel = new St.Label({ - text: '', - visible: false, - y_align: Clutter.ActorAlign.CENTER, - style: 'font-size: 11px; font-weight: 600; color: white;', - }); - this._deliveryChip = new St.DrawingArea({ - width: BADGE_CHIP_SIZE, - height: BADGE_CHIP_SIZE, - visible: false, - }); - this._deliveryChip.connect('repaint', (area) => { - const cr = area.get_context(); - drawBadgeChip(cr, this._delivery, true, this._fillColor); - cr.$dispose(); - }); - this._targetChip = new St.DrawingArea({ - width: BADGE_CHIP_SIZE, - height: BADGE_CHIP_SIZE, - visible: false, - }); - this._targetChip.connect('repaint', (area) => { - const cr = area.get_context(); - drawBadgeChip(cr, this._target, false, this._fillColor); - cr.$dispose(); - }); - this._badge = new St.BoxLayout({ - visible: false, - reactive: false, - can_focus: false, - style: badgeStyle(this._fillColor), - }); - this._badge.add_child(this._badgeLabel); - this._badge.add_child(this._deliveryChip); - this._badge.add_child(this._targetChip); - Main.layoutManager.addTopChrome(this._badge); - this._cursorX = null; - this._cursorY = null; - this._badgeRevealedAt = null; - this._modifierFadeAt = null; - this._frameId = GLib.timeout_add(GLib.PRIORITY_DEFAULT, 33, () => { - if (!this._cursor) return GLib.SOURCE_REMOVE; - const now = nowSeconds(); - if (this._endingAt !== null && now >= this._endingAt) this._setCursorState('idle', '', ''); - if ( - ONE_SHOT_ACTIONS.has(this._action) && - now - this._actionStarted >= ACTION_DURATIONS[this._action] - ) - this._setCursorState('idle', '', ''); - if (this._cursor.visible) this._cursor.queue_repaint(); - let labelAlpha = this._badgeRevealedAt === null ? 0 : 1; - if (this._badgeRevealedAt !== null) { - const badgeElapsed = now - this._badgeRevealedAt; - if ( - badgeElapsed > BADGE_HOLD_SECONDS && - badgeElapsed < BADGE_HOLD_SECONDS + BADGE_FADE_SECONDS - ) { - const fade = (badgeElapsed - BADGE_HOLD_SECONDS) / BADGE_FADE_SECONDS; - labelAlpha = 1 - easeInOut(fade); - } else if (badgeElapsed >= BADGE_HOLD_SECONDS + BADGE_FADE_SECONDS) { - labelAlpha = 0; - this._badgeRevealedAt = null; - } - } - let chipAlpha = this._delivery || this._target ? 1 : 0; - if (this._modifierFadeAt !== null) { - const fade = (now - this._modifierFadeAt) / BADGE_FADE_SECONDS; - chipAlpha = 1 - easeInOut(fade); - if (fade >= 1) { - chipAlpha = 0; - this._modifierFadeAt = null; - this._delivery = ''; - this._target = ''; - this._updateModifierChips(); - } - } - if (this._badgeLabel) { - this._badgeLabel.opacity = Math.round(255 * labelAlpha); - if (labelAlpha <= 0.001 && this._badgeLabel.visible) { - this._badgeLabel.hide(); - if (this._cursorX !== null && this._cursorY !== null) - this._positionBadge(this._cursorX, this._cursorY, 0); - } - } - if (this._deliveryChip) this._deliveryChip.opacity = Math.round(255 * chipAlpha); - if (this._targetChip) this._targetChip.opacity = Math.round(255 * chipAlpha); - if (this._badge) { - if (labelAlpha > 0.001 || chipAlpha > 0.001) { - this._badge.opacity = 255; - this._badge.show(); - } else { - this._badge.hide(); - } - } - return GLib.SOURCE_CONTINUE; - }); - } - disable() { - if (this._frameId) { - GLib.source_remove(this._frameId); - this._frameId = 0; - } - if (this._cursor) { - this._cursor.destroy(); - this._cursor = null; - } - if (this._badge) { - this._badge.destroy(); - this._badge = null; - } - this._badgeLabel = null; - this._deliveryChip = null; - this._targetChip = null; - if (this._glowSurface) { - this._glowSurface.finish(); - this._glowSurface = null; - } - if (this._impl) { - this._impl.unexport(); - this._impl = null; - } - if (this._nameId) { - Gio.bus_unown_name(this._nameId); - this._nameId = 0; - } - } - GetVersion() { - return 8; - } - GetRects() { - const actors = global.get_window_actors(); - const actorByWindow = new Map(); - for (const actor of actors) { - if (actor.meta_window) actorByWindow.set(actor.meta_window, actor); - } - const windows = global.display.sort_windows_by_stacking([...actorByWindow.keys()]); - const focusedWindow = global.display.focus_window; - const out = []; - for (let stacking = 0; stacking < windows.length; stacking++) { - const w = windows[stacking]; - const actor = actorByWindow.get(w); - const r = w.get_frame_rect(); - let buffer = r; - try { - buffer = w.get_buffer_rect(); - } catch (_error) { - // Older Shell releases may not expose the buffer rectangle. - } - const minimized = Boolean(w.minimized); - out.push({ - id: w.get_stable_sequence(), - pid: w.get_pid(), - title: w.get_title() ?? '', - x: r.x, - y: r.y, - w: r.width, - h: r.height, - buffer_x: buffer.x, - buffer_y: buffer.y, - focused: focusedWindow === w, - minimized, - visible: Boolean(actor?.visible) && !minimized, - stacking, - }); - } - return JSON.stringify(out); - } - async CaptureAsync(_params, invocation) { - try { - const shooter = new Shell.Screenshot(); - const stream = Gio.MemoryOutputStream.new_resizable(); - const [width, height] = global.display.get_size(); - if (width <= 0 || height <= 0) throw new Error(`Invalid display size ${width}x${height}`); - // GNOME 50's stage-content capture copies the real cursor sprite - // after painting the stage. Remote/headless pointer seats can - // expose a 0x0 sprite, and that copy can crash Shell. The stream - // API has an explicit include_cursor flag, so keep cursor capture - // disabled; cua-driver draws its own agent cursor. - await shooter.screenshot(false, stream); - stream.close(null); - const encoded = GLib.base64_encode(stream.steal_as_bytes().get_data()); - invocation.return_value(new GLib.Variant('(s)', [encoded])); - } catch (error) { - invocation.return_dbus_error('org.cua.WinRects.CaptureFailed', String(error)); - } - } - ActivateAsync([id], invocation) { - const target = global - .get_window_actors() - .map((actor) => actor.meta_window) - .find((window) => window?.get_stable_sequence() === id); - if (!target) { - invocation.return_value(new GLib.Variant('(b)', [false])); - return; - } - target.activate(global.get_current_time()); - GLib.timeout_add(GLib.PRIORITY_DEFAULT, 100, () => { - invocation.return_value(new GLib.Variant('(b)', [global.display.focus_window === target])); - return GLib.SOURCE_REMOVE; - }); - } - MoveCursor(x, y) { - if (!this._cursor) return; - const revealBadge = !this._cursor.visible; - this._cursorX = x; - this._cursorY = y; - this._cursor.show(); - this._cursor.ease({ - x: x - ACTOR_CENTER, - y: y - ACTOR_CENTER, - duration: 480, - mode: Clutter.AnimationMode.EASE_OUT_CUBIC, - }); - this._positionBadge(x, y, 480); - if (revealBadge) this._revealBadge(); - } - ClickPulse(x, y) { - if (!this._cursor) return; - const revealBadge = !this._cursor.visible; - this._cursorX = x; - this._cursorY = y; - this._cursor.set_position(x - ACTOR_CENTER, y - ACTOR_CENTER); - this._positionBadge(x, y, 0); - if (['idle', 'navigate', 'click'].includes(this._action)) - this._setCursorState('click', this._delivery, this._target); - this._cursor.show(); - this._cursor.queue_repaint(); - if (revealBadge) this._revealBadge(); - } - SetCursorColor(fillColor) { - const match = /^#([0-9a-fA-F]{6})$/.exec(fillColor); - if (!match) return; - const rgb = Number.parseInt(match[1], 16); - this._fillColor = [(rgb >> 16) & 0xff, (rgb >> 8) & 0xff, rgb & 0xff]; - this._fillColorCss = `#${match[1].toLowerCase()}`; - if (this._badge) this._badge.set_style(badgeStyle(this._fillColor)); - this._deliveryChip?.queue_repaint(); - this._targetChip?.queue_repaint(); - if (this._glowSurface) this._glowSurface.finish(); - this._glowSurface = createGlowSurface(this._fillColor); - if (this._cursor) this._cursor.queue_repaint(); - } - SetCursorState(action, delivery, target, active) { - if (!ACTIONS.has(action)) return; - if (active) { - this._setCursorState(action, delivery, target); - } else if (this._action === action && !ONE_SHOT_ACTIONS.has(action)) { - this._endingAt = nowSeconds() + 0.4; - } - } - SetSessionLabel(label) { - if (!this._badgeLabel || !this._badge) return; - this._badgeLabel.set_text(label); - if (label.length === 0 || !this._cursor?.visible) { - this._badgeLabel.hide(); - this._badgeRevealedAt = null; - if (!this._delivery && !this._target) this._badge.hide(); - return; - } - if (this._cursorX !== null && this._cursorY !== null) { - this._positionBadge(this._cursorX, this._cursorY, 0); - this._revealBadge(); - } - } - _positionBadge(x, y, duration) { - if (!this._badge) return; - const [, naturalWidth] = this._badge.get_preferred_width(-1); - const badgeX = Math.round(x - naturalWidth / 2); - const badgeY = Math.round(y + BADGE_Y_OFFSET); - if (duration > 0) { - this._badge.ease({ - x: badgeX, - y: badgeY, - duration, - mode: Clutter.AnimationMode.EASE_OUT_CUBIC, - }); - } else { - this._badge.set_position(badgeX, badgeY); - } - } - _revealBadge() { - if (!this._badge || !this._badgeLabel || this._badgeLabel.get_text().length === 0) return; - this._badgeLabel.show(); - if (this._cursorX !== null && this._cursorY !== null) - this._positionBadge(this._cursorX, this._cursorY, 0); - this._badgeRevealedAt = nowSeconds(); - this._badge.opacity = 255; - this._badge.show(); - } - _setCursorState(action, delivery, target) { - this._action = ACTIONS.has(action) ? action : 'idle'; - const nextDelivery = delivery ?? ''; - const nextTarget = target ?? ''; - if (nextDelivery || nextTarget) { - this._delivery = nextDelivery; - this._target = nextTarget; - this._modifierFadeAt = null; - } else if ((this._delivery || this._target) && this._modifierFadeAt === null) { - this._modifierFadeAt = nowSeconds(); - } - this._updateModifierChips(); - this._actionStarted = nowSeconds(); - this._endingAt = null; - if (this._cursor) this._cursor.queue_repaint(); - if (this._cursorX !== null && this._cursorY !== null) - this._positionBadge(this._cursorX, this._cursorY, 0); - } - _updateModifierChips() { - if (this._deliveryChip) { - this._deliveryChip.visible = this._delivery.length > 0; - this._deliveryChip.queue_repaint(); - } - if (this._targetChip) { - this._targetChip.visible = this._target.length > 0; - this._targetChip.queue_repaint(); - } - } - HideCursor() { - if (this._cursor) this._cursor.hide(); - if (this._badge) this._badge.hide(); - this._badgeRevealedAt = null; - this._modifierFadeAt = null; - this._delivery = ''; - this._target = ''; - this._updateModifierChips(); - } -} diff --git a/packages/cua-driver/wayland-helper/winrects@cua/metadata.json b/packages/cua-driver/wayland-helper/winrects@cua/metadata.json deleted file mode 100644 index 3faf86b779c..00000000000 --- a/packages/cua-driver/wayland-helper/winrects@cua/metadata.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "uuid": "winrects@cua", - "name": "WinRects", - "description": "Expose window geometry, capture, cursor, and verified activation for cua-driver on Wayland.", - "shell-version": ["45", "46", "47", "48", "49", "50"], - "version": 8 -} diff --git a/packages/node-repl/README.md b/packages/node-repl/README.md index 0e7e7e847ff..ea6ae0fb9a4 100644 --- a/packages/node-repl/README.md +++ b/packages/node-repl/README.md @@ -1,7 +1,7 @@ # @qwen-code/node-repl-mcp A standalone [Model Context Protocol](https://modelcontextprotocol.io) server that -exposes a **session-persistent Node.js REPL** as five MCP tools. It runs a real +exposes a **session-persistent Node.js REPL** as two MCP tools. It runs a real Node.js kernel in a dedicated child process; top-level bindings, closures, and module state persist across calls within a session. @@ -10,13 +10,17 @@ client (Qwen Code via `mcpServers`, Claude, Codex, etc.) can run it. ## Tools -| Tool | Description | -| ------------------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `node_repl` | Start one JavaScript cell. `{ code, timeout_ms?, yield_time_ms?, title? }`; yields a cell ID if it remains active. | -| `node_repl_wait` | Wait for the active cell by ID without cancelling it. | -| `node_repl_cancel` | Cancel the active cell by ID; terminate an unresponsive kernel after five seconds. | -| `node_repl_reset` | Terminate the kernel process and discard all bindings/module state. | -| `node_repl_add_node_module_dir` | Register an extra `node_modules` directory for bare-package resolution. | +| Tool | Description | +| ---------- | ---------------------------------------------------------------------------------------------- | +| `js` | Execute JavaScript to completion. `{ code, timeout_ms?, title? }`, default timeout 30 seconds. | +| `js_reset` | Discard bindings and restart the kernel on the next call. | + +The first call preloads global `cua` and emits the current platform's runtime +documentation. Start with `var app = await cua.getApp("Application")`; it displays +AX state automatically. Use `app.getAXState()`, `app.getScreenshot()`, or +`app.getAXStateAndScreenshot()` for observations, with `emit:false` to suppress +automatic display. Actions return void; observe to confirm the result. +Call `cua.rewriteDocumentation()` after context compaction. `cua.desktop` provides explicit foreground desktop input; `cua.getBrowser/getTab/createBrowserTab` wrap the existing browser SDK. The SDK and browser SDK are installed as matching dependencies. Graceful reset/disposal closes the native session and releases held desktop keys before terminating the kernel. ### Cell semantics @@ -31,7 +35,7 @@ client (Qwen Code via `mcpServers`, Claude, Codex, etc.) can run it. helpers that captured its binding read the replacement. Helper assignments and direct assignments share the same binding, including within one cell. - Bare packages resolve from the session `cwd` `node_modules` plus any directory - registered via `node_repl_add_node_module_dir`; package entrypoints use Node + configured by the host with `QWEN_NODE_REPL_MODULE_DIRS`; package entrypoints use Node singleton caching. Local `.js`/`.mjs` reload on each execution. - Node builtins are importable except `process`/`node:process`. Use `(await import('node:module')).createRequire(import.meta.url)` for CommonJS or @@ -41,7 +45,7 @@ client (Qwen Code via `mcpServers`, Claude, Codex, etc.) can run it. not committed. If the kernel does not return a terminal result within five seconds of cancellation, the host terminates it and reports that all bindings were lost. A subsequent cell starts a fresh kernel. External actions may have - completed, so verify external state before retrying. `node_repl_reset` or a + completed, so verify external state before retrying. `js_reset` or a real process crash also discards all bindings. - When the kernel is retained, runtime errors keep completed statement/declarator checkpoints; cancellation and timeout restore binding values from cell entry. @@ -54,23 +58,131 @@ client (Qwen Code via `mcpServers`, Claude, Codex, etc.) can run it. ## Usage -Once published, the packaged `bin` is the simplest entry point: +### Requirements and installation + +Install Node.js **22 or later**, including npm, and the MCP client you want to +use. Run the server on the computer whose desktop you want to control. +The commands below target **0.26.0** and require that version's npm packages +and native GitHub Release (`cua-driver-rs-v0.26.0`) to be published. + +`npx` downloads the server on first launch. It automatically installs +`@qwen-code/cua-sdk@0.26.0` and `@qwen-code/browser-use@0.26.0`; the SDK's +installation downloads the matching native payload and verifies its SHA-256 +checksum. Allow npm installation scripts and network access to npm and GitHub +Releases. No separate SDK install, manual SDK import, skill installation, +or standalone `qwen-cua-driver` CLI installation is needed. + +Claude Code and Codex do not require Qwen Code to use this server. These +component versions do not require a Qwen Code CLI or Web Shell release with +the same version. For direct native MCP tools instead of JavaScript, see the +[CUA Driver README](../cua-driver/README.md#agent-integration). + +### Qwen Code + +On macOS/Linux: + +```bash +qwen mcp add --scope user node-repl npx -y @qwen-code/node-repl-mcp@0.26.0 +``` + +This writes the server entry to `~/.qwen/settings.json`. Use `--scope project` +for the current project's `.qwen/settings.json` instead. The equivalent +settings entry is: ```jsonc -// qwen-code settings.json (or any MCP client) { "mcpServers": { "node-repl": { "command": "npx", - "args": ["-y", "@qwen-code/node-repl-mcp"], - "cwd": "/your/workspace", - "env": { "QWEN_NODE_REPL_ROOTS": "/extra/node_modules/parent" }, + "args": ["-y", "@qwen-code/node-repl-mcp@0.26.0"], }, }, } ``` -For local development against a build in this repo, point at `dist/index.js`: +### Claude Code + +On macOS/Linux: + +```bash +claude mcp add --scope user --transport stdio node-repl -- npx -y @qwen-code/node-repl-mcp@0.26.0 +``` + +User scope is stored in `~/.claude.json`. Use `--scope project` to write the +current project's `.mcp.json` instead; its `mcpServers` entry can use the same +JSON shape as above. See the +[Claude Code MCP documentation](https://code.claude.com/docs/en/mcp). + +### Codex + +On macOS/Linux: + +```bash +codex mcp add node-repl -- npx -y @qwen-code/node-repl-mcp@0.26.0 +``` + +Alternatively, add the following to `~/.codex/config.toml`. The longer startup +timeout allows time for the first npm/native payload download; add it to the +existing table if you registered the server with the CLI. + +```toml +[mcp_servers.node-repl] +command = "npx" +args = ["-y", "@qwen-code/node-repl-mcp@0.26.0"] +startup_timeout_sec = 120 +``` + +Use an absolute `npx` executable path if the client's `PATH` does not include +your Node.js installation. See the +[official OpenAI MCP documentation](https://learn.chatgpt.com/docs/extend/mcp). + +### Windows client commands + +In PowerShell, use `cmd.exe` to launch npm's `npx` command. Run the command for +your chosen client: + +```powershell +# Qwen Code +qwen mcp add --scope user node-repl cmd.exe /d /c npx -y @qwen-code/node-repl-mcp@0.26.0 + +# Claude Code +claude mcp add --scope user --transport stdio node-repl -- cmd.exe /d /c npx -y @qwen-code/node-repl-mcp@0.26.0 + +# Codex +codex mcp add node-repl -- cmd.exe /d /c npx -y @qwen-code/node-repl-mcp@0.26.0 +``` + +For manual JSON/TOML configuration on Windows, set `command` to `"cmd.exe"` +and `args` to `["/d", "/c", "npx", "-y", "@qwen-code/node-repl-mcp@0.26.0"]`. + +### Permissions and connection check + +Restart the client, then check `qwen mcp list`, `claude mcp list`, or +`codex mcp list`. Inside the session, use `/mcp` to check that `node-repl` +is connected and exposes `js` and `js_reset`. Ask the agent to execute this +first `js` call: + +```js +await cua.getState(); +``` + +It displays the current platform's instructions and application inventory. +Continue with `var app = await cua.getApp('Application name')` using an +application from that observation. The global `cua` is already loaded. + +On macOS, grant Accessibility and Screen Recording to the responsible host +shown by macOS (the client, terminal, or Node process, depending on how you +launched it). Automation may also be requested for document/Finder actions. +These SDK grants follow the host's OS responsibility chain; the standalone +`QwenCuaDriver.app` grant does not replace them. On Linux, pass the desktop +environment as described below. Browser automation uses Chrome 125 or later on +macOS/Linux and requires the +[Qwen Code browser extension](https://chromewebstore.google.com/detail/qwen-code/hdhmmjclhibojdddmancfgbkleahfaph) +installed in the Chrome profile you want to control. + +### Local development and environment + +For a build in this repo, point at `dist/index.js`: ```jsonc { @@ -90,6 +202,7 @@ resolution (`<cwd>/node_modules`). Environment: - `QWEN_NODE_REPL_ROOTS` — extra readable roots (path-list, OS delimiter). +- `QWEN_NODE_REPL_MODULE_DIRS` — host-configured `node_modules` directories (path-list, OS delimiter); retained across reset. - `QWEN_NODE_REPL_DEBUG` — set truthy for stderr debug logging. ### Linux desktop sessions @@ -100,9 +213,10 @@ variables removed by the MCP client cannot be recovered by the SDK. For Codex, add the following allowlist to the existing server entry in `config.toml`: ```toml -[mcp_servers.node_repl] +[mcp_servers.node-repl] command = "npx" -args = ["-y", "@qwen-code/node-repl-mcp"] +args = ["-y", "@qwen-code/node-repl-mcp@0.26.0"] +startup_timeout_sec = 120 env_vars = ["DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS", "AT_SPI_BUS_ADDRESS", "XDG_RUNTIME_DIR", "WAYLAND_DISPLAY", "GTK_MODULES", "QT_ACCESSIBILITY", "QT_LINUX_ACCESSIBILITY_ALWAYS_ON"] ``` @@ -111,6 +225,9 @@ explicit `env` map, populate it from that same desktop session; display numbers, authentication paths, and bus addresses vary by machine and user. Restart the MCP server after changing its environment. An X11 connection failure is reported as `desktop_unavailable` by CUA discovery, rather than as an empty desktop. +Linux supports X11 applications, including XWayland; native Wayland +applications are unsupported. Add `env_vars` to the existing table rather +than creating a second table for the same server. Uncaught errors preserve a string `code` and bounded `details` text in MCP output, including action and verification diagnostics. External side effects may already @@ -119,11 +236,11 @@ have happened when an error is thrown; use those diagnostics before retrying. ## Desktop relay `node-repl-mcp desktop-relay` lends this server to a Qwen Code session running -somewhere else, typically a headless dev box, so the computer-use skill can +somewhere else, typically a headless dev box, so the Computer Use runtime can drive the desktop you are sitting at. On macOS: ```bash -npx -y @qwen-code/node-repl-mcp@0.1.7 desktop-relay install +npx -y @qwen-code/node-repl-mcp@0.26.0 desktop-relay install ``` This installs the server and `@qwen-code/cua-sdk` under `~/.qwen/desktop-relay` @@ -133,7 +250,7 @@ until something connects, and each connection gets its own short-lived process. - **Web Shell.** In a session served by a remote `qwen serve` (started with `QWEN_SERVE_CLIENT_MCP_OVER_WS=1`), choose **Use this computer** in the sidebar footer. A native dialog on this computer asks for approval; once allowed, the - relay starts `node_repl` here and registers it for that one session over the + relay starts the JavaScript MCP server here and registers it for that one session over the daemon's reverse tool channel. Disconnect from the same entry. An approved session can run code on this computer with your permissions and see @@ -177,3 +294,7 @@ transform, protocol, security policy, kernel manager). The qwen-coupled result converter was replaced by `output-adapter.ts`, which emits MCP content blocks. The empty-in-production trusted-package / sha256-pinning layer was removed entirely: this runtime has no trusted-package or capability mechanism. + +Relayed `js` calls have a 20-second execution limit so cancellation and the +terminal response fit the daemon's 30-second reverse-channel deadline. Local +stdio calls retain the default 30-second limit. diff --git a/packages/node-repl/package.json b/packages/node-repl/package.json index 8e9295a8dd2..79242ca68d0 100644 --- a/packages/node-repl/package.json +++ b/packages/node-repl/package.json @@ -1,7 +1,7 @@ { "name": "@qwen-code/node-repl-mcp", - "version": "0.1.7", - "description": "Standalone MCP server exposing a session-persistent Node.js REPL (node_repl) — independent of qwen-code core.", + "version": "0.26.0", + "description": "Standalone MCP server exposing persistent JavaScript (js/js_reset) with the Computer Use runtime.", "repository": { "type": "git", "url": "git+https://github.com/QwenLM/qwen-code.git", @@ -45,7 +45,8 @@ "tree-sitter-wasms": "^0.1.13", "web-tree-sitter": "^0.24.7", "ws": "^8.18.0", - "zod": "^3.25.76" + "zod": "^3.25.76", + "@qwen-code/cua-sdk": "0.26.0" }, "devDependencies": { "@types/node": "^22.13.10", diff --git a/packages/node-repl/scripts/lifecycle-smoke.mjs b/packages/node-repl/scripts/lifecycle-smoke.mjs index c61c24dc008..ad2df0c1b36 100644 --- a/packages/node-repl/scripts/lifecycle-smoke.mjs +++ b/packages/node-repl/scripts/lifecycle-smoke.mjs @@ -53,7 +53,7 @@ send({ jsonrpc: '2.0', id: 2, method: 'tools/call', - params: { name: 'node_repl', arguments: { code: 'nodeRepl.write("up");' } }, + params: { name: 'js', arguments: { code: 'nodeRepl.write("up");' } }, }); await new Promise((resolve) => { diff --git a/packages/node-repl/scripts/mcp-smoke.mjs b/packages/node-repl/scripts/mcp-smoke.mjs index ecb0d2305d0..a5c65ba2cc7 100644 --- a/packages/node-repl/scripts/mcp-smoke.mjs +++ b/packages/node-repl/scripts/mcp-smoke.mjs @@ -40,18 +40,17 @@ try { const tools = await client.listTools(); const names = tools.tools.map((t) => t.name).sort(); check( - 'lists the five node_repl tools', - names.join(',') === - 'node_repl,node_repl_add_node_module_dir,node_repl_cancel,node_repl_reset,node_repl_wait', + 'lists the two JavaScript tools', + names.join(',') === 'js,js_reset', names.join(','), ); await client.callTool({ - name: 'node_repl', + name: 'js', arguments: { code: 'const x = 40;' }, }); const r = await client.callTool({ - name: 'node_repl', + name: 'js', arguments: { code: 'nodeRepl.write(String(x + 2));' }, }); check( @@ -60,9 +59,9 @@ try { textOf(r).trim(), ); - await client.callTool({ name: 'node_repl_reset', arguments: {} }); + await client.callTool({ name: 'js_reset', arguments: {} }); const r2 = await client.callTool({ - name: 'node_repl', + name: 'js', arguments: { code: 'nodeRepl.write(typeof x);' }, }); check( diff --git a/packages/node-repl/scripts/verify-package.mjs b/packages/node-repl/scripts/verify-package.mjs index 02b40356ecd..aebda061233 100644 --- a/packages/node-repl/scripts/verify-package.mjs +++ b/packages/node-repl/scripts/verify-package.mjs @@ -48,6 +48,12 @@ function run(command, args, options = {}) { } const outputDirectory = path.resolve(valueAfter('--output-dir')); +const localDependencies = ['--sdk-tarball', '--browser-tarball'].flatMap( + (flag) => { + const index = process.argv.indexOf(flag); + return index < 0 ? [] : [path.resolve(valueAfter(flag))]; + }, +); const manifest = JSON.parse( readFileSync(path.join(packageRoot, 'package.json'), 'utf8'), ); @@ -156,6 +162,7 @@ try { [ 'install', tarball, + ...localDependencies, '--no-audit', '--no-fund', `--registry=${npmRegistry}`, @@ -197,7 +204,7 @@ try { if ( instructions.length === 0 || instructions.length >= 256 || - !instructions.includes('One persistent JavaScript kernel') || + !instructions.includes('runtime documentation') || instructions.includes('Computer Use') ) { throw new Error( @@ -206,28 +213,29 @@ try { } const tools = await client.listTools(); const names = tools.tools.map((tool) => tool.name).sort(); - if ( - names.join(',') !== - 'node_repl,node_repl_add_node_module_dir,node_repl_cancel,node_repl_reset,node_repl_wait' - ) { + if (names.join(',') !== 'js,js_reset') { throw new Error(`unexpected MCP tools: ${names.join(',')}`); } - await client.callTool({ - name: 'node_repl', + const initial = await client.callTool({ + name: 'js', arguments: { code: 'const releaseValue = 40;' }, }); + if (initial.isError || !textOf(initial).includes('cua.desktop')) + throw new Error( + 'clean package did not preload matching runtime documentation', + ); const persisted = await client.callTool({ - name: 'node_repl', + name: 'js', arguments: { code: 'nodeRepl.write(String(releaseValue + 2));' }, }); if (!textOf(persisted).includes('42')) { throw new Error('packed MCP server did not preserve cell bindings'); } - await client.callTool({ name: 'node_repl_reset', arguments: {} }); + await client.callTool({ name: 'js_reset', arguments: {} }); const reset = await client.callTool({ - name: 'node_repl', + name: 'js', arguments: { code: 'nodeRepl.write(typeof releaseValue);' }, }); if (!textOf(reset).includes('undefined')) { diff --git a/packages/node-repl/src/desktop-relay/constants.ts b/packages/node-repl/src/desktop-relay/constants.ts index 531470c631a..ef57bbec48f 100644 --- a/packages/node-repl/src/desktop-relay/constants.ts +++ b/packages/node-repl/src/desktop-relay/constants.ts @@ -22,8 +22,8 @@ export const DESKTOP_RELAY_LABEL = 'com.qwencode.desktop-relay'; */ export const DESKTOP_RELAY_SERVER_NAME = 'desktop-node-repl'; -/** Kept equal to the SDK version the bundled computer-use skill pins. */ -export const DESKTOP_RELAY_CUA_SDK_VERSION = '0.20.11'; +/** Same version as the standalone runtime and its SDK dependency. */ +export const DESKTOP_RELAY_CUA_SDK_VERSION = '0.26.0'; /** * The daemon drops `/acp` frames above 10 MB; leave room for the @@ -31,14 +31,8 @@ export const DESKTOP_RELAY_CUA_SDK_VERSION = '0.20.11'; */ export const MAX_RELAYED_REPLY_BYTES = 9 * 1024 * 1024; -/** - * Below the daemon's 30 s per-message round-trip limit on the reverse channel - * (`CLIENT_MCP_MESSAGE_TIMEOUT_MS` in packages/core), so a relayed call's - * reply fits its frame: a longer yield would time the frame out while the - * cell kept driving the desktop. A local stdio node_repl keeps its own 60 s - * maximum (`MAX_YIELD_TIME_MS` in mcp-server.ts). - */ -export const MAX_RELAYED_YIELD_TIME_MS = 25_000; +/** Leaves cancellation grace and transport time within the daemon's 30 s limit. */ +export const MAX_RELAYED_TIMEOUT_MS = 20_000; export function defaultRelayHome(): string { return path.join(os.homedir(), '.qwen', 'desktop-relay'); diff --git a/packages/node-repl/src/desktop-relay/mcp-child-relay.test.ts b/packages/node-repl/src/desktop-relay/mcp-child-relay.test.ts index da3afb2cc69..3c4d6364f6b 100644 --- a/packages/node-repl/src/desktop-relay/mcp-child-relay.test.ts +++ b/packages/node-repl/src/desktop-relay/mcp-child-relay.test.ts @@ -5,7 +5,7 @@ */ import { describe, expect, it } from 'vitest'; -import { MAX_RELAYED_YIELD_TIME_MS } from './constants.js'; +import { MAX_RELAYED_TIMEOUT_MS } from './constants.js'; import { McpChildRelay, type ChildChannel, @@ -178,7 +178,7 @@ describe('McpChildRelay', () => { expect(child.closed).toBe(true); }); - it('rewrites a yield above the channel budget down to it', async () => { + it('bounds execution and cancellation within the channel timeout', async () => { const child = new FakeChild(); const relay = new McpChildRelay(child); const pending = relay.handle({ @@ -186,19 +186,18 @@ describe('McpChildRelay', () => { id: 1, method: 'tools/call', params: { - name: 'node_repl', - arguments: { code: 'longRunning()', yield_time_ms: 60_000 }, + name: 'js', + arguments: { code: 'longRunning()', timeout_ms: 60_000 }, }, }); - // The daemon answers one reverse-channel frame for at most 30 s; a longer - // yield would time the frame out while the cell kept driving the desktop. + // Execution plus cancellation grace must fit within one reverse-channel frame. expect(child.last()).toMatchObject({ method: 'tools/call', params: { - name: 'node_repl', + name: 'js', arguments: { code: 'longRunning()', - yield_time_ms: MAX_RELAYED_YIELD_TIME_MS, + timeout_ms: MAX_RELAYED_TIMEOUT_MS, }, }, }); @@ -206,7 +205,7 @@ describe('McpChildRelay', () => { await expect(pending).resolves.toMatchObject({ id: 1, result: {} }); }); - it('leaves a yield inside the channel budget untouched', async () => { + it('leaves shorter execution timeouts untouched', async () => { const child = new FakeChild(); const relay = new McpChildRelay(child); const pending = relay.handle({ @@ -214,12 +213,12 @@ describe('McpChildRelay', () => { id: 1, method: 'tools/call', params: { - name: 'node_repl', - arguments: { code: 'fast()', yield_time_ms: 5_000 }, + name: 'js', + arguments: { code: 'fast()', timeout_ms: 5_000 }, }, }); expect(child.last()).toMatchObject({ - params: { arguments: { code: 'fast()', yield_time_ms: 5_000 } }, + params: { arguments: { code: 'fast()', timeout_ms: 5_000 } }, }); child.reply({ jsonrpc: '2.0', id: child.last().id, result: {} }); await expect(pending).resolves.toMatchObject({ id: 1, result: {} }); diff --git a/packages/node-repl/src/desktop-relay/mcp-child-relay.ts b/packages/node-repl/src/desktop-relay/mcp-child-relay.ts index 397f4a2c949..6b724fc78fe 100644 --- a/packages/node-repl/src/desktop-relay/mcp-child-relay.ts +++ b/packages/node-repl/src/desktop-relay/mcp-child-relay.ts @@ -6,7 +6,7 @@ import { MAX_RELAYED_REPLY_BYTES, - MAX_RELAYED_YIELD_TIME_MS, + MAX_RELAYED_TIMEOUT_MS, } from './constants.js'; export type JsonRpcId = string | number; @@ -53,26 +53,23 @@ interface Pending { resolve: (reply: JsonRpcMessage | undefined) => void; } -/** - * A relayed `tools/call` reply must fit the daemon's per-message round-trip - * budget, so a `yield_time_ms` above it is rewritten down; the caller can - * keep waiting with `node_repl_wait`. - */ -function clampYieldTime(message: JsonRpcMessage): JsonRpcMessage { +function clampExecutionTime(message: JsonRpcMessage): JsonRpcMessage { if (message.method !== 'tools/call') return message; - const params = message.params as { arguments?: unknown } | undefined; - const args = params?.arguments; + const params = message.params as + | { name?: string; arguments?: unknown } + | undefined; + if (params?.name !== 'js') return message; + const args = params.arguments; if (args === null || typeof args !== 'object') return message; const record = args as Record<string, unknown>; - const yieldTime = record['yield_time_ms']; - if (typeof yieldTime !== 'number' || yieldTime <= MAX_RELAYED_YIELD_TIME_MS) { + const timeout = record['timeout_ms']; + if (typeof timeout === 'number' && timeout <= MAX_RELAYED_TIMEOUT_MS) return message; - } return { ...message, params: { ...params, - arguments: { ...record, yield_time_ms: MAX_RELAYED_YIELD_TIME_MS }, + arguments: { ...record, timeout_ms: MAX_RELAYED_TIMEOUT_MS }, }, }; } @@ -204,7 +201,7 @@ export class McpChildRelay { return new Promise((resolve) => { this.pending.set(relayId, { originalId, resolve }); try { - this.child.send({ ...clampYieldTime(message), id: relayId }); + this.child.send({ ...clampExecutionTime(message), id: relayId }); } catch (error) { this.settle( relayId, diff --git a/packages/node-repl/src/index.ts b/packages/node-repl/src/index.ts index 8c48d41ebf6..8601b14d24e 100644 --- a/packages/node-repl/src/index.ts +++ b/packages/node-repl/src/index.ts @@ -29,9 +29,7 @@ async function main(): Promise<void> { const shutdown = (code = 0) => { if (shuttingDown) return; shuttingDown = true; - dispose(); - void server - .close() + void Promise.all([dispose(), server.close()]) .catch(() => undefined) .finally(() => process.exit(code)); }; @@ -45,7 +43,9 @@ async function main(): Promise<void> { process.stdin.once('end', () => shutdown()); process.stdin.once('close', () => shutdown()); // Best-effort synchronous cleanup for any other exit path. - process.on('exit', () => dispose()); + process.on('exit', () => { + void dispose(); + }); const transport = new StdioServerTransport(); await server.connect(transport); diff --git a/packages/node-repl/src/kernel-manager.test.ts b/packages/node-repl/src/kernel-manager.test.ts index fffaecc043d..922275eb688 100644 --- a/packages/node-repl/src/kernel-manager.test.ts +++ b/packages/node-repl/src/kernel-manager.test.ts @@ -80,12 +80,35 @@ beforeEach(() => { manager = makeManager(); }); -afterEach(() => { - manager.dispose(); +afterEach(async () => { + await manager.dispose(); fs.rmSync(workDir, { recursive: true, force: true }); }); describe('NodeReplKernelManager', () => { + it('awaits Computer Use session cleanup on reset and disposal', async () => { + const marker = path.join(workDir, 'cua-cleanup.txt'); + createEsmPackage( + path.join(workDir, 'node_modules'), + 'cua-cleanup-fixture', + ` + import fs from 'node:fs'; + import { setTimeout } from 'node:timers/promises'; + export const cua = { async [Symbol.asyncDispose]() { + await setTimeout(30); + fs.appendFileSync(${JSON.stringify(marker)}, ${JSON.stringify('released\n')}); + } }; + `, + ); + const install = `globalThis.cua = (await import('cua-cleanup-fixture')).cua;`; + expect((await run(install)).status).toBe('ok'); + await manager.reset(); + expect(fs.readFileSync(marker, 'utf8')).toBe('released\n'); + expect((await run(install)).status).toBe('ok'); + await manager.dispose(); + expect(fs.readFileSync(marker, 'utf8')).toBe('released\nreleased\n'); + }); + it('preserves bounded error diagnostics without invoking getters or custom inspectors', async () => { const result = await run(` var invoked = 0; @@ -1221,7 +1244,7 @@ describe('NodeReplKernelManager', () => { fs.mkdirSync(readableRoot); const original = path.join(readableRoot, 'original.png'); fs.writeFileSync(original, Buffer.from(PNG_BASE64, 'base64')); - manager.dispose(); + await manager.dispose(); manager = makeManager({ readableRoots: [readableRoot] }); expect( ( @@ -1558,7 +1581,7 @@ describe('NodeReplKernelManager', () => { async () => { const volatileCwd = path.join(workDir, 'volatile-cwd'); fs.mkdirSync(volatileCwd); - manager.dispose(); + await manager.dispose(); manager = new NodeReplKernelManager({ cwd: volatileCwd, homeDir: os.homedir(), @@ -1607,11 +1630,11 @@ describe('NodeReplKernelManager', () => { await new Promise((resolve) => setTimeout(resolve, 100)); const queuedReset = manager.reset(); - manager.dispose(); + const disposal = manager.dispose(); expect((await pending).status).toBe('cancelled'); await expect(queuedReset).rejects.toThrow(/disposed/); - await new Promise((resolve) => setTimeout(resolve, 100)); + await disposal; expect(() => process.kill(pid, 0)).toThrow(); expect(fs.existsSync(tmpDir)).toBe(false); await expect(run('1;')).rejects.toThrow(/disposed/); diff --git a/packages/node-repl/src/kernel-manager.ts b/packages/node-repl/src/kernel-manager.ts index 380803f67eb..9f93bf1cbe5 100644 --- a/packages/node-repl/src/kernel-manager.ts +++ b/packages/node-repl/src/kernel-manager.ts @@ -31,7 +31,7 @@ const WINDOWS_TASKKILL = `${process.env['SystemRoot'] || 'C:\\Windows'}\\System3 const READY_TIMEOUT_MS = 15_000; const ADD_ROOT_TIMEOUT_MS = 10_000; const CANCEL_GRACE_MS = 5_000; -const TERMINATE_GRACE_MS = 500; +const TERMINATE_GRACE_MS = 2500; const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; const MAX_RAW_TEXT_BYTES = 16 * 1024 * 1024; const MAX_RAW_TEXT_EVENTS = 128 * 1024; @@ -96,6 +96,7 @@ export interface NodeReplExecRequest { } export interface KernelManagerOptions { + preloadCua?: boolean; cwd: string; homeDir: string; tmpRootDir: string; @@ -175,6 +176,7 @@ export class NodeReplKernelManager { private inflight: InflightExec | null = null; private operationChain: Promise<unknown> = Promise.resolve(); private disposed = false; + private disposal: Promise<void> | undefined; private sessionTmpDir: string | null = null; private pendingReady: PendingReady | null = null; private readonly pendingAddRoot = new Map< @@ -265,8 +267,8 @@ export class NodeReplKernelManager { return run; } - dispose(): void { - if (this.disposed) return; + dispose(): Promise<void> { + if (this.disposal) return this.disposal; this.disposed = true; const inflight = this.inflight; if (inflight) { @@ -280,11 +282,13 @@ export class NodeReplKernelManager { this.rejectPending('node_repl task disposed'); const handle = this.kernel; this.kernel = null; - if (handle) { - this.signalProcessTree(handle, true); - } - this.cleanupTmpDir(); - debugLogger.debug('[node-repl] disposed'); + this.disposal = ( + handle ? this.terminateHandle(handle) : Promise.resolve() + ).finally(() => { + this.cleanupTmpDir(); + debugLogger.debug('[node-repl] disposed'); + }); + return this.disposal; } private async execInner( @@ -631,6 +635,7 @@ export class NodeReplKernelManager { handle.toKernel.write( encodeFrame({ type: 'init', + preloadCua: this.options.preloadCua ?? false, generation: handle.generation, cwd: this.options.cwd, homeDir: this.options.homeDir, @@ -995,9 +1000,12 @@ export class NodeReplKernelManager { } catch { // The hard stop below remains authoritative. } - this.signalProcessTree(handle, false); let exited = await this.waitForExit(handle.child, TERMINATE_GRACE_MS); let hardKill = false; + if (!exited) { + this.signalProcessTree(handle, false); + exited = await this.waitForExit(handle.child, 500); + } if (!exited) { hardKill = true; this.signalProcessTree(handle, true); diff --git a/packages/node-repl/src/mcp-server.test.ts b/packages/node-repl/src/mcp-server.test.ts index 7d52c42e5f1..59edb47d43a 100644 --- a/packages/node-repl/src/mcp-server.test.ts +++ b/packages/node-repl/src/mcp-server.test.ts @@ -7,7 +7,7 @@ import os from 'node:os'; import path from 'node:path'; import fs from 'node:fs'; -import { fileURLToPath } from 'node:url'; +import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; @@ -17,15 +17,14 @@ import { resolveContextFromEnv, } from './mcp-server.js'; -async function connected() { - const tmpRootDir = fs.mkdtempSync( - path.join(os.tmpdir(), 'qwen-node-repl-mcp-'), - ); +async function connected(moduleRoots: string[] = []) { + const tmpRootDir = fs.mkdtempSync(path.join(os.tmpdir(), 'qwen-js-mcp-')); const { server, dispose } = createNodeReplMcpServer({ - cwd: process.cwd(), + cwd: tmpRootDir, homeDir: os.homedir(), tmpRootDir, - readableRoots: [process.cwd()], + readableRoots: [tmpRootDir], + moduleRoots, }); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); @@ -43,7 +42,6 @@ async function connected() { }, }; } - function textOf(result: unknown): string { const content = ( result as { content?: Array<{ type: string; text?: string }> } @@ -54,372 +52,249 @@ function textOf(result: unknown): string { .join(''); } -describe('node_repl MCP server', () => { - it('exposes the five focused tools with correct schemas', async () => { +describe('JavaScript MCP server', () => { + it('exposes only js and js_reset with compact entry instructions', async () => { const { client, close } = await connected(); try { const { tools } = await client.listTools(); - const names = tools.map((t) => t.name).sort(); - expect(names).toEqual([ - 'node_repl', - 'node_repl_add_node_module_dir', - 'node_repl_cancel', - 'node_repl_reset', - 'node_repl_wait', + expect(tools.map((tool) => tool.name).sort()).toEqual(['js', 'js_reset']); + const js = tools.find((tool) => tool.name === 'js')!; + expect(Object.keys(js.inputSchema.properties ?? {}).sort()).toEqual([ + 'code', + 'timeout_ms', + 'title', ]); - const repl = tools.find((t) => t.name === 'node_repl'); - expect(repl?.inputSchema['required']).toEqual(['code']); - expect(repl?.inputSchema['additionalProperties']).toBe(false); - // All four declared params must be advertised. + expect(js.inputSchema.required).toEqual(['code']); expect( - Object.keys( - (repl?.inputSchema['properties'] ?? {}) as Record<string, unknown>, - ).sort(), - ).toEqual(['code', 'timeout_ms', 'title', 'yield_time_ms']); - // Arbitrary code execution must be advertised as destructive/open-world. - expect(repl?.annotations).toMatchObject({ - readOnlyHint: false, + (js.inputSchema.properties?.['timeout_ms'] as { default?: number }) + ?.default, + ).toBe(30000); + expect(js.description).toContain('global cua'); + expect(js.annotations).toMatchObject({ destructiveHint: true, - idempotentHint: false, openWorldHint: true, }); - // reset destroys session state but is idempotent and closed-world. - expect( - tools.find((t) => t.name === 'node_repl_reset')?.annotations, - ).toMatchObject({ - destructiveHint: true, - idempotentHint: true, - openWorldHint: false, - }); - // registering a module dir is additive. - expect( - tools.find((t) => t.name === 'node_repl_add_node_module_dir') - ?.annotations, - ).toMatchObject({ destructiveHint: false, idempotentHint: true }); - expect( - tools.find((t) => t.name === 'node_repl_wait')?.annotations, - ).toMatchObject({ readOnlyHint: true, openWorldHint: true }); - expect( - tools.find((t) => t.name === 'node_repl_cancel')?.annotations, - ).toMatchObject({ idempotentHint: false, openWorldHint: true }); - } finally { - await close(); - } - }); - - it('advertises the real package version', async () => { - const { client, close } = await connected(); - try { - const expected = JSON.parse( - fs.readFileSync( - fileURLToPath(new URL('../package.json', import.meta.url)), - 'utf8', - ), - ).version as string; - expect(client.getServerVersion()?.version).toBe(expected); - expect(expected).toMatch(/^\d+\.\d+\.\d+/); - } finally { - await close(); - } - }); - - it('advertises only the compact Node REPL runtime contract', async () => { - const { client, close } = await connected(); - try { expect(client.getInstructions()).toBe(NODE_REPL_INSTRUCTIONS); - expect(NODE_REPL_INSTRUCTIONS.length).toBeLessThan(256); - expect(NODE_REPL_INSTRUCTIONS).not.toMatch( - /Computer Use|cua-sdk|RecreationBench/, - ); - const { tools } = await client.listTools(); - const description = tools.find( - (tool) => tool.name === 'node_repl', - )?.description; - expect(description).toContain('globalThis'); - expect(description).toContain('node_repl_cancel'); - expect(description).toContain('nodeRepl.signal'); - expect(description).toContain('do not commit'); - expect(description).toContain('cancellation does not roll back effects'); - expect(description).toContain( - 'Runtime errors retain completed statement state', + expect(client.getServerVersion()?.version).toBe( + JSON.parse( + fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'), + ).version, ); } finally { await close(); } }); - it('reports a stuck cell timeout and allows a fresh cell', async () => { + it('preloads cua from an unrelated cwd, emits documentation once, and repeats after reset', async () => { + const { client, close } = await connected(); + try { + const first = await client.callTool({ + name: 'js', + arguments: { + code: 'var count = 41; nodeRepl.write(typeof cua.getApp);', + }, + }); + expect(first.isError).not.toBe(true); + expect(textOf(first)).toContain('Computer Use JavaScript runtime'); + expect(textOf(first)).toContain('function'); + const next = await client.callTool({ + name: 'js', + arguments: { code: 'nodeRepl.write(++count);' }, + }); + expect(textOf(next)).toContain('42'); + expect(textOf(next)).not.toContain('Computer Use JavaScript runtime'); + const rewritten = await client.callTool({ + name: 'js', + arguments: { code: 'cua.rewriteDocumentation();' }, + }); + expect(textOf(rewritten)).toContain('Computer Use JavaScript runtime'); + await client.callTool({ name: 'js_reset', arguments: {} }); + const reset = await client.callTool({ + name: 'js', + arguments: { code: 'nodeRepl.write(typeof count);' }, + }); + expect(textOf(reset)).toContain('Computer Use JavaScript runtime'); + expect(textOf(reset)).toContain('undefined'); + } finally { + await close(); + } + }); + + it('returns a terminal result in one call and recovers from timeout without replay', async () => { const { client, close } = await connected(); try { await client.callTool({ - name: 'node_repl', - arguments: { code: 'const beforeHang = 42;' }, + name: 'js', + arguments: { code: 'var count = 0;' }, }); - const running = await client.callTool({ - name: 'node_repl', + const timeout = await client.callTool({ + name: 'js', + arguments: { + code: 'await new Promise(() => {}); count++;', + timeout_ms: 100, + }, + }); + expect(timeout.isError).toBe(true); + expect(textOf(timeout)).toContain('timeout'); + const next = await client.callTool({ + name: 'js', + arguments: { + code: 'await new Promise(r => setTimeout(r, 50)); nodeRepl.write(count);', + }, + }); + expect(next.isError).not.toBe(true); + expect(textOf(next)).toContain('0'); + expect(textOf(next)).not.toContain('still running'); + } finally { + await close(); + } + }); + + it('replaces a stuck kernel and allows a new call', async () => { + const { client, close } = await connected(); + try { + const result = await client.callTool({ + name: 'js', arguments: { code: 'await nodeRepl.signal.waitUntil(new Promise(() => {}));', timeout_ms: 100, - yield_time_ms: 10, }, }); - const match = textOf(running).match( - /cell ([0-9a-f]{8}-[0-9a-f-]{27}) is still running/i, - ); - expect(match?.[1]).toBeTruthy(); - const terminal = await client.callTool({ - name: 'node_repl_wait', - arguments: { cell_id: match![1], yield_time_ms: 7_000 }, - }); - expect(terminal.isError).toBe(true); - expect(textOf(terminal)).toContain('[node_repl timeout]'); - expect(textOf(terminal)).toContain('bindings were lost'); - expect(textOf(terminal)).toContain('verify external state'); - - const recovered = await client.callTool({ - name: 'node_repl', - arguments: { code: 'nodeRepl.write(typeof beforeHang);' }, - }); - expect(recovered.isError).not.toBe(true); - expect(textOf(recovered)).toBe('undefined'); - } finally { - await close(); - } - }, 15_000); - - it('yields, cancels, and retains the persistent kernel state', async () => { - const { client, close } = await connected(); - try { - await client.callTool({ - name: 'node_repl', - arguments: { - code: 'globalThis.marker = {}; const marker = globalThis.marker;', - }, - }); - const running = await client.callTool({ - name: 'node_repl', - arguments: { - code: 'await new Promise((resolve) => setTimeout(resolve, 60_000)); const mustNotCommit = true;', - yield_time_ms: 10, - timeout_ms: 60_000, - }, - }); - const match = textOf(running).match( - /cell ([0-9a-f]{8}-[0-9a-f-]{27}) is still running/i, - ); - expect(match?.[1]).toBeTruthy(); - - const cancelled = await client.callTool({ - name: 'node_repl_cancel', - arguments: { cell_id: match![1], yield_time_ms: 5_000 }, - }); - expect((cancelled as { isError?: boolean }).isError).toBe(true); - expect(textOf(cancelled)).toMatch(/cancelled/i); - - const retained = await client.callTool({ - name: 'node_repl', - arguments: { - code: 'nodeRepl.write(`${marker === globalThis.marker}|${typeof mustNotCommit}`);', - }, - }); - expect(textOf(retained)).toContain('true|undefined'); - } finally { - await close(); - } - }); - - it('waits for the only active cell and rejects conflicting lifecycle calls', async () => { - const { client, close } = await connected(); - try { - const running = await client.callTool({ - name: 'node_repl', - arguments: { - code: 'await new Promise((resolve) => setTimeout(resolve, 100)); nodeRepl.write("finished");', - yield_time_ms: 10, - }, - }); - const match = textOf(running).match( - /cell ([0-9a-f]{8}-[0-9a-f-]{27}) is still running/i, - ); - expect(match?.[1]).toBeTruthy(); - - const conflict = await client.callTool({ - name: 'node_repl', - arguments: { code: 'nodeRepl.write("must-not-run");' }, - }); - expect((conflict as { isError?: boolean }).isError).toBe(true); - expect(textOf(conflict)).toMatch(/already has active cell/i); - - const reset = await client.callTool({ - name: 'node_repl_reset', - arguments: {}, - }); - expect((reset as { isError?: boolean }).isError).toBe(true); - expect(textOf(reset)).toMatch(/active; cancel it/i); - - const completed = await client.callTool({ - name: 'node_repl_wait', - arguments: { cell_id: match![1], yield_time_ms: 5_000 }, - }); - expect((completed as { isError?: boolean }).isError).not.toBe(true); - expect(textOf(completed)).toContain('finished'); - + expect(result.isError).toBe(true); + expect(textOf(result)).toContain('timeout'); const next = await client.callTool({ - name: 'node_repl', - arguments: { code: 'nodeRepl.write("next");' }, + name: 'js', + arguments: { code: 'nodeRepl.write("fresh");' }, }); - expect(textOf(next)).toContain('next'); + expect(next.isError).not.toBe(true); + expect(textOf(next)).toContain('fresh'); + expect(textOf(next)).toContain('Computer Use JavaScript runtime'); } finally { await close(); } - }); + }, 15000); - it('persists bindings across calls and clears them on reset', async () => { - const { client, close } = await connected(); - try { - await client.callTool({ - name: 'node_repl', - arguments: { code: 'const persisted = 7;' }, - }); - const read = await client.callTool({ - name: 'node_repl', - arguments: { code: 'nodeRepl.write(String(persisted * 6));' }, - }); - expect(textOf(read)).toContain('42'); - - await client.callTool({ name: 'node_repl_reset', arguments: {} }); - const after = await client.callTool({ - name: 'node_repl', - arguments: { code: 'nodeRepl.write(typeof persisted);' }, - }); - expect(textOf(after)).toContain('undefined'); - } finally { - await close(); - } - }); - - it('reports a runtime error as isError with the message', async () => { - const { client, close } = await connected(); - try { - const result = await client.callTool({ - name: 'node_repl', - arguments: { code: 'throw new Error("kaboom");' }, - }); - expect((result as { isError?: boolean }).isError).toBe(true); - expect(textOf(result)).toContain('kaboom'); - } finally { - await close(); - } - }); - - it('rejects a timeout_ms above the allowed maximum', async () => { - const { client, close } = await connected(); - try { - const result = await client.callTool({ - name: 'node_repl', - arguments: { code: '1;', timeout_ms: 999_999_999 }, - }); - expect((result as { isError?: boolean }).isError).toBe(true); - expect(textOf(result)).toMatch(/timeout_ms/); - } finally { - await close(); - } - }); - - it('signals success with actionable guidance when a cell prints nothing', async () => { - const { client, close } = await connected(); - try { - const result = await client.callTool({ - name: 'node_repl', - arguments: { code: '1 + 1;' }, - }); - expect((result as { isError?: boolean }).isError).toBeFalsy(); - const text = textOf(result); - expect(text.trim().length).toBeGreaterThan(0); - // Must actually tell the model how to return a value. - expect(text).toMatch(/nodeRepl\.write/); - } finally { - await close(); - } - }); - - it('dispose() kills the kernel child process', async () => { - const tmpRootDir = fs.mkdtempSync( - path.join(os.tmpdir(), 'qwen-node-repl-dispose-'), + it('does not run user code after a timeout during first-call preloading', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'qwen-js-preload-')); + const entered = path.join(dir, 'entered'); + const release = path.join(dir, 'release'); + const runtime = path.join(dir, 'runtime.mjs'); + const loader = path.join(dir, 'loader.mjs'); + const register = path.join(dir, 'register.mjs'); + fs.writeFileSync( + runtime, + `import fs from 'node:fs'; + fs.writeFileSync(${JSON.stringify(entered)}, 'entered'); + while (!fs.existsSync(${JSON.stringify(release)})) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + export function createCua() { return { rewriteDocumentation() {} }; }`, ); - const { server, dispose } = createNodeReplMcpServer({ - cwd: process.cwd(), - homeDir: os.homedir(), - tmpRootDir, - readableRoots: [process.cwd()], - }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - const client = new Client({ name: 'test', version: '0.0.0' }); - await Promise.all([ - client.connect(clientTransport), - server.connect(serverTransport), - ]); + fs.writeFileSync( + loader, + `export function resolve(specifier, context, nextResolve) { + if (specifier === '@qwen-code/cua-sdk/runtime') { + return { url: ${JSON.stringify(pathToFileURL(runtime).href)}, shortCircuit: true }; + } + return nextResolve(specifier, context); + }`, + ); + fs.writeFileSync( + register, + `import { register } from 'node:module'; + register(${JSON.stringify(pathToFileURL(loader).href)}, import.meta.url);`, + ); + const { client, close } = await connected(); + const previousOptions = process.env['NODE_OPTIONS']; + process.env['NODE_OPTIONS'] = [ + previousOptions, + `--import=${pathToFileURL(register).href}`, + ] + .filter(Boolean) + .join(' '); try { - await client.callTool({ - name: 'node_repl', - arguments: { code: 'nodeRepl.write("up");' }, - }); - // Find the live kernel pid, then prove dispose() reaps it. - const pidResult = await client.callTool({ - name: 'node_repl', + const pending = client.callTool({ + name: 'js', arguments: { - code: 'const p = await import("node:os"); nodeRepl.write("alive");', + code: 'globalThis.preloadEffect = true;', + timeout_ms: 100, }, }); - expect(textOf(pidResult)).toContain('alive'); - - dispose(); - await new Promise((r) => setTimeout(r, 500)); - - // After dispose the manager refuses further work. - const after = await client.callTool({ - name: 'node_repl', - arguments: { code: 'nodeRepl.write("should not run");' }, + const deadline = Date.now() + 5000; + while (!fs.existsSync(entered) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(fs.existsSync(entered)).toBe(true); + const result = await pending; + expect(result.isError).toBe(true); + expect(textOf(result)).toContain('timeout'); + fs.writeFileSync(release, 'release'); + const next = await client.callTool({ + name: 'js', + arguments: { + code: 'nodeRepl.write(typeof globalThis.preloadEffect);', + }, }); - expect((after as { isError?: boolean }).isError).toBe(true); - } finally { - await client.close(); - fs.rmSync(tmpRootDir, { recursive: true, force: true }); - } - }); - - it('reports a bad module root as isError instead of throwing', async () => { - const { client, close } = await connected(); - try { - const result = await client.callTool({ - name: 'node_repl_add_node_module_dir', - arguments: { path: '/definitely/not/a/node_modules_dir' }, - }); - expect((result as { isError?: boolean }).isError).toBe(true); - expect(textOf(result)).toMatch(/node_modules/); + expect(next.isError).not.toBe(true); + expect(textOf(next)).toContain('undefined'); } finally { + fs.writeFileSync(release, 'release'); + if (previousOptions === undefined) delete process.env['NODE_OPTIONS']; + else process.env['NODE_OPTIONS'] = previousOptions; await close(); + fs.rmSync(dir, { recursive: true, force: true }); } }); - it('registers a valid node_modules directory', async () => { - const { client, close } = await connected(); - const root = path.join( - fs.mkdtempSync(path.join(os.tmpdir(), 'qwen-node-repl-root-')), - 'node_modules', + it('retains host-configured module roots across reset', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'qwen-js-module-')); + const root = path.join(dir, 'node_modules'); + const pkg = path.join(root, 'host-fixture'); + fs.mkdirSync(pkg, { recursive: true }); + fs.writeFileSync( + path.join(pkg, 'package.json'), + JSON.stringify({ + name: 'host-fixture', + type: 'module', + exports: './index.js', + }), ); - fs.mkdirSync(root); + fs.writeFileSync(path.join(pkg, 'index.js'), 'export const value = 73;'); + const { client, close } = await connected([root]); try { - const result = await client.callTool({ - name: 'node_repl_add_node_module_dir', - arguments: { path: root }, + for (let i = 0; i < 2; i++) { + const result = await client.callTool({ + name: 'js', + arguments: { + code: 'nodeRepl.write((await import("host-fixture")).value);', + }, + }); + expect(result.isError).not.toBe(true); + expect(textOf(result)).toContain('73'); + await client.callTool({ name: 'js_reset', arguments: {} }); + } + } finally { + await close(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reports runtime errors and rejects invalid timeouts', async () => { + const { client, close } = await connected(); + try { + const error = await client.callTool({ + name: 'js', + arguments: { code: 'throw new Error("fixture failure");' }, }); - expect((result as { isError?: boolean }).isError).toBeFalsy(); - expect(textOf(result)).toMatch(/Registered module root/); + expect(error.isError).toBe(true); + expect(textOf(error)).toContain('fixture failure'); + const invalid = await client.callTool({ + name: 'js', + arguments: { code: 'nodeRepl.write(1)', timeout_ms: 600001 }, + }); + expect(invalid.isError).toBe(true); } finally { await close(); - fs.rmSync(path.dirname(root), { recursive: true, force: true }); } }); }); diff --git a/packages/node-repl/src/mcp-server.ts b/packages/node-repl/src/mcp-server.ts index 91482293def..fe739f0f105 100644 --- a/packages/node-repl/src/mcp-server.ts +++ b/packages/node-repl/src/mcp-server.ts @@ -7,7 +7,6 @@ import os from 'node:os'; import path from 'node:path'; import { readFileSync } from 'node:fs'; -import { randomUUID } from 'node:crypto'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; import { z } from 'zod'; @@ -15,14 +14,11 @@ import { NodeReplKernelManager } from './kernel-manager.js'; import { NodeReplSecurityPolicy } from './security-policy.js'; import { convertOutcomeToMcpResult } from './output-adapter.js'; import { createDebugLogger } from './debug-log.js'; -import type { NodeReplExecOutcome } from './kernel-manager.js'; const debugLogger = createDebugLogger('NODE_REPL'); -const DEFAULT_TIMEOUT_MS = 10 * 60_000; +const DEFAULT_TIMEOUT_MS = 30_000; const MAX_TIMEOUT_MS = 10 * 60_000; -const DEFAULT_YIELD_TIME_MS = 10_000; -const MAX_YIELD_TIME_MS = 60_000; const MAX_TITLE_LENGTH = 80; export interface NodeReplServerContext { @@ -34,62 +30,17 @@ export interface NodeReplServerContext { tmpRootDir: string; /** Directories the cell may read via file: URLs / emitImage. */ readableRoots: string[]; + moduleRoots?: string[]; } -const NODE_REPL_DESCRIPTION = [ - 'Execute JavaScript in a session-persistent Node.js REPL. Top-level bindings', - '(const/let/var/function/class) persist across calls in the same session, so', - 'you can build up state incrementally. Top-level await is supported; the', - 'default execution limit is 10 minutes. A call yields with a cell ID when', - 'work is still running; use node_repl_wait or node_repl_cancel with that ID.', - 'Omit yield_time_ms for ordinary calls to use the 10-second wait. Shorter', - 'yields do not make a cell faster and can require an extra wait call; use', - 'them only when you need to regain control before the cell completes.', - '', - 'Output: plain expression results are NOT returned — use nodeRepl.write(value)', - 'for explicit text and nodeRepl.emitImage(png|jpeg|webp) for images; console.*', - 'is also captured. nodeRepl.cwd/homeDir/tmpDir and nodeRepl.getHeapStatus() are', - 'available.', - '', - 'Modules: top-level static import is NOT allowed — use dynamic await import().', - 'Bare packages resolve from the session cwd node_modules and any directory', - 'registered via node_repl_add_node_module_dir; package entrypoints use Node', - 'singleton caching. Local .js/.mjs reload on each execution. Node builtins are', - 'importable except process/node:process. Use', - "(await import('node:module')).createRequire(import.meta.url) for CommonJS or", - 'native addons — note this require() is not subject to the process denial or', - 'the module-root containment that the import path enforces.', - '', - 'Timeout and cancellation normally stop only the active cell and retain the', - 'kernel. If it does not stop within 5 seconds, it is terminated and bindings', - 'are lost. Verify external state before retrying actions. An explicit reset', - 'or a real process crash also discards bindings. For rerunnable', - 'declarations prefer assignment, var, a fresh name, or block scope.', -].join('\n'); - -const NODE_REPL_RUNTIME_RULES = [ - 'This server is one session-persistent JavaScript kernel. Reuse imported', - 'objects and helpers across cells. Top-level const, let, function, and class', - 'bindings persist and cannot be redeclared; update persistent state with', - 'explicit globalThis properties or assignment, and put disposable locals in', - 'an explicit block. Never use the same name for a lexical binding and a', - 'globalThis property because the lexical binding wins. Top-level var may be', - 'redeclared. Use await import() instead of static import.', - '', - 'Return only decision-relevant output with nodeRepl.write(string) and images', - 'with nodeRepl.emitImage(...). A running cell returns an ID; use', - 'node_repl_wait to continue waiting or node_repl_cancel to abort it. Timeout', - 'and cancellation normally retain the kernel and existing bindings but do not commit', - 'new bindings from that cell. Pass nodeRepl.signal to cancellable async APIs', - 'so cancellation waits for their terminal result. If the kernel does not stop', - 'within 5 seconds, it is terminated and bindings are lost; verify external', - 'state before retrying actions. Reset and a real process', - 'crash discard state. Runtime errors retain completed statement state;', - 'cancellation does not roll back effects already performed before it.', -].join('\n'); +const JS_DESCRIPTION = + 'Execute JavaScript in a persistent Node.js session with top-level await. ' + + 'The first call preloads global cua and displays runtime documentation. ' + + 'Use cua.getApp(name) to start Computer Use, or cua.rewriteDocumentation() ' + + 'after context compaction. Use nodeRepl.write(value) for explicit output.'; export const NODE_REPL_INSTRUCTIONS = - "One persistent JavaScript kernel. Follow node_repl's runtime rules."; + 'Use js for persistent JavaScript and follow the runtime documentation returned on the first call.'; /** * Read the package version at runtime so it cannot drift from package.json. @@ -110,60 +61,13 @@ function resolvePackageVersion(): string { return '0.0.0'; } -interface ActiveCell { - id: string; - controller: AbortController; - result: Promise<NodeReplExecOutcome>; - outcome?: NodeReplExecOutcome; - error?: Error; -} - -function activeCellResult(cell: ActiveCell): CallToolResult { - return { - content: [ - { - type: 'text', - text: - `node_repl cell ${cell.id} is still running. ` + - 'Use node_repl_wait or node_repl_cancel with this cell_id.', - }, - ], - }; -} - function cellError(message: string): CallToolResult { return { isError: true, content: [{ type: 'text', text: message }] }; } -async function waitForCell( - cell: ActiveCell, - yieldTimeMs: number, -): Promise<NodeReplExecOutcome | null> { - if (cell.outcome) return cell.outcome; - if (cell.error) throw cell.error; - return new Promise((resolve, reject) => { - const timer = setTimeout(() => resolve(null), yieldTimeMs); - void cell.result.then( - (outcome) => { - clearTimeout(timer); - resolve(outcome); - }, - (error: unknown) => { - clearTimeout(timer); - reject(error instanceof Error ? error : new Error(String(error))); - }, - ); - }); -} - -/** - * Builds an McpServer exposing the five node_repl tools backed by a single - * shared kernel manager. Call `dispose()` on the returned handle (or close the - * server) to tear down the kernel. - */ export function createNodeReplMcpServer(context: NodeReplServerContext): { server: McpServer; - dispose: () => void; + dispose: () => Promise<void>; } { const manager = new NodeReplKernelManager({ cwd: context.cwd, @@ -171,22 +75,25 @@ export function createNodeReplMcpServer(context: NodeReplServerContext): { tmpRootDir: context.tmpRootDir, policy: NodeReplSecurityPolicy.default(), readableRoots: context.readableRoots, + preloadCua: true, }); - let activeCell: ActiveCell | null = null; + const initialized = (async () => { + for (const root of context.moduleRoots ?? []) + await manager.addModuleRoot(root); + })(); + void initialized.catch(() => undefined); + let active: AbortController | undefined; const server = new McpServer( - { - name: 'node-repl', - version: resolvePackageVersion(), - }, + { name: 'node-repl', version: resolvePackageVersion() }, { instructions: NODE_REPL_INSTRUCTIONS }, ); server.registerTool( - 'node_repl', + 'js', { - title: 'Node REPL', - description: `${NODE_REPL_RUNTIME_RULES}\n\n${NODE_REPL_DESCRIPTION}`, + title: 'JavaScript', + description: JS_DESCRIPTION, annotations: { readOnlyHint: false, destructiveHint: true, @@ -200,203 +107,52 @@ export function createNodeReplMcpServer(context: NodeReplServerContext): { .int() .min(1) .max(MAX_TIMEOUT_MS) - .optional() + .default(DEFAULT_TIMEOUT_MS) .describe(`Execution timeout in ms (default ${DEFAULT_TIMEOUT_MS}).`), - yield_time_ms: z - .number() - .int() - .min(1) - .max(MAX_YIELD_TIME_MS) - .optional() - .describe( - `Wait before yielding a running cell ID (default ${DEFAULT_YIELD_TIME_MS}ms). Omit for ordinary calls; a shorter wait does not speed up execution.`, - ), title: z .string() .min(1) .max(MAX_TITLE_LENGTH) .optional() - .describe('Optional short label for this cell.'), + .describe('Short user-facing description of this call.'), }, }, - async ( - { code, timeout_ms, yield_time_ms, title }, - extra, - ): Promise<CallToolResult> => { - if (title) { - debugLogger.debug(`exec cell: ${title}`); - } - if (activeCell) { + async ({ code, timeout_ms, title }, extra): Promise<CallToolResult> => { + if (active) return cellError( - `node_repl already has active cell ${activeCell.id}; wait for or cancel it before starting another cell.`, + 'A js call is already running. Wait for its terminal result.', ); - } + if (title) debugLogger.debug(`exec cell: ${title}`); const controller = new AbortController(); - let result: Promise<NodeReplExecOutcome>; + active = controller; + const onAbort = () => controller.abort(); + extra.signal.addEventListener('abort', onAbort, { once: true }); + if (extra.signal.aborted) controller.abort(); try { - result = manager.exec({ + await initialized; + const outcome = await manager.exec({ code, timeoutMs: timeout_ms ?? DEFAULT_TIMEOUT_MS, signal: controller.signal, }); + return convertOutcomeToMcpResult(outcome); } catch (error) { return cellError( error instanceof Error ? error.message : String(error), ); - } - const cell: ActiveCell = { id: randomUUID(), controller, result }; - void cell.result.then( - (outcome) => { - cell.outcome = outcome; - }, - (error: unknown) => { - cell.error = - error instanceof Error ? error : new Error(String(error)); - }, - ); - activeCell = cell; - const onAbort = () => cell.controller.abort(); - extra?.signal?.addEventListener('abort', onAbort, { once: true }); - try { - let outcome: NodeReplExecOutcome | null; - try { - outcome = await waitForCell( - cell, - yield_time_ms ?? DEFAULT_YIELD_TIME_MS, - ); - } catch (error) { - if (activeCell === cell) activeCell = null; - return cellError( - `node_repl cell ${cell.id} failed: ${error instanceof Error ? error.message : String(error)}`, - ); - } - if (!outcome) return activeCellResult(cell); - if (activeCell === cell) activeCell = null; - return convertOutcomeToMcpResult(outcome); } finally { - extra?.signal?.removeEventListener('abort', onAbort); + extra.signal.removeEventListener('abort', onAbort); + if (active === controller) active = undefined; } }, ); - const cellIdSchema = z - .string() - .uuid() - .describe('Cell ID returned by node_repl.'); - server.registerTool( - 'node_repl_wait', + 'js_reset', { - title: 'Wait for Node REPL cell', + title: 'Reset JavaScript session', description: - 'Wait for the active cell without cancelling it. Returns the final ' + - 'result or the same running cell ID when the yield interval expires.', - annotations: { - readOnlyHint: true, - destructiveHint: false, - idempotentHint: true, - openWorldHint: true, - }, - inputSchema: { - cell_id: cellIdSchema, - yield_time_ms: z - .number() - .int() - .min(1) - .max(MAX_YIELD_TIME_MS) - .optional(), - }, - }, - async ({ cell_id, yield_time_ms }): Promise<CallToolResult> => { - const cell = activeCell; - if (!cell || cell.id !== cell_id) { - return cellError(`node_repl has no active cell ${cell_id}.`); - } - let outcome: NodeReplExecOutcome | null; - try { - outcome = await waitForCell( - cell, - yield_time_ms ?? DEFAULT_YIELD_TIME_MS, - ); - } catch (error) { - if (activeCell === cell) activeCell = null; - return cellError( - `node_repl cell ${cell.id} failed: ${error instanceof Error ? error.message : String(error)}`, - ); - } - if (!outcome) return activeCellResult(cell); - if (activeCell === cell) activeCell = null; - return convertOutcomeToMcpResult(outcome); - }, - ); - - server.registerTool( - 'node_repl_cancel', - { - title: 'Cancel Node REPL cell', - description: - 'Cancel the active cell. Normally retains the kernel and earlier ' + - 'bindings; if it does not stop within 5 seconds, terminate it and ' + - 'discard bindings. Verify external state before retrying actions.', - annotations: { - readOnlyHint: false, - destructiveHint: true, - idempotentHint: false, - openWorldHint: true, - }, - inputSchema: { - cell_id: cellIdSchema, - yield_time_ms: z - .number() - .int() - .min(1) - .max(MAX_YIELD_TIME_MS) - .optional(), - }, - }, - async ({ cell_id, yield_time_ms }): Promise<CallToolResult> => { - const cell = activeCell; - if (!cell || cell.id !== cell_id) { - return cellError(`node_repl has no active cell ${cell_id}.`); - } - cell.controller.abort(); - let outcome: NodeReplExecOutcome | null; - try { - outcome = await waitForCell( - cell, - yield_time_ms ?? DEFAULT_YIELD_TIME_MS, - ); - } catch (error) { - if (activeCell === cell) activeCell = null; - return cellError( - `node_repl cell ${cell.id} failed after cancellation: ${error instanceof Error ? error.message : String(error)}`, - ); - } - if (!outcome) { - return { - content: [ - { - type: 'text', - text: - `Cancellation was requested for node_repl cell ${cell.id}; ` + - 'use node_repl_wait to receive its terminal result.', - }, - ], - }; - } - if (activeCell === cell) activeCell = null; - return convertOutcomeToMcpResult(outcome); - }, - ); - - server.registerTool( - 'node_repl_reset', - { - title: 'Reset Node REPL', - description: - 'Terminate the current Node REPL kernel process and discard all ' + - 'bindings and module state. The next node_repl call lazily starts a ' + - 'fresh kernel.', + 'Discard all bindings and module state. The next js call starts a fresh session and displays documentation again.', annotations: { readOnlyHint: false, destructiveHint: true, @@ -406,92 +162,33 @@ export function createNodeReplMcpServer(context: NodeReplServerContext): { inputSchema: {}, }, async (): Promise<CallToolResult> => { - if (activeCell) { - return cellError( - `node_repl cell ${activeCell.id} is active; cancel it before resetting the kernel.`, - ); - } - const rootCount = manager.getModuleRoots().length; - await manager.reset(); - return { - content: [ - { - type: 'text', - text: - `node_repl kernel reset (generation ${manager.getGeneration()}). ` + - `All bindings were discarded. ${rootCount} registered module ` + - `root(s) were retained.`, - }, - ], - }; - }, - ); - - server.registerTool( - 'node_repl_add_node_module_dir', - { - title: 'Add Node REPL module directory', - description: - 'Register an additional node_modules directory (absolute path, final ' + - 'segment must be node_modules) for bare-package resolution in the Node ' + - 'REPL. The directory need not exist yet. This only widens the ' + - 'bare-package resolution search path; it grants no additional authority.', - annotations: { - readOnlyHint: false, - destructiveHint: false, - idempotentHint: true, - openWorldHint: false, - }, - inputSchema: { - path: z - .string() - .min(1) - .describe('Absolute path to a node_modules directory.'), - }, - }, - async ({ path: modulePath }): Promise<CallToolResult> => { + active?.abort(); try { - const registration = await manager.addModuleRoot(modulePath); + await initialized; + await manager.reset(); return { content: [ { type: 'text', - text: registration.added - ? `Registered module root: ${registration.path}` - : `Module root already registered: ${registration.path}`, + text: 'JavaScript session reset. All bindings were discarded; host module roots were retained.', }, ], }; } catch (error) { - return { - isError: true, - content: [ - { - type: 'text', - text: `Failed to register module root: ${ - error instanceof Error ? error.message : String(error) - }`, - }, - ], - }; + return cellError( + error instanceof Error ? error.message : String(error), + ); } }, ); - const dispose = () => { - try { - activeCell?.controller.abort(); - manager.dispose(); - } catch (error) { - debugLogger.warn( - `[node-repl] manager dispose failed: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } + return { + server, + async dispose() { + active?.abort(); + await manager.dispose(); + }, }; - - return { server, dispose }; } /** @@ -511,5 +208,9 @@ export function resolveContextFromEnv(): NodeReplServerContext { homeDir: os.homedir(), tmpRootDir: path.join(os.tmpdir(), 'qwen-node-repl'), readableRoots: [cwd, ...extraRoots], + moduleRoots: (process.env['QWEN_NODE_REPL_MODULE_DIRS'] ?? '') + .split(path.delimiter) + .map((entry) => entry.trim()) + .filter(Boolean), }; } diff --git a/packages/node-repl/src/node-repl.lines.test.ts b/packages/node-repl/src/node-repl.lines.test.ts index 52f59e68297..859ed723c54 100644 --- a/packages/node-repl/src/node-repl.lines.test.ts +++ b/packages/node-repl/src/node-repl.lines.test.ts @@ -30,8 +30,8 @@ function makeManager(): NodeReplKernelManager { return manager; } -afterEach(() => { - while (managers.length > 0) managers.pop()?.dispose(); +afterEach(async () => { + await Promise.all(managers.splice(0).map((manager) => manager.dispose())); }); /** First `<cellfile>:<line>:<col>` frame from the error stack. */ diff --git a/packages/node-repl/src/node-repl.napi.test.ts b/packages/node-repl/src/node-repl.napi.test.ts index 2705b219101..d038ec39499 100644 --- a/packages/node-repl/src/node-repl.napi.test.ts +++ b/packages/node-repl/src/node-repl.napi.test.ts @@ -107,8 +107,8 @@ beforeAll(async () => { } }, 120_000); -afterAll(() => { - manager?.dispose(); +afterAll(async () => { + await manager?.dispose(); // Clean the build output so it isn't left dirtying the tree. fs.rmSync(path.join(fixtureDir, 'build'), { recursive: true, force: true }); }); diff --git a/packages/node-repl/src/node-repl.reload.test.ts b/packages/node-repl/src/node-repl.reload.test.ts index f5ba168b032..b971a32cb10 100644 --- a/packages/node-repl/src/node-repl.reload.test.ts +++ b/packages/node-repl/src/node-repl.reload.test.ts @@ -38,8 +38,8 @@ function makeTmpDir(): string { return dir; } -afterEach(() => { - while (managers.length > 0) managers.pop()?.dispose(); +afterEach(async () => { + await Promise.all(managers.splice(0).map((manager) => manager.dispose())); while (tmpDirs.length > 0) { const dir = tmpDirs.pop(); if (dir) fs.rmSync(dir, { recursive: true, force: true }); diff --git a/packages/node-repl/src/node-repl.scale.test.ts b/packages/node-repl/src/node-repl.scale.test.ts index b49e47ada2e..98223e18684 100644 --- a/packages/node-repl/src/node-repl.scale.test.ts +++ b/packages/node-repl/src/node-repl.scale.test.ts @@ -29,10 +29,8 @@ function makeManager(): NodeReplKernelManager { return manager; } -afterEach(() => { - while (managers.length > 0) { - managers.pop()?.dispose(); - } +afterEach(async () => { + await Promise.all(managers.splice(0).map((manager) => manager.dispose())); }); function firstText(events: Array<{ type: string; text?: string }>): string { @@ -123,9 +121,7 @@ describe('node_repl scale & isolation', () => { // Dispose and confirm every process tree is gone and temp dirs removed. const tmpDirs = kernels.map((m) => m.getSessionTmpDir()); const livePids = kernels.map((m) => m.getKernelPid()); - while (managers.length > 0) managers.pop()?.dispose(); - // Give SIGKILL a moment to reap. - await new Promise((r) => setTimeout(r, 500)); + await Promise.all(managers.splice(0).map((manager) => manager.dispose())); for (const pid of livePids) { if (typeof pid !== 'number') continue; diff --git a/packages/node-repl/src/node-repl.semantics.test.ts b/packages/node-repl/src/node-repl.semantics.test.ts index 386fb37a8cd..97509b1c2f1 100644 --- a/packages/node-repl/src/node-repl.semantics.test.ts +++ b/packages/node-repl/src/node-repl.semantics.test.ts @@ -25,8 +25,8 @@ function makeManager(): NodeReplKernelManager { return manager; } -afterEach(() => { - while (managers.length > 0) managers.pop()?.dispose(); +afterEach(async () => { + await Promise.all(managers.splice(0).map((manager) => manager.dispose())); }); function textOf(events: Array<{ type: string; text?: string }>): string { @@ -436,7 +436,7 @@ describe('binding semantics through the real kernel', () => { // The kernel child's cwd is `work`; on Windows a directory that is any // process's cwd cannot be removed (EBUSY). Dispose first, then retry // the removal across the child's asynchronous teardown. - manager.dispose(); + await manager.dispose(); fs.rmSync(store, { recursive: true, force: true, diff --git a/packages/node-repl/src/protocol.ts b/packages/node-repl/src/protocol.ts index 59ae5a5b565..e13c0b107d1 100644 --- a/packages/node-repl/src/protocol.ts +++ b/packages/node-repl/src/protocol.ts @@ -22,6 +22,7 @@ export interface NodeReplModuleRoot { export interface InitMessage { type: 'init'; + preloadCua?: boolean; generation: number; cwd: string; homeDir: string; diff --git a/packages/node-repl/src/runtime/kernel.mjs b/packages/node-repl/src/runtime/kernel.mjs index 1a0524a05bd..311470cbaad 100644 --- a/packages/node-repl/src/runtime/kernel.mjs +++ b/packages/node-repl/src/runtime/kernel.mjs @@ -806,6 +806,7 @@ function initialize(message) { cwd: String(message.cwd), homeDir: String(message.homeDir), tmpDir: String(message.tmpDir), + preloadCua: message.preloadCua === true, moduleRoots: [...message.moduleRoots], readableRoots: [...message.readableRoots], }; @@ -973,7 +974,22 @@ async function handleExec(message) { pendingCancelExecId = null; activeExec.cancel(); } - const evaluation = cell.evaluate({ timeout: message.timeoutMs }); + const evaluation = (async () => { + if (config.preloadCua && !untrustedContext.cua) { + const { createCua } = await import('@qwen-code/cua-sdk/runtime'); + if (exec.cancelRequested || activeExec !== exec) + throw new CellCancelledError(); + const runtime = untrustedContext.nodeRepl; + Object.defineProperty(untrustedContext, 'cua', { + value: createCua(runtime), + enumerable: true, + }); + untrustedContext.cua.rewriteDocumentation(); + } + if (exec.cancelRequested || activeExec !== exec) + throw new CellCancelledError(); + await cell.evaluate({ timeout: message.timeoutMs }); + })(); void evaluation.catch(() => undefined); await Promise.race([evaluation, cancellation]); if (activeExec.cancelRequested) throw new CellCancelledError(); @@ -1061,10 +1077,20 @@ function handleAddModuleRoot(message) { }); } -function shutdown(removeSessionTmpDir = false) { +async function shutdown(removeSessionTmpDir = false) { if (shuttingDown) return; shuttingDown = true; + activeExec?.abortController.abort(); clearAllTimers(); + const forceExit = hostSetTimeout(() => process.exit(0), 2000); + forceExit.unref(); + try { + await untrustedContext?.cua?.[Symbol.asyncDispose]?.(); + } catch (error) { + process.stderr.write( + `Computer Use cleanup failed: ${error?.message ?? String(error)}\n`, + ); + } if (removeSessionTmpDir) { try { if (config?.tmpDir) { diff --git a/packages/web-shell/client/desktop-relay/desktop-relay-client.ts b/packages/web-shell/client/desktop-relay/desktop-relay-client.ts index 6db427deeed..b9ecee7f957 100644 --- a/packages/web-shell/client/desktop-relay/desktop-relay-client.ts +++ b/packages/web-shell/client/desktop-relay/desktop-relay-client.ts @@ -22,7 +22,7 @@ export const DESKTOP_RELAY_URL = 'http://127.0.0.1:47821'; * must publish that version before the setup command can be used. */ export const DESKTOP_RELAY_INSTALL_COMMAND = - 'npx -y @qwen-code/node-repl-mcp@0.1.7 desktop-relay install'; + 'npx -y @qwen-code/node-repl-mcp@0.26.0 desktop-relay install'; export type DesktopRelayRemotePhase = | 'connecting' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d07e204234..964b10d8076 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,7 +24,7 @@ overrides: '@agentclientprotocol/sdk>zod': 3.25.76 openapi-typescript>supports-color: 7.2.0 -pnpmfileChecksum: sha256-kM8+keBiS7IPtr0EFrsWpw7AG3oIBDtPmZfAnOLx5ww= +pnpmfileChecksum: sha256-3b7imv8ATmRczQtGoBMZ/l3GHung5L8OdgCSNoEoXe0= importers: @@ -1253,6 +1253,28 @@ importers: specifier: 1.2.0-beta.10 version: 1.2.0-beta.10 + packages/cua-driver/typescript: + dependencies: + '@qwen-code/browser-use': + specifier: workspace:* + version: link:../../browser-use + '@ubjs/core': + specifier: 0.31.0-3 + version: 0.31.0-3 + tar: + specifier: 7.5.22 + version: 7.5.22 + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.20.1 + typescript: + specifier: 5.8.3 + version: 5.8.3 + uniffi-bindgen-react-native: + specifier: 0.31.0-3 + version: 0.31.0-3 + packages/mobile-mcp: dependencies: '@modelcontextprotocol/sdk': @@ -1340,6 +1362,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.30.0 version: 1.30.0(supports-color@7.2.0)(zod@3.25.76) + '@qwen-code/cua-sdk': + specifier: workspace:* + version: link:../cua-driver/typescript tree-sitter-wasms: specifier: ^0.1.13 version: 0.1.13 @@ -5348,6 +5373,9 @@ packages: resolution: {integrity: sha512-jIXhD0eWQ1JA6ln/5Dltyx22UxWNrw0hZmhy2rlv6m6KgF7kplHx3g0fzi09lNmTJQRR91OlemYp3xFnvDK9og==} engines: {node: '>=20.0.0'} + '@ubjs/core@0.31.0-3': + resolution: {integrity: sha512-39XrJgUZ2VVb561sSnkXPhczNoeBsNiSRArecsV0JE7CJq69ajFkcn9/tBAUS2NpgHkLIDU+z6Ks2+1wXnboxg==} + '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} deprecated: Potential CWE-502 - Update to 1.3.1 or higher @@ -10500,6 +10528,10 @@ packages: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} + uniffi-bindgen-react-native@0.31.0-3: + resolution: {integrity: sha512-br7giBJRr/j00rdMXhOZGMGuDWMAVUcxC9O3lco0KRqzEAxUz00+gegPW5tKamvKbrnj7NB682XUCYrcagtxFA==} + hasBin: true + unified@11.0.5: resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} @@ -14823,6 +14855,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@ubjs/core@0.31.0-3': {} + '@ungap/structured-clone@1.3.0': {} '@upsetjs/venn.js@2.0.0': @@ -21211,6 +21245,8 @@ snapshots: unicorn-magic@0.3.0: {} + uniffi-bindgen-react-native@0.31.0-3: {} + unified@11.0.5: dependencies: '@types/unist': 3.0.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a639ce88b55..74864561477 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -3,6 +3,7 @@ # channel join the pnpm layout silently while npm still ignores it. packages: - 'packages/*' + - 'packages/cua-driver/typescript' - 'packages/channels/base' - 'packages/channels/dingtalk' - 'packages/channels/dws' diff --git a/scripts/copy-browser-use-assets.js b/scripts/copy-browser-use-assets.js index 8c60fafbd76..b70be252058 100644 --- a/scripts/copy-browser-use-assets.js +++ b/scripts/copy-browser-use-assets.js @@ -7,17 +7,9 @@ import fs from 'node:fs'; import { createRequire } from 'node:module'; import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -/** - * Copy the built Browser Use runtime (SDK bundle, Native Host, its setup - * script, NOTICE and the pinned playwright-core) into `skillDir/runtime`. - * - * Two callers: `copy_bundle_assets.js` stages it into `dist/bundled` for the - * shipped CLI, and `scripts/dev.js` (or running this file directly) stages it - * next to the source SKILL.md for `npm run dev`. No build step writes into the - * source tree; the dev copy is git-ignored. - */ +// Assemble an isolated Browser SDK fixture for dependency and kernel tests. +// Production loads the published package through the preloaded cua runtime. export function copyBrowserUseAssets(root, skillDir) { const browserUseDir = path.join(root, 'packages', 'browser-use'); const runtimeFiles = [ @@ -69,15 +61,3 @@ export function copyBrowserUseAssets(root, skillDir) { { recursive: true }, ); } - -if ( - process.argv[1] && - fs.existsSync(process.argv[1]) && - fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) -) { - const root = fileURLToPath(new URL('..', import.meta.url)); - copyBrowserUseAssets( - root, - path.join(root, 'packages/core/src/skills/bundled/browser-use'), - ); -} diff --git a/scripts/copy_bundle_assets.js b/scripts/copy_bundle_assets.js index cf394f62a17..db7b7c85170 100644 --- a/scripts/copy_bundle_assets.js +++ b/scripts/copy_bundle_assets.js @@ -39,7 +39,6 @@ import { createRequire } from 'node:module'; import { fileURLToPath } from 'node:url'; import { glob } from 'glob'; import fs from 'node:fs'; -import { copyBrowserUseAssets } from './copy-browser-use-assets.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const defaultRoot = join(__dirname, '..'); @@ -553,11 +552,9 @@ export function copyBundleAssets({ root = defaultRoot } = {}) { skipEntry: (entry, sourcePath) => isBundledSkillTestFile(entry) || entry === 'DESIGN.md' || - sourcePath === join(bundledSkillsDir, 'browser-use', 'runtime'), + sourcePath === join(bundledSkillsDir, 'browser-use') || + sourcePath === join(bundledSkillsDir, 'computer-use'), }); - if (existsSync(join(bundledSkillsDir, 'browser-use', 'SKILL.md'))) { - copyBrowserUseAssets(root, join(destBundledDir, 'browser-use')); - } console.log('Copied bundled skills to dist/bundled/'); } else { console.warn( diff --git a/scripts/copy_files.js b/scripts/copy_files.js index fb621bd642a..8c7e4f1c200 100644 --- a/scripts/copy_files.js +++ b/scripts/copy_files.js @@ -23,7 +23,6 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { copyBrowserUseAssets } from './copy-browser-use-assets.js'; const extensionsToCopy = ['.md', '.json', '.sb']; @@ -51,7 +50,12 @@ function copyFilesRecursive(source, target, rootSourceDir) { const normalizedPath = relativePath.replace(/\\/g, '/'); if (item.isDirectory()) { - if (normalizedPath === 'skills/bundled/browser-use/runtime') continue; + if ( + ['skills/bundled/browser-use', 'skills/bundled/computer-use'].includes( + normalizedPath, + ) + ) + continue; copyFilesRecursive(sourcePath, targetPath, rootSourceDir); } else { const ext = path.extname(item.name); @@ -78,18 +82,15 @@ export function copyFiles({ root = process.cwd() } = {}) { process.exit(1); } + for (const name of ['browser-use', 'computer-use']) { + fs.rmSync(path.join(targetDir, 'skills/bundled', name), { + recursive: true, + force: true, + }); + } copyFilesRecursive(sourceDir, targetDir, sourceDir); const packageName = path.basename(root); - if ( - packageName === 'core' && - fs.existsSync(path.join(sourceDir, 'skills/bundled/browser-use/SKILL.md')) - ) { - copyBrowserUseAssets( - path.resolve(root, '../..'), - path.join(targetDir, 'skills/bundled/browser-use'), - ); - } // Copy example extensions into the bundle. if (packageName === 'cli') { const examplesSource = path.join( diff --git a/scripts/dev.js b/scripts/dev.js index d4d4b46c95a..0c964b7d67b 100755 --- a/scripts/dev.js +++ b/scripts/dev.js @@ -29,7 +29,6 @@ import { readdirSync, } from 'node:fs'; import { tmpdir, platform } from 'node:os'; -import { copyBrowserUseAssets } from './copy-browser-use-assets.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const root = join(__dirname, '..'); @@ -60,31 +59,6 @@ if (existsSync(userDocsTarget) && !existsSync(qcHelperDocsLink)) { } } -// Stage the Browser Use skill runtime next to its SKILL.md for dev mode. The -// bundle step does the same into dist/bundled; only dev mode reads the skill -// from the source tree, so this copy exists nowhere else (it is git-ignored, -// and no build step writes it). It needs the built browser-use package. The -// informational passthrough flags never reach a skill, so they skip the copy -// and stay a fast path. -const passthroughArgs = process.argv.slice(2); -const informationalOnly = passthroughArgs.some((argument) => - ['--version', '-v', '--help', '-h'].includes(argument), -); -if (!informationalOnly) { - try { - copyBrowserUseAssets( - root, - join(root, 'packages', 'core', 'src', 'skills', 'bundled', 'browser-use'), - ); - } catch (error) { - console.warn( - `Browser Use skill runtime not staged for dev mode: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } -} - // Entry point for the CLI const cliEntry = join(cliPackageDir, 'index.ts'); diff --git a/scripts/release-packages.mjs b/scripts/release-packages.mjs index 4831200176e..3d805066799 100644 --- a/scripts/release-packages.mjs +++ b/scripts/release-packages.mjs @@ -12,6 +12,8 @@ import { getWorkspacePackageJsonPaths } from './workspaces.js'; export const INDEPENDENT_PACKAGES = [ '@qwen-code/sdk', '@qwen-code/mobile-mcp', + '@qwen-code/browser-use', + '@qwen-code/cua-sdk', '@qwen-code/node-repl-mcp', '@qwen-code/qwen-live', ]; diff --git a/scripts/tests/browser-use-assets.test.js b/scripts/tests/browser-use-assets.test.js index a10ab13f2a5..b3d942138c1 100644 --- a/scripts/tests/browser-use-assets.test.js +++ b/scripts/tests/browser-use-assets.test.js @@ -5,7 +5,6 @@ */ import fs from 'node:fs'; -import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -23,7 +22,7 @@ const runtimeFiles = [ ]; const playwrightPath = 'packages/browser-use/node_modules/playwright-core'; -describe('browser-use builtin resources', () => { +describe('isolated browser runtime fixtures', () => { let root; beforeEach(() => { @@ -66,65 +65,77 @@ describe('browser-use builtin resources', () => { fs.rmSync(root, { force: true, recursive: true }); }); - it.each([ - ['dev', skillPath], - ['core', 'packages/core/dist/src/skills/bundled/browser-use'], - ['bundle', 'dist/bundled/browser-use'], - ])( - 'stages the complete runtime for %s and removes stale generated files', - (mode, target) => { - write(`${target}/runtime/stale.json`, '{}\n'); - if (mode === 'dev') { - copyBrowserUseAssets(root, path.join(root, target)); - } else { - write(`${skillPath}/runtime/source-only.json`, '{}\n'); - const readdirSync = fs.readdirSync; - vi.spyOn(fs, 'readdirSync').mockImplementation((directory, ...args) => { - if (directory === path.join(root, skillPath, 'runtime')) { - throw new Error( - 'General asset copying must not walk generated runtime', - ); - } - return readdirSync(directory, ...args); - }); - if (mode === 'core') - copyFiles({ root: path.join(root, 'packages/core') }); - else copyBundleAssets({ root }); - } + it('copies an isolated runtime fixture and removes stale generated files', () => { + const target = 'isolated-fixture'; + write(`${target}/runtime/stale.json`, '{}\n'); + copyBrowserUseAssets(root, path.join(root, target)); - const runtimeDir = path.join(root, target, 'runtime'); - expect(fs.readdirSync(runtimeDir).sort()).toEqual([ - 'NOTICE', - 'index.js', - 'native-host.js', - 'node_modules', - 'scripts', - ]); - expect(fs.readdirSync(path.join(runtimeDir, 'scripts'))).toEqual([ - 'native-host-setup.js', - ]); - for (const file of runtimeFiles) { - expect(fs.readFileSync(path.join(runtimeDir, file), 'utf8')).toBe( - `// ${file}\n`, - ); - } - for (const file of [ - 'package.json', - 'lib/server/injected.js', - 'LICENSE', - 'NOTICE', - 'types/types.d.ts', - 'bin/install.sh', - ]) { - expect( - fs.readFileSync( - path.join(runtimeDir, 'node_modules/playwright-core', file), - ), - ).toEqual(fs.readFileSync(path.join(root, playwrightPath, file))); - } - expect(fs.readFileSync(path.join(runtimeDir, 'NOTICE'), 'utf8')).toBe( - 'Browser use notice\n', + const runtimeDir = path.join(root, target, 'runtime'); + expect(fs.readdirSync(runtimeDir).sort()).toEqual([ + 'NOTICE', + 'index.js', + 'native-host.js', + 'node_modules', + 'scripts', + ]); + expect(fs.readdirSync(path.join(runtimeDir, 'scripts'))).toEqual([ + 'native-host-setup.js', + ]); + for (const file of runtimeFiles) { + expect(fs.readFileSync(path.join(runtimeDir, file), 'utf8')).toBe( + `// ${file}\n`, ); + } + for (const file of [ + 'package.json', + 'lib/server/injected.js', + 'LICENSE', + 'NOTICE', + 'types/types.d.ts', + 'bin/install.sh', + ]) { + expect( + fs.readFileSync( + path.join(runtimeDir, 'node_modules/playwright-core', file), + ), + ).toEqual(fs.readFileSync(path.join(root, playwrightPath, file))); + } + expect(fs.readFileSync(path.join(runtimeDir, 'NOTICE'), 'utf8')).toBe( + 'Browser use notice\n', + ); + }); + + it.each(['core', 'bundle'])( + 'excludes retired skills from %s even with stale source and output assets', + (mode) => { + const target = + mode === 'core' + ? 'packages/core/dist/src/skills/bundled' + : 'dist/bundled'; + for (const name of ['browser-use', 'computer-use']) { + write( + `packages/core/src/skills/bundled/${name}/SKILL.md`, + '# Retired\n', + ); + write( + `packages/core/src/skills/bundled/${name}/runtime/stale.json`, + '{}\n', + ); + write(`${target}/${name}/SKILL.md`, '# Stale output\n'); + } + write('packages/core/src/skills/bundled/current/SKILL.md', '# Current\n'); + fs.rmSync(path.join(root, 'packages/browser-use/dist'), { + recursive: true, + }); + if (mode === 'core') + copyFiles({ root: path.join(root, 'packages/core') }); + else copyBundleAssets({ root }); + for (const name of ['browser-use', 'computer-use']) { + expect(fs.existsSync(path.join(root, target, name))).toBe(false); + } + expect( + fs.readFileSync(path.join(root, target, 'current/SKILL.md'), 'utf8'), + ).toBe('# Current\n'); }, ); @@ -139,37 +150,6 @@ describe('browser-use builtin resources', () => { }, ); - it('stages the dev runtime when the copy script runs from another directory', () => { - const scriptPath = path.join(root, 'scripts/copy-browser-use-assets.js'); - fs.mkdirSync(path.dirname(scriptPath), { recursive: true }); - fs.copyFileSync( - new URL('../copy-browser-use-assets.js', import.meta.url), - scriptPath, - ); - write('package.json', '{"type":"module"}\n'); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: tmpdir(), - encoding: 'utf8', - }); - - expect(result.status, result.stderr).toBe(0); - for (const file of runtimeFiles) { - expect( - fs.readFileSync(path.join(root, skillPath, 'runtime', file), 'utf8'), - ).toBe(`// ${file}\n`); - } - expect( - fs.existsSync( - path.join( - root, - skillPath, - 'runtime/node_modules/playwright-core/LICENSE', - ), - ), - ).toBe(true); - }); - it('uses the SDK-local pinned Playwright instead of the hoisted version', () => { write( 'node_modules/playwright-core/package.json', @@ -208,17 +188,6 @@ describe('browser-use builtin resources', () => { ).toThrow('Browser-use requires playwright-core 1.2.3, but resolved 9.9.9'); }); - it('does not silently omit resources when the builtin skill exists', () => { - fs.rmSync(path.join(root, 'packages/browser-use/dist/index.js')); - - expect(() => copyBundleAssets({ root })).toThrow( - 'Browser-use runtime not found', - ); - expect(() => copyFiles({ root: path.join(root, 'packages/core') })).toThrow( - 'Browser-use runtime not found', - ); - }); - it('declares the browser-use build dependency for recursive scheduling', () => { const core = JSON.parse( fs.readFileSync( diff --git a/scripts/tests/cua-driver-release-workflow.test.js b/scripts/tests/cua-driver-release-workflow.test.js index feeab65359b..87f30ab7f71 100644 --- a/scripts/tests/cua-driver-release-workflow.test.js +++ b/scripts/tests/cua-driver-release-workflow.test.js @@ -24,7 +24,7 @@ const powershellInstaller = readFileSync( describe('CUA SDK release workflow', () => { it('fails closed when a driver production dispatch disables notarization', () => { expect(workflow).toContain( - '"${{ inputs.dry_run }}" != "true" && "${{ inputs.node_repl_only }}" != "true" && "${{ inputs.notarize }}" != "true"', + '"${{ inputs.dry_run }}" != "true" && "${{ inputs.notarize }}" != "true"', ); }); @@ -41,9 +41,7 @@ describe('CUA SDK release workflow', () => { it('smokes the high-level ComputerUse runtime from a clean package', () => { expect(packageVerifier).toContain('await ComputerUse.create('); - expect(packageVerifier).toContain( - 'await computer.supportsObservationRevision()', - ); + expect(packageVerifier).toContain('await computer.appCall("click"'); expect(workflow).toContain('await ComputerUse.create('); expect(workflow).toContain('await computer.close()'); }); diff --git a/scripts/tests/dev.test.js b/scripts/tests/dev.test.js index 627372c12ee..45303846ce6 100644 --- a/scripts/tests/dev.test.js +++ b/scripts/tests/dev.test.js @@ -103,8 +103,8 @@ describe('scripts/dev.js launcher', () => { expect(options).toEqual(expect.objectContaining({ shell: false })); }); - it.each(['--version', '--help'])( - 'launches %s without building or staging Browser Use', + it.each(['--version', '--help', '-p'])( + 'launches %s without building or staging retired skill assets', async (flag) => { process.argv = ['node', 'scripts/dev.js', flag]; diff --git a/scripts/tests/package-assets.test.js b/scripts/tests/package-assets.test.js index d8a09930edf..93189472f8f 100644 --- a/scripts/tests/package-assets.test.js +++ b/scripts/tests/package-assets.test.js @@ -797,7 +797,7 @@ describe('package asset scripts', () => { }, ); - it('copies Computer Use platform references to both CLI and core distributions', () => { + it('copies bundled skill references to both CLI and core distributions', () => { const rootDir = createFixtureRoot(); const coreDir = path.join(rootDir, 'packages', 'core'); const resources = [ @@ -808,7 +808,7 @@ describe('package asset scripts', () => { for (const resource of resources) { writeFile( rootDir, - `packages/core/src/skills/bundled/computer-use/${resource}`, + `packages/core/src/skills/bundled/fixture-skill/${resource}`, resource, ); } @@ -818,13 +818,13 @@ describe('package asset scripts', () => { for (const resource of resources) { expect( readFileSync( - path.join(rootDir, 'dist/bundled/computer-use', resource), + path.join(rootDir, 'dist/bundled/fixture-skill', resource), 'utf8', ), ).toBe(resource); expect( readFileSync( - path.join(coreDir, 'dist/src/skills/bundled/computer-use', resource), + path.join(coreDir, 'dist/src/skills/bundled/fixture-skill', resource), 'utf8', ), ).toBe(resource); diff --git a/scripts/tests/package-scripts.test.js b/scripts/tests/package-scripts.test.js index 9ba4ddf1421..41d05aff4d1 100644 --- a/scripts/tests/package-scripts.test.js +++ b/scripts/tests/package-scripts.test.js @@ -1915,6 +1915,11 @@ describe('package scripts', () => { 'release-sdk', 'Publish @qwen-code/sdk', ], + [ + '.github/workflows/cd-cua-driver.yml', + 'publish-browser', + 'Publish immutable Browser SDK tarball', + ], [ '.github/workflows/cd-cua-driver.yml', 'publish-sdk', diff --git a/scripts/tests/release-versioning.test.js b/scripts/tests/release-versioning.test.js index 5b542d912d2..877329bb17a 100644 --- a/scripts/tests/release-versioning.test.js +++ b/scripts/tests/release-versioning.test.js @@ -116,6 +116,15 @@ it.each(['patch', '99.0.0-preview.1'])( } expect(after.private, path).toBe(before.private); } + for (const path of [ + 'packages/browser-use/package.json', + 'packages/cua-driver/typescript/package.json', + 'packages/node-repl/package.json', + ]) { + expect(read(path), path).toEqual( + JSON.parse(readFileSync(join(root, path), 'utf8')), + ); + } expect(read(extensionPath).version).toBe(expected); for (const path of ['package.json', 'packages/cli/package.json']) { expect(read(path).config.sandboxImageUri).toBe( diff --git a/scripts/tests/release-workflow.test.js b/scripts/tests/release-workflow.test.js index ac84e076c22..93745dd77a3 100644 --- a/scripts/tests/release-workflow.test.js +++ b/scripts/tests/release-workflow.test.js @@ -58,15 +58,17 @@ const cuaReleaseWorkflow = readFileSync( '.github/workflows/cd-cua-driver.yml', 'utf8', ); +const cuaReleaseJobs = parse(cuaReleaseWorkflow).jobs; const nodeReplPackage = JSON.parse( readFileSync('packages/node-repl/package.json', 'utf8'), ); const cuaSdkPackage = JSON.parse( readFileSync('packages/cua-driver/typescript/package.json', 'utf8'), ); -const cuaSdkPackageLock = JSON.parse( - readFileSync('packages/cua-driver/typescript/package-lock.json', 'utf8'), +const browserUsePackage = JSON.parse( + readFileSync('packages/browser-use/package.json', 'utf8'), ); +const workspaceLock = parse(readFileSync('pnpm-lock.yaml', 'utf8')); const cuaInstallScript = readFileSync( 'packages/cua-driver/scripts/install.sh', 'utf8', @@ -97,17 +99,35 @@ const liveHostOssWorkflow = readFileSync( ); describe('CUA release workflow', () => { - it('keeps the Node REPL package independently versioned', () => { + it('excludes the Computer Use distribution from the CLI release', () => { + for (const name of [ + '@qwen-code/browser-use', + '@qwen-code/cua-sdk', + '@qwen-code/node-repl-mcp', + ]) { + expect(RELEASE_WORKSPACES).not.toContain(name); + } + }); + + it('keeps the Computer Use distribution on one version', () => { expect(nodeReplPackage.name).toBe('@qwen-code/node-repl-mcp'); - expect(nodeReplPackage.version).toBe('0.1.7'); + expect(nodeReplPackage.version).toBe(cuaSdkPackage.version); + expect(browserUsePackage.version).toBe(cuaSdkPackage.version); + expect(nodeReplPackage.dependencies['@qwen-code/cua-sdk']).toBe( + cuaSdkPackage.version, + ); + expect(cuaSdkPackage.dependencies['@qwen-code/browser-use']).toBe( + cuaSdkPackage.version, + ); expect(cuaReleaseWorkflow).toContain( "node_repl_version: '${{ steps.release.outputs.node_repl_version }}'", ); expect(cuaReleaseWorkflow).not.toContain( 'NODE_REPL_VERSION does not match release version', ); - expect(cuaSdkPackageLock.version).toBe(cuaSdkPackage.version); - expect(cuaSdkPackageLock.packages[''].version).toBe(cuaSdkPackage.version); + expect( + workspaceLock.importers['packages/cua-driver/typescript'], + ).toBeDefined(); }); it('dry-runs and clean-installs the packed Node REPL MCP server', () => { @@ -117,9 +137,15 @@ describe('CUA release workflow', () => { }); it('publishes the verified Node REPL tarball immutably with provenance', () => { - expect(cuaReleaseWorkflow).toMatch( - /publish-node-repl:[\s\S]*?needs: \['validate-version', 'verify-node-repl-package', 'release'\][\s\S]*?npm view "@qwen-code\/node-repl-mcp@\$\{VERSION\}" dist\.integrity[\s\S]*?npm publish "\$TARBALL" --provenance --access public --tag "\$NPM_TAG"[\s\S]*?Verify npm registry integrity/, + const job = cuaReleaseJobs['publish-node-repl']; + expect(job.needs).toContain('publish-sdk'); + const commands = job.steps.map((step) => step.run ?? '').join('\n'); + expect(commands).toMatch( + /npm view "@qwen-code\/node-repl-mcp@\$\{VERSION\}" dist\.integrity[\s\S]*?npm publish "\$TARBALL" --provenance --access public --tag "\$NPM_TAG"/, ); + expect( + job.steps.some((step) => step.name === 'Verify npm registry integrity'), + ).toBe(true); }); it('keeps installer version changes in the feature PR', () => { @@ -150,32 +176,30 @@ describe('CUA release workflow', () => { ); }); - it('pins exact Computer Use package versions across the skill and user guide', () => { + it('pins exact runtime registration and dependency versions', () => { expect(computerUseGuide).toContain( `@qwen-code/node-repl-mcp@${nodeReplPackage.version}`, ); expect(computerUseGuide).toContain( - `@qwen-code/cua-sdk@${cuaSdkPackage.version}`, + 'installs the matching SDK and browser SDK as dependencies', ); - expect(cuaReleaseWorkflow).toContain('SKILL_NODE_REPL_VERSION=$(sed -nE'); + expect(cuaReleaseWorkflow).not.toContain('SKILL_NODE_REPL_VERSION='); expect(cuaReleaseWorkflow).toContain( 'USER_GUIDE_NODE_REPL_VERSION=$(sed -nE', ); - expect(cuaReleaseWorkflow).toContain('SKILL_SDK_VERSION=$(sed -nE'); - expect(cuaReleaseWorkflow).toContain('USER_GUIDE_SDK_VERSION=$(sed -nE'); + expect(cuaReleaseWorkflow).not.toContain('SKILL_SDK_VERSION='); + expect(cuaReleaseWorkflow).toContain('SDK_BROWSER_VERSION='); + expect(cuaReleaseWorkflow).not.toContain('USER_GUIDE_SDK_VERSION='); expect(cuaReleaseWorkflow).not.toContain( 'grep -Fq "@qwen-code/node-repl-mcp@', ); expect(cuaReleaseWorkflow).not.toContain('grep -Fq "@qwen-code/cua-sdk@'); }); - it('bootstraps only Node REPL without replacing an existing CUA release', () => { - expect(cuaReleaseWorkflow).toMatch( - /node_repl_only:[\s\S]*?type: 'boolean'[\s\S]*?default: false/, - ); - expect(cuaReleaseWorkflow).toMatch( - /publish-node-repl:[\s\S]*?needs\.release\.result == 'success'[\s\S]*?inputs\.node_repl_only == true[\s\S]*?inputs\.dry_run == false/, - ); + it('releases dependencies before the same-version runtime', () => { + expect(cuaReleaseWorkflow).not.toContain('node_repl_only'); + expect(cuaReleaseJobs['publish-sdk'].needs).toContain('publish-browser'); + expect(cuaReleaseJobs['publish-node-repl'].needs).toContain('publish-sdk'); expect(cuaReleaseWorkflow).toContain( 'A production dispatch must run from protected main', );