mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-11-07 06:19:43 -10:00
feat(cua): streamline computer use and port native reliability fixes (#14846)
* feat(cua): preload the JavaScript computer use runtime * refactor(cua): remove the native Wayland backend * feat(cua): move app computer use into the driver * feat(cua): preserve background delivery and port platform fixes * feat(cua): add explicit desktop input and browser runtime facade * feat(cua): package the unified 0.26.0 computer use runtime * fix(cua): finalize cross-platform native reliability * fix(cua): isolate runtime releases from CLI versioning * fix(cua): keep Electron out of root workspace installs * fix(cua): synchronize consumers and async cleanup tests * fix(browser): migrate smoke to preloaded cua runtime * docs(cua): document MCP setup for Qwen Code Claude Code and Codex
This commit is contained in:
303 files changed
+28408
-27909
No files matched your search
@@ -17,7 +17,7 @@
|
|||||||
3480 audio-capture-prebuilds.yml
|
3480 audio-capture-prebuilds.yml
|
||||||
9023 auto-minimize-spam.yml
|
9023 auto-minimize-spam.yml
|
||||||
9256 build-and-publish-image.yml
|
9256 build-and-publish-image.yml
|
||||||
50773 cd-cua-driver.yml
|
55276 cd-cua-driver.yml
|
||||||
2222 cd-mobile-mcp.yml
|
2222 cd-mobile-mcp.yml
|
||||||
147136 ci.yml
|
147136 ci.yml
|
||||||
4193 codeql.yml
|
4193 codeql.yml
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ name: 'CD: Qwen CUA Driver'
|
|||||||
# Linux : cua-driver-rs-<v>-linux-{x86_64,arm64}.tar.gz + *-binary.tar.gz
|
# Linux : cua-driver-rs-<v>-linux-{x86_64,arm64}.tar.gz + *-binary.tar.gz
|
||||||
# Windows: cua-driver-rs-<v>-windows-{x86_64,arm64}.zip + *-binary.zip
|
# Windows: cua-driver-rs-<v>-windows-{x86_64,arm64}.zip + *-binary.zip
|
||||||
# npm : @qwen-code/cua-sdk downloads and verifies the matching *-binary
|
# npm : @qwen-code/cua-sdk downloads and verifies the matching *-binary
|
||||||
# archive above. @qwen-code/node-repl-mcp is independently
|
# archive above. SDK, browser SDK and Node REPL share one version
|
||||||
# versioned and ships from the same verified workflow.
|
# and ship from the same verified workflow.
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
tags: ['cua-driver-rs-v*']
|
tags: ['cua-driver-rs-v*']
|
||||||
@@ -30,7 +30,7 @@ on:
|
|||||||
version:
|
version:
|
||||||
description: 'Version to release (without v prefix)'
|
description: 'Version to release (without v prefix)'
|
||||||
required: true
|
required: true
|
||||||
default: '0.20.11'
|
default: '0.26.0'
|
||||||
notarize:
|
notarize:
|
||||||
description:
|
description:
|
||||||
'Codesign + notarize the macOS artifacts (false to skip during
|
'Codesign + notarize the macOS artifacts (false to skip during
|
||||||
@@ -40,18 +40,11 @@ on:
|
|||||||
default: true
|
default: true
|
||||||
dry_run:
|
dry_run:
|
||||||
description:
|
description:
|
||||||
'Build release assets and clean-install both npm packages without
|
'Build release assets and clean-install all three npm packages without
|
||||||
creating a GitHub Release, npm version, or tag.'
|
creating a GitHub Release, npm version, or tag.'
|
||||||
required: false
|
required: false
|
||||||
type: 'boolean'
|
type: 'boolean'
|
||||||
default: true
|
default: true
|
||||||
node_repl_only:
|
|
||||||
description:
|
|
||||||
'Only verify/publish @qwen-code/node-repl-mcp; do not build or release
|
|
||||||
cua-driver assets.'
|
|
||||||
required: false
|
|
||||||
type: 'boolean'
|
|
||||||
default: false
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: 'write'
|
contents: 'write'
|
||||||
jobs:
|
jobs:
|
||||||
@@ -109,8 +102,6 @@ jobs:
|
|||||||
# shellcheck disable=SC2016
|
# shellcheck disable=SC2016
|
||||||
README_PS1_VERSION=$(sed -nE 's/^\$env:CUA_DRIVER_RS_VERSION = "([^"]+)"$/\1/p' packages/cua-driver/README.md)
|
README_PS1_VERSION=$(sed -nE 's/^\$env:CUA_DRIVER_RS_VERSION = "([^"]+)"$/\1/p' packages/cua-driver/README.md)
|
||||||
README_EXPECTED_VERSION=$(sed -nE 's/^Expected: qwen-cua-driver ([0-9A-Za-z.-]+)\.$/\1/p' packages/cua-driver/README.md)
|
README_EXPECTED_VERSION=$(sed -nE 's/^Expected: qwen-cua-driver ([0-9A-Za-z.-]+)\.$/\1/p' packages/cua-driver/README.md)
|
||||||
SKILL_SDK_VERSION=$(sed -nE 's/^npm install --no-save --package-lock=false @qwen-code\/cua-sdk@([^ ]+)$/\1/p' packages/core/src/skills/bundled/computer-use/SKILL.md | sort -u)
|
|
||||||
USER_GUIDE_SDK_VERSION=$(sed -nE 's/^npm install --no-save --package-lock=false @qwen-code\/cua-sdk@([^ ]+)$/\1/p' docs/users/features/computer-use.md)
|
|
||||||
for PAIR in \
|
for PAIR in \
|
||||||
"shell installer:$INSTALL_SH_VERSION" \
|
"shell installer:$INSTALL_SH_VERSION" \
|
||||||
"PowerShell installer:$INSTALL_PS1_VERSION" \
|
"PowerShell installer:$INSTALL_PS1_VERSION" \
|
||||||
@@ -119,9 +110,7 @@ jobs:
|
|||||||
"driver skill:$DRIVER_SKILL_VERSION" \
|
"driver skill:$DRIVER_SKILL_VERSION" \
|
||||||
"README shell command:$README_SH_VERSION" \
|
"README shell command:$README_SH_VERSION" \
|
||||||
"README PowerShell command:$README_PS1_VERSION" \
|
"README PowerShell command:$README_PS1_VERSION" \
|
||||||
"README expected output:$README_EXPECTED_VERSION" \
|
"README expected output:$README_EXPECTED_VERSION"; do
|
||||||
"Computer Use skill SDK:$SKILL_SDK_VERSION" \
|
|
||||||
"Computer Use user guide SDK:$USER_GUIDE_SDK_VERSION"; do
|
|
||||||
LABEL=${PAIR%%:*}
|
LABEL=${PAIR%%:*}
|
||||||
VALUE=${PAIR#*:}
|
VALUE=${PAIR#*:}
|
||||||
if [[ "$VALUE" != "$VERSION" ]]; then
|
if [[ "$VALUE" != "$VERSION" ]]; then
|
||||||
@@ -140,26 +129,26 @@ jobs:
|
|||||||
echo "::error::Invalid Node REPL package version $NODE_REPL_VERSION"
|
echo "::error::Invalid Node REPL package version $NODE_REPL_VERSION"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
SKILL_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' packages/core/src/skills/bundled/computer-use/SKILL.md)
|
BROWSER_VERSION=$(node -p "require('./packages/browser-use/package.json').version")
|
||||||
USER_GUIDE_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' docs/users/features/computer-use.md)
|
SDK_BROWSER_VERSION=$(node -p "require('./packages/cua-driver/typescript/package.json').dependencies['@qwen-code/browser-use']")
|
||||||
BROWSER_USE_SKILL_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' packages/core/src/skills/bundled/browser-use/SKILL.md)
|
NODE_SDK_VERSION=$(node -p "require('./packages/node-repl/package.json').dependencies['@qwen-code/cua-sdk']")
|
||||||
for PAIR in \
|
for VALUE in "$NODE_REPL_VERSION" "$BROWSER_VERSION" "$SDK_BROWSER_VERSION" "$NODE_SDK_VERSION"; do
|
||||||
"Computer Use skill Node REPL:$SKILL_NODE_REPL_VERSION" \
|
if [[ "$VALUE" != "$VERSION" ]]; then
|
||||||
"Computer Use user guide Node REPL:$USER_GUIDE_NODE_REPL_VERSION" \
|
echo "::error::SDK, browser SDK, Node REPL and native release must use the same version $VERSION; found $VALUE"
|
||||||
"Browser Use skill Node REPL:$BROWSER_USE_SKILL_NODE_REPL_VERSION"; do
|
|
||||||
LABEL=${PAIR%%:*}
|
|
||||||
VALUE=${PAIR#*:}
|
|
||||||
if [[ "$VALUE" != "$NODE_REPL_VERSION" ]]; then
|
|
||||||
echo "::error::$LABEL version $VALUE does not match Node REPL version $NODE_REPL_VERSION"
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
USER_GUIDE_NODE_REPL_VERSION=$(sed -nE 's/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@([^ ]+)$/\1/p' docs/users/features/computer-use.md)
|
||||||
|
if [[ "$USER_GUIDE_NODE_REPL_VERSION" != "$NODE_REPL_VERSION" ]]; then
|
||||||
|
echo "::error::Computer Use user guide Node REPL version $USER_GUIDE_NODE_REPL_VERSION does not match Node REPL version $NODE_REPL_VERSION"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "$GITHUB_REF" != "refs/heads/main" ]]; then
|
if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "$GITHUB_REF" != "refs/heads/main" ]]; then
|
||||||
echo "::error::A production dispatch must run from protected main"
|
echo "::error::A production dispatch must run from protected main"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "${{ inputs.node_repl_only }}" != "true" && "${{ inputs.notarize }}" != "true" ]]; then
|
if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "${{ inputs.notarize }}" != "true" ]]; then
|
||||||
echo "::error::A production dispatch must enable macOS codesigning and notarization"
|
echo "::error::A production dispatch must enable macOS codesigning and notarization"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -173,25 +162,17 @@ jobs:
|
|||||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||||
echo "node_repl_version=$NODE_REPL_VERSION" >> "$GITHUB_OUTPUT"
|
echo "node_repl_version=$NODE_REPL_VERSION" >> "$GITHUB_OUTPUT"
|
||||||
- uses: 'dtolnay/[email protected]'
|
- uses: 'dtolnay/[email protected]'
|
||||||
if:
|
|
||||||
"github.event_name != 'workflow_dispatch' || inputs.node_repl_only ==
|
|
||||||
false"
|
|
||||||
- name: 'Test model payload filtering'
|
- name: 'Test model payload filtering'
|
||||||
if:
|
|
||||||
"github.event_name != 'workflow_dispatch' || inputs.node_repl_only ==
|
|
||||||
false"
|
|
||||||
working-directory: 'packages/cua-driver/rust'
|
working-directory: 'packages/cua-driver/rust'
|
||||||
run: 'cargo test -p cua-driver-core --locked model_payload'
|
run: 'cargo test -p cua-driver-core --locked model_payload'
|
||||||
|
|
||||||
# ── Linux (x86_64 + arm64) ───────────────────────────────────────────────
|
# ── Linux (x86_64 + arm64) ───────────────────────────────────────────────
|
||||||
# Built inside debian:11 (glibc 2.31) so binaries run on Debian 11+,
|
# Built inside debian:11 (glibc 2.31) so binaries run on Debian 11+,
|
||||||
# Ubuntu 20.04+, RHEL/Rocky 8+. arm64 builds natively on a hosted ARM
|
# Ubuntu 20.04+, RHEL/Rocky 8+. arm64 builds natively on a hosted ARM
|
||||||
# runner to avoid hand-wiring a cross linker for the X11/Wayland C deps.
|
# runner to avoid hand-wiring a cross linker for the X11 C deps.
|
||||||
build-linux:
|
build-linux:
|
||||||
name: 'linux-${{ matrix.arch }}'
|
name: 'linux-${{ matrix.arch }}'
|
||||||
needs: ['validate-version']
|
needs: ['validate-version']
|
||||||
if: "github.event_name != 'workflow_dispatch' || inputs.node_repl_only ==
|
|
||||||
false"
|
|
||||||
runs-on: '${{ matrix.runner }}'
|
runs-on: '${{ matrix.runner }}'
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
@@ -219,14 +200,11 @@ jobs:
|
|||||||
apt-get -o Acquire::Retries=3 update
|
apt-get -o Acquire::Retries=3 update
|
||||||
apt-get -o Acquire::Retries=3 install -y --no-install-recommends \
|
apt-get -o Acquire::Retries=3 install -y --no-install-recommends \
|
||||||
git ca-certificates curl build-essential pkg-config \
|
git ca-certificates curl build-essential pkg-config \
|
||||||
libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev \
|
libx11-dev libxi-dev libxtst-dev libxext-dev
|
||||||
libxkbcommon-dev
|
|
||||||
- uses: 'actions/checkout@v4'
|
- uses: 'actions/checkout@v4'
|
||||||
- uses: 'actions/setup-node@v4'
|
- uses: 'actions/setup-node@v4'
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: 'npm'
|
|
||||||
cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json'
|
|
||||||
- name: 'Determine version'
|
- name: 'Determine version'
|
||||||
id: 'version'
|
id: 'version'
|
||||||
shell: 'bash'
|
shell: 'bash'
|
||||||
@@ -253,11 +231,11 @@ jobs:
|
|||||||
cua-driver-linux-${{ matrix.arch }}-
|
cua-driver-linux-${{ matrix.arch }}-
|
||||||
- name: 'Build (release)'
|
- name: 'Build (release)'
|
||||||
working-directory: 'packages/cua-driver/rust'
|
working-directory: 'packages/cua-driver/rust'
|
||||||
run: 'cargo build -p cua-driver -p cursor-theme-cli -p cua-driver-sdk --release --locked --features cua-driver/portal-input --target ${{ matrix.target }}'
|
run: 'cargo build -p cua-driver -p cursor-theme-cli -p cua-driver-sdk --release --locked --target ${{ matrix.target }}'
|
||||||
- name: 'Build Electron-compatible Node runtime'
|
- name: 'Build Electron-compatible Node runtime'
|
||||||
working-directory: 'packages/cua-driver'
|
working-directory: 'packages/cua-driver'
|
||||||
run: |
|
run: |
|
||||||
npm --prefix typescript ci --ignore-scripts
|
corepack pnpm -C ../.. install --frozen-lockfile --ignore-scripts --prefer-offline
|
||||||
node scripts/build-node-runtime.mjs \
|
node scripts/build-node-runtime.mjs \
|
||||||
--target "${{ matrix.target }}" \
|
--target "${{ matrix.target }}" \
|
||||||
--output "rust/target/${{ matrix.target }}/release/cua_driver_node_runtime.node"
|
--output "rust/target/${{ matrix.target }}/release/cua_driver_node_runtime.node"
|
||||||
@@ -273,12 +251,11 @@ jobs:
|
|||||||
cp "target/${TARGET}/release/libcua_driver_sdk.so" "release/${STAGE}/"
|
cp "target/${TARGET}/release/libcua_driver_sdk.so" "release/${STAGE}/"
|
||||||
cp "target/${TARGET}/release/cua_driver_node_runtime.node" "release/${STAGE}/"
|
cp "target/${TARGET}/release/cua_driver_node_runtime.node" "release/${STAGE}/"
|
||||||
cp include/cua_driver_abi.h "release/${STAGE}/"
|
cp include/cua_driver_abi.h "release/${STAGE}/"
|
||||||
cp -R ../wayland-helper "release/${STAGE}/wayland-helper"
|
|
||||||
cp ../../../LICENSE "release/${STAGE}/LICENSE"
|
cp ../../../LICENSE "release/${STAGE}/LICENSE"
|
||||||
(cd release && tar -czf "${STAGE}.tar.gz" "${STAGE}")
|
(cd release && tar -czf "${STAGE}.tar.gz" "${STAGE}")
|
||||||
tar -czf "release/${STAGE}-binary.tar.gz" -C "release/${STAGE}" \
|
tar -czf "release/${STAGE}-binary.tar.gz" -C "release/${STAGE}" \
|
||||||
qwen-cua-driver cua-cursor-theme libcua_driver_sdk.so \
|
qwen-cua-driver cua-cursor-theme libcua_driver_sdk.so \
|
||||||
cua_driver_node_runtime.node cua_driver_abi.h wayland-helper
|
cua_driver_node_runtime.node cua_driver_abi.h
|
||||||
ls -lh "release/${STAGE}.tar.gz" "release/${STAGE}-binary.tar.gz"
|
ls -lh "release/${STAGE}.tar.gz" "release/${STAGE}-binary.tar.gz"
|
||||||
- uses: 'actions/upload-artifact@v4'
|
- uses: 'actions/upload-artifact@v4'
|
||||||
with:
|
with:
|
||||||
@@ -290,8 +267,6 @@ jobs:
|
|||||||
build-windows:
|
build-windows:
|
||||||
name: 'windows-${{ matrix.arch }}'
|
name: 'windows-${{ matrix.arch }}'
|
||||||
needs: ['validate-version']
|
needs: ['validate-version']
|
||||||
if: "github.event_name != 'workflow_dispatch' || inputs.node_repl_only ==
|
|
||||||
false"
|
|
||||||
runs-on: 'windows-latest'
|
runs-on: 'windows-latest'
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
@@ -306,8 +281,6 @@ jobs:
|
|||||||
- uses: 'actions/setup-node@v4'
|
- uses: 'actions/setup-node@v4'
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: 'npm'
|
|
||||||
cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json'
|
|
||||||
- name: 'Determine version'
|
- name: 'Determine version'
|
||||||
id: 'version'
|
id: 'version'
|
||||||
shell: 'bash'
|
shell: 'bash'
|
||||||
@@ -344,7 +317,7 @@ jobs:
|
|||||||
working-directory: 'packages/cua-driver'
|
working-directory: 'packages/cua-driver'
|
||||||
shell: 'bash'
|
shell: 'bash'
|
||||||
run: |
|
run: |
|
||||||
npm --prefix typescript ci --ignore-scripts
|
corepack pnpm -C ../.. install --frozen-lockfile --ignore-scripts --prefer-offline
|
||||||
node scripts/build-node-runtime.mjs \
|
node scripts/build-node-runtime.mjs \
|
||||||
--target "${{ matrix.target }}" \
|
--target "${{ matrix.target }}" \
|
||||||
--output "rust/target/${{ matrix.target }}/release/cua_driver_node_runtime.node"
|
--output "rust/target/${{ matrix.target }}/release/cua_driver_node_runtime.node"
|
||||||
@@ -387,8 +360,6 @@ jobs:
|
|||||||
build-macos:
|
build-macos:
|
||||||
name: 'macos-universal'
|
name: 'macos-universal'
|
||||||
needs: ['validate-version']
|
needs: ['validate-version']
|
||||||
if: "github.event_name != 'workflow_dispatch' || inputs.node_repl_only ==
|
|
||||||
false"
|
|
||||||
runs-on: 'macos-26'
|
runs-on: 'macos-26'
|
||||||
env:
|
env:
|
||||||
# Sign+notarize on tag push (real release); on manual dispatch honor the
|
# Sign+notarize on tag push (real release); on manual dispatch honor the
|
||||||
@@ -399,8 +370,6 @@ jobs:
|
|||||||
- uses: 'actions/setup-node@v4'
|
- uses: 'actions/setup-node@v4'
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: 'npm'
|
|
||||||
cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json'
|
|
||||||
- name: 'Determine version'
|
- name: 'Determine version'
|
||||||
id: 'version'
|
id: 'version'
|
||||||
shell: 'bash'
|
shell: 'bash'
|
||||||
@@ -436,7 +405,7 @@ jobs:
|
|||||||
- name: 'Build Electron-compatible Node runtimes'
|
- name: 'Build Electron-compatible Node runtimes'
|
||||||
working-directory: 'packages/cua-driver'
|
working-directory: 'packages/cua-driver'
|
||||||
run: |
|
run: |
|
||||||
npm --prefix typescript ci --ignore-scripts
|
corepack pnpm -C ../.. install --frozen-lockfile --ignore-scripts --prefer-offline
|
||||||
node scripts/build-node-runtime.mjs \
|
node scripts/build-node-runtime.mjs \
|
||||||
--target aarch64-apple-darwin \
|
--target aarch64-apple-darwin \
|
||||||
--output rust/target/aarch64-apple-darwin/release/cua_driver_node_runtime.node
|
--output rust/target/aarch64-apple-darwin/release/cua_driver_node_runtime.node
|
||||||
@@ -624,7 +593,13 @@ jobs:
|
|||||||
|
|
||||||
verify-sdk-package:
|
verify-sdk-package:
|
||||||
name: '@qwen-code/cua-sdk package dry-run'
|
name: '@qwen-code/cua-sdk package dry-run'
|
||||||
needs: ['validate-version', 'build-linux', 'verify-release-artifacts']
|
needs:
|
||||||
|
[
|
||||||
|
'validate-version',
|
||||||
|
'build-linux',
|
||||||
|
'verify-release-artifacts',
|
||||||
|
'verify-browser-package',
|
||||||
|
]
|
||||||
runs-on: 'ubuntu-latest'
|
runs-on: 'ubuntu-latest'
|
||||||
permissions:
|
permissions:
|
||||||
contents: 'read'
|
contents: 'read'
|
||||||
@@ -633,8 +608,6 @@ jobs:
|
|||||||
- uses: 'actions/setup-node@v4'
|
- uses: 'actions/setup-node@v4'
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: 'npm'
|
|
||||||
cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json'
|
|
||||||
registry-url: 'https://registry.npmjs.org'
|
registry-url: 'https://registry.npmjs.org'
|
||||||
scope: '@qwen-code'
|
scope: '@qwen-code'
|
||||||
- name: 'Install npm 11'
|
- name: 'Install npm 11'
|
||||||
@@ -664,9 +637,14 @@ jobs:
|
|||||||
(cd "$RELEASE_DIR" && sha256sum "$(basename "$ARCHIVE")" > checksums.txt)
|
(cd "$RELEASE_DIR" && sha256sum "$(basename "$ARCHIVE")" > checksums.txt)
|
||||||
echo "directory=$NATIVE_DIR" >> "$GITHUB_OUTPUT"
|
echo "directory=$NATIVE_DIR" >> "$GITHUB_OUTPUT"
|
||||||
echo "release_directory=$RELEASE_DIR" >> "$GITHUB_OUTPUT"
|
echo "release_directory=$RELEASE_DIR" >> "$GITHUB_OUTPUT"
|
||||||
- name: 'Install SDK dependencies'
|
- name: 'Install SDK workspace dependencies'
|
||||||
working-directory: 'packages/cua-driver/typescript'
|
run: |
|
||||||
run: 'npm ci --ignore-scripts --no-audit --no-fund'
|
corepack pnpm install --frozen-lockfile --ignore-scripts --prefer-offline
|
||||||
|
npm run build --workspace @qwen-code/browser-use
|
||||||
|
- uses: 'actions/download-artifact@v4'
|
||||||
|
with:
|
||||||
|
name: 'browser-use-npm-${{ needs.validate-version.outputs.version }}'
|
||||||
|
path: '${{ runner.temp }}/browser-use-package'
|
||||||
- name: 'Test SDK and native loader'
|
- name: 'Test SDK and native loader'
|
||||||
working-directory: 'packages/cua-driver/typescript'
|
working-directory: 'packages/cua-driver/typescript'
|
||||||
env:
|
env:
|
||||||
@@ -678,6 +656,7 @@ jobs:
|
|||||||
QWEN_CUA_SDK_NATIVE_DIR: '${{ steps.native.outputs.directory }}'
|
QWEN_CUA_SDK_NATIVE_DIR: '${{ steps.native.outputs.directory }}'
|
||||||
run: |
|
run: |
|
||||||
node packages/cua-driver/scripts/verify-cua-sdk-package.mjs \
|
node packages/cua-driver/scripts/verify-cua-sdk-package.mjs \
|
||||||
|
--browser-tarball "${RUNNER_TEMP}/browser-use-package/qwen-code-browser-use-${{ needs.validate-version.outputs.version }}.tgz" \
|
||||||
--native-dir "${QWEN_CUA_SDK_NATIVE_DIR}" \
|
--native-dir "${QWEN_CUA_SDK_NATIVE_DIR}" \
|
||||||
--release-dir "${{ steps.native.outputs.release_directory }}" \
|
--release-dir "${{ steps.native.outputs.release_directory }}" \
|
||||||
--output-dir "${RUNNER_TEMP}/cua-sdk-package"
|
--output-dir "${RUNNER_TEMP}/cua-sdk-package"
|
||||||
@@ -688,9 +667,39 @@ jobs:
|
|||||||
if-no-files-found: 'error'
|
if-no-files-found: 'error'
|
||||||
retention-days: 7
|
retention-days: 7
|
||||||
|
|
||||||
|
verify-browser-package:
|
||||||
|
name: '@qwen-code/browser-use package dry-run'
|
||||||
|
needs: ['validate-version']
|
||||||
|
runs-on: 'ubuntu-latest'
|
||||||
|
permissions:
|
||||||
|
contents: 'read'
|
||||||
|
steps:
|
||||||
|
- uses: 'actions/checkout@v4'
|
||||||
|
- uses: 'actions/setup-node@v4'
|
||||||
|
with:
|
||||||
|
node-version: '22'
|
||||||
|
- name: 'Install workspace dependencies'
|
||||||
|
run: 'corepack pnpm install --frozen-lockfile --ignore-scripts --prefer-offline'
|
||||||
|
- name: 'Test and pack existing browser SDK'
|
||||||
|
run: |
|
||||||
|
npm run test --workspace @qwen-code/browser-use
|
||||||
|
node packages/browser-use/scripts/verify-package.mjs --output-dir "${RUNNER_TEMP}/browser-use-package"
|
||||||
|
- uses: 'actions/upload-artifact@v4'
|
||||||
|
with:
|
||||||
|
name: 'browser-use-npm-${{ needs.validate-version.outputs.version }}'
|
||||||
|
path: '${{ runner.temp }}/browser-use-package/*.tgz'
|
||||||
|
if-no-files-found: 'error'
|
||||||
|
retention-days: 7
|
||||||
|
|
||||||
verify-node-repl-package:
|
verify-node-repl-package:
|
||||||
name: '@qwen-code/node-repl-mcp package dry-run'
|
name: '@qwen-code/node-repl-mcp package dry-run'
|
||||||
needs: ['validate-version']
|
needs:
|
||||||
|
[
|
||||||
|
'validate-version',
|
||||||
|
'verify-sdk-package',
|
||||||
|
'verify-browser-package',
|
||||||
|
'build-linux',
|
||||||
|
]
|
||||||
runs-on: 'ubuntu-latest'
|
runs-on: 'ubuntu-latest'
|
||||||
permissions:
|
permissions:
|
||||||
contents: 'read'
|
contents: 'read'
|
||||||
@@ -705,6 +714,24 @@ jobs:
|
|||||||
run: 'npm install --global [email protected]'
|
run: 'npm install --global [email protected]'
|
||||||
- name: 'Install workspace dependencies'
|
- name: 'Install workspace dependencies'
|
||||||
run: 'corepack pnpm install --frozen-lockfile --ignore-scripts --prefer-offline --reporter=append-only'
|
run: 'corepack pnpm install --frozen-lockfile --ignore-scripts --prefer-offline --reporter=append-only'
|
||||||
|
- uses: 'actions/download-artifact@v4'
|
||||||
|
with:
|
||||||
|
name: 'cua-sdk-npm-${{ needs.validate-version.outputs.version }}'
|
||||||
|
path: '${{ runner.temp }}/cua-sdk-package'
|
||||||
|
- uses: 'actions/download-artifact@v4'
|
||||||
|
with:
|
||||||
|
name: 'browser-use-npm-${{ needs.validate-version.outputs.version }}'
|
||||||
|
path: '${{ runner.temp }}/browser-use-package'
|
||||||
|
- uses: 'actions/download-artifact@v4'
|
||||||
|
with:
|
||||||
|
name: 'cua-driver-rs-linux-x86_64'
|
||||||
|
path: '${{ runner.temp }}/cua-driver-linux'
|
||||||
|
- name: 'Prepare matching native payload and SDK declarations'
|
||||||
|
run: |
|
||||||
|
mkdir -p "${RUNNER_TEMP}/native"
|
||||||
|
tar -xzf "${RUNNER_TEMP}/cua-driver-linux/cua-driver-rs-${{ needs.validate-version.outputs.version }}-linux-x86_64-binary.tar.gz" -C "${RUNNER_TEMP}/native"
|
||||||
|
npm run build --workspace @qwen-code/browser-use
|
||||||
|
npm run build --workspace @qwen-code/cua-sdk
|
||||||
- name: 'Test standalone Node REPL package'
|
- name: 'Test standalone Node REPL package'
|
||||||
working-directory: 'packages/node-repl'
|
working-directory: 'packages/node-repl'
|
||||||
run: |
|
run: |
|
||||||
@@ -715,8 +742,12 @@ jobs:
|
|||||||
npm run smoke:mcp
|
npm run smoke:mcp
|
||||||
npm run smoke:lifecycle
|
npm run smoke:lifecycle
|
||||||
- name: 'Pack, publish-dry-run, and clean-install Node REPL tarball'
|
- name: 'Pack, publish-dry-run, and clean-install Node REPL tarball'
|
||||||
|
env:
|
||||||
|
QWEN_CUA_SDK_NATIVE_DIR: '${{ runner.temp }}/native'
|
||||||
run: |
|
run: |
|
||||||
node packages/node-repl/scripts/verify-package.mjs \
|
node packages/node-repl/scripts/verify-package.mjs \
|
||||||
|
--sdk-tarball "${RUNNER_TEMP}/cua-sdk-package/qwen-code-cua-sdk-${{ needs.validate-version.outputs.version }}.tgz" \
|
||||||
|
--browser-tarball "${RUNNER_TEMP}/browser-use-package/qwen-code-browser-use-${{ needs.validate-version.outputs.version }}.tgz" \
|
||||||
--output-dir "${RUNNER_TEMP}/node-repl-package"
|
--output-dir "${RUNNER_TEMP}/node-repl-package"
|
||||||
- uses: 'actions/upload-artifact@v4'
|
- uses: 'actions/upload-artifact@v4'
|
||||||
with:
|
with:
|
||||||
@@ -736,6 +767,7 @@ jobs:
|
|||||||
'build-macos',
|
'build-macos',
|
||||||
'verify-release-artifacts',
|
'verify-release-artifacts',
|
||||||
'verify-sdk-package',
|
'verify-sdk-package',
|
||||||
|
'verify-browser-package',
|
||||||
'verify-node-repl-package',
|
'verify-node-repl-package',
|
||||||
]
|
]
|
||||||
runs-on: 'ubuntu-latest'
|
runs-on: 'ubuntu-latest'
|
||||||
@@ -743,8 +775,7 @@ jobs:
|
|||||||
version: '${{ steps.version.outputs.version }}'
|
version: '${{ steps.version.outputs.version }}'
|
||||||
if:
|
if:
|
||||||
"startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name
|
"startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name
|
||||||
== 'workflow_dispatch' && inputs.dry_run == false &&
|
== 'workflow_dispatch' && inputs.dry_run == false)"
|
||||||
inputs.node_repl_only == false)"
|
|
||||||
steps:
|
steps:
|
||||||
- uses: 'actions/checkout@v4'
|
- uses: 'actions/checkout@v4'
|
||||||
- uses: 'actions/download-artifact@v4'
|
- uses: 'actions/download-artifact@v4'
|
||||||
@@ -845,7 +876,7 @@ jobs:
|
|||||||
on the target machine before UIAccess activation
|
on the target machine before UIAccess activation
|
||||||
- **Node.js**: the same workflow publishes `@qwen-code/cua-sdk`
|
- **Node.js**: the same workflow publishes `@qwen-code/cua-sdk`
|
||||||
after a clean install against these release assets and publishes
|
after a clean install against these release assets and publishes
|
||||||
the independently versioned `@qwen-code/node-repl-mcp` package
|
the same-version browser SDK and `@qwen-code/node-repl-mcp` package
|
||||||
after its MCP protocol smoke tests
|
after its MCP protocol smoke tests
|
||||||
|
|
||||||
Enable relative coordinates: `CUA_DRIVER_RS_COORDINATE_SPACE=1`
|
Enable relative coordinates: `CUA_DRIVER_RS_COORDINATE_SPACE=1`
|
||||||
@@ -856,11 +887,11 @@ jobs:
|
|||||||
|
|
||||||
publish-sdk:
|
publish-sdk:
|
||||||
name: 'Publish @qwen-code/cua-sdk'
|
name: 'Publish @qwen-code/cua-sdk'
|
||||||
needs: ['validate-version', 'verify-sdk-package', 'release']
|
needs:
|
||||||
|
['validate-version', 'verify-sdk-package', 'release', 'publish-browser']
|
||||||
if:
|
if:
|
||||||
"startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name
|
"startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name
|
||||||
== 'workflow_dispatch' && inputs.dry_run == false &&
|
== 'workflow_dispatch' && inputs.dry_run == false)"
|
||||||
inputs.node_repl_only == false)"
|
|
||||||
runs-on: 'ubuntu-latest'
|
runs-on: 'ubuntu-latest'
|
||||||
environment:
|
environment:
|
||||||
name: 'production-release'
|
name: 'production-release'
|
||||||
@@ -917,13 +948,17 @@ jobs:
|
|||||||
if (!Array.isArray(listing.tools) || listing.tools.length === 0) {
|
if (!Array.isArray(listing.tools) || listing.tools.length === 0) {
|
||||||
throw new Error("native SDK returned no tools");
|
throw new Error("native SDK returned no tools");
|
||||||
}
|
}
|
||||||
const revisionSupported = await computer.supportsObservationRevision();
|
let safeRefusal = false;
|
||||||
if (typeof revisionSupported !== "boolean") {
|
try {
|
||||||
throw new Error("ComputerUse capability probe returned an invalid result");
|
await computer.appCall("click", { scope: "desktop", x: 1, y: 1 });
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code !== "screenshot_required" || error.sent !== false) throw error;
|
||||||
|
safeRefusal = true;
|
||||||
}
|
}
|
||||||
|
if (!safeRefusal || listing.tools.length !== 19) throw new Error("Computer Use session/manifest contract mismatch");
|
||||||
console.log(JSON.stringify({
|
console.log(JSON.stringify({
|
||||||
toolCount: listing.tools.length,
|
toolCount: listing.tools.length,
|
||||||
revisionSupported,
|
safeRefusal,
|
||||||
}));
|
}));
|
||||||
} finally {
|
} finally {
|
||||||
await computer.close();
|
await computer.close();
|
||||||
@@ -972,15 +1007,81 @@ jobs:
|
|||||||
echo '::error::Published npm version did not become visible'
|
echo '::error::Published npm version did not become visible'
|
||||||
exit 1
|
exit 1
|
||||||
|
|
||||||
|
publish-browser:
|
||||||
|
name: 'Publish @qwen-code/browser-use'
|
||||||
|
needs: ['validate-version', 'verify-browser-package', 'release']
|
||||||
|
if: "startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false)"
|
||||||
|
runs-on: 'ubuntu-latest'
|
||||||
|
environment:
|
||||||
|
name: 'production-release'
|
||||||
|
url: 'https://www.npmjs.com/package/@qwen-code/browser-use/v/${{
|
||||||
|
needs.validate-version.outputs.version }}'
|
||||||
|
permissions:
|
||||||
|
contents: 'read'
|
||||||
|
id-token: 'write'
|
||||||
|
steps:
|
||||||
|
- uses: 'actions/setup-node@v4'
|
||||||
|
with:
|
||||||
|
node-version: '22'
|
||||||
|
registry-url: 'https://registry.npmjs.org'
|
||||||
|
scope: '@qwen-code'
|
||||||
|
- name: 'Install npm 11'
|
||||||
|
run: 'npm install --global [email protected]'
|
||||||
|
- uses: 'actions/download-artifact@v4'
|
||||||
|
with:
|
||||||
|
name: 'browser-use-npm-${{
|
||||||
|
needs.validate-version.outputs.version }}'
|
||||||
|
path: '${{ runner.temp }}/browser-use-package'
|
||||||
|
- name: 'Publish immutable Browser SDK tarball'
|
||||||
|
env:
|
||||||
|
VERSION: '${{ needs.validate-version.outputs.version }}'
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
TARBALL=$(find "${RUNNER_TEMP}/browser-use-package" -type f -name '*.tgz' -print -quit)
|
||||||
|
if [[ -z "$TARBALL" ]]; then
|
||||||
|
echo '::error::Packed @qwen-code/browser-use tarball is missing'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
LOCAL_INTEGRITY="sha512-$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A)"
|
||||||
|
REMOTE_INTEGRITY=$(npm view "@qwen-code/browser-use@${VERSION}" dist.integrity 2>/dev/null || true)
|
||||||
|
if [[ -n "$REMOTE_INTEGRITY" ]]; then
|
||||||
|
if [[ "$REMOTE_INTEGRITY" != "$LOCAL_INTEGRITY" ]]; then
|
||||||
|
echo "::error::@qwen-code/browser-use@${VERSION} already exists with different integrity"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "@qwen-code/browser-use@${VERSION} already contains this exact tarball"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
NPM_TAG=latest
|
||||||
|
[[ "$VERSION" == *-* ]] && NPM_TAG=next
|
||||||
|
npm publish "$TARBALL" --provenance --access public --tag "$NPM_TAG"
|
||||||
|
- name: 'Verify npm registry integrity'
|
||||||
|
env:
|
||||||
|
VERSION: '${{ needs.validate-version.outputs.version }}'
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
TARBALL=$(find "${RUNNER_TEMP}/browser-use-package" -type f -name '*.tgz' -print -quit)
|
||||||
|
LOCAL_INTEGRITY="sha512-$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A)"
|
||||||
|
for attempt in {1..20}; do
|
||||||
|
REMOTE_INTEGRITY=$(npm view "@qwen-code/browser-use@${VERSION}" dist.integrity 2>/dev/null || true)
|
||||||
|
if [[ "$REMOTE_INTEGRITY" == "$LOCAL_INTEGRITY" ]]; then
|
||||||
|
echo "Verified npm registry integrity on attempt $attempt"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [[ -n "$REMOTE_INTEGRITY" ]]; then
|
||||||
|
echo '::error::Published npm artifact integrity does not match the verified tarball'
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 10
|
||||||
|
done
|
||||||
|
echo '::error::Published npm version did not become visible'
|
||||||
|
exit 1
|
||||||
|
|
||||||
publish-node-repl:
|
publish-node-repl:
|
||||||
name: 'Publish @qwen-code/node-repl-mcp'
|
name: 'Publish @qwen-code/node-repl-mcp'
|
||||||
needs: ['validate-version', 'verify-node-repl-package', 'release']
|
needs:
|
||||||
if: "always() && needs.validate-version.result == 'success' &&
|
['validate-version', 'verify-node-repl-package', 'release', 'publish-sdk']
|
||||||
needs.verify-node-repl-package.result == 'success' &&
|
if: "startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false)"
|
||||||
(needs.release.result == 'success' || (github.event_name ==
|
|
||||||
'workflow_dispatch' && inputs.node_repl_only == true)) &&
|
|
||||||
(startsWith(github.ref, 'refs/tags/cua-driver-rs-v') ||
|
|
||||||
(github.event_name == 'workflow_dispatch' && inputs.dry_run == false))"
|
|
||||||
runs-on: 'ubuntu-latest'
|
runs-on: 'ubuntu-latest'
|
||||||
environment:
|
environment:
|
||||||
name: 'production-release'
|
name: 'production-release'
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ export const workspacePackageNames = new Set([
|
|||||||
'@qwen-code/channel-wecom',
|
'@qwen-code/channel-wecom',
|
||||||
'@qwen-code/channel-weixin',
|
'@qwen-code/channel-weixin',
|
||||||
'@qwen-code/chrome-bridge',
|
'@qwen-code/chrome-bridge',
|
||||||
|
'@qwen-code/cua-sdk',
|
||||||
'@qwen-code/external-context',
|
'@qwen-code/external-context',
|
||||||
'@qwen-code/external-context-mem0',
|
'@qwen-code/external-context-mem0',
|
||||||
'@qwen-code/test-failure-explorer',
|
'@qwen-code/test-failure-explorer',
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ Runs the CLI via `tsx` with `DEV=true`. Changes to `packages/core` or
|
|||||||
`packages/cli` are reflected immediately without rebuilding. Browser Use reuses
|
`packages/cli` are reflected immediately without rebuilding. Browser Use reuses
|
||||||
its built runtime, like other compiled workspace dependencies. After editing
|
its built runtime, like other compiled workspace dependencies. After editing
|
||||||
Browser Use, run `npm run build --workspace=@qwen-code/browser-use` to rebuild
|
Browser Use, run `npm run build --workspace=@qwen-code/browser-use` to rebuild
|
||||||
and stage its runtime for dev. Normal `npm install` prepares it automatically.
|
its runtime for dev. Normal `npm install` prepares it automatically.
|
||||||
|
|
||||||
### Unit Testing
|
### Unit Testing
|
||||||
|
|
||||||
|
|||||||
@@ -105,6 +105,7 @@ For automation and custom integrations:
|
|||||||
- **Headless** — Run `qwen -p "..."` in scripts, CI/CD, or batch jobs.
|
- **Headless** — Run `qwen -p "..."` in scripts, CI/CD, or batch jobs.
|
||||||
- **SDKs** — Build with [TypeScript](./packages/sdk-typescript/README.md), [Python](./packages/sdk-python/README.md), or [Java](./packages/sdk-java/qwencode/README.md).
|
- **SDKs** — Build with [TypeScript](./packages/sdk-typescript/README.md), [Python](./packages/sdk-python/README.md), or [Java](./packages/sdk-java/qwencode/README.md).
|
||||||
- **Daemon** — Run `qwen serve` to connect clients over HTTP + SSE (ACP). _(experimental)_ [Daemon guide](https://qwenlm.github.io/qwen-code-docs/en/users/qwen-serve/).
|
- **Daemon** — Run `qwen serve` to connect clients over HTTP + SSE (ACP). _(experimental)_ [Daemon guide](https://qwenlm.github.io/qwen-code-docs/en/users/qwen-serve/).
|
||||||
|
- **Computer Use** — Set up [native MCP tools](./packages/cua-driver/README.md#agent-integration) or [Node REPL + SDK](./packages/node-repl/README.md#usage) in Qwen Code, Claude Code, or Codex.
|
||||||
|
|
||||||
<details>
|
<details>
|
||||||
<summary>SDK example (Python)</summary>
|
<summary>SDK example (Python)</summary>
|
||||||
|
|||||||
@@ -0,0 +1,332 @@
|
|||||||
|
# qwen-code Computer Use upgrade design and decisions (2026-10-15–16)
|
||||||
|
|
||||||
|
[English](cua-upgrade-2026-10.md) | [简体中文](cua-upgrade-2026-10.zh-CN.md)
|
||||||
|
|
||||||
|
This document records the background, analysis and decisions from these two days for implementation and later reference.
|
||||||
|
|
||||||
|
- Decisions are dated. Detailed evidence is listed in the document index.
|
||||||
|
- **[Measured]** means supported by tests or data; **[Inference]** means inferred from code or observations; **[Pending]** means verification is required.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. Background
|
||||||
|
|
||||||
|
**Scope**
|
||||||
|
|
||||||
|
- `packages/cua-driver`: the Rust driver forked from trycua/cua `cua-driver-rs-v0.20.0`, distributed as qwen-cua-driver / QwenCuaDriver.app.
|
||||||
|
- `@qwen-code/cua-sdk` (`packages/cua-driver/typescript`, then 0.20.11).
|
||||||
|
- `@qwen-code/node-repl-mcp` (`packages/node-repl`, then 0.1.7).
|
||||||
|
- The bundled `computer-use` skill.
|
||||||
|
- The browser SDK `@qwen-code/browser-use` (0.25.2), included under topic 5.
|
||||||
|
|
||||||
|
**Why upgrade**
|
||||||
|
|
||||||
|
1. Other implementations have progressed:
|
||||||
|
- Codex `cua_repl` exposes a `js` entry, supplies documentation at runtime, automatically displays state from `getApp`, and returns void from actions.
|
||||||
|
- Upstream cua-driver reached 0.35.4, with 499 commits between the two versions.
|
||||||
|
2. RecreationBench and OSWorld-V2 exposed these problems:
|
||||||
|
- **Comparable quality, lower efficiency**: across the final 12 RecreationBench groups and 450 results, our quality and token use were comparable to official Codex and QoderCU, but elapsed time was about 1.5 times higher. Input per round was 110–130k, close to the 128k limit, making cost roughly proportional to round count. **[Measured]**
|
||||||
|
- **Too many rounds**: in the first macOS stable20 round, our generation count was 2.47 times the official implementation's, and input per generation was 2.05 times higher. On 41 tasks passed by all three implementations, calls were official 4, Qoder 12 and ours 28. The OSWorld-V2 candidate made about twice as many requests as direct mode. **[Measured]**
|
||||||
|
- **Cost of always foregrounding macOS**: commit `0ceac09` on 09-13 changed macOS input to activate B before every input, based on a misattributed Codex Windows document. Total time rose 37%; one five-action cell took 8.61 s versus official 1.61 s. **[Measured]**
|
||||||
|
|
||||||
|
**Overall goal, decided 10-15**: complete computer-use tasks with fewer rounds, fewer tokens and clear, concise steps.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Overall direction (10-15)
|
||||||
|
|
||||||
|
1. Supply runtime documentation in place of skills and simplify node-repl by removing wait / cancel, following Codex.
|
||||||
|
2. Maintain one underlying service. The driver remains directly callable; the node-repl path wraps that same driver through the SDK. Upstream synchronization means porting or referencing specific fixes and capabilities rather than merging its entire codebase.
|
||||||
|
3. Exclude upstream `run_script` / `run_steps` and its upper-level interfaces. Synchronize native operation coverage, reliability and efficiency.
|
||||||
|
4. Decide foreground/background behavior separately for each platform, as described in section 2.
|
||||||
|
5. Evaluate rounds, tokens and clarity of steps.
|
||||||
|
|
||||||
|
**Addition, 10-16**: direct driver MCP use, without node-repl, must remain fully usable and is equally important.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Foreground / background (topic 4)
|
||||||
|
|
||||||
|
The shared scenario is a user typing in A while the agent operates B behind A.
|
||||||
|
|
||||||
|
### 2.1 macOS (10-15)
|
||||||
|
|
||||||
|
**Analysis**
|
||||||
|
|
||||||
|
- There are three ways to deliver input to B:
|
||||||
|
1. AX accessibility actions such as press, value writes and focus changes, without foregrounding B.
|
||||||
|
2. Process-directed events, optionally telling B that it has been activated through “synthetic focus,” while the actual system foreground remains unchanged.
|
||||||
|
3. Global events, requiring B to become the actual foreground and affecting the user's keys and pointer.
|
||||||
|
- Codex and upstream primarily use 1 and 2; ours primarily uses 3 after `0ceac09`.
|
||||||
|
- Our implementation waits a **fixed 1 s after each action** to check for new windows (`window_change_detector.rs:152`).
|
||||||
|
|
||||||
|
**Decisions**
|
||||||
|
|
||||||
|
- Combine AX and process-directed events, drawing on Codex and upstream.
|
||||||
|
- Initially omit an automatic foreground/global-event route; Codex and upstream do not use it for this path.
|
||||||
|
- Verify post-action waits: if Codex completes correctly without waiting, that wait may explain the efficiency gap.
|
||||||
|
- Initially test Codex's directed drag and align paste with Codex.
|
||||||
|
- Design tests on mini using official Codex and upstream 0.35.4 as comparison targets.
|
||||||
|
|
||||||
|
**mini smoke observations, 10-15/16**: one run per case; **[Measured]**, but preliminary. The formal batch finished and still required analysis at the time.
|
||||||
|
|
||||||
|
- **Codex returns after dispatch without a fixed wait**: control actions took about 100–150 ms. Opening a sheet and typing without another observation succeeded 3/3. The roughly 2 s smoke delay came from an unanswered system authorization dialog and disappeared after it was handled.
|
||||||
|
- **Codex synthetic focus works**: B considers itself active and keyboard-focused while system foreground stays A. It can paste with `cmd+v` and press Return in a background sheet; upstream cannot do those two actions but explicitly refuses them.
|
||||||
|
- **Codex gaps**:
|
||||||
|
- `typeText` lost Chinese and emoji, 0/3, while reporting success.
|
||||||
|
- Coordinate right-click reached neither A nor B.
|
||||||
|
- Drag/drop and list reorder reported success without effect; two of three runs swallowed one user key. Slider drag alone worked.
|
||||||
|
- **Upstream problems**:
|
||||||
|
- Popup `setValue` worked, but text-field `setValue` did not trigger the application's change callback.
|
||||||
|
- AXPress on a text field returned -25206 without fallback.
|
||||||
|
- Each action took about 1 s.
|
||||||
|
- **Ours, forced foreground** passed the most cases but took about 1.1 s per action, foregrounded B and moved the pointer. Restoring A closed menus just opened in B.
|
||||||
|
|
||||||
|
**Implementation details decided from this evidence**
|
||||||
|
|
||||||
|
- Remove the fixed post-action wait. Return after dispatch and move waiting into observation, as described in topic 3.
|
||||||
|
- Explicitly refuse background drag/drop and list reorder, suggesting alternatives. Use `setValue` for sliders rather than Codex's instantaneous drag.
|
||||||
|
- Align paste with Codex: temporarily use and then restore the clipboard.
|
||||||
|
- Use directed keys for ASCII; use AX insertion at the caret plus readback for non-ASCII to avoid Codex's lost text. **[Pending]**
|
||||||
|
- Implement popup `setValue` by opening the menu and selecting the item, porting upstream behavior.
|
||||||
|
- Use AXPress for controls that support press; otherwise use a directed click with synthetic focus.
|
||||||
|
- Port upstream background protection:
|
||||||
|
- Do not send A a synthetic deactivation record (#4943).
|
||||||
|
- `key_focus_guard` (#4943 #4955 #5091 #5129).
|
||||||
|
- Restore A after B activates itself, except when the user moved the mouse or modifiers within 1 s (#4976).
|
||||||
|
- Explicitly stamp mouse modifier state (#5051).
|
||||||
|
- Save and rename with Apple Events and readback (#5057).
|
||||||
|
- Refuse typing without a focused text field using `no_focused_text_input`.
|
||||||
|
- Return `write_not_applied` when readback shows no write.
|
||||||
|
- Merge these protections with existing `app_focus.rs` synthetic focus into one observer.
|
||||||
|
- Do not implement Codex's visible cursor at this stage; no decision to include it was made.
|
||||||
|
|
||||||
|
### 2.2 Windows (10-15/16)
|
||||||
|
|
||||||
|
**Analysis**
|
||||||
|
|
||||||
|
- Four routes deliver input to B:
|
||||||
|
1. UIA semantics without foreground activation.
|
||||||
|
2. PostMessage window messages without activation, often ignored by Chromium, WPF and Tk.
|
||||||
|
3. Synthetic pointer injection.
|
||||||
|
4. SendInput, which requires the actual foreground.
|
||||||
|
- Current implementations:
|
||||||
|
- **Codex** activates B, with up to 20 attempts and an Alt technique, then uses SendInput. It does not guarantee restoring A and monitors user input.
|
||||||
|
- **Upstream** prefers background and returns `background_unavailable` for known ignored combinations.
|
||||||
|
- **Our App layer** always activates B and uses SendInput, bypassing UIA Invoke even for element clicks, then synchronously restores A. It depends on a signed UIAccess worker.
|
||||||
|
- The fork already contains upstream background routes and ignored-delivery classifications; App currently forces foreground.
|
||||||
|
- Known defects:
|
||||||
|
- Foreground typing waits only 40 ms before restoring A; trailing characters can enter A.
|
||||||
|
- Text is sent as one batch without checking foreground between characters.
|
||||||
|
- **Default Windows npm installation is broken**: the distributed worker is unsigned, while the installer requires a valid signature (`install-native.mjs:157`).
|
||||||
|
|
||||||
|
**Decision: hybrid mode**
|
||||||
|
|
||||||
|
1. Try UIA semantics, window messages and synthetic pointer input first. **Activate B and use SendInput only after pre-dispatch `background_unavailable`, with no side effect.** Dispatched but unconfirmed input reports uncertainty and is never retried.
|
||||||
|
2. Keep verified restoration of A and preserve a user-selected C. Port #4500, waiting for B to drain input before restoring, and #4901, checking foreground per character.
|
||||||
|
3. Stop and report an error if the user acts during foreground delivery, following Codex. Do not intercept physical input.
|
||||||
|
4. Initially use upstream's elevated daemon with ordinary-user tokens for applications it launches. Retain UIAccess code; signing is being requested.
|
||||||
|
5. `typeText` always inserts at the caret; `setValue` always replaces the whole value.
|
||||||
|
|
||||||
|
Other fixes: #3671 avoids fallback after timeout and duplicate clicks; #4774 implements native combo selection; refuse Tk clicks; set VK_LWIN flags; bound worker pipe reads.
|
||||||
|
|
||||||
|
### 2.3 Linux (10-16)
|
||||||
|
|
||||||
|
**Analysis**
|
||||||
|
|
||||||
|
- X11 and Wayland are different display protocols.
|
||||||
|
- X11 has a central server and no application isolation, allowing cross-window input, screenshots and global simulation.
|
||||||
|
- Wayland delegates to the compositor, isolates applications, and requires compositor-specific automation access.
|
||||||
|
- Codex supports only X11; its 26.1007 documentation and code contain no Wayland implementation.
|
||||||
|
- Upstream has full X11 support; Wayland is disabled by default and remains limited when enabled.
|
||||||
|
- X11 has four input routes:
|
||||||
|
1. AT-SPI semantics.
|
||||||
|
2. Directed SendEvent plus FocusIn, used by Codex and often ignored by GTK/Chromium.
|
||||||
|
3. A virtual second pointer through MPX + uinput, used by upstream and requiring real Xorg and uinput.
|
||||||
|
4. Foreground XTest, currently used by ours.
|
||||||
|
|
||||||
|
**Decisions**
|
||||||
|
|
||||||
|
- **Remove Wayland entirely and support only X11**:
|
||||||
|
- Remove 21 fork files: `platform-linux/src/wayland/*`, `video_wayland.rs`, `wayland-helper/`, libei examples, Wayland tests and documentation.
|
||||||
|
- Pure Wayland sessions report “X11 only.”
|
||||||
|
- X11 applications under XWayland remain supported.
|
||||||
|
- Do not synchronize upstream Wayland changes.
|
||||||
|
- **Design for real shared desktops**; a test machine is only a test environment. Foreground transactions retain “restore A, preserve C.”
|
||||||
|
- **Use the Windows structure**:
|
||||||
|
1. AT-SPI.
|
||||||
|
2. Virtual second pointer where supported.
|
||||||
|
3. Foreground XTest only after `background_unavailable`.
|
||||||
|
4. Stop on user input during foreground delivery.
|
||||||
|
5. Never retry dispatched but unconfirmed input.
|
||||||
|
- Use directed events only for Qt / Tk / Xt applications known to accept them.
|
||||||
|
- **Audit and port X11 / AT-SPI fixes after upstream 0.20.0**, checking whether the fork already contains each:
|
||||||
|
- Input: #3882, #3761, #3762, #3864, #3618, #4248, #4188 and Chinese text loss #5139 / #5158.
|
||||||
|
- MPX: #3601, #2888, #5070.
|
||||||
|
- AT-SPI: #4903, #4984, #5074, #4375, GTK combo `set_value`, #4292.
|
||||||
|
- Other: #3758, #4281, #3814, #4529, #4715, #3658, #3778.
|
||||||
|
- Repair fork defects as well:
|
||||||
|
- Send FocusIn only to Qt5, and follow Codex's FocusOut conditions.
|
||||||
|
- Restore focus after X11 activation fails.
|
||||||
|
- Fix per-window scaling identified 09-17, with 164 / 76 px offsets; double scaling in `mouse_button_up`; X11 Page_Up / Page_Down; and on-demand Chromium classification.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Topics 1–7 (10-16)
|
||||||
|
|
||||||
|
### Topic 1: runtime documentation replaces skills; simplify node-repl, following Codex
|
||||||
|
|
||||||
|
- **Tools**: expose only `js` (`code`, `timeout_ms` default 30 s, `title`) and `js_reset`. `add_node_module_dir` is host configuration. Remove wait / cancel and follow Codex timeout behavior.
|
||||||
|
- **Guidance**:
|
||||||
|
- MCP instructions contain one sentence.
|
||||||
|
- The `js` description explains only the entry point.
|
||||||
|
- First execution imports the SDK, creates global `cua` and prints core documentation for the current platform only.
|
||||||
|
- After compaction, the model calls `cua.rewriteDocumentation()`.
|
||||||
|
- **Entry**: `getApp` displays full state; observations display automatically, with `emit:false` to disable display.
|
||||||
|
- Remove the bundled computer-use and browser-use `SKILL.md` files afterward. Deleting these content files is the sole exception to leaving qwen-code itself untouched.
|
||||||
|
|
||||||
|
**Evidence**: historical wait requests used only to retrieve results occurred 1008 times and consumed 130 million tokens; Codex's five-layer guidance was measured through the MCP handshake.
|
||||||
|
|
||||||
|
### Topic 2: driver as the only backend; SDK as a thin wrapper
|
||||||
|
|
||||||
|
- **2a Move into the driver**:
|
||||||
|
- Application/window resolution: accept `app` directly and resolve in one round trip, replacing at least three SDK round trips per action.
|
||||||
|
- Short IDs.
|
||||||
|
- Observation rendering, character limits and diff.
|
||||||
|
- Foreground/background policy.
|
||||||
|
- Structured errors.
|
||||||
|
- The SDK retains global `cua`, typed methods, automatic display, runtime documentation and serialization per application. Generate MCP descriptions and runtime documentation from one interface definition.
|
||||||
|
- **2b Simplify tools**: expose a Codex-like set of core operations by default. Upstream recording, cursor themes, zoom, replay, administration and `browser_*` stay disabled by default and are opt-in.
|
||||||
|
- **2c** MCP tool names and parameters correspond one-to-one to JS methods.
|
||||||
|
|
||||||
|
### Topic 3: observation
|
||||||
|
|
||||||
|
- Split into `getAXState / getScreenshot / getAXStateAndScreenshot`, following Codex. Return a string, remove `.elements` and display automatically.
|
||||||
|
- Do not capture screenshots for text-only observations. Remove the SDK's fixed `includeScreenshot:true`; Linux may save 1.6–2 s per observation. **[Inference from the 09-18 diagnosis]**
|
||||||
|
- Return actions after dispatch. Wait for stability during observation, initially 300–500 ms, with the final limit based on tests.
|
||||||
|
- First `getApp` and window changes return full AX without a screenshot; later observations default to diff. **All three platforms use diff**, intentionally differing from Codex's full Linux/Windows observations.
|
||||||
|
- Also repair per-window screenshot mapping, avoid recapture after incomplete AX (4.4–5.3 s in Chrome), match image MIME to bytes, and wake Chromium AT-SPI before its first tree read (#5074).
|
||||||
|
|
||||||
|
### Topic 4: action results and errors
|
||||||
|
|
||||||
|
- Success returns `void`, without `effect`. Confirm the result through observation within the same code invocation.
|
||||||
|
- Failures throw `{code, reason, sent: true/false, next-step advice}` from the driver, preserved by SDK and direct MCP. Initial codes:
|
||||||
|
|
||||||
|
| Code | Meaning |
|
||||||
|
| -------------------------------------- | -------------------------------------------- |
|
||||||
|
| `element_stale` | ID from an older snapshot; no input sent |
|
||||||
|
| `not_settable` | Control cannot be written directly |
|
||||||
|
| `write_not_applied` | Readback did not change |
|
||||||
|
| `no_focused_text_input` | Focus is outside a text field |
|
||||||
|
| `target_occluded` | Target point is covered |
|
||||||
|
| `unsupported_on_platform` | Operation unsupported on this platform |
|
||||||
|
| `user_input_detected` | Foreground delivery stopped on user activity |
|
||||||
|
| `foreground_unavailable` | Foreground activation failed; no input sent |
|
||||||
|
| `app_not_running` / `window_not_found` | Application not running / no window found |
|
||||||
|
|
||||||
|
- Do not expose `background_unavailable` to the model; the driver handles its permitted foreground fallback.
|
||||||
|
- IDs are valid only for the selected window's latest snapshot. Refuse old IDs before dispatch.
|
||||||
|
- Sent but unconfirmed input does not throw: return normally with “Sent but not confirmed; observe before retrying.”
|
||||||
|
|
||||||
|
**Evidence**: in macOS smoke tests, three distinct failures were rewritten by the SDK into the same generic message, forcing another observation to determine the cause.
|
||||||
|
|
||||||
|
### Topic 5: capability gaps
|
||||||
|
|
||||||
|
- **5a Browser**: reuse and migrate existing `@qwen-code/browser-use`. It already runs in node-repl, using Playwright over CDP through the Qwen Chrome extension to the user's Chrome, with void actions and coded errors. Expose `getBrowser / getTab / createBrowserTab` under `cua`, **without functional changes or upstream browser code**. Do not add Windows or diff support yet.
|
||||||
|
- **5b** Add explicit `cua.desktop`: full-display screenshots and screen-coordinate input. This naturally uses foreground and is selected only for explicit desktop operations.
|
||||||
|
- **5c** Add click modifiers, `click(p, {key})`, and `key_down / key_up`.
|
||||||
|
- **5d** Do not add per-application authorization.
|
||||||
|
|
||||||
|
### Topic 6: distribution and versions
|
||||||
|
|
||||||
|
- **6a** Leave qwen-code cli/core unchanged; node-repl remains separately registered MCP.
|
||||||
|
- **6b** Give SDK, native packages and browser SDK one version. Stop following upstream version numbers; record “ported from upstream #xxxx” in the changelog.
|
||||||
|
- **6c** Download native payloads on demand and verify SHA256 using the existing mechanism.
|
||||||
|
|
||||||
|
### Topic 7: evaluation
|
||||||
|
|
||||||
|
- Record tool and model evaluation separately; the user moved OSWorld evaluation after release on 2026-10-20:
|
||||||
|
- **Layer 1, tools without models**: compare the candidate, Codex and upstream on the same cases for correctness, action latency, interference and success-without-effect. The driver merge gate is at least the better comparator. The user clarified on 2026-10-20 that the observed transient macOS target/menu cover is acceptable and expected; record it and recovery separately rather than applying a strict zero-cover blocker. This does not accept wrong-target input, changed click semantics or replay after uncertain delivery.
|
||||||
|
- **Layer 2, small model/OSWorld evaluation after release**: primary metrics are **rounds, tokens and wall-clock time**, provided success rate does not regress. This is post-release follow-up rather than a pre-release gate; no improvement or nonregression result is claimed before execution.
|
||||||
|
- **Tasks**:
|
||||||
|
- macOS and Windows use 20 OSWorld-V2 tasks selected in two AP groups (`sh-prod-1`, Qoder + Node REPL scaffold, `azure.gpt-6-astra`):
|
||||||
|
- `group-5cff67aa55934a09a45160e3194b5801-s1`: macOS, mean 0.26, one full-score task.
|
||||||
|
- `group-bc7a3539903b4735bc3508e04e5791d5-s1`: Windows, mean 0.12, zero full-score tasks.
|
||||||
|
- Both use task_v2_001, 010, 020, 022, 027, 038, 044, 046, 051, 052, 055, 059, 060, 067, 076, 080, 088, 096, 101, 107.
|
||||||
|
- Linux initially uses the same 20 IDs, replacing missing/inapplicable tasks from the Linux set. **[Pending verification]**
|
||||||
|
- **Rules**: change one variable at a time; record infrastructure failures separately and exclude/rerun them; missing usage yields lower bounds only; reuse existing comparator trajectories.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Implementation checklist by package
|
||||||
|
|
||||||
|
**cua-driver, Rust**
|
||||||
|
|
||||||
|
- Platform dispatch: macOS section 2.1, Windows hybrid section 2.2 and Linux X11 hybrid section 2.3. The driver decides; SDK no longer forces foreground.
|
||||||
|
- Remove Wayland and reject pure Wayland sessions.
|
||||||
|
- Move app/window resolution, short IDs, rendering/diff on all three platforms, waiting and structured errors into the driver.
|
||||||
|
- Simplify default MCP tools; align tools and JS methods; generate descriptions and documentation from one interface.
|
||||||
|
- Add desktop operations, click modifiers and `key_down / key_up`.
|
||||||
|
- Port macOS protection, Windows #4500 #4901 #3671 #4774 and the Linux X11 / AT-SPI list.
|
||||||
|
- Use the elevated Windows daemon topology; installation no longer depends on signing.
|
||||||
|
|
||||||
|
**cua-sdk, TS**
|
||||||
|
|
||||||
|
- Thin wrapper: global `cua`, typed methods, automatic display with `emit:false`, platform documentation, `rewriteDocumentation()` and per-app serialization.
|
||||||
|
- Remove App business logic: forced foreground, generic error rewriting and fixed screenshots.
|
||||||
|
- Wrap browser-use through `cua.getBrowser / getTab / createBrowserTab`.
|
||||||
|
|
||||||
|
**node-repl-mcp**
|
||||||
|
|
||||||
|
- `js` + `js_reset`, default 30 s; remove wait / cancel; host-only `add_node_module_dir`.
|
||||||
|
- Preload `cua` and documentation in the startup banner.
|
||||||
|
|
||||||
|
**browser-use**: reference and migrate only, without functional changes.
|
||||||
|
|
||||||
|
**qwen-code itself**: delete only the two bundled computer-use/browser-use `SKILL.md` files; no code changes.
|
||||||
|
|
||||||
|
**Versions**: one shared version and changelog port references.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Test environment snapshot (10-16)
|
||||||
|
|
||||||
|
| Platform | Machine | Status at that time |
|
||||||
|
| -------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| macOS | mini, 30.221.187.223, `/Users/tutu/cua-fgbg-20261015` | Five harness problems repaired. Formal three-implementation × three-run batch, covering fixture cases and TextEdit / Excel / Finder / Chrome, finished pending analysis. Upstream T13 / T14 / T16 awaited Automation approval; T15 lacked LibreOffice. |
|
||||||
|
| Windows | windows-ecs-new, 8.217.1.105, `C:\cua-fgbg-20261016` | Upstream 0.35.4 and ours 0.20.11 prepared; new test certificate `83923F67…AB98`, valid through 2027-01-14, with signed/timestamped worker. SentinelA / UserTyper / Sampler compiled. Test processes still needed interactive desktop session 2; the borrowed-token launcher failed. An explicit choice remained between a one-off `schtasks /it` task and a runner started by the user in remote desktop. |
|
||||||
|
| Linux | linux-ecs, 47.76.93.28 | Dedicated `cua-fgbg-osworld-20261016` container using OSWorld-V2, QEMU software emulation and ports 127.0.0.1:50916 / 58916; existing containers preserved. Harness pending. |
|
||||||
|
|
||||||
|
AP at that time: mini ap-client upgraded from ArtLab to 0.5.1 and passed its version gate. The two group exports were reported at `/Users/tutu/cua-fgbg-20261015/ap-groups/`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Corrections made during discussion
|
||||||
|
|
||||||
|
- “Codex waits conditionally for about 2 s after actions” was incorrect. An unanswered authorization dialog delayed screenshot capture until timeout; the formal batch had no such wait.
|
||||||
|
- The “official: input methods activate the target window” quotation used for `0ceac09` came from Codex's **Windows** SDK, not macOS.
|
||||||
|
- The launcher borrowing an explorer token still produced session-0 children. Fixture results from that session were invalidated.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Document index
|
||||||
|
|
||||||
|
| Document | Contents |
|
||||||
|
| -------------------------------------------------------------------------- | ------------------------------------------------------------------------- |
|
||||||
|
| `BRIEF.md` | Research brief |
|
||||||
|
| `findings/code-architecture.md` | Existing architecture |
|
||||||
|
| `findings/codex-latest-delta.md` | Codex 26.924 → 26.1007 changes |
|
||||||
|
| `findings/upstream-delta.md` | Upstream 0.20 → 0.35.4 delta and synchronization simulation |
|
||||||
|
| `findings/design-discussions.md` | Historical decision ledger, including conflicting 09-17 / 09-21 decisions |
|
||||||
|
| `findings/fgbg-mechanics.md` | Platform foreground/background mechanisms at code level |
|
||||||
|
| `findings/fgbg-evidence.md` | Experiments and decision evidence |
|
||||||
|
| `findings/rb-final-comparison.md`, `rb-sdk-rounds.md` | RecreationBench comparison and round analysis |
|
||||||
|
| `findings/osworld-stable20.md` | OSWorld-V2 and macOS stable20 evidence |
|
||||||
|
| `topic4-fgbg/codex-fgbg-2026-09-29-reconstructed.md` | Reconstructed 09-29 Codex platform analysis |
|
||||||
|
| `topic4-fgbg/macos.md`, `macos-test-plan.md`, `macos-shakeout-20261015.md` | macOS analysis, plan and smoke observations |
|
||||||
|
| `topic4-fgbg/windows.md`, `windows-test-plan.md` | Windows analysis, decisions and plan |
|
||||||
|
| `topic4-fgbg/linux.md` | Linux analysis and decisions |
|
||||||
|
|
||||||
|
## 8. Current execution scope (2026-10-19)
|
||||||
|
|
||||||
|
The user prioritizes the first four verification items: macOS, Windows and Linux X11 native acceptance, followed by runtime, installation and package checks. OSWorld evaluation is scheduled after release by the user’s 2026-10-20 decision; future model runs use `gpt-6-sol`. On 2026-10-20 the user requested a fresh same-mini Codex tool comparison and clarified acceptance of observed transient macOS cover. All remaining native GUI tests run on mini. The 10-16 environment table above is historical. Current evidence and candidate identities are recorded in the [acceptance ledger](cua-upgrade-acceptance-2026-10.md#acceptance-clarification-2026-10-20). The user now authorizes pushing this feature branch and submitting its PR to main. OSWorld results are not a release prerequisite; production release execution is a separate action and no benchmark result is marked passed.
|
||||||
@@ -0,0 +1,347 @@
|
|||||||
|
# qwen-code Computer Use 升级方案与讨论记录(2026-10-15 ~ 10-16)
|
||||||
|
|
||||||
|
[English](cua-upgrade-2026-10.md) | [简体中文](cua-upgrade-2026-10.zh-CN.md)
|
||||||
|
|
||||||
|
本文汇总这两天讨论的背景、分析过程和全部决定,供后续实现和查阅。
|
||||||
|
|
||||||
|
- 每条决定都标了日期。细节和证据见文末"文档索引"里的分题文档。
|
||||||
|
- 标注约定:**【实测】** 有测试或数据支撑;**【推断】** 是根据代码或现象做的判断;**【待测】** 需要后续测试确认。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. 背景
|
||||||
|
|
||||||
|
**范围**
|
||||||
|
|
||||||
|
- `packages/cua-driver`:Rust driver。从 trycua/cua 的 `cua-driver-rs-v0.20.0` fork 而来,以 qwen-cua-driver / QwenCuaDriver.app 发布。
|
||||||
|
- `@qwen-code/cua-sdk`(`packages/cua-driver/typescript`,当前 0.20.11)。
|
||||||
|
- `@qwen-code/node-repl-mcp`(`packages/node-repl`,当前 0.1.7)。
|
||||||
|
- bundled `computer-use` skill。
|
||||||
|
- 浏览器 SDK `@qwen-code/browser-use`(0.25.2),在议题 5 里纳入。
|
||||||
|
|
||||||
|
**为什么升级**
|
||||||
|
|
||||||
|
1. 市面上的方案有了新进展:
|
||||||
|
- Codex 的 `cua_repl`:模型只看到 `js` 工具,文档在运行时下发,`getApp` 自动输出状态,动作返回 void;
|
||||||
|
- 上游 cua-driver 到了 0.35.4,两个版本之间有 499 个提交。
|
||||||
|
2. RecreationBench 和 OSWorld-V2 实验暴露的问题:
|
||||||
|
- **质量接近,效率差**:RecreationBench 最终 12 组、450 个结果里,我们和 Codex 官方、QoderCU 质量相当,token 接近,但用时约 1.5 倍;单轮输入在 110–130k,接近 128k 上限,所以成本几乎和轮次成正比【实测】。
|
||||||
|
- **轮次多**:macOS stable20 第一轮里,我们的生成次数是官方的 2.47 倍,单次输入是 2.05 倍【实测】。三方都通过的 41 题,调用数分别是官方 4、Qoder 12、我们 28【实测】。OSWorld-V2 的候选方案请求数约为 direct 的 2 倍【实测】。
|
||||||
|
- **macOS 一律前台的代价**:09-13 的提交 `0ceac09` 把 macOS 输入改成"一律先激活 B 再输入",依据是一句被误认的 Codex Windows 文档。改后总时间 +37%;一个 cell 里 5 个动作,我们 8.61 s,官方 1.61 s【实测】。
|
||||||
|
|
||||||
|
**总目标(10-15 定)**:用尽可能少的轮次、少的 token、清晰简洁的步骤完成 computer-use 任务。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 总体方向(10-15 定)
|
||||||
|
|
||||||
|
1. 和 Codex 一样:运行时下发文档,替代 skill;简化 node-repl 工具(去掉 wait / cancel)。
|
||||||
|
2. 只维护一套底层服务:driver 始终可以作为底层工具直接调用;node-repl 路径就是 SDK 包装同一个 driver。和上游的同步从"代码合并"改为"移植或引用具体的修复和功能"。
|
||||||
|
3. 不跟进上游的 `run_script` / `run_steps` 和它的上层调用接口。同步只看底层操作的丰富度、稳定性和效率。
|
||||||
|
4. 前台/后台按平台单独讨论(见第 2 节)。
|
||||||
|
5. 评测目标:少轮次、少 token、步骤清晰。
|
||||||
|
|
||||||
|
**补充(10-16)**:driver 作为不经过 node-repl、直接调用的 MCP 工具,也必须完整可用。这条路径同样重要。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 前台 / 后台(议题 4)
|
||||||
|
|
||||||
|
场景统一为:用户在 A 里打字,Agent 要操作被 A 挡住的 B。
|
||||||
|
|
||||||
|
### 2.1 macOS(10-15 定)
|
||||||
|
|
||||||
|
**分析**
|
||||||
|
|
||||||
|
- 把输入送到 B 只有三条路:
|
||||||
|
1. AX 辅助功能动作:按下、写值、设焦点,不需要 B 在前台;
|
||||||
|
2. 定向事件:事件指定投给 B 的进程,必要时先单独告诉 B"你已被激活"(下称"合成焦点"),系统真实前台不变;
|
||||||
|
3. 全局事件:B 必须先成为真前台,用户的按键和指针都会被影响。
|
||||||
|
- Codex 和上游都以 1、2 为主,我们反而以 3 为主(`0ceac09` 之后)。
|
||||||
|
- 我们每个动作后**固定等 1 s**,看有没有新窗口出现(`window_change_detector.rs:152`)。
|
||||||
|
|
||||||
|
**决定**
|
||||||
|
|
||||||
|
- 设计"AX + 定向事件"相结合的方案,借鉴 Codex 和上游。
|
||||||
|
- **前台(全局事件)路径先不出现**,Codex 和上游都不用它。
|
||||||
|
- 核实动作后的等待:"如果 Codex 没有任何等待也能正确完成任务,这个等待可能就是影响效率的点"。
|
||||||
|
- 拖动先试 Codex 的定向拖动;paste 对齐 Codex。
|
||||||
|
- 设计测试任务,用 Codex 官方和上游 0.35.4 的结果作为预期,在 mini 上验证。
|
||||||
|
|
||||||
|
**mini 冒烟的观察**(10-15/16,每个用例 1 次,【实测】但只是预备数据;正式批次已跑完,结果待整理)
|
||||||
|
|
||||||
|
- **Codex 动作发出就返回,没有固定等待**:控件动作约 100–150 ms;点开 sheet 后不观察、直接在 sheet 里输入,3/3 成功。冒烟时看到的约 2 s 等待,是 Codex 服务的系统授权弹窗没人处理造成的,弹窗点掉后就消失了。
|
||||||
|
- **Codex 的合成焦点有效**:B 自认为激活、成为键盘焦点,系统前台仍是 A。靠这一步,它能在后台 `cmd+v` 粘贴、在 sheet 里按回车;上游做不到这两件事,但会明确拒绝,不假装成功。
|
||||||
|
- **Codex 的缺口**:
|
||||||
|
- `typeText` 丢中文和 emoji(0/3),接口照样返回成功;
|
||||||
|
- 坐标右键 A 和 B 都收不到;
|
||||||
|
- 拖放和列表重排返回成功但没生效,而且 3 次里 2 次吞掉了用户的一个按键;只有拖滑块成功。
|
||||||
|
- **上游的问题**:
|
||||||
|
- 弹出菜单的 setValue 能用,但文本框 setValue 不触发应用自己的变更回调;
|
||||||
|
- AXPress 对文本框报错 -25206,而且不回退;
|
||||||
|
- 每个动作约 1 s。
|
||||||
|
- **我们(强制前台)**:通过的用例最多,但每个动作约 1.1 s,B 被切到前台、指针被移动;还 A 时会把 B 刚打开的菜单关掉。
|
||||||
|
|
||||||
|
**据此确定的实现细则**(属于已定方向内的细化)
|
||||||
|
|
||||||
|
- 去掉动作后的固定等待,动作发出即返回;等待放进观察(见议题 3)。
|
||||||
|
- 拖放和列表重排在后台**明确拒绝**,并提示替代路径;滑块改用 setValue。不用 Codex 的瞬间拖动。
|
||||||
|
- paste 对齐 Codex:临时使用剪贴板,事后恢复。
|
||||||
|
- 输入:ASCII 走定向按键;非 ASCII 走 AX 在光标处插入并读回(避开 Codex 丢字)【待测】。
|
||||||
|
- 弹出菜单的 setValue 改为"打开菜单并选中该项"(移植上游)。
|
||||||
|
- 点击:支持"按下"动作的控件用 AXPress;其余用定向点击加合成焦点。
|
||||||
|
- 移植上游的后台保护:
|
||||||
|
- 不给 A 发失焦记录(#4943);
|
||||||
|
- key_focus_guard(#4943 #4955 #5091 #5129);
|
||||||
|
- B 自激活后切回 A,用户 1 s 内动过鼠标或修饰键就不切(#4976);
|
||||||
|
- 鼠标事件显式写修饰键状态(#5051);
|
||||||
|
- 保存和改名用 Apple Events 并读回验证(#5057);
|
||||||
|
- 焦点不在文本框时拒绝打字 `no_focused_text_input`;
|
||||||
|
- 写入未生效报 `write_not_applied`。
|
||||||
|
|
||||||
|
这些保护要和现有的 `app_focus.rs` 合成焦点合并成一个观察者。
|
||||||
|
|
||||||
|
- Codex 的可见光标:先不做(未表态即按不做处理)。
|
||||||
|
|
||||||
|
### 2.2 Windows(10-15/16 定)
|
||||||
|
|
||||||
|
**分析**
|
||||||
|
|
||||||
|
- 把输入送到 B 有四条路:
|
||||||
|
1. UIA 语义操作,不需要前台;
|
||||||
|
2. 窗口消息 PostMessage,不需要前台,但 Chromium、WPF、Tk 等经常不认;
|
||||||
|
3. 合成指针注入;
|
||||||
|
4. SendInput,必须是真前台。
|
||||||
|
- 三方现状:
|
||||||
|
- **Codex**:先激活 B(最多 20 轮,带 Alt 技巧),再用 SendInput;不保证还 A;有用户输入监测。
|
||||||
|
- **上游**:后台优先,已知会被丢弃的组合直接返回 `background_unavailable`。
|
||||||
|
- **我们 App**:一律激活 B + SendInput(连元素点击都跳过 UIA Invoke),再同步还 A,依赖签名的 UIAccess worker。
|
||||||
|
- 我们 fork 其实已经带着上游的后台实现(丢弃判定表等),只是 App 层强制成了前台。
|
||||||
|
- 已知缺陷:
|
||||||
|
- 前台打字只等 40 ms 就还 A,末尾的字可能进 A;
|
||||||
|
- 文本一次发完,中途不检查前台;
|
||||||
|
- **Windows 默认 npm 安装是坏的**:发布的 worker 未签名,安装器却要求签名有效(`install-native.mjs:157`)。
|
||||||
|
|
||||||
|
**决定:混合模式**
|
||||||
|
|
||||||
|
1. 先走上游式后台:UIA 语义 → 窗口消息 → 合成指针。**只有返回 `background_unavailable`(派发前拒绝,还没有副作用)时,才激活 B 并用 SendInput**。已派发但效果未确认的,报"未确认",不重试。
|
||||||
|
2. 还 A:保留"读回确认,用户已切到 C 就不拉回",补上 #4500(等 B 读完输入再还)和 #4901(逐字检查前台)。
|
||||||
|
3. 前台段期间用户在操作:与 Codex 一致,检测到就停下报错,不拦截物理输入。
|
||||||
|
4. 权限:先改用上游的"高权限 daemon + 它启动的应用用普通用户 token"。UIAccess 代码保留,签名在申请中。
|
||||||
|
5. 语义:`typeText` 一律在光标处插入;`setValue` 一律整体替换。
|
||||||
|
|
||||||
|
其余小修:#3671(超时不回退,避免双击)、#4774(combo box 真实选择)、Tk 点击拒绝、VK_LWIN 标志、worker 管道读取加超时。
|
||||||
|
|
||||||
|
### 2.3 Linux(10-16 定)
|
||||||
|
|
||||||
|
**分析**
|
||||||
|
|
||||||
|
- X11 和 Wayland 是两种显示协议。
|
||||||
|
- X11 有中心的 X server,应用之间不隔离,可以给别的窗口发事件、截屏、模拟全局输入。
|
||||||
|
- Wayland 由合成器统一管理,应用相互隔离,自动化只能靠各家合成器开的口子。
|
||||||
|
- Codex:只做 X11(26.1007 文档和代码里没有 Wayland)。
|
||||||
|
- 上游:X11 完整支持;Wayland 默认关闭,打开后也很受限。
|
||||||
|
- X11 上把输入送到 B 有四条路:
|
||||||
|
1. AT-SPI 语义操作;
|
||||||
|
2. 定向事件 SendEvent 加 FocusIn(Codex 的做法;GTK、Chromium 常常不认);
|
||||||
|
3. 虚拟第二指针 MPX + uinput(上游的做法,需要真实 Xorg 和 uinput);
|
||||||
|
4. 前台 XTest(我们现在的做法)。
|
||||||
|
|
||||||
|
**决定**
|
||||||
|
|
||||||
|
- **完全放弃 Wayland,只支持 X11**:
|
||||||
|
- 删除 fork 的 Wayland 后端,共 21 个文件(`platform-linux/src/wayland/*`、`video_wayland.rs`、`wayland-helper/`、libei 示例、Wayland 测试和文档);
|
||||||
|
- 纯 Wayland 会话直接报"只支持 X11";
|
||||||
|
- XWayland 下的 X11 应用照常支持;
|
||||||
|
- 不再同步上游的 Wayland 提交。
|
||||||
|
- **按真实使用场景设计**:要考虑共享桌面,测试机只是测试环境。前台段保留"还 A + 检查 C"。
|
||||||
|
- **与 Windows 同一结构**:
|
||||||
|
1. AT-SPI;
|
||||||
|
2. 虚拟第二指针(机器支持时);
|
||||||
|
3. 只有返回 `background_unavailable` 时才转前台 XTest;
|
||||||
|
4. 前台段期间检测到用户输入就停下;
|
||||||
|
5. 已派发未确认的不重试。
|
||||||
|
|
||||||
|
定向事件只用于已知能认的 Qt / Tk / Xt 应用。
|
||||||
|
|
||||||
|
- **同步上游 0.20.0 之后的 X11 / AT-SPI 修复**(逐个核对 fork 是否已有):
|
||||||
|
- 输入:#3882、#3761、#3762、#3864、#3618、#4248、#4188、中文丢字 #5139 / #5158;
|
||||||
|
- MPX:#3601、#2888、#5070;
|
||||||
|
- AT-SPI:#4903、#4984、#5074、#4375、GTK combo set_value、#4292;
|
||||||
|
- 其他:#3758、#4281、#3814、#4529、#4715、#3658、#3778。
|
||||||
|
- fork 自身的缺陷一起修:
|
||||||
|
- FocusIn 只对 Qt5 发,FocusOut 按 Codex 的条件发;
|
||||||
|
- X11 激活失败后还原焦点;
|
||||||
|
- 09-17 发现的坐标缩放按窗口区分(164 / 76 px 偏移)、`mouse_button_up` 重复缩放、X11 的 Page_Up / Page_Down、Chromium 进程分类按需执行。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 总议题 1–7(10-16 定)
|
||||||
|
|
||||||
|
### 议题 1:运行时文档替代 skill + 简化 node-repl —— 全部与 Codex 一致
|
||||||
|
|
||||||
|
- **工具**:模型只看到 `js`(`code`、`timeout_ms` 默认 30 s、`title`)和 `js_reset`。`add_node_module_dir` 只由宿主配置。删除 wait / cancel,超时行为照 Codex。
|
||||||
|
- **引导**:
|
||||||
|
- MCP instructions 只有一句话;
|
||||||
|
- `js` 工具描述只讲入口;
|
||||||
|
- 第一次执行时自动 import SDK、建好全局 `cua`,输出核心文档(只输出当前平台的部分);
|
||||||
|
- 上下文压缩后由模型调 `cua.rewriteDocumentation()`。
|
||||||
|
- **入口**:`getApp` 自动输出状态;观察方法默认自动显示,`emit:false` 关闭。
|
||||||
|
- 改完之后,从 bundled skill 里删掉 computer-use 和 browser-use 两个 `SKILL.md`。这是"不动 qwen-code 本体"的唯一例外,只删内容文件。
|
||||||
|
|
||||||
|
**依据**:历史上"只为拿结果"的 wait 请求有 1008 次、1.3 亿 token;Codex 的五层引导经 MCP 握手实测。
|
||||||
|
|
||||||
|
### 议题 2:driver 是唯一底层,SDK 只做包装
|
||||||
|
|
||||||
|
- **2a 下沉到 driver**:
|
||||||
|
- 应用和窗口的解析:动作直接接受 `app`,一次往返完成;现在 SDK 每个动作至少 3 次往返;
|
||||||
|
- 短 ID;
|
||||||
|
- 观察的渲染、字符上限和 diff;
|
||||||
|
- 前后台策略;
|
||||||
|
- 结构化错误。
|
||||||
|
|
||||||
|
SDK 只保留全局 `cua`、类型化方法、自动显示、运行时文档和同一应用的串行。一份接口说明同时生成 MCP 工具描述和运行时文档。
|
||||||
|
|
||||||
|
- **2b 精简工具面**:默认只暴露类似 Codex 的核心十几个工具。上游的大量工具(录屏、光标主题、zoom、轨迹回放、管理类、`browser_*`)默认关闭,按需打开。
|
||||||
|
- **2c** MCP 工具名和参数与 JS 方法一一对应。
|
||||||
|
|
||||||
|
### 议题 3:观察
|
||||||
|
|
||||||
|
- 照 Codex 拆成 `getAXState / getScreenshot / getAXStateAndScreenshot`,返回字符串,去掉 `.elements`,默认自动显示。
|
||||||
|
- 只要文本时不截图:去掉 SDK 固定传的 `includeScreenshot:true`。Linux 上每次观察能省 1.6–2 s【推断,依据 09-18 定位】。
|
||||||
|
- 动作发出即返回;观察时等界面稳定再采集,上限按测试数据定,先从 300–500 ms 起步。
|
||||||
|
- 首次 `getApp` 和切换窗口时给全量,只输出 AX、不带截图;之后默认 diff。**三平台都用 diff**(有意不跟 Codex:Codex 在 Linux / Windows 上始终全量)。
|
||||||
|
- 一并修复:
|
||||||
|
- 截图坐标映射按窗口区分;
|
||||||
|
- AX 不完整时不自动重截(Chrome 上 4.4–5.3 s);
|
||||||
|
- 图片 MIME 声明与实际一致;
|
||||||
|
- Chromium 的 AT-SPI 首次读树前先唤醒(#5074)。
|
||||||
|
|
||||||
|
### 议题 4:动作返回值和错误
|
||||||
|
|
||||||
|
- 成功返回 `void`,去掉 `effect`。结果靠同一次调用里的观察确认。
|
||||||
|
- 失败抛出 `{code, reason, sent: true/false, 下一步建议}`。driver 给出,SDK 原样透传,MCP 路径字段相同。初始错误码:
|
||||||
|
|
||||||
|
| 错误码 | 含义 |
|
||||||
|
| -------------------------------------- | ------------------------------ |
|
||||||
|
| `element_stale` | 编号来自旧快照,没发送 |
|
||||||
|
| `not_settable` | 该控件不能直接写值 |
|
||||||
|
| `write_not_applied` | 写入后读回没变 |
|
||||||
|
| `no_focused_text_input` | 焦点不在文本框 |
|
||||||
|
| `target_occluded` | 点击点被挡住 |
|
||||||
|
| `unsupported_on_platform` | 本平台不支持该操作 |
|
||||||
|
| `user_input_detected` | 前台段检测到用户在操作,已停下 |
|
||||||
|
| `foreground_unavailable` | 转前台失败,没发送 |
|
||||||
|
| `app_not_running` / `window_not_found` | 应用没运行 / 找不到窗口 |
|
||||||
|
|
||||||
|
`background_unavailable` 不暴露给模型(driver 会自动转前台)。
|
||||||
|
|
||||||
|
- 元素编号只在该窗口最新一次快照内有效,用旧编号在发送前报错。
|
||||||
|
- "已发送但确认不了"不抛异常:动作照常返回,输出里加一行"已发送但未确认生效,重试前先看状态"。
|
||||||
|
|
||||||
|
**依据**:mac 冒烟里三种不同的失败被 SDK 改写成同一句通用文案,模型只能再观察一轮去猜。
|
||||||
|
|
||||||
|
### 议题 5:能力缺口
|
||||||
|
|
||||||
|
- **5a 浏览器**:直接引用、迁移仓库里的 `@qwen-code/browser-use`。它已经跑在 node-repl 里:Playwright 经 CDP 连到 Qwen Chrome 扩展,再到用户的 Chrome;动作返回 void,错误带错误码。在 `cua` 下以 `getBrowser / getTab / createBrowserTab` 接入,**不做功能改动**,不用上游 cua-driver 的浏览器内容。暂不补 Windows,暂不加 diff。
|
||||||
|
- **5b** 加显式的 `cua.desktop`:全屏截图和按屏幕坐标的输入。天然走前台,只在明确操作桌面时使用。
|
||||||
|
- **5c** 补"点击时按住修饰键"(`click(p, {key})`)和 `key_down / key_up`。
|
||||||
|
- **5d** 不做按应用授权。
|
||||||
|
|
||||||
|
### 议题 6:分发与版本
|
||||||
|
|
||||||
|
- **6a** 这次不动 qwen-code 本体(cli/core)。node-repl 仍作为单独注册的 MCP。
|
||||||
|
- **6b** SDK、原生包、浏览器 SDK 统一一个版本号。driver 不再跟上游版本号,changelog 里写"移植自上游 #xxxx"。
|
||||||
|
- **6c** 原生包按需下载,校验 sha256(沿用现有机制)。
|
||||||
|
|
||||||
|
### 议题 7:评测
|
||||||
|
|
||||||
|
- 工具层与模型层分别记录;用户于 2026-10-20 将 OSWorld 评测安排到发布后:
|
||||||
|
- **第 1 层:工具层,不用模型**。同一套用例对比新方案、Codex 和上游,看正确性、每个动作耗时、打扰项、返回成功但没生效的次数。作为 driver 改动的合入门槛:至少达到 Codex 和上游中较好的那个。用户于 2026-10-20 澄清,已观察到的 macOS 目标窗口/菜单短暂遮挡可接受且符合预期;逐项记录遮挡与恢复,不再以严格零遮挡阻塞。错误目标输入、改变点击语义或不确定后重放仍不被接受。
|
||||||
|
- **第 2 层:发布后的小规模模型/OSWorld 评测**。主指标是**轮次、token、墙钟时间**,前提是成功率不降。此项为发布后跟进,不再作为发布前门槛;实际执行前不声称性能提升或成功率不回退。
|
||||||
|
- **任务集**:
|
||||||
|
- mac 和 Windows 用 AP 上两个 group 筛过的 OSWorld-V2 20 题(`sh-prod-1`,scaffold 为 Qoder + Node REPL,模型 `azure.gpt-6-astra`):
|
||||||
|
- `group-5cff67aa55934a09a45160e3194b5801-s1`:macOS,平均 0.26,满分 1 题;
|
||||||
|
- `group-bc7a3539903b4735bc3508e04e5791d5-s1`:Windows,平均 0.12,满分 0 题。
|
||||||
|
- 两组是同一组题号:task_v2_001、010、020、022、027、038、044、046、051、052、055、059、060、067、076、080、088、096、101、107。
|
||||||
|
- Linux 计划用同样这 20 个题号,缺的或不适用的从 Linux 全集补齐【待核实】。
|
||||||
|
- **规则**:
|
||||||
|
- 每次只改一个变量;
|
||||||
|
- 基础设施失败单独记录、剔除重跑;
|
||||||
|
- usage 缺失的轮次只算下限;
|
||||||
|
- 对照组复用已有轨迹。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 实施清单(按包)
|
||||||
|
|
||||||
|
**cua-driver(Rust)**
|
||||||
|
|
||||||
|
- 前后台:macOS(2.1)、Windows 混合(2.2)、Linux X11 混合(2.3)。driver 自动决策,不再接受 SDK 强制前台。
|
||||||
|
- 删除 Wayland 后端;纯 Wayland 会话直接报错。
|
||||||
|
- 下沉:应用 / 窗口解析、短 ID、观察渲染与 diff(三平台)、等待逻辑(动作即返回,观察里等)、结构化错误码。
|
||||||
|
- 精简默认 MCP 工具面;工具和 JS 方法一一对应;一份接口说明生成工具描述和运行时文档。
|
||||||
|
- 新增:`desktop` 操作、点击按住修饰键、`key_down / key_up`。
|
||||||
|
- 移植上游修复:mac 保护、Windows 修复(#4500 #4901 #3671 #4774 等)、Linux X11 / AT-SPI 修复清单。
|
||||||
|
- Windows 改用高权限 daemon 形态;安装不再依赖签名。
|
||||||
|
|
||||||
|
**cua-sdk(TS)**
|
||||||
|
|
||||||
|
- 变成薄包装:全局 `cua`、类型化方法、自动显示(`emit:false`)、运行时文档(按平台)、`rewriteDocumentation()`、同一应用串行。
|
||||||
|
- 删除 App 层的业务逻辑:强制前台、通用错误改写、固定截图。
|
||||||
|
- `cua.getBrowser / getTab / createBrowserTab` 包装 browser-use。
|
||||||
|
|
||||||
|
**node-repl-mcp**
|
||||||
|
|
||||||
|
- 工具改为 `js` + `js_reset`,默认 30 s,删 wait / cancel;`add_node_module_dir` 只由宿主配置。
|
||||||
|
- 启动 banner 预载 `cua` 并输出文档。
|
||||||
|
|
||||||
|
**browser-use**:只做引用和迁移,不改功能。
|
||||||
|
|
||||||
|
**qwen-code 本体**:只删两个 bundled `SKILL.md`(computer-use、browser-use),代码不动。
|
||||||
|
|
||||||
|
**版本**:统一版本号;changelog 记录移植来源。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 测试环境现状(截至 10-16)
|
||||||
|
|
||||||
|
| 平台 | 机器 | 状态 |
|
||||||
|
| ------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| macOS | mini(30.221.187.223),`/Users/tutu/cua-fgbg-20261015` | harness 已修好 5 个问题。正式批次(三方 × 3 次,fixture 用例加 TextEdit / Excel / Finder / Chrome)**已跑完,结果待整理**。上游的 T13 / T14 / T16 需要人工点 Automation 授权,未跑;T15 因 mini 没装 LibreOffice 未跑 |
|
||||||
|
| Windows | windows-ecs-new(8.217.1.105),`C:\cua-fgbg-20261016` | 上游 0.35.4 和我们 0.20.11 已就绪:换了新的测试签名证书 `83923F67…AB98`,有效到 2027-01-14,worker 已签名并带时间戳。SentinelA / UserTyper / Sampler 已编译。**卡点:测试进程要在交互桌面 session 2 里运行**,借令牌的启动器没跑通,需要选一种方式(一次性计划任务 `schtasks /it`,或用户在远程桌面启动常驻执行器) |
|
||||||
|
| Linux | linux-ecs(47.76.93.28) | 新起了容器 `cua-fgbg-osworld-20261016`(OSWorld-V2 镜像,QEMU 软件模拟,端口 127.0.0.1:50916 / 58916),不影响已有容器。harness 待搭 |
|
||||||
|
|
||||||
|
AP:mini 上的 ap-client 已从 ArtLab 升级到 0.5.1,版本检查通过;两个 group 的导出在 mini 的 `/Users/tutu/cua-fgbg-20261015/ap-groups/`。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 讨论过程中的更正
|
||||||
|
|
||||||
|
- "Codex 动作后有约 2 s 的条件等待"是错的:冒烟时 Codex 服务的授权弹窗没人处理,截图卡到超时。正式批次里没有这个等待。
|
||||||
|
- `0ceac09` 改成一律前台时引用的"官方:输入方法自行激活目标窗口",出自 Codex 的 **Windows** SDK,不是 macOS。
|
||||||
|
- Windows 上借 explorer 令牌的启动器,实测子进程仍在 session 0,之前在那上面跑出的 fixture 结果作废。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 文档索引
|
||||||
|
|
||||||
|
| 文档 | 内容 |
|
||||||
|
| -------------------------------------------------------------------------- | -------------------------------------------------- |
|
||||||
|
| `BRIEF.md` | 调研任务说明 |
|
||||||
|
| `findings/code-architecture.md` | 现有代码结构 |
|
||||||
|
| `findings/codex-latest-delta.md` | Codex 26.924 → 26.1007 的变化 |
|
||||||
|
| `findings/upstream-delta.md` | 上游 0.20 → 0.35.4 的差异与同步模拟 |
|
||||||
|
| `findings/design-discussions.md` | 历史设计讨论账本(09-17 / 09-21 两次决定的冲突等) |
|
||||||
|
| `findings/fgbg-mechanics.md` | 三平台前后台机制(代码级) |
|
||||||
|
| `findings/fgbg-evidence.md` | 前后台相关的实验与决策证据 |
|
||||||
|
| `findings/rb-final-comparison.md`、`rb-sdk-rounds.md` | RecreationBench 对比与轮次分析 |
|
||||||
|
| `findings/osworld-stable20.md` | OSWorld-V2 与 macOS stable20 证据 |
|
||||||
|
| `topic4-fgbg/codex-fgbg-2026-09-29-reconstructed.md` | 09-29 Codex 三平台前后台逆向(重建版) |
|
||||||
|
| `topic4-fgbg/macos.md`、`macos-test-plan.md`、`macos-shakeout-20261015.md` | macOS 分析、测试方案、冒烟记录 |
|
||||||
|
| `topic4-fgbg/windows.md`、`windows-test-plan.md` | Windows 分析与决定、测试方案 |
|
||||||
|
| `topic4-fgbg/linux.md` | Linux 分析与决定 |
|
||||||
|
|
||||||
|
## 8. 当前执行范围(2026-10-19)
|
||||||
|
|
||||||
|
用户现阶段优先前四项验证:macOS、Windows、Linux X11 真机验收,以及运行时、安装与打包检查。用户于 2026-10-20 明确将 OSWorld 评测放在发布后,未来模型运行改用 `gpt-6-sol`。用户于 2026-10-20 要求重新进行同机 Codex 工具对照,并澄清已观察的 macOS 短暂遮挡可接受;剩余真机 GUI 测试均在 mini 执行。上方 10-16 环境表为历史快照。当前证据与候选身份见[验收账本](cua-upgrade-acceptance-2026-10.zh-CN.md#验收口径澄清2026-10-20)。用户现已授权推送此功能分支并向 main 提交本次 PR。OSWorld 结果不再是发布前置条件;正式发布执行为独立动作,未将任何评测结果标为通过。
|
||||||
@@ -0,0 +1,229 @@
|
|||||||
|
# Computer Use upgrade acceptance record (2026-10-16)
|
||||||
|
|
||||||
|
[English](cua-upgrade-acceptance-2026-10.md) | [简体中文](cua-upgrade-acceptance-2026-10.zh-CN.md)
|
||||||
|
|
||||||
|
Implementation follows the [approved decisions](cua-upgrade-2026-10.md) and [platform port ledger](cua-upgrade-platform-ports.md). Work is on `lazzy/cua-upgrade-2026-10`, based on `a8eb74fdf5b60851e1fc4a8e258e043273df4411`. Version `0.26.0` is the local development target and has not been released. At the F31 native checkpoint, no push, PR creation, shared-branch merge or release workflow had occurred; isolated local merge-validation commits are identified below. The subsequent PR submission decision is recorded next.
|
||||||
|
|
||||||
|
## PR submission and evaluation schedule (2026-10-20)
|
||||||
|
|
||||||
|
The user authorized this feature PR and explicitly moved OSWorld evaluation after release. The native/runtime acceptance facts below retain their candidate identities and limitations; absent benchmark results are not marked passed. The release audit also separates the three Computer Use npm packages from CLI version bumps and publication, while preserving their shared 0.26.0 version and dependency publication order. Production release execution has not been requested in this submission step.
|
||||||
|
|
||||||
|
## Acceptance clarification (2026-10-20)
|
||||||
|
|
||||||
|
The user confirmed mini unlocked, clarified that the previously observed transient macOS target/menu cover is acceptable and expected, and requested a fresh Codex comparison. Strict zero-cover is no longer a blocker. Historical strict-oracle failures below remain factual measurements of cover; they are not evidence that cover disappeared or that macOS makes prevention impossible. Functional correctness, exact target ownership, requested modifier semantics, uncertainty/no replay and recovery remain separate checks.
|
||||||
|
|
||||||
|
All remaining native GUI tests and the new comparison run on mini. The earlier local Writer preflight sent zero document actions, and the subsequently abandoned local comparator preparation sent zero Codex actions and was cleaned. Local work covers source, compilation, packages and merge validation. OSWorld/model evaluation remains deferred. The immutable clarification and new run reports are kept in the worktree's ignored evidence directories.
|
||||||
|
|
||||||
|
The adopted F31 repair passes two fresh mini Writer background workflows: the original observer reads the exact seed body unfocused before one public App click and focused afterward; one input changes the live text without changing the file; one toolbar Save persists exactly that text in valid FODT XML. The prior v27 negative and rejected F28/F30 experiments remain separate. F31 uses existing native focus preparation and a read-only caret query before the existing AXFocused setter; the public acknowledgement stays conservative and no uncertain action is replayed. Its native tree is `6abc23df3e6066a73f5110bfe77c041def4cc90852b5c5d6bde9ae537033eec3` (453 native plus 12 support inputs). Each successful Writer case has one document window; existing ambiguous-keyboard-destination protection remains and these cases do not prove arbitrary multi-window focus transitions.
|
||||||
|
|
||||||
|
Fresh v27 mini canvas, popup and context-menu effects pass 3/3 with receiver modifiers and recovery recorded; the F31 change is restricted to official LibreOffice text areas. The requested official Codex comparison is inconclusive: even the installed matching bundled CLI and existing mini consumer profile fail at application binding with app-server exit `-10005`, before any native action. A separate ordinary stdio initialization succeeds, so the cause is unresolved. No Codex functional failure, performance comparison or zero-cover conclusion follows. Comparator-owned services, sockets and fixtures were removed while the shared remote service was preserved.
|
||||||
|
|
||||||
|
## Local implementation
|
||||||
|
|
||||||
|
P1–P6 implementation and packaging changes are in an isolated worktree. Node REPL exposes only `js/js_reset`, defaults to 30 seconds, and preloads global `cua` and platform documentation. The driver owns app/window resolution, short IDs, observation diffs on all three platforms, screenshot coordinates and dispatch errors. Direct MCP exposes 19 default operations. macOS retains background AX and directed events; Windows/Linux may use foreground fallback only after explicit pre-dispatch refusal. Explicit desktop input and held keys belong to a session; normal end/reset awaits release. Browser integration returns the existing browser-use SDK objects without changing browser behavior.
|
||||||
|
|
||||||
|
Native Wayland implementation is removed; X11 applications, including X11 apps under XWayland, remain supported. SDK, native packages, browser-use and Node REPL share version `0.26.0`. Standalone npm installations pin dependencies and verify native download checksums. Default Windows installation does not require a signed UIAccess worker; explicit opt-in still checks signing and a safe installation path. Qwen CLI/core product changes consist solely of removing two bundled `SKILL.md` files.
|
||||||
|
|
||||||
|
Local checkpoints: P1 `cf89555254`, P2 `d2995a50c5`, P3 `b7c4d9c86c`, P4 `56a3ec2ba5`, P5 `fcc473d4a6`; P6 is carried by the commit containing this record.
|
||||||
|
|
||||||
|
## Evidence obtained
|
||||||
|
|
||||||
|
| Scope | Result and limits |
|
||||||
|
| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
|
| Repository build/types | Root build and typecheck pass, including final browser-use type checking. |
|
||||||
|
| Node REPL | 68 relevant tests pass, including asynchronous reset/dispose cleanup in real child processes. An actual standalone tarball installed into a consumer directory without repository dependencies passes documentation preload, variable persistence and reset checks. |
|
||||||
|
| SDK / browser-use | SDK: 51 pass, 6 skip. Browser: 594 pass. Wrapper tests use the original SDK; they do not establish Chrome GUI success. All three npm packages were actually packed; publishing was dry-run only. |
|
||||||
|
| Linux native | Locked all-target checking and SDK/driver builds pass in an isolated ARM64 container. Coordinate tests: 38; Computer Use: 5. Real Xvfb desktop chain: 96/96; App coordinate chain: 29/29. Actual display resize was skipped. |
|
||||||
|
| Linux first installation | Real archive/checksum HTTP download, cache hit, incomplete-cache repair, bad-SHA refusal, missing-checksum refusal and postinstall: 14/14. The actual downloaded SDK/bridge loads from cache and passes GTK input, screenshot and key-release checks: 14/14. This does not use native overrides or development symlinks. |
|
||||||
|
| macOS native | All-target checking and ARM64/Intel SDK/bridge builds pass. Both combined universal files contain arm64/x86_64. Loading was executed only on ARM64; Intel runtime and notarization were not verified. |
|
||||||
|
| Windows | GNU cross all-target checking passes. P6 did not execute desktop/UAC tests. Pure-state regressions establish that post-dispatch UIA errors and worker disconnection no longer cause a second click. |
|
||||||
|
| Release configuration | Shared-version constraints, dependency publication order and production gates pass. The workflow was not triggered. |
|
||||||
|
|
||||||
|
Closing review repaired three consumer defects: an old normalization environment variable transformed canonical pixel coordinates twice; a post-dispatch Windows UIA error was considered safe to click again; direct MCP initialize still described obsolete tools and normalized coordinates. Failed reproductions and post-fix evidence are retained separately.
|
||||||
|
|
||||||
|
The final P6 native source manifest is v5: 688 files, tree SHA-256 `c8a662cdc5c53eaa4cbbd14b146b8e844a86fc060d1b8f16a8f7bb4014c7dbee`. It again passes Linux/Windows GNU all-target checks, actual MCP initialize/list 11/11, exact desktop chain 96/96, actual download 14/14 and cached SDK GUI chain 14/14. SDK/driver were rebuilt for v5; the bridge explicitly reuses the fresh v2 build because its source and ABI did not change. Earlier v2/v3/v4 results retain their original identities.
|
||||||
|
|
||||||
|
Raw logs and independent reports are in `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/` in this worktree. Main report: `.qwen/e2e-tests/cua-upgrade-2026-10.md`; P6 report: `p6-validation.md`; self-audit: `.qwen/investigations/cua-upgrade-2026-10/parent-self-audit.md`.
|
||||||
|
|
||||||
|
After P6 fixes, two complete diff/consumer audit passes found no further issues, and the independent P6 boundary review was CLEAN. Codex `/review` is unavailable in this environment; Qwen Review was not used as a substitute.
|
||||||
|
|
||||||
|
## Historical P7 checkpoints (2026-10-19)
|
||||||
|
|
||||||
|
mini is unlocked and executes actual AppKit input, screenshot and key-lifecycle tests through the existing GUI Node host. Windows has built the real MSVC driver, SDK and pinned Node bridge. Linux executes GTK/AT-SPI tests in an Ubuntu Xorg guest on ECS. Earlier lock-screen and cross-compilation-only limits no longer describe the whole current state; historical P6 results retain their artifact identities.
|
||||||
|
|
||||||
|
Review of upstream `0.20.0 → 0.35.4` found applicable fixes beyond the P4 ledger. Current changes and evidence include:
|
||||||
|
|
||||||
|
| Fix | Evidence and remaining limits |
|
||||||
|
| ------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| Linux top-left window coordinate offset (#4967) | Same GTK scene hits an adjacent row before the fix and the exact target afterward. |
|
||||||
|
| Linux missing `_NET_WM_PID` (#4214) | Removing the property loses discovery/observation before the fix. XRes preserves the correct PID, window, AX and screenshot afterward. |
|
||||||
|
| Linux temporary keyboard mapping lifetime | Candidate and upstream both lose remapped Unicode while the GTK main loop is busy for 3 seconds. The queue-barrier candidate preserves input in busy GTK/Chrome scenes. Final v5b timeout returns normal sent/unconfirmed without replay; missing protocol support refuses before input. This does not substitute for physical MPX or all toolkit/IME acceptance. |
|
||||||
|
| macOS non-finite window dimensions (#5125) | Actual CFNumber tests fail before and pass after, retaining normal and negative coordinates. |
|
||||||
|
| macOS screenshot PATH (#3755) | Narrow PATH reproduces ENOENT on the real machine; the candidate returns an actual 1920×1080 PNG under narrow and normal PATH. |
|
||||||
|
| macOS raw clicks (#2907/#4690/#5137) | Same-PID parent/sheet reproduces lost middle-click targeting and duplicate events. Two real-machine reruns restore exact-window targeting and the correct event-pair counts. Default App middle-click was already correct and is a separate route. |
|
||||||
|
| macOS Electron background scrolling (#4861/#5079) | Normal v4 SDK passes seven native cases: raw page/inner × AX/pixel and default App inner retain page +404px or inner +114px, with the other region unchanged. Exhausted inner raw/App refuse with actual sent:false and no offset changes. v2/v3's reversed -96px down movement remains the before result. Read-only native diagnostics identify ROW35 above the viewport as a false bottom sliver; v4 excludes the wrong document-order candidate. Each case makes one public request, without replay, foreground fallback or browser APIs. Main routes retain uncertain ACKs. Six geometry unit tests pass. v2's controlled 43 cases and the later controlled-program startup timeout remain separate; 44 passes are not claimed. Normal v7 independently repeats and passes all seven GUI cases. This does not establish all directions, AX layouts or human concurrency. |
|
||||||
|
| Element token/native-object binding (#3616/#3873) | Actual core/token/cache controlled interleavings reproduce old tokens reading new objects. Capture/resolve/reverse cases pass afterward and all three consumers compile. Windows native-v2 MSVC cache: 5 pass, 2 explicit skips; tokens: 30 pass. No GUI click was performed by this reproduction. |
|
||||||
|
| MCP schema (#4888) | Actual default 19 and EXTRA 70 wire declarations pass offline compatibility checks; runtime still refuses invalid target combinations. No real model provider was requested. |
|
||||||
|
| Windows GUI-host console (#5136) | MSVC tests verify no console appears for a console-less host and inheritance remains for a console host. |
|
||||||
|
| Windows screenshot failure cleanup (#4333) | Actual GDI allocation/fault tests pass without handle growth over repeated failures. Interactive screenshot acceptance remains separate. |
|
||||||
|
| Windows shortcut environment variables (#3975) | Real IShellLink returns an unusable literal path before, and resolves the target while retaining arguments after. |
|
||||||
|
| Implicit-session idle recovery (#4283) | The same real MCP connection resumes after about 34 seconds. Explicitly ended/named sessions remain terminated; old element tokens are invalid. |
|
||||||
|
| Recording ownership/stop/time (#3632/#3714/#5144/#5145) | Public entrypoints verify named ownership, encoder errors and timing. Independent retest finds an omitted implicit connection; a second fix passes same/cross-connection and lifecycle regressions. |
|
||||||
|
| macOS recording completion (#4238) | Real-machine before results include false empty recordings and files growing after success. Five candidate 120ms recordings decode fully immediately; file SHA remains unchanged one second later. |
|
||||||
|
| Recording AX budgets/session lock (#4215) | Actual ToolRegistry with a controlled provider establishes that a hung after-capture no longer blocks state/stop. Six cases cover the 2-second return ceiling, same-PID busy refusal, late-result discard and stop→start isolation. Two native MSVC tests cover recording-thread COM initialization and release order, with all-target checking. This is not a GUI AX-hang test. |
|
||||||
|
| macOS Retina metadata (#3328) | At the same actual display mode, 3600px/1800pt, the public interface returns 1 before and 2 after. This does not imply default click coordinates were wrong. |
|
||||||
|
| macOS accessory-app identification (#3456) | On mini, the same installed accessory app loses running status in both default/running_only before; native-v3 returns actual PID 9402 through both public entrypoints. Exact installation paths and every running instance are retained; another copy sharing a bundle ID is not substituted. |
|
||||||
|
| Symlinked output directories (#4210) | Manifest and public ToolRegistry tests refuse a new file below actual `/tmp`-style directory aliases before and resolve its authorized path with one dispatch after. Outside-root, broken-link and file-parent cases still refuse. This is not a GUI screenshot test. |
|
||||||
|
| Windows named-pipe cleanup (#4279) | A private Win32 pipe reproduces creation failure and recovery after release. Source adds bounded retry and shared PID/SDK cleanup. Native-v4 MSVC driver/SDK build, 6 cache tests and 30 token tests pass; production daemon retry/cleanup still needs the interactive desktop entrypoint. |
|
||||||
|
| Uninstall MCP configuration ownership (upstream `ff770f5d5`, etc.) | Isolated fixtures verify five foreign configuration classes are retained and removal uses the actual executable of the selected payload. No complete uninstall ran. Configuration may already be committed if later deletion fails; original-byte backups permit recovery. |
|
||||||
|
| Typed-schema numeric formats (#3462) | The upstream regression fails before and passes after. Inputs/outputs retain valid formats and literals; 37 contract tests and 1 generator test pass. |
|
||||||
|
| Windows installer ancestor protection (#4398) | Twelve independent controlled cleanup/legacy-delegation checks pass. PID cleanup protects ancestors and skips task End when a driver ancestor exists. Sealed UnitOnly passes without changing PATH or performing actual task/full installation. |
|
||||||
|
| Windows autostart/isolated installation (#3180/#4284) | 32 controlled tests, 11 native MSVC unit tests and all-target checks with restored compile fixtures pass. Cancellation/null no longer reports success; replacement does not first delete the task; isolated release installation preserves the shared task. Real UAC, scheduler atomicity, login startup and full installation remain separate. |
|
||||||
|
| macOS release-install recovery (#3297) | Actual shell branches reproduce copy failure losing the old app before. Eighteen isolated scenarios and 350 independent checks pass after, covering staged/post-copy verification, copy/rename/link failure, retained backup on restore failure, first install, TERM and repeated cleanup. Permanent regression tests include original-byte restoration. These are controlled filesystem transactions, not actual install/signing/permission operations. No automatic TCC reset or SIGKILL/power-loss recovery is claimed. |
|
||||||
|
| Chromium AX readiness (#3783) | Sixteen controlled actual-function cases, 2 old/new publication races and 2 actual core budget-propagation tests pass. A shared 500ms readiness budget adds no polling for actions/screenshots/recording. Real Chrome refuses opt-in; the accepted path remains unverified. |
|
||||||
|
| AppKit launch-window recovery (#4309) | The 1-second active-search budget shares the original observation start. Actual-function controlled tests: 27; G17 outer-consumer regressions: 49. A fix removes active retry after sleep exhausts the budget. One Mac unit test and Mac/SDK all-target checks pass. Final App body/native IPC are outside this ceiling; real AppKit launch timing remains unverified. |
|
||||||
|
| Ordinary macOS AX traversal budget (#3882) | Controlled actual traversal reproduces raw/App reading after 1 second before. After: 27 traversal scenarios and 3 node-admission checks; preparation is separate, an exactly-at-deadline complete tree is not falsely truncated, and recording absolute budgets/legacy node limits remain intact. Actual core ToolRegistry/default App/Mac version/cache/token consumers pass 6 controlled snapshots and 22 independent checks: old-ID refusal, stable resolved bindings, partial-result warning and cleanup. Six tree, 4 cache and 27 core version unit tests plus Mac/SDK all-target checks pass. Full platform/SDK/GUI slow-AX behavior remains unverified; no whole-call 1-second hard deadline is claimed. |
|
||||||
|
| macOS backing-store capture (#4996) | Thirty-two controlled after checks pass; current-Space public/private raw/App routes return correct PNGs. A later normal candidate captures B on Space68 while A is foreground on Space1: raw/App images and five ROIs pass and are byte-identical. All 120 capture samples retain Space1 and foreground/topmost A; synthetic input reaches A 6/6. Normal binaries do not record the private route, so causality is not inferred. The full batch remains FAIL because its fullscreen-exit-button precondition was false; cleanup passes after owned processes exit. |
|
||||||
|
| macOS omitted AXWindow recovery (#4068/#4085/#4509) | Four native files add exact PID/SPI identity, restricted remote-token recovery and default App/tree/facts consumers. Three Mac unit tests, Mac/SDK all-target checks, 24 independent boundary checks and 49 actual outer-consumer checks pass. Actual omitted-window acceptance remains incomplete. |
|
||||||
|
| macOS special-key flags | After real Space preparation fails, a no-delivery native reproduction identifies missing structural flags shared by candidate/upstream. Four named-key builders pass native 128-keycode comparisons, explicit modifier/release and ambient-isolation checks; 11 unit tests and Mac/SDK all-target checks pass. One normal candidate Ctrl+Left is independently observed to return to Space1 about 638ms from call start. There is no resend; B does not receive the system-consumed Left, and its actual event flags are not inferred. |
|
||||||
|
|
||||||
|
The stable Linux nested-Xorg focus/end-session batch passes 31 checks; one insufficient TCG-timing case remains inconclusive. Unicode and mapping restoration pass under normal response. A later 3-second GTK-main-loop pause makes 13 remapped Unicode keypresses become NoSymbol in both candidate/upstream, independently identifying premature map restoration. The original frozen primary-GNOME batch remains an infrastructure confound. Client ping passes in GTK/Chrome, but Chrome responds before DOM input completes, so it cannot establish semantic success. Native-v5a/v5b separately validate mechanism and final response: timeout can still lose text and requires observation before another action.
|
||||||
|
|
||||||
|
MPX now confirms the complete ordered batch from the exact virtual keyboard. Unconfirmed/failed devices retain pending state; their next request refuses before mapping, focus or input. Initial warm-up also needs complete confirmation before temporary-key planning. Independent controlled lifecycle tests improve from 7/10 violations before to 10/10 passes after; 13 event-matcher cases, 12 Linux native tests and all-target checking pass. Real kernel inter-batch delay, hot-unplug and physical MPX were not established by those controlled results. Current v7 native regression adds 16/16 MPX and 7/7 key-alias passes; dedicated real uinput/evdev/Xorg acceptance is ongoing.
|
||||||
|
|
||||||
|
macOS T04 raises the target over the user's window throughout three native-v3 runs, about 3.1–3.3 seconds, while the user window remains front/key and receives all synthetic input. Dual-query diagnostics distinguish the complete window list from actual visible order. Fix 1 switches to visible order, but reactivation succeeds without restoring order and the rerun fails. Fix 2 raises only the current foreground AXFocusedWindow: three runs remove sustained cover but retain 99.4, 200.5 and 192.8ms transients. **The zero-interference gate has not passed.** One preparation-only diagnostic observes no cover, without excluding short transients or guard masking. A separate requested-pair-only switch to public CGEventPostToPid confirms actual click effect via source-PID mouseDown, native field editor and final forced AX focus, yet A observes 194.110ms cover and independent WindowServer ordering 196.695ms. Public publishing does not demonstrate a remedy and is not adopted in production. Synthetic keys reach A 10/10; only one key occurs within the click and none during observed cover, so this does not prove human-input safety during cover. T06 menu/T10 target self-activation regressions pass, as does exact Unicode readback through explicit Desktop foreground input. Later exact-window SPI tightening belongs to a new candidate. The independent two-top-level-window fixture has not started successfully; parent/sheet tests do not replace it.
|
||||||
|
|
||||||
|
Electron native scrolling has the main-route port and seven-case real-machine evidence, including the reproduced exhausted-inner reversal. Without visible anchors the original geometry heuristic remains; reordered/multi-column AX graphs remain outside the proof. Ordinary AX partial-result and actual-consumer controlled checks are complete; full SDK/native slow-AX boundaries remain. Cross-Space capture has the effect evidence above. Exact AX-window recovery, accepted Chromium readiness and AppKit launch recovery still require their own native cases. Cross-Space AX returns 85 elements with explicit incompleteness and does not prove remote-token recovery. Default App selection, pre-action checking and observation forwarding remain in scope; changing only the raw walker cannot establish completion.
|
||||||
|
|
||||||
|
Full-diff review also finds extra native references not owned by the macOS recording hook. Independent actual walker/hook/cache tests combine controlled AX data with real CoreFoundation retain counts: three before calls leak 1, 2 and 3 references; repeated complete/incomplete after calls return to zero, existing tokens remain valid and seven JSON payloads differ only in elapsed time. The fix reuses the unpublished snapshot owner without registering/replacing action tokens. Three native regression tests pass; all 39 evidence entries are verified byte-for-byte. This is not native GUI/AX acceptance and does not cover serialization failure or actual budget truncation.
|
||||||
|
|
||||||
|
In the same review, independent complete-tool-module tests with controlled providers reproduce raw double/right-click replay through pixels after semantic effect followed by AX error. Once AXOpen/AXShowMenu is attempted, the candidate returns normal sent/unconfirmed with a warning and does not replay. Successful semantic and unadvertised-semantic pointer routes remain. Six double-click and 4 right-click controlled cases pass; three uncertain responses retain unverifiable/accessibility and warnings through actual core public projection and typed validation. A mistaken actual-foreground report for a foreground request is also fixed: AX without a foreground transaction reports background. The added native projection regression and existing route test pass. Default App's early forwarding is a control, not evidence it shares the bug. An actual AX server effect-then-error GUI proof is still pending.
|
||||||
|
|
||||||
|
G34 explicitly rejects #5124/#4856's blanket native omnibox paste/newline refusal: the approved selected App/window contract allows native input at current focus. ASCII Return and non-ASCII AX insertion are tested separately; insertion readback does not prove navigation. Hotkey, pressKey and paste are reviewed individually. #4835 cold foreground field/selection differences remain a narrow raw/EXTRA reproduction task, unreachable through default App; no AllWindows or fixed stabilization sleep is introduced.
|
||||||
|
|
||||||
|
G07's latest production-entrypoint test exits during SDK creation in SSH Session0 for lack of an interactive user session, in 23ms, before reaching named-pipe acceptance. No private pipe exists and the shared driver is unchanged. Pipes themselves can work in Session0; that does not make this entrypoint reachable. G23's second standalone startup diagnostic compiles but has no readiness/stage logs after its single 8-second launch. Another diagnostic of the same binary and ordinary Node launch remains unready after that window. One 1-second sample exits 0 with 89 `_dyld_start +0` main-thread samples and unavailable images; it does not locate the wait mechanism. Neither case dispatches AXPress; the later diagnostic creates no SDK/AX session. No TCC/EDR cause is inferred or security settings changed on that basis. An existing Electron host with a task-owned native addon is being prepared as a new fixture.
|
||||||
|
|
||||||
|
All 499 upstream changes are classified; the former insufficient-evidence and unreviewed-P4-reference categories are empty. Applicable differences still need reproduction, porting or explicit disposition; classification is not acceptance. The latest 2026-10-19 whole-JavaScript build/typecheck pass; SDK: 51 pass, 6 explicit skips. The initial missing-corepack failure remains, followed by success using a task-local entrypoint and existing pinned local caches, without global installation. Normal v7 has 453 native inputs plus 12 exact compile-support files, tree `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693`. Mac/SDK, Linux/driver/SDK and Windows GNU/driver/SDK all-target checks and the default SDK build pass. Linux x64 and Windows MSVC driver/SDK releases now also build from that exact manifest. Windows adds 2 COM, 11 autostart and 1 bounded-capture pass. Linux's current normal direct-MCP GUI tests pass GTK 7/7 and Chrome 3/3. The unchanged core retains its exact-source eight-check checkpoint. Earlier omitted-fixture failures are retained.
|
||||||
|
|
||||||
|
Normal v7 independently passes G05's seven GUI cases using SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe`. Independent fixture IPC confirms inner +114px/page +404px and unchanged exhausted offsets. All 143 ordering/pointer/clipboard-count samples retain A; owned processes/LaunchAgents are cleaned and shared processes retain their lifetimes. The root verifies all 634 referenced evidence entries. Sampling is 8.444–12.213Hz, not proof of zero transient interference or human concurrency.
|
||||||
|
|
||||||
|
P7's first full audit covered 89 changed paths and found the recording/replay issues above. A later complete clean pass was followed by a second pass finding missing bilingual design documents; translation repair resets the consecutive-clean count. Two complete clean passes over the final updated scope are still required. Full three-platform regression remains ongoing. Detailed before/after results, invalid batches and artifact identities remain in `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/p7-*`; P6 results do not establish acceptance of subsequent P7 changes.
|
||||||
|
|
||||||
|
## Historical external acceptance checkpoint
|
||||||
|
|
||||||
|
Windows ordinary SSH is restored and current console/runner/shared-driver lifetimes are rechecked. The user explicitly approves the task-owned `\QwenCuaP7Acceptance` task: no triggers, existing Administrator InteractiveToken, Limited, no stored password, at most 2 hours, verified deletion afterward. Current v7 interactive acceptance is being prepared. No other process token is borrowed. The user also approves normal Automation dialogs for the existing mini Node host controlling Finder/TextEdit, and test-machine permissions needed for this work; no TCC database edits are required. Linux's dedicated real uinput/evdev/Xorg chain is a separate topology from the nested guest. Nested Xorg alone does not establish physical user input, MPX or XWayland acceptance.
|
||||||
|
|
||||||
|
Three-way tool comparison and OSWorld-20 model evaluation remain incomplete. Earlier provider 400s remain historical failures. Current gpt-6.1-sol passes global qwen 0.25.2 text preflight and actual CLI→MCP→Node REPL execution. The third effective synthetic-image preflight returns a real PNG and correctly identifies all four color/shape/position facts, using 2 model requests and 1 tool call. The first two timeout/transcription refusals produced no image and remain separate. No native desktop session was created, so this is not GUI acceptance.
|
||||||
|
|
||||||
|
All 20 local task definitions are located and frozen; they default to Linux and only two contain explicit Windows branches. A bounded mini search of 1,003 non-credential text files did not locate the historical adaptation. The existing stable20 set uses different tasks/scaffolding. The user now explicitly authorizes read/export of historical groups `group-5cff67aa55934a09a45160e3194b5801-s1` (macOS) and `group-bc7a3539903b4735bc3508e04e5791d5-s1` (Windows). Each actual AP request passes the exact official mini AP Client 0.5.1 version gate. Group metadata, jobs and evaluations are recovered; artifact export is ongoing. Raw exports stay private on mini; credentials are not copied into this repository or local artifacts. Both groups contain 20 terminal Succeeded jobs, which does not imply task success: macOS mean score 0.2613, 1/20 passed; Windows 0.1206, 0/20 passed. Their template commits differ (`d1b8341d17659205b9aa735e0390f54301be0dc7` / `b70f9f90b3df26f177d78279bc942490f7cc37e7`) and both use azure.gpt-6-astra, unlike the current gpt-6.1-sol preflight. No new AP job, retry or cancellation is authorized or performed. Current candidate OSWorld execution count remains 0, and no matched success-rate, token, turn, wall-time or Codex/upstream improvement claim is made. These pre-release gates remain active.
|
||||||
|
|
||||||
|
## Historical v13 P7 acceptance snapshot (2026-10-19)
|
||||||
|
|
||||||
|
This historical snapshot superseded the earlier checkpoints at v13. Its source identities and open items describe that checkpoint; the current scope and later verification are recorded below.
|
||||||
|
|
||||||
|
### Current candidate and distribution
|
||||||
|
|
||||||
|
The current native freeze is v13: 453 production/build inputs plus 12 exact compile-support files, tree SHA-256 `c43e1b300706eff833a9999f889b7f7c418a414b6863d3b3c102ca339770f83c`. Linux uses its fresh v13 CLI and SDK. macOS tests use the normal default-feature v11 SDK (`4131a02af8d64a88572e981dbd9dfa5e7d0eaa9ec5ac5509be341c159d59ce9d`); Windows uses the native MSVC v10 executable (`c5ca0054810eb4512986aef0f6231e81497ada15d1809bcb9bd5521016ef3fca`) and SDK (`91f871c885e13e856ee8284af0efed149232abbf58d9c7fc4584dee3d1f02283`). The intervening Linux-only changes require a target-specific source-equivalence audit before those other artifacts establish current acceptance.
|
||||||
|
|
||||||
|
All three publishable packages pass local npm pack dry runs at 0.26.0: CUA SDK 34 files, Browser Use 9 files and Node REPL MCP 46 files. The whole JavaScript build and typecheck pass, with SDK 51 pass and 6 explicit skips. The complete current working-tree snapshot merges without conflict with main `a250fd033941cab10c88d6eb8beb5756405193b9`; no branch/ref, push, PR or release was created by that object-only check. Final formatting and two complete clean source-audit passes remain required.
|
||||||
|
|
||||||
|
### Native fixes and actual effects
|
||||||
|
|
||||||
|
- macOS G23 now has an actual AppKit AX server proof: four v10 cases cover raw/default App with a handler returning NO or raising NSException after toggling. Each performs one request and one effect. The exception response is normal sent/unconfirmed with no pointer replay. Returning NO can still produce AX success and is kept separate. A v11 regression is pending.
|
||||||
|
- macOS default App T04 now uses exact editable AX focus for a single unmodified left click on a supported native field. Three fresh v11 click-only cases independently confirm the field editor and fresh AX focus without value changes or observed cover. All observed canary characters reach A, but none of those click request windows contains a new keyDown; they do not prove input concurrency. The full click-and-type regression passes; a higher-frequency overlap run is ongoing. Pointer-only diagnostic variants that still raised B are not adopted or counted as fixed.
|
||||||
|
- Linux MPX successful delivery now carries an explicit background/LinuxUinput execution record, avoiding a public action_outcome_mismatch after text had actually arrived. Ordered raw-key confirmation, pending-device refusal and session cleanup prevent replay after timeout.
|
||||||
|
- Openbox 3.6.1 promotes a virtual master's core FocusIn to global activation. Fresh zero-key controls attribute this to the WM/Xorg interaction. v12 refuses MPX keyboard before map/device/input work on that identified WM. Six native requests return background_unavailable/sent:false with zero input. The public hybrid route may then use foreground once. It restores the original foreground, core focus, pointer, map and stacking top; a lower-window order change remains observable and is not counted as background zero interference. Mutter 42.9 preserves the stronger complete order in the corresponding background tests.
|
||||||
|
- Linux AT-SPI InsertText requires a UTF-8 byte length while caret offsets remain character positions. The earlier implementation and upstream 0.35.4 truncated real mixed text. v13 fixes the shared length conversion before dispatch. Ten native/public/SDK/facade cases now read back exact CJK, emoji, combining characters and nonzero-caret insertion with zero raw key events. Readback does not change the runtime's conservative sent/unconfirmed projection.
|
||||||
|
|
||||||
|
### Platform verification and remaining native work
|
||||||
|
|
||||||
|
Windows v10 completes 14 native regressions, 68 direct-MCP/SDK/native/Edge checks, 10 production Node REPL kernel checks and 12 ordinary private production-serve checks. The last two cover actual kernel lifecycle and pipe ownership respectively, not a Node REPL MCP-wire GUI session or injected later-instance pipe failure. An ordinary serve supports low-level CuaDriver.connect; the retained ComputerUse.connect factory still requires authenticated session ownership and cannot bind that ordinary serve. Documentation now states that boundary. WinForms background samples preserve A and pointer; Edge typing deliberately uses the one permitted foreground fallback. Upstream 0.35.4 meets the expected results on 12 native and 8 Edge checks. No verified Windows Codex entrypoint was available.
|
||||||
|
|
||||||
|
Linux v13 passes 53 focused native tests, with one pre-existing ignored test, all-target checks and release builds. Ten semantic Unicode/caret cases and a real-kernel MPX smoke pass. v12 separately covers responsive/blocked/timeout/pending-refusal/forget-and-fresh lifecycle through native, SDK and Node facade consumers on private real Xorg displays. The external-input interruption proof uses a separate owned uinput device, not a human physical device. Neither a software-rendered Mutter container nor nested Xorg is presented as shared GNOME, native Wayland, physical-human concurrency or VM performance evidence.
|
||||||
|
|
||||||
|
The macOS v11 full fixture matrix has run, with its detailed independent readback still in progress. Remaining observations are retained as failures or limitations: sheet typing refuses with same_pid_keyboard_ambiguity; WKWebView typing refuses with no_focused_text_input; two short layer-order overlap samples need attribution. A non-document native field's AXValue is replaced while the fixture delegate callback stays absent; that stronger callback oracle remains false, and no second input is added to manufacture it. Normal TextEdit save and Finder rename still need their approved Automation regression. Clipboard generation alone cannot establish restored clipboard contents.
|
||||||
|
|
||||||
|
The temporary Windows task has been deleted and read back absent, its runner has exited, and all owned helpers are gone. Limited task configuration is recorded separately from the built-in Administrator's actual High integrity. Shared driver/process lifetimes are preserved. Linux input processes and temporary devices are removed, while the private display fixtures remain for final cleanup. macOS owns a desktop mutex and cleans exact run-owned process/LaunchAgent identities. Synthetic canaries and sampled focus/order observations are always distinguished from physical-user proof.
|
||||||
|
|
||||||
|
### Matched model evaluation
|
||||||
|
|
||||||
|
The approved read-only AP export is complete using the official mini AP Client 0.5.1 with the version gate before each actual request. Raw credential-bearing exports stay private on mini. Both historical groups use qodercli 1.1.52 and azure.gpt-6-astra. Their image/template versions differ. macOS scores average 0.26128 with 1/20 passed; Windows 0.12058 with 0/20 passed. Terminal Succeeded is not task success. Historical token counters are unavailable zero-filled fields, not zero consumption; only 13 Windows traces have complete final turn/time metrics.
|
||||||
|
|
||||||
|
The exact baked macOS pilot cases/plugins have been recovered, with all 20 instructions matching the AP exports. Windows matches 19/20: its historical 001 instruction names Calendar.app, while the recovered Windows override correctly uses Thunderbird Calendar. Base-layer seeds, genuine runner/generic scoring, websites and dependencies are being restored with complete layer hashes and platform-specific overlays. Setup must preserve existing files/profiles, use run-owned listeners/process cleanup, and fail preflight when Calendar persistence, application, codec or website requirements are missing. Scorer/gold files are never supplied to the autonomous model.
|
||||||
|
|
||||||
|
A fresh matched comparison must keep the harness, task seed bytes, instruction/rubric, model/provider and budgets constant across the intended old/new tool stacks. Historical Qoder results remain a reference, not that controlled baseline. The current gpt-6.1-sol global Qwen CLI passes text, real Node REPL and synthetic-image transport checks. A separately deployed production Node REPL MCP process on mini also passes initialize, exact js/js_reset listing, real kernel/preload, reset and EOF cleanup over the existing SSH connection without model credentials or a native GUI session. Actual candidate OSWorld task execution remains zero. Success-rate nonregression and round/token/wall-time gates remain open; no performance or three-way parity claim is made.
|
||||||
|
|
||||||
|
## Historical first-four scope (2026-10-19)
|
||||||
|
|
||||||
|
The user now prioritizes native macOS, Windows and Linux X11 acceptance plus runtime, installation and package checks. Codex/upstream comparisons and OSWorld/model evaluation are deferred. Future model runs are configured for `gpt-6-sol`; no new model/AP task is part of this scope. Completing these four items will not establish the deferred model gates or authorize publication.
|
||||||
|
|
||||||
|
### Candidate identity and completed checks
|
||||||
|
|
||||||
|
The validated v21 native source freeze has 453 production/build inputs and 12 support files, tree SHA-256 `11b0fafab2419448a16542a73cde08cea58ee67b3a7b922fdfe12fee456f9dad`. Relative to v20, only the CLI entry changes: when effective telemetry consent is false, a finite command runs directly instead of starting an unnecessary observer child. Consent precedence and the enabled path are preserved. The macOS backend remains the v20 SDK: pre-signing SDK binary SHA-256 `886bf05463a6c8273d0125b28482ee4f5ba8e9c1cffe5b4d5230e5fef2e77eee`, or `134b334ea8b7e912e51ac0c27383d71b6c245e17b8f99f69cbee6e08c07fb064` after the candidate Developer ID signature. Code-section equivalence is independently checked. Windows and Linux backend inputs remain target-equivalent to their v19 acceptance artifacts; new CLI checks keep their own version labels.
|
||||||
|
|
||||||
|
- macOS v20 passes fresh sheet typing and single-Return confirmation, WebKit click/type/submit, exact paste with clipboard type/byte restoration, and safe negative/positive two-window input. Production Node REPL MCP passes eight real wire/lifecycle checks with exact deployed source/dependency identities. TextEdit Unicode save and Finder semantic rename have independent C9 disk readback. Finder inline keyboard rename safely refuses when native parent/window ownership cannot be proved. The internal key-window preparation contains NaN-location mouse records: the old Swift JSON logger omitted them, so earlier claims of no internal mouse events are withdrawn.
|
||||||
|
- Windows v19 passes actual MSVC checking/building and 37 focused tests, retaining the independently identified native GUI regressions. A production MCP-wire action times out after its one real WinForms effect; no replay occurs, and reset/EOF cleanup passes. Complete CI-shaped packages pass owned fresh, official-old-byte layout upgrade, repeat, uninstall and purge checks. Unsigned explicit UIAccess deployment is rejected; neither signed UIAccess nor ARM64 native execution is claimed. The current isolated uninstaller passes 14 actual Windows PowerShell regressions, including a task mixing owned Exec and foreign ComHandler actions. Shared processes and the deleted temporary task remain protected.
|
||||||
|
- Linux v19 passes 53 focused tests, one explicit ignored test, actual release builds and native MCP initialization/listing of 19 tools. Real Mutter/Xorg/uinput input preserves the complete sampled focus/order and exact Unicode; production Node MCP passes 15 checks with real AX readback. CLI installation and SDK cold/warm cache tests use complete packages. Openbox fallback restores the original foreground/top window but changes lower stacking order; it is not counted as background zero interference. Synthetic owned input is distinct from physical-human concurrency.
|
||||||
|
- All three npm package dry runs pass at 0.26.0; SDK tests pass 51 with six explicit skips. The repaired Node MCP preload test holds SDK evaluation behind an observed barrier before timing out, then checks the next real request; the full MCP file passes eight tests, with 113 Node tests covered across the retained runs. Independent mutation evidence detects a late continuation. Installer Python checks pass 57 with five explicit skips and four subtests.
|
||||||
|
|
||||||
|
A subsequent v22 freeze (`de5d5030e7dbf5b3f64c7946dc8d335ee2d988db14e8fe0994226d2f6f980245`) adds an Excel-only Apple Events compatibility fix and corrects the window-order comment. Excel uses full-name1773 and saved1842 rather than the missing Standard Suite path/modified properties; a case-sensitive full-path guard precedes its one save. Exact and mismatched-path read-only argv probes pass. Six existing Apple Events unit tests, Mac/SDK/CLI all-target checks and the ARM release build pass; the SDK binary is `04d996ff2b51337bf07a72da408dee0ea0b987081b209bb680e28e8eb3f92499`. ARM and Intel release builds pass. Fresh v22 native Excel save also passes: one save returns in 316.586 ms, the modified flag changes true to false, and an independent XLSX oracle confirms the exact A1 marker with B1 unchanged. Both earlier dirty documents remain unchanged. Its click still produces 48.937083 ms of observed cover, so this is a save-function pass, not zero-interference acceptance. The updated v22 Mac package checks below also pass within their disclosed scope. Windows/Linux target inputs are unchanged.
|
||||||
|
|
||||||
|
### Runtime, installer and package verification
|
||||||
|
|
||||||
|
The v21 Windows/Linux CLI appendix now passes actual builds, existing consent/disabled tests, default-off and explicit-off finite commands, native MCP initialization/listing/EOF and current owned install/repeat/purge. Its 287 files and all four complete package archives have an independent parent hash/member readback.
|
||||||
|
|
||||||
|
On mochi, the coherent v22 macOS candidate uses the existing product Developer ID team `NF4574S59H` for CLI, theme, SDK and bundle, without timestamp or notarization. The signed SDK hash is `5a0dd6d011e633fee9bd6b751444ee82430cc0c8632a5ab43e5c4143cfa93946`; packaged code sections match the fresh ARM and Intel inputs. ARM and Rosetta Intel finite/native MCP checks, installed ARM checks, SDK cold/warm installation and twelve production Node MCP checks pass. Fresh install, upgrade from a private complete signed 0.17 bundle, and repeat install pass fourteen assertions; the replacement satisfies the old signing requirement. The original global bundle and v21 artifacts remain unchanged. The parent independently verifies all 636 regular evidence files (738 tar entries including 102 directories), all four package archives and their members, native identities and all-architecture signatures for CLI, theme, SDK and app. The unchanged CI NAPI artifact retains its documented unsigned Intel-slice boundary.
|
||||||
|
|
||||||
|
These checks complete the fourth item for v22 within the disclosed owned/local candidate scope: macOS ARM plus Rosetta Intel, and Windows/Linux x64 with verified unchanged target inputs. Installer tests use documented destination/writeability/download/stop-helper seams and real filesystem, signing and replacement operations. The two actual rollback fault cases are retained from v21 under the unchanged installer source, not newly executed v22 faults. Shared services are preserved; this does not establish stock global replacement, Windows signed UIAccess, ARM hardware coverage or official notarization. Earlier installed ad-hoc ARM failures remain retained; the coherent Developer ID candidates pass that installed runtime path without a security-policy change. The baseline absolute nested-local SDK import defect is retained with identical base/current loader bytes; installed bare-package and global `cua` consumers pass separately.
|
||||||
|
|
||||||
|
### Remaining gates and evidence limits
|
||||||
|
|
||||||
|
macOS canvas pointer input still raises the target over A for roughly 100–200 ms despite preserving foreground identity and canary delivery. Menu coverage also remains open. Exact effect plus eventual restoration does not satisfy zero cover. Authentication, public-post, key-preparation, default mouse-subtype and PrivateSource diagnostic variants retain coverage and are rejected; private variants never replace the normal artifact. PrivateSource produces one exact native effect but still about 199 ms of coverage. A read-only causal review does not establish system impossibility; none of these pointer factors addresses AX-only popup coverage. A fresh whole-preparation-omission counterfactual still covers A for about 206 ms and loses the native canvas effect, so that candidate is rejected too. The subsequent read-only review finds no evidence-backed next external-driver repair hypothesis; it does not prove system impossibility.
|
||||||
|
|
||||||
|
A fresh normal-v20 slider alternative passes: one `setValue('80')` produces both value80 and the independent native callback80, with no observed disturbance. Excel's original selected-grid state and LibreOffice's newly opened document state return `no_focused_text_input` before dispatch; neither is a saved-document pass. Excel Edit subsequently confirms the typed marker through its native formula-editor description and Return through A2. An initial save remains unknown after an Automation prompt and caller deadline; after normal authorization, a fresh document’s one save returns `automation_failed (-50)`, `sent:true`, with no disk marker. The v22 native save fix is verified above; its pointer coverage remains a separate failure. LibreOffice’s public body click leaves a group focused; a separate read-only body probe reports settable AXValue/AXFocused but actual focus false. No replacement input or saved-document pass is claimed. The native still-launching public chain now passes: an independently identified installed accessory fixture is still launching when its one `getApp` starts, with over 900 ms of native launch delay remaining; it returns the exact AX marker in 1.056 seconds, followed by independent finished-launching proof. Twenty action-plus-settle order samples show no owned coverage and six synthetic canaries arrive at A. This does not prove which retry ran or that the old 400 ms implementation failed: the first AX IPC may itself block until readiness. Earlier no-request and scratch-helper-exclusion attempts are retained separately. One pre-overlap MCP sample has an external System Settings window above A; without per-RPC absolute timing it is not attributed to the candidate or reported as uninterrupted A-on-top.
|
||||||
|
|
||||||
|
Immutable reports, raw failures and manifests remain under `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/`, including `p7-normal-v20-native-regression-independent-report.md`, `p7-windows-first4-final-v19-20261019.md`, `p7-linux-final19-first4-acceptance.md`, `p7-mac-zero-cover-causal-boundary-review.md` and the versioned package appendices. Evidence-only clean passes do not count as the required two final full-source audit passes. The earlier main merge check is historical; final source validation and mergeability must cover the completed tree. No push, PR, merge or release has occurred.
|
||||||
|
|
||||||
|
### Historical v25 checkpoint (2026-10-20)
|
||||||
|
|
||||||
|
The current 453 native plus12 support inputs have tree SHA-256 `6faac284650aeffaa64c5fa6b220ab1799ae25f69f0e68845e21159bf7d50e5a`. Linux v24 repairs partial-send classification at prefix/first-character/key-release/flush boundaries, admits MPX pending input only after focus validation, and cleans up failed press/probe release. Actual Linux focused tests, all-target checking, CLI/SDK release, private install, finite MCP and SDK cold/warm checks pass. Four fresh Xorg/Mutter cases exercise public CLI Unicode and named keys, the current SDK C ABI trusted-session seam, and one paused-target uncertain request. The last case delivers only `a` after resume, correctly remains sent/unconfirmed, and has no extra physical events at return, after1.7seconds or session end; it is not a complete-text success. The parent verifies38 evidence hashes, original callbacks, full distinct old/new Xorg logs and owned-container cleanup. Earlier tail-log count overclaims retain their correction and originals.
|
||||||
|
|
||||||
|
v25 changes only macOS exact-target resolution relative to v24. A real LibreOffice failure showed a same-window app-focused `AXGroup` explicitly reporting false while its text body actually reported true. This precise condition now invokes the existing bounded, complete, unique text-focus proof; PID/window binding, cross-window rejection and final rechecks remain. ARM and Intel CLI/SDK release builds, six existing exact-target units and all-target checks pass; the new ARM SDK has a verified Developer ID signature. The new coherent universal Developer ID package passes ARM/Rosetta finite commands and direct MCP, packed SDK cold/warm installation, production Node MCP, and14 owned fresh/official0.17-upgrade/repeat assertions. The parent independently verifies four archives and all38 regular member occurrences,11 result reports,465 source/support pins and five all-architecture signatures. Signing remains offline without timestamp/notarization, and installation uses the disclosed private-prefix/local-endpoint seams. Complete current-source audits and final merge preparation remain separate checks. Windows/Linux/shared inputs are unchanged by this Mac repair.
|
||||||
|
|
||||||
|
Fresh original T15 confirms one public ASCII insertion through independent body AXValue and selection readback. Its one Save times out waiting for OS Automation permission, reports sent:true, and has no independent disk marker; no Save is replayed. LO reports AXEdited=false even with the changed body, so that attribute is not a save oracle. The local canary process has no posting permission:121 posting attempts and zero receiving Sentinel keydowns do not prove canary delivery or uninterrupted user typing. Numeric foreground/top/pointer samples retain their bounded observation scope. Earlier LO focus observations above remain historical; positive insertion now passes, while saved-document acceptance stays open.
|
||||||
|
|
||||||
|
Strict pointer/menu zero-cover still fails. A fresh signed own-window bit21 control returns Set0 but reads the requested bit unchanged, then Clear0 and exact baseline restoration. It does not validate a foreign-window remedy. Earlier unsigned-helper SIGKILL and its empty logs remain stage-unknown, and no unsupported OS-wide impossibility claim follows. All-platform acceptance and PR readiness remain incomplete until these native gates, final checks and current-tree merge preparation pass. No push, PR or release is authorized.
|
||||||
|
|
||||||
|
### v26 LibreOffice pointer safety checkpoint (2026-10-20)
|
||||||
|
|
||||||
|
The v25 checkpoint above is historical. Current v26 has native tree SHA-256 `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3`; its 465-input manifest is `f16523c3823554f6d0e74d257ee232d768b7b8cebe17a80bf6b787d2591b4042`. Only the three macOS click tools changed relative to v25. Windows, Linux, shared native inputs and the JavaScript SDK are unchanged.
|
||||||
|
|
||||||
|
A fresh default public Calc request targeted C5, but three independent exact-window Name Box reads returned B28. The original sent/unconfirmed request was not replayed. Exact LibreOffice 26.2.6.3 source reads the global `NSEvent.mouseLocation` for VCL mouse callbacks; the observed screenshot-coordinate inversion was coherent. This establishes an incompatible pointer-reading mechanism, without proving the precise native callback responsible for B28.
|
||||||
|
|
||||||
|
The repair adapts upstream #4864's refusal principle using the fork's existing NSRunningApplication identity for `org.libreoffice.script`. It prevents PID-routed pixel click, right-click, double-click and element-pointer alternatives from acting at an unrelated hardware-pointer position. It preserves advertised AX actions and safe editable-focus/AX-selection paths. The old raw foreground pointer routes are also refused because they do not move the hardware pointer. This is narrower than upstream's mapped-library detector: rebranded or unbundled VCL processes are not identified by this rule. Canonical public Mac actions remain background-only and do not expose a new JavaScript foreground option.
|
||||||
|
|
||||||
|
An independent new-document v26 request returns public `background_unavailable`, `sent:false` and request `committed:false`; three bounded same-window Name Box reads remain A1. This passes safe-refusal verification, not C5 background-selection success or raw AXPress runtime equivalence. The original stricter harness outcome remains failed because it expected internal `effect`/`toolkit` fields that the existing public error projection deliberately reduces to code/reason/sent/next. Its raw output and the separate contract-based adjudication are both retained. SDK `operation.dispatched:true` means entry into the FFI request lifecycle, not physical input delivery.
|
||||||
|
|
||||||
|
Current ARM click14, AX-dispatch1 and double-click1 focused tests, all-target checks and release CLI/SDK builds pass; the ARM SDK has a verified offline Developer ID signature. Current-source packaging and further native regressions are recorded separately. The temporary local validation commit `a9fe798b6dac938392f450f2edf55456a6dfc6e0` retains the clean merge of fixed main `5dcf7c7cb162e825d95d531081f67ee60da3c0e1`. Its complete 8 GiB build script passes in201.876seconds and normal root typecheck in72.948seconds, including integration and without NODE_OPTIONS for typecheck. These are verification timings, not a performance comparison. Earlier default4GiB merge/main failures remain failures; that command was not rerun or marked green for v26.
|
||||||
|
|
||||||
|
The independent v25 public-toolbar Save passed actual FODT XML verification; the original Cmd-S/AppleEvents Save remains UNKNOWN and was never replayed. A later own-window bit21 positive control passed, but the trusted normal-caller foreign-window experiment did not change the bit and was rejected after exact restoration. Historical Codex canvas samples received Command despite an unmodified client request, with unknown origin, so they cannot establish an ordinary unmodified-click baseline; its composite popup/context samples also retain actual cover. Strict macOS pointer/menu zero-cover remains FAIL. Current-source audits, native acceptance and PR readiness must be assessed separately; no shipping P7 commit, push, PR or release has occurred.
|
||||||
|
|
||||||
|
### Historical v27 Writer focus candidate — not native accepted (2026-10-20)
|
||||||
|
|
||||||
|
The v26 package checkpoint is complete: ARM/Intel builds, focused checks, coherent universal archives and all11 non-GUI runtime/install report groups pass. A fresh independent readback verifies92 evidence files,38 regular archive members,465 source inputs and5 all-architecture signatures. These package/protocol results do not close Writer or zero-cover acceptance.
|
||||||
|
|
||||||
|
A source-slice reproduction executes the actual v26 predicates and shows that an official LibreOffice AXTextArea with no advertised action is refused even when its value/focus attributes are writable. A separate target-only, read-only query of the preserved v25 Writer process confirms exactly one same-window body, empty action list, and successful true AXValue/AXFocused settable flags. Its application-focused element is AXGroup while the body reports AXFocused=true. This is real old-instance property evidence, not a current action pass.
|
||||||
|
|
||||||
|
The intermediate v27 candidate changes only click.rs from v26: the default App single-unmodified-left-click branch may focus a known official LibreOffice AXTextArea through the existing editable-focus path, with both settable gates and current PID/window ownership checks intact. Other applications' text areas remain excluded. The raw click implementation, generic role helper and original tests are byte-identical to v26. Focus/value confirmation is unchanged: the AXGroup mismatch can still produce an unconfirmed result, followed by observation; there is no pixel fallback, replay, caret relocation or stronger success claim. Native tree is `c52f7cb6ac11aa65a9d7a226c5d4e38f55ee52820bcc4085ed4314b006b40538`, manifest SHA-256 `a3aed6e4f6b77b532395729b611f9817de06bac65c6d64c4811e0ed17c870ad1` (465 inputs). Focused builds/review are separate from the still-open fresh Writer click/type GUI test.
|
||||||
|
|
||||||
|
The latest Writer Stage1 stopped before LibreOffice or document actions because the owned sentinel could not obtain the foreground. System-window content remains unknown after Computer Use rejected access to that system app; no alternate channel bypass was used. Mini still reports locked in a fresh authenticated read-only check. Strict pointer/menu zero-cover remains FAIL. The successful full8GiB build/typecheck belongs to local merged checkpoint a9fe798b; it is not a full-build claim for the development checkout. Only the four later Markdown updates and this Mac Rust delta are intended for a refreshed local validation snapshot; any inherited build evidence must state that boundary. A new validation commit also changes the generated Git commit constant, so strict byte-identical compiler-input equivalence is not claimed. No final P7 shipping commit, push, PR or release.
|
||||||
|
|
||||||
|
## F31 native and distribution checkpoint (2026-10-20)
|
||||||
|
|
||||||
|
The accepted Writer source adds nine guarded lines to v27 click.rs and exactly matches the mini-tested SDK `091df63289f3542c8d1518f0be0db458863b7f87d4002de281579fca377509ab`. Both fresh documents independently transition from body AXFocused=false to true after one App click. Their one input and one toolbar Save persist exact XML, with saved hashes `83d5fd0c80a0e654d18f55b6d181bfb6be6ca1463a09b1885cad45393071cdb2` and `c4d557fa50ab5d7f449891d8fd139f0da24334880a44b31268bdd911f7ab01d1`. The parent verifies all 46 sealed evidence files and the first native read in each case. Pointer and text-clipboard state stay at each case's starting values; sampling gaps of up to 122.5/118.1 ms do not prove uninterrupted zero cover. The earlier v27 desktop click/input/save positive is a separate explicit foreground workflow. The original Cmd-S/AppleEvents request remains UNKNOWN and was not replayed.
|
||||||
|
|
||||||
|
F31 ARM/Intel CLI and SDK release builds and relevant native/all-target checks pass. The coherent universal packages pass all 11 non-GUI report groups: ARM/Rosetta finite commands and direct MCP, packed SDK cold/warm installation, production Node MCP, private owned installation/replacement and installed ARM/Rosetta commands/MCP. The final manifests retain exact slice/archive/signature identities and the disclosed offline Developer ID signing and private installer seams. A task-specific caffeinate identity preflight failed before these checks launched; its correction and the subsequent actual full run are preserved separately. Windows v21 and Linux v24 retain their original native identities; the intervening Mac changes do not alter their target inputs.
|
||||||
|
|
||||||
|
Formal Codex review of the new delta produced one multi-window concern, recorded and triaged rather than described as zero findings. The existing preparation sends process-scoped focus records, so its keyboard ambiguity gate remains; two single-window positives do not establish arbitrary multi-window support. Earlier complete source audits keep their pinned scope; final updated-delta audits, latest-main default build/typecheck, source equivalence and PR draft are recorded in the ignored final readiness evidence. No local source/build result substitutes for a native effect.
|
||||||
|
|
||||||
|
The first-four scope is platform-native effects plus runtime/installation/distribution verification, with the documented safe refusals and uncertainty limits. G17/G18/G24 accepted-opt-in native coverage, the later-instance pipe-fault injection and physical-human concurrency retain their individual unverified limits. The requested new Codex behavior comparison is inconclusive before action; matched model/OSWorld success, tokens, turns and timing remain deferred. None is silently scored as a pass, and no publication is authorized.
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
# Computer Use 升级验收记录(2026-10-16)
|
||||||
|
|
||||||
|
[English](cua-upgrade-acceptance-2026-10.md) | [简体中文](cua-upgrade-acceptance-2026-10.zh-CN.md)
|
||||||
|
|
||||||
|
实现以 [讨论结论](cua-upgrade-2026-10.md) 和 [平台移植清单](cua-upgrade-platform-ports.md) 为准。工作位于 `lazzy/cua-upgrade-2026-10`,基于 `a8eb74fdf5b60851e1fc4a8e258e043273df4411`。版本 `0.26.0` 是开发目标,尚未发布;F31 原生检查点完成时尚未推送、创建 PR、合入共享分支或运行发布工作流,隔离的本地合并验证提交分别记录于下文。此后的 PR 提交决定见下一节。
|
||||||
|
|
||||||
|
## PR 提交与评测安排(2026-10-20)
|
||||||
|
|
||||||
|
用户已授权提交本次功能 PR,并明确将 OSWorld 评测放到发布后。下文原生/运行时验收事实保留原候选身份和边界,未执行的评测不记为通过。发布审计还将三个 Computer Use npm 包从 CLI 版本更新及发布中排除,保留其统一 0.26.0 及依赖发布顺序。本次提交步骤没有要求执行正式发布。
|
||||||
|
|
||||||
|
## 验收口径澄清(2026-10-20)
|
||||||
|
|
||||||
|
用户确认 mini 已解锁,澄清此前观察到的 macOS 目标窗口/菜单短暂遮挡可接受且符合预期,并要求重新对比 Codex。严格零遮挡不再是阻塞项。下文历史严格判据的失败仍作为真实遮挡测量保留,不表示遮挡已消失,也不证明 macOS 无法避免它。功能正确性、精确目标归属、请求修饰键语义、不确定后不重放与恢复分别核验。
|
||||||
|
|
||||||
|
剩余真机 GUI 测试和新对照均在 mini 上执行。此前本机 Writer 准备阶段未发送文档动作;随后放弃的本机对照准备也未发送 Codex 动作,并已清理。本地工作限于源码、编译、包及合并验证。OSWorld/模型评测仍延期。不可变澄清记录与新用例报告保留在工作树的忽略证据目录。
|
||||||
|
|
||||||
|
已采纳的 F31 修复通过 mini 两个新实例的 Writer 后台流程:原 observer 在一次公开 App 点击前读取精确种子正文未聚焦,点击后读取已聚焦;一次输入改变正文且文件不变;一次工具栏 Save 将完全一致的正文保存为有效 FODT XML。此前 v27 负例及已否决的 F28/F30 试验分别保留。F31 在既有 AXFocused 写入前使用现有原生焦点准备和只读 caret 查询;公开回执仍保守,未确认动作不重放。原生 tree 为 `6abc23df3e6066a73f5110bfe77c041def4cc90852b5c5d6bde9ae537033eec3`(453 项原生加 12 项支持输入)。两个成功 Writer 用例各有一个文档窗口;既有键盘目标歧义保护保留,不据此宣称任意多窗口焦点切换成功。
|
||||||
|
|
||||||
|
mini 新 v27 画布、下拉和右键菜单原生效果 3/3 通过,接收修饰键及恢复均有记录;F31 改动仅限官方 LibreOffice 文本区。新要求的官方 Codex 对比结论为无法比较:即使使用当前安装包匹配的 bundled CLI 和 mini 既有消费者 profile,仍在应用绑定阶段以 app-server exit `-10005` 退出,未派发任何原生动作。另行普通 stdio 初始化成功,原因尚未确定。不据此推断 Codex 功能失败、性能差异或零遮挡。对照自有服务、socket 和夹具已清理,共享远程服务保留。
|
||||||
|
|
||||||
|
## 本地实现
|
||||||
|
|
||||||
|
P1–P6 的代码和打包改动已落在独立 worktree。Node REPL 只暴露 `js/js_reset`,默认 30 秒,预载全局 `cua` 和当前平台文档。Driver 管理应用/窗口解析、短 ID、三平台观察 diff、截图坐标和派发错误;直接 MCP 默认 19 个操作。macOS 保留后台 AX/定向事件;Windows/Linux 仅在明确派发前拒绝时转前台。显式桌面输入和 held keys 绑定会话,正常结束/reset 等待释放。浏览器接口直接返回仓库 browser-use SDK 的对象,未修改浏览器功能。
|
||||||
|
|
||||||
|
Native Wayland 实现已删除,X11 应用(包括 XWayland 中的 X11 应用)保留。SDK、原生、browser-use 和 Node REPL 统一为 `0.26.0`;独立 npm 安装使用固定版本依赖,原生包按校验和下载。Windows 默认安装无需签名 UIAccess worker;显式 opt-in 仍校验签名和安全安装路径。Qwen CLI/core 的产品差异只有删除两个 bundled `SKILL.md`。
|
||||||
|
|
||||||
|
本地检查点:P1 `cf89555254`、P2 `d2995a50c5`、P3 `b7c4d9c86c`、P4 `56a3ec2ba5`、P5 `fcc473d4a6`;P6 由本记录所在提交承载。
|
||||||
|
|
||||||
|
## 已取得的证据
|
||||||
|
|
||||||
|
| 范围 | 结果与边界 |
|
||||||
|
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| 仓库构建/类型 | 根目录 build、typecheck 通过;最终 browser-use 类型检查通过。 |
|
||||||
|
| Node REPL | 68 个相关测试通过,含真实子进程 reset/dispose 异步清理。独立 tarball 在无仓库依赖的消费者目录安装,通过文档预载、变量持久化和 reset 验证。 |
|
||||||
|
| SDK / browser-use | SDK 51 通过、6 跳过;浏览器 594 通过。浏览器包装测试使用原 SDK,无 Chrome GUI 成功声明。三个 npm 包均实际打包,仅执行 publish dry-run。 |
|
||||||
|
| Linux 原生 | 独立 ARM64 容器中 locked all-target check、SDK/driver 构建通过。规范坐标测试 38、Computer Use 测试 5 通过。Xvfb 真实桌面链 96/96、App 坐标链 29/29;实际 display resize 跳过。 |
|
||||||
|
| Linux 首次安装 | 实际 archive/checksums HTTP 下载、缓存命中、残缺修复、坏 SHA 拒绝、缺 checksum 拒绝及 postinstall 14/14;从下载缓存加载真实 SDK/桥并完成 GTK 输入/截图/按键释放链 14/14。不是 native override 或开发 symlink 验收。 |
|
||||||
|
| macOS 原生 | all-target check、ARM64 和 Intel SDK/桥构建通过,组合的两个 universal 文件均包含 arm64/x86_64。仅在 ARM64 主机执行加载;未声称 Intel 运行或 notarization 验证。 |
|
||||||
|
| Windows | GNU 交叉 all-target check 通过;未执行 Windows 桌面/UAC 测试。派发后 UIA 错误和 worker 断连的纯状态回归确认不再发送第二次点击。 |
|
||||||
|
| 发布配置 | 同版本约束、依赖发布顺序和 production gate 测试通过。工作流没有被触发。 |
|
||||||
|
|
||||||
|
收尾审查修复了三个消费者问题:旧归一化环境变量重复变换规范像素坐标;Windows UIA 派发后错误被当成可再次点击;直接 MCP initialize 仍给出旧工具和归一化说明。对应失败复现与修复后证据分别保留。
|
||||||
|
|
||||||
|
最终原生源清单为 P6 v5:688 文件,树 SHA-256 `c8a662cdc5c53eaa4cbbd14b146b8e844a86fc060d1b8f16a8f7bb4014c7dbee`。v5 再次通过 Linux/Windows GNU all-target、真实 MCP initialize/list 11/11、精确桌面链 96/96、实际下载 14/14 和缓存加载 SDK GUI 链 14/14。v5 SDK/driver 重新构建;桥因源和 ABI 未变,明确复用 v2 的 fresh build。早期 v2/v3/v4 证据仍保留各自版本。
|
||||||
|
|
||||||
|
详细原始日志与独立测试报告保存在当前 worktree 的 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/`,主报告为 `.qwen/e2e-tests/cua-upgrade-2026-10.md`,P6 报告为 `p6-validation.md`;自审记录位于 `.qwen/investigations/cua-upgrade-2026-10/parent-self-audit.md`。
|
||||||
|
|
||||||
|
完整差异和消费者自审在修复后连续两轮无新发现,独立测试工程师的 P6 边界复核为 CLEAN。当前环境未提供 Codex `/review` 工作流,未使用 Qwen Review 代替。
|
||||||
|
|
||||||
|
## P7 历史检查点(2026-10-19)
|
||||||
|
|
||||||
|
mini 已解锁,并通过既有 GUI Node 宿主执行真实 AppKit 输入、截图和按键生命周期测试。Windows 已完成真实 MSVC driver、SDK 和固定版本 Node bridge 构建;Linux 在 ECS 内的 Ubuntu Xorg guest 执行真实 GTK/AT-SPI 测试。早期“锁屏”与“仅交叉编译”的限制不再描述全部当前状态,但 P6 的历史结果仍按原产物保留。
|
||||||
|
|
||||||
|
上游 `0.20.0 → 0.35.4` 的核对发现 P4 清单之外仍有适用修复。当前源改动及证据包括:
|
||||||
|
|
||||||
|
| 修复 | 证据及剩余边界 |
|
||||||
|
| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| Linux 左上角窗口坐标偏移(#4967) | GTK 同场景修复前点到相邻行,修复后精确命中目标行。 |
|
||||||
|
| Linux 缺少 `_NET_WM_PID`(#4214) | 同窗口移除属性后,修复前丢失发现与观察,修复后通过 XRes 保持正确 PID、窗口、AX 和截图。 |
|
||||||
|
| Linux 临时键盘映射生命周期 | 候选与上游共有:目标 GTK 主循环忙碌 3 秒时丢失 remapped Unicode。队列屏障候选在 GTK/Chrome 忙碌场景完整输入;最终 v5b 对超时正常返回 sent/unconfirmed,无回放;无协议支持在输入前拒绝。未代替物理 MPX 或所有 toolkit/IME 验收。 |
|
||||||
|
| macOS 非有限窗口尺寸(#5125) | 真实 CFNumber 测试修复前失败、修复后通过,保留正常和负坐标。 |
|
||||||
|
| macOS 截图 PATH(#3755) | 真机窄 PATH 复现 ENOENT;新候选在窄 PATH 和正常 PATH 均返回真实 1920×1080 PNG。 |
|
||||||
|
| macOS raw 点击(#2907/#4690/#5137) | 同 PID 父窗/sheet 复现中键丢窗口和双份事件;新候选两轮真机复测恢复精确窗口与正确目标事件对数。默认 App 中键原本正常,未混为同一路径。 |
|
||||||
|
| macOS Electron 后台滚动(#4861/#5079) | 正常 v4 SDK 七格真机通过:raw page/inner × AX/pixel 和默认 App inner 保持页面 +404px、内层 +114px,另一滚动区不动;内层到底时 raw/App 均真实拒绝、sent:false、两偏移不变。此前 v2/v3 的 down 逆向 -96px 保留为 before;只读原生诊断确认上方 ROW35 被误报为底边片段,v4 以可见文档顺序排除错误候选。每格一次请求,无重放、前台回退或浏览器接口,主路径仍为未确认 ACK。六项几何单测通过;v2 受控 43 项与后续受控程序启动超时独立保留,未称 44 项通过。普通 v7 独立重跑七格并全部通过;不是全部方向、AX 布局或真人并发验收。 |
|
||||||
|
| 元素 token 与原生对象绑定(#3616/#3873) | 真实 core/token/cache 受控交错复现旧 token 读取新对象;新候选通过 capture/resolve/reverse 三种交错,三端消费者编译通过;Windows native-v2 MSVC 缓存回归 5 通过、2 明确跳过,token 30 项通过。该复现未执行 GUI 点击。 |
|
||||||
|
| MCP schema(#4888) | 默认 19 与 EXTRA 70 个真实 wire 声明通过离线兼容检查;运行时仍拒绝非法目标组合。未请求真实模型 provider。 |
|
||||||
|
| Windows GUI 宿主控制台(#5136) | MSVC 测试确认无控制台宿主不弹控制台,有控制台宿主保留继承。 |
|
||||||
|
| Windows 截图失败资源清理(#4333) | 实际 GDI 分配及失败注入测试通过,连续失败无 handle 增长;交互桌面截图尚未验收。 |
|
||||||
|
| Windows 快捷方式环境变量(#3975) | 真实 IShellLink 修复前返回不可用字面路径,修复后解析目标且保留参数。 |
|
||||||
|
| 隐式会话空闲恢复(#4283) | 同一真实 MCP 连接等待约 34 秒后恢复调用;显式结束/命名会话保持终止,旧元素 token 失效。 |
|
||||||
|
| 录制归属/停止/时间(#3632/#3714/#5144/#5145) | 公共入口验证命名归属、encoder 错误和时间修复;独立复测发现隐式连接遗漏,二次修复通过同连接/跨连接/lifecycle 回归。 |
|
||||||
|
| macOS 录屏完成时机(#4238) | 真机修复前出现 false empty 和成功返回后文件增长;新候选 5 次 120ms 短录制均立即可完整解码,一秒后文件 SHA 不变。 |
|
||||||
|
| 录制 AX 预算与会话锁(#4215) | 真实 ToolRegistry 加受控 provider 的独立 before/after 证明:挂起的 after 捕获不再阻塞 state/stop;2 秒返回上限、同 PID 忙碌拒绝、迟到结果丢弃和 stop→start 隔离共 6 项通过。Windows 录制线程 COM 初始化/节点释放顺序两项原生 MSVC 测试及全目标检查通过;不是 GUI AX 挂起验收。 |
|
||||||
|
| macOS Retina 缩放元数据(#3328) | 同一真实显示模式 3600px/1800pt,公共接口修复前返回 1,修复后返回 2;未据此推断默认点击坐标错误。 |
|
||||||
|
| macOS 辅助应用识别(#3456) | mini 同一已安装 accessory 应用,修复前 default/running_only 均丢失运行状态;native-v3 两个公共入口均返回实际 PID 9402。保留精确安装路径与所有运行实例,不以同 bundle ID 误认其他副本。 |
|
||||||
|
| 输出路径含目录符号链接(#4210) | manifest 与公共 ToolRegistry 分别验证:修复前拒绝实际 `/tmp` 等目录别名下的新文件,修复后规范化授权路径并仅派发一次;目录外、断链和文件父节点仍拒绝。不是 GUI 截图验收。 |
|
||||||
|
| Windows named pipe 错误清理(#4279) | 私有 Win32 pipe 复现创建失败与释放后恢复;源码加入有界重试及统一 PID/SDK 清理。native-v4 实际 MSVC driver/SDK 构建通过,缓存 6 项、token 30 项通过;生产 daemon 重试/清理运行验收仍待交互桌面入口。 |
|
||||||
|
| 卸载 MCP 配置归属(上游 `ff770f5d5` 等) | 独立临时 fixture 验证五类 foreign 配置不再误删,按本次选定待删除 payload 的真实可执行路径清理。没有执行完整卸载;后续删除失败时配置可能已先提交,原字节备份可恢复。 |
|
||||||
|
| typed schema 数值格式(#3462) | 上游回归用例修复前失败、修复后通过;输入/输出保留合法格式与字面值,contract 37 项及生成器 1 项通过。 |
|
||||||
|
| Windows 安装器祖先进程保护(#4398) | 独立 12 项受控清理/legacy 委托验证通过;按 PID 清理并保护祖先,存在 driver 祖先时跳过任务级 End。密封 UnitOnly 通过,未改变用户 PATH 或执行真实任务/完整安装。 |
|
||||||
|
| Windows 自动启动与隔离安装(#3180/#4284) | 32 项独立受控测试、MSVC 11 项原生单测、补齐编译夹具后的全目标检查通过。取消/null 不再报成功、替换前不先删除任务、隔离 release 安装保留共享任务。真实 UAC、Task Scheduler 原子性、登录启动和完整安装仍未验。 |
|
||||||
|
| macOS release 安装失败恢复(#3297) | 两项实际 shell 分支 before 复现复制失败丢旧应用;适配后 18 个隔离场景及 350 项独立核验通过,涵盖 staged/复制后校验、复制/rename/link 失败、恢复失败保留备份、首次安装、TERM 与重复清理。已加入旧字节恢复等长期回归测试。只验证受控文件事务,没有执行真实安装/签名/权限操作;不移植自动 TCC reset,不声称 SIGKILL/断电自动恢复。 |
|
||||||
|
| Chromium AX readiness(#3783) | 16 项实际函数受控测试、2 项旧/新发布竞态对照、2 项实际 core 预算传播测试通过;共享 500ms readiness 预算,不为动作/截图/录制新增轮询。真实 Chrome 拒绝 opt-in,accepted 路径尚未验收。 |
|
||||||
|
| AppKit 启动窗口恢复(#4309) | 1 秒主动查找额度共享原观察起点;27 项实际函数受控验证和 G17 外层消费回归 49 项通过。修复睡眠后预算已耗尽仍主动重试的问题;Mac 单测 1 项及 Mac/SDK 全目标检查通过。最终 App body 和原生 IPC 不受此硬限时约束,真实 AppKit 启动时序尚未验收。 |
|
||||||
|
| 普通 macOS AX 遍历预算(#3882) | 真实遍历函数受控 before 确认普通 raw/App 在 1 秒后继续读取;修复后 27 个遍历场景和 3 个节点准入控制通过,准备时间独立、完整树恰好到期不误报截断,录制绝对预算和 legacy 节点上限行为保留。真实 core ToolRegistry、默认 App、Mac 版本/cache/token 消费链完成 6 次受控快照与 22 项独立核验,旧 ID 拒绝、已解析绑定不漂移、部分结果警告及清理均通过。Mac 树 6 项、缓存 4 项、core 版本比较 27 项单测及 Mac/SDK 全目标检查通过。完整平台/SDK/GUI 路径仍待验收;没有整体调用 1 秒硬截止或真机慢 AX 延迟结论。 |
|
||||||
|
| macOS backing-store 截图(#4996) | 独立受控 after 32 项通过;当前 Space 下 public/private raw/App 四链返回正确 PNG。后续正常候选在 B 位于 Space68、前台 A 位于 Space1 时,raw/App 截图内容与五处 ROI 通过,二者字节一致;capture 段 120 个采样均保持 Space1 和 A 前台/顶层,模拟输入 6/6 到 A。正常库未记录实际私有路由,不能据此归因;全屏退出按钮预置不成立使通用整轮结果仍为 FAIL,自有进程终止后清理通过。 |
|
||||||
|
| macOS 遗漏 AXWindow 恢复(#4068/#4085/#4509) | 四个原生文件已补精确 PID/SPI ID、受限 remote-token 和默认 App/树/facts 消费者;Mac 单测 3 项与 Mac/SDK 全目标检查通过,独立探测边界 24 项与外层实际消费函数 49 项通过。真实遗漏 AXWindow 验收尚未完成。 |
|
||||||
|
| macOS 特殊键标志 | 真机 Space 准备失败后,无投递原生复现确认候选和上游共有结构位丢失;四条 named-key builder 修复后通过原生 128 键码对照、显式修饰/释放与环境隔离验证,原生单测 11 项及 Mac/SDK 全目标检查通过。正常新候选单次 Ctrl+Left 后,独立 Space oracle 在调用起点后约 638ms 确认返回 Space1;没有补发,B 未收到被系统消费的 Left,不推定其真实事件 flags。 |
|
||||||
|
|
||||||
|
Linux 稳定 nested Xorg 的焦点/结束会话批次 31 项通过,一项 TCG 时序不足保留为 inconclusive;正常响应时 Unicode 与映射恢复通过。后续受控暂停 GTK 主循环 3 秒时,候选和上游都出现 13 个 remapped Unicode 按键变为 NoSymbol,已独立定位临时映射恢复过早;原主 GNOME 冻结批次仍单独保留为环境混杂。客户端 ping 实验在 GTK/Chrome 通过,但 Chrome 回应早于 DOM 输入完成,不能作为语义成功证明;native-v5a/v5b 已分别验证机制与最终返回契约,超时仍可能丢字,必须先观察再决定下一步。
|
||||||
|
|
||||||
|
MPX 已改为确认准确虚拟键盘的完整有序批次;未确认/发送失败的设备保留 pending,下一请求在改映射、焦点或输入前拒绝。首次设备预热也须完整确认,并先于临时键码规划。独立受控生命周期测试由 before 的 7/10 违反变为 after 10/10 通过,13 项事件匹配回归、12 项 Linux 原生单测及全目标检查通过。真实内核跨批延迟、设备热拔插和物理 MPX 仍未验收。
|
||||||
|
|
||||||
|
macOS T04 在 native-v3 的三轮独立真机测试中均发生目标窗口升层,持续覆盖用户窗口至测试结束(约 3.1–3.3 秒);用户窗口保持 front/key,模拟输入全部到达。双查询诊断确认全窗口列表与实际可见层序不同;第一版改为可见列表后,重新激活应用仍返回成功却未恢复层序,首轮失败。第二版只提升当前前台的 AXFocusedWindow,三轮已消除持续覆盖,但仍有 99.4、200.5、192.8 ms 瞬时覆盖,**零干扰门槛未通过**。后续一次只发准备事件的独立诊断未观察到覆盖,不能排除采样及现有 guard 掩盖短暂升层。另一次仅把请求 pair 改用公共 CGEventPostToPid:精确源 PID 的 mouseDown、原生 field editor 与强制最终 AX 焦点均确认点击生效,但 A 自身仍观测到 194.110 ms 覆盖,独立 WindowServer 层序口径为 196.695 ms;公共 publisher 未显示补救效果,未改生产代码。该格 10/10 模拟按键到 A,实际点击调用内仅一键、已观测覆盖期间零键,不证明覆盖时真人输入安全。T06 菜单、T10 目标自激活回归通过;显式 Desktop 前台 Unicode 输入精确读回。后续精确窗口 SPI 校验收紧仍须归入新候选验证,不混用前述产物证据。独立双顶层窗口夹具尚未成功启动,父窗/sheet 测试不替代该场景。
|
||||||
|
|
||||||
|
Electron 后台原生滚动已完成上述主路径移植及七格真机验证,包括已复现的内层逆向边界。无可见锚点时保留原几何策略,重排或多列 AX 图谱仍不在此次证明范围。普通 macOS AX 树的有界部分结果及实际消费者受控验证已完成,完整平台/SDK 真机慢 AX 边界待验。跨 Space 截图已有上述真机效果证据;准确 AX 窗口恢复、Chromium accepted readiness 和 AppKit 启动恢复仍须完成各自真机边界。此次跨 Space 公共 AX 返回 85 个元素但明确不完整,未证明 remote-token 实际恢复。默认 App 选窗、动作前校验和观察结果转发均属于核验范围,不能只改 raw walker 就报完成。其余可选原生/安装路径的取舍见逐提交覆盖台账。
|
||||||
|
|
||||||
|
全差异自审还发现 macOS 录制钩子未接管可操作节点的额外原生引用。独立测试复用实际 walker、hook 与缓存,受控 AX 数据加真实 CoreFoundation 引用计数证明:修复前三次调用累计泄漏 1、2、3 份,修复后完整及不完整树的重复调用均归零,既有 token 保持有效,七份 JSON 除耗时外一致。修复复用现有未发布快照的所有权对象,不注册或替换动作 token;三项原生回归测试通过。39 项证据已逐字核验;不将受控引用测试计为真实 GUI/AX 验收,也未验证序列化失败或真实预算截断。
|
||||||
|
|
||||||
|
同轮独立测试用完整实际工具模块配合受控 provider,复现 raw 双击和右键在语义效果发生后返回 AX 错误时继续投递像素动作。已尝试的 AXOpen/AXShowMenu 现在直接返回 sent/unconfirmed 正常响应和警告,不回放;成功语义路径及未提供语义动作时的像素路径保留。完整原生工具模块的双击六格、右键四格受控复验通过,三项不确定响应经真实 core 公开投影和 typed 校验后仍为 unverifiable/accessibility,警告保留。复核发现的 foreground 请求被误记为实际前台方式也已修复:未执行前台事务的 AX 路径明确报告 background。新增原生投影回归与既有路由单测通过。默认 App 提前转发是对照,不推定默认 App 受影响;尚无真实 AX 服务器“生效后报错”的 GUI 证明。
|
||||||
|
|
||||||
|
G34 的 #5124/#4856 一律禁止 native omnibox 粘贴/换行策略明确不采纳:已定合同允许 selected App/window 当前焦点的合法原生输入,ASCII Return 与非 ASCII AX 插入分开验证,插入读回不证明导航。hotkey、pressKey、paste 三个入口分别核对。#4835 的冷前台字段/选区差异仅属 raw/EXTRA,默认 App 不可达,保留窄范围待复现,未引入 AllWindows 或固定稳定等待。
|
||||||
|
|
||||||
|
G07 最新生产入口实测表明 SSH Session0 在 SDK 创建时就因缺少交互用户会话而退出(23ms),尚未进入 named-pipe 接受循环;私有 pipe 不存在,共享驱动未变。管道本身可运行于 Session0,但这不等于当前生产入口可达。G23 第二版独立启动诊断正常编译,唯一启动 8 秒无 readiness 或阶段日志。后续同一 binary、相同普通 Node 启动路径的一格诊断,原 8 秒窗口结束后仍未就绪;一次 1 秒进程采样 exit0,仅得到 89 条 `_dyld_start +0` 主线程采样,映像信息不可用,不能定位具体等待机制。独立 Electron 宿主加任务自有 native addon 夹具正在准备;此前两格仍未执行,两格均未派发 AXPress,后续诊断完全不创建 SDK/AX 会话,没有据此推断 TCC/EDR 原因或修改安全设置。
|
||||||
|
|
||||||
|
499 条上游变更已完成逐项分类,原“证据不足”和“仅引用 P4 未复核”项已归零;仍有适用差异待复现、移植或明确取舍,分类完成不等于验收完成。2026-10-19 最新 JavaScript 整库 build、typecheck 通过,SDK 测试 51 通过、6 明确跳过;缺少 corepack 的首次构建失败保留,后续使用任务内入口和本机已有工具/固定缓存版本完成,未全局安装。当前普通 v7 的 453 文件统一原生清单通过 Mac/SDK、Linux/driver/SDK 和 Windows GNU/driver/SDK all-target 检查,默认 SDK 构建通过;12 个精确编译支持文件另列,树为 `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693`。Linux x64 和 Windows MSVC 的 driver/SDK release 均从此清单构建通过;Windows 本轮 2 项 COM、11 项 autostart、1 项有界捕获回归通过。Linux 当前正常 direct-MCP GUI 批次 GTK 7/7、Chrome 3/3 通过。相对 G05 v4 新增上述录制引用及 raw 语义错误不回放修复;v7 独立七格滚动真机全部通过,使用 SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe`。独立 fixture IPC 确认 inner +114px/page +404px、耗尽边界偏移不变;143 个层序/鼠标/剪贴板计数采样均保持 A,自有进程和 LaunchAgent 已清理,共享进程生命周期不变;主代理逐字核验 634 项引用。8.444–12.213Hz 采样不证明零瞬时干扰或真人并发。未变更 core 沿用八项通过的精确源检查点。12 个精确编译支持文件另附清单,早期遗漏夹具的失败保留。P7 第一轮已覆盖 89 个当前变更路径并发现上述问题;后续完整一轮干净审查之后,第二轮发现设计文档缺少双语版本,补齐翻译后连续干净计数重置。最终更新范围的完整两轮干净自审尚未完成。三端完整真机回归仍在进行。详细 before/after、无效批次与产物身份保存在 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/p7-*`;不得把历史 P6 检查结果用于证明后续 P7 改动已全部通过。
|
||||||
|
|
||||||
|
## 外部验收历史检查点
|
||||||
|
|
||||||
|
Windows 普通 SSH 已恢复,当前 console/runner/共享驱动生命周期已核验。用户明确批准独占临时任务 `\QwenCuaP7Acceptance`:无触发器、现有 Administrator InteractiveToken、Limited、不保存密码、最长两小时,完成后核验并删除;当前 v7 交互验收正在准备。未借用其他进程令牌。用户也批准 mini 现有 Node 控制 Finder/TextEdit 的正常 Automation 对话框,以及本任务所需的测试机权限;无需修改 TCC 数据库。Linux 独占的真实 uinput/evdev/Xorg 链与原 nested guest 分开记录,nested Xorg 本身不证明物理用户输入、MPX 或 XWayland 验收。
|
||||||
|
|
||||||
|
工具层三方对照与 OSWorld 20 题模型评测尚未完成。此前两个模型的 provider 400 保留为历史失败记录;2026-10-19 当前配置的 gpt-6.1-sol 已通过全局 qwen 0.25.2 文本预检和真实 CLI→MCP→Node REPL 单次代码执行。合成图像预检第三次有效调用返回实际 PNG,模型正确识别四项颜色、形状和位置(2 次模型请求、1 次工具调用);前两次超时/代码转录拒绝均未输出图像,独立保留。未创建原生桌面会话,不能视为 GUI 验收。20/20 个本地题目定义已定位并冻结,均默认 Linux,仅两题有明确 Windows 分支;历史 macOS/Windows 适配正在恢复。mini 两处既定线索的有界只读核查覆盖 1003 个非凭据文本文件,未命中目标组;已有 stable20 是另一版题目与 scaffold,不能替代。执行数为 0,当前模型与历史基线的 gpt-6-astra 不同。用户已明确批准只读导出历史 macOS 组 `group-5cff67aa55934a09a45160e3194b5801-s1` 与 Windows 组 `group-bc7a3539903b4735bc3508e04e5791d5-s1`。每次真实 AP 请求均通过 mini 官方 AP Client 0.5.1 的准确入口门禁,已取回 group/jobs/eval,artifact 导出进行中。原始导出仅留 mini 私有目录,不向本仓库或本地证据复制凭据。两组各有 20 个 terminal Succeeded job,但不等于题目成功:Mac 均分 0.2613、1/20 通过,Windows 0.1206、0/20 通过。模板提交分别为 `d1b8341d17659205b9aa735e0390f54301be0dc7` 与 `b70f9f90b3df26f177d78279bc942490f7cc37e7`,二者均用 azure.gpt-6-astra,与当前预检模型不同。未授权或执行任何新 AP 提交、重试或取消,没有匹配条件下成功率、token、轮次、耗时或 Codex/上游改进结论。这些发布前验收条件仍然有效。
|
||||||
|
|
||||||
|
## P7 v13 历史验收快照(2026-10-19)
|
||||||
|
|
||||||
|
本节是 v13 当时取代早期检查点的历史快照。其中身份和开放项仅描述该检查点,当前范围及后续验证见下方。
|
||||||
|
|
||||||
|
### 当前候选与分发
|
||||||
|
|
||||||
|
当前原生冻结版为 v13:453 个生产/构建输入和 12 个精确编译支持文件,tree SHA-256 为 `c43e1b300706eff833a9999f889b7f7c418a414b6863d3b3c102ca339770f83c`。Linux 使用新编译的 v13 CLI/SDK。macOS 测试使用正常默认特性的 v11 SDK(`4131a02af8d64a88572e981dbd9dfa5e7d0eaa9ec5ac5509be341c159d59ce9d`);Windows 使用原生 MSVC v10 可执行文件(`c5ca0054810eb4512986aef0f6231e81497ada15d1809bcb9bd5521016ef3fca`)与 SDK(`91f871c885e13e856ee8284af0efed149232abbf58d9c7fc4584dee3d1f02283`)。中间仅 Linux 的改动仍需逐目标编译输入等同性审计,才能让其他平台的产物支持当前验收。
|
||||||
|
|
||||||
|
三个待发布包的 npm pack 本地 dry run 均通过,版本统一 0.26.0:CUA SDK 34 文件、Browser Use 9 文件、Node REPL MCP 46 文件。全库 JavaScript build/typecheck 通过,SDK 51 项通过、6 项明确跳过。完整当前工作树快照与 main `a250fd033941cab10c88d6eb8beb5756405193b9` 的合并无冲突;该检查仅创建 Git 对象,没有更新分支/ref,没有 push、PR 或发布。最终格式检查和两轮完整连续干净的源码自审仍待完成。
|
||||||
|
|
||||||
|
### 原生修复与实际效果
|
||||||
|
|
||||||
|
- macOS G23 已取得真实 AppKit AX server 证据:v10 四格覆盖 raw/default App,handler 切换状态后返回 NO 或抛 NSException。每格仅一次请求、一次效果。异常返回正常 sent/unconfirmed,不重放指针。返回 NO 仍可能得到 AX 成功,单独保留。v11 回归待执行。
|
||||||
|
- macOS 默认 App T04 对支持的原生文本框,单次无修饰左击走精确可编辑 AX 焦点。v11 三次 fresh 点击用例独立确认 field editor 与 fresh AX 焦点、值不变、未观察到遮挡。已观察的 canary 字符均到 A,但三次 click 请求窗口都没有新 keyDown,不能证明输入并发。完整 click 后 type 回归通过,正在补高频重叠测试。仍抬高 B 的纯指针诊断变体未采用,也不计修复通过。
|
||||||
|
- Linux MPX 成功派发现在携带明确 background/LinuxUinput 执行记录,避免实际文字已到达后公共接口返回 action_outcome_mismatch。完整有序 raw-key 确认、pending 设备拒绝与会话清理阻止超时后的重放。
|
||||||
|
- Openbox 3.6.1 将虚拟 master 的 core FocusIn 提升为全局激活。fresh 零按键对照归因到 WM/Xorg 行为。v12 对该已识别 WM 在映射、创建设备和输入前拒绝 MPX 键盘;六项原生请求返回 background_unavailable/sent:false,零输入。公共 hybrid 路径随后可仅一次走 foreground。原前台、core focus、指针、映射和层序最上窗口恢复;低层窗口顺序仍发生可观察变化,不计后台零打扰。Mutter 42.9 对应后台测试保持更强的完整层序。
|
||||||
|
- Linux AT-SPI InsertText 的长度单位为 UTF-8 字节,而 caret offset 仍是字符位置。旧实现与上游 0.35.4 都会截断真实混合文本。v13 在派发前修复共享长度换算。十项 native/public/SDK/facade 用例精确回读中文、emoji、组合字符与非零 caret 插入,零 raw-key 事件。读回不改变运行时保守的 sent/unconfirmed 投影。
|
||||||
|
|
||||||
|
### 平台验证与剩余原生工作
|
||||||
|
|
||||||
|
Windows v10 完成 14 项原生回归、68 项 direct MCP/SDK/native/Edge 检查、10 项生产 Node REPL kernel 检查和 12 项普通私有 production serve 检查。后两者分别验证真实 kernel 生命周期及管道归属,并非 Node REPL MCP wire GUI 会话或注入后续 pipe instance 创建失败。普通 serve 支持低层 CuaDriver.connect;保留的 ComputerUse.connect 仍要求认证会话归属,不能绑定该普通 serve,文档已写明。WinForms 后台样本保持 A 与指针;Edge 输入按合同走唯一允许的 foreground fallback。上游 0.35.4 的 12 项 native 与 8 项 Edge 检查达到各自预期。没有可核验的 Windows Codex 入口。
|
||||||
|
|
||||||
|
Linux v13 的 53 项 focused native 测试、all-target 检查和 release 构建通过,一个既有 ignored 测试保留。十项语义 Unicode/caret 用例及 real-kernel MPX smoke 通过。v12 在私有真实 Xorg 显示上另行覆盖 native/SDK/Node facade 的响应、阻塞、超时、pending 拒绝、forget 后 fresh 生命周期。外部输入中止使用独立自有 uinput 设备,并非真人物理设备。软件渲染 Mutter 容器与 nested Xorg 均不冒充共享 GNOME、原生 Wayland、真人并发或 VM 性能证据。
|
||||||
|
|
||||||
|
macOS v11 完整 fixture 矩阵已执行,详细独立复核仍在进行。剩余观察保留为失败或限制:sheet 输入被 same_pid_keyboard_ambiguity 拒绝;WKWebView 输入被 no_focused_text_input 拒绝;两项短层序重叠样本待归因。非文档原生文本框 AXValue 已替换,但 fixture delegate 回调缺失;该更强回调 oracle 仍为 false,不增加第二次输入制造回调。正常 TextEdit 保存与 Finder 重命名仍需完成已获批的 Automation 回归。仅凭剪贴板 generation 不能证明内容已恢复。
|
||||||
|
|
||||||
|
Windows 临时任务已删除并读回 absent,runner 已退出,自有辅助进程均清理。Limited 任务配置与内置 Administrator 实际 High integrity 分开记录。共享 driver/进程寿命保持。Linux 输入进程与临时设备已移除,私有显示 fixture 保留供最终清理。macOS 持桌面互斥锁,仅清理精确归属本轮的进程与 LaunchAgent 身份。合成 canary 与采样的焦点/层序观察始终与物理真人证明区分。
|
||||||
|
|
||||||
|
### 匹配模型评测
|
||||||
|
|
||||||
|
已获批的只读 AP 导出完成,每次实际请求前都对 mini 上官方 AP Client 0.5.1 执行版本门禁。含凭据的原始导出仅在 mini 私有位置保留。两组历史评测使用 qodercli 1.1.52 和 azure.gpt-6-astra,镜像/template 版本不同。macOS 平均分 0.26128、1/20 passed;Windows 0.12058、0/20 passed。任务状态 Succeeded 不等于任务成功。历史 token 是不可用的零填字段,不是零消耗;Windows 仅 13 条 trace 有完整最终轮次/时间指标。
|
||||||
|
|
||||||
|
已找回精确 baked macOS pilot cases/plugins,20 项指令均与 AP 导出相同。Windows 19/20 相同:历史 001 指令写 Calendar.app,而恢复的 Windows override 正确指定 Thunderbird Calendar。基础镜像层的 seed、真正 runner/generic scorer、网站与依赖正在按完整 layer hash 和各平台 overlay 恢复。setup 必须保护现有文件/profile,仅清理本轮 listener/进程;Calendar 持久化、应用、codec 或网站依赖缺失必须让 preflight 失败。scorer/gold 不提供给自主模型。
|
||||||
|
|
||||||
|
fresh 匹配比较必须固定 harness、任务 seed 字节、指令/rubric、模型/provider 与预算,只改变预定的新旧工具栈。历史 Qoder 结果仅为参考,不能充当该控制基线。当前 gpt-6.1-sol 的全局 Qwen CLI 文字、真实 Node REPL 和合成图片传输检查通过。mini 独立部署的生产 Node REPL MCP 也已通过现有 SSH 连接验证 initialize、精确 js/js_reset 列表、真实 kernel/preload、reset 与 EOF 清理,未转移模型凭据、未创建原生 GUI 会话。当前候选 OSWorld 实际执行数仍为零。成功率不回退与轮次/token/耗时门槛继续开放;未宣称性能提升或三方持平。
|
||||||
|
|
||||||
|
## 历史前四项范围(2026-10-19)
|
||||||
|
|
||||||
|
用户现阶段优先要求完成 macOS、Windows、Linux X11 真机验收,以及运行时、安装与打包检查。Codex/上游比较及 OSWorld/模型评测延后。未来模型调用已配置为 `gpt-6-sol`,本范围不新增模型/AP 任务。前四项通过不等于延后的模型门槛通过,也不授权发布。
|
||||||
|
|
||||||
|
### 候选身份与已完成检查
|
||||||
|
|
||||||
|
已验证的 v21 原生源码冻结版本包含 453 个生产/构建输入与 12 个支持文件,tree SHA-256 为 `11b0fafab2419448a16542a73cde08cea58ee67b3a7b922fdfe12fee456f9dad`。相对 v20 仅 CLI 入口改变:有效遥测偏好为 false 时直接执行有限命令,省去无用的观察子进程。偏好优先级与启用路径保持。macOS 后端仍为 v20 SDK:签名前 SDK 二进制 SHA-256 为 `886bf05463a6c8273d0125b28482ee4f5ba8e9c1cffe5b4d5230e5fef2e77eee`,候选 Developer ID 签名后为 `134b334ea8b7e912e51ac0c27383d71b6c245e17b8f99f69cbee6e08c07fb064`,代码段等同性已独立核验。Windows/Linux 后端输入与各自 v19 验收产物逐目标等同;新 CLI 检查保留自己的版本标签。
|
||||||
|
|
||||||
|
- macOS v20 fresh 通过 sheet 输入与单次 Return 确认、WebKit 点击/输入/提交、精确粘贴与剪贴板类型/字节恢复,以及双窗安全拒绝/正向输入。生产 Node REPL MCP 的八项真实 wire/生命周期检查通过,部署源码/依赖身份精确。C9 TextEdit Unicode 保存与 Finder 语义重命名另有独立磁盘读回。Finder inline 键盘重命名无法证明原生 parent/window 归属时安全拒绝。内部 key-window 准备包含 NaN 坐标 mouse records;旧 Swift JSON logger 会漏记,先前“无内部鼠标事件”的说法已撤回。
|
||||||
|
- Windows v19 实际 MSVC 检查/构建、37 项 focused 测试通过,保留各自明确身份的原生 GUI 回归。生产 MCP wire 动作在 WinForms 一次真实生效后超时,没有重放,reset/EOF 清理通过。完整 CI 形态包通过自有 fresh、正式旧字节布局升级、repeat、uninstall/purge。显式部署 unsigned UIAccess 被拒绝,不声称真实 signed UIAccess 或 ARM64 原生执行。当前隔离卸载器通过 14 项实际 Windows PowerShell 回归,包括 owned Exec 与 foreign ComHandler 混合 task。共享进程受保护,临时任务已删除。
|
||||||
|
- Linux v19 的 53 项 focused 测试、一个明确 ignored 测试、实际 release 构建,以及 native MCP 初始化并列出 19 项工具通过。真实 Mutter/Xorg/uinput 输入保持完整采样焦点/层序并精确输入 Unicode;生产 Node MCP 的 15 项真实 AX 读回检查通过。CLI 安装与 SDK 冷/热缓存测试使用完整包。Openbox fallback 恢复原前台/最上窗口,但低层层序改变,不计后台零打扰。自有合成输入与物理真人并发分开。
|
||||||
|
- 三个 npm 包在 0.26.0 的 dry run 通过;SDK 51 项通过、六项明确跳过。修复后的 Node MCP preload 测试先将 SDK 求值阻塞在已观察的 barrier 后再触发超时,并验证下一次真实请求;整个 MCP 文件八项通过,保留批次共覆盖 113 项 Node 测试。独立 mutation 证据能检出迟到 continuation。安装器 Python 检查 57 项通过、五项明确跳过、四项 subtest。
|
||||||
|
|
||||||
|
后续 v22 冻结版本(`de5d5030e7dbf5b3f64c7946dc8d335ee2d988db14e8fe0994226d2f6f980245`)新增仅 Excel 的 Apple Events 兼容修复,并修正窗口层序注释。Excel 使用 full-name1773 和 saved1842,替代缺失的 Standard Suite 路径/modified 属性;唯一保存前严格匹配完整路径及大小写。精确路径和不匹配路径的只读 argv 检查通过。六项既有 Apple Events 单测、Mac/SDK/CLI all-target 检查与 ARM release 构建通过;SDK 二进制哈希为 `04d996ff2b51337bf07a72da408dee0ea0b987081b209bb680e28e8eb3f92499`。ARM 与 Intel release 构建通过。fresh v22 原生 Excel 保存也已通过:唯一保存 316.586 毫秒返回,modified 从 true 变为 false,独立 XLSX oracle 确认 A1 精确标记且 B1 不变;两个旧 dirty 文档未改。该格点击仍观察到 48.937083 毫秒覆盖,因此只计保存功能通过,不计零打扰。以下更新后的 v22 Mac 包检查也已在披露范围内通过。Windows/Linux 目标输入未变。
|
||||||
|
|
||||||
|
### 运行时、安装器与包验证
|
||||||
|
|
||||||
|
v21 Windows/Linux CLI 附录现已通过实际构建、既有遥测偏好/关闭测试、默认关闭与显式关闭有限命令、native MCP 初始化/列表/EOF 和当前自有安装/repeat/purge。287 份文件与四个完整包已由主控独立核验哈希及成员。
|
||||||
|
|
||||||
|
mochi 上一致的 v22 macOS 候选对 CLI、theme、SDK 与 bundle 使用现有产品 Developer ID 团队 `NF4574S59H`,无时间戳、未公证。签名后 SDK 哈希为 `5a0dd6d011e633fee9bd6b751444ee82430cc0c8632a5ab43e5c4143cfa93946`,打包代码段与 fresh ARM/Intel 输入相同。ARM 与 Rosetta Intel 的有限命令/native MCP、安装后 ARM、SDK 冷热安装和生产 Node MCP 十二项均通过。新装、实际完整签名 0.17 bundle 私有副本升级、重复安装共十四项断言通过,替换产物满足旧签名要求;原全局 bundle 与 v21 产物不变。主控独立核验了全部 636 个普通证据文件(tar 总共 738 个条目,含 102 个目录)、四个包及其成员、原生身份,以及 CLI/theme/SDK/app 的全架构签名。未改动的 CI NAPI 产物保留其 Intel slice 未签名的已披露边界。
|
||||||
|
|
||||||
|
这些检查在披露的自有/本地候选范围内完成 v22 第四项:macOS ARM 加 Rosetta Intel,Windows/Linux x64 通过目标输入未变核验。安装测试采用已记录的目标路径/可写性/下载/stop-helper seams,文件系统、签名与替换操作真实执行。两项真实 rollback 故障用例保留自 v21,安装器源码未变,不冒称本轮 fresh v22 故障测试。共享服务受保护,证据不延伸为 stock 全局替换、Windows signed UIAccess、ARM 硬件覆盖或正式公证。此前 installed ad-hoc ARM 启动失败仍保留,一致 Developer ID 候选无需改变安全策略即可通过该安装后运行路径。原有绝对路径 nested-local SDK 导入缺陷保留,base/current loader 字节相同;installed bare-package 与全局 `cua` consumer 另行通过。
|
||||||
|
|
||||||
|
### 剩余门槛与证据边界
|
||||||
|
|
||||||
|
macOS 画布指针输入仍使目标覆盖 A 约 100–200 毫秒,虽然前台身份与 canary 到达保持。菜单覆盖也仍开放。实际效果正确、最终恢复不满足零覆盖。authentication、public-post、key-preparation、默认 mouse subtype 与 PrivateSource 诊断变体仍有覆盖,均已否决;私有变体不替换正常产物。PrivateSource 虽然原生效果精确一次,仍覆盖约 199 毫秒。只读因果复核不能证明系统不可实现;这些指针因子都没有处理 AX-only popup 覆盖。后续 fresh 完整省略 preparation 的反事实仍覆盖约 206 毫秒,并丢失原生画布效果,因此也否决该候选。之后的只读复核没有找到有证据支撑的下一项外部 driver 修复假设;这不证明系统不可实现。
|
||||||
|
|
||||||
|
正常 v20 fresh 滑块替代路径通过:唯一 `setValue('80')` 同时得到 value80 与独立原生 callback80,未观察到打扰。Excel 原有选中网格状态与 LibreOffice 刚打开文档的状态均在派发前返回 `no_focused_text_input`,不计文档保存成功;Excel Edit 后续通过原生公式编辑栏 description 确认输入标记,通过 A2 确认 Return。首次保存遇到 Automation 弹窗及 caller deadline,结果未知;正常授权后,新文档唯一保存明确返回 `automation_failed (-50)`、`sent:true`,磁盘没有标记;以上 v22 原生保存修复已验证,指针覆盖仍为独立失败。LibreOffice 公共正文 click 后仍聚焦 group;另一次只读正文检查确认 AXValue/AXFocused 可写,但实际焦点为 false。不以替代写入或保存成功计分。原生 still-launching 公共链现已通过:独立确认的已安装 accessory 夹具在唯一 `getApp` 开始时仍未完成启动,真实启动延迟尚余 900 毫秒以上;1.056 秒后返回精确 AX 标记,独立观察随后确认启动完成。动作加 settle 的 20 份层序采样未见自有窗口遮挡,六次合成 canary 均到 A。这不能证明具体执行了哪次 retry,也不能证明旧 400 毫秒实现失败:首个 AX IPC 本身可能阻塞至 ready。先前无请求与 scratch helper 被排除的尝试单独保留。MCP 一项 overlap 前采样有外部 System Settings 窗口在 A 上方,缺少逐 RPC 绝对时间,不能归因给候选,也不能称全程 A 在最上。
|
||||||
|
|
||||||
|
不可变报告、失败原始文件和 manifest 保留在 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/`,包括 `p7-normal-v20-native-regression-independent-report.md`、`p7-windows-first4-final-v19-20261019.md`、`p7-linux-final19-first4-acceptance.md`、`p7-mac-zero-cover-causal-boundary-review.md` 及各版本打包附录。仅证据的 clean pass 不计入最终完整源码连续两轮自审。先前 main 合并检查属于历史记录,最终源码验证和可合并性仍需覆盖完成后的整树。没有 push、PR、merge 或 release。
|
||||||
|
|
||||||
|
### 历史 v25 检查点(2026-10-20)
|
||||||
|
|
||||||
|
当前453个原生输入加12个支持文件的 tree SHA-256 为 `6faac284650aeffaa64c5fa6b220ab1799ae25f69f0e68845e21159bf7d50e5a`。Linux v24 修复前缀/首字符/按键释放/flush 的部分派发分类、MPX 焦点准入后的 pending,以及失败按下/探测键释放清理。实际 Linux 定向测试、all-target、CLI/SDK release、私有安装、有限 MCP 及 SDK 冷热检查通过。fresh Xorg/Mutter 四例覆盖公开 CLI 中文/命名键、当前 SDK C ABI trusted-session 路径,以及暂停自有目标后的单次不确定请求。最后一例恢复后仅得到 `a`,回执正确保持已发送未确认;返回、等待1.7秒及结束会话三点没有新增物理事件,不算完整文字送达。主控核验38份证据、原回调、不同旧/新 Xorg 完整日志及自有容器清理;早期尾部片段计数夸大保留原件和勘误。
|
||||||
|
|
||||||
|
v25 相对 v24 只改 Mac 精确焦点解析。真实 LibreOffice 失败显示:同窗应用焦点指向明确 AXFocused=false 的 AXGroup,而正文实际 AXFocused=true。仅此精确条件转入现有有界、完整且唯一的文本焦点证明;PID/窗口绑定、跨窗拒绝和末尾复核均保留。ARM 与 Intel CLI/SDK release、六项既有精确目标测试及 all-target 检查通过;新 ARM SDK 的 Developer ID 签名已核验。一致的通用 Developer ID 新包已通过 ARM/Rosetta 有限命令与直接 MCP、打包 SDK 冷热安装、生产 Node MCP,以及14项自有新装/正式0.17升级/重复安装断言。主控独立核验四份归档及全部38个普通文件成员实例、11份结果报告、465个源码/支持 pin 和五项全架构签名。签名保持离线、无时间戳/公证,安装使用已披露私有 prefix/本地下载 seam。完整当前源码自审与最终合并准备仍独立进行;该 Mac 修复未改 Windows/Linux/共享输入。
|
||||||
|
|
||||||
|
fresh 原始 T15 的一次公开 ASCII 插入,经独立正文 AXValue 和选区读回确认。唯一 Save 因等待系统 Automation 权限超时,报告 sent:true,独立磁盘没有标记;未重放 Save。LO 即使正文已改变仍报告 AXEdited=false,不能把该属性当保存 oracle。本机 canary 进程没有投递权限:121次 posting 尝试与接收端零 keydown 不能证明 canary 送达或用户输入不中断。数值前台/顶层/指针采样只保留其有界观察范围。上方 LO 焦点结果保留历史身份;正向插入现已通过,文稿落盘验收仍开放。
|
||||||
|
|
||||||
|
严格指针/菜单零遮挡仍失败。fresh 签名自有窗口 bit21 正控中,Set 返回0但该位读数未变,随后一次 Clear 返回0且基线精确恢复;不能采用为外部窗口修复。早期无签名 helper 的 SIGKILL/空日志仍阶段未知,不据此断言整个系统不可能。真机门槛、最终检查及当前树合并准备未全部通过前,整体验收和 PR 准备均未完成;没有授权 push、PR 或发布。
|
||||||
|
|
||||||
|
### v26 LibreOffice 指针安全检查点(2026-10-20)
|
||||||
|
|
||||||
|
上面的 v25 检查点属于历史记录。当前 v26 原生 tree SHA-256 为 `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3`,465项输入 manifest 为 `f16523c3823554f6d0e74d257ee232d768b7b8cebe17a80bf6b787d2591b4042`。相对 v25 仅改三个 Mac 点击工具,Windows、Linux、共享原生输入与 JavaScript SDK 均未改变。
|
||||||
|
|
||||||
|
新建 Calc 文档的一次默认公开点击请求以 C5 为目标,但三次同窗独立 Name Box 读数均为 B28。原请求已发送未确认,未重放。精确 LibreOffice 26.2.6.3 源码通过全局 `NSEvent.mouseLocation` 计算 VCL 鼠标回调位置;本次截图坐标反算一致。这确立了全局指针读取与定向事件不兼容的机制,尚不能唯一证明产生 B28 的具体原生回调。
|
||||||
|
|
||||||
|
本修复借鉴上游 #4864 的拒绝原则,使用本 fork 现有 NSRunningApplication 身份识别 `org.libreoffice.script`。其 PID 定向像素点击、右键、双击及元素像素回退不再作用于无关的硬件指针位置;已广告 AX 动作与安全的可编辑焦点/AX选区路径保留。旧 raw foreground 指针后端没有移动硬件指针,因此也拒绝。该识别比上游映射库检测更窄,不覆盖改名或无 bundle 的 VCL 进程。规范公开 Mac 动作仍固定后台,不新增 JavaScript foreground 选项。
|
||||||
|
|
||||||
|
另一个新文档的 v26 请求明确返回公开 `background_unavailable`、`sent:false` 和请求级 `committed:false`,三次有界同窗 Name Box 读数均保持 A1。通过的是安全拒绝验证,不是后台 C5 选区成功,也不是 raw AXPress 真机等价。原严格 harness 因错误期待公开错误投影不会保留的内部 effect/toolkit 字段而失败,原输出与独立按现有 code/reason/sent/next 契约作出的判定分别保留。SDK 的 `operation.dispatched:true` 仅表示进入 FFI 请求生命周期,不是物理输入证据。
|
||||||
|
|
||||||
|
当前 ARM click14、AX-dispatch1、double-click1 定向测试、all-target 检查及 CLI/SDK release 构建通过,ARM SDK 离线 Developer ID 签名已核验。当前源码的包验证与其他真机回归另行记录。临时本地验证提交 `a9fe798b6dac938392f450f2edf55456a6dfc6e0` 保留固定主线 `5dcf7c7cb162e825d95d531081f67ee60da3c0e1` 的实际干净合并;完整8GiB构建脚本201.876秒、正常root typecheck72.948秒均通过,后者含integration且不带NODE_OPTIONS。这些是验证耗时,不是性能对比。此前默认4GiB合并候选/主线失败均保留,v26未重跑或宣称该命令通过。
|
||||||
|
|
||||||
|
独立 v25 公开工具栏保存已通过实际 FODT XML 核验,原 Cmd-S/AppleEvents 保存仍 UNKNOWN、从未重放。后续自有窗口 bit21 正控通过,但已信任宿主以正常 caller 修改外窗的试验未改变该位,精确恢复后否决。历史 Codex 画布样本在调用未指定修饰键时收到 Command,来源未知,不能作为普通无修饰点击基线;其 popup/context 组合样本也保留实际遮挡失败。严格 Mac 指针/菜单零遮挡仍 FAIL。源码自审、真机验收和 PR 就绪分别判定;未形成最终 P7 发布提交,没有 push、PR 或 release。
|
||||||
|
|
||||||
|
### 历史 v27 Writer 焦点候选——未经真机验收(2026-10-20)
|
||||||
|
|
||||||
|
v26包检查点已完成:ARM/Intel构建、定向检查、一致通用归档与全部11组非GUI运行时/安装报告通过。新独立读回核验92份证据、38个归档普通成员、465项源码输入和5项全架构签名。这些包/协议结果不关闭Writer或零遮挡验收。
|
||||||
|
|
||||||
|
受控源码切片执行实际v26谓词,确认官方LibreOffice的无广告action AXTextArea即使value/focus属性可写仍会被拒绝。另一次针对保留v25 Writer进程的目标限定只读查询,确认唯一同窗正文、空action及AXValue/AXFocused settable成功且为true。应用级focused element为AXGroup,正文却报告AXFocused=true。这是旧实例真实属性,不是当前动作通过。
|
||||||
|
|
||||||
|
中间v27候选相对v26只改click.rs:默认App的单次无修饰左键分支可通过既有可编辑焦点路径聚焦已识别的官方LibreOffice AXTextArea,保留两项settable及当前PID/窗口归属检查,其他应用的textarea仍排除;raw点击实现、通用角色helper及原测试与v26逐字节相同。焦点/值确认规则未放宽:AXGroup不匹配仍可返回未确认,之后须观察;没有像素fallback、重放、光标位置改写或更强成功声明。原生tree为`c52f7cb6ac11aa65a9d7a226c5d4e38f55ee52820bcc4085ed4314b006b40538`,manifest SHA-256为`a3aed6e4f6b77b532395729b611f9817de06bac65c6d64c4811e0ed17c870ad1`,共465输入。定向构建/review与尚未完成的fresh Writer点击/输入GUI验收分开。
|
||||||
|
|
||||||
|
最新Writer Stage1因自有sentinel无法取得前台,在LibreOffice及文档动作启动前停止。Computer Use拒绝访问该系统app,系统窗口内容仍未知,没有换通道绕过;mini经新认证只读检查仍锁屏。指针/菜单严格零遮挡仍FAIL。完整8GiB构建/typecheck成功属于本地合并检查点a9fe798b,不是开发checkout全库构建通过。仅计划将之后四份Markdown更新与该Mac Rust增量同步至本地验证快照;继承构建证据时须明确此边界;新验证提交还会改变构建生成的Git commit常量,不能声称编译输入严格逐字节等同。没有最终P7发布提交、push、PR或release。
|
||||||
|
|
||||||
|
## F31 原生与分发检查点(2026-10-20)
|
||||||
|
|
||||||
|
已采纳 Writer 源码相对 v27 的 click.rs 增加九行受门禁保护的逻辑,与 mini 实测 SDK `091df63289f3542c8d1518f0be0db458863b7f87d4002de281579fca377509ab` 完全对应。两个新文档均在一次 App 点击后,独立读到正文 AXFocused=false→true;各自一次输入和一次工具栏 Save 保存精确 XML,文件 hash 分别为 `83d5fd0c80a0e654d18f55b6d181bfb6be6ca1463a09b1885cad45393071cdb2` 和 `c4d557fa50ab5d7f449891d8fd139f0da24334880a44b31268bdd911f7ab01d1`。主控核验全部 46 份封存证据及每例首次原生读回。指针与文本剪贴板保留各例起始值;最大 122.5/118.1ms 采样间隙不证明全程零遮挡。此前 v27 桌面点击/输入/保存属于另一显式前台流程;原 Cmd-S/AppleEvents 请求仍 UNKNOWN,从未重放。
|
||||||
|
|
||||||
|
F31 ARM/Intel CLI 与 SDK release 构建、相关原生及 all-target 检查通过。一致通用包通过全部 11 组非 GUI 报告:ARM/Rosetta 有限命令及直接 MCP、packed SDK 冷热安装、生产 Node MCP、私有自有安装/替换,以及安装后 ARM/Rosetta 命令/MCP。最终 manifest 保留精确 slice/归档/签名身份,并保留离线 Developer ID 签名和私有安装器 seam 边界。一次任务自加的 caffeinate 身份预检在这些检查启动前失败,勘误与随后实际完整运行分别保留。Windows v21 与 Linux v24 保留原真机身份,此后 Mac 改动未改变其目标输入。
|
||||||
|
|
||||||
|
新差异的正式 Codex review 提出一项多窗口疑问,已记录并 triage,不宣称零反馈。现有准备会发送进程级焦点 records,因此保留键盘歧义门禁;两个单窗口正例不证明任意多窗口支持。此前完整源码自审保留固定范围;最终更新差异自审、最新 main 默认构建/typecheck、源码等价与 PR 草稿另存忽略的最终就绪证据。本地源码/构建结果不代替原生效果。
|
||||||
|
|
||||||
|
前四项范围为三平台原生效果及运行时/安装/分发验证,保留已记录的安全拒绝和不确定性边界。G17/G18/G24 accepted-opt-in 原生覆盖、later-instance 管道故障注入及物理真人并发仍分别未经验证。新要求的 Codex 行为对照在动作前受阻,无法比较;同条件模型/OSWorld 成功率、token、轮次与耗时仍延期。均不默记通过,也未授权发布。
|
||||||
@@ -0,0 +1,564 @@
|
|||||||
|
# Computer Use platform ports (2026-10)
|
||||||
|
|
||||||
|
[English](cua-upgrade-platform-ports.md) | [简体中文](cua-upgrade-platform-ports.zh-CN.md)
|
||||||
|
|
||||||
|
This is the implementation ledger for the approved [upgrade design](./cua-upgrade-2026-10.md). Code adapted from trycua/cua through `cua-driver-rs-v0.35.4` is MIT licensed, Copyright (c) 2025 Cua AI. Only the native driver library is a port source. The upstream bench package and its license are not included.
|
||||||
|
|
||||||
|
## Dispatch contract
|
||||||
|
|
||||||
|
App input resolves an exact application/window inside the driver. macOS uses accessibility and process-directed events; there is no automatic global foreground fallback. Windows and Linux may try the foreground rung once, exclusively after a native `background_unavailable` error with `sent:false`. An actuator error after a possible dispatch never replays input. Explicit desktop input is a separate foreground surface.
|
||||||
|
|
||||||
|
Foreground transactions observe physical input without consuming it. They stop remaining input on detection, preserve an external foreground C, and restore prior A only while the transaction still owns B. Windows drains the target input queue before restoring; text checks the target between UTF-16 units. Linux confirms both EWMH activation and core input focus before XTest.
|
||||||
|
|
||||||
|
Success acknowledges dispatch, not application effects. Input calls do not wait for screenshot changes. The next observation supplies effect evidence. A held press and a multi-click gesture retain only the timing required to deliver the gesture.
|
||||||
|
|
||||||
|
## macOS
|
||||||
|
|
||||||
|
| Upstream reference | Adaptation |
|
||||||
|
| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
|
| #4943 | Do not send A a synthetic deactivation notice when directing focus to B. |
|
||||||
|
| #4943, #4955, #5091, #5129 | Key-focus protection shares the existing per-process `app_focus` observer; bounded tail/order correction and intentional-focus epochs prevent competing observers. |
|
||||||
|
| #4976 | Physical mouse/modifier activity grants the user's switch priority. The existing activation guard checks the same physical-input state. |
|
||||||
|
| #5051 | Directed mouse events explicitly set modifier flags, including the empty set. |
|
||||||
|
| #5057 | Existing-file saves and Finder item renames use bounded Apple Events and readback. Arguments are passed as script arguments, not concatenated into AppleScript. Automation consent follows normal macOS TCC. |
|
||||||
|
| Popup selection | Open a popup, select a unique matching option through AX, close a menu opened by this call on refusal. Safari's existing DOM fallback remains restricted to the actual foreground document. |
|
||||||
|
| Text insertion | ASCII uses directed events. Non-ASCII requires a focused editable element and AX selected-text support; expected insertion is read back. No keyboard replay after an uncertain AX write. |
|
||||||
|
| Document model | A verified AX write to a document-backed AppKit field registers an edit through a directed, net-zero editing shim; refusal is reported if registration cannot be verified. |
|
||||||
|
|
||||||
|
Background drag refuses before dispatch. Sliders use `setValue`. Paste keeps the clipboard restoration transaction. The old fixed one-second post-action wait is removed. Neither AX acknowledgement nor event delivery alone is reported as a confirmed effect.
|
||||||
|
|
||||||
|
P7 controlled-provider tests of the complete native tool modules reproduced
|
||||||
|
raw double-click and right-click replay after an advertised AXOpen/AXShowMenu
|
||||||
|
applied an effect but returned an error. Once
|
||||||
|
either AX action is attempted, its error now returns a normal sent/unconfirmed
|
||||||
|
acknowledgement and warning; no pointer route follows. Successful semantic
|
||||||
|
actions and a pointer route selected because no action is advertised retain
|
||||||
|
their behavior. These AX acknowledgements explicitly identify background
|
||||||
|
delivery, even when the raw request selected foreground, because this semantic
|
||||||
|
branch performs no foreground transaction. Independent controlled tests execute
|
||||||
|
both complete native tool modules and the actual core public projection: all
|
||||||
|
six double-click and four right-click cases pass, with three uncertain cases
|
||||||
|
remaining unverifiable/accessibility and preserving their warning. Default App
|
||||||
|
early forwarding is a control; these tests do not demonstrate an App defect or
|
||||||
|
an actual macOS AX server returning an error after an effect.
|
||||||
|
|
||||||
|
## Windows
|
||||||
|
|
||||||
|
| Upstream reference | Adaptation |
|
||||||
|
| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| #4500 | Attach to the actual target input queue and use a sentinel read to bound queue draining before restoration. |
|
||||||
|
| #4901 | Check foreground ownership and physical input between UTF-16 units; stop after a stalled unit without replay. |
|
||||||
|
| #3671 | A timed-out UIA point action is distinct from a preflight absence of a suitable pattern, so an uncertain action cannot become a second coordinate click. |
|
||||||
|
| #4774 | Native combo selection uses the control's selection and notification messages and verifies the selected value. |
|
||||||
|
| Tk | Refuse the unsupported background pointer rung before input; the shared App dispatcher may then use foreground once. |
|
||||||
|
| Extended keys | Left/right Windows keys carry the extended-key flag. |
|
||||||
|
| Worker IPC | Bound worker response reading; a timeout says input may have been sent. UIAccess routing is explicitly opt-in. |
|
||||||
|
| Standard-user launch | The elevated daemon derives and verifies a Medium token from its own token with administrator membership deny-only. No other process token is opened or borrowed. Direct executable and shell-helper launches use that token; failures refuse rather than launching elevated. |
|
||||||
|
|
||||||
|
The normal installer/UAC flow is retained. Cross-compilation is build evidence only; it does not prove a normal interactive Windows desktop, UAC behavior, token facts at runtime, or shared-desktop interference.
|
||||||
|
|
||||||
|
## Linux
|
||||||
|
|
||||||
|
Native Wayland support, helpers, dependencies and installer payloads are removed. X11 and XWayland X11 applications share the same X11 implementation. A compositor-native Wayland surface is unsupported.
|
||||||
|
|
||||||
|
| Upstream reference | Adaptation / existing equivalent |
|
||||||
|
| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| #3882 | MPX pointer/keyboard dispatch, toolkit-aware refusal, occlusion and focus guards, no input shields. Existing fork exact-window checks remain. |
|
||||||
|
| #3761 | Flush/check key press before hold timing; unconditional final server round-trip before closing a short-lived connection. A later P7 reproduction shows that server delivery alone is insufficient for temporary keymaps; the client-queue fix below remains under validation. |
|
||||||
|
| #3762 | Check pointer presses/releases, keep gesture timing, release a queued button/modifiers after an error. |
|
||||||
|
| #3864 | Preserve the fork's exact observed bus/object click identities and per-window snapshot validation. Session-owned MPX device persistence comes from the MPX umbrella rather than this identity patch. |
|
||||||
|
| #3618 | Modified foreground drag brackets the gesture with real modifier transitions and cleanup. Unsupported modified background drag refuses before dispatch. |
|
||||||
|
| #4248 | The new App contract has no post-action effect classifier: closing B's dialog cannot be mislabeled a no-op because the WM parks focus. Existing exact-target foreground restoration preserves C and does not infer confirmation from unattributed popups. The upstream screenshot/window-change wait is intentionally not copied. |
|
||||||
|
| #4188 | Exclude application/frame/window shells from coordinate AX activation; recognize both NUL-separated and space-joined Chromium helper command lines. |
|
||||||
|
| #5139, #5158 | Unicode keysyms and full-character preflight; reserve/remap spare keycodes before the first event; serialize temporary keymap changes. No silently skipped characters. |
|
||||||
|
| #3601 | Master ownership includes boot, PID namespace, UID and process start identity. Reap only stale devices proven to belong to this driver family. |
|
||||||
|
| #2888 | Conservatively refuse MPX on KDE where device hotplug may reconfigure physical input. No automatic permission changes to `/dev/uinput`. |
|
||||||
|
| #5070 | Background keyboard refusal names a missing AT-SPI bus and its normal setup path; the diagnostic bus probe is bounded. |
|
||||||
|
| #4903 | Remove AT-SPI P2P feature/peer calls; use one registered accessibility connection. |
|
||||||
|
| #4984 | Rebase tree bounds only against a valid complete screen frame; reject incomplete/zero-size frame geometry. |
|
||||||
|
| #5074 | Wake an AT-SPI application with attributes before its first child walk. |
|
||||||
|
| #4375 | First semantic walk budget is two seconds; later walks one second. No generic full-tree retry for an incomplete observation. |
|
||||||
|
| GTK combo | `Selection.SelectChild` picks a unique option and its selected-child name is read back. No focus/foreground change for selection. |
|
||||||
|
| #4292 | Preserve text content separately from labels; prefer numeric Value for numeric controls and do not read password Text. |
|
||||||
|
| #3758 | Decode DirectColor screenshots with the X colormap rather than treating them as TrueColor masks. |
|
||||||
|
| #4281 | A successful kill waits for actual process exit; zombies count as exited. |
|
||||||
|
| #3814 | A delivered exact-window frame explicitly reports `screenshot_frame_valid:true`. |
|
||||||
|
| #4529 | Prepare overlays unmapped with empty input/bounding shapes; map only while painting visible content and unmap idle frames. |
|
||||||
|
| #4715 | Non-Linux stub registry no longer advertises the removed screenshot tool; portable menu schemas do not pass through cursor-only schema assertions. |
|
||||||
|
| #3658 | Enter/Return/KP/Numpad aliases share the terminal key gate and key mapping. |
|
||||||
|
| #3778 | Existing opt-in native browser tools share the session cursor registry; inactive tabs do not move the visible cursor. This does not add upstream browser tools to the default surface or change the repository Browser SDK. |
|
||||||
|
|
||||||
|
Fork repairs also preserve per-window screenshot scaling, do not scale an omitted mouse-up position twice, restrict Qt synthetic FocusIn to Qt5, condition FocusOut on actual core focus ownership, and classify Chromium only when a route needs the information.
|
||||||
|
|
||||||
|
### P7 Linux temporary keymap lifetime
|
||||||
|
|
||||||
|
Both the candidate and upstream lose remapped Unicode characters when an owned
|
||||||
|
GTK application's event loop is paused for three seconds: the server delivers
|
||||||
|
the keycodes, the driver restores the map, and the client later translates them
|
||||||
|
as NoSymbol. This is independently reproduced in nested Xorg, separate from the
|
||||||
|
unstable main GNOME session.
|
||||||
|
|
||||||
|
The candidate now prepares an X11 client-queue acknowledgement before changing
|
||||||
|
any mapping or sending input. Only operations needing a spare keycode pay this
|
||||||
|
cost. The target or a same-X-client ancestor must advertise `_NET_WM_PING`;
|
||||||
|
another connection in the same process does not qualify. XTest also checks the
|
||||||
|
focused client before each character and before/after the acknowledgement.
|
||||||
|
After dispatch, the exact nonce echo on the root window releases the temporary
|
||||||
|
maps. A five-second timeout restores the maps and returns a normal
|
||||||
|
sent-but-unconfirmed acknowledgement, with no replay. A detected focus change
|
||||||
|
during partial typing still stops remaining input with an error. Missing protocol
|
||||||
|
support refuses before input. The existing ASCII path retains its server
|
||||||
|
round-trip, with no added ping. Sparse pointer modifier maps refuse before input
|
||||||
|
because a borrowed keycode does not establish a real modifier mapping.
|
||||||
|
|
||||||
|
MPX additionally requires raw-key server-delivery confirmation before sending
|
||||||
|
the client ping. A controlled reproduction found that the last-key predicate
|
||||||
|
accepted an earlier repeated release after only two of six planned events. The
|
||||||
|
revised predicate requires every transition in order from the actual virtual
|
||||||
|
slave, including repeated keys and Shift. A legacy zero source ID is accepted
|
||||||
|
only if the event's device ID is that exact slave, never the shared master.
|
||||||
|
Unexpected same-source transitions leave delivery unconfirmed. A retained
|
||||||
|
virtual keyboard with unconfirmed input refuses the next request before
|
||||||
|
mapping, focus or dispatch; normal session cleanup removes that state with the
|
||||||
|
device. Device warm-up must also confirm its full batch, before the planner
|
||||||
|
can borrow a keycode. The existing keymap-to-MPX lock order is preserved.
|
||||||
|
Independent controlled lifecycle cases pass 10/10, matcher cases 13/13 and
|
||||||
|
actual Linux unit tests 16/16 after the warm-up preflight below. Kernel backlog
|
||||||
|
and physical-device validation remain pending; the controlled tests replace OS
|
||||||
|
boundaries with stubs.
|
||||||
|
Before warm-up, XKB must identify the actual slave and prove X248 has no symbol
|
||||||
|
in any group or level. An occupied, invalid or unreadable mapping refuses with
|
||||||
|
zero emissions and retains the pending state. The fixed keycode is never
|
||||||
|
temporarily remapped to make this check pass. Independent checks cover 20
|
||||||
|
mapping/query cases and three retained-pending entrypoint cases; C header and
|
||||||
|
Rust layout measurements agree. The real read-only Xephyr slave's empty-group
|
||||||
|
mapping is a shape control, not evidence for a new uinput device.
|
||||||
|
No focus grab, input shield, clipboard change, permanent keymap or new helper
|
||||||
|
process is introduced. GTK and Chrome probes establish the queue mechanism;
|
||||||
|
Chrome's echo precedes DOM input completion, so it is explicitly not semantic
|
||||||
|
success. Native-v5b GTK responsive, three-second blocked, timeout, wrong-nonce
|
||||||
|
and unsupported-protocol cases pass their respective effect/refusal/uncertainty
|
||||||
|
checks. Chrome's responsive/blocked/timeout mechanism checks use v5a; v5b changes
|
||||||
|
only uncertainty reporting, which is tested separately. Physical MPX remains
|
||||||
|
unverified.
|
||||||
|
|
||||||
|
## P7: upstream correctness closure
|
||||||
|
|
||||||
|
### Ordinary macOS AX walk budget (G31 / upstream #3882)
|
||||||
|
|
||||||
|
The ordinary raw and default App observation currently limit tree depth and
|
||||||
|
node count, but only the recorder has a time budget. Adapt the upstream
|
||||||
|
one-second node-walk budget to the ordinary observation entry. Start it at the
|
||||||
|
first admitted, nonduplicate node, after window selection and readiness work.
|
||||||
|
Keep the separate Chromium readiness and AppKit launch budgets, the existing
|
||||||
|
native messaging timeout, and the outer observation timeout. This cooperative
|
||||||
|
limit cannot cancel a blocked native read or guarantee a one-second total call.
|
||||||
|
|
||||||
|
A deliberate time cutoff must preserve the completed prefix and mark the
|
||||||
|
observation truncated and incomplete. Successful attempted reads retain
|
||||||
|
`read_complete`, matching the existing node/depth-cap semantics; an actual AX
|
||||||
|
read failure still clears it. Bounded revisions return a full replacement when
|
||||||
|
the prefix changes, so missing rows are not deletion evidence. Raw tokens must
|
||||||
|
remain bound to their own immutable captured objects across later captures.
|
||||||
|
Verify these consumers with actual walker results before accepting the fix.
|
||||||
|
|
||||||
|
The legacy node-only walker and recording's absolute deadline keep their
|
||||||
|
existing behavior. No public timeout parameter, new output field, worker or
|
||||||
|
action retry is introduced. A finite slow-node reproduction and setup-delay
|
||||||
|
control precede implementation; native latency remains a separate acceptance
|
||||||
|
boundary.
|
||||||
|
|
||||||
|
### Windows installer parent protection (G32 / upstream #4398)
|
||||||
|
|
||||||
|
The release installer's shared daemon cleanup can kill the driver process that
|
||||||
|
launched `update --apply`, including its installer child, and can misclassify
|
||||||
|
that protected parent as a stale daemon. The independent reproduction executes
|
||||||
|
the exact PowerShell functions with all process, task and pipe boundaries
|
||||||
|
stubbed; no real process or scheduled task is stopped.
|
||||||
|
|
||||||
|
Adapt the upstream live ancestor walk, rejecting a reused parent PID whose
|
||||||
|
creation time is later than its child's. Cleanup targets only non-ancestor
|
||||||
|
driver PIDs and excludes ancestors from the survivor/repair list. If a driver
|
||||||
|
ancestor is present, skip the task-wide `/End` operation as well: a PID filter
|
||||||
|
cannot protect an ancestor from Task Scheduler termination. An unavailable
|
||||||
|
ancestor query stops cleanup before any termination. Legacy migration uses
|
||||||
|
the same helper instead of a second image-name tree-kill implementation.
|
||||||
|
The separate `-local` installer identity is unchanged.
|
||||||
|
|
||||||
|
Validate ancestor and unrelated-process cases, stale reporting, PID reuse,
|
||||||
|
cycles, query errors and legacy delegation through isolated command stubs.
|
||||||
|
The regression script gains an explicit unit-only entry that exits before
|
||||||
|
builds and real Task Scheduler tests. This source change neither registers
|
||||||
|
nor changes a scheduled task or Windows permissions.
|
||||||
|
|
||||||
|
### Windows autostart failure reporting (G33 / upstream #3180 and #4284)
|
||||||
|
|
||||||
|
Replace a task with `Register-ScheduledTask -Force` instead of deleting it
|
||||||
|
first. Elevation must return a failure when UAC is cancelled, no process is
|
||||||
|
returned or the child has no exit code; otherwise forward the child's actual
|
||||||
|
exit code. Quote the executable as a literal PowerShell argument. Reuse the
|
||||||
|
existing task-status query to verify successful registration; an unavailable
|
||||||
|
query reports that registration could not be verified, rather than printing
|
||||||
|
an unconditional success. Cancelled or unverified elevation is not retried
|
||||||
|
automatically.
|
||||||
|
|
||||||
|
Both Windows installers should print registration success only after their
|
||||||
|
registration call succeeds. An isolated release install selected by
|
||||||
|
`CUA_DRIVER_RS_HOME` or `CUA_DRIVER_RS_INSTALL_DIR` leaves the shared daemon and
|
||||||
|
autostart task alone unless
|
||||||
|
`-AutoStart` was explicitly requested. This guard also surrounds the shared
|
||||||
|
daemon cleanup before registration; guarding only the closing autostart block
|
||||||
|
would still terminate the ordinary installation's daemon. Default installation
|
||||||
|
and the separate local product identity retain their existing routing.
|
||||||
|
|
||||||
|
Independent before/after tests execute the actual PowerShell payloads and
|
||||||
|
installer blocks against task/UAC/cleanup stubs. They must cover cancellation,
|
||||||
|
null/failed children, preservation of an existing task on registration failure,
|
||||||
|
accurate summaries and isolated/default/explicit parameter combinations. These
|
||||||
|
tests do not perform UAC, task registration or a live installation. Separately authorized native task and owned installation lifecycle tests have since run; their identities and limits are recorded in the current acceptance appendix.
|
||||||
|
|
||||||
|
### macOS background window order (T04)
|
||||||
|
|
||||||
|
The on-screen CGWindow query is authoritative for visible stacking order;
|
||||||
|
the all-window query can disagree even for windows it marks on-screen.
|
||||||
|
Real desktop diagnostics also found that reactivating the already-frontmost
|
||||||
|
application can return success after a second without restoring its order.
|
||||||
|
The guard will instead request AXRaise only on that application's current
|
||||||
|
AXFocusedWindow, after checking its PID and exact visible layer-0 window ID.
|
||||||
|
WindowServer must still name the same front application, and recent hardware
|
||||||
|
mouse/modifier activity prevents restoration. AX calls use a 100 ms messaging
|
||||||
|
timeout, references remain owned for the call, and the existing bounded watch
|
||||||
|
limits remain. This changes neither key focus nor the background action's
|
||||||
|
dispatch path. API acceptance alone is not proof of restored order.
|
||||||
|
|
||||||
|
A single owned-window primitive probe restored visible order about 23 ms
|
||||||
|
after AXRaise while preserving all owned synthetic typing. Three production fix-2 rounds
|
||||||
|
eliminated sustained covering but retained 99.4, 200.5 and 192.8 ms of transient
|
||||||
|
covering, so the zero-interference gate remains open. All owned synthetic typing stayed
|
||||||
|
in A; popup/context, intentional activation and explicit foreground regressions
|
||||||
|
passed. A subsequent strict-ID diagnostic observed the rise after the requested
|
||||||
|
mouse-down, before mouse-up; the activation/warm-up phase showed no rise in that
|
||||||
|
run. This supports further investigation of requested pointer delivery, not
|
||||||
|
removing the warm-up on speculation. The earlier activation-based fix failed
|
||||||
|
and remains recorded.
|
||||||
|
|
||||||
|
One later preparation-only cell skipped both requested events and observed no
|
||||||
|
covering; unchanged guards and sampling still limit that negative result. A
|
||||||
|
separate task-only cell retained preparation and changed only the requested
|
||||||
|
pair to one public CGEventPostToPid publisher per event. Exact-source receipt,
|
||||||
|
native field-editor state and forced final AX focus verified the intended click,
|
||||||
|
but SentinelA still observed 194.110 ms of covering; the independent sampled
|
||||||
|
WindowServer-order interval was 196.695 ms. This transport change is not a
|
||||||
|
demonstrated remedy and was not applied to production. All ten synthetic
|
||||||
|
canaries reached A, but only one was sent within the click call and none during
|
||||||
|
the observed covering interval. This does not establish human-input safety
|
||||||
|
during covering or general zero interference.
|
||||||
|
|
||||||
|
### Off-Space window observation (G17 / upstream #4996)
|
||||||
|
|
||||||
|
Add the upstream single-window SkyLight backing-store capture to the shared
|
||||||
|
capture layer. A known off-Space target tries it before the public SCK/shell
|
||||||
|
chain; current-Space and unknown targets retain public-first ordering. Resolve
|
||||||
|
the private symbol dynamically, retain the returned image independently of its
|
||||||
|
owning CFArray, reject empty or fully transparent images, and preserve the
|
||||||
|
existing image dimension limits and PNG conversion. This adds no process,
|
||||||
|
permission, activation or Space switch.
|
||||||
|
|
||||||
|
The private route must validate the existing WindowCaptureIdentity before and
|
||||||
|
after capture, and refuse attached-window or unresolved attachment geometry.
|
||||||
|
It shares the existing one-worker gate and the three-second native capture
|
||||||
|
budget with SCK; an unreturned native call retains the permit, including after
|
||||||
|
the caller times out. Public shell compatibility retains its current behavior.
|
||||||
|
All consumers continue through the same capture function. A backing-store image
|
||||||
|
can be stale, particularly for off-Space Chromium, so raw and default App
|
||||||
|
observations must carry that limitation rather than imply a new frame proves
|
||||||
|
an action's effect.
|
||||||
|
|
||||||
|
Verification first executes the current entry point with controlled native
|
||||||
|
boundaries, then checks routing, identity changes, attachment refusal, CF
|
||||||
|
ownership, transparent pixels and worker limits. Those checks do not replace
|
||||||
|
real off-Space capture or the separate exact AX remote-token recovery work.
|
||||||
|
|
||||||
|
### Exact omitted AXWindow recovery (G17 / upstream #4068, #4085, #4509)
|
||||||
|
|
||||||
|
After public window, focus/main and attached-sheet discovery fail, a bounded
|
||||||
|
remote-token probe may recover the requested AXWindow. It requires a current
|
||||||
|
same-PID layer-0 WindowServer window known to be off-Space, or known to be on
|
||||||
|
the current Space but off-screen. Unknown membership does not qualify. Each
|
||||||
|
candidate must have the AXWindow role, exact PID and exact direct SPI window
|
||||||
|
ID; frame/title approximation cannot establish this identity. The optional
|
||||||
|
private symbol is resolved dynamically. The 300 ms probe budget is clipped
|
||||||
|
to any earlier recording deadline, with at most 50 ms per native message and
|
||||||
|
2,000 candidate tokens. Native calls remain cooperatively bounded.
|
||||||
|
|
||||||
|
The tree and fresh action facts retain their existing completeness, ownership,
|
||||||
|
minimized/hidden and keyboard-ambiguity gates. Default App selection tries the
|
||||||
|
probe only once after public selection/reopen has finished, and only with one
|
||||||
|
candidate among all same-PID layer-0 windows. A recovered root still resolves
|
||||||
|
its attached sheet; an incomplete, ambiguous or foreign sheet refuses selection.
|
||||||
|
An exactly proven App surface can be selected despite an incomplete global AX
|
||||||
|
array, as with the existing focused-window path; its subsequent tree/facts must
|
||||||
|
still pass their stronger completeness gates. No input is sent, window activated,
|
||||||
|
Space switched, or AX object cached across requests by this recovery.
|
||||||
|
|
||||||
|
### Chromium observation readiness (G24 / upstream #3783)
|
||||||
|
|
||||||
|
An accepted modern accessibility opt-in remains pending until AXWebArea is
|
||||||
|
observed. Default App AX observations share one 500 ms readiness deadline
|
||||||
|
across window selection and tree walking. Recording, input facts and pure
|
||||||
|
screenshots request opt-in without polling or consuming a wait attempt.
|
||||||
|
Positive evidence returns immediately; pending results have a 30-second
|
||||||
|
cooldown and at most three wait attempts per process lifetime. The cache is
|
||||||
|
keyed by PID and kernel start stamp, with lifetime validation serialized with
|
||||||
|
publication so a stale completion cannot overwrite a newer process's readiness.
|
||||||
|
|
||||||
|
Independent controlled verification covers 16 readiness cases, the old/new
|
||||||
|
publication race, and two actual core propagation tests. Real Chrome rejected
|
||||||
|
the opt-in in the available fixture, so accepted-path native readiness remains
|
||||||
|
unverified. The 500 ms budget is not a whole-observation latency bound. The
|
||||||
|
separate AppKit still-launching recovery is described below.
|
||||||
|
|
||||||
|
### Native named-key flags found during G17 acceptance
|
||||||
|
|
||||||
|
The global chord, desktop held-key transition and both PID named-key builders
|
||||||
|
overwrite CoreGraphics flags to isolate explicitly requested modifiers. This
|
||||||
|
also strips the intrinsic Function/NumericPad bits of special keys. Independent
|
||||||
|
native event construction reproduced the loss without posting any input; the
|
||||||
|
upstream global helper shares it. Preserve the keycode-specific structural bits
|
||||||
|
and add only explicit or session-owned modifiers. Do not copy factory flags,
|
||||||
|
which may contain unrelated physical modifiers. Fn itself remains a held
|
||||||
|
modifier, so its release has no intrinsic Fn bit. Unicode and bare forwarding
|
||||||
|
keep their existing event semantics. Routing, timing and single-dispatch
|
||||||
|
behavior are unchanged. Real Space navigation needs a separate effect oracle.
|
||||||
|
|
||||||
|
### AppKit launching-window lookup (G24 / upstream #4309)
|
||||||
|
|
||||||
|
The separate AppKit recovery uses a one-second native lookup allowance, the
|
||||||
|
exact upstream default. Its absolute deadline shares the existing observation
|
||||||
|
start: subtract the named Chromium readiness budget from that deadline, then
|
||||||
|
add the launch allowance. Chromium still receives only 500 ms. No readiness
|
||||||
|
deadline means no launch wait, including recording and action facts. This
|
||||||
|
internal coupling is covered by a shared-anchor regression test.
|
||||||
|
|
||||||
|
Only unresolved windows in a process known to be still launching receive a
|
||||||
|
retry. Default App selection waits for natural resolution instead of issuing
|
||||||
|
another reopen request; exact tree discovery retains its full scope/ownership
|
||||||
|
checks. Retry pauses are at most 50 ms and leave room for the previous attempt's
|
||||||
|
duration. Each discarded attempt releases its AX objects and incomplete state;
|
||||||
|
the final attempt determines the result. A tree walk tries the G17 remote-token
|
||||||
|
fallback at most once across retries. This is a cooperative retry allowance,
|
||||||
|
not a hard bound on native calls or the complete observation. Existing finished
|
||||||
|
application reopening, node-walk budgets and outer timeouts remain separate.
|
||||||
|
|
||||||
|
Independent verification passed 27 controlled actual-function cases and 49
|
||||||
|
G17 consumer regressions. It caught a sleep-overrun defect; the repaired loop
|
||||||
|
checks its deadline again before another active AX retry. The default App
|
||||||
|
wrapper still calls its ordinary final body, which may itself resolve AX.
|
||||||
|
A later native G24 fixture establishes still-launching state at the single public request, followed by the correct AX marker in 1.056 seconds and independent finished-launching readback. This validates that cold public chain, but does not identify the retry responsible, prove the old 400 ms path fails, or impose a hard one-second bound: the first AX IPC may itself wait for readiness.
|
||||||
|
|
||||||
|
### Recording AX capture budget (G30 / upstream #4215)
|
||||||
|
|
||||||
|
Recording AX callbacks run outside the recording-session lock with a fixed
|
||||||
|
two-second caller backstop. macOS also gives the recording tree walker a
|
||||||
|
one-second cooperative node deadline, retaining a completed prefix. Ordinary
|
||||||
|
observations keep their existing budgets. At most four native capture workers
|
||||||
|
may run, with one per PID; a timeout holds its slot until the provider actually
|
||||||
|
returns. Late results are discarded. The worker inherits the requesting
|
||||||
|
runtime, authorization and trusted session evidence, with no primary-runtime
|
||||||
|
fallback. The callback registration signature remains unchanged.
|
||||||
|
|
||||||
|
Before and after capture, the recording verifies that the generation and owner
|
||||||
|
still match. Only the second check and artifact commit hold the session lock,
|
||||||
|
so stopping or replacing a recording cannot admit a late capture into either
|
||||||
|
session. Screenshot privacy rules and video-stop behavior retain their existing
|
||||||
|
boundaries. This adds native threads, not a process or public budget option.
|
||||||
|
|
||||||
|
The new Windows recording thread owns its COM apartment: successful COM
|
||||||
|
initialization must be paired with uninitialization after all recording-only
|
||||||
|
node bindings are released. It must not release an apartment when initialization
|
||||||
|
fails, including a conflicting apartment mode, or publish recording nodes into
|
||||||
|
the action cache. Other UIA callers retain their existing lifecycle.
|
||||||
|
|
||||||
|
The macOS recording hook also owns the walk's extra actionable-element retains.
|
||||||
|
It uses the existing unpublished snapshot owner to release them after JSON
|
||||||
|
serialization, without replacing action tokens. Independent actual-walker/hook
|
||||||
|
tests with controlled AX data and real CoreFoundation reference counts reproduce
|
||||||
|
one leaked retain per call before the fix and zero after repeated complete and
|
||||||
|
incomplete captures. Existing tokens remain valid and seven payloads match
|
||||||
|
apart from elapsed time. Three native regression tests and macOS/SDK all-target
|
||||||
|
checks pass. These are ownership tests; native GUI capture, serialization failure
|
||||||
|
and an actual elapsed-budget truncation are not established by this fixture.
|
||||||
|
|
||||||
|
### Snapshot identity (G18 / upstream #3616 and #3873)
|
||||||
|
|
||||||
|
Upstream #3616 / #3873 require the snapshot identity and its native element
|
||||||
|
bindings to agree. A controlled concurrent observation reproduced an old App
|
||||||
|
short ID resolving to a replacement native object: token resolution and the
|
||||||
|
per-window cache lookup were separate operations.
|
||||||
|
|
||||||
|
The existing token registry will publish the window, index membership, token
|
||||||
|
generation and an immutable owned native payload in one entry. Resolution keeps
|
||||||
|
an `Arc` to that payload. Platform actions read a bound cache view, including
|
||||||
|
Windows kind/role/geometry and Linux identity/actions, instead of the latest
|
||||||
|
mutable window cache. This covers default App calls, raw native calls and their
|
||||||
|
interleaving without holding a GUI lock across an observation or action. Token
|
||||||
|
format, runtime isolation, same-window invalidation and the existing LRU stay
|
||||||
|
unchanged; no upstream SnapshotStore or new public API is introduced.
|
||||||
|
|
||||||
|
Publication changes affect the three `get_window_state` implementations and
|
||||||
|
native element caches. Consumers include click/double/right click, keyboard
|
||||||
|
input, setValue, scrolling, secondary actions and Mac selectText. Both Mac
|
||||||
|
`ClickTool` and the separate `invoke_app_click` path use the bound view. Recording
|
||||||
|
markers also retain the exact binding rather than resolving a bare index again.
|
||||||
|
Pending captures own their native references and release them on failure or
|
||||||
|
cancellation. Runtime/target cleanup releases registry ownership; a previously
|
||||||
|
resolved action keeps its own payload alive until it finishes.
|
||||||
|
|
||||||
|
A degraded Mac AX-window observation revokes the previous target snapshot in
|
||||||
|
the calling runtime; it does not publish an unresolved tree or revoke another
|
||||||
|
runtime's observation. Each platform cache's `clear_target` also revokes this
|
||||||
|
runtime's tokens. Existing watchers separately clear all runtime lanes only
|
||||||
|
after proving that the native window or process has closed. Session dispatch
|
||||||
|
guards defer end cleanup until the observation finishes; cancellation releases
|
||||||
|
unpublished pending payloads rather than publishing after cleanup.
|
||||||
|
|
||||||
|
Verification pauses publication and lookup around a competing snapshot, checks
|
||||||
|
old-object identity or a stale refusal, and covers cross-session/raw/App calls,
|
||||||
|
sparse Linux indices, Windows UIA/MSAA metadata, retain/release accounting and
|
||||||
|
screenshot-only observations. Native desktop acceptance remains separate.
|
||||||
|
|
||||||
|
### Implicit session recovery after idle cleanup (G08 / upstream #4283)
|
||||||
|
|
||||||
|
An idle sweep currently leaves the still-connected unnamed MCP/SDK lifecycle
|
||||||
|
permanently ended. Record whether a tombstone came from idle reclamation of its
|
||||||
|
owner's implicit lifecycle. Only that same owner may recreate it, at the existing
|
||||||
|
session admission boundary, after all cleanup hooks complete. Recheck under the
|
||||||
|
tombstone lock before removing it and inserting the next lifecycle record.
|
||||||
|
Named sessions, explicit end, transport exit and revocation remain terminal.
|
||||||
|
The daemon adapter's early mirror must admit this one recoverable state and
|
||||||
|
follow core revival notifications. Direct SDK and MCP share core admission.
|
||||||
|
|
||||||
|
Recreation must not restore prior resources. Snapshot entries retain the private
|
||||||
|
publishing session identity so end cleanup can retire only its native bindings;
|
||||||
|
App short IDs, capture scope, grants and recording already have owner cleanup.
|
||||||
|
Test unfinished cleanup, wrong owner, named/explicit/transport termination,
|
||||||
|
concurrent admission and an old raw snapshot token after recreation, then repeat
|
||||||
|
the actual direct MCP idle-TTL reproduction.
|
||||||
|
|
||||||
|
### Native Electron background scrolling (G05)
|
||||||
|
|
||||||
|
Upstream #4861/#5079 reveal content through native accessibility when Electron
|
||||||
|
ignores background wheel input. This is a native application operation; it does
|
||||||
|
not use the upstream browser API. Reproduce the candidate's current refusal and
|
||||||
|
compare the same owned scroll fixture with upstream before implementation.
|
||||||
|
The native before now confirms five candidate refusals with unchanged offsets;
|
||||||
|
stock 0.35.4 moves the inner pane by 114 pixels and the page by 404 pixels in
|
||||||
|
four corresponding single-request cases.
|
||||||
|
|
||||||
|
Adapt the geometry and clamped-row selection while preserving the
|
||||||
|
approved dispatch contract: prove the exact window and native element ancestry,
|
||||||
|
choose the innermost relevant scroll container, and send at most one
|
||||||
|
`AXScrollToVisible` action. Do not copy the upstream fixed 150 ms effect wait or
|
||||||
|
its retry on another container after an unconfirmed dispatch. Return dispatch
|
||||||
|
acknowledgement and use the next observation for effect evidence. A preflight
|
||||||
|
absence may refuse; an uncertain action must not be replayed. Existing Browser
|
||||||
|
SDK behavior remains outside this change.
|
||||||
|
|
||||||
|
Selection is read-only and bounded by 4,000 descendants, depth 80 and a private
|
||||||
|
one-second cooperative search budget. Failure to select safely refuses before
|
||||||
|
dispatch. The selected native proxy and process mutation lease live in the
|
||||||
|
blocking worker through the final exact-target check and one AX action, including
|
||||||
|
when its async caller is cancelled. A native AX error after dispatch is an
|
||||||
|
uncertain acknowledgement with a warning; it never starts another route. These
|
||||||
|
bounds do not impose a hard deadline on an in-flight native read.
|
||||||
|
|
||||||
|
The normal v3 SDK passes five native Electron fixture cases: raw inner/page
|
||||||
|
scrolling through AX and pixel targets, plus default App inner scrolling. Each
|
||||||
|
case uses one request; independent fixture IPC observes inner +114 pixels or
|
||||||
|
page +404 pixels while the other region remains unchanged. The original
|
||||||
|
unconfirmed acknowledgement is preserved. A separate exhausted-inner fixture
|
||||||
|
reveals the before defect: both v2 and v3 move the inner pane up 96 pixels
|
||||||
|
after a single down request. Normal v4 repeats the five main cases and passes
|
||||||
|
raw/App exhausted-inner refusals with actual sent:false and unchanged offsets.
|
||||||
|
These seven cases do not establish all-direction behavior,
|
||||||
|
human concurrency or absence of shorter unsampled interference. The v2
|
||||||
|
controlled 43-case result and v3 four geometry unit tests remain distinct from
|
||||||
|
the v3 controlled executable's startup timeout.
|
||||||
|
|
||||||
|
The exhausted-inner diagnostic identifies an earlier row reported at the
|
||||||
|
bottom edge as a one-pixel sliver. The native container is correct; revealing
|
||||||
|
that row reverses the intended movement. The v4 selector uses its existing
|
||||||
|
visible-leaf set to reject clamped targets on the wrong side in document
|
||||||
|
order: down/right after the last visible leaf, up/left before the first.
|
||||||
|
It adds no native reads or actions and leaves real-frame selection unchanged.
|
||||||
|
When no visible anchor exists, the previous heuristic remains; reordered or
|
||||||
|
multicolumn AX graphs still need separate evidence. This is a scoped repair
|
||||||
|
for the reproduced case, not a universal direction guarantee.
|
||||||
|
|
||||||
|
Normal v7 independently repeats all seven native GUI cases with source tree `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693` and SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe`: seven pass, without replay or foreground fallback. Independent fixture IPC confirms the same +114px/+404px offsets and unchanged exhausted boundaries. The 143 samples retain foreground/topmost A, pointer position and clipboard count. All 634 referenced entries are verified. Sampling at 8.444–12.213Hz does not establish absence of shorter interference or human concurrency.
|
||||||
|
|
||||||
|
### macOS release replacement recovery (G36)
|
||||||
|
|
||||||
|
The actual release-install branch loses the previous bundle if copying fails:
|
||||||
|
two isolated before cases confirm that failure before copying or after a partial
|
||||||
|
copy leaves a broken executable symlink. Adapt #3297's staged validation and
|
||||||
|
backup/rollback, using the existing bundle path and signing identity. Verify the
|
||||||
|
downloaded bundle before touching the installed application, keep its previous
|
||||||
|
bytes in a sibling backup until the copied bundle and executable are validated,
|
||||||
|
and restore them on copy/verification failure or a handled interruption. Preserve
|
||||||
|
the backup and report its path if restoration fails. A process kill or power loss
|
||||||
|
cannot execute the shell trap and is not claimed to recover automatically.
|
||||||
|
|
||||||
|
Compare the replacement against the previous designated signing requirement to
|
||||||
|
report permission continuity accurately. Do not import upstream automatic TCC
|
||||||
|
resets: installation never changes authorization records. Defer old local-build
|
||||||
|
cleanup on macOS until replacement succeeds. This keeps the existing installer,
|
||||||
|
daemon, bundle location and permission owner; no new service or helper process
|
||||||
|
architecture is introduced. Validation uses isolated directories and controlled
|
||||||
|
system-command boundaries, not a real application or permission installation.
|
||||||
|
|
||||||
|
### Additional verified upstream ports
|
||||||
|
|
||||||
|
| Reference | Adaptation and evidence |
|
||||||
|
| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| #4967 / #4214 | Linux top-level frame rebasing and local XRes PID fallback. Real GTK before/after fixes wrong-row input and missing windows without `_NET_WM_PID`; scoped to the tested Xorg environment. |
|
||||||
|
| #2907 / #4690 / #5137 | macOS window clicks select one native delivery route; middle clicks retain the exact window. Two fresh mini parent/sheet rounds remove duplicate target events. |
|
||||||
|
| #3755 / #5125 | macOS screenshot uses the absolute system executable; window bounds reject non-finite native numbers. Restricted-PATH real capture and native CFNumber regression pass. |
|
||||||
|
| #5136 / #4333 / #3975 | Windows console inheritance, GDI failure cleanup and resolved shortcut environment paths. Actual MSVC tests pass; interactive desktop behavior is a separate pending gate. |
|
||||||
|
| #3277 / #4985 | Embedded driver preserves explicit telemetry and update-check preferences through its environment filter. |
|
||||||
|
| #3462 / #4888 | Shared typed schema generation removes Schemars-only numeric formats; input declarations use the supported dialect while runtime validation remains authoritative. Declaration tests and offline MCP wire checks are distinct from provider acceptance. |
|
||||||
|
| #3632 | Recording reserves turns only for its owner session or implicit trusted connection; lifecycle calls are omitted. Independent testing caught an implicit-owner omission; the second fix passes public registry checks for named owners, implicit connection owners, foreign connections and lifecycle exclusion. |
|
||||||
|
| #4238 | ScreenCaptureKit stop waits for the recording-output finish callback. Real mini baseline reproduces false empty-file failures and post-return growth; five new-candidate 120ms recordings decode immediately and retain the same file hash after one second. |
|
||||||
|
| #3328 | Display scaling reads the current mode's backing pixels divided by points. The public tool reports 2 instead of 1 on the same real Retina display; coordinate dispatch is a separate contract. |
|
||||||
|
| #3714 / #5144 / #5145 | Encoder exit/timeout becomes an explicit error, and recording timestamps preserve one start baseline with ISO-8601 action timestamps. Public API probes verify the failure details and output metadata. |
|
||||||
|
|
||||||
|
This table records implemented groups, not a claim that every upstream commit has
|
||||||
|
been reconciled. The full inventory and remaining native gaps are tracked in the
|
||||||
|
acceptance evidence ledger. A successful cross-platform build does not replace
|
||||||
|
real desktop effect and interference checks.
|
||||||
|
|
||||||
|
## Validation boundaries
|
||||||
|
|
||||||
|
See the ignored `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/` reports for immutable source manifests, build logs, protocol calls and fixture readback. The current [acceptance snapshot](cua-upgrade-acceptance-2026-10.md#acceptance-clarification-2026-10-20) separates earlier controlled/GNU/Xvfb checks from the later actual Windows MSVC and interactive-desktop tests, Linux Xorg/uinput/Mutter tests and macOS native fixtures. None establishes physical-human concurrency. The user clarified that observed transient macOS target/menu cover is acceptable; retain measurements and recovery separately from functional correctness. Model task success, token counts and timing parity require actual second-layer evidence. The user scheduled OSWorld after release on 2026-10-20; it is not a pre-release gate.
|
||||||
|
|
||||||
|
Later acceptance repairs supplement the upstream ports. Exact AppKit key-window records fix the reproduced sheet Return failure; they contain internal NaN-location mouse events and do not solve pointer/window coverage. Isolated Windows uninstall ownership now covers every task action and only the selected process/install roots, verified on actual PowerShell. Finite CLI commands skip their observation self-wrapper when effective telemetry consent is false; existing consent precedence and enabled behavior remain. This CLI-only v21 change is verified separately from unchanged SDK backend artifacts. Package/runtime appendices retain local developer signing and installer seam limits; they do not establish official signing/notarization or a production release.
|
||||||
|
|
||||||
|
### Acceptance-driven input boundary repairs
|
||||||
|
|
||||||
|
Linux v24 distinguishes pre-dispatch refusal from uncertain delivery throughout text prefixes, first/named-key presses, releases and flushes. Event-emission errors carry sent uncertainty even when the first requested press fails; preinput focus checks still refuse before input. MPX pending requested input is retained only after target-focus admission. A failed press and the focus-probe key each get bounded best-effort release cleanup, retaining uncertainty instead of claiming completed text. Real focused tests and fresh Xorg/Mutter cases separately verify this behavior; an uncertain paused-target result is never automatically replayed or scored as complete text.
|
||||||
|
|
||||||
|
Mac v25 handles an app-focused same-window AXGroup that explicitly reports AXFocused=false while a genuine text descendant holds focus. It delegates only this shape to the existing complete, unique, bounded actual-focus search. Unknown focus, cross-window focus and other roles keep their existing behavior; PID/window checks and final focus/window revalidation remain. Actual LibreOffice ASCII insertion passes. The original Cmd-S/AppleEvents Save remains UNKNOWN; an independent v25 toolbar Save has actual FODT evidence. Accepted transient target/menu cover remains measured in the acceptance record. AX settable/edited flags alone do not establish an applied edit or saved file.
|
||||||
|
|
||||||
|
### v26 pointer-reading target refusal
|
||||||
|
|
||||||
|
A real Calc request selected B28 instead of requested C5. Exact LibreOffice VCL source computes mouse callback coordinates from the global hardware pointer, which PID-routed clicks do not move. Following upstream #4864's safety principle, the fork now identifies the official `org.libreoffice.script` app through its existing NSRunningApplication boundary and refuses routed pixel click/right-click/double-click alternatives before input. Raw foreground PID routes also refuse; merely activating a window does not fix this mismatch. Advertised AX actions and safe AX editable-focus/selection remain available. This scoped identity rule is not upstream's general mapped-library detector, so it makes no claim for rebranded/unbundled VCL targets. Canonical public Mac delivery stays background-only.
|
||||||
|
|
||||||
|
The v26 tree `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3` changes only these three Mac tools from v25. A fresh public-contract refusal returns background_unavailable/sent:false and leaves the same exact-window Name Box at A1 in three bounded reads. This is a safety refusal, not successful C5 background selection; raw AXPress runtime equivalence remains separate. The historical strict zero-cover predicate was not met, but the user subsequently clarified that observed transient target/menu cover is acceptable. See the v26 acceptance checkpoint for native/source/build evidence and the retained original harness-assertion failure.
|
||||||
|
|
||||||
|
### v27 scoped Writer editable focus
|
||||||
|
|
||||||
|
The official LibreOffice Writer body exposes AXTextArea with no advertised action and writable AXValue/AXFocused. The intermediate v27 candidate admits only this bundle's text area into the default App branch's editable-focus path, retaining both attribute gates, exact ownership, single unmodified left-click conditions and value-preserving focus readback. Generic text areas do not gain this shortcut; the raw path is unchanged from v26. Fresh mini native reads show that this admission alone does not establish focus, despite a successful AX setter status.
|
||||||
|
|
||||||
|
F31 adds existing exact-window background keyboard preparation and an existing read-only selection-range query before the single AXFocused setter. The same-version LibreOffice caret getter initializes its cursor cache/context, which explains the purpose of the otherwise non-obvious read. There is no range write, caret relocation, text replacement or pixel fallback. The original observer, which makes no caret query, reads the same seed body false before and true after one fresh mini App click. One input and one toolbar Save then produce exact live and saved XML text. Conservative confirmation and sent/unconfirmed results remain: observe before the next action, never replay. A preparation failure can follow native focus records and must not be treated as proof that nothing was sent. Rejected prep-only and range-write experiments retain their own source and native evidence.
|
||||||
|
|
||||||
|
Preparation retains the existing keyboard-target ambiguity gate because it can send a PID-scoped ReturnKeyFocus notification before the exact-window records. An AX element binding alone does not prove that notification safe with competing keyboard destinations. The two fresh Writer positives each have one document window; arbitrary multi-window focus switching is not established or enabled by deleting the safety gate.
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
# Computer Use 平台移植记录(2026-10)
|
||||||
|
|
||||||
|
[English](cua-upgrade-platform-ports.md) | [简体中文](cua-upgrade-platform-ports.zh-CN.md)
|
||||||
|
|
||||||
|
本文是已批准[升级方案](./cua-upgrade-2026-10.zh-CN.md)的实施台账。移植来源为 trycua/cua 截至 `cua-driver-rs-v0.35.4` 的 MIT 代码,Copyright (c) 2025 Cua AI。仅原生 driver 库作为来源,不纳入上游 bench 包及其许可证。
|
||||||
|
|
||||||
|
## 派发合同
|
||||||
|
|
||||||
|
App 输入在 driver 内解析精确应用和窗口。macOS 使用辅助功能与进程定向事件,不自动回退全局前台。Windows/Linux 仅在原生层返回派发前 `background_unavailable` 且 `sent:false` 时,允许一次前台路径。可能已经派发后的执行器错误不回放。显式桌面输入是独立前台接口。
|
||||||
|
|
||||||
|
前台事务观察物理输入而不消耗它;检测到后停止剩余输入,保留用户选择的前台 C,仅在仍拥有 B 时恢复原 A。Windows 恢复前等待目标输入队列排空,并在 UTF-16 单元间检查目标。Linux 在 XTest 前同时确认 EWMH 激活和核心输入焦点。
|
||||||
|
|
||||||
|
成功表示派发确认,不表示应用效果已确认。输入调用不等待截图变化,下一次观察提供效果证据。持续按压和多击仅保留完成手势所需的时序。
|
||||||
|
|
||||||
|
## macOS
|
||||||
|
|
||||||
|
| 上游来源 | 适配 |
|
||||||
|
| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
|
| #4943 | 定向焦点到 B 时,不给 A 发送合成失焦通知。 |
|
||||||
|
| #4943, #4955, #5091, #5129 | 键盘焦点保护复用进程级 `app_focus` 观察者;有界尾部/层序恢复和主动焦点 epoch 避免观察者竞争。 |
|
||||||
|
| #4976 | 物理鼠标/修饰键活动优先保留用户切换;现有激活保护检查同一物理输入状态。 |
|
||||||
|
| #5051 | 定向鼠标事件显式设置修饰键,包括空集合。 |
|
||||||
|
| #5057 | 保存已有文件和 Finder 改名使用有界 Apple Events 并读回;参数作为脚本参数传入,不拼接 AppleScript;Automation 走正常 TCC 授权。 |
|
||||||
|
| 弹出选择 | AX 打开菜单并选择唯一匹配项;拒绝时关闭本次打开的菜单。Safari 原有 DOM 回退仍仅限实际前台文档。 |
|
||||||
|
| 文本插入 | ASCII 使用定向事件;非 ASCII 要求聚焦可编辑元素和 AX selected-text 支持,并读回预期插入;不确定 AX 写入后不重放键盘。 |
|
||||||
|
| 文档模型 | 已验证的 AppKit 文档字段 AX 写入,通过定向且净变化为零的编辑 shim 注册修改;无法验证注册时拒绝。 |
|
||||||
|
|
||||||
|
后台拖动在派发前拒绝;滑块使用 `setValue`;粘贴保留剪贴板恢复事务。删除旧的动作后固定一秒等待。AX 确认或事件投递本身都不算效果确认。
|
||||||
|
|
||||||
|
P7 对完整原生工具模块的受控 provider 测试复现:raw 双击/右键的 AXOpen/AXShowMenu 先产生效果再报错,旧实现继续发送鼠标动作。现在一旦尝试该 AX 动作,错误直接返回正常 sent/unconfirmed 响应与警告,不进入鼠标路径。成功语义路径及未提供语义动作时选择的鼠标路径保持原行为。该语义分支未执行前台事务,因此即使请求 foreground,也明确报告实际 background。独立测试执行完整模块及真实 core 公开投影,双击六格、右键四格均通过,三项不确定结果保持 unverifiable/accessibility 并保留警告。默认 App 的提前转发仅作对照,不证明 App 缺陷或真实 AX 服务器生效后报错。
|
||||||
|
|
||||||
|
## Windows
|
||||||
|
|
||||||
|
| 上游来源 | 适配 |
|
||||||
|
| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| #4500 | 连接实际目标输入队列,以 sentinel 读有界等待排空再恢复。 |
|
||||||
|
| #4901 | UTF-16 单元之间检查前台所有权和物理输入;单元停滞后停止,不回放。 |
|
||||||
|
| #3671 | UIA 点动作超时与派发前不存在合适 pattern 分开,不确定动作不能变成第二次坐标点击。 |
|
||||||
|
| #4774 | 原生 combo 使用控件选择/通知消息,并验证选中值。 |
|
||||||
|
| Tk | 输入前拒绝不支持的后台鼠标路径,随后共享 App 派发器可前台执行一次。 |
|
||||||
|
| 扩展键 | 左右 Windows 键保留 extended-key 标志。 |
|
||||||
|
| Worker IPC | 有界读取响应;超时说明可能已发送。UIAccess 路由明确 opt-in。 |
|
||||||
|
| 普通用户启动 | 高权限 daemon 从自身 token 派生并验证 Medium token,管理员成员为 deny-only;不打开/借用其他进程 token。直接可执行文件和 shell helper 均使用它,失败则拒绝,不高权限启动应用。 |
|
||||||
|
|
||||||
|
保留正常安装/UAC 流程。交叉编译仅证明构建,不证明交互桌面、UAC、运行时 token 或共享桌面打扰。
|
||||||
|
|
||||||
|
## Linux
|
||||||
|
|
||||||
|
删除原生 Wayland、helper、依赖和安装载荷。X11 和 XWayland 中的 X11 应用共用 X11 实现;不支持 compositor 原生 Wayland 表面。
|
||||||
|
|
||||||
|
| 上游来源 | 适配或现有等效实现 |
|
||||||
|
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
|
| #3882 | MPX 鼠标/键盘、toolkit 拒绝规则、遮挡和焦点保护,不使用输入屏障;保留 fork 精确窗口检查。 |
|
||||||
|
| #3761 | hold 计时前 flush/check 按下;短连接关闭前无条件 server round-trip。P7 后续复现表明临时映射仅靠 server 送达不够,下面的客户端队列修复仍需验收。 |
|
||||||
|
| #3762 | 检查按下/释放,保留时序,出错后释放已排队按钮和修饰键。 |
|
||||||
|
| #3864 | 保留观察到的精确 bus/object 点击身份和窗口快照校验;session-owned MPX 持久化来自 MPX 总体适配,而非此身份补丁。 |
|
||||||
|
| #3618 | 修饰键前台拖动用真实键状态包围并清理;不支持的后台组合在派发前拒绝。 |
|
||||||
|
| #4248 | 新 App 合同无动作后效果分类器,不会因 WM 暂存焦点把关闭 B 对话框误判无效果。精确前台恢复保留 C,不从未知 popup 推断确认;不移植上游截图/窗口变化等待。 |
|
||||||
|
| #4188 | 坐标 AX 激活排除 application/frame/window 外壳;识别 NUL 分隔和空格拼接的 Chromium helper 命令行。 |
|
||||||
|
| #5139, #5158 | Unicode keysym、全文字符预检;首个事件前预留/重映射空键码,序列化临时映射,不静默漏字。 |
|
||||||
|
| #3601 | master 所有权包含 boot、PID namespace、UID、进程开始身份;仅回收已证明属于本 driver 家族的过期设备。 |
|
||||||
|
| #2888 | KDE 设备热插拔可能重配物理输入,因此保守拒绝 MPX;不自动改变 `/dev/uinput` 权限。 |
|
||||||
|
| #5070 | 后台键盘拒绝明确指出缺 AT-SPI bus 和正常配置路径;诊断探测有界。 |
|
||||||
|
| #4903 | 删除 AT-SPI P2P feature/peer 调用,使用一个注册连接。 |
|
||||||
|
| #4984 | 仅基于有效完整屏幕 frame 重定位;拒绝不完整或零尺寸几何。 |
|
||||||
|
| #5074 | 首次子树遍历前通过 attributes 唤醒应用。 |
|
||||||
|
| #4375 | 首次语义遍历两秒,之后一秒;不对不完整观察做通用整树重试。 |
|
||||||
|
| GTK combo | `Selection.SelectChild` 选择唯一选项,读回 selected-child 名称,不改焦点/前台。 |
|
||||||
|
| #4292 | 文本与标签分开;数字控件优先 numeric Value,不读取密码 Text。 |
|
||||||
|
| #3758 | DirectColor 截图用 X colormap 解码,不当作 TrueColor mask。 |
|
||||||
|
| #4281 | kill 成功等待实际退出;zombie 算已退出。 |
|
||||||
|
| #3814 | 精确窗口截图明确 `screenshot_frame_valid:true`。 |
|
||||||
|
| #4529 | overlay 未映射时准备空 input/bounding shape,仅绘制可见内容时映射,空闲帧取消映射。 |
|
||||||
|
| #4715 | 非 Linux stub 不再声明已删除截图工具;portable menu schema 不走 cursor-only 断言。 |
|
||||||
|
| #3658 | Enter/Return/KP/Numpad 别名共用终端 gate 和映射。 |
|
||||||
|
| #3778 | 既有 opt-in 原生浏览器工具共用 session cursor;非活动 tab 不移动可见光标,不把上游浏览器工具加入默认接口,也不改仓库 Browser SDK。 |
|
||||||
|
|
||||||
|
fork 修复还包括按窗口保留截图缩放,不重复缩放省略位置的 mouse-up,Qt 合成 FocusIn 限 Qt5,按实际核心焦点决定 FocusOut,仅在路由需要时分类 Chromium。
|
||||||
|
|
||||||
|
### P7 Linux 临时键盘映射生命周期
|
||||||
|
|
||||||
|
在自有 GTK 主循环暂停三秒时,候选和上游都丢失重映射 Unicode:server 已送键码,driver 恢复映射,客户端之后把它译为 NoSymbol。此复现使用独立 nested Xorg,与不稳定主 GNOME 分开。
|
||||||
|
|
||||||
|
候选在修改映射或输入前准备 X11 客户端队列确认,仅需要空键码的操作付出此成本。目标或同 X client 的祖先必须声明 `_NET_WM_PING`;同进程另一连接不算。XTest 在每字及确认前后检查焦点 client。派发后收到 root 上精确 nonce 回声再释放临时映射。五秒超时恢复映射并正常返回已发送但未确认,不回放;部分输入期间发现焦点变化仍停止后续输入并报错;缺协议支持则输入前拒绝。ASCII 保留 server round-trip,不增加 ping。鼠标修饰键不在现有 map 时提前拒绝,因为借键码不能建立真实 modifier mapping。
|
||||||
|
|
||||||
|
MPX 在 ping 前还必须确认 raw-key server delivery。受控复现发现旧 last-key 谓词在六个计划事件只到两个时,把较早的重复 release 当成完成。新实现要求准确虚拟 slave 的每次有序转换,包括重复键和 Shift。旧版零 source ID 仅在 device ID 等于该 slave 时允许,不能用共享 master。相同 source 的意外转换使结果未确认;保留设备的 pending 状态,下一请求在映射、焦点或输入前拒绝,正常 session 清理随设备删除。预热也确认完整批次,且先于借键码;保留 keymap→MPX 锁顺序。独立生命周期 10/10、匹配器 13/13、下述预热校验后的 Linux 单测 16/16 通过。内核积压和物理设备仍待验;受控测试替换了 OS 边界。
|
||||||
|
|
||||||
|
预热前 XKB 必须识别准确 slave,证明 X248 所有 group/level 都无符号。占用、无效或不可读时零输入拒绝并保持 pending;不临时重映射固定键使检查通过。独立验证覆盖 20 个 map/query 和三个 pending 入口;C header 与 Rust layout 一致。真实只读 Xephyr slave 的 empty-group map 仅为形状对照,不代表新 uinput 设备。
|
||||||
|
|
||||||
|
不引入焦点 grab、输入屏障、剪贴板变更、永久映射或新 helper 进程。GTK/Chrome 证明队列机制;Chrome 回声早于 DOM 输入结束,明确不是语义成功。native-v5b 的 GTK 响应、暂停三秒、超时、错误 nonce、缺协议分别通过效果/拒绝/不确定检查。Chrome 机制检查使用 v5a;v5b 仅改不确定结果报告,另有验证。物理 MPX 未验证。
|
||||||
|
|
||||||
|
## P7:上游正确性收口
|
||||||
|
|
||||||
|
### 普通 macOS AX 遍历预算(G31 / 上游 #3882)
|
||||||
|
|
||||||
|
原 raw 和默认 App 观察仅限制深度/节点数,时间预算仅用于录制。适配上游一秒节点遍历预算,从窗口选择和 readiness 之后第一个非重复、准入节点开始。Chromium readiness、AppKit launch、原生 messaging timeout 和外层观察 timeout 分开;此协作限制不能取消挂起的原生读,也不保证整体一秒返回。
|
||||||
|
|
||||||
|
主动截断保留完整前缀,标记 truncated/incomplete。成功尝试的读保留 `read_complete`,与既有节点/深度上限一致;真实 AX 错误清除它。前缀变化时 bounded revision 返回完整替换,缺行不算删除。raw token 跨后续观察仍绑定自己的不可变对象;接受前用真实 walker 结果验证消费者。
|
||||||
|
|
||||||
|
legacy 节点预算与录制绝对期限保留。不新增公开 timeout 参数、输出字段、worker 或动作重试。先做有限慢节点复现与准备延迟对照,再实现;原生延迟另行验收。
|
||||||
|
|
||||||
|
### Windows 安装器祖先保护(G32 / 上游 #4398)
|
||||||
|
|
||||||
|
共享清理可能结束发起 `update --apply` 的 driver 和 installer 子进程,并把受保护父进程误判为 stale。独立复现执行真实 PowerShell 函数,进程/task/pipe 全部 stub,不停止真实进程或任务。
|
||||||
|
|
||||||
|
适配 live ancestor walk,父进程创建时间晚于子进程则视为 PID 重用而拒绝继续。仅清理非祖先 driver PID,survivor/repair 列表排除祖先。存在 driver 祖先时跳过 task-wide `/End`,PID filter 不能保护其免受 Task Scheduler 终止。祖先查询不可用时任何结束动作前停止。legacy migration 复用同一 helper,移除另一份 image-name tree kill;独立 `-local` 产品身份不变。
|
||||||
|
|
||||||
|
隔离 stub 验证祖先、无关进程、stale、PID 重用、cycle、查询错误及 legacy 委托。回归脚本新增 unit-only,在构建/真实任务测试前退出。此源修改本身不注册/改变任务或 Windows 权限。
|
||||||
|
|
||||||
|
### Windows 自动启动失败报告(G33 / 上游 #3180 和 #4284)
|
||||||
|
|
||||||
|
以 `Register-ScheduledTask -Force` 替换任务,避免先删。UAC 取消、无进程或无 exit code 均失败,否则转发子进程实际码;可执行文件作为 PowerShell 字面参数引用。复用状态查询验证注册;查询不可用时报告无法确认,不无条件成功。不自动重试取消或未确认的提升。
|
||||||
|
|
||||||
|
两个安装器仅在注册调用成功后提示成功。用 `CUA_DRIVER_RS_HOME` 或 `CUA_DRIVER_RS_INSTALL_DIR` 选择隔离 release 安装时,除非显式 `-AutoStart`,保留共享 daemon/task。此 guard 也包围注册前共享清理,仅保护结尾会仍杀普通安装的 daemon。默认安装与独立 local 身份保留原路由。
|
||||||
|
|
||||||
|
独立 before/after 执行真实 PowerShell payload 和 installer block,task/UAC/cleanup 为 stub;覆盖取消、null/失败子进程、注册失败保留旧任务、摘要及隔离/默认/显式参数组合。这些不是 UAC、真实注册或完整安装;后续已在独立授权下执行真实任务与自有安装生命周期测试,身份及边界见当前验收附录。
|
||||||
|
|
||||||
|
### macOS 后台窗口层序(T04)
|
||||||
|
|
||||||
|
可见层序以 on-screen CGWindow 查询为准;all-window 查询对标为 on-screen 的行仍可能排序不同。真机诊断还发现重新激活已在前台的应用,一秒后返回成功却不恢复层序。guard 改为仅对其当前 AXFocusedWindow 请求 AXRaise,核验 PID 与精确可见 layer-0 ID。WindowServer 必须仍指向同一前台应用,近期物理鼠标/修饰键活动阻止恢复;AX messaging timeout 为 100ms,引用全程持有,保留现有有界观察。不改键盘焦点或后台派发路径;API 接受不证明层序恢复。
|
||||||
|
|
||||||
|
单个自有窗口原语探测在 AXRaise 约 23ms 后恢复可见层序,保留全部模拟输入。生产 fix-2 三轮消除持续覆盖,但仍有 99.4、200.5、192.8ms 瞬态,零干扰门槛未关闭。全部模拟输入到 A;popup/context、主动激活、显式前台回归通过。后续 strict-ID 诊断在请求 mouse-down 后、mouse-up 前观察到升层,激活/预热阶段未出现;支持继续调查请求的鼠标投递,不能据此猜测移除预热。早期激活式修复失败并保留。
|
||||||
|
|
||||||
|
后续 preparation-only 一格跳过两个请求事件,未观测覆盖,但现有 guard 和采样限制该阴性结果。另一 task-only 格保留准备,仅把请求 pair 改为每事件一次公共 CGEventPostToPid;精确源 PID 事件、原生 field-editor 和强制最终 AX focus 确认点击,却仍被 SentinelA 观测到 194.110ms 覆盖,独立 WindowServer 采样口径 196.695ms。未证明此传输变化能修复,因此未入生产。十个模拟 canary 全到 A,但调用内仅一个,覆盖期间零个;不证明覆盖时真人输入安全或普遍零干扰。
|
||||||
|
|
||||||
|
### 跨 Space 窗口观察(G17 / 上游 #4996)
|
||||||
|
|
||||||
|
共享 capture 层新增上游单窗口 SkyLight backing-store 捕获。已知 off-Space 时先试私有路径,再 public SCK/shell;当前或未知 Space 保留 public-first。动态解析私有符号,返回图像从其 CFArray 独立 retain,拒绝空或全透明图,保留尺寸限制和 PNG 转换。不新增进程、权限、激活或 Space 切换。
|
||||||
|
|
||||||
|
私有路径 capture 前后校验原有 WindowCaptureIdentity,拒绝 attached-window 或不明确的 attachment 几何。与 SCK 共用一 worker gate 和三秒原生预算;未返回调用在外层超时后仍持 permit。public shell 兼容行为不变。所有消费者共用 capture 函数。backing-store 可能过时,尤其 off-Space Chromium;raw 和默认 App 观察必须说明,不能把新图当作动作效果证明。
|
||||||
|
|
||||||
|
先以受控原生边界运行真实入口,验证顺序、身份变化、attachment 拒绝、CF 所有权、透明像素和 worker 上限。这不代替真实 off-Space capture,也不代替独立精确 AX remote-token 恢复。
|
||||||
|
|
||||||
|
### 精确遗漏 AXWindow 恢复(G17 / 上游 #4068、#4085、#4509)
|
||||||
|
|
||||||
|
公共窗口、focus/main、attached-sheet 发现失败后,可用有界 remote-token 恢复指定 AXWindow。要求实时同 PID、layer-0 WindowServer 窗口,已知 off-Space 或已知当前 Space 但 off-screen;未知 membership 不允许。候选必须 AXWindow role、精确 PID、直接 SPI window ID,frame/title 近似不能证明身份。私有符号动态解析;300ms 预算被更早的 recording deadline 截短,每原生消息至多 50ms、至多 2,000 tokens,仍是协作边界。
|
||||||
|
|
||||||
|
tree 和 fresh action facts 保留完整性、owner、minimized/hidden、键盘歧义 gate。默认 App 在 public selection/reopen 后仅一次探测,且所有同 PID layer-0 窗口中仅一个候选。恢复 root 仍需解析 attached sheet;不完整、歧义或外来 sheet 拒绝。与现有 focused-window 路径一样,精确已证明的 App 表面可以在全局 AX array 不完整时被选中,但后续 tree/facts 必须满足更严格完整性检查。此恢复不发输入、不激活/切 Space、不跨请求缓存 AX 对象。
|
||||||
|
|
||||||
|
### Chromium 观察 readiness(G24 / 上游 #3783)
|
||||||
|
|
||||||
|
现代 opt-in 接受后仍为 pending,直到观测 AXWebArea。默认 App AX 观察在选窗和遍历间共用 500ms deadline。录制、动作 facts 和纯截图仅 opt-in,不轮询或消耗等待次数。正证据立即返回;pending cooldown 30 秒,每进程生命周期至多三次等待。cache 以 PID+kernel start stamp 为键,lifetime 检查与发布序列化,旧完成不能覆盖新进程 readiness。
|
||||||
|
|
||||||
|
独立受控验证覆盖 16 个 readiness、旧/新发布竞态及两个真实 core 传播测试。现有 Chrome fixture 拒绝 opt-in,accepted-path 真机未验证。500ms 不是整个观察延迟界限;另有下面的 AppKit still-launching 恢复。
|
||||||
|
|
||||||
|
### G17 验收发现的原生命名键标志
|
||||||
|
|
||||||
|
global chord、desktop held-key 与两个 PID named-key builder 覆写 flags 以隔离显式修饰键,也误删特殊键内在 Function/NumericPad 位。独立原生 event construction 无投递复现,上游 global helper 共有。保留按键码结构位,仅加显式或 session-owned modifier;不复制可能混入物理状态的 factory flags。Fn 本身仍是持有修饰键,释放不自带 Fn。Unicode 与裸转发语义不变,路由、时序、单次派发不变;真实 Space 导航另需效果 oracle。
|
||||||
|
|
||||||
|
### AppKit 启动窗口查找(G24 / 上游 #4309)
|
||||||
|
|
||||||
|
独立 AppKit 恢复使用上游默认一秒原生 lookup allowance。absolute deadline 共享现有观察起点:从 readiness deadline 减 Chromium budget,再加 launch allowance;Chromium 仍仅 500ms。没有 readiness deadline 就不等待 launch,包括录制和动作 facts;共享起点有回归测试。
|
||||||
|
|
||||||
|
仅已知 still-launching 进程的 unresolved window 重试。默认 App 等自然解析,不再 reopen;精确 tree discovery 保留 scope/owner 校验。pause 至多 50ms,给上次 attempt 耗时留余量;丢弃 attempt 释放 AX 和不完整状态,最终 attempt 决定结果。tree 跨重试至多一次 G17 remote-token。它是协作 retry allowance,不硬约束原生调用/整体观察;已完成应用 reopen、node-walk 与外层 timeout 分开。
|
||||||
|
|
||||||
|
独立 27 个 actual-function 受控场景、49 个 G17 consumer 回归通过,发现并修复 sleep-overrun:睡眠后、主动 AX retry 前再查 deadline。默认 App wrapper 仍调用普通最终 body,可能再次解析 AX。后续原生 G24 夹具确认唯一公共请求开始时仍在启动,1.056 秒后获得正确 AX 标记,并独立确认启动完成。这验证该冷启动公共链,不能确定具体哪次 retry 生效、证明旧 400ms 路径失败,或给出硬一秒上界;首个 AX IPC 本身可能等待 ready。
|
||||||
|
|
||||||
|
### 录制 AX 捕获预算(G30 / 上游 #4215)
|
||||||
|
|
||||||
|
录制 callback 在 session lock 外执行,调用方固定两秒 backstop。macOS tree 另有一秒协作 node deadline,保留已完成前缀;普通观察保留各自预算。至多四个 worker、每 PID 一个;超时后直到 provider 真返回才释放 slot,迟到结果丢弃。worker 继承请求 runtime、authorization 和 trusted session evidence,不回退 primary。callback 注册签名不变。
|
||||||
|
|
||||||
|
捕获前后检查 generation/owner;仅第二次检查和提交 artifacts 持 session lock,stop/替换不能让旧捕获写入任一新会话。截图隐私与 video stop 边界不变。增加原生线程,不增加进程或公开 budget。
|
||||||
|
|
||||||
|
新 Windows recording thread 拥有 COM apartment:成功初始化后,录制专用 raw node binding 全部释放,再 uninitialize;初始化失败或 mode 冲突不释放 apartment,不发布录制节点到 action cache。其他 UIA 调用生命周期不变。
|
||||||
|
|
||||||
|
macOS hook 同时接管 walker 对 actionable 节点的额外 retain,复用未发布 snapshot owner 在 JSON 序列化后释放,不替换 action token。实际 walker/hook 加受控 AX、真实 CF count 独立测试复现旧版每次漏一份,修复后重复完整/不完整捕获零泄漏,旧 token 有效,七份 payload 除耗时一致。三项原生回归、Mac/SDK all-target 通过。这是所有权测试,不证明 GUI、序列化失败或实际 elapsed-budget 截断。
|
||||||
|
|
||||||
|
### 快照身份(G18 / 上游 #3616 和 #3873)
|
||||||
|
|
||||||
|
上游要求快照身份与原生 binding 一致。受控并发观察复现旧 App 短 ID 命中替换对象:token 解析和窗口 cache 查找原为两步。
|
||||||
|
|
||||||
|
现有 token registry 一次发布 window、index membership、token generation 和不可变自有 native payload;解析持有其 `Arc`。平台 action 使用 bound cache,包括 Windows kind/role/geometry 和 Linux identity/actions,不读最新可变窗口 cache。覆盖默认 App、raw 及交错,无需跨观察/动作持 GUI lock。token 格式、runtime 隔离、同窗口失效和 LRU 不变,不引入上游 SnapshotStore 或新公开 API。
|
||||||
|
|
||||||
|
发布影响三端 `get_window_state` 与元素 cache;消费者包括 click/double/right、键盘、setValue、scroll、secondary action、Mac selectText。Mac `ClickTool` 和独立 `invoke_app_click` 都用 bound view。录制 marker 保留准确 binding,不重新解析裸 index。pending capture 拥有原生引用,在失败/取消时释放;runtime/target 清理 registry 所有权,已解析 action 自有 payload 到结束。
|
||||||
|
|
||||||
|
退化 Mac AX-window 观察仅撤销调用 runtime 的旧 target,不发布 unresolved tree,不撤销别的 runtime。三端 `clear_target` 同时撤销当前 runtime token。watcher 只有证明窗口/进程已关闭才跨 runtime 清理。session dispatch guard 延后 end cleanup 至观察结束;取消释放未发布 payload,不在 cleanup 后发布。
|
||||||
|
|
||||||
|
验证在竞争快照前后暂停 publication/lookup,要求旧对象身份或 stale 拒绝,覆盖跨 session/raw/App、稀疏 Linux index、Windows UIA/MSAA metadata、引用计数和 screenshot-only。原生桌面验收另计。
|
||||||
|
|
||||||
|
### 空闲清理后的隐式会话恢复(G08 / 上游 #4283)
|
||||||
|
|
||||||
|
原 idle sweep 会让仍连接的 unnamed MCP/SDK 生命周期永久结束。tombstone 记录是否来自 owner implicit lifecycle 的 idle 回收;仅同 owner 在 cleanup 完成后、原 admission 边界重建,在 tombstone lock 下复核后删旧并插入下一 episode。命名、显式 end、transport exit、revoke 保持终止。daemon adapter 早期 mirror 允许此唯一 recoverable 状态并跟随 core revive。SDK/MCP 共用 admission。
|
||||||
|
|
||||||
|
重建不恢复旧资源。snapshot 带私有 publishing session 身份,仅清理该 owner 的 binding;App 短 ID、capture scope、grant、recording 已有 owner cleanup。测试未完成 cleanup、错误 owner、命名/显式/transport 终止、并发 admission、重建后旧 raw token,再重复真实 direct MCP idle-TTL 复现。
|
||||||
|
|
||||||
|
### 原生 Electron 后台滚动(G05)
|
||||||
|
|
||||||
|
Electron 忽略后台 wheel 时,上游 #4861/#5079 通过原生辅助功能 reveal。它是应用原生动作,不用上游 browser API。先复现候选拒绝并与上游比较;native before 确认候选五次拒绝、偏移不变,stock 0.35.4 四次单请求分别使 inner+114/page+404。
|
||||||
|
|
||||||
|
适配几何与 clamped-row 选择,保留合同:证明精确窗口/原生祖先,选最内层相关容器,至多一次 `AXScrollToVisible`。不复制上游固定 150ms effect wait 或不确定派发后换容器重试。返回派发 ACK,下一观察取证;preflight 不存在可拒绝,不确定不能回放。Browser SDK 不在变更内。
|
||||||
|
|
||||||
|
只读选择限 4,000 descendants、depth80、私有一秒协作搜索;安全选择失败则派发前拒绝。native proxy 与进程 mutation lease 在 blocking worker 中保留到最终 exact gate 和唯一 AX action,包括 async caller 取消。AX 错误后返回不确定 ACK/警告,不换路由;限制不硬截止挂起的原生读。
|
||||||
|
|
||||||
|
普通 v3 通过五格真实 Electron fixture:raw inner/page×AX/pixel,加默认 App inner;每格一次请求,独立 fixture IPC 读 inner+114 或 page+404,另一层不变,保持未确认 ACK。独立 exhausted-inner 复现 v2/v3 单次 down 却向上96px。普通 v4 重跑五主格,并通过 raw/App exhausted 拒绝,真实 `sent:false`、偏移不变。七格不证明全部方向、真人并发或更短采样间隔的干扰。v2 受控43项、v3四个几何单测与 v3 受控程序启动超时分开保留。
|
||||||
|
|
||||||
|
exhausted 诊断确认容器正确,但上方旧 row 被报为底边1px sliver,reveal 使方向反转。v4 用既有 visible-leaf 集合排除文档顺序错误侧候选:down/right 在最后可见 leaf 之后,up/left 在第一 leaf 之前。不新增原生读/动作,不改真实 frame 选择。没有可见锚点时保留旧 heuristic,重排/多列 AX 图仍需另验;这是限定修复,不是普遍方向保证。
|
||||||
|
|
||||||
|
普通 v7 以源树 `70001802535ebff45ed2f9bc8a0a55b0ba7edfb6f07d1c76aae1b5eb6a002693` 和 SDK SHA-256 `4d2b048131c42055991f1716b41691715da9abf3b7b176fc125548a12dff60fe` 独立重跑七格真实 GUI,全部通过,无回放或前台回退。独立 fixture IPC 确认同样 +114px/+404px 和耗尽边界偏移不变;143 个采样保持 A 前台/顶层、鼠标位置与剪贴板计数。634 项引用已逐字核验。8.444–12.213Hz 采样不证明不存在更短干扰或真人并发。
|
||||||
|
|
||||||
|
### macOS release 替换恢复(G36)
|
||||||
|
|
||||||
|
真实 release 分支在复制失败时丢旧 bundle;两个隔离 before 场景确认复制前或部分复制失败都留下坏 executable symlink。适配 #3297 的 staged validation/backup/rollback,保留现有 bundle 路径和签名身份。动旧应用前验证下载 bundle,copied bundle 和 executable 通过前保留 sibling backup;复制/验证失败或可处理信号恢复旧字节。恢复失败保留 backup 并报路径。process kill/断电不能运行 trap,不声称自动恢复。
|
||||||
|
|
||||||
|
以旧 designated signing requirement 验证 replacement,准确报告权限连续性。不移植自动 TCC reset,安装不改授权记录;Mac local-build cleanup 延后到 replacement 成功。保留 installer、daemon、bundle location、permission owner,不引入新 service/helper 架构。验证使用隔离目录和受控系统命令,不是真实应用/权限安装。
|
||||||
|
|
||||||
|
### 其他已验证上游移植
|
||||||
|
|
||||||
|
| 来源 | 适配和证据 |
|
||||||
|
| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| #4967 / #4214 | Linux 顶层 frame 重定位和本地 XRes PID fallback;真实 GTK before/after 修正错行输入及无 `_NET_WM_PID` 的窗口发现,限实测 Xorg 环境。 |
|
||||||
|
| #2907 / #4690 / #5137 | Mac 窗口点击选单一 native 路由,中键保留精确 window;mini 两轮 parent/sheet 去除重复事件。 |
|
||||||
|
| #3755 / #5125 | Mac 截图用绝对系统路径,窗口几何拒绝非有限数;窄 PATH 真截图和 CFNumber 回归通过。 |
|
||||||
|
| #5136 / #4333 / #3975 | Windows console 继承、GDI 失败清理、shortcut 环境路径解析;实际 MSVC 测试通过,交互桌面另待验。 |
|
||||||
|
| #3277 / #4985 | embedded 环境过滤保留显式 telemetry/update-check 偏好。 |
|
||||||
|
| #3462 / #4888 | typed schema 去 Schemars 专用 numeric format,声明用兼容 dialect,runtime 校验仍权威;声明测试/离线 MCP wire 与真实 provider 分开。 |
|
||||||
|
| #3632 | 录制仅 owner session/隐式 trusted connection 的动作,忽略 lifecycle;独立测试发现隐式 owner 漏洞,二次修复通过命名/隐式/外来连接/lifecycle 公共 registry 验证。 |
|
||||||
|
| #4238 | SCK stop 等 recording-output 完成 callback;mini baseline 复现误报空文件及返回后增长,新候选五次120ms录制均立即解码,一秒后 hash 不变。 |
|
||||||
|
| #3328 | display scale 用当前 mode backing pixels/points;同 Retina 显示器公开值由1改2,坐标派发另有合同。 |
|
||||||
|
| #3714 / #5144 / #5145 | encoder exit/timeout 明确报错;recording 时间用同一开始基线、action ISO-8601;公开 API 探测验证细节和 metadata。 |
|
||||||
|
|
||||||
|
此表记录已实现组,不声称每个上游提交均已收口。完整清单与原生缺口见 acceptance ledger;跨平台构建不代替真实效果/干扰。
|
||||||
|
|
||||||
|
## 验证边界
|
||||||
|
|
||||||
|
忽略的 `.qwen/e2e-tests/cua-upgrade-2026-10-artifacts/` 保存不可变 source manifest、构建日志、协议调用和 fixture 读回。[当前验收快照](cua-upgrade-acceptance-2026-10.zh-CN.md#验收口径澄清2026-10-20) 将早期受控/GNU/Xvfb 检查与后来的真实 Windows MSVC/交互桌面、Linux Xorg/uinput/Mutter、macOS 原生 fixture 分开。均不证明物理真人并发。用户澄清已观察到的 macOS 目标窗口/菜单短暂遮挡可接受;遮挡测量、恢复与功能正确性分别保留。模型成功率、token 与时长等效性仍须实际第二层证据。用户于 2026-10-20 将 OSWorld 安排在发布后,不再作为发布前门槛。
|
||||||
|
|
||||||
|
后续验收修复补充上游移植:精确 AppKit key-window records 修复已复现的 sheet Return 失败,内部含 NaN 坐标 mouse 事件,并未解决指针/窗口覆盖。Windows 隔离卸载归属检查覆盖全部 task action,进程/安装范围限选定根目录,已在真实 PowerShell 验证。有限 CLI 命令在有效遥测偏好为 false 时跳过观察 self-wrapper,保留既有偏好优先级和启用行为;该 v21 CLI 改动与未变的 SDK 后端分开验证。运行时/打包附录保留本地开发签名与 installer seam 边界,不证明官方签名/notarization 或正式发布。
|
||||||
|
|
||||||
|
### 验收驱动的输入边界修复
|
||||||
|
|
||||||
|
Linux v24 在文字前缀、首字符/命名键按下、释放及 flush 全链区分派发前拒绝与不确定送达。事件发送错误即使发生在首次请求按下也保留 sent 不确定性,输入前焦点检查仍可在派发前拒绝。MPX 仅在目标焦点准入后保留请求 pending。失败按下及焦点探测键各作有界 best-effort 释放清理,不把不确定性宣称为完整文字完成。真实定向测试和 fresh Xorg/Mutter 用例分开验证;暂停目标的不确定结果不自动重放,也不计完整送达。
|
||||||
|
|
||||||
|
Mac v25 处理同窗应用焦点 AXGroup 明确 AXFocused=false、实际文本后代持有焦点的形态,仅将此条件转入现有完整、唯一、有界的实际焦点搜索。未知焦点、跨窗焦点及其他角色保留原行为;PID/窗口检查和末尾焦点/窗口复核均不放宽。LibreOffice ASCII 插入已实际通过。原 Cmd-S/AppleEvents 保存仍 UNKNOWN;独立 v25 工具栏 Save 有真实 FODT 证据。可接受的目标窗口/菜单短暂遮挡仍保留测量。单独 AX settable/edited 标志不证明编辑生效或文件已保存。
|
||||||
|
|
||||||
|
### v26 全局指针读取目标的拒绝
|
||||||
|
|
||||||
|
真实 Calc 请求以 C5 为目标,却选中 B28。精确 LibreOffice VCL 源码用全局硬件指针计算鼠标回调坐标,而 PID 定向点击不移动硬件指针。借鉴上游 #4864 的安全原则,本 fork 通过现有 NSRunningApplication 边界识别正式 `org.libreoffice.script`,在输入前拒绝其定向像素点击、右键、双击及元素像素回退。旧 raw foreground PID 路径也拒绝,仅激活窗口不能解决坐标不兼容。已广告 AX 动作与安全的 AX 可编辑焦点/选区路径保留。这是限定应用身份的规则,不是上游通用映射库检测,不声明覆盖改名或无 bundle 的 VCL 目标。规范公开 Mac 投递仍固定后台。
|
||||||
|
|
||||||
|
v26 tree `9bb4ffe40e1801d750f5a71ae20ea354293fe8bff01f0191554a7ca1facaabe3` 相对 v25 仅改这三个 Mac 工具。独立新文档的公开契约回归返回 background_unavailable/sent:false,三次有界同窗 Name Box 读数均保持 A1。通过的是安全拒绝,不是后台 C5 选区成功;raw AXPress 真机等价仍另行判定。历史严格零遮挡判据未满足,但用户随后澄清已观察到的目标窗口/菜单短暂遮挡可接受。原生/源码/构建证据与保留的原 harness 断言失败见 v26 验收检查点。
|
||||||
|
|
||||||
|
### v27限定Writer可编辑焦点
|
||||||
|
|
||||||
|
官方 LibreOffice Writer 正文暴露无广告 action 的 AXTextArea,且 AXValue/AXFocused 可写。中间 v27 候选只允许此 bundle 的 textarea 进入默认 App 分支的可编辑焦点路径,保留两项属性门禁、精确归属、单次无修饰左键条件及保持原值的焦点读回。通用 textarea 不获得该捷径,raw 路径保持 v26 原样。mini 新实例的原生读取证明,仅增加准入仍无法建立焦点,尽管 AX setter 返回成功。
|
||||||
|
|
||||||
|
F31 在单次 AXFocused setter 前增加既有精确窗口的后台键盘焦点准备和既有只读选区查询。同版本 LibreOffice 的 caret getter 会初始化 cursor 缓存/context,这说明了该读取的目的。不写选区、不移动 caret、不替换正文、不回退像素。未查询 caret 的原 observer 在 mini 新实例的一次 App 点击前后读到同一种子正文 false→true,随后一次输入和一次工具栏 Save 得到完全一致的实时正文与已保存 XML。保守确认及 sent/unconfirmed 回执保留:先观察后续状态,不重放。准备失败前可能已发送原生焦点 records,不能据此宣称未派发。已否决的仅准备、写选区试验分别保留源码和真机证据。
|
||||||
|
|
||||||
|
准备过程保留既有键盘目标歧义门禁,因为精确窗口 records 前可能发送 PID 级 ReturnKeyFocus 通知。单独 AX 元素绑定不能证明在多个键盘目标间发送该通知安全。两个新 Writer 正例各有一个文档窗口,不通过删除安全门禁来启用或宣称任意多窗口焦点切换。
|
||||||
@@ -1,129 +1,81 @@
|
|||||||
# Computer Use
|
# Computer Use
|
||||||
|
|
||||||
Qwen Code includes a `computer-use` skill that teaches the model how to
|
Computer Use runs through the standalone Node REPL MCP server. Its first `js` call loads global `cua` and displays documentation for the current platform. Computer Use and Browser Use no longer require bundled skills.
|
||||||
operate desktop applications through two separately installed packages:
|
|
||||||
|
|
||||||
```text
|
```text
|
||||||
bundled computer-use skill
|
Qwen Code / any MCP client
|
||||||
-> @qwen-code/node-repl-mcp
|
-> @qwen-code/node-repl-mcp (js, js_reset)
|
||||||
-> @qwen-code/cua-sdk/computer-use
|
-> @qwen-code/cua-sdk/runtime
|
||||||
-> native cua-driver accessibility backend
|
-> native Qwen CUA Driver (desktop applications)
|
||||||
|
-> @qwen-code/browser-use (Qwen browser extension)
|
||||||
```
|
```
|
||||||
|
|
||||||
Qwen Code does not bundle the MCP server, SDK, or native driver. The skill
|
## Setup
|
||||||
installs the external packages automatically when they are missing.
|
|
||||||
|
|
||||||
> [!warning]
|
Node.js 22 or later and npm are required. Register the MCP server, then restart Qwen Code:
|
||||||
>
|
|
||||||
> Computer Use can read application UI and control mouse and keyboard input.
|
|
||||||
> Use it only in trusted environments and review MCP approvals carefully.
|
|
||||||
|
|
||||||
## Automatic setup
|
|
||||||
|
|
||||||
Node.js 22 or later and npm are required.
|
|
||||||
|
|
||||||
When first used, the skill runs these commands itself:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
qwen mcp add --scope user node-repl npx -y @qwen-code/node-repl-mcp@0.1.7
|
qwen mcp add --scope user node-repl npx -y @qwen-code/node-repl-mcp@0.26.0
|
||||||
npm install --no-save --package-lock=false @qwen-code/[email protected]
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Restart Qwen Code after the MCP server is first added. The skill then resumes
|
The server installs the matching SDK and browser SDK as dependencies. The native SDK payload is downloaded for the current platform and verified against the matching release's SHA-256 checksums. It is cached outside the npm package; no workspace SDK bootstrap command is needed. Version 0.26.0 is a local development target until its release is published.
|
||||||
the desktop task through `node_repl`.
|
|
||||||
|
|
||||||
The SDK installation leaves `package.json` and the lockfile unchanged, but it
|
## Application tasks
|
||||||
does write to the workspace's `node_modules`. Its postinstall downloads and
|
|
||||||
verifies the native payload for the current platform.
|
|
||||||
|
|
||||||
Removing the MCP configuration or workspace SDK installation disables the
|
|
||||||
execution path; there is no legacy fallback.
|
|
||||||
|
|
||||||
## Use
|
|
||||||
|
|
||||||
Ask Qwen Code to use `$computer-use` for the desktop task. After bootstrap, it
|
|
||||||
uses the app workflow on macOS:
|
|
||||||
|
|
||||||
1. binds the application with `computer.getApp(nameOrIdentifierOrPath)`;
|
|
||||||
2. reads `app.getState()` for compact accessibility text, followed by automatic
|
|
||||||
incremental updates;
|
|
||||||
3. performs one or more actions using the short element IDs in that text;
|
|
||||||
4. fetches the latest state before deciding what to do next; and
|
|
||||||
5. closes the SDK client and resets the REPL only when no other persistent
|
|
||||||
state is needed.
|
|
||||||
|
|
||||||
The driver is the only component that computes observation diffs. Model code
|
|
||||||
uses the typed SDK methods and does not dispatch arbitrary driver tool names.
|
|
||||||
The app handle tracks the current window and dialogs, keeps native element
|
|
||||||
identity internally, and delegates input to the native driver. Model code does
|
|
||||||
not choose foreground/background modes. Unconfirmed actions are not replayed.
|
|
||||||
`getState()` can open a discovered stopped app; actions never restart it.
|
|
||||||
Existing exact-window APIs remain available on Windows and Linux.
|
|
||||||
|
|
||||||
```js
|
```js
|
||||||
const app = await computer.getApp('Microsoft Excel');
|
var app = await cua.getApp('Microsoft Excel');
|
||||||
nodeRepl.write((await app.getState()).text);
|
// Use a short ID from the automatically displayed AX state.
|
||||||
// Use an element ID from the returned state.
|
|
||||||
await app.click(37);
|
await app.click(37);
|
||||||
await app.typeText('hello');
|
await app.typeText('hello');
|
||||||
nodeRepl.write((await app.getState()).text);
|
await app.getAXState();
|
||||||
```
|
```
|
||||||
|
|
||||||
Refresh state after opening or closing a dialog before reusing element IDs.
|
`getApp` displays full AX text without an image. `getAXState` displays subsequent diffs. `getScreenshot` captures only the current window; `getAXStateAndScreenshot` captures both. Observations return strings and display themselves; `{emit:false}` suppresses output. Request a screenshot before using `{x,y}` coordinates, which are pixels in that window's current screenshot.
|
||||||
Each App state refresh captures the current screenshot internally. The default
|
|
||||||
return keeps it hidden; request it explicitly with
|
|
||||||
`app.getState({ includeScreenshot: true })` when the model needs the image.
|
|
||||||
|
|
||||||
## Permissions
|
The driver resolves applications/windows, owns current short IDs and coordinates, and chooses input delivery. macOS uses accessibility and directed events. Windows and Linux try background delivery first and use foreground input only after a refusal that proves nothing was sent. Foreground input stops when physical user activity is detected; it restores the prior application only if the user has not switched elsewhere. A dispatch acknowledgement does not prove the UI changed: observe before retrying.
|
||||||
|
|
||||||
The Node REPL is an MCP server that executes model-authored JavaScript with
|
Linux supports X11, including X11 applications under XWayland. Native Wayland applications are unsupported. Pass DISPLAY, XAUTHORITY and the desktop accessibility session bus to the server.
|
||||||
ordinary Node.js authority. Its calls follow Qwen Code's normal
|
|
||||||
[MCP approval flow](./approval-mode.md). The SDK also enforces native
|
|
||||||
authorization.
|
|
||||||
|
|
||||||
On macOS, accessibility observation and input require Accessibility permission.
|
## Desktop and browser tasks
|
||||||
Screenshots additionally require Screen Recording permission. macOS may
|
|
||||||
attribute the grant to the terminal or IDE that launched Qwen Code. Windows and
|
|
||||||
Linux use their platform accessibility and input facilities.
|
|
||||||
|
|
||||||
## Use the computer in front of you from a remote session
|
Use the explicit desktop interface for full-screen tasks:
|
||||||
|
|
||||||
When Qwen Code runs on a headless machine (a dev box, a server), the skill can
|
```js
|
||||||
still drive the desktop you are sitting at: that computer lends its own
|
await cua.desktop.getScreenshot();
|
||||||
`node_repl` to one remote session. It works on macOS today.
|
await cua.desktop.click({ x: 300, y: 200 }, { key: 'shift' });
|
||||||
|
```
|
||||||
|
|
||||||
Set it up once on your computer:
|
Desktop coordinates are pixels in the latest primary-display screenshot. Desktop input controls the visible foreground desktop and can move the real pointer. For held keys, always use `try/finally` with `cua.desktop.keyDown(key)` and `keyUp(key)`. Held keys belong to the calling session and are released on graceful reset/end. While holding keys, use `keyDown/keyUp` for chords and desktop pointer actions without explicit modifiers; release them before text, ordinary shortcuts or application input. Abrupt OS process termination cannot guarantee cleanup.
|
||||||
|
|
||||||
|
```js
|
||||||
|
var browser = await cua.getBrowser('Chrome');
|
||||||
|
// Browser documentation and tabs are displayed on first selection.
|
||||||
|
var tab = await cua.getTab('TAB_ID', { browser: 'Chrome' });
|
||||||
|
await tab.playwright.domSnapshot();
|
||||||
|
```
|
||||||
|
|
||||||
|
`getBrowser`, `getTab`, and `createBrowserTab` return the existing repository browser-use SDK handles. Follow the selected browser's runtime documentation and connect the Qwen browser extension as before. The integration adds no browser windows API, DOM diff or new Windows support.
|
||||||
|
|
||||||
|
## Permissions and lifecycle
|
||||||
|
|
||||||
|
The server executes JavaScript with ordinary Node.js permissions and follows the client's MCP approval flow. On macOS, Accessibility is required for input/AX observation and Screen Recording for screenshots; Automation may be requested for document save/Finder rename. Grants belong to the host process's OS responsibility chain. On Windows, the normal installed daemon uses the existing UAC/logon-task path; applications it launches use a token derived from its own token at normal-user integrity. UIAccess is optional: explicitly setting `CUA_DRIVER_USE_UIACCESS=1` requires a trusted signed worker under Program Files and fails if unavailable. Default npm installation does not stage that worker or require its signature.
|
||||||
|
|
||||||
|
`js` completes in one call and defaults to a 30-second timeout. A timeout can leave external effects; observe before retrying. `js_reset` discards bindings and gracefully closes the owning native/browser session. After context compaction call `cua.rewriteDocumentation()` before continuing.
|
||||||
|
|
||||||
|
## Remote desktop relay
|
||||||
|
|
||||||
|
For an existing macOS desktop relay setup, install its matching server:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npx -y @qwen-code/node-repl-mcp@0.1.7 desktop-relay install
|
npx -y @qwen-code/node-repl-mcp@0.26.0 desktop-relay install
|
||||||
```
|
```
|
||||||
|
|
||||||
This installs `node_repl` and the SDK under `~/.qwen/desktop-relay` and registers
|
This uses the existing local launchd socket at `127.0.0.1:47821`. In a secure Web Shell, choose **Use this computer**, then **Connect this computer**. The desktop prompts for each connection. The remote daemon must enable `QWEN_SERVE_CLIENT_MCP_OVER_WS=1`.
|
||||||
a launchd socket on `127.0.0.1:47821`. Nothing runs in the background; launchd
|
|
||||||
starts a short-lived process only when something connects.
|
|
||||||
|
|
||||||
- **From the Web Shell.** The remote daemon must run with
|
## Direct MCP and troubleshooting
|
||||||
`QWEN_SERVE_CLIENT_MCP_OVER_WS=1`, and the Web Shell must be a secure page
|
|
||||||
(https, or `http://localhost` through an SSH tunnel). In a session, choose
|
|
||||||
**Use this computer** in the sidebar footer, then **Connect this computer**.
|
|
||||||
|
|
||||||
A dialog on your computer asks you to allow every connection. An allowed session
|
`qwen-cua-driver mcp` exposes the same 19 native Computer Use operations directly. App calls use `app`; desktop-capable calls use `scope:"desktop"` instead. `key_down/key_up` are desktop-only. Advanced native tools remain available through the host opt-in `CUA_DRIVER_EXTRA_TOOLS=1`.
|
||||||
can run code on your computer with your permissions and see and control its
|
|
||||||
screen, just like local Computer Use, until you disconnect it or it ends. macOS
|
|
||||||
asks to allow `node` under Accessibility and Screen Recording the first time.
|
|
||||||
|
|
||||||
## Troubleshooting
|
If the server is unavailable, check `qwen mcp list` and restart the client after adding it. If capture fails, check the current desktop session and OS permissions. If native download fails, check the exact release version/checksum asset or supply a locally built matching payload through `QWEN_CUA_SDK_NATIVE_DIR`. A locked desktop cannot provide GUI acceptance evidence.
|
||||||
|
|
||||||
- If `node_repl` is still unavailable after automatic setup, restart Qwen Code
|
See [MCP servers](./mcp.md), [Approval Mode](./approval-mode.md) and [Sandboxing](./sandbox.md).
|
||||||
and verify the server with `qwen mcp list`.
|
|
||||||
- If the SDK import still fails after automatic setup, confirm Qwen Code is
|
|
||||||
running from the workspace where the package was installed.
|
|
||||||
- After a timeout, cancellation, reset, or kernel crash, bootstrap the SDK
|
|
||||||
client again and request fresh state.
|
|
||||||
|
|
||||||
## See also
|
|
||||||
|
|
||||||
- [Skills](./skills.md)
|
|
||||||
- [MCP servers](./mcp.md)
|
|
||||||
- [Approval Mode](./approval-mode.md)
|
|
||||||
- [Sandboxing](./sandbox.md)
|
|
||||||
@@ -9,6 +9,7 @@
|
|||||||
"type": "module",
|
"type": "module",
|
||||||
"workspaces": [
|
"workspaces": [
|
||||||
"packages/*",
|
"packages/*",
|
||||||
|
"packages/cua-driver/typescript",
|
||||||
"packages/channels/base",
|
"packages/channels/base",
|
||||||
"packages/channels/telegram",
|
"packages/channels/telegram",
|
||||||
"packages/channels/weixin",
|
"packages/channels/weixin",
|
||||||
|
|||||||
@@ -0,0 +1,203 @@
|
|||||||
|
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright 2025 Google LLC
|
||||||
|
Copyright 2025 Qwen
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
@@ -31,8 +31,8 @@ const nodeBanner =
|
|||||||
// reads process.env/process.platform, so bind `process` through createRequire,
|
// reads process.env/process.platform, so bind `process` through createRequire,
|
||||||
// the one route the kernel leaves open. The load check at the bottom of this
|
// the one route the kernel leaves open. The load check at the bottom of this
|
||||||
// file runs in Node's main realm, where `process` is always a global, so it
|
// file runs in Node's main realm, where `process` is always a global, so it
|
||||||
// cannot detect this binding going missing; src/skill-runtime-kernel.test.ts
|
// cannot detect this binding going missing; src/runtime-kernel.test.ts
|
||||||
// runs the skill's first cell through the real kernel for that.
|
// imports the SDK through the real kernel for that.
|
||||||
const kernelBanner =
|
const kernelBanner =
|
||||||
"import { createRequire as __qwenCreateRequire } from 'node:module'; const process = __qwenCreateRequire(import.meta.url)('node:process');";
|
"import { createRequire as __qwenCreateRequire } from 'node:module'; const process = __qwenCreateRequire(import.meta.url)('node:process');";
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@qwen-code/browser-use",
|
"name": "@qwen-code/browser-use",
|
||||||
"private": true,
|
"version": "0.26.0",
|
||||||
"version": "0.25.3",
|
|
||||||
"description": "Browser Use SDK for the Qwen Code Node REPL",
|
"description": "Browser Use SDK for the Qwen Code Node REPL",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
@@ -26,7 +25,8 @@
|
|||||||
"dist/sdk/types.d.ts",
|
"dist/sdk/types.d.ts",
|
||||||
"dist/native-host.js",
|
"dist/native-host.js",
|
||||||
"dist/scripts/native-host-setup.js",
|
"dist/scripts/native-host-setup.js",
|
||||||
"NOTICE"
|
"NOTICE",
|
||||||
|
"LICENSE"
|
||||||
],
|
],
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=22"
|
"node": ">=22"
|
||||||
@@ -40,7 +40,8 @@
|
|||||||
"smoke:qwen-saucedemo": "npm run build --workspace=@qwen-code/node-repl-mcp && npm run build --workspace=@qwen-code/chrome-bridge && npm run build && npm --prefix ../.. run bundle && node dist/scripts/smoke-qwen-saucedemo.js",
|
"smoke:qwen-saucedemo": "npm run build --workspace=@qwen-code/node-repl-mcp && npm run build --workspace=@qwen-code/chrome-bridge && npm run build && npm --prefix ../.. run bundle && node dist/scripts/smoke-qwen-saucedemo.js",
|
||||||
"test:ci": "vitest run",
|
"test:ci": "vitest run",
|
||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
"test": "vitest run"
|
"test": "vitest run",
|
||||||
|
"prepack": "npm run build"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^22.0.0",
|
"@types/node": "^22.0.0",
|
||||||
@@ -51,5 +52,8 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"playwright-core": "1.62.1"
|
"playwright-core": "1.62.1"
|
||||||
|
},
|
||||||
|
"publishConfig": {
|
||||||
|
"access": "public"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,7 +5,17 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { readFileSync } from 'node:fs';
|
import {
|
||||||
|
mkdtempSync,
|
||||||
|
mkdirSync,
|
||||||
|
readFileSync,
|
||||||
|
rmSync,
|
||||||
|
writeFileSync,
|
||||||
|
} from 'node:fs';
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
import { runInNewContext } from 'node:vm';
|
import { runInNewContext } from 'node:vm';
|
||||||
import { describe, expect, it, vi } from 'vitest';
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
import { transformSync } from 'esbuild';
|
import { transformSync } from 'esbuild';
|
||||||
@@ -59,6 +69,87 @@ describe('preflight screenshot validation', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('smoke artifact preflight', () => {
|
||||||
|
it.each([false, true])(
|
||||||
|
'resolves import-only CUA exports before checking Browser SDK artifacts (missing host: %s)',
|
||||||
|
(missingHost) => {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), 'qwen-smoke-preflight-'));
|
||||||
|
const write = (name: string, text = '') => {
|
||||||
|
const target = join(root, name);
|
||||||
|
mkdirSync(dirname(target), { recursive: true });
|
||||||
|
writeFileSync(target, text);
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
write('dist/cli.js');
|
||||||
|
write('packages/node-repl/dist/index.js');
|
||||||
|
const sdk = 'packages/node-repl/node_modules/@qwen-code/cua-sdk';
|
||||||
|
write(
|
||||||
|
`${sdk}/package.json`,
|
||||||
|
JSON.stringify({
|
||||||
|
type: 'module',
|
||||||
|
exports: { './runtime': { import: './lib/runtime-entry.js' } },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
write(`${sdk}/lib/runtime-entry.js`);
|
||||||
|
const browser = `${sdk}/node_modules/@qwen-code/browser-use`;
|
||||||
|
write(
|
||||||
|
`${browser}/package.json`,
|
||||||
|
JSON.stringify({ exports: { './package.json': './package.json' } }),
|
||||||
|
);
|
||||||
|
write(`${browser}/dist/index.js`);
|
||||||
|
if (!missingHost) write(`${browser}/dist/native-host.js`);
|
||||||
|
write(`${browser}/node_modules/playwright-core/package.json`, '{}');
|
||||||
|
const smoke = readFileSync(
|
||||||
|
new URL('./smoke-qwen-saucedemo.ts', import.meta.url),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
const start = smoke.indexOf('const packageRoot =');
|
||||||
|
const end = smoke.indexOf("await withManagedChrome('sauce'");
|
||||||
|
assert.ok(start >= 0 && end > start);
|
||||||
|
const builtUrl = pathToFileURL(
|
||||||
|
join(
|
||||||
|
root,
|
||||||
|
'packages/browser-use/dist/scripts/smoke-qwen-saucedemo.js',
|
||||||
|
),
|
||||||
|
).href;
|
||||||
|
const code = transformSync(
|
||||||
|
smoke
|
||||||
|
.slice(start, end)
|
||||||
|
.replaceAll('import.meta.url', JSON.stringify(builtUrl)),
|
||||||
|
{ loader: 'ts', target: 'node22' },
|
||||||
|
).code;
|
||||||
|
const result = spawnSync(
|
||||||
|
process.execPath,
|
||||||
|
[
|
||||||
|
'--input-type=module',
|
||||||
|
'--eval',
|
||||||
|
`
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { access } from 'node:fs/promises';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
|
import { dirname, join, resolve } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
${code}
|
||||||
|
`,
|
||||||
|
],
|
||||||
|
{ cwd: tmpdir(), encoding: 'utf8', timeout: 20_000 },
|
||||||
|
);
|
||||||
|
expect(result.error).toBeUndefined();
|
||||||
|
if (missingHost) {
|
||||||
|
expect(result.status).not.toBe(0);
|
||||||
|
expect(result.stderr).toContain('ENOENT');
|
||||||
|
expect(result.stderr).toContain('native-host.js');
|
||||||
|
} else {
|
||||||
|
expect(result.status, result.stderr).toBe(0);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
describe('smoke observation claims', () => {
|
describe('smoke observation claims', () => {
|
||||||
const smoke = readFileSync(
|
const smoke = readFileSync(
|
||||||
new URL('./smoke-qwen-saucedemo.ts', import.meta.url),
|
new URL('./smoke-qwen-saucedemo.ts', import.meta.url),
|
||||||
@@ -66,33 +157,61 @@ describe('smoke observation claims', () => {
|
|||||||
);
|
);
|
||||||
const checks = smoke.match(/const checks = \{[\s\S]*?\n {4}\};/)?.[0];
|
const checks = smoke.match(/const checks = \{[\s\S]*?\n {4}\};/)?.[0];
|
||||||
assert.ok(checks, 'Smoke checks must be exercised');
|
assert.ok(checks, 'Smoke checks must be exercised');
|
||||||
const helperStart = smoke.indexOf('function stagedRuntimeImported(');
|
const helperStart = smoke.indexOf('function preloadedBrowserUsed(');
|
||||||
const helperEnd = smoke.indexOf('\n}\n', helperStart) + 3;
|
const helperEnd = smoke.indexOf('\n}\n', helperStart) + 3;
|
||||||
assert.ok(helperStart >= 0, 'stagedRuntimeImported must be exercised');
|
assert.ok(helperStart >= 0, 'preloadedBrowserUsed must be exercised');
|
||||||
const stagedRuntimeImported = runInNewContext(
|
const preloadedBrowserUsed = runInNewContext(
|
||||||
transformSync(smoke.slice(helperStart, helperEnd), {
|
transformSync(smoke.slice(helperStart, helperEnd), {
|
||||||
loader: 'ts',
|
loader: 'ts',
|
||||||
target: 'node22',
|
target: 'node22',
|
||||||
}).code + '\nstagedRuntimeImported;',
|
}).code + '\npreloadedBrowserUsed;',
|
||||||
{},
|
{},
|
||||||
) as (
|
) as (calls: Array<{ code: string; output: string }>) => boolean;
|
||||||
calls: Array<{ code: string; output: string }>,
|
|
||||||
browserRuntimeRoot: string,
|
const bootstrap = {
|
||||||
builtinSkillRoot: string,
|
code: 'var browser = await cua.getBrowser();',
|
||||||
) => boolean;
|
output: '# Qwen Browser Use\nBrowser documentation',
|
||||||
|
};
|
||||||
|
|
||||||
it.each([
|
it.each([
|
||||||
'// .type( .click( .finalize(',
|
['the preloaded browser bootstrap', [bootstrap], true],
|
||||||
'nodeRepl.write(".type( .click( .finalize(");',
|
['no calls', [], false],
|
||||||
])('does not report SDK setup from unrelated action text: %s', (code) => {
|
['a failed or unobserved bootstrap', [{ ...bootstrap, output: '' }], false],
|
||||||
|
[
|
||||||
|
'a comment',
|
||||||
|
[{ ...bootstrap, code: '// await cua.getBrowser();' }],
|
||||||
|
false,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'printed code',
|
||||||
|
[{ ...bootstrap, code: 'nodeRepl.write("cua.getBrowser()");' }],
|
||||||
|
false,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'a manually imported SDK',
|
||||||
|
[
|
||||||
|
{
|
||||||
|
...bootstrap,
|
||||||
|
code: "await (await import('/elsewhere/index.js')).setupBrowserRuntime();",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
false,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'setup preceding the bootstrap',
|
||||||
|
[{ code: 'var cua = replacement;', output: '' }, bootstrap],
|
||||||
|
false,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'a fallback import after bootstrap',
|
||||||
|
[bootstrap, { code: "await import('/elsewhere/index.js');", output: '' }],
|
||||||
|
false,
|
||||||
|
],
|
||||||
|
])('reports preloaded Browser Use for %s', (_label, calls, expected) => {
|
||||||
const result = runInNewContext(checks + '\nchecks;', {
|
const result = runInNewContext(checks + '\nchecks;', {
|
||||||
calls: [{ code, output: '' }],
|
calls,
|
||||||
joinedCode: code,
|
|
||||||
joinedOutput: 'Total: $36.69',
|
joinedOutput: 'Total: $36.69',
|
||||||
browserRuntimeEntry: '/skill/runtime/index.js',
|
preloadedBrowserUsed,
|
||||||
browserRuntimeRoot: '/skill/runtime',
|
|
||||||
builtinSkillRoot: '/skill',
|
|
||||||
stagedRuntimeImported,
|
|
||||||
events: [],
|
events: [],
|
||||||
moduleDirectoryRegistrationRequested: () => false,
|
moduleDirectoryRegistrationRequested: () => false,
|
||||||
completion: {
|
completion: {
|
||||||
@@ -102,73 +221,10 @@ describe('smoke observation claims', () => {
|
|||||||
cartMatchesLowestThree: true,
|
cartMatchesLowestThree: true,
|
||||||
finalAnswer: '$36.69',
|
finalAnswer: '$36.69',
|
||||||
}) as Record<string, boolean>;
|
}) as Record<string, boolean>;
|
||||||
|
expect(result.preloadedBrowserUsed).toBe(expected);
|
||||||
expect(result.browserSdkImported).toBe(false);
|
|
||||||
expect(result.checkoutCompletionObserved).toBe(true);
|
expect(result.checkoutCompletionObserved).toBe(true);
|
||||||
expect(result.reportedTotalIsExpected).toBe(true);
|
expect(result.reportedTotalIsExpected).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it.each([
|
|
||||||
[
|
|
||||||
'a literal entry path',
|
|
||||||
"await (await import('/skill/runtime/index.js')).setupBrowserRuntime();",
|
|
||||||
true,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'an entry built from the staged runtime directory',
|
|
||||||
"const SKILL_BASE = '/skill/runtime';\nawait (await import(SKILL_BASE + '/index.js')).setupBrowserRuntime();",
|
|
||||||
true,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'an entry built from the skill root',
|
|
||||||
"const base = '/skill';\nglobalThis.SKILL_BASE = base;\nawait (await import(SKILL_BASE + '/runtime/index.js')).setupBrowserRuntime();",
|
|
||||||
true,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'a runtime loaded from somewhere else',
|
|
||||||
"await (await import('/elsewhere/runtime/index.js')).setupBrowserRuntime();",
|
|
||||||
false,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'the skill root without setup',
|
|
||||||
"import('/skill/runtime/index.js');",
|
|
||||||
false,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'a staged import that threw and fell back to a foreign entry',
|
|
||||||
"let sdk; try { sdk = await import('/skill/runtime/index.js'); } catch { sdk = await import('/repo/packages/browser-use/dist/index.js'); } await sdk.setupBrowserRuntime();",
|
|
||||||
false,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'the setup export reached through an alias',
|
|
||||||
"const { setupBrowserRuntime: setup } = await import('/skill/runtime/index.js');\nawait setup();",
|
|
||||||
true,
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'the skill root echoed in prose next to a foreign import',
|
|
||||||
"// staged at /skill/runtime/index.js\nawait (await import('/elsewhere/index.js')).setupBrowserRuntime();",
|
|
||||||
false,
|
|
||||||
],
|
|
||||||
])('reports SDK setup for %s', (_label, code, expected) => {
|
|
||||||
const result = runInNewContext(checks + '\nchecks;', {
|
|
||||||
calls: [{ code, output: '' }],
|
|
||||||
joinedCode: code,
|
|
||||||
joinedOutput: 'Total: $36.69',
|
|
||||||
browserRuntimeEntry: '/skill/runtime/index.js',
|
|
||||||
browserRuntimeRoot: '/skill/runtime',
|
|
||||||
builtinSkillRoot: '/skill',
|
|
||||||
stagedRuntimeImported,
|
|
||||||
events: [],
|
|
||||||
moduleDirectoryRegistrationRequested: () => false,
|
|
||||||
completion: {
|
|
||||||
url: 'https://www.saucedemo.com/checkout-complete.html',
|
|
||||||
snapshot: 'Thank you for your order!',
|
|
||||||
},
|
|
||||||
cartMatchesLowestThree: true,
|
|
||||||
finalAnswer: '$36.69',
|
|
||||||
}) as Record<string, boolean>;
|
|
||||||
expect(result.browserSdkImported).toBe(expected);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('smoke price reads', () => {
|
describe('smoke price reads', () => {
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
* SPDX-License-Identifier: Apache-2.0
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { spawn } from 'node:child_process';
|
import { spawn, spawnSync } from 'node:child_process';
|
||||||
import {
|
import {
|
||||||
access,
|
access,
|
||||||
chmod,
|
chmod,
|
||||||
@@ -16,6 +16,7 @@ import {
|
|||||||
rm,
|
rm,
|
||||||
writeFile,
|
writeFile,
|
||||||
} from 'node:fs/promises';
|
} from 'node:fs/promises';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import { dirname, join, resolve } from 'node:path';
|
import { dirname, join, resolve } from 'node:path';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
@@ -32,7 +33,8 @@ import {
|
|||||||
|
|
||||||
const PROMPT = [
|
const PROMPT = [
|
||||||
'这是 Qwen Browser Use 的隔离 SauceDemo 冒烟测试。',
|
'这是 Qwen Browser Use 的隔离 SauceDemo 冒烟测试。',
|
||||||
'使用 browser-use skill 和标准 node_repl,',
|
'使用预加载的 cua 和 Node REPL 的 js 工具。',
|
||||||
|
'首次 js 调用只执行 var browser = await cua.getBrowser();,阅读返回文档后再继续;不要手动导入 SDK 或注册模块目录。',
|
||||||
'用 standard_user / secret_sauce 登录 https://www.saucedemo.com/,',
|
'用 standard_user / secret_sauce 登录 https://www.saucedemo.com/,',
|
||||||
'动态识别价格最低的三件商品并加入购物车,完成整个结账流程,',
|
'动态识别价格最低的三件商品并加入购物车,完成整个结账流程,',
|
||||||
'提交后验证完成页,最后告诉我含税总价。',
|
'提交后验证完成页,最后告诉我含税总价。',
|
||||||
@@ -52,33 +54,48 @@ interface CommandResult {
|
|||||||
durationMs: number;
|
durationMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const apiKey = process.env['DASHSCOPE_API_KEY'];
|
||||||
|
assert(apiKey, 'DASHSCOPE_API_KEY is required');
|
||||||
|
const baseUrl = process.env['DASHSCOPE_BASE_URL'];
|
||||||
|
assert(baseUrl, 'DASHSCOPE_BASE_URL is required');
|
||||||
|
const model = process.env['QWEN_BROWSER_USE_MODEL'];
|
||||||
|
assert(model, 'QWEN_BROWSER_USE_MODEL is required');
|
||||||
|
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
|
||||||
|
const repositoryRoot = resolve(packageRoot, '../..');
|
||||||
|
const qwenEntry = join(repositoryRoot, 'dist/cli.js');
|
||||||
|
const nodeReplEntry = join(repositoryRoot, 'packages/node-repl/dist/index.js');
|
||||||
|
// Resolve import-only exports from the MCP host, independent of the smoke cwd.
|
||||||
|
const cuaResolution = spawnSync(
|
||||||
|
process.execPath,
|
||||||
|
[
|
||||||
|
'--input-type=module',
|
||||||
|
'--eval',
|
||||||
|
"process.stdout.write(import.meta.resolve('@qwen-code/cua-sdk/runtime'))",
|
||||||
|
],
|
||||||
|
{ cwd: dirname(nodeReplEntry), encoding: 'utf8', timeout: 10_000 },
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
cuaResolution.status === 0,
|
||||||
|
'Could not resolve CUA runtime: ' +
|
||||||
|
(cuaResolution.error?.message ?? cuaResolution.stderr),
|
||||||
|
);
|
||||||
|
const cuaRuntimeEntry = fileURLToPath(cuaResolution.stdout);
|
||||||
|
const browserManifest = createRequire(cuaRuntimeEntry).resolve(
|
||||||
|
'@qwen-code/browser-use/package.json',
|
||||||
|
);
|
||||||
|
const browserPackageRoot = dirname(browserManifest);
|
||||||
|
await Promise.all(
|
||||||
|
[
|
||||||
|
qwenEntry,
|
||||||
|
nodeReplEntry,
|
||||||
|
cuaRuntimeEntry,
|
||||||
|
join(browserPackageRoot, 'dist/index.js'),
|
||||||
|
join(browserPackageRoot, 'dist/native-host.js'),
|
||||||
|
createRequire(browserManifest).resolve('playwright-core/package.json'),
|
||||||
|
].map((path) => access(path)),
|
||||||
|
);
|
||||||
|
|
||||||
await withManagedChrome('sauce', async (chrome) => {
|
await withManagedChrome('sauce', async (chrome) => {
|
||||||
const apiKey = process.env['DASHSCOPE_API_KEY'];
|
|
||||||
assert(apiKey, 'DASHSCOPE_API_KEY is required');
|
|
||||||
const baseUrl = process.env['DASHSCOPE_BASE_URL'];
|
|
||||||
assert(baseUrl, 'DASHSCOPE_BASE_URL is required');
|
|
||||||
const model = process.env['QWEN_BROWSER_USE_MODEL'];
|
|
||||||
assert(model, 'QWEN_BROWSER_USE_MODEL is required');
|
|
||||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
|
|
||||||
const repositoryRoot = resolve(packageRoot, '../..');
|
|
||||||
const qwenEntry = join(repositoryRoot, 'dist/cli.js');
|
|
||||||
const nodeReplEntry = join(
|
|
||||||
repositoryRoot,
|
|
||||||
'packages/node-repl/dist/index.js',
|
|
||||||
);
|
|
||||||
const builtinSkillRoot = join(repositoryRoot, 'dist/bundled/browser-use');
|
|
||||||
const browserRuntimeRoot = join(builtinSkillRoot, 'runtime');
|
|
||||||
const browserRuntimeEntry = join(browserRuntimeRoot, 'index.js');
|
|
||||||
await Promise.all(
|
|
||||||
[
|
|
||||||
qwenEntry,
|
|
||||||
nodeReplEntry,
|
|
||||||
join(builtinSkillRoot, 'SKILL.md'),
|
|
||||||
browserRuntimeEntry,
|
|
||||||
join(browserRuntimeRoot, 'native-host.js'),
|
|
||||||
join(browserRuntimeRoot, 'node_modules/playwright-core/package.json'),
|
|
||||||
].map((path) => access(path)),
|
|
||||||
);
|
|
||||||
const runId = new Date()
|
const runId = new Date()
|
||||||
.toISOString()
|
.toISOString()
|
||||||
.replaceAll(':', '')
|
.replaceAll(':', '')
|
||||||
@@ -158,7 +175,7 @@ await withManagedChrome('sauce', async (chrome) => {
|
|||||||
'--allowed-mcp-server-names',
|
'--allowed-mcp-server-names',
|
||||||
'node-repl',
|
'node-repl',
|
||||||
'--core-tools',
|
'--core-tools',
|
||||||
'skill,tool_search',
|
'tool_search',
|
||||||
'--approval-mode',
|
'--approval-mode',
|
||||||
'yolo',
|
'yolo',
|
||||||
'--max-wall-time',
|
'--max-wall-time',
|
||||||
@@ -230,11 +247,7 @@ await withManagedChrome('sauce', async (chrome) => {
|
|||||||
const joinedOutput = calls.map((call) => call.output).join('\n');
|
const joinedOutput = calls.map((call) => call.output).join('\n');
|
||||||
const checks = {
|
const checks = {
|
||||||
existingNodeReplUsed: calls.length > 0,
|
existingNodeReplUsed: calls.length > 0,
|
||||||
browserSdkImported: stagedRuntimeImported(
|
preloadedBrowserUsed: preloadedBrowserUsed(calls),
|
||||||
calls,
|
|
||||||
browserRuntimeRoot,
|
|
||||||
builtinSkillRoot,
|
|
||||||
),
|
|
||||||
noModuleDirectoryRegistered:
|
noModuleDirectoryRegistered:
|
||||||
!moduleDirectoryRegistrationRequested(events),
|
!moduleDirectoryRegistrationRequested(events),
|
||||||
checkoutCompletionObserved:
|
checkoutCompletionObserved:
|
||||||
@@ -364,46 +377,18 @@ function assertNoSecret(secret: string, ...values: string[]): void {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// A run proves it loaded the staged built-in runtime only through an import
|
// The first successful cell must acquire the preloaded browser and receive
|
||||||
// expression that names that runtime: a literal path under the staged
|
// its documentation. A manual import fallback does not exercise that entry.
|
||||||
// runtime directory, or a path the same or an earlier cell assembled from the
|
function preloadedBrowserUsed(calls: NodeReplCall[]): boolean {
|
||||||
// skill root the CLI substituted into SKILL.md. The cell must also reach the
|
const first = calls[0];
|
||||||
// setup export (a destructuring alias still names it). A literal entry loaded
|
return (
|
||||||
// from anywhere else — the fallback a model reaches for when the staged
|
first !== undefined &&
|
||||||
// import throws — disqualifies the run, because it is not the shipped
|
/^\s*(?:var|let|const)\s+\w+\s*=\s*await\s+cua\.getBrowser\(\)\s*;?\s*$/.test(
|
||||||
// runtime that was exercised.
|
first.code,
|
||||||
function stagedRuntimeImported(
|
) &&
|
||||||
calls: NodeReplCall[],
|
first.output.includes('# Qwen Browser Use') &&
|
||||||
browserRuntimeRoot: string,
|
calls.every((call) => !/\bimport\s*\(/.test(call.code))
|
||||||
builtinSkillRoot: string,
|
);
|
||||||
): boolean {
|
|
||||||
const entryFile = /\/(?:index|native-host)\.js/;
|
|
||||||
let rootBound = false;
|
|
||||||
let staged = false;
|
|
||||||
for (const call of calls) {
|
|
||||||
for (const root of [builtinSkillRoot, browserRuntimeRoot]) {
|
|
||||||
if (
|
|
||||||
call.code.includes(`'${root}'`) ||
|
|
||||||
call.code.includes(`"${root}"`) ||
|
|
||||||
call.code.includes(`\`${root}\``)
|
|
||||||
)
|
|
||||||
rootBound = true;
|
|
||||||
}
|
|
||||||
let importsStaged = false;
|
|
||||||
for (const match of call.code.matchAll(/import\(([^()]*)\)/g)) {
|
|
||||||
const expression = match[1] ?? '';
|
|
||||||
const literal = /^\s*(['"`])([^'"`$]+)\1\s*$/.exec(expression)?.[2];
|
|
||||||
if (literal !== undefined) {
|
|
||||||
if (literal.startsWith(`${browserRuntimeRoot}/`)) importsStaged = true;
|
|
||||||
else if (entryFile.test(literal)) return false;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (/index\.js/.test(expression) && rootBound) importsStaged = true;
|
|
||||||
}
|
|
||||||
if (importsStaged && call.code.includes('setupBrowserRuntime'))
|
|
||||||
staged = true;
|
|
||||||
}
|
|
||||||
return staged;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The prompt makes the model confirm cart.html with a separate tab.url() call
|
// The prompt makes the model confirm cart.html with a separate tab.url() call
|
||||||
|
|||||||
@@ -86,23 +86,25 @@ describe('smoke transcript tool uses', () => {
|
|||||||
it('collects REPL cells invoked directly and through the deferred-tool bridge', () => {
|
it('collects REPL cells invoked directly and through the deferred-tool bridge', () => {
|
||||||
expect(
|
expect(
|
||||||
collectSuccessfulNodeReplCalls([
|
collectSuccessfulNodeReplCalls([
|
||||||
toolUse('a', 'mcp__node-repl__node_repl', { code: 'direct' }),
|
toolUse('a', 'mcp__node-repl__js', { code: 'direct' }),
|
||||||
toolUse('b', 'tool_call', {
|
toolUse('b', 'tool_call', {
|
||||||
name: 'mcp__node-repl__node_repl',
|
name: 'mcp__node-repl__js',
|
||||||
arguments: { code: 'bridged' },
|
arguments: { code: 'bridged' },
|
||||||
}),
|
}),
|
||||||
toolUse('c', 'tool_call', {
|
toolUse('c', 'tool_call', {
|
||||||
name: 'mcp__node-repl__node_repl',
|
name: 'mcp__node-repl__js',
|
||||||
arguments: { code: 'failed' },
|
arguments: { code: 'failed' },
|
||||||
}),
|
}),
|
||||||
toolUse('d', 'tool_call', {
|
toolUse('d', 'tool_call', {
|
||||||
name: 'mcp__node-repl__node_repl_reset',
|
name: 'mcp__node-repl__js_reset',
|
||||||
arguments: { code: 'other tool' },
|
arguments: { code: 'other tool' },
|
||||||
}),
|
}),
|
||||||
toolResult('a', 'out-a'),
|
toolResult('a', 'out-a'),
|
||||||
toolResult('b', 'out-b'),
|
toolResult('b', 'out-b'),
|
||||||
toolResult('c', 'out-c', true),
|
toolResult('c', 'out-c', true),
|
||||||
toolResult('d', 'out-d'),
|
toolResult('d', 'out-d'),
|
||||||
|
toolUse('legacy', 'mcp__node-repl__node_repl', { code: 'retired' }),
|
||||||
|
toolResult('legacy', 'old-output'),
|
||||||
]),
|
]),
|
||||||
).toEqual([
|
).toEqual([
|
||||||
{ code: 'direct', output: 'out-a' },
|
{ code: 'direct', output: 'out-a' },
|
||||||
@@ -128,12 +130,12 @@ describe('smoke transcript tool uses', () => {
|
|||||||
],
|
],
|
||||||
[
|
[
|
||||||
'an ordinary REPL cell',
|
'an ordinary REPL cell',
|
||||||
toolUse('t', 'mcp__node-repl__node_repl', { code: '1' }),
|
toolUse('t', 'mcp__node-repl__js', { code: '1' }),
|
||||||
false,
|
false,
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
'a bridged call to another tool',
|
'a bridged call to another tool',
|
||||||
toolUse('t', 'tool_call', { name: 'mcp__node-repl__node_repl_reset' }),
|
toolUse('t', 'tool_call', { name: 'mcp__node-repl__js_reset' }),
|
||||||
false,
|
false,
|
||||||
],
|
],
|
||||||
])(
|
])(
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ export function collectSuccessfulNodeReplCalls(
|
|||||||
for (const { id, name, input } of toolUses(events)) {
|
for (const { id, name, input } of toolUses(events)) {
|
||||||
if (
|
if (
|
||||||
typeof id === 'string' &&
|
typeof id === 'string' &&
|
||||||
name === 'mcp__node-repl__node_repl' &&
|
name === 'mcp__node-repl__js' &&
|
||||||
isRecord(input) &&
|
isRecord(input) &&
|
||||||
typeof input.code === 'string'
|
typeof input.code === 'string'
|
||||||
) {
|
) {
|
||||||
@@ -91,7 +91,7 @@ export function collectSuccessfulNodeReplCalls(
|
|||||||
return successful;
|
return successful;
|
||||||
}
|
}
|
||||||
|
|
||||||
// The bundled SDK loads its own dependencies. A run that registered a module
|
// The preloaded SDK loads its own dependencies. A run that registered a module
|
||||||
// directory would pass even if that stopped working, so it does not count.
|
// directory would pass even if that stopped working, so it does not count.
|
||||||
export function moduleDirectoryRegistrationRequested(
|
export function moduleDirectoryRegistrationRequested(
|
||||||
events: unknown[],
|
events: unknown[],
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
import { mkdirSync, readFileSync } from 'node:fs';
|
||||||
|
import { dirname, join, resolve } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||||
|
const index = process.argv.indexOf('--output-dir');
|
||||||
|
if (index < 0 || !process.argv[index + 1])
|
||||||
|
throw new Error('missing --output-dir');
|
||||||
|
const output = resolve(process.argv[index + 1]);
|
||||||
|
mkdirSync(output, { recursive: true });
|
||||||
|
function run(args, capture = false) {
|
||||||
|
const result = spawnSync('npm', args, {
|
||||||
|
cwd: root,
|
||||||
|
encoding: 'utf8',
|
||||||
|
stdio: capture ? 'pipe' : 'inherit',
|
||||||
|
});
|
||||||
|
if (result.error) throw result.error;
|
||||||
|
if (result.status !== 0)
|
||||||
|
throw new Error(result.stderr || `npm exited ${result.status}`);
|
||||||
|
return result.stdout;
|
||||||
|
}
|
||||||
|
const manifest = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'));
|
||||||
|
const [packed] = JSON.parse(
|
||||||
|
run(['pack', '.', '--pack-destination', output, '--json'], true),
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
packed.name !== '@qwen-code/browser-use' ||
|
||||||
|
packed.version !== manifest.version
|
||||||
|
)
|
||||||
|
throw new Error('browser package identity mismatch');
|
||||||
|
const files = new Set(packed.files.map((file) => file.path));
|
||||||
|
for (const name of [
|
||||||
|
'LICENSE',
|
||||||
|
'NOTICE',
|
||||||
|
'dist/index.js',
|
||||||
|
'dist/index.d.ts',
|
||||||
|
'dist/core/primitives.d.ts',
|
||||||
|
'dist/sdk/types.d.ts',
|
||||||
|
'dist/native-host.js',
|
||||||
|
'dist/scripts/native-host-setup.js',
|
||||||
|
]) {
|
||||||
|
if (!files.has(name)) throw new Error(`browser package missing ${name}`);
|
||||||
|
}
|
||||||
|
run([
|
||||||
|
'publish',
|
||||||
|
join(output, packed.filename),
|
||||||
|
'--dry-run',
|
||||||
|
'--access',
|
||||||
|
'public',
|
||||||
|
'--registry=https://registry.npmjs.org',
|
||||||
|
]);
|
||||||
|
process.stdout.write(
|
||||||
|
JSON.stringify({
|
||||||
|
name: packed.name,
|
||||||
|
version: packed.version,
|
||||||
|
tarball: join(output, packed.filename),
|
||||||
|
integrity: packed.integrity,
|
||||||
|
}) + '\n',
|
||||||
|
);
|
||||||
+11
-28
@@ -11,18 +11,9 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
|
|||||||
import type { TestContext } from 'vitest';
|
import type { TestContext } from 'vitest';
|
||||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
// The bundled skill has the model import the staged runtime/index.js from
|
// Import the packaged browser runtime inside the real Node REPL kernel,
|
||||||
// inside the node_repl kernel's untrusted vm context, which has no `process`
|
// including its pinned Playwright dependency and its VM process binding.
|
||||||
// global. build.mjs's own load check imports dist/index.js in Node's main
|
|
||||||
// realm and therefore cannot see that difference, so this test runs the
|
|
||||||
// skill's first cell, verbatim from SKILL.md, through the real kernel built in
|
|
||||||
// packages/node-repl/dist against a runtime it stages itself, the way the
|
|
||||||
// bundle step and `npm run dev` do, into a temporary skill directory.
|
|
||||||
const repoRoot = fileURLToPath(new URL('../../../', import.meta.url));
|
const repoRoot = fileURLToPath(new URL('../../../', import.meta.url));
|
||||||
const skillFile = path.join(
|
|
||||||
repoRoot,
|
|
||||||
'packages/core/src/skills/bundled/browser-use/SKILL.md',
|
|
||||||
);
|
|
||||||
const browserUseDist = path.join(repoRoot, 'packages/browser-use/dist');
|
const browserUseDist = path.join(repoRoot, 'packages/browser-use/dist');
|
||||||
const nodeReplDist = path.join(repoRoot, 'packages/node-repl/dist');
|
const nodeReplDist = path.join(repoRoot, 'packages/node-repl/dist');
|
||||||
const requiredArtifacts = [
|
const requiredArtifacts = [
|
||||||
@@ -40,7 +31,7 @@ interface KernelOutcome {
|
|||||||
|
|
||||||
interface Kernel {
|
interface Kernel {
|
||||||
exec(request: { code: string; timeoutMs: number }): Promise<KernelOutcome>;
|
exec(request: { code: string; timeoutMs: number }): Promise<KernelOutcome>;
|
||||||
dispose(): void;
|
dispose(): Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface KernelManagerModule {
|
interface KernelManagerModule {
|
||||||
@@ -61,23 +52,15 @@ interface CopyAssetsModule {
|
|||||||
copyBrowserUseAssets(root: string, skillDir: string): void;
|
copyBrowserUseAssets(root: string, skillDir: string): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
function firstSkillCell(skillBase: string): string {
|
|
||||||
const skill = fs.readFileSync(skillFile, 'utf8');
|
|
||||||
const cell = /```js\n([\s\S]*?)```/.exec(skill)?.[1];
|
|
||||||
if (cell === undefined) throw new Error('SKILL.md has no ```js cell');
|
|
||||||
expect(cell).toContain("import('/absolute/skill/base/runtime/index.js')");
|
|
||||||
return cell.replaceAll('/absolute/skill/base', skillBase);
|
|
||||||
}
|
|
||||||
|
|
||||||
describe(
|
describe(
|
||||||
'bundled skill runtime inside the node_repl kernel',
|
'browser runtime inside the node_repl kernel',
|
||||||
{ timeout: 90_000 },
|
{ timeout: 90_000 },
|
||||||
() => {
|
() => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
vi.unstubAllEnvs();
|
vi.unstubAllEnvs();
|
||||||
});
|
});
|
||||||
|
|
||||||
async function runFirstSkillCell(
|
async function runBrowserRuntime(
|
||||||
context: TestContext,
|
context: TestContext,
|
||||||
prepareRuntime: (runtimeDir: string) => void = () => {},
|
prepareRuntime: (runtimeDir: string) => void = () => {},
|
||||||
): Promise<{ outcome: KernelOutcome; runtimeDir: string }> {
|
): Promise<{ outcome: KernelOutcome; runtimeDir: string }> {
|
||||||
@@ -129,12 +112,12 @@ describe(
|
|||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const outcome = await manager.exec({
|
const outcome = await manager.exec({
|
||||||
code: firstSkillCell(skillBase),
|
code: `var browserRuntime = await import(${JSON.stringify(pathToFileURL(path.join(runtimeDir, 'index.js')).href)}); var browserRuntimeAgent = await browserRuntime.setupBrowserRuntime(); await browserRuntimeAgent.browsers.list();`,
|
||||||
timeoutMs: 60_000,
|
timeoutMs: 60_000,
|
||||||
});
|
});
|
||||||
return { outcome, runtimeDir };
|
return { outcome, runtimeDir };
|
||||||
} finally {
|
} finally {
|
||||||
manager.dispose();
|
await manager.dispose();
|
||||||
fs.rmSync(tmpRootDir, { recursive: true, force: true });
|
fs.rmSync(tmpRootDir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -152,8 +135,8 @@ describe(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
it("runs SKILL.md's first cell up to the Chrome bridge without a ReferenceError", async (context) => {
|
it('loads the SDK up to the Chrome bridge without a ReferenceError', async (context) => {
|
||||||
const { outcome } = await runFirstSkillCell(context);
|
const { outcome } = await runBrowserRuntime(context);
|
||||||
|
|
||||||
// A `ReferenceError: process is not defined` here means the staged
|
// A `ReferenceError: process is not defined` here means the staged
|
||||||
// runtime lost the kernel-realm `process` binding that build.mjs adds
|
// runtime lost the kernel-realm `process` binding that build.mjs adds
|
||||||
@@ -167,7 +150,7 @@ describe(
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('reports an incomplete runtime when its bundled playwright-core is missing', async (context) => {
|
it('reports an incomplete runtime when its bundled playwright-core is missing', async (context) => {
|
||||||
const { outcome, runtimeDir } = await runFirstSkillCell(
|
const { outcome, runtimeDir } = await runBrowserRuntime(
|
||||||
context,
|
context,
|
||||||
(runtime) => {
|
(runtime) => {
|
||||||
fs.rmSync(path.join(runtime, 'node_modules/playwright-core'), {
|
fs.rmSync(path.join(runtime, 'node_modules/playwright-core'), {
|
||||||
@@ -187,7 +170,7 @@ describe(
|
|||||||
it('refuses a playwright-core other than the pinned version', async (context) => {
|
it('refuses a playwright-core other than the pinned version', async (context) => {
|
||||||
// Stands in for a missing bundled copy that Node's lookup replaced with
|
// Stands in for a missing bundled copy that Node's lookup replaced with
|
||||||
// an unrelated install further up the directory tree.
|
// an unrelated install further up the directory tree.
|
||||||
const { outcome, runtimeDir } = await runFirstSkillCell(
|
const { outcome, runtimeDir } = await runBrowserRuntime(
|
||||||
context,
|
context,
|
||||||
(runtime) => {
|
(runtime) => {
|
||||||
const manifestPath = path.join(
|
const manifestPath = path.join(
|
||||||
@@ -14,6 +14,8 @@ import {
|
|||||||
type AvailableSkillEntry,
|
type AvailableSkillEntry,
|
||||||
} from '../tools/skill-utils.js';
|
} from '../tools/skill-utils.js';
|
||||||
import { parseSkillContent } from './skill-load.js';
|
import { parseSkillContent } from './skill-load.js';
|
||||||
|
import { SkillManager } from './skill-manager.js';
|
||||||
|
import { makeFakeConfig } from '../test-utils/config.js';
|
||||||
|
|
||||||
// Bundled skills are loaded from disk at runtime by SkillManager. A typo in
|
// Bundled skills are loaded from disk at runtime by SkillManager. A typo in
|
||||||
// frontmatter (missing `description`, malformed YAML, broken `---` delimiter,
|
// frontmatter (missing `description`, malformed YAML, broken `---` delimiter,
|
||||||
@@ -30,7 +32,11 @@ const bundledDir = path.join(
|
|||||||
|
|
||||||
const skillNames = fs
|
const skillNames = fs
|
||||||
.readdirSync(bundledDir, { withFileTypes: true })
|
.readdirSync(bundledDir, { withFileTypes: true })
|
||||||
.filter((d) => d.isDirectory())
|
.filter(
|
||||||
|
(d) =>
|
||||||
|
d.isDirectory() &&
|
||||||
|
fs.existsSync(path.join(bundledDir, d.name, 'SKILL.md')),
|
||||||
|
)
|
||||||
.map((d) => d.name);
|
.map((d) => d.name);
|
||||||
|
|
||||||
describe('bundled SKILL.md files', () => {
|
describe('bundled SKILL.md files', () => {
|
||||||
@@ -38,6 +44,15 @@ describe('bundled SKILL.md files', () => {
|
|||||||
expect(skillNames.length).toBeGreaterThan(0);
|
expect(skillNames.length).toBeGreaterThan(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each(['browser-use', 'computer-use'])(
|
||||||
|
'keeps retired %s documentation out of the bundled skill catalog',
|
||||||
|
async (name) => {
|
||||||
|
expect(skillNames).not.toContain(name);
|
||||||
|
const manager = new SkillManager(makeFakeConfig());
|
||||||
|
await expect(manager.loadSkill(name, 'bundled')).resolves.toBeNull();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
it.each(skillNames)('%s/SKILL.md parses with required fields', (name) => {
|
it.each(skillNames)('%s/SKILL.md parses with required fields', (name) => {
|
||||||
const file = path.join(bundledDir, name, 'SKILL.md');
|
const file = path.join(bundledDir, name, 'SKILL.md');
|
||||||
const content = fs.readFileSync(file, 'utf8');
|
const content = fs.readFileSync(file, 'utf8');
|
||||||
|
|||||||
@@ -1,159 +0,0 @@
|
|||||||
---
|
|
||||||
name: browser-use
|
|
||||||
description: Control the user's Chrome through the existing persistent Node REPL and the Qwen Browser SDK.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Browser Use
|
|
||||||
|
|
||||||
Browser Use is bundled with Qwen Code and uses the generic Node REPL MCP tools
|
|
||||||
and the Qwen Chrome extension. Do not install a separate Qwen extension or
|
|
||||||
start a separate Browser Use MCP server.
|
|
||||||
|
|
||||||
## Setup
|
|
||||||
|
|
||||||
Install and enable the Qwen Code Chrome extension in the profile the user wants
|
|
||||||
to use. On macOS and Linux, first use of this SDK automatically registers the
|
|
||||||
shared Native Messaging host in the user's installation directory. Windows is
|
|
||||||
not supported: no Native Messaging host is ever registered there, so the
|
|
||||||
extension has nothing to connect back to. Calls that start a browser session
|
|
||||||
fail with `BROWSER_DISCONNECTED`, and `browsers.list()` waits out the connect
|
|
||||||
budget and returns an empty list: on Windows an empty list means the platform
|
|
||||||
is unsupported, not that no profile exists. If the user is on Windows, tell them
|
|
||||||
Browser Use needs macOS or Linux and stop. Do not retry, and do not send them to
|
|
||||||
the Chrome Web Store or to `chrome://extensions`; neither can fix this. A browser
|
|
||||||
task opts into this local setup, which can finish before the extension connects.
|
|
||||||
The SDK checks the actual connection and protocol instead of reading Chrome's
|
|
||||||
extension preferences.
|
|
||||||
|
|
||||||
If the extension does not connect on macOS or Linux, ask the user to open Chrome
|
|
||||||
and check the extension in the intended profile. It installs from the Chrome Web
|
|
||||||
Store:
|
|
||||||
https://chromewebstore.google.com/detail/qwen-code/hdhmmjclhibojdddmancfgbkleahfaph
|
|
||||||
|
|
||||||
If the store reports it is not available in the user's region, they can build
|
|
||||||
it from `packages/chrome-extension` in the Qwen Code repository (its
|
|
||||||
README) and load `dist/extension` through `chrome://extensions` (Developer mode
|
|
||||||
→ Load unpacked). After installation or enabling it, retry the connection.
|
|
||||||
|
|
||||||
A usable Host of the same protocol is reused. A Host installed by a newer Qwen
|
|
||||||
Code is never downgraded: if setup reports one, tell the user to update Qwen
|
|
||||||
Code and stop. The Host files persist after Qwen exits. The user can inspect or
|
|
||||||
remove them with `node <skill-base>/runtime/scripts/native-host-setup.js status`
|
|
||||||
or `uninstall`; `install` explicitly switches the Host to this Qwen Code's copy
|
|
||||||
from the next time Chrome starts it. A later Browser Use initialization can
|
|
||||||
register the Host again. If required Host files cannot be read or written,
|
|
||||||
report the failing action and path so the user can resolve access and retry.
|
|
||||||
|
|
||||||
Multiple Qwen sessions can use the same profile, each controlling its own tabs.
|
|
||||||
A tab held by another session reports `TAB_OWNERSHIP_CONFLICT`. Use another tab
|
|
||||||
or wait for that session to release it. The setup below binds the runtime to
|
|
||||||
the default profile through `browserAgent.browsers.get('chrome')`. Only when the
|
|
||||||
user names a specific Chrome profile, call `browserAgent.browsers.list()` before
|
|
||||||
that line and pass the returned ID to `browserAgent.browsers.get(id)` instead.
|
|
||||||
The runtime stays bound to the first profile it connects to; to switch profiles,
|
|
||||||
call `node_repl_reset` and run the setup again.
|
|
||||||
|
|
||||||
If `node_repl` is unavailable, configure it with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
qwen mcp add --scope user node-repl npx -y @qwen-code/[email protected]
|
|
||||||
```
|
|
||||||
|
|
||||||
Then tell the user to restart Qwen Code and stop. Do not start a separate
|
|
||||||
Browser Use MCP server. Screenshot metadata requires `@qwen-code/node-repl-mcp`
|
|
||||||
0.1.7 or later, so keep this exact pin.
|
|
||||||
|
|
||||||
Qwen reports the absolute `Base directory for this skill` when loading this
|
|
||||||
file. Use that directory as `<skill-base>`. The bundled SDK ships with its
|
|
||||||
dependencies: import it directly, without installing packages or registering
|
|
||||||
module directories, replacing the example skill base below with that absolute
|
|
||||||
path. If the import reports a missing file or an incomplete runtime, stop and
|
|
||||||
report an incomplete Browser Use runtime instead of installing dependencies
|
|
||||||
into the workspace:
|
|
||||||
|
|
||||||
```js
|
|
||||||
globalThis.browserAgent ??= await (
|
|
||||||
await import('/absolute/skill/base/runtime/index.js')
|
|
||||||
).setupBrowserRuntime();
|
|
||||||
globalThis.browser ??= await browserAgent.browsers.get('chrome');
|
|
||||||
nodeRepl.write(await browser.documentation());
|
|
||||||
```
|
|
||||||
|
|
||||||
Create a tab or claim an exact result returned by `openTabs()`:
|
|
||||||
|
|
||||||
```js
|
|
||||||
globalThis.tab = await browser.tabs.new();
|
|
||||||
await tab.goto('https://example.com/');
|
|
||||||
nodeRepl.write(await tab.playwright.domSnapshot());
|
|
||||||
```
|
|
||||||
|
|
||||||
```js
|
|
||||||
const candidates = await browser.user.openTabs();
|
|
||||||
nodeRepl.write(candidates);
|
|
||||||
globalThis.tab = await browser.user.claimTab(
|
|
||||||
candidates.find((candidate) => candidate.url === 'https://example.com/'),
|
|
||||||
);
|
|
||||||
```
|
|
||||||
|
|
||||||
## Interaction
|
|
||||||
|
|
||||||
Observe before acting and observe again after acting. Prefer semantic
|
|
||||||
Playwright locators. Use `tab.dom_cua.get_visible_dom()` and its `node_id`
|
|
||||||
values when snapshot refs are clearer, and use `tab.cua` for visual coordinate
|
|
||||||
targets.
|
|
||||||
|
|
||||||
```js
|
|
||||||
await tab.playwright.getByRole('button', { name: 'Continue' }).click();
|
|
||||||
await tab.playwright.getByLabel('Email').type('[email protected]');
|
|
||||||
await tab.dom_cua.click({ node_id: 'e4' });
|
|
||||||
await tab.dom_cua.type({ text: 'hello' });
|
|
||||||
```
|
|
||||||
|
|
||||||
When an action should navigate, arm the Playwright watcher around it:
|
|
||||||
|
|
||||||
```js
|
|
||||||
await tab.playwright.expectNavigation(
|
|
||||||
() => tab.playwright.getByRole('link', { name: 'Next' }).click(),
|
|
||||||
{ url: '**/next.html' },
|
|
||||||
);
|
|
||||||
```
|
|
||||||
|
|
||||||
Render screenshots with their metadata:
|
|
||||||
|
|
||||||
```js
|
|
||||||
await nodeRepl.emitImage(await tab.screenshot());
|
|
||||||
```
|
|
||||||
|
|
||||||
The metadata reaches the model only through `@qwen-code/node-repl-mcp` 0.1.6
|
|
||||||
or later; an older server accepts the call and silently drops it. If
|
|
||||||
`node_repl` was registered before this skill existed, check its command with
|
|
||||||
`qwen mcp list`, and when it pins an older version, re-register it with the
|
|
||||||
`qwen mcp add` command from Setup above and ask the user to restart Qwen Code.
|
|
||||||
|
|
||||||
Use `tab.dev.logs()` for bounded console diagnostics. Arm
|
|
||||||
`tab.playwright.waitForEvent('download' | 'filechooser')` before the action that
|
|
||||||
triggers it. Handle JavaScript dialogs through `tab.getJsDialog()`. Use a
|
|
||||||
focused `tab.playwright.evaluate()` only when locators cannot obtain the needed
|
|
||||||
data.
|
|
||||||
|
|
||||||
Use `browser.user.history()` only when the task needs browser history, with a
|
|
||||||
focused query and bounded result count.
|
|
||||||
|
|
||||||
## Finish
|
|
||||||
|
|
||||||
Treat `browser.tabs.finalize({ keep })` as the final browser action of the turn.
|
|
||||||
Omitted agent-created tabs close; omitted claimed tabs are released without
|
|
||||||
closing. `keep` is the complete set for this call. Keep a user-facing result as
|
|
||||||
`deliverable`, and keep a live page under control as `handoff`. Repeat a
|
|
||||||
handoff in each later turn that still needs it; omitting it from the next
|
|
||||||
finalize call or closing the runtime closes an agent-created tab.
|
|
||||||
|
|
||||||
```js
|
|
||||||
await browser.tabs.finalize({
|
|
||||||
keep: [{ tab, status: 'deliverable' }],
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
The Node kernel preserves `globalThis` and top-level bindings across cells.
|
|
||||||
Keep cells short, await every Browser SDK promise, and use `nodeRepl.write()` to
|
|
||||||
return text or structured data.
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
/**
|
|
||||||
* @license
|
|
||||||
* Copyright 2026 Qwen Team
|
|
||||||
* SPDX-License-Identifier: Apache-2.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import { fileURLToPath } from 'node:url';
|
|
||||||
import { runInNewContext } from 'node:vm';
|
|
||||||
import { describe, expect, it, vi } from 'vitest';
|
|
||||||
import { makeFakeConfig } from '../../../test-utils/config.js';
|
|
||||||
import { SkillManager } from '../../skill-manager.js';
|
|
||||||
|
|
||||||
const skillUrl = new URL('./SKILL.md', import.meta.url);
|
|
||||||
const skill = fs.readFileSync(skillUrl, 'utf8');
|
|
||||||
const nodeReplPackage = JSON.parse(
|
|
||||||
fs.readFileSync(
|
|
||||||
new URL('../../../../../node-repl/package.json', import.meta.url),
|
|
||||||
'utf8',
|
|
||||||
),
|
|
||||||
) as { version: string };
|
|
||||||
|
|
||||||
describe('bundled browser-use skill', () => {
|
|
||||||
it('is discovered without a Qwen extension', async () => {
|
|
||||||
const manager = new SkillManager(makeFakeConfig());
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
manager.loadSkill('browser-use', 'bundled'),
|
|
||||||
).resolves.toMatchObject({
|
|
||||||
name: 'browser-use',
|
|
||||||
level: 'bundled',
|
|
||||||
filePath: fileURLToPath(skillUrl),
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('loads its bundled runtime through the generic Node REPL', () => {
|
|
||||||
expect(skill).toContain('If `node_repl` is unavailable');
|
|
||||||
expect(skill).toContain('qwen mcp add --scope user node-repl');
|
|
||||||
expect(skill).toContain(
|
|
||||||
`@qwen-code/node-repl-mcp@${nodeReplPackage.version}`,
|
|
||||||
);
|
|
||||||
expect(skill).not.toContain('@qwen-code/node-repl-mcp@latest');
|
|
||||||
// The SDK loads its own bundled dependencies, so setup must not send the
|
|
||||||
// model through a module-directory registration first.
|
|
||||||
expect(skill).not.toContain('node_repl_add_node_module_dir');
|
|
||||||
expect(skill).not.toContain('node_modules');
|
|
||||||
expect(skill).toContain("import('/absolute/skill/base/runtime/index.js')");
|
|
||||||
expect(skill).not.toContain('<extension-root>');
|
|
||||||
expect(skill).not.toContain('qwen extensions install');
|
|
||||||
expect(skill).not.toContain('npm install --no-save');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('pins the Node REPL version that carries screenshot metadata', () => {
|
|
||||||
const pin = skill.match(
|
|
||||||
/^qwen mcp add --scope user node-repl npx -y @qwen-code\/node-repl-mcp@(\S+)$/m,
|
|
||||||
)?.[1];
|
|
||||||
expect(pin).toBe(nodeReplPackage.version);
|
|
||||||
expect(skill.replace(/\s+/g, ' ')).toContain(
|
|
||||||
`Screenshot metadata requires \`@qwen-code/node-repl-mcp\` ${nodeReplPackage.version} or later`,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('names where the Qwen Chrome extension comes from', () => {
|
|
||||||
expect(skill).toContain(
|
|
||||||
'https://chromewebstore.google.com/detail/qwen-code/hdhmmjclhibojdddmancfgbkleahfaph',
|
|
||||||
);
|
|
||||||
expect(skill).toContain('`packages/chrome-extension`');
|
|
||||||
expect(skill).toContain('`dist/extension`');
|
|
||||||
expect(skill).toContain('Load unpacked');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('tells the agent that Windows is unsupported', () => {
|
|
||||||
const prose = skill.replace(/\s+/g, ' ');
|
|
||||||
expect(prose).toContain(
|
|
||||||
'Windows is not supported: no Native Messaging host is ever registered there',
|
|
||||||
);
|
|
||||||
expect(prose).toContain(
|
|
||||||
'Calls that start a browser session fail with `BROWSER_DISCONNECTED`, and `browsers.list()` waits out the connect budget and returns an empty list',
|
|
||||||
);
|
|
||||||
expect(prose).toContain(
|
|
||||||
'tell them Browser Use needs macOS or Linux and stop',
|
|
||||||
);
|
|
||||||
expect(prose).toContain(
|
|
||||||
'Do not retry, and do not send them to the Chrome Web Store',
|
|
||||||
);
|
|
||||||
expect(prose).toContain(
|
|
||||||
'If the extension does not connect on macOS or Linux',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('uses the current Browser SDK contract', () => {
|
|
||||||
expect(skill).toContain('setupBrowserRuntime()');
|
|
||||||
expect(skill).toContain('nodeRepl.write');
|
|
||||||
expect(skill).toContain('browser.tabs.finalize');
|
|
||||||
expect(skill).toContain('complete set');
|
|
||||||
expect(skill).toContain('handoff in each later turn');
|
|
||||||
expect(skill).toContain('node_id');
|
|
||||||
expect(skill).not.toContain('setupBrowserRuntime(nodeRepl)');
|
|
||||||
expect(skill).not.toContain('dev.network');
|
|
||||||
expect(skill).not.toContain('markDeliverable');
|
|
||||||
expect(skill).not.toContain('markHandoff');
|
|
||||||
expect(skill).not.toContain('domSnapshot({ filter:');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('passes the complete screenshot and metadata to emitImage', async () => {
|
|
||||||
const example = [...skill.matchAll(/```js\n([\s\S]*?)```/g)].find(
|
|
||||||
([, code]) => code.includes('tab.screenshot()'),
|
|
||||||
)?.[1];
|
|
||||||
expect(example).toBeDefined();
|
|
||||||
|
|
||||||
const shot = {
|
|
||||||
bytes: new Uint8Array([1, 2, 3]),
|
|
||||||
mimeType: 'image/jpeg',
|
|
||||||
metadata: {
|
|
||||||
width: 1280,
|
|
||||||
height: 720,
|
|
||||||
viewport: { width: 1280, height: 720 },
|
|
||||||
devicePixelRatio: 2,
|
|
||||||
coordinateSpace: 'css-pixels',
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const screenshot = vi.fn().mockResolvedValue(shot);
|
|
||||||
const output: unknown[] = [];
|
|
||||||
|
|
||||||
await runInNewContext(`(async () => {\n${example}\n})()`, {
|
|
||||||
tab: { screenshot },
|
|
||||||
nodeRepl: {
|
|
||||||
write: (text: string) => output.push({ text }),
|
|
||||||
emitImage: async (image: unknown) => {
|
|
||||||
output.push({ image });
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(screenshot).toHaveBeenCalledTimes(1);
|
|
||||||
expect(output).toEqual([{ image: shot }]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,295 +0,0 @@
|
|||||||
---
|
|
||||||
name: computer-use
|
|
||||||
description: Control local desktop applications through Computer Use for tasks that require reading or operating app UI. Prefer purpose-built connectors, APIs, or CLIs when available.
|
|
||||||
---
|
|
||||||
|
|
||||||
## node_repl + @qwen-code/cua-sdk (Computer Use)
|
|
||||||
|
|
||||||
- Use `node_repl` (JavaScript) for all Computer Use actions.
|
|
||||||
- If both servers are available, use `node_repl` from the
|
|
||||||
`desktop-node-repl` MCP server; the regular server runs on the Qwen Code host.
|
|
||||||
- Do not use other technologies besides `node_repl` for computer interactions, unless specifically requested by the user (e.g. AppleScript, `osascript`, JXA, System Events, synthesized input).
|
|
||||||
- Prefer a dedicated plugin or skill when it can complete the task; use Computer Use for app interactions that are not exposed through a more specific interface.
|
|
||||||
- `node_repl` state is persistent across calls.
|
|
||||||
- For text output, use `nodeRepl.write(...)`. It takes a string; use `JSON.stringify(...)` only for textual metadata. For an observation, write its `.text` and emit its screenshots as images; do not stringify an observation or driver result containing image bytes.
|
|
||||||
- Omit `yield_time_ms` for ordinary UI calls to use the default 10-second wait. A shorter yield does not speed up the action and can add a `node_repl_wait` round. Use a shorter yield only when you need control back before completion; if a cell is still running, collect its result with `node_repl_wait` before issuing dependent actions.
|
|
||||||
|
|
||||||
## Forwarding results in Codex code mode
|
|
||||||
|
|
||||||
When calling `node_repl` through `tools.*` inside Codex's outer `functions.exec`,
|
|
||||||
forward each returned `content` block by its type. `nodeRepl.emitImage(...)`
|
|
||||||
produces an MCP image block; the outer script must pass that block to `image()`
|
|
||||||
for the model to receive an image. Prefer the desktop relay tool when it is
|
|
||||||
present; otherwise use the regular `node-repl` server:
|
|
||||||
|
|
||||||
```js
|
|
||||||
// A code-mode `tools` object throws on an unknown key, so probe with `in`:
|
|
||||||
// reading an unbound tool would abort the script before any fallback ran.
|
|
||||||
const DESKTOP_NODE_REPL = 'mcp__desktop_node_repl__node_repl';
|
|
||||||
const nodeReplTool =
|
|
||||||
DESKTOP_NODE_REPL in tools
|
|
||||||
? tools[DESKTOP_NODE_REPL]
|
|
||||||
: tools.mcp__node_repl__node_repl;
|
|
||||||
const result = await nodeReplTool({ code });
|
|
||||||
for (const block of result.content ?? []) {
|
|
||||||
if (block.type === 'text') {
|
|
||||||
text(block.text);
|
|
||||||
} else if (block.type === 'image') {
|
|
||||||
image(block);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Here `code` is the JavaScript to run in the persistent Node REPL. Keep the
|
|
||||||
forwarding loop in the outer code-mode script, outside that `code` string.
|
|
||||||
Apply the same loop to `node_repl_wait` results: images may arrive only when a
|
|
||||||
running cell completes. Forward text blocks too, including running-cell IDs and
|
|
||||||
errors. Direct MCP tool calls do not need this outer forwarding loop.
|
|
||||||
|
|
||||||
Do not use `text(result)`, `text(block)`, or `JSON.stringify(result)` to forward
|
|
||||||
an MCP result containing images: this turns image base64 into text, consuming
|
|
||||||
context without showing the image. `image()` accepts one image block, not the
|
|
||||||
whole result, so do not use `image(result)` either.
|
|
||||||
|
|
||||||
## Bootstrap
|
|
||||||
|
|
||||||
If `desktop-node-repl` is connected, skip the installation commands below: it
|
|
||||||
already provides `node_repl` and the SDK on the connected computer. Continue
|
|
||||||
with the `computer` initialization below. Otherwise, if `node_repl` is
|
|
||||||
unavailable, run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
qwen mcp add --scope user node-repl npx -y @qwen-code/[email protected]
|
|
||||||
npm install --no-save --package-lock=false @qwen-code/[email protected]
|
|
||||||
```
|
|
||||||
|
|
||||||
Tell the user to restart Qwen Code, then stop. If only the SDK import is missing,
|
|
||||||
run the second command and retry.
|
|
||||||
|
|
||||||
Reuse an existing `computer` connected to the intended desktop. Otherwise import
|
|
||||||
once per fresh `node_repl` session. The same App workflow below applies to macOS,
|
|
||||||
Linux and Windows; no additional platform resource is needed:
|
|
||||||
|
|
||||||
```js
|
|
||||||
globalThis.computer = await (
|
|
||||||
await import('@qwen-code/cua-sdk/computer-use')
|
|
||||||
).ComputerUse.create();
|
|
||||||
var platform = await computer.getPlatform();
|
|
||||||
nodeRepl.write(`Connected platform: ${platform}`);
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the connected platform for shortcuts, not the CLI or Node host operating system.
|
|
||||||
When the task identifies an app, combine initialization with `computer.getApp()`
|
|
||||||
and its first `getState()` in the same call. Read that state before editing or input.
|
|
||||||
|
|
||||||
## API surface
|
|
||||||
|
|
||||||
```ts
|
|
||||||
type Point = number | { x: number; y: number };
|
|
||||||
type ComputerUse = {
|
|
||||||
getPlatform: () => Promise<'macos' | 'linux' | 'windows'>;
|
|
||||||
getApp: (nameOrIdentifierOrPath: string) => Promise<App>;
|
|
||||||
listApps: () => Promise<
|
|
||||||
Array<{ id: string; displayName: string; isRunning: boolean }>
|
|
||||||
>;
|
|
||||||
close: () => Promise<void>;
|
|
||||||
};
|
|
||||||
type App = {
|
|
||||||
getState: (options?: {
|
|
||||||
disableDiff?: boolean;
|
|
||||||
includeScreenshot?: boolean;
|
|
||||||
maxTextChars?: number;
|
|
||||||
}) => Promise<State>;
|
|
||||||
click: (
|
|
||||||
point: Point,
|
|
||||||
options?: { button?: 'left' | 'right' | 'middle'; count?: number },
|
|
||||||
) => Promise<object>;
|
|
||||||
doubleClick: (point: Point) => Promise<object>;
|
|
||||||
rightClick: (
|
|
||||||
point: Point,
|
|
||||||
options?: { modifier?: string[] },
|
|
||||||
) => Promise<object>;
|
|
||||||
setValue: (element: number, value: string) => Promise<object>;
|
|
||||||
performSecondaryAction: (element: number, action: string) => Promise<object>;
|
|
||||||
typeText: (text: string) => Promise<object>;
|
|
||||||
paste: (
|
|
||||||
text: string,
|
|
||||||
options?: { format?: 'text' | 'md' | 'html'; signal?: AbortSignal },
|
|
||||||
) => Promise<object>;
|
|
||||||
selectText: (
|
|
||||||
element: number,
|
|
||||||
text: string,
|
|
||||||
options?: {
|
|
||||||
prefix?: string;
|
|
||||||
suffix?: string;
|
|
||||||
selection?: 'text' | 'cursor_before' | 'cursor_after';
|
|
||||||
signal?: AbortSignal;
|
|
||||||
},
|
|
||||||
) => Promise<object>;
|
|
||||||
pressKey: (
|
|
||||||
key: string,
|
|
||||||
options?: { modifiers?: string[] },
|
|
||||||
) => Promise<object>;
|
|
||||||
hotkey: (keys: string[]) => Promise<object>;
|
|
||||||
scroll: (
|
|
||||||
point: Point,
|
|
||||||
options: { direction: 'up' | 'down' | 'left' | 'right'; amount?: number },
|
|
||||||
) => Promise<object>;
|
|
||||||
drag: (options: {
|
|
||||||
fromX: number;
|
|
||||||
fromY: number;
|
|
||||||
toX: number;
|
|
||||||
toY: number;
|
|
||||||
}) => Promise<object>;
|
|
||||||
};
|
|
||||||
type State = {
|
|
||||||
app: string;
|
|
||||||
window: string;
|
|
||||||
mode: 'full' | 'diff' | 'no_change';
|
|
||||||
text: string;
|
|
||||||
screenshot?: { images: Array<{ mimeType: string; dataBase64: string }> };
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
## Workflow
|
|
||||||
|
|
||||||
`paste` and `selectText` currently require macOS. On Linux/Windows, use the shared
|
|
||||||
`typeText`, `setValue`, `pressKey` and observed actions. Unsupported text methods
|
|
||||||
fail with `unsupported_platform`; do not retry them as window failures.
|
|
||||||
|
|
||||||
### 1. Initialize
|
|
||||||
|
|
||||||
If initialization already bound the task's app and returned its state, reuse
|
|
||||||
that `app` and observation. Otherwise, bind the app named by the task, then read its state.
|
|
||||||
`getApp()` binds identity; `getState()` can open a discovered stopped app. Combine these
|
|
||||||
steps in one Node REPL call:
|
|
||||||
|
|
||||||
```js
|
|
||||||
var app = await computer.getApp('Microsoft Excel');
|
|
||||||
nodeRepl.write((await app.getState()).text);
|
|
||||||
```
|
|
||||||
|
|
||||||
The app handle tracks its current window and dialog. Read the returned window
|
|
||||||
title to confirm the intended document. If the app is unknown or its name is
|
|
||||||
ambiguous, discover applications with `computer.listApps()` and use a matching
|
|
||||||
application `id` when it distinguishes the app. Two running instances may have
|
|
||||||
the same ID; retrying that ID cannot resolve the ambiguity. Ask the user to keep
|
|
||||||
only the intended instance open rather than guessing a target or retrying it.
|
|
||||||
|
|
||||||
AX text uses short numeric IDs, such as `[37] TextField "Name"`. Use IDs from
|
|
||||||
the current observation for element actions. IDs can change when the app's
|
|
||||||
window or session changes. Disabled and static-text rows are observation-only.
|
|
||||||
|
|
||||||
For token efficiency, the accessibility tree will be returned
|
|
||||||
as a diff when appropriate. Prefer this default diff output. A full state
|
|
||||||
replaces the previous state; a diff updates it; no-change preserves it. If you
|
|
||||||
need a full replacement, use `disableDiff: true` only when the previous state
|
|
||||||
is unavailable or no longer useful. Do not disregard the text and then assume
|
|
||||||
that a subsequent diff will reproduce the information you skipped.
|
|
||||||
|
|
||||||
Returned text defaults to at most 12,000 characters. Set `maxTextChars` (minimum 512) to adjust the limit. A truncation notice means some captured rows were
|
|
||||||
omitted; request `app.getState({ disableDiff: true, maxTextChars: 24000 })` when
|
|
||||||
you need more full text. An omitted row does not prove an element is absent.
|
|
||||||
Traversal-limited captures cover only the captured nodes: identical captures
|
|
||||||
can return no-change, while changes return full captured state. Use current
|
|
||||||
captured IDs; after a read failure, use only IDs from the latest observation.
|
|
||||||
|
|
||||||
### 2. Actions using app
|
|
||||||
|
|
||||||
After performing one or more UI actions, call `app.getState()` before deciding
|
|
||||||
what to do next. Batch actions whose target remains the same, then print only
|
|
||||||
the state needed for the next decision:
|
|
||||||
|
|
||||||
```js
|
|
||||||
await app.click(37);
|
|
||||||
await app.hotkey([platform === 'macos' ? 'super' : 'ctrl', 'a']);
|
|
||||||
await app.typeText('hello');
|
|
||||||
await app.pressKey('Return');
|
|
||||||
nodeRepl.write((await app.getState()).text);
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the actual ID from your observation; `37` is only an example.
|
|
||||||
|
|
||||||
An observation is a decision boundary. When the current state already identifies
|
|
||||||
the controls and the next actions are known, combine those actions and saving
|
|
||||||
in the same call. Read state after the batch. End the batch at a new dialog,
|
|
||||||
menu, changed target or uncertain result; use that state before choosing the
|
|
||||||
next action. Do not split a known sequence merely to put each action in its own
|
|
||||||
call.
|
|
||||||
|
|
||||||
- Prefer element IDs to coordinates. `setValue(id, value)` changes a writable control, and `performSecondaryAction(id, action)` invokes a secondary action listed for that element. Use an observed action name rather than guessing.
|
|
||||||
- When an action opens or closes a dialog, sheet or menu, end the batch and call `app.getState()` to read the new window and IDs before continuing.
|
|
||||||
- `No open application window.` means the app is still running without a document window. If closing it completed the task, finish instead of retrying actions; otherwise open the intended file or window first.
|
|
||||||
- An action error can occur after the UI already changed. Read state before deciding whether to retry. Partial, unconfirmed or cancelled actions must not be blindly repeated.
|
|
||||||
- Coordinate actions use pixels in this app's current screenshot, with `(0, 0)` at its top-left. Every App observation refreshes that frame internally. Request `includeScreenshot: true` when you need to inspect the image, especially after a window change. Do not infer coordinates from another window or desktop screenshot.
|
|
||||||
- `pressKey` sends one key, optionally with modifiers. `hotkey` sends a combination such as `['super', 's']`. Use the connected platform's appropriate shortcut: macOS generally uses `super`; Linux/Windows generally use `ctrl`.
|
|
||||||
- App input manages any required activation internally and restores the previous focus, unless the user has moved it elsewhere. If the platform cannot confirm the target or restore focus, the action reports an error. There is no delivery-mode choice and no automatic replay after an uncertain result.
|
|
||||||
- On Linux, some compositor sessions cannot confirm an exact App target. An `app_window_unavailable` result can mean the session does not support this workflow; ask the user to use a supported desktop session instead of retrying input.
|
|
||||||
- Literal `\n` or `\r` in `typeText` sends Return. In a composer or form this may submit rather than insert a newline.
|
|
||||||
- If AX is incomplete or does not explain the interface, request a screenshot and inspect it. Only currently captured actionable IDs can be used for element actions.
|
|
||||||
|
|
||||||
## Paste and select text
|
|
||||||
|
|
||||||
These App methods are available on macOS only. Read current state first and use
|
|
||||||
an observed element ID for selection.
|
|
||||||
|
|
||||||
```js
|
|
||||||
await app.selectText(37, 'draft', { prefix: 'Status: ' });
|
|
||||||
nodeRepl.write((await app.getState()).text);
|
|
||||||
```
|
|
||||||
|
|
||||||
Use a real ID and text from your observation; the example assumes that the
|
|
||||||
selected element contains exactly one matching `draft` immediately after
|
|
||||||
`Status: `.
|
|
||||||
|
|
||||||
After confirming the selection, replace it with plain text:
|
|
||||||
|
|
||||||
```js
|
|
||||||
await app.paste('ready');
|
|
||||||
nodeRepl.write((await app.getState()).text);
|
|
||||||
```
|
|
||||||
|
|
||||||
- `selectText(element, text)` selects one exact, case-sensitive match. Optional
|
|
||||||
`prefix` and `suffix` must be immediately adjacent to that match. Missing or
|
|
||||||
ambiguous matches fail; provide enough observed context to identify one.
|
|
||||||
- `selection` defaults to `'text'`. Use `'cursor_before'` or `'cursor_after'` to
|
|
||||||
place the insertion point at a match boundary instead of selecting the text.
|
|
||||||
The element must support a writable text selection; static labels may not.
|
|
||||||
- `paste(text)` defaults to plain text. Use `{ format: 'md' }` for Markdown or
|
|
||||||
`{ format: 'html' }` for HTML; the receiving app determines which supplied
|
|
||||||
format it accepts. Use plain text when markup should remain literal.
|
|
||||||
- Paste uses the clipboard temporarily. It restores the previous clipboard only
|
|
||||||
while it still owns that transaction; a newer external clipboard change is
|
|
||||||
preserved.
|
|
||||||
- Both methods accept an optional `signal`. A completed dispatch, error or
|
|
||||||
cancellation does not by itself prove what changed. Observe state before
|
|
||||||
deciding whether to retry; do not blindly repeat unconfirmed text operations.
|
|
||||||
|
|
||||||
## Reading screenshots
|
|
||||||
|
|
||||||
Every App observation captures the current screenshot internally, independent
|
|
||||||
of whether AX returns full state, a diff or no-change. The default return omits
|
|
||||||
the image. `includeScreenshot: true` exposes it when visual inspection is needed.
|
|
||||||
Prefer the text-only default when it identifies the controls and confirms the
|
|
||||||
requested change. Request an image to resolve missing or ambiguous information,
|
|
||||||
choose coordinates or verify an appearance that AX does not describe.
|
|
||||||
|
|
||||||
```js
|
|
||||||
var state = await app.getState({ includeScreenshot: true });
|
|
||||||
nodeRepl.write(state.text);
|
|
||||||
for (const image of state.screenshot?.images ?? []) {
|
|
||||||
await nodeRepl.emitImage(`data:${image.mimeType};base64,${image.dataBase64}`);
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Include connection cleanup at the end of the call that emits the final
|
|
||||||
verification. Inspect that result before reporting success; reconnect and
|
|
||||||
continue if it reveals unfinished work. A separate cleanup-only model turn is
|
|
||||||
unnecessary:
|
|
||||||
|
|
||||||
```js
|
|
||||||
await computer.close();
|
|
||||||
globalThis.computer = undefined;
|
|
||||||
```
|
|
||||||
|
|
||||||
Reset the Node REPL only when no other persistent state is needed.
|
|
||||||
@@ -1,143 +0,0 @@
|
|||||||
/**
|
|
||||||
* @license
|
|
||||||
* Copyright 2026 Qwen
|
|
||||||
* SPDX-License-Identifier: Apache-2.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
import * as fs from 'node:fs';
|
|
||||||
import * as path from 'node:path';
|
|
||||||
import { fileURLToPath } from 'node:url';
|
|
||||||
import { runInNewContext } from 'node:vm';
|
|
||||||
import { describe, expect, it, vi } from 'vitest';
|
|
||||||
import { parseSkillContent } from '../../skill-load.js';
|
|
||||||
|
|
||||||
function loadComputerUseSkill() {
|
|
||||||
const skillPath = path.join(
|
|
||||||
path.dirname(fileURLToPath(import.meta.url)),
|
|
||||||
'SKILL.md',
|
|
||||||
);
|
|
||||||
const config = parseSkillContent(
|
|
||||||
fs.readFileSync(skillPath, 'utf-8'),
|
|
||||||
skillPath,
|
|
||||||
);
|
|
||||||
return { config, body: config.body };
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('bundled computer-use skill', () => {
|
|
||||||
it.each([true, false])(
|
|
||||||
'runs the forwarding example with desktop relay available=%s',
|
|
||||||
async (desktopAvailable) => {
|
|
||||||
const { body } = loadComputerUseSkill();
|
|
||||||
const example = body.match(/```js\n([\s\S]*?)\n```/)?.[1];
|
|
||||||
expect(example).toBeDefined();
|
|
||||||
const screenshot = {
|
|
||||||
type: 'image',
|
|
||||||
data: 'screenshot',
|
|
||||||
mimeType: 'image/png',
|
|
||||||
};
|
|
||||||
const result = {
|
|
||||||
content: [{ type: 'text', text: 'macos' }, screenshot],
|
|
||||||
};
|
|
||||||
const desktop = vi.fn().mockResolvedValue(result);
|
|
||||||
const regular = vi.fn().mockResolvedValue(result);
|
|
||||||
const text = vi.fn();
|
|
||||||
const image = vi.fn();
|
|
||||||
// Mirror the code-mode host (host.ts): a Proxy that throws on an
|
|
||||||
// unknown key, so an unbound desktop tool must be probed with `in`.
|
|
||||||
const toolTarget = Object.assign(Object.create(null), {
|
|
||||||
mcp__node_repl__node_repl: regular,
|
|
||||||
...(desktopAvailable
|
|
||||||
? { mcp__desktop_node_repl__node_repl: desktop }
|
|
||||||
: {}),
|
|
||||||
}) as Record<string, unknown>;
|
|
||||||
const tools = new Proxy(toolTarget, {
|
|
||||||
get(target, property) {
|
|
||||||
if (typeof property === 'string' && !(property in target)) {
|
|
||||||
throw new Error(
|
|
||||||
`Unknown or unavailable code mode tool: ${property}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return Reflect.get(target, property) as unknown;
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await runInNewContext(`(async () => {${example}})()`, {
|
|
||||||
tools,
|
|
||||||
ALL_TOOLS: desktopAvailable
|
|
||||||
? [{ name: 'mcp__desktop_node_repl__node_repl' }]
|
|
||||||
: [],
|
|
||||||
code: 'return platform',
|
|
||||||
text,
|
|
||||||
image,
|
|
||||||
});
|
|
||||||
expect(desktopAvailable ? desktop : regular).toHaveBeenCalledWith({
|
|
||||||
code: 'return platform',
|
|
||||||
});
|
|
||||||
expect(desktopAvailable ? regular : desktop).not.toHaveBeenCalled();
|
|
||||||
expect(text).toHaveBeenCalledWith('macos');
|
|
||||||
expect(image).toHaveBeenCalledWith(screenshot);
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
it('loads a self-contained App workflow for every connected platform', () => {
|
|
||||||
const { config, body } = loadComputerUseSkill();
|
|
||||||
expect(config.name).toBe('computer-use');
|
|
||||||
expect(config.allowedTools).toBeUndefined();
|
|
||||||
expect(body).toContain('`desktop-node-repl` MCP server');
|
|
||||||
expect(body).toContain('mcp__desktop_node_repl__node_repl');
|
|
||||||
expect(body).toContain('skip the installation commands below');
|
|
||||||
expect(body).toContain('ComputerUse.create()');
|
|
||||||
expect(body).toContain('await computer.getPlatform()');
|
|
||||||
expect(body).toContain('computer.getApp(');
|
|
||||||
expect(body).toContain('app.getState(');
|
|
||||||
expect(body).toContain('app.click(37)');
|
|
||||||
expect(body).not.toMatch(
|
|
||||||
/references\/|computer\.observeWindow\(|computer\.listWindows\(|elementToken|windowId/,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('preserves batching, incremental observation and safe refresh guidance', () => {
|
|
||||||
const { body } = loadComputerUseSkill();
|
|
||||||
expect(body).toMatch(/After performing one or more UI actions/);
|
|
||||||
expect(body).toMatch(/Batch actions whose target remains the same/);
|
|
||||||
expect(body).toContain('Prefer this default diff output');
|
|
||||||
expect(body).toContain('disableDiff: true');
|
|
||||||
expect(body).toMatch(/window or session changes/);
|
|
||||||
expect(body).toContain('maxTextChars?: number');
|
|
||||||
expect(body).toContain('12,000 characters');
|
|
||||||
expect(body).toContain('Only currently captured actionable IDs');
|
|
||||||
expect(body).toMatch(
|
|
||||||
/Partial, unconfirmed or cancelled actions must not be blindly repeated/,
|
|
||||||
);
|
|
||||||
expect(body).not.toMatch(
|
|
||||||
/RecreationBench|benchmark|evaluator|score|failure count/i,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('requests screenshots separately and keeps the persistent REPL lifecycle', () => {
|
|
||||||
const { body } = loadComputerUseSkill();
|
|
||||||
const screenshotSection = body.split('## Reading screenshots')[1];
|
|
||||||
expect(screenshotSection).toContain('includeScreenshot: true');
|
|
||||||
expect(screenshotSection).not.toContain('disableDiff');
|
|
||||||
expect(screenshotSection).toContain('image.dataBase64');
|
|
||||||
expect(screenshotSection).toContain('nodeRepl.write(state.text)');
|
|
||||||
expect(body).toContain('await computer.close()');
|
|
||||||
expect(body).toContain(
|
|
||||||
'Reset the Node REPL only when no other persistent state is needed.',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('documents the macOS text methods and their uncertainty boundaries', () => {
|
|
||||||
const { body } = loadComputerUseSkill();
|
|
||||||
expect(body).toContain('app.selectText(37,');
|
|
||||||
expect(body).toContain("await app.paste('ready')");
|
|
||||||
expect(body).toContain("format?: 'text' | 'md' | 'html'");
|
|
||||||
expect(body).toContain(
|
|
||||||
"selection?: 'text' | 'cursor_before' | 'cursor_after'",
|
|
||||||
);
|
|
||||||
expect(body).toContain('immediately adjacent');
|
|
||||||
expect(body).toContain('Missing or');
|
|
||||||
expect(body).toContain('ambiguous matches fail');
|
|
||||||
expect(body).toContain('Observe state before');
|
|
||||||
expect(body).toContain('newer external clipboard change');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
interface:
|
|
||||||
display_name: 'Computer Use'
|
|
||||||
short_description: 'Control desktop apps through the Qwen CUA SDK'
|
|
||||||
default_prompt: 'Use $computer-use to inspect and operate the requested desktop application safely.'
|
|
||||||
@@ -34,3 +34,27 @@ already in the upstream delta, so:
|
|||||||
|
|
||||||
If `<new-ref>` is still *before* a PR merges, that row stays; the cherry-pick
|
If `<new-ref>` is still *before* a PR merges, that row stays; the cherry-pick
|
||||||
keeps carrying the fix.
|
keeps carrying the fix.
|
||||||
|
|
||||||
|
|
||||||
|
## Qwen 0.26.0 platform ports
|
||||||
|
|
||||||
|
The base remains `cua-driver-rs-v0.20.0`; this is a Qwen release version, not a new wholesale upstream snapshot. Selected MIT driver changes from upstream 0.35.4 were adapted to this fork. Native Wayland was removed. No upstream bench code is included.
|
||||||
|
|
||||||
|
The complete per-PR port ledger, implementation differences and verification limits are in [the platform port record](../../docs/design/cua-upgrade-platform-ports.md). macOS ports cover #4943, #4955, #4976, #5051, #5057, #5091 and #5129. Windows covers #4500, #4901, #3671 and #4774. Linux covers #3882, #3761, #3762, #3864, #3618, #4248, #4188, #5139, #5158, #3601, #2888, #5070, #4903, #4984, #5074, #4375, #4292, #3758, #4281, #3814, #4529, #4715, #3658 and #3778.
|
||||||
|
|
||||||
|
The default 19-operation Computer Use contract, driver-owned observations/dispatch, Node REPL runtime documentation, desktop key ownership and browser-use wrapper are Qwen integration changes. The legacy native tool surface remains a host opt-in.
|
||||||
|
|
||||||
|
### P7 follow-up ports
|
||||||
|
|
||||||
|
The native audit also added scoped adaptations for omitted-window capture and AX identity (#4996, #4068, #4085, #4509), Chromium AX readiness (#3783), AppKit launch-window recovery (#4309), ordinary macOS AX walk budgets (#3882), native Electron background scrolling (#4861, #5079), recording capture budgets (#4215), immutable snapshot identity (#3616, #3873), implicit idle-session recovery (#4283), macOS release replacement recovery (#3297), Windows installer ancestor protection (#4398), and Windows autostart error handling and isolated installation (#3180, #4284). These are adapted implementations, not a claim that each entire upstream commit was copied or that all native acceptance has passed. The macOS installer preserves authorization records; upstream automatic TCC resets are not included.
|
||||||
|
|
||||||
|
The [platform port record](../../docs/design/cua-upgrade-platform-ports.md) describes the retained fork contracts and additional ports. The [acceptance record](../../docs/design/cua-upgrade-acceptance-2026-10.md) separates controlled tests, native execution and outstanding validation. The upstream base tag stays unchanged until a complete snapshot synchronization occurs.
|
||||||
|
|
||||||
|
### Additional Qwen acceptance repairs
|
||||||
|
|
||||||
|
These are fork repairs found while testing the adapted implementation, rather than additional whole upstream ports:
|
||||||
|
|
||||||
|
- Preserve a normal uncertain acknowledgement after an attempted AX semantic action; never replay an AX error through a pointer actuator. Apple Events save and Finder rename carry their actual background transport and independent readback evidence.
|
||||||
|
- Resolve exact native macOS window identity before keyboard delivery, establish its AppKit key-window state, and omit the separate titlebar activation click. The native focus records themselves produce internal NaN-location mouse events; they are distinct from the requested pointer action. This does not claim that pointer or menu window covering has been eliminated.
|
||||||
|
- Use byte lengths for AT-SPI text insertion and preserve character-based caret offsets. Keep MPX mappings and input ownership until native completion, including the uncertain timeout path; reject unsupported WM delivery before dispatch.
|
||||||
|
- Limit an isolated Windows uninstall to its selected runtime and owned task actions. Unknown or mixed non-Exec task actions preserve the shared task; legacy default-install cleanup retains its existing scope.
|
||||||
+118
-20
@@ -11,21 +11,41 @@ documented in [`.vendored-patches.md`](.vendored-patches.md) and
|
|||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
|
The commands below target release **0.26.0** and require its GitHub Release
|
||||||
|
(`cua-driver-rs-v0.26.0`) to be published. The native driver does not require
|
||||||
|
Node.js or npm. Install the MCP client you want to use separately.
|
||||||
|
|
||||||
macOS and Linux:
|
macOS and Linux:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
CUA_DRIVER_RS_VERSION=0.20.11 \
|
CUA_DRIVER_RS_VERSION=0.26.0 \
|
||||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.sh)"
|
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.sh)"
|
||||||
```
|
```
|
||||||
|
|
||||||
Windows PowerShell:
|
Windows PowerShell:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
$env:CUA_DRIVER_RS_VERSION = "0.20.11"
|
$env:CUA_DRIVER_RS_VERSION = "0.26.0"
|
||||||
irm https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.ps1 | iex
|
irm https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.ps1 | iex
|
||||||
```
|
```
|
||||||
|
|
||||||
Expected: qwen-cua-driver 0.20.11.
|
The default executable is `~/.local/bin/qwen-cua-driver` on macOS/Linux and
|
||||||
|
`%LOCALAPPDATA%\Programs\Qwen\qwen-cua-driver\bin\qwen-cua-driver.exe` on
|
||||||
|
Windows. Check the installation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# macOS / Linux
|
||||||
|
"$HOME/.local/bin/qwen-cua-driver" --version
|
||||||
|
```
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# Windows PowerShell
|
||||||
|
$driver = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver\bin\qwen-cua-driver.exe"
|
||||||
|
& $driver --version
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `qwen-cua-driver 0.26.0`. If you chose a custom install directory,
|
||||||
|
use its absolute executable path in the client commands below.
|
||||||
|
|
||||||
The released product uses Qwen-owned identities throughout:
|
The released product uses Qwen-owned identities throughout:
|
||||||
|
|
||||||
@@ -44,20 +64,99 @@ that state directory.
|
|||||||
|
|
||||||
## Agent integration
|
## Agent integration
|
||||||
|
|
||||||
Run the MCP server directly:
|
Choose an interface for your agent:
|
||||||
|
|
||||||
|
| Interface | Install and configure |
|
||||||
|
| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
|
| Direct native MCP tools | Install this driver, then follow the client instructions below. |
|
||||||
|
| Persistent JavaScript with preloaded `cua` (Node REPL + SDK) | Follow the [Node REPL README](../node-repl/README.md#usage). It installs the matching SDK dependencies automatically and does not require this standalone driver installation. |
|
||||||
|
|
||||||
|
Both interfaces work with Qwen Code, Claude Code, and Codex over stdio MCP.
|
||||||
|
Claude Code and Codex can use them independently of Qwen Code. The version
|
||||||
|
`0.26.0` here belongs to the CUA components; it does not require a Qwen Code
|
||||||
|
CLI or Web Shell release with that version.
|
||||||
|
|
||||||
|
### Qwen Code
|
||||||
|
|
||||||
|
On macOS/Linux, register the installed executable at user scope:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
qwen-cua-driver mcp
|
qwen mcp add --scope user cua-driver "$HOME/.local/bin/qwen-cua-driver" mcp
|
||||||
```
|
```
|
||||||
|
|
||||||
Register it with Qwen Code:
|
This writes `mcpServers.cua-driver` in `~/.qwen/settings.json`. Use
|
||||||
|
`--scope project` for the current project's `.qwen/settings.json` instead.
|
||||||
|
|
||||||
|
### Claude Code
|
||||||
|
|
||||||
|
On macOS/Linux:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
qwen mcp add cua-driver qwen-cua-driver mcp
|
claude mcp add --scope user --transport stdio cua-driver -- "$HOME/.local/bin/qwen-cua-driver" mcp
|
||||||
```
|
```
|
||||||
|
|
||||||
Other MCP clients can use the same executable and arguments. Shell-oriented
|
User scope is stored in `~/.claude.json`. Use `--scope project` to write the
|
||||||
automation can call tools through `qwen-cua-driver call`.
|
current project's `.mcp.json` instead. See the
|
||||||
|
[Claude Code MCP documentation](https://code.claude.com/docs/en/mcp).
|
||||||
|
|
||||||
|
### Codex
|
||||||
|
|
||||||
|
On macOS/Linux:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
codex mcp add cua-driver -- "$HOME/.local/bin/qwen-cua-driver" mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
Alternatively, add this table to `~/.codex/config.toml`, replacing the path
|
||||||
|
with your actual installed executable. TOML command strings are literal paths;
|
||||||
|
do not use `$HOME` or `~` in place of the absolute path.
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[mcp_servers.cua-driver]
|
||||||
|
command = "/absolute/path/to/.local/bin/qwen-cua-driver"
|
||||||
|
args = ["mcp"]
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [official OpenAI MCP documentation](https://learn.chatgpt.com/docs/extend/mcp).
|
||||||
|
|
||||||
|
### Windows client commands
|
||||||
|
|
||||||
|
In PowerShell, use the installed executable's absolute path. Run the command
|
||||||
|
for your chosen client:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$driver = Join-Path $env:LOCALAPPDATA "Programs\Qwen\qwen-cua-driver\bin\qwen-cua-driver.exe"
|
||||||
|
|
||||||
|
# Qwen Code
|
||||||
|
qwen mcp add --scope user cua-driver $driver mcp
|
||||||
|
|
||||||
|
# Claude Code
|
||||||
|
claude mcp add --scope user --transport stdio cua-driver -- $driver mcp
|
||||||
|
|
||||||
|
# Codex
|
||||||
|
codex mcp add cua-driver -- $driver mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
### Permissions and connection check
|
||||||
|
|
||||||
|
Restart the client after registration. Check its server list with
|
||||||
|
`qwen mcp list`, `claude mcp list`, or `codex mcp list`, and use `/mcp` inside
|
||||||
|
the session to inspect the connection and available tools. The default native
|
||||||
|
MCP interface exposes 19 Computer Use tools; advanced tools require the
|
||||||
|
explicit `CUA_DRIVER_EXTRA_TOOLS=1` opt-in.
|
||||||
|
|
||||||
|
On macOS, grant Accessibility and Screen Recording to
|
||||||
|
`/Applications/QwenCuaDriver.app`; normal `mcp` uses its app-owned daemon.
|
||||||
|
On Windows, run the installer in the logged-in desktop user's session and
|
||||||
|
allow its UAC prompt. On Linux, use an X11 desktop (including X11 apps under
|
||||||
|
XWayland); native Wayland apps are unsupported. Launch the client with that
|
||||||
|
desktop session's environment. Codex needs the matching environment allowlist
|
||||||
|
shown in [Linux desktop sessions](../node-repl/README.md#linux-desktop-sessions),
|
||||||
|
applied to its `mcp_servers.cua-driver` table.
|
||||||
|
|
||||||
|
Clients start `qwen-cua-driver mcp` themselves; a separately started MCP
|
||||||
|
process is unnecessary. Shell-oriented automation can instead call tools
|
||||||
|
through `qwen-cua-driver call`.
|
||||||
|
|
||||||
## 0.20 runtime and SDK surface
|
## 0.20 runtime and SDK surface
|
||||||
|
|
||||||
@@ -156,15 +255,15 @@ release workflow.
|
|||||||
|
|
||||||
## Repository layout
|
## Repository layout
|
||||||
|
|
||||||
| Path | Purpose |
|
| Path | Purpose |
|
||||||
| --- | --- |
|
| ----------------- | -------------------------------------------------------------------- |
|
||||||
| `rust/` | Cargo workspace for the CLI, daemon, SDK, platform crates, and tests |
|
| `rust/` | Cargo workspace for the CLI, daemon, SDK, platform crates, and tests |
|
||||||
| `python/` | Python SDK and bundled-runtime wrapper |
|
| `python/` | Python SDK and bundled-runtime wrapper |
|
||||||
| `typescript/` | TypeScript SDK and native runtime loader |
|
| `typescript/` | TypeScript SDK and native runtime loader |
|
||||||
| `contract/` | Generated portable contract and fixtures |
|
| `contract/` | Generated portable contract and fixtures |
|
||||||
| `tests/fixtures/` | Cross-platform GUI harness applications |
|
| `tests/fixtures/` | Cross-platform GUI harness applications |
|
||||||
| `scripts/` | Release/local install, uninstall, generation, and sync helpers |
|
| `scripts/` | Release/local install, uninstall, generation, and sync helpers |
|
||||||
| `docs/` | Package-local architecture and behavior notes |
|
| `docs/` | Package-local architecture and behavior notes |
|
||||||
|
|
||||||
Start with [`rust/README.md`](rust/README.md),
|
Start with [`rust/README.md`](rust/README.md),
|
||||||
[`docs/test-matrix.md`](docs/test-matrix.md), and
|
[`docs/test-matrix.md`](docs/test-matrix.md), and
|
||||||
@@ -181,5 +280,4 @@ cargo test -p cua-driver-core
|
|||||||
```
|
```
|
||||||
|
|
||||||
Generated contract and language bindings must be checked with the package-local
|
Generated contract and language bindings must be checked with the package-local
|
||||||
scripts before release. Signed/notarized macOS, Windows UIAccess, Linux X11 and
|
scripts before release. Signed/notarized macOS, Windows UIAccess, Linux X11, and real MCP/model verification remain platform release gates.
|
||||||
Wayland, and real MCP/model verification remain platform release gates.
|
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,10 @@
|
|||||||
# Cua Driver 0.13.2 Overnight Stabilization Plan
|
# Cua Driver 0.13.2 Overnight Stabilization Plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Status:** Draft for review
|
**Status:** Draft for review
|
||||||
**Baseline:** Cua Driver 0.13.1 is already released
|
**Baseline:** Cua Driver 0.13.1 is already released
|
||||||
**Source baseline:** `origin/main` at `1e77ab5536eb88ca4cb7d3b6cc1fb51eea03f0e4`
|
**Source baseline:** `origin/main` at `1e77ab5536eb88ca4cb7d3b6cc1fb51eea03f0e4`
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ when that is the background-safe route.
|
|||||||
| Windows/Win32 | `64e82449` | Run `29257963004`: 122/122 rows, 99 delivered and 23 exact refusals |
|
| Windows/Win32 | `64e82449` | Run `29257963004`: 122/122 rows, 99 delivered and 23 exact refusals |
|
||||||
| macOS/Quartz | `0b9b5d94` and `64e82449` | Logged-in local matrix: 138 deliveries and 6 exact refusals passed; one cursor-interrupted row passed alone on the final head, for effective 145/145 coverage |
|
| macOS/Quartz | `0b9b5d94` and `64e82449` | Logged-in local matrix: 138 deliveries and 6 exact refusals passed; one cursor-interrupted row passed alone on the final head, for effective 145/145 coverage |
|
||||||
| Linux/X11 | `268a6ab1` | Run `29254936043`: 116/116 rows, 75 delivered and 41 exact refusals |
|
| Linux/X11 | `268a6ab1` | Run `29254936043`: 116/116 rows, 75 delivered and 41 exact refusals |
|
||||||
| Linux/Sway | `268a6ab1` and `64e82449` | Run `29255208927` passed native and capture 36/36; run `29257961614` passed shared 80/80, for effective 116/116 coverage |
|
|
||||||
|
|
||||||
## Shared web harnesses
|
## Shared web harnesses
|
||||||
|
|
||||||
@@ -36,14 +35,10 @@ Foreground rows listed here are delivered for both AX and PX.
|
|||||||
| macOS | WKWebView | left/right/double click AX/PX, type text AX/PX, press key AX/PX, hotkey AX/PX, scroll AX/PX, child window AX/PX, editor save AX | drag PX: `background_unavailable` |
|
| macOS | WKWebView | left/right/double click AX/PX, type text AX/PX, press key AX/PX, hotkey AX/PX, scroll AX/PX, child window AX/PX, editor save AX | drag PX: `background_unavailable` |
|
||||||
| Linux X11 | Electron | Background AX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Background PX left click, AX/PX right and double click, PX child window and drag, AX/PX keyboard and scroll, and AX editor save return exact `background_unavailable` |
|
| Linux X11 | Electron | Background AX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Background PX left click, AX/PX right and double click, PX child window and drag, AX/PX keyboard and scroll, and AX editor save return exact `background_unavailable` |
|
||||||
| Linux X11 | Tauri | Background AX/PX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Other background pointer, keyboard, scroll, drag, and editor-save shapes return exact `background_unavailable` |
|
| Linux X11 | Tauri | Background AX/PX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Other background pointer, keyboard, scroll, drag, and editor-save shapes return exact `background_unavailable` |
|
||||||
| Linux Sway | Electron | The full 40-cell catalog has accepted fixture-owned evidence; safe semantic and compositor routes deliver | Focus-bound and raw background shapes without a target-addressed route return their declared exact refusal |
|
|
||||||
| Linux Sway | Tauri | The full 40-cell WebKitGTK catalog has accepted fixture-owned evidence, including live compositor geometry and roleless WebProcess targeting | Focus-bound and raw background shapes without a target-addressed route return their declared exact refusal |
|
|
||||||
|
|
||||||
All shared background rows require fixture state, focus, z-order, and
|
All shared background rows require fixture state, focus, z-order, and
|
||||||
no-leaked-input evidence. Windows, macOS, and X11 also require cursor
|
no-leaked-input evidence. Windows, macOS, and X11 also require cursor
|
||||||
preservation. Wayland reports the cursor oracle as unsupported until the
|
preservation.
|
||||||
environment can prove it; [issue #2194](https://github.com/trycua/cua/issues/2194)
|
|
||||||
tracks possible observer routes.
|
|
||||||
Foreground rows require fixture state and retain a video plus before/after turn
|
Foreground rows require fixture state and retain a video plus before/after turn
|
||||||
evidence.
|
evidence.
|
||||||
|
|
||||||
@@ -69,16 +64,13 @@ integrity-level harness can exercise it.
|
|||||||
|
|
||||||
## Native Linux
|
## Native Linux
|
||||||
|
|
||||||
Linux is recorded per display server and compositor because Wayland protocols
|
Linux supports X11, including X11 applications under XWayland. Native Wayland
|
||||||
are capabilities, not one uniform API.
|
backends and their validation lanes have been removed. The exact-source runs
|
||||||
|
below are historical baselines; acceptance of the upgrade requires new evidence.
|
||||||
|
|
||||||
| Environment | Proven contracts | Refusals and gaps |
|
| Environment | Proven contracts | Refusals and gaps |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| X11/Openbox | Exact run `29254936043` passed all 116 declared outcomes: 75 deliveries and 41 exact refusals. GTK3 AT-SPI actions and values, foreground XTest pointer/keyboard routes, capture, desktop scope, and the complete Electron/Tauri catalog all reached fixture-owned oracles. | Xvfb does not prove the real-Xorg MPX/uinput background pointer route. Toolkits that reject XSendEvent retain exact refusals. |
|
| X11/Openbox | Exact run `29254936043` passed all 116 declared outcomes: 75 deliveries and 41 exact refusals. GTK3 AT-SPI actions and values, foreground XTest pointer/keyboard routes, capture, desktop scope, and the complete Electron/Tauri catalog all reached fixture-owned oracles. | Xvfb does not prove the real-Xorg MPX/uinput background pointer route. Toolkits that reject XSendEvent retain exact refusals. |
|
||||||
| Sway/wlroots | Runs `29255208927` and `29257961614` passed the complete 116 outcomes: native and capture 36/36, then shared 80/80. The Electron, Tauri, GTK3, capture, and desktop-scope catalogs use live Sway identity/geometry and external fixture oracles. | Stock Wayland cannot target raw focus-bound input at an occluded surface. Those rows retain exact refusals; another wlroots compositor is not assumed equivalent until a material divergence is reported and tested. Exact cursor preservation remains unproven and is tracked in [issue #2194](https://github.com/trycua/cua/issues/2194). |
|
|
||||||
| GNOME/Mutter | A real GNOME 46 Wayland run passed the full GTK3 matrix, 31/31. The WinRects helper supplies stable window ids, frame and buffer geometry, stacking, verified activation, stage capture, and the compositor cursor. AT-SPI handles semantic actions; persistent portal/libei sessions deliver foreground PX click, right/double click, drag, scroll, type, key, and hotkey. Background rows either deliver through AT-SPI with focus and leak guards or return the declared exact refusal. | The helper requires installation plus one Shell-session restart. Without it, target-bound foreground input refuses. Portal video and a shared Electron/Tauri GNOME run remain open. |
|
|
||||||
| KDE/KWin | A Plasma 6 session reached GTK AT-SPI discovery, generic toplevel discovery, and portal-interface preflight. Portal input is compiled into release binaries. | No behavioral matrix is accepted. A target-addressable KWin activation adapter is not implemented, so foreground portal/libei input refuses instead of injecting into the wrong focused app. |
|
|
||||||
| Nested `cua-compositor` | The optional backend owns its nested session and is covered by typed shared, native, and capture catalogs. Run `29197643541` passed native GTK3 31/31; run `29197887387` passed capture/scope 5/5. Full shared run `29199596935` passed 26/36 Electron rows before the occluded-subtree and wheel-frame repairs. | The lane remains experimental. The accepted full shared run still has 10 failures; focused replacement runs reduce that set but do not promote the environment. Unicode text and a canonical parallel-drag row remain unproven. This private route is not evidence of a stock-Wayland capability. |
|
|
||||||
|
|
||||||
PX background left-click rows may resolve the screen point to an actionable
|
PX background left-click rows may resolve the screen point to an actionable
|
||||||
AT-SPI node. Such a pass proves the public PX-addressed behavior and its desktop
|
AT-SPI node. Such a pass proves the public PX-addressed behavior and its desktop
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Browser Cross-Platform Hardening Journal
|
# Browser Cross-Platform Hardening Journal
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
This journal records implementation and validation evidence for the browser
|
This journal records implementation and validation evidence for the browser
|
||||||
support hardening branch. It is not a public support contract.
|
support hardening branch. It is not a public support contract.
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Existing-Profile Browser Attachment and Reconnect Plan
|
# Existing-Profile Browser Attachment and Reconnect Plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
- **State:** Implemented and accepted from exact source SHA `7fa6c80f` across macOS, Windows, Linux X11, and Linux Wayland on Sway
|
- **State:** Implemented and accepted from exact source SHA `7fa6c80f` across macOS, Windows, Linux X11, and Linux Wayland on Sway
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Browser Semantic State Plan
|
# Browser Semantic State Plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
- **State:** Implemented; cross-platform acceptance in progress
|
- **State:** Implemented; cross-platform acceptance in progress
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Browser Tool Implementation Journal
|
# Browser Tool Implementation Journal
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
This journal records implementation and validation evidence for the first-class
|
This journal records implementation and validation evidence for the first-class
|
||||||
browser tools. It intentionally omits machine identities, credentials, private
|
browser tools. It intentionally omits machine identities, credentials, private
|
||||||
URLs, profile paths, and raw DevTools endpoint identifiers.
|
URLs, profile paths, and raw DevTools endpoint identifiers.
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Cua Driver Browser Tool Implementation Plan
|
# Cua Driver Browser Tool Implementation Plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
- **State:** Browser-tool v2 roadmap implemented locally; acceptance evidence recorded in the implementation journal
|
- **State:** Browser-tool v2 roadmap implemented locally; acceptance evidence recorded in the implementation journal
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Cursor modifier badge restructuring plan
|
# Cursor modifier badge restructuring plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Status:** Implemented in [PR #2677](https://github.com/trycua/cua/pull/2677); validation complete
|
**Status:** Implemented in [PR #2677](https://github.com/trycua/cua/pull/2677); validation complete
|
||||||
**Base:** `main` at `b249df857994ebaebf9f02248e8312b3bd2a5dd2`
|
**Base:** `main` at `b249df857994ebaebf9f02248e8312b3bd2a5dd2`
|
||||||
**Scope:** Move delivery and target modifiers from around the agent pointer into the session badge while preserving action animations beside the pointer.
|
**Scope:** Move delivery and target modifiers from around the agent pointer into the session badge while preserving action animations beside the pointer.
|
||||||
|
|||||||
@@ -24,11 +24,8 @@ fades over 400 milliseconds while the cursor stays available. Moving the human
|
|||||||
pointer over the synthetic cursor temporarily reveals the name again, and
|
pointer over the synthetic cursor temporarily reveals the name again, and
|
||||||
moving it away hides the name.
|
moving it away hides the name.
|
||||||
Hiding and revealing the cursor also shows the badge again. Hover reveal uses
|
Hiding and revealing the cursor also shows the badge again. Hover reveal uses
|
||||||
the native global-pointer APIs on macOS, Windows, and X11. Stock Wayland does
|
the native global-pointer APIs on macOS, Windows, and X11. The click-through
|
||||||
not expose another client's global pointer position, so native Wayland keeps
|
overlay never captures pointer input to fake this capability.
|
||||||
the timed reveal but cannot offer hover reveal without a compositor-owned
|
|
||||||
adapter. The click-through overlay never captures pointer input to fake this
|
|
||||||
capability.
|
|
||||||
|
|
||||||
Delivery and target context appears as up to two chips inside the same badge.
|
Delivery and target context appears as up to two chips inside the same badge.
|
||||||
The delivery chip comes first and is filled; the target chip comes second and
|
The delivery chip comes first and is filled; the target chip comes second and
|
||||||
@@ -48,8 +45,7 @@ move the agent cursor to that target before dispatch. Actions without a spatial
|
|||||||
target, such as typing into the already focused desktop application, keep the
|
target, such as typing into the already focused desktop application, keep the
|
||||||
cursor at its current location and only change the semantic animation.
|
cursor at its current location and only change the semantic animation.
|
||||||
During a drag, the native gesture's endpoints, step count, and wall-clock
|
During a drag, the native gesture's endpoints, step count, and wall-clock
|
||||||
duration drive the same shared pointer-anchor transform on macOS, Windows, X11,
|
duration drive the same shared pointer-anchor transform on macOS, Windows, and X11, so the synthetic cursor follows the held
|
||||||
and supported Wayland input routes, so the synthetic cursor follows the held
|
|
||||||
gesture instead of moving only after release.
|
gesture instead of moving only after release.
|
||||||
|
|
||||||
The cursor is a visual aid for people watching an agent. It is not a security
|
The cursor is a visual aid for people watching an agent. It is not a security
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Cua Driver Permission Modes, Capability Policy, and Human Consent Plan
|
# Cua Driver Permission Modes, Capability Policy, and Human Consent Plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
- **State:** Reviewed draft / local; implementation plan, not yet a plan of record
|
- **State:** Reviewed draft / local; implementation plan, not yet a plan of record
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Driver Permission Modes Implementation Journal
|
# Driver Permission Modes Implementation Journal
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
> Contract update: the shipped canonical middle-mode name is `bounded`, with
|
> Contract update: the shipped canonical middle-mode name is `bounded`, with
|
||||||
> `autonomous` retained as a compatibility alias. The interactive CLI danger
|
> `autonomous` retained as a compatibility alias. The interactive CLI danger
|
||||||
> flag now selects `unrestricted` as well as acknowledging its risk; embedded
|
> flag now selects `unrestricted` as well as acknowledging its risk; embedded
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# E2E Convergence Journal
|
# E2E Convergence Journal
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Historical record.** Use `action-support.md` and the hosted platform support
|
**Historical record.** Use `action-support.md` and the hosted platform support
|
||||||
reference for current support claims.
|
reference for current support claims.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Linux representative desktop validation
|
# Linux representative desktop validation
|
||||||
|
|
||||||
Hosted CI owns the canonical Xvfb/Openbox and headless Sway environments. Some
|
Hosted CI owns the canonical Xvfb/Openbox environment. Some
|
||||||
contracts require a real user desktop and therefore run only when a maintainer
|
contracts require a real user desktop and therefore run only when a maintainer
|
||||||
explicitly provisions the corresponding environment.
|
explicitly provisions the corresponding environment.
|
||||||
|
|
||||||
@@ -8,10 +8,7 @@ explicitly provisions the corresponding environment.
|
|||||||
|
|
||||||
| Environment | Required evidence | Current status | Preflight |
|
| Environment | Required evidence | Current status | Preflight |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| GNOME/Mutter | WinRects geometry and activation, portal/libei input, portal recording, and shared renderer apps | Native GTK behavior, capture, and desktop scope are accepted; shared renderers and portal video remain open | Wayland user session, enabled WinRects helper, and portal grant |
|
| Real Xorg | MPX/uinput behavior that Xvfb cannot provide | Not yet validated | Xorg session with `/dev/uinput` access |
|
||||||
| KDE/KWin | KWin-specific activation and portal behavior | Plasma 6 session startup, GTK AT-SPI discovery, and portal interfaces were observed; no behavioral matrix is accepted | Plasma/KWin 6 Wayland session; Plasma 5.27 is rejected |
|
|
||||||
| Real Xorg | MPX/uinput behavior that Xvfb cannot provide | Not yet validated | Non-Wayland Xorg session with `/dev/uinput` access |
|
|
||||||
| DRM/EGL renderer | Representative WebKitGTK/Tauri accessibility tree | Not available on the hosted Sway or representative desktop hosts used so far | A real `/dev/dri/renderD128`; software-only headless rendering is rejected |
|
|
||||||
|
|
||||||
## Source ownership
|
## Source ownership
|
||||||
|
|
||||||
@@ -32,8 +29,6 @@ equivalent terminal inside that user's session. The wrapper rejects the wrong
|
|||||||
desktop generation before building fixtures.
|
desktop generation before building fixtures.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scripts/ci/linux/run-rust-e2e-desktop.sh gnome
|
|
||||||
scripts/ci/linux/run-rust-e2e-desktop.sh kde
|
|
||||||
scripts/ci/linux/run-rust-e2e-desktop.sh xorg
|
scripts/ci/linux/run-rust-e2e-desktop.sh xorg
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Linux support completion plan
|
# Linux support completion plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Status:** Historical implementation plan. Current outcomes live in
|
**Status:** Historical implementation plan. Current outcomes live in
|
||||||
`action-support.md`; remaining work lives in the
|
`action-support.md`; remaining work lives in the
|
||||||
[public platform roadmap](https://cua.ai/docs/reference/cua-driver/platform-roadmap).
|
[public platform roadmap](https://cua.ai/docs/reference/cua-driver/platform-roadmap).
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Active Permission Adapters and Trusted Per-Session Modes
|
# Active Permission Adapters and Trusted Per-Session Modes
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Status:** Reviewed; implementation sequencing accepted
|
**Status:** Reviewed; implementation sequencing accepted
|
||||||
|
|
||||||
**Issues:** #2385 and #2437
|
**Issues:** #2385 and #2437
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Permission Simplification Execution Journal
|
# Permission Simplification Execution Journal
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Status:** Active
|
**Status:** Active
|
||||||
|
|
||||||
**Branch:** `codex/permission-simplification-0130`
|
**Branch:** `codex/permission-simplification-0130`
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# PR #2161 Landing Plan - Second-Wave E2E Convergence
|
# PR #2161 Landing Plan - Second-Wave E2E Convergence
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Historical execution plan.** This file records how the original draft was
|
**Historical execution plan.** This file records how the original draft was
|
||||||
split and landed. It is not the current test or platform support reference.
|
split and landed. It is not the current test or platform support reference.
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Cua Driver SDK Rust source-of-truth and UniFFI evaluation plan
|
# Cua Driver SDK Rust source-of-truth and UniFFI evaluation plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
Status: superseded by the final package-boundary decision in PR #2341
|
Status: superseded by the final package-boundary decision in PR #2341
|
||||||
|
|
||||||
Applies to: PR #2341 and follow-up SDK/runtime work
|
Applies to: PR #2341 and follow-up SDK/runtime work
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Rust E2E Harness Convergence Plan
|
# Rust E2E Harness Convergence Plan
|
||||||
|
|
||||||
|
> Historical record, superseded by the 2026-10 X11-only upgrade. Native Wayland
|
||||||
|
> implementations, helpers, dependencies and validation lanes described below
|
||||||
|
> have been removed; these historical results do not describe current support.
|
||||||
|
|
||||||
|
|
||||||
**Historical plan.** Use `test-harnesses-guide.md`, `test-matrix.md`, and
|
**Historical plan.** Use `test-harnesses-guide.md`, `test-matrix.md`, and
|
||||||
`e2e-ci-reporting.md` for the implemented contributor workflow.
|
`e2e-ci-reporting.md` for the implemented contributor workflow.
|
||||||
|
|
||||||
|
|||||||
@@ -222,26 +222,13 @@ Runner: `scripts/ci/linux/run-rust-e2e.sh`
|
|||||||
| Shared app matrix | `cross_platform_behavior_test.rs` | Electron and Tauri |
|
| Shared app matrix | `cross_platform_behavior_test.rs` | Electron and Tauri |
|
||||||
| Native controls | `harness_gtk3_test.rs` | Repo-local GTK3 app |
|
| Native controls | `harness_gtk3_test.rs` | Repo-local GTK3 app |
|
||||||
| Capture contract | `capture_contract_test.rs` | Linux capture backend |
|
| Capture contract | `capture_contract_test.rs` | Linux capture backend |
|
||||||
| Desktop scope | `desktop_scope_linux_test.rs` | X11/Wayland desktop scope |
|
| Desktop scope | `desktop_scope_linux_test.rs` | X11 desktop scope |
|
||||||
|
|
||||||
Linux has separate X11 and Wayland concerns. Nix supplies the reproducible
|
Linux requires an authenticated X11 session and its accessibility session bus.
|
||||||
build and desktop environment, but the E2E test still needs an actual X11 or
|
Xvfb/Openbox can cover capture, AT-SPI and foreground input. Real Xorg is
|
||||||
Wayland session. Linux does not need GIF output; MP4, screenshots, accessibility
|
required to verify MPX/uinput. A DISPLAY with surrounding Wayland environment
|
||||||
trees, trajectories, and logs are the useful evidence.
|
variables exercises X11 selection only; it does not prove real XWayland.
|
||||||
|
Linux evidence includes MP4, screenshots, accessibility trees, trajectories and logs.
|
||||||
Wayland results are compositor-specific. The hosted lane uses Sway to prove
|
|
||||||
wlroots protocols. GNOME requires the optional WinRects Shell helper for
|
|
||||||
authoritative frame and buffer geometry, observation, capture, and verified
|
|
||||||
target activation. A portal/libei grant persists until the user revokes it, so
|
|
||||||
subsequent driver processes do not reopen the consent dialog.
|
|
||||||
KDE requires a future target-addressable KWin adapter; portal availability by
|
|
||||||
itself is not evidence that input can be sent safely to a named window.
|
|
||||||
Standard Wayland does not expose the physical pointer position, so canonical
|
|
||||||
Wayland rows do not claim the real-cursor preservation oracle. Focus, full
|
|
||||||
occlusion, sentinel input isolation, liveness, and fixture-state oracles remain
|
|
||||||
mandatory. [Issue #2194](https://github.com/trycua/cua/issues/2194) tracks
|
|
||||||
compositor, portal/libei, sentinel, and capture-based ways to add a proven
|
|
||||||
cursor observer where the environment supports one.
|
|
||||||
|
|
||||||
## AX, PX, and Delivery
|
## AX, PX, and Delivery
|
||||||
|
|
||||||
@@ -302,10 +289,7 @@ preflight deliberately sends input to the sentinel and deliberately raises the
|
|||||||
background target. The lane proceeds only if the leaked input and transient
|
background target. The lane proceeds only if the leaked input and transient
|
||||||
focus loss are observed, the sentinel is restored, and it once again fully
|
focus loss are observed, the sentinel is restored, and it once again fully
|
||||||
occludes the target. Windows, macOS, and X11 require the sentinel's live focus
|
occludes the target. Windows, macOS, and X11 require the sentinel's live focus
|
||||||
journal to report the loss. Wayland uses the compositor-backed native focus
|
journal to report the loss. This positive control prevents a broken
|
||||||
observer because Electron/Ozone does not reliably emit a DOM `blur` event for
|
|
||||||
an external surface focus transition. The sentinel heartbeat and leaked-input
|
|
||||||
journal remain mandatory on Wayland. This positive control prevents a broken
|
|
||||||
guard from making every background row look green.
|
guard from making every background row look green.
|
||||||
|
|
||||||
A focus assertion can prove "no focus steal" while failing to prove that a
|
A focus assertion can prove "no focus steal" while failing to prove that a
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ Every harness E2E result should identify these dimensions:
|
|||||||
| Dimension | Values |
|
| Dimension | Values |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| OS | `windows`, `macos`, `linux` |
|
| OS | `windows`, `macos`, `linux` |
|
||||||
| Window system | Win32/UIA, AppKit/AX, X11/AT-SPI, Wayland/AT-SPI, WebView/CDP |
|
| Window system | Win32/UIA, AppKit/AX, X11/AT-SPI, WebView/CDP |
|
||||||
| Harness | Electron, Tauri, WPF, WinUI3, WebView2, AppKit, SwiftUI, WKWebView, GTK3 |
|
| Harness | Electron, Tauri, WPF, WinUI3, WebView2, AppKit, SwiftUI, WKWebView, GTK3 |
|
||||||
| Action targeting | `ax`, `px`, `page`, `not_applicable` |
|
| Action targeting | `ax`, `px`, `page`, `not_applicable` |
|
||||||
| Delivery | `background`, `foreground`, `N/A` |
|
| Delivery | `background`, `foreground`, `N/A` |
|
||||||
@@ -140,13 +140,12 @@ transient-panel AX discovery gap.
|
|||||||
|
|
||||||
| Harness | Source test | Window system | Coverage |
|
| Harness | Source test | Window system | Coverage |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| Electron | `cross_platform_behavior_test.rs` | X11 and hosted Sway | Shared web action matrix |
|
| Electron | `cross_platform_behavior_test.rs` | X11 | Shared web action matrix |
|
||||||
| Tauri | `cross_platform_behavior_test.rs` | X11 and hosted Sway | Shared web action matrix |
|
| Tauri | `cross_platform_behavior_test.rs` | X11 | Shared web action matrix |
|
||||||
| GTK3 | `harness_gtk3_test.rs` | X11/AT-SPI and Wayland/AT-SPI where configured | Native GTK controls and input |
|
| GTK3 | `harness_gtk3_test.rs` | X11/AT-SPI | Native GTK controls and input |
|
||||||
| Desktop target | `desktop_scope_linux_test.rs` | X11/Wayland | Per-call window/desktop modality, screen-absolute action delivery, and strict rejection |
|
| Desktop target | `desktop_scope_linux_test.rs` | X11 | Per-call window/desktop modality, screen-absolute action delivery, and strict rejection |
|
||||||
|
|
||||||
Nix provides the Linux build and desktop environment. X11 and Wayland are
|
Linux requires an authenticated X11 display and its AT-SPI session bus.
|
||||||
separate matrix dimensions because their capture and input contracts differ.
|
|
||||||
Linux runs do not produce GIF output. Every canonical GUI cell retains an MP4
|
Linux runs do not produce GIF output. Every canonical GUI cell retains an MP4
|
||||||
and trajectory alongside screenshots, AX trees, structured results, and driver
|
and trajectory alongside screenshots, AX trees, structured results, and driver
|
||||||
logs for E2E evidence.
|
logs for E2E evidence.
|
||||||
@@ -186,9 +185,7 @@ allowed, and the desktop-side-effect oracles pass.
|
|||||||
| Windows unit/compile | `windows-latest` | Package-scoped `cargo test --all-targets --no-run --locked` |
|
| Windows unit/compile | `windows-latest` | Package-scoped `cargo test --all-targets --no-run --locked` |
|
||||||
| Windows Harness E2E | Active Windows user session | `scripts/ci/windows/run-rust-e2e.ps1 -RequireGui` |
|
| Windows Harness E2E | Active Windows user session | `scripts/ci/windows/run-rust-e2e.ps1 -RequireGui` |
|
||||||
| Linux X11 Harness E2E | Nix X11 session | `scripts/ci/linux/run-rust-e2e.sh` |
|
| Linux X11 Harness E2E | Nix X11 session | `scripts/ci/linux/run-rust-e2e.sh` |
|
||||||
| Linux Sway Harness E2E | Controlled wlroots session | `scripts/ci/linux/run-rust-e2e-wayland.sh` |
|
| Linux representative desktop E2E | Existing Xorg login | `scripts/ci/linux/run-rust-e2e-desktop.sh <desktop>` |
|
||||||
| Linux nested-compositor E2E | Controlled experimental session | `scripts/ci/linux/run-rust-e2e-inject.sh` |
|
|
||||||
| Linux representative desktop E2E | Existing GNOME, KDE, or Xorg login | `scripts/ci/linux/run-rust-e2e-desktop.sh <desktop>` |
|
|
||||||
| macOS Harness E2E | Maintainer Lume SIP-off worker with a logged-in session and inherited grants | `packages/cua-driver/tests/runners/macos-lume/run-all.sh` |
|
| macOS Harness E2E | Maintainer Lume SIP-off worker with a logged-in session and inherited grants | `packages/cua-driver/tests/runners/macos-lume/run-all.sh` |
|
||||||
|
|
||||||
Workflows select private execution lanes. Rust source owns scenario definitions,
|
Workflows select private execution lanes. Rust source owns scenario definitions,
|
||||||
|
|||||||
@@ -1,102 +0,0 @@
|
|||||||
# Wayland Compositor Identity Plan
|
|
||||||
|
|
||||||
This is an architecture explanation and evidence plan for expanding exact
|
|
||||||
browser attachment beyond Sway. It does not claim generic Wayland support.
|
|
||||||
|
|
||||||
## Why Compositor Identity Is Required
|
|
||||||
|
|
||||||
Existing-profile setup can open an internal browser page and press one exact
|
|
||||||
accessibility control. Before doing so, Cua Driver must prove that the native
|
|
||||||
toplevel, browser PID, geometry, AT-SPI application/window, product descriptor,
|
|
||||||
and loopback endpoint all describe the same browser generation.
|
|
||||||
|
|
||||||
Title, application ID, or screen-coordinate similarity alone is insufficient.
|
|
||||||
An attacker or unrelated window can reuse those values. When the compositor
|
|
||||||
cannot attest exact identity, mutation must refuse before opening the setup
|
|
||||||
page or sending input.
|
|
||||||
|
|
||||||
## Accepted Sway Route
|
|
||||||
|
|
||||||
The existing Sway adapter uses compositor IPC to correlate the exact container,
|
|
||||||
PID, and rectangle with the AT-SPI tree. Setup may briefly focus that attested
|
|
||||||
container for fixed browser-owned internal navigation, then restores the prior
|
|
||||||
container. Canonical Sway evidence accepts Google Chrome, Chromium, and
|
|
||||||
Microsoft Edge after each product completed the same full matrix with playable
|
|
||||||
video and no unexpected results.
|
|
||||||
|
|
||||||
## Accepted GNOME Route
|
|
||||||
|
|
||||||
The maintained WinRects extension v4 exposes the narrow identity route Cua
|
|
||||||
Driver needs on GNOME Shell and Mutter: a stable window identifier, PID,
|
|
||||||
geometry, focus and stacking state, and exact activation.
|
|
||||||
|
|
||||||
The Linux adapter resolves the extension's immutable unique D-Bus owner,
|
|
||||||
verifies that the owner is the current user's system-installed `gnome-shell`
|
|
||||||
process, requires helper API v4 for browser-sensitive calls, and addresses that
|
|
||||||
unique owner directly. This prevents an unrelated same-session process from
|
|
||||||
replacing the public bus name between verification and activation. Setup and
|
|
||||||
consent briefly activate one exact window, then restore and verify the prior
|
|
||||||
Shell focus.
|
|
||||||
|
|
||||||
The route is accepted for Google Chrome after the full standalone-browser
|
|
||||||
matrix recorded 13 delivered behaviors, 3 policy refusals, 0 failures, and 0
|
|
||||||
skips with 16 playable videos at one source commit. Other Chromium-family
|
|
||||||
products still require their own product-specific evidence on GNOME.
|
|
||||||
|
|
||||||
The adapter refuses when the component is missing, incompatible, disabled, or
|
|
||||||
unable to distinguish multiple windows for one browser process. Enabling a
|
|
||||||
generic unsafe-mode switch is not an acceptable production dependency.
|
|
||||||
|
|
||||||
## Candidate Compositors
|
|
||||||
|
|
||||||
### KDE Plasma and KWin
|
|
||||||
|
|
||||||
KWin 6 scripting exposes a per-window internal ID, PID, frame geometry,
|
|
||||||
stacking order, active state, and workspace activation. See the official
|
|
||||||
[KWin scripting API](https://develop.kde.org/docs/plasma/kwin/api/).
|
|
||||||
|
|
||||||
A maintained Cua Driver KWin component could return a versioned, read-only
|
|
||||||
identity record and perform one exact focus-and-restore transition. The adapter
|
|
||||||
must bind its D-Bus peer, KWin generation, internal window ID, PID, geometry,
|
|
||||||
and AT-SPI window. A generic user-provided script or title lookup is not enough.
|
|
||||||
|
|
||||||
### Other Compositors
|
|
||||||
|
|
||||||
The standard `ext-foreign-toplevel-list-v1` protocol exposes an opaque stable
|
|
||||||
toplevel identifier, title, and application ID, but not PID or geometry. See
|
|
||||||
the [foreign toplevel protocol](https://wayland.app/protocols/ext-foreign-toplevel-list-v1).
|
|
||||||
It can contribute one part of a proof, but cannot independently authorize
|
|
||||||
mutation.
|
|
||||||
|
|
||||||
wlroots compositors may expose additional management or compositor IPC. Each
|
|
||||||
route remains compositor-specific until it proves all required fields. No
|
|
||||||
screen-coordinate fallback is allowed.
|
|
||||||
|
|
||||||
## Threat Cases
|
|
||||||
|
|
||||||
Each adapter and harness must cover:
|
|
||||||
|
|
||||||
- two same-title browser windows with equal or near-equal geometry;
|
|
||||||
- PID reuse or browser restart between approval and mutation;
|
|
||||||
- a decoy application with the same application ID or title;
|
|
||||||
- a window moving, changing workspace, minimizing, or closing during setup;
|
|
||||||
- stale compositor object IDs after remap;
|
|
||||||
- an incomplete or mismatched AT-SPI tree;
|
|
||||||
- endpoint ownership or generation drift; and
|
|
||||||
- focus restoration failure after a bounded setup action.
|
|
||||||
|
|
||||||
Any ambiguity produces a stable, non-mutating structured refusal.
|
|
||||||
|
|
||||||
## Release Acceptance Per Compositor
|
|
||||||
|
|
||||||
A compositor is listed as supported only after the canonical matrix records:
|
|
||||||
|
|
||||||
1. exact compositor, distribution, browser product, and version provenance;
|
|
||||||
2. setup, attach, reconnect, multi-tab, stale-ref, background type, trusted
|
|
||||||
input policy, and ambiguous-window rows;
|
|
||||||
3. external page-state oracles rather than driver success responses;
|
|
||||||
4. continuous focus, z-order or occlusion, and no-leaked-input guards;
|
|
||||||
5. cursor preservation where the compositor exposes a trustworthy readback;
|
|
||||||
6. before and after evidence plus playable video at the exact source commit;
|
|
||||||
7. explicit environment results for missing APIs or accessibility trees; and
|
|
||||||
8. generic and unknown Wayland sessions refusing before mutation.
|
|
||||||
@@ -1,3 +1,15 @@
|
|||||||
|
# 0.26.0 (2026-10-16)
|
||||||
|
|
||||||
|
Qwen Computer Use release; no longer follows the upstream version number. The SDK, native payload, browser SDK and Node REPL use this same version.
|
||||||
|
|
||||||
|
- Driver-owned app/window resolution, current short IDs, AX diffs/budgets and structured dispatch errors; compact 19-tool direct MCP surface.
|
||||||
|
- Background-first platform delivery, guarded foreground fallback on Windows/Linux, and explicit desktop input with session-owned held-key cleanup. Native Wayland removed.
|
||||||
|
- Runtime documentation and browser-use facade replace the two bundled Computer Use/Browser Use skill entrypoints. Browser functionality is unchanged.
|
||||||
|
- Windows UIAccess is optional; default installation does not require a signed worker. Native archives remain on-demand downloads with SHA-256 verification.
|
||||||
|
- Selected upstream MIT fixes are recorded per PR in `.vendored-patches.md` and `docs/design/cua-upgrade-platform-ports.md`, including platform adaptations and runtime evidence gaps.
|
||||||
|
|
||||||
|
This local development checkpoint does not claim published assets, GUI parity or model benchmark improvement.
|
||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
## [0.20.0](https://github.com/trycua/cua/compare/cua-driver-rs-v0.19.3...cua-driver-rs-v0.20.0) (2026-08-15)
|
## [0.20.0](https://github.com/trycua/cua/compare/cua-driver-rs-v0.19.3...cua-driver-rs-v0.20.0) (2026-08-15)
|
||||||
|
|||||||
Generated
+19
-579
@@ -46,16 +46,6 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "annotate-snippets"
|
|
||||||
version = "0.9.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e"
|
|
||||||
dependencies = [
|
|
||||||
"unicode-width",
|
|
||||||
"yansi-term",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anstream"
|
name = "anstream"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
@@ -142,21 +132,6 @@ version = "0.7.6"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
|
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "ashpd"
|
|
||||||
version = "0.13.11"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "340e0f6bf7f9ee78549c61454f1460a3ed97c011902ee76b58301bbc6d502a32"
|
|
||||||
dependencies = [
|
|
||||||
"enumflags2",
|
|
||||||
"futures-util",
|
|
||||||
"getrandom 0.4.2",
|
|
||||||
"serde",
|
|
||||||
"serde_repr",
|
|
||||||
"tokio",
|
|
||||||
"zbus",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "askama"
|
name = "askama"
|
||||||
version = "0.14.0"
|
version = "0.14.0"
|
||||||
@@ -181,7 +156,7 @@ dependencies = [
|
|||||||
"memchr",
|
"memchr",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"rustc-hash 2.1.3",
|
"rustc-hash",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_derive",
|
"serde_derive",
|
||||||
"syn",
|
"syn",
|
||||||
@@ -365,7 +340,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "6bf601cccedfffec598ec2db1f9d6745885458bccc0e8916d7023f017c94b3d0"
|
checksum = "6bf601cccedfffec598ec2db1f9d6745885458bccc0e8916d7023f017c94b3d0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atspi-common",
|
"atspi-common",
|
||||||
"atspi-connection",
|
|
||||||
"atspi-proxies",
|
"atspi-proxies",
|
||||||
"zbus",
|
"zbus",
|
||||||
]
|
]
|
||||||
@@ -430,27 +404,6 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bindgen"
|
|
||||||
version = "0.69.5"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
|
|
||||||
dependencies = [
|
|
||||||
"annotate-snippets",
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"cexpr",
|
|
||||||
"clang-sys",
|
|
||||||
"itertools",
|
|
||||||
"lazy_static",
|
|
||||||
"lazycell",
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"regex",
|
|
||||||
"rustc-hash 1.1.0",
|
|
||||||
"shlex",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bit-set"
|
name = "bit-set"
|
||||||
version = "0.8.0"
|
version = "0.8.0"
|
||||||
@@ -582,19 +535,6 @@ version = "1.11.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
|
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "calloop"
|
|
||||||
version = "0.14.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"polling",
|
|
||||||
"rustix",
|
|
||||||
"slab",
|
|
||||||
"tracing",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "camino"
|
name = "camino"
|
||||||
version = "1.2.4"
|
version = "1.2.4"
|
||||||
@@ -658,25 +598,6 @@ dependencies = [
|
|||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "cexpr"
|
|
||||||
version = "0.6.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
|
||||||
dependencies = [
|
|
||||||
"nom 7.1.3",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "cfg-expr"
|
|
||||||
version = "0.15.8"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02"
|
|
||||||
dependencies = [
|
|
||||||
"smallvec",
|
|
||||||
"target-lexicon",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cfg-if"
|
name = "cfg-if"
|
||||||
version = "1.0.4"
|
version = "1.0.4"
|
||||||
@@ -717,17 +638,6 @@ dependencies = [
|
|||||||
"phf",
|
"phf",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "clang-sys"
|
|
||||||
version = "1.8.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
|
|
||||||
dependencies = [
|
|
||||||
"glob",
|
|
||||||
"libc",
|
|
||||||
"libloading",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "4.6.1"
|
version = "4.6.1"
|
||||||
@@ -781,7 +691,6 @@ dependencies = [
|
|||||||
"objc2-foundation 0.3.2",
|
"objc2-foundation 0.3.2",
|
||||||
"objc2-ui-kit",
|
"objc2-ui-kit",
|
||||||
"windows 0.59.0",
|
"windows 0.59.0",
|
||||||
"wl-clipboard-rs",
|
|
||||||
"x11rb",
|
"x11rb",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -819,15 +728,6 @@ dependencies = [
|
|||||||
"crossbeam-utils",
|
"crossbeam-utils",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "convert_case"
|
|
||||||
version = "0.6.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
|
|
||||||
dependencies = [
|
|
||||||
"unicode-segmentation",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cookie"
|
name = "cookie"
|
||||||
version = "0.18.1"
|
version = "0.18.1"
|
||||||
@@ -839,15 +739,6 @@ dependencies = [
|
|||||||
"version_check",
|
"version_check",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "cookie-factory"
|
|
||||||
version = "0.3.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2"
|
|
||||||
dependencies = [
|
|
||||||
"futures",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cookie_store"
|
name = "cookie_store"
|
||||||
version = "0.22.1"
|
version = "0.22.1"
|
||||||
@@ -939,15 +830,6 @@ version = "1.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
|
checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "crossbeam-channel"
|
|
||||||
version = "0.5.15"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
|
|
||||||
dependencies = [
|
|
||||||
"crossbeam-utils",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crossbeam-queue"
|
name = "crossbeam-queue"
|
||||||
version = "0.3.12"
|
version = "0.3.12"
|
||||||
@@ -981,7 +863,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cua-driver"
|
name = "cua-driver"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1017,7 +899,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cua-driver-bindgen"
|
name = "cua-driver-bindgen"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cbindgen",
|
"cbindgen",
|
||||||
"uniffi",
|
"uniffi",
|
||||||
@@ -1025,7 +907,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cua-driver-contract"
|
name = "cua-driver-contract"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"schemars",
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -1035,7 +917,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cua-driver-core"
|
name = "cua-driver-core"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1053,6 +935,7 @@ dependencies = [
|
|||||||
"sha2",
|
"sha2",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
|
"time",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-tungstenite",
|
"tokio-tungstenite",
|
||||||
"tracing",
|
"tracing",
|
||||||
@@ -1064,7 +947,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cua-driver-sdk"
|
name = "cua-driver-sdk"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"core-foundation",
|
"core-foundation",
|
||||||
@@ -1086,7 +969,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cua-driver-testkit"
|
name = "cua-driver-testkit"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"core-foundation",
|
"core-foundation",
|
||||||
"libc",
|
"libc",
|
||||||
@@ -1100,7 +983,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cua-driver-uia"
|
name = "cua-driver-uia"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"cua-driver-core",
|
"cua-driver-core",
|
||||||
@@ -1115,7 +998,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cursor-overlay"
|
name = "cursor-overlay"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"cua-driver-contract",
|
"cua-driver-contract",
|
||||||
@@ -1132,7 +1015,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cursor-theme-cli"
|
name = "cursor-theme-cli"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"cursor-overlay",
|
"cursor-overlay",
|
||||||
@@ -1170,27 +1053,6 @@ dependencies = [
|
|||||||
"crypto-common",
|
"crypto-common",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "dirs"
|
|
||||||
version = "5.0.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
|
|
||||||
dependencies = [
|
|
||||||
"dirs-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "dirs-sys"
|
|
||||||
version = "0.4.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"option-ext",
|
|
||||||
"redox_users",
|
|
||||||
"windows-sys 0.48.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dispatch2"
|
name = "dispatch2"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
@@ -1231,24 +1093,12 @@ dependencies = [
|
|||||||
"futures-util",
|
"futures-util",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "downcast-rs"
|
|
||||||
version = "1.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dyn-clone"
|
name = "dyn-clone"
|
||||||
version = "1.0.20"
|
version = "1.0.20"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "either"
|
|
||||||
version = "1.16.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "email_address"
|
name = "email_address"
|
||||||
version = "0.2.9"
|
version = "0.2.9"
|
||||||
@@ -1348,7 +1198,7 @@ dependencies = [
|
|||||||
"bitvec",
|
"bitvec",
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"libc",
|
"libc",
|
||||||
"nix 0.23.2",
|
"nix",
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1421,12 +1271,6 @@ version = "0.1.9"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fixedbitset"
|
|
||||||
version = "0.5.7"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "flate2"
|
name = "flate2"
|
||||||
version = "1.1.9"
|
version = "1.1.9"
|
||||||
@@ -1552,48 +1396,12 @@ version = "2.0.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "futures"
|
|
||||||
version = "0.3.32"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
|
|
||||||
dependencies = [
|
|
||||||
"futures-channel",
|
|
||||||
"futures-core",
|
|
||||||
"futures-executor",
|
|
||||||
"futures-io",
|
|
||||||
"futures-sink",
|
|
||||||
"futures-task",
|
|
||||||
"futures-util",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "futures-channel"
|
|
||||||
version = "0.3.32"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
|
|
||||||
dependencies = [
|
|
||||||
"futures-core",
|
|
||||||
"futures-sink",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-core"
|
name = "futures-core"
|
||||||
version = "0.3.32"
|
version = "0.3.32"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
|
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "futures-executor"
|
|
||||||
version = "0.3.32"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
|
|
||||||
dependencies = [
|
|
||||||
"futures-core",
|
|
||||||
"futures-task",
|
|
||||||
"futures-util",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-io"
|
name = "futures-io"
|
||||||
version = "0.3.32"
|
version = "0.3.32"
|
||||||
@@ -1642,13 +1450,10 @@ version = "0.3.32"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"futures-channel",
|
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-io",
|
|
||||||
"futures-macro",
|
"futures-macro",
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
"futures-task",
|
"futures-task",
|
||||||
"memchr",
|
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"slab",
|
"slab",
|
||||||
]
|
]
|
||||||
@@ -2008,15 +1813,6 @@ version = "1.70.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "itertools"
|
|
||||||
version = "0.12.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
|
|
||||||
dependencies = [
|
|
||||||
"either",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "itoa"
|
name = "itoa"
|
||||||
version = "1.0.18"
|
version = "1.0.18"
|
||||||
@@ -2087,12 +1883,6 @@ version = "1.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "lazycell"
|
|
||||||
version = "1.3.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "leb128fmt"
|
name = "leb128fmt"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -2105,53 +1895,6 @@ version = "0.2.186"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "libloading"
|
|
||||||
version = "0.8.9"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
"windows-link 0.2.1",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "libredox"
|
|
||||||
version = "0.1.17"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "libspa"
|
|
||||||
version = "0.8.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "65f3a4b81b2a2d8c7f300643676202debd1b7c929dbf5c9bb89402ea11d19810"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"cc",
|
|
||||||
"convert_case",
|
|
||||||
"cookie-factory",
|
|
||||||
"libc",
|
|
||||||
"libspa-sys",
|
|
||||||
"nix 0.27.1",
|
|
||||||
"nom 7.1.3",
|
|
||||||
"system-deps",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "libspa-sys"
|
|
||||||
version = "0.8.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "bf0d9716420364790e85cbb9d3ac2c950bde16a7dd36f3209b7dfdfc4a24d01f"
|
|
||||||
dependencies = [
|
|
||||||
"bindgen",
|
|
||||||
"cc",
|
|
||||||
"system-deps",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "linux-raw-sys"
|
name = "linux-raw-sys"
|
||||||
version = "0.12.1"
|
version = "0.12.1"
|
||||||
@@ -2325,17 +2068,6 @@ dependencies = [
|
|||||||
"memoffset 0.6.5",
|
"memoffset 0.6.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "nix"
|
|
||||||
version = "0.27.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"cfg-if",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nom"
|
name = "nom"
|
||||||
version = "1.2.4"
|
version = "1.2.4"
|
||||||
@@ -2352,15 +2084,6 @@ dependencies = [
|
|||||||
"minimal-lexical",
|
"minimal-lexical",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "nom"
|
|
||||||
version = "8.0.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
|
|
||||||
dependencies = [
|
|
||||||
"memchr",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nu-ansi-term"
|
name = "nu-ansi-term"
|
||||||
version = "0.50.3"
|
version = "0.50.3"
|
||||||
@@ -2796,12 +2519,6 @@ dependencies = [
|
|||||||
"vcpkg",
|
"vcpkg",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "option-ext"
|
|
||||||
version = "0.2.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ordered-stream"
|
name = "ordered-stream"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
@@ -2812,16 +2529,6 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "os_pipe"
|
|
||||||
version = "1.2.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"windows-sys 0.61.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "outref"
|
name = "outref"
|
||||||
version = "0.5.2"
|
version = "0.5.2"
|
||||||
@@ -2863,17 +2570,6 @@ version = "2.3.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "petgraph"
|
|
||||||
version = "0.8.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
|
|
||||||
dependencies = [
|
|
||||||
"fixedbitset",
|
|
||||||
"hashbrown 0.15.5",
|
|
||||||
"indexmap",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "phf"
|
name = "phf"
|
||||||
version = "0.12.1"
|
version = "0.12.1"
|
||||||
@@ -2900,7 +2596,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pip-preview"
|
name = "pip-preview"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -2918,34 +2614,6 @@ dependencies = [
|
|||||||
"futures-io",
|
"futures-io",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pipewire"
|
|
||||||
version = "0.8.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "08e645ba5c45109106d56610b3ee60eb13a6f2beb8b74f8dc8186cf261788dda"
|
|
||||||
dependencies = [
|
|
||||||
"anyhow",
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"libc",
|
|
||||||
"libspa",
|
|
||||||
"libspa-sys",
|
|
||||||
"nix 0.27.1",
|
|
||||||
"once_cell",
|
|
||||||
"pipewire-sys",
|
|
||||||
"thiserror 1.0.69",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pipewire-sys"
|
|
||||||
version = "0.8.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "849e188f90b1dda88fe2bfe1ad31fe5f158af2c98f80fb5d13726c44f3f01112"
|
|
||||||
dependencies = [
|
|
||||||
"bindgen",
|
|
||||||
"libspa-sys",
|
|
||||||
"system-deps",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pkg-config"
|
name = "pkg-config"
|
||||||
version = "0.3.33"
|
version = "0.3.33"
|
||||||
@@ -2960,53 +2628,39 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "platform-linux"
|
name = "platform-linux"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"ashpd",
|
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"atspi",
|
"atspi",
|
||||||
|
"atspi-connection",
|
||||||
"base64",
|
"base64",
|
||||||
"calloop",
|
|
||||||
"clipboard-rs",
|
"clipboard-rs",
|
||||||
"crossbeam-channel",
|
|
||||||
"cua-driver-contract",
|
"cua-driver-contract",
|
||||||
"cua-driver-core",
|
"cua-driver-core",
|
||||||
"cursor-overlay",
|
"cursor-overlay",
|
||||||
"dirs",
|
|
||||||
"enumflags2",
|
|
||||||
"evdev",
|
"evdev",
|
||||||
"image",
|
"image",
|
||||||
"libc",
|
"libc",
|
||||||
"libspa",
|
|
||||||
"memmap2",
|
"memmap2",
|
||||||
"meval",
|
"meval",
|
||||||
"pip-preview",
|
"pip-preview",
|
||||||
"pipewire",
|
|
||||||
"reis",
|
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"sha2",
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
"tiny-skia",
|
"tiny-skia",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
|
||||||
"url",
|
|
||||||
"uuid",
|
"uuid",
|
||||||
"wayland-backend",
|
|
||||||
"wayland-client",
|
|
||||||
"wayland-protocols",
|
|
||||||
"wayland-protocols-wlr",
|
|
||||||
"wayland-scanner",
|
|
||||||
"x11",
|
"x11",
|
||||||
"x11rb",
|
"x11rb",
|
||||||
"xkbcommon",
|
|
||||||
"zbus",
|
"zbus",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "platform-macos"
|
name = "platform-macos"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -3042,7 +2696,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "platform-windows"
|
name = "platform-windows"
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -3056,6 +2710,7 @@ dependencies = [
|
|||||||
"pip-preview",
|
"pip-preview",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"tempfile",
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
"tiny-skia",
|
"tiny-skia",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -3279,17 +2934,6 @@ dependencies = [
|
|||||||
"bitflags 2.11.1",
|
"bitflags 2.11.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "redox_users"
|
|
||||||
version = "0.4.6"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
|
|
||||||
dependencies = [
|
|
||||||
"getrandom 0.2.17",
|
|
||||||
"libredox",
|
|
||||||
"thiserror 1.0.69",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ref-cast"
|
name = "ref-cast"
|
||||||
version = "1.0.25"
|
version = "1.0.25"
|
||||||
@@ -3389,19 +3033,6 @@ dependencies = [
|
|||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "reis"
|
|
||||||
version = "0.7.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "81f3fedd2777cde52c1be5e572efbec485eac7b801c47820eda388d4f13b9c4b"
|
|
||||||
dependencies = [
|
|
||||||
"enumflags2",
|
|
||||||
"futures-util",
|
|
||||||
"log",
|
|
||||||
"rustix",
|
|
||||||
"tokio",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ring"
|
name = "ring"
|
||||||
version = "0.17.14"
|
version = "0.17.14"
|
||||||
@@ -3416,12 +3047,6 @@ dependencies = [
|
|||||||
"windows-sys 0.52.0",
|
"windows-sys 0.52.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rustc-hash"
|
|
||||||
version = "1.1.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc-hash"
|
name = "rustc-hash"
|
||||||
version = "2.1.3"
|
version = "2.1.3"
|
||||||
@@ -3863,19 +3488,6 @@ dependencies = [
|
|||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "system-deps"
|
|
||||||
version = "6.2.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-expr",
|
|
||||||
"heck",
|
|
||||||
"pkg-config",
|
|
||||||
"toml 0.8.23",
|
|
||||||
"version-compare",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tap"
|
name = "tap"
|
||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
@@ -3892,12 +3504,6 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "target-lexicon"
|
|
||||||
version = "0.12.16"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tempfile"
|
name = "tempfile"
|
||||||
version = "3.27.0"
|
version = "3.27.0"
|
||||||
@@ -4209,7 +3815,6 @@ version = "0.1.44"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
|
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"log",
|
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"tracing-attributes",
|
"tracing-attributes",
|
||||||
"tracing-core",
|
"tracing-core",
|
||||||
@@ -4265,17 +3870,6 @@ dependencies = [
|
|||||||
"tracing-log",
|
"tracing-log",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tree_magic_mini"
|
|
||||||
version = "3.2.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6"
|
|
||||||
dependencies = [
|
|
||||||
"memchr",
|
|
||||||
"nom 8.0.0",
|
|
||||||
"petgraph",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ttf-parser"
|
name = "ttf-parser"
|
||||||
version = "0.21.1"
|
version = "0.21.1"
|
||||||
@@ -4342,18 +3936,6 @@ version = "1.0.24"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "unicode-segmentation"
|
|
||||||
version = "1.13.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "unicode-width"
|
|
||||||
version = "0.1.14"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-xid"
|
name = "unicode-xid"
|
||||||
version = "0.2.6"
|
version = "0.2.6"
|
||||||
@@ -4597,12 +4179,6 @@ version = "0.2.15"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "version-compare"
|
|
||||||
version = "0.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "version_check"
|
name = "version_check"
|
||||||
version = "0.9.5"
|
version = "0.9.5"
|
||||||
@@ -4738,76 +4314,6 @@ dependencies = [
|
|||||||
"semver",
|
"semver",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wayland-backend"
|
|
||||||
version = "0.3.15"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d"
|
|
||||||
dependencies = [
|
|
||||||
"cc",
|
|
||||||
"downcast-rs",
|
|
||||||
"rustix",
|
|
||||||
"smallvec",
|
|
||||||
"wayland-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wayland-client"
|
|
||||||
version = "0.31.14"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"rustix",
|
|
||||||
"wayland-backend",
|
|
||||||
"wayland-scanner",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wayland-protocols"
|
|
||||||
version = "0.32.12"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"wayland-backend",
|
|
||||||
"wayland-client",
|
|
||||||
"wayland-scanner",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wayland-protocols-wlr"
|
|
||||||
version = "0.3.12"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.11.1",
|
|
||||||
"wayland-backend",
|
|
||||||
"wayland-client",
|
|
||||||
"wayland-protocols",
|
|
||||||
"wayland-scanner",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wayland-scanner"
|
|
||||||
version = "0.31.10"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quick-xml",
|
|
||||||
"quote",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wayland-sys"
|
|
||||||
version = "0.31.11"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be"
|
|
||||||
dependencies = [
|
|
||||||
"pkg-config",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "webpki-roots"
|
name = "webpki-roots"
|
||||||
version = "1.0.7"
|
version = "1.0.7"
|
||||||
@@ -4832,28 +4338,6 @@ version = "0.1.12"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
|
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "winapi"
|
|
||||||
version = "0.3.9"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
|
||||||
dependencies = [
|
|
||||||
"winapi-i686-pc-windows-gnu",
|
|
||||||
"winapi-x86_64-pc-windows-gnu",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "winapi-i686-pc-windows-gnu"
|
|
||||||
version = "0.4.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "winapi-x86_64-pc-windows-gnu"
|
|
||||||
version = "0.4.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows"
|
name = "windows"
|
||||||
version = "0.58.0"
|
version = "0.58.0"
|
||||||
@@ -5422,24 +4906,6 @@ dependencies = [
|
|||||||
"wasmparser",
|
"wasmparser",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wl-clipboard-rs"
|
|
||||||
version = "0.9.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e9651471a32e87d96ef3a127715382b2d11cc7c8bb9822ded8a7cc94072eb0a3"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"log",
|
|
||||||
"os_pipe",
|
|
||||||
"rustix",
|
|
||||||
"thiserror 2.0.18",
|
|
||||||
"tree_magic_mini",
|
|
||||||
"wayland-backend",
|
|
||||||
"wayland-client",
|
|
||||||
"wayland-protocols",
|
|
||||||
"wayland-protocols-wlr",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "writeable"
|
name = "writeable"
|
||||||
version = "0.6.3"
|
version = "0.6.3"
|
||||||
@@ -5482,32 +4948,6 @@ version = "0.13.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd"
|
checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "xkbcommon"
|
|
||||||
version = "0.9.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a7a974f48060a14e95705c01f24ad9c3345022f4d97441b8a36beb7ed5c4a02d"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"memmap2",
|
|
||||||
"xkeysym",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "xkeysym"
|
|
||||||
version = "0.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "yansi-term"
|
|
||||||
version = "0.1.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1"
|
|
||||||
dependencies = [
|
|
||||||
"winapi",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "yoke"
|
name = "yoke"
|
||||||
version = "0.8.2"
|
version = "0.8.2"
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ members = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.20.11"
|
version = "0.26.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
authors = ["trycua"]
|
authors = ["trycua"]
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ implementations, testkit, and helper crates.
|
|||||||
| `cua-driver-core` | Shared protocol, tool models, config, and CDP helpers |
|
| `cua-driver-core` | Shared protocol, tool models, config, and CDP helpers |
|
||||||
| `platform-macos` | macOS AX, capture, input, browser, and daemon support |
|
| `platform-macos` | macOS AX, capture, input, browser, and daemon support |
|
||||||
| `platform-windows` | Windows UIA, capture, input, overlay, and diagnostics |
|
| `platform-windows` | Windows UIA, capture, input, overlay, and diagnostics |
|
||||||
| `platform-linux` | Linux AT-SPI, X11/Wayland, capture, and input support |
|
| `platform-linux` | Linux AT-SPI, X11, capture, and input support |
|
||||||
| `cua-driver-testkit` | Test-only helpers for spawning the daemon and parsing responses |
|
| `cua-driver-testkit` | Test-only helpers for spawning the daemon and parsing responses |
|
||||||
| `cua-driver-uia` | Windows UIAccess worker |
|
| `cua-driver-uia` | Windows UIAccess worker |
|
||||||
| `cursor-overlay` | Cross-platform semantic cursor renderer and bounded compiled-theme loader |
|
| `cursor-overlay` | Cross-platform semantic cursor renderer and bounded compiled-theme loader |
|
||||||
|
|||||||
@@ -419,7 +419,6 @@ capabilities, or existing-profile consent.
|
|||||||
| Chrome / Edge on Windows | Exact binding, refs, navigation, typing, trusted or explicit DOM click | Must run in an interactive user session, not Session 0 |
|
| Chrome / Edge on Windows | Exact binding, refs, navigation, typing, trusted or explicit DOM click | Must run in an interactive user session, not Session 0 |
|
||||||
| Chrome / Edge on macOS | Exact binding, refs, navigation, typing, explicit DOM click | Trusted standalone click refuses to preserve background posture |
|
| Chrome / Edge on macOS | Exact binding, refs, navigation, typing, explicit DOM click | Trusted standalone click refuses to preserve background posture |
|
||||||
| Chrome / Chromium on Linux X11 | Exact binding, refs, navigation, typing, explicit DOM click | Trusted standalone click refuses to preserve background posture |
|
| Chrome / Chromium on Linux X11 | Exact binding, refs, navigation, typing, explicit DOM click | Trusted standalone click refuses to preserve background posture |
|
||||||
| Chromium on validated Wayland setups | Exact binding only when compositor identity is provable | Generic/ambiguous compositor identity refuses mutation |
|
|
||||||
| Electron | Exact single-page routes where endpoint and host relationship are proven | Do not infer support for arbitrary embedded webviews |
|
| Electron | Exact single-page routes where endpoint and host relationship are proven | Do not infer support for arbitrary embedded webviews |
|
||||||
| Safari / Firefox | Native window state only | Typed page mutation is not supported yet |
|
| Safari / Firefox | Native window state only | Typed page mutation is not supported yet |
|
||||||
| WebView2 / Tauri / other embedded webviews | Native AX/PX fallback unless an exact route is reported | Host/renderer correlation may refuse |
|
| WebView2 / Tauri / other embedded webviews | Native AX/PX fallback unless an exact route is reported | Host/renderer correlation may refuse |
|
||||||
|
|||||||
@@ -10,9 +10,6 @@ session recording.
|
|||||||
|
|
||||||
On X11, `set_window_frame({pid, window_id, x, y, width, height})` sends an
|
On X11, `set_window_frame({pid, window_id, x, y, width, height})` sends an
|
||||||
EWMH window-manager request and confirms it against `list_windows` geometry.
|
EWMH window-manager request and confirms it against `list_windows` geometry.
|
||||||
Wayland has no portable protocol for setting another application's top-level
|
|
||||||
geometry, so this tool refuses there unless a future compositor-owned adapter
|
|
||||||
can provide exact targeting and readback.
|
|
||||||
|
|
||||||
AT-SPI is talked to natively over D-Bus (the `atspi`/zbus crate) — no
|
AT-SPI is talked to natively over D-Bus (the `atspi`/zbus crate) — no
|
||||||
`pyatspi` or GObject-introspection typelibs are required at runtime.
|
`pyatspi` or GObject-introspection typelibs are required at runtime.
|
||||||
@@ -73,14 +70,9 @@ Keystroke / XSendEvent / XTest / foreground rungs return
|
|||||||
`effect:"unverifiable"` unless another publishable readback exists. Confirm
|
`effect:"unverifiable"` unless another publishable readback exists. Confirm
|
||||||
those through `verify_state` or multimodal reading.
|
those through `verify_state` or multimodal reading.
|
||||||
|
|
||||||
**Escalation** — action responses use the cross-platform closed
|
**Escalation** — use pixels for a non-AX surface when supported. Retry in
|
||||||
`escalation:{target, reason}` shape. See `SKILL.md` → behavior matrix. On a
|
foreground only after a pre-dispatch `background_unavailable` refusal.
|
||||||
standard Wayland compositor the Linux-specific target is **`foreground`**
|
An unconfirmed dispatched action must be observed before any further action.
|
||||||
(raw background pixels cannot target an unfocused window); the opt-in nested
|
|
||||||
compositor is a separate environment. Use **`pixel` on X11** (an element px
|
|
||||||
action — background pixel click — lands via
|
|
||||||
AT-SPI `do_action`-at-point off the screenshot already in the snapshot — the
|
|
||||||
matrix below).
|
|
||||||
|
|
||||||
## Perception and the ax/px action choice
|
## Perception and the ax/px action choice
|
||||||
|
|
||||||
@@ -110,10 +102,7 @@ Modality is chosen at **action time**, by how you address the target:
|
|||||||
already in the `get_window_state` response. Best-effort; caller-confirmed.
|
already in the `get_window_state` response. Best-effort; caller-confirmed.
|
||||||
|
|
||||||
`get_window_state` returning `degraded:true` (empty AT-SPI walk) is the cue to
|
`get_window_state` returning `degraded:true` (empty AT-SPI walk) is the cue to
|
||||||
do an **element px action** off that same screenshot (X11) or escalate to
|
use pixels from the same screenshot when the target supports that route.
|
||||||
`delivery_mode:"foreground"` (standard Wayland: raw background pixels cannot
|
|
||||||
target an unfocused window there). The nested compositor has its own
|
|
||||||
experimental per-surface routes.
|
|
||||||
|
|
||||||
## Cross-platform schema residuals (Linux)
|
## Cross-platform schema residuals (Linux)
|
||||||
|
|
||||||
@@ -141,7 +130,7 @@ Use `invoke_menu({pid, window_id, path:[...]})` for a known GTK/Qt application
|
|||||||
menu command. It activates the exact target only for the duration of the
|
menu command. It activates the exact target only for the duration of the
|
||||||
operation, resolves each labelled AT-SPI menu descendant again after the prior
|
operation, resolves each labelled AT-SPI menu descendant again after the prior
|
||||||
menu expands, and refuses missing, duplicate, disabled, or non-actionable
|
menu expands, and refuses missing, duplicate, disabled, or non-actionable
|
||||||
segments. It works through AT-SPI on both X11 and Wayland and never falls back
|
segments. It works through AT-SPI on X11 and never falls back
|
||||||
to coordinates. Verify the command's semantic effect from fresh state; the
|
to coordinates. Verify the command's semantic effect from fresh state; the
|
||||||
native `do_action` acknowledgement alone is not task completion.
|
native `do_action` acknowledgement alone is not task completion.
|
||||||
|
|
||||||
@@ -200,49 +189,23 @@ uinput, escalate to `delivery_mode:"foreground"`. (`type_text` in the
|
|||||||
`background` rung is focus-dependent for non-editable widgets; that's the one
|
`background` rung is focus-dependent for non-editable widgets; that's the one
|
||||||
genuine background limitation, and `foreground` is the documented escalation.)
|
genuine background limitation, and `foreground` is the documented escalation.)
|
||||||
|
|
||||||
## Wayland
|
## Display support
|
||||||
|
|
||||||
Set `CUA_DRIVER_RS_ENABLE_WAYLAND=1` to enable native Wayland support. The
|
Only X11 is supported. Native Wayland backends, compositor helpers, portal
|
||||||
driver selects a backend from compositor capabilities:
|
input and their opt-in variables have been removed. X11 applications under
|
||||||
|
XWayland remain supported through their authenticated DISPLAY; native Wayland
|
||||||
- Sway and other wlroots compositors use foreign-toplevel discovery,
|
applications cannot be targeted. No DISPLAY produces an explicit X11-only error.
|
||||||
wlr-screencopy, virtual pointer, and virtual keyboard protocols.
|
|
||||||
- GNOME/Mutter uses the bundled WinRects Shell helper for target geometry and
|
|
||||||
activation, plus portal/libei for foreground raw input.
|
|
||||||
- KDE/KWin uses AT-SPI and portal facilities where available. Target-specific
|
|
||||||
foreground activation remains experimental, so unsafe raw input refuses.
|
|
||||||
- The optional `cua-compositor` is a separate nested session enabled
|
|
||||||
explicitly for controlled automation. GNOME and KDE never switch into it.
|
|
||||||
|
|
||||||
Sway recording works through the wlroots recorder path and is exercised by the
|
|
||||||
canonical harness runner. Portal-backed GNOME recording is still an evidence
|
|
||||||
gap. Capture and recording availability therefore depend on the compositor,
|
|
||||||
installed helpers, and portal grant.
|
|
||||||
|
|
||||||
Standard Wayland has no general client protocol for raw input to an arbitrary
|
|
||||||
occluded surface. Background AX actions can still deliver through AT-SPI, and
|
|
||||||
a PX left click can deliver when hit-testing resolves to an actionable AT-SPI
|
|
||||||
control. Other focus-bound background pointer and keyboard shapes return an
|
|
||||||
exact `background_unavailable` result. They do not report success after a
|
|
||||||
silent drop.
|
|
||||||
|
|
||||||
Use `delivery_mode:"foreground"` for raw Wayland input. The driver activates
|
|
||||||
the selected target through a verified compositor adapter before dispatch. If
|
|
||||||
the compositor has no target-addressable activation or input backend, the call
|
|
||||||
refuses before sending input. Reconstructing coordinates alone does not make
|
|
||||||
raw background PX possible on a standard compositor.
|
|
||||||
|
|
||||||
## Quick triage
|
## Quick triage
|
||||||
|
|
||||||
If a tool call surprises you on Linux:
|
If a tool call surprises you on Linux:
|
||||||
|
|
||||||
1. `qwen-cua-driver doctor` — reports the display server (X11 / Wayland),
|
1. `qwen-cua-driver doctor` — reports X11 availability,
|
||||||
**whether `org.a11y.Bus` actually answers on the session bus** (not just
|
**whether `org.a11y.Bus` actually answers on the session bus** (not just
|
||||||
"is there a bus"), the discovered `DBUS_SESSION_BUS_ADDRESS`, and
|
"is there a bus"), the discovered `DBUS_SESSION_BUS_ADDRESS`, and
|
||||||
`ffmpeg` availability (for recording).
|
`ffmpeg` availability (for recording).
|
||||||
2. Check `XDG_SESSION_TYPE` — `x11` is fully supported; `wayland`
|
2. Check `DISPLAY` and `XAUTHORITY`. A usable X11 display takes precedence
|
||||||
needs `CUA_DRIVER_RS_ENABLE_WAYLAND=1` for the native backend,
|
regardless of the surrounding session's `XDG_SESSION_TYPE`.
|
||||||
else XWayland.
|
|
||||||
3. **Empty AT-SPI tree** (`get_window_state` returns `degraded:true`) — in
|
3. **Empty AT-SPI tree** (`get_window_state` returns `degraded:true`) — in
|
||||||
order of likelihood: (a) the daemon isn't on the desktop session bus
|
order of likelihood: (a) the daemon isn't on the desktop session bus
|
||||||
(headless / container / `runuser` / root-against-user-session — see
|
(headless / container / `runuser` / root-against-user-session — see
|
||||||
@@ -269,10 +232,6 @@ ask the user.
|
|||||||
| Environment | Proven baseline | Main limits |
|
| Environment | Proven baseline | Main limits |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| X11/Openbox | AT-SPI trees and actions, foreground pointer and keyboard input, window and desktop capture, and video | Raw background delivery remains toolkit-specific; unsupported shapes refuse |
|
| X11/Openbox | AT-SPI trees and actions, foreground pointer and keyboard input, window and desktop capture, and video | Raw background delivery remains toolkit-specific; unsupported shapes refuse |
|
||||||
| Sway/wlroots | AT-SPI, native discovery, full-display and cropped-window screencopy, foreground input, semantic background actions, and video | Raw background pointer and keyboard input remains focus-bound |
|
|
||||||
| GNOME/Mutter | AT-SPI, WinRects geometry and activation, capture, and portal/libei foreground input | Requires the helper and portal grant; portal video parity remains open |
|
|
||||||
| KDE/KWin | AT-SPI and generic discovery where exposed | Target-specific activation and behavioral coverage remain experimental |
|
|
||||||
| Nested `cua-compositor` | Versioned direct per-surface input, native GTK 31/31, capture/scope 5/5, and partial Electron coverage | The complete shared matrix remains experimental; do not infer standard-Wayland support |
|
|
||||||
|
|
||||||
See `SKILL.md` for the cross-platform loop (snapshot-before-AND-after,
|
See `SKILL.md` for the cross-platform loop (snapshot-before-AND-after,
|
||||||
pixel-click contract, failure modes) and `RECORDING.md` for session
|
pixel-click contract, failure modes) and `RECORDING.md` for session
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
---
|
---
|
||||||
name: cua-driver
|
name: cua-driver
|
||||||
description: Drive a native GUI app (macOS, Windows, Linux) via the Qwen Cua Driver CLI (default) or MCP server; snapshot its accessibility tree, act through snapshot-bound element tokens, native menu paths, exact window geometry, or pixel coordinates, and verify from fresh state. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real application on the host.
|
description: Drive a native GUI app (macOS, Windows, Linux) via the Qwen Cua Driver CLI (default) or MCP server; snapshot its accessibility tree, act through snapshot-bound element tokens, native menu paths, exact window geometry, or pixel coordinates, and verify from fresh state. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real application on the host.
|
||||||
version: 0.20.11 # x-release-please-version
|
version: 0.26.0 # x-release-please-version
|
||||||
metadata:
|
metadata:
|
||||||
openclaw:
|
openclaw:
|
||||||
requires:
|
requires:
|
||||||
@@ -17,9 +17,6 @@ metadata:
|
|||||||
- name: CUA_DRIVER_PATH
|
- name: CUA_DRIVER_PATH
|
||||||
required: false
|
required: false
|
||||||
description: Optional path to a qwen-cua-driver binary used by an embedding host.
|
description: Optional path to a qwen-cua-driver binary used by an embedding host.
|
||||||
- name: CUA_DRIVER_RS_ENABLE_WAYLAND
|
|
||||||
required: false
|
|
||||||
description: Set to 1 to enable the native Wayland backend.
|
|
||||||
- name: CUA_DRIVER_RS_MCP_HTTP_PORT
|
- name: CUA_DRIVER_RS_MCP_HTTP_PORT
|
||||||
required: false
|
required: false
|
||||||
description: Optional port for the local MCP HTTP endpoint.
|
description: Optional port for the local MCP HTTP endpoint.
|
||||||
@@ -52,7 +49,7 @@ directory:
|
|||||||
ApplicationFrameHost hosting, layered UIA+PostMessage click chain,
|
ApplicationFrameHost hosting, layered UIA+PostMessage click chain,
|
||||||
Session 0 isolation, Windows-specific focus-steal vectors).
|
Session 0 isolation, Windows-specific focus-steal vectors).
|
||||||
- **Linux** — read `LINUX.md` (X11 background input via AT-SPI +
|
- **Linux** — read `LINUX.md` (X11 background input via AT-SPI +
|
||||||
XSendEvent and compositor-specific Wayland capabilities).
|
XSendEvent (including X11 apps under XWayland)).
|
||||||
|
|
||||||
Cross-cutting topics also have their own files:
|
Cross-cutting topics also have their own files:
|
||||||
|
|
||||||
@@ -552,13 +549,6 @@ surface like Electron/Chromium/canvas), returns `degraded: true`
|
|||||||
plus an observation-specific escalation hint — normally pointing at pixels (you
|
plus an observation-specific escalation hint — normally pointing at pixels (you
|
||||||
still have the screenshot from the same call to click off).
|
still have the screenshot from the same call to click off).
|
||||||
|
|
||||||
**Platform nuance for action escalation.** On **Wayland** an unfocused
|
|
||||||
window cannot be pixel-targeted in the background (libei →
|
|
||||||
`background_unavailable`), so the action target is
|
|
||||||
**`foreground`, not `pixel`**. macOS, X11, and most Windows surfaces
|
|
||||||
can pixel-target in the background, so they target `pixel`. See
|
|
||||||
`LINUX.md` / `WINDOWS.md`.
|
|
||||||
|
|
||||||
## The verify-then-escalate ladder (algorithm)
|
## The verify-then-escalate ladder (algorithm)
|
||||||
|
|
||||||
Every snapshot already hands you both the tree and the screenshot, so
|
Every snapshot already hands you both the tree and the screenshot, so
|
||||||
@@ -607,8 +597,6 @@ if resp.effect == "suspected_noop"
|
|||||||
if resp.escalation.target == "foreground"
|
if resp.escalation.target == "foreground"
|
||||||
or the px action still did nothing:
|
or the px action still did nothing:
|
||||||
re-call the same action with delivery_mode:"foreground"
|
re-call the same action with delivery_mode:"foreground"
|
||||||
# on Wayland this is the ONLY escalation — px-bg can't target an
|
|
||||||
# unfocused window there; see LINUX.md
|
|
||||||
verify again
|
verify again
|
||||||
|
|
||||||
# Route 5 — per-call desktop fallback
|
# Route 5 — per-call desktop fallback
|
||||||
@@ -817,7 +805,7 @@ the response's `snapshot_id` with `element_index`; bare indices fail closed in
|
|||||||
`list_windows.z_index` uses one portable convention: higher integer
|
`list_windows.z_index` uses one portable convention: higher integer
|
||||||
values are closer to the front. Select a frontmost candidate with the
|
values are closer to the front. Select a frontmost candidate with the
|
||||||
maximum non-null value. If all values are `null` (as they can be on
|
maximum non-null value. If all values are `null` (as they can be on
|
||||||
native Wayland), use an explicit fallback; never treat `null` as zero
|
unavailable stacking metadata), use an explicit fallback; never treat `null` as zero
|
||||||
or infer stacking from array order. The `windows` records returned by
|
or infer stacking from array order. The `windows` records returned by
|
||||||
`launch_app` use the same convention.
|
`launch_app` use the same convention.
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
0.20.11
|
0.26.0
|
||||||
@@ -20,17 +20,31 @@ fn run() -> Result<(), String> {
|
|||||||
.find(|arg| !arg.starts_with('-'))
|
.find(|arg| !arg.starts_with('-'))
|
||||||
.map(String::as_str)
|
.map(String::as_str)
|
||||||
.unwrap_or("all");
|
.unwrap_or("all");
|
||||||
if !matches!(target, "all" | "manifest") {
|
if !matches!(target, "all" | "manifest" | "computer-use") {
|
||||||
return Err("usage: cua-contract-gen [all|manifest] [--check]".into());
|
return Err("usage: cua-contract-gen [all|manifest|computer-use] [--check]".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut contents = serde_json::to_string_pretty(&manifest()).map_err(|e| e.to_string())?;
|
if target != "computer-use" {
|
||||||
contents.push('\n');
|
let mut contents = serde_json::to_string_pretty(&manifest()).map_err(|e| e.to_string())?;
|
||||||
write_or_check(
|
contents.push('\n');
|
||||||
&driver_root()?.join("contract/manifest.json"),
|
write_or_check(
|
||||||
&contents,
|
&driver_root()?.join("contract/manifest.json"),
|
||||||
check,
|
&contents,
|
||||||
)
|
check,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
if target != "manifest" {
|
||||||
|
let mut contents =
|
||||||
|
serde_json::to_string_pretty(&cua_driver_contract::computer_use::manifest())
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
|
contents.push('\n');
|
||||||
|
write_or_check(
|
||||||
|
&driver_root()?.join("typescript/computer-use/interface.json"),
|
||||||
|
&contents,
|
||||||
|
check,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn driver_root() -> Result<PathBuf, String> {
|
fn driver_root() -> Result<PathBuf, String> {
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
use serde::Serialize;
|
||||||
|
use serde_json::{json, Map, Value};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
pub struct ComputerUseTool {
|
||||||
|
pub name: &'static str,
|
||||||
|
pub method: &'static str,
|
||||||
|
pub description: &'static str,
|
||||||
|
pub read_only: bool,
|
||||||
|
pub input_schema: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tool(
|
||||||
|
name: &'static str,
|
||||||
|
method: &'static str,
|
||||||
|
description: &'static str,
|
||||||
|
read_only: bool,
|
||||||
|
parameters: Value,
|
||||||
|
required: &[&str],
|
||||||
|
) -> ComputerUseTool {
|
||||||
|
let mut properties = Map::from_iter([
|
||||||
|
(
|
||||||
|
"app".into(),
|
||||||
|
json!({"type":"string","minLength":1,"description":"Exact application name, identifier or absolute path from get_state/get_app."}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"session".into(),
|
||||||
|
json!({"type":"string","description":crate::MULTI_CALL_SESSION_DESCRIPTION}),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
if name == "get_state" {
|
||||||
|
properties.remove("app");
|
||||||
|
}
|
||||||
|
properties.extend(parameters.as_object().expect("object schema").clone());
|
||||||
|
let desktop = matches!(
|
||||||
|
name,
|
||||||
|
"get_screenshot"
|
||||||
|
| "click"
|
||||||
|
| "double_click"
|
||||||
|
| "right_click"
|
||||||
|
| "scroll"
|
||||||
|
| "drag"
|
||||||
|
| "type_text"
|
||||||
|
| "press_key"
|
||||||
|
| "hotkey"
|
||||||
|
| "key_down"
|
||||||
|
| "key_up"
|
||||||
|
);
|
||||||
|
if desktop {
|
||||||
|
properties.insert("scope".into(), json!({"type":"string","enum":["desktop"],"description":"Explicit foreground desktop input. Coordinates use the latest primary-display screenshot pixels. Omit app."}));
|
||||||
|
if matches!(name, "key_down" | "key_up") {
|
||||||
|
properties.remove("app");
|
||||||
|
} else {
|
||||||
|
properties.get_mut("app").unwrap()["description"] = json!(
|
||||||
|
"Exact application name, identifier or absolute path. Supply exactly one of app or scope:desktop."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let schema = json!({
|
||||||
|
"type":"object", "properties":properties,
|
||||||
|
"required":required.iter().filter(|field| !desktop || **field != "app").collect::<Vec<_>>(), "additionalProperties":false,
|
||||||
|
});
|
||||||
|
ComputerUseTool {
|
||||||
|
name,
|
||||||
|
method,
|
||||||
|
description,
|
||||||
|
read_only,
|
||||||
|
input_schema: schema,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn tools() -> Vec<ComputerUseTool> {
|
||||||
|
let observation = json!({
|
||||||
|
"force_full":{"type":"boolean","default":false},
|
||||||
|
"max_text_chars":{"type":"integer","minimum":512,"maximum":1000000,"default":12000},
|
||||||
|
"max_elements":{"type":"integer","minimum":1},
|
||||||
|
"max_depth":{"type":"integer","minimum":1},
|
||||||
|
});
|
||||||
|
let mut initial_observation = observation.clone();
|
||||||
|
initial_observation
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("force_full");
|
||||||
|
let point = json!({
|
||||||
|
"element":{"type":"integer","minimum":0,"description":"Short ID from the latest observation of this app window."},
|
||||||
|
"x":{"type":"number","description":"Window screenshot pixel x. Requires a current screenshot."},
|
||||||
|
"y":{"type":"number","description":"Window screenshot pixel y. Requires a current screenshot."},
|
||||||
|
});
|
||||||
|
let mut click = point.clone();
|
||||||
|
click["key"] = json!({"type":"array","items":{"type":"string"},"description":"Modifier keys held during the click, such as [\"shift\"] or [\"ctrl\",\"shift\"]."});
|
||||||
|
click["button"] = json!({"type":"string","enum":["left","middle","right"],"default":"left"});
|
||||||
|
click["count"] = json!({"type":"integer","minimum":1,"maximum":3,"default":1});
|
||||||
|
let mut scroll = point;
|
||||||
|
scroll["direction"] = json!({"type":"string","enum":["up","down","left","right"]});
|
||||||
|
scroll["amount"] = json!({"type":"integer","minimum":1,"maximum":50,"default":3});
|
||||||
|
let text = json!({"text":{"type":"string"}});
|
||||||
|
vec![
|
||||||
|
tool("get_state", "getState", "List applications and their windows. Use an exact app selector for subsequent calls.", true, json!({}), &[]),
|
||||||
|
tool("get_app", "getApp", "Resolve an application and return its full AX state without an image. A stopped, uniquely identified application may be opened for observation; actions never relaunch it.", false, initial_observation, &["app"]),
|
||||||
|
tool("get_ax_state", "getAXState", "Observe AX text without capturing pixels. Uses a diff after the first observation; force_full requests a complete view. Short IDs belong to this app window's current observation.", true, observation.clone(), &["app"]),
|
||||||
|
tool("get_screenshot", "getScreenshot", "Observe the current application window screenshot without an AX walk. Coordinates in actions use this image's pixels.", true, json!({}), &["app"]),
|
||||||
|
tool("get_ax_state_and_screenshot", "getAXStateAndScreenshot", "Observe AX text and the current window image together. Uses an AX diff unless force_full is set.", true, observation, &["app"]),
|
||||||
|
tool("click", "click", "Click a current short element ID or screenshot point. Optional key holds modifiers during the click. Observe the result before retrying an unconfirmed action.", false, click, &["app"]),
|
||||||
|
tool("double_click", "doubleClick", "Double-click a current short element ID or screenshot point, then observe the result.", false, json!({"element":{"type":"integer","minimum":0},"x":{"type":"number"},"y":{"type":"number"}}), &["app"]),
|
||||||
|
tool("right_click", "rightClick", "Right-click a current short element ID or screenshot point, then observe the result.", false, json!({"element":{"type":"integer","minimum":0},"x":{"type":"number"},"y":{"type":"number"}}), &["app"]),
|
||||||
|
tool("scroll", "scroll", "Scroll at a current element or screenshot point. Observe the result before sending another scroll.", false, scroll, &["app","direction"]),
|
||||||
|
tool("drag", "drag", "Drag between current window screenshot points. Unsupported background gestures refuse before sending input; use semantic value operations when available.", false, json!({"from_x":{"type":"number"},"from_y":{"type":"number"},"to_x":{"type":"number"},"to_y":{"type":"number"}}), &["app","from_x","from_y","to_x","to_y"]),
|
||||||
|
tool("type_text", "typeText", "Insert text at the current text caret. This does not replace the whole field; use set_value for replacement.", false, text.clone(), &["app","text"]),
|
||||||
|
tool("set_value", "setValue", "Replace the entire value of the current short element ID and read back the result. Refuses a non-settable element or a write that was not applied.", false, json!({"element":{"type":"integer","minimum":0},"value":{"type":"string"}}), &["app","element","value"]),
|
||||||
|
tool("press_key", "pressKey", "Press a key in the selected app. Optional modifiers form a chord. Observe before retrying an unconfirmed dispatch.", false, json!({"key":{"type":"string"},"modifiers":{"type":"array","items":{"type":"string"}}}), &["app","key"]),
|
||||||
|
tool("hotkey", "hotkey", "Press a modifier/key chord in the selected app.", false, json!({"keys":{"type":"array","minItems":1,"items":{"type":"string"}}}), &["app","keys"]),
|
||||||
|
tool("key_down", "desktop.keyDown", "Hold a desktop key for this session. Release with key_up in finally. Session end releases owned keys. While keys are held, use key_down/key_up for chords and desktop pointer actions without explicit modifiers.", false, json!({"key":{"type":"string","minLength":1}}), &["scope","key"]),
|
||||||
|
tool("key_up", "desktop.keyUp", "Release a desktop key held by this session. Another session cannot release it.", false, json!({"key":{"type":"string","minLength":1}}), &["scope","key"]),
|
||||||
|
tool("paste", "paste", "Paste text at the caret using a temporary clipboard value, then restore the previous clipboard even if the operation fails.", false, text, &["app","text"]),
|
||||||
|
tool("select_text", "selectText", "Select one unambiguous text match inside a current short element ID. Does not change the field value.", false, json!({"element":{"type":"integer","minimum":0},"text":{"type":"string"}}), &["app","element","text"]),
|
||||||
|
tool("perform_secondary_action", "performSecondaryAction", "Perform an explicitly named secondary accessibility action on a current short element ID.", false, json!({"element":{"type":"integer","minimum":0},"action":{"type":"string"}}), &["app","element","action"]),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn manifest() -> Value {
|
||||||
|
json!({"version":1,"tools":tools()})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Versioned dispatch acknowledgement for the app surface; no effect is claimed.
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct AppActionOutput {
|
||||||
|
pub computer_use: u8,
|
||||||
|
pub sent: bool,
|
||||||
|
}
|
||||||
|
impl AppActionOutput {
|
||||||
|
pub fn validate(&self) -> Result<(), String> {
|
||||||
|
if self.computer_use != 1 || !self.sent {
|
||||||
|
return Err("invalid Computer Use dispatch acknowledgement".into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_app_call(name: &str, args: &Value) -> bool {
|
||||||
|
matches!(
|
||||||
|
name,
|
||||||
|
"get_state" | "get_app" | "get_ax_state" | "get_screenshot" | "get_ax_state_and_screenshot"
|
||||||
|
) || (args.get("app").is_some() || args["scope"] == "desktop")
|
||||||
|
&& tools().iter().any(|tool| tool.name == name)
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@
|
|||||||
//! These types are transport-free. The contract generator derives JSON Schema
|
//! These types are transport-free. The contract generator derives JSON Schema
|
||||||
//! from them, and live Rust handlers deserialize the same types before acting.
|
//! from them, and live Rust handlers deserialize the same types before acting.
|
||||||
|
|
||||||
use crate::CursorThemeSelection;
|
use crate::{schema_settings, CursorThemeSelection};
|
||||||
use schemars::{json_schema, JsonSchema, Schema, SchemaGenerator};
|
use schemars::{json_schema, JsonSchema, Schema, SchemaGenerator};
|
||||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@@ -15,42 +15,89 @@ pub trait ToolInput: Serialize + DeserializeOwned + JsonSchema {
|
|||||||
const TOOL_NAME: &'static str;
|
const TOOL_NAME: &'static str;
|
||||||
|
|
||||||
fn input_schema() -> Value {
|
fn input_schema() -> Value {
|
||||||
let settings = schemars::generate::SchemaSettings::draft2020_12().with(|settings| {
|
let schema = schema_settings()
|
||||||
settings.meta_schema = None;
|
.into_generator()
|
||||||
settings.inline_subschemas = true;
|
.into_root_schema_for::<Self>();
|
||||||
});
|
|
||||||
let schema = settings.into_generator().into_root_schema_for::<Self>();
|
|
||||||
let mut value = serde_json::to_value(schema).expect("tool input schema serializes");
|
let mut value = serde_json::to_value(schema).expect("tool input schema serializes");
|
||||||
normalize_schema(&mut value);
|
normalize_input_schema(&mut value);
|
||||||
value
|
value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_schema(value: &mut Value) {
|
pub fn normalize_input_schema(value: &mut Value) {
|
||||||
match value {
|
let Some(object) = value.as_object_mut() else {
|
||||||
Value::Object(object) => {
|
return;
|
||||||
object.remove("title");
|
};
|
||||||
if object.get("type").and_then(Value::as_str) == Some("object") {
|
object.remove("title");
|
||||||
object
|
if object.get("default").is_some_and(Value::is_null) {
|
||||||
.entry("properties")
|
object.remove("default");
|
||||||
.or_insert_with(|| Value::Object(serde_json::Map::new()));
|
}
|
||||||
object
|
if let Some(types) = object.get("type").and_then(Value::as_array) {
|
||||||
.entry("required")
|
let kept: Vec<_> = types
|
||||||
.or_insert_with(|| Value::Array(Vec::new()));
|
.iter()
|
||||||
object
|
.filter(|kind| kind.as_str() != Some("null"))
|
||||||
.entry("additionalProperties")
|
.cloned()
|
||||||
.or_insert(Value::Bool(true));
|
.collect();
|
||||||
|
if let [kind] = kept.as_slice() {
|
||||||
|
object.insert("type".into(), kind.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(variants) = object.get("anyOf").and_then(Value::as_array) {
|
||||||
|
let kept: Vec<_> = variants
|
||||||
|
.iter()
|
||||||
|
.filter(|schema| schema["type"] != "null")
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
if let [Value::Object(schema)] = kept.as_slice() {
|
||||||
|
let schema = schema.clone();
|
||||||
|
object.remove("anyOf");
|
||||||
|
object.extend(schema);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(constant) = object.remove("const") {
|
||||||
|
match constant {
|
||||||
|
Value::String(text) => {
|
||||||
|
object.insert("type".into(), serde_json::json!("string"));
|
||||||
|
object.insert("enum".into(), serde_json::json!([text]));
|
||||||
}
|
}
|
||||||
for child in object.values_mut() {
|
Value::Number(number) => {
|
||||||
normalize_schema(child);
|
object.insert("minimum".into(), serde_json::json!(number));
|
||||||
|
object.insert("maximum".into(), serde_json::json!(number));
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
object.insert("const".into(), other);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Value::Array(values) => {
|
}
|
||||||
for child in values {
|
if object.get("type").and_then(Value::as_str) == Some("object") {
|
||||||
normalize_schema(child);
|
object
|
||||||
|
.entry("properties")
|
||||||
|
.or_insert_with(|| serde_json::json!({}));
|
||||||
|
object
|
||||||
|
.entry("required")
|
||||||
|
.or_insert_with(|| serde_json::json!([]));
|
||||||
|
object
|
||||||
|
.entry("additionalProperties")
|
||||||
|
.or_insert(Value::Bool(true));
|
||||||
|
}
|
||||||
|
for key in ["properties", "$defs"] {
|
||||||
|
if let Some(children) = object.get_mut(key).and_then(Value::as_object_mut) {
|
||||||
|
for child in children.values_mut() {
|
||||||
|
normalize_input_schema(child);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key in ["items", "additionalProperties"] {
|
||||||
|
if let Some(child) = object.get_mut(key).filter(|child| child.is_object()) {
|
||||||
|
normalize_input_schema(child);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key in ["anyOf", "oneOf", "allOf"] {
|
||||||
|
if let Some(children) = object.get_mut(key).and_then(Value::as_array_mut) {
|
||||||
|
for child in children {
|
||||||
|
normalize_input_schema(child);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -246,23 +293,38 @@ fn desktop_scope_schema(generator: &mut SchemaGenerator) -> Schema {
|
|||||||
/// `display_id="primary"` is the portable desktop target in this release.
|
/// `display_id="primary"` is the portable desktop target in this release.
|
||||||
/// Platforms that cannot address another display reject it explicitly rather
|
/// Platforms that cannot address another display reject it explicitly rather
|
||||||
/// than silently changing coordinate spaces.
|
/// than silently changing coordinate spaces.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, uniffi::Enum)]
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
pub enum ActionTarget {
|
pub enum ActionTarget {
|
||||||
Window { pid: u32, window_id: u64 },
|
Window { pid: u32, window_id: u64 },
|
||||||
Desktop { display_id: String },
|
Desktop { display_id: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl JsonSchema for ActionTarget {
|
||||||
|
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||||
|
"ActionTarget".into()
|
||||||
|
}
|
||||||
|
fn json_schema(_: &mut SchemaGenerator) -> Schema {
|
||||||
|
json_schema!({
|
||||||
|
"type": "object",
|
||||||
|
"description": "Exact window or desktop target. Window requires pid and window_id; desktop requires display_id.",
|
||||||
|
"required": ["kind"],
|
||||||
|
"properties": {
|
||||||
|
"kind": {"type":"string","enum":["window","desktop"]},
|
||||||
|
"pid": {"type":"integer","minimum":0},
|
||||||
|
"window_id": {"type":"integer","minimum":0},
|
||||||
|
"display_id": {"type":"string","description":"Use primary for the primary display."}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn action_target_schema() -> Value {
|
pub fn action_target_schema() -> Value {
|
||||||
let settings = schemars::generate::SchemaSettings::draft2020_12().with(|settings| {
|
let schema = schema_settings()
|
||||||
settings.meta_schema = None;
|
|
||||||
settings.inline_subschemas = true;
|
|
||||||
});
|
|
||||||
let schema = settings
|
|
||||||
.into_generator()
|
.into_generator()
|
||||||
.into_root_schema_for::<ActionTarget>();
|
.into_root_schema_for::<ActionTarget>();
|
||||||
let mut value = serde_json::to_value(schema).expect("action target schema serializes");
|
let mut value = serde_json::to_value(schema).expect("action target schema serializes");
|
||||||
normalize_schema(&mut value);
|
normalize_input_schema(&mut value);
|
||||||
value
|
value
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1275,14 +1337,39 @@ mod tests {
|
|||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn input_schema_normalization_keeps_property_names_and_literal_values() {
|
||||||
|
let mut schema = json!({
|
||||||
|
"type":"object", "title":"Drop annotation", "properties": {
|
||||||
|
"title": {"type":["string","null"], "default":null},
|
||||||
|
"const": {"type":"object", "default":{"title":"literal","const":1}},
|
||||||
|
"version": {"type":"integer","const":1}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
normalize_input_schema(&mut schema);
|
||||||
|
assert_eq!(schema["properties"]["title"], json!({"type":"string"}));
|
||||||
|
assert_eq!(
|
||||||
|
schema["properties"]["const"]["default"],
|
||||||
|
json!({"title":"literal","const":1})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
schema["properties"]["version"],
|
||||||
|
json!({"type":"integer","minimum":1,"maximum":1})
|
||||||
|
);
|
||||||
|
assert!(schema.get("title").is_none());
|
||||||
|
assert!(serde_json::from_value::<ActionTarget>(json!({"kind":"window","pid":1})).is_err());
|
||||||
|
assert!(serde_json::from_value::<ActionTarget>(json!({"kind":"desktop"})).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn generated_click_schema_matches_driver_dialect() {
|
fn generated_click_schema_matches_driver_dialect() {
|
||||||
let schema = ClickInput::input_schema();
|
let schema = ClickInput::input_schema();
|
||||||
assert_eq!(schema["type"], "object");
|
assert_eq!(schema["type"], "object");
|
||||||
assert_eq!(schema["additionalProperties"], false);
|
assert_eq!(schema["additionalProperties"], false);
|
||||||
assert_eq!(schema["required"], json!(["x", "y"]));
|
assert_eq!(schema["required"], json!(["x", "y"]));
|
||||||
assert_eq!(schema["properties"]["scope"]["const"], "desktop");
|
assert_eq!(schema["properties"]["scope"]["enum"], json!(["desktop"]));
|
||||||
assert!(schema["properties"]["target"]["anyOf"].is_array());
|
assert_eq!(schema["properties"]["target"]["type"], "object");
|
||||||
|
assert!(schema["properties"]["target"].get("anyOf").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
schema["properties"]["button"],
|
schema["properties"]["button"],
|
||||||
json!({ "type": "string", "enum": ["left", "right", "middle"] })
|
json!({ "type": "string", "enum": ["left", "right", "middle"] })
|
||||||
|
|||||||
@@ -8,12 +8,14 @@
|
|||||||
//! typed inputs/results and versioned declarations used by the live runtime
|
//! typed inputs/results and versioned declarations used by the live runtime
|
||||||
//! and to generate experimental client SDKs.
|
//! and to generate experimental client SDKs.
|
||||||
|
|
||||||
|
use schemars::{generate::SchemaSettings, transform::RecursiveTransform, Schema};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
pub mod compatibility;
|
pub mod compatibility;
|
||||||
|
pub mod computer_use;
|
||||||
pub mod cursor;
|
pub mod cursor;
|
||||||
mod cursor_tools;
|
mod cursor_tools;
|
||||||
mod desktop;
|
mod desktop;
|
||||||
@@ -22,23 +24,41 @@ mod outputs;
|
|||||||
mod session;
|
mod session;
|
||||||
mod verification;
|
mod verification;
|
||||||
|
|
||||||
|
pub(crate) fn schema_settings() -> SchemaSettings {
|
||||||
|
SchemaSettings::draft2020_12()
|
||||||
|
.with(|settings| {
|
||||||
|
settings.meta_schema = None;
|
||||||
|
settings.inline_subschemas = true;
|
||||||
|
})
|
||||||
|
.with_transform(RecursiveTransform(drop_schemars_numeric_format))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drop_schemars_numeric_format(schema: &mut Schema) {
|
||||||
|
if matches!(
|
||||||
|
schema.get("format").and_then(Value::as_str),
|
||||||
|
Some("uint32" | "uint64" | "double")
|
||||||
|
) {
|
||||||
|
schema.remove("format");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub use cursor::{
|
pub use cursor::{
|
||||||
classify_cursor_semantics, CursorAction, CursorDelivery, CursorPlayback, CursorReducedMotion,
|
classify_cursor_semantics, CursorAction, CursorDelivery, CursorPlayback, CursorReducedMotion,
|
||||||
CursorSemantics, CursorTarget, CursorThemeSelection,
|
CursorSemantics, CursorTarget, CursorThemeSelection,
|
||||||
};
|
};
|
||||||
pub use inputs::{
|
pub use inputs::{
|
||||||
action_target_schema, ActionTarget, CaptureScope, ClickButton, ClickInput, ClipboardReadInput,
|
action_target_schema, normalize_input_schema, ActionTarget, CaptureScope, ClickButton,
|
||||||
ClipboardWriteInput, DeliveryMode, DesktopScope, DoubleClickInput, DragInput, EndSessionInput,
|
ClickInput, ClipboardReadInput, ClipboardWriteInput, DeliveryMode, DesktopScope,
|
||||||
EscalateSessionInput, EscalationReason, GetAgentCursorStateInput, GetCursorPositionInput,
|
DoubleClickInput, DragInput, EndSessionInput, EscalateSessionInput, EscalationReason,
|
||||||
GetDesktopStateInput, GetScreenSizeInput, GetSessionInput, GetSessionStateInput,
|
GetAgentCursorStateInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput,
|
||||||
GetWindowStateInput, HotkeyInput, InvokeMenuInput, LaunchAppInput, ListAppsInput,
|
GetSessionInput, GetSessionStateInput, GetWindowStateInput, HotkeyInput, InvokeMenuInput,
|
||||||
ListSessionsInput, ListWindowsInput, MoveCursorInput, ObservationRevisionInput, PasteFormat,
|
LaunchAppInput, ListAppsInput, ListSessionsInput, ListWindowsInput, MoveCursorInput,
|
||||||
PasteInput, PerformSecondaryActionInput, PressKeyInput, RightClickInput, ScrollBy,
|
ObservationRevisionInput, PasteFormat, PasteInput, PerformSecondaryActionInput, PressKeyInput,
|
||||||
ScrollDirection, ScrollInput, SelectTextInput, SetAgentCursorEnabledInput,
|
RightClickInput, ScrollBy, ScrollDirection, ScrollInput, SelectTextInput,
|
||||||
SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetValueInput, SetWindowFrameInput,
|
SetAgentCursorEnabledInput, SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetValueInput,
|
||||||
StartSessionInput, TextSelection, ToolInput, TypeTextInput, WindowClickInput, WindowDragInput,
|
SetWindowFrameInput, StartSessionInput, TextSelection, ToolInput, TypeTextInput,
|
||||||
WindowHotkeyInput, WindowPressKeyInput, WindowScrollInput, WindowTypeTextInput,
|
WindowClickInput, WindowDragInput, WindowHotkeyInput, WindowPressKeyInput, WindowScrollInput,
|
||||||
MULTI_CALL_SESSION_DESCRIPTION,
|
WindowTypeTextInput, MULTI_CALL_SESSION_DESCRIPTION,
|
||||||
};
|
};
|
||||||
pub use outputs::{
|
pub use outputs::{
|
||||||
advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode,
|
advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode,
|
||||||
@@ -347,6 +367,72 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
struct NumericFormatFixture;
|
||||||
|
|
||||||
|
impl schemars::JsonSchema for NumericFormatFixture {
|
||||||
|
fn schema_name() -> std::borrow::Cow<'static, str> {
|
||||||
|
"NumericFormatFixture".into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_schema(_: &mut schemars::SchemaGenerator) -> Schema {
|
||||||
|
schemars::json_schema!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"count": {"type": "integer", "format": "uint32"},
|
||||||
|
"frame": {"type": "integer", "format": "uint64"},
|
||||||
|
"ratio": {"type": "number", "format": "double"},
|
||||||
|
"created_at": {"type": "string", "format": "date-time"},
|
||||||
|
"format": {"type": "string"},
|
||||||
|
"annotation": {
|
||||||
|
"const": {"format": "uint32"},
|
||||||
|
"default": {"format": "uint64"},
|
||||||
|
"examples": [{"format": "double"}]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["format"]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ToolInput for NumericFormatFixture {
|
||||||
|
const TOOL_NAME: &'static str = "numeric_format_fixture";
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ToolOutput for NumericFormatFixture {}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn input_and_output_schemas_drop_only_schemars_numeric_formats() {
|
||||||
|
let schemas = [
|
||||||
|
<NumericFormatFixture as ToolInput>::input_schema(),
|
||||||
|
<NumericFormatFixture as ToolOutput>::output_schema(),
|
||||||
|
];
|
||||||
|
|
||||||
|
for schema in schemas {
|
||||||
|
for property in ["count", "frame", "ratio"] {
|
||||||
|
assert!(schema["properties"][property].get("format").is_none());
|
||||||
|
}
|
||||||
|
assert_eq!(schema["properties"]["created_at"]["format"], "date-time");
|
||||||
|
assert_eq!(
|
||||||
|
schema["properties"]["format"],
|
||||||
|
serde_json::json!({"type": "string"})
|
||||||
|
);
|
||||||
|
assert_eq!(schema["required"], serde_json::json!(["format"]));
|
||||||
|
assert_eq!(
|
||||||
|
schema["properties"]["annotation"]["const"],
|
||||||
|
serde_json::json!({"format": "uint32"})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
schema["properties"]["annotation"]["default"],
|
||||||
|
serde_json::json!({"format": "uint64"})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
schema["properties"]["annotation"]["examples"],
|
||||||
|
serde_json::json!([{"format": "double"}])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn manifest_is_sorted_and_versioned() {
|
fn manifest_is_sorted_and_versioned() {
|
||||||
let manifest = manifest();
|
let manifest = manifest();
|
||||||
@@ -402,8 +488,8 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
contract.input_schema["properties"]["expect"]["items"]["properties"]["element"]
|
contract.input_schema["properties"]["expect"]["items"]["properties"]["element"]
|
||||||
["properties"]["exists"]["enum"],
|
["properties"]["exists"]["type"],
|
||||||
serde_json::json!([true])
|
serde_json::json!("boolean")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
// Copyright (c) 2026 Cua AI, Inc.
|
// Copyright (c) 2026 Cua AI, Inc.
|
||||||
|
|
||||||
use crate::{CaptureScope, EscalationReason, Platform};
|
use crate::{schema_settings, CaptureScope, EscalationReason, Platform};
|
||||||
use schemars::{generate::SchemaSettings, JsonSchema};
|
use schemars::JsonSchema;
|
||||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -56,11 +56,12 @@ pub fn advertised_output_schema(success: Value) -> Value {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn output_schema_with_additional_properties<T: JsonSchema>(additional_properties: bool) -> Value {
|
fn output_schema_with_additional_properties<T: JsonSchema>(additional_properties: bool) -> Value {
|
||||||
let mut settings = SchemaSettings::draft2020_12();
|
let mut schema = serde_json::to_value(
|
||||||
settings.inline_subschemas = true;
|
schema_settings()
|
||||||
settings.meta_schema = None;
|
.into_generator()
|
||||||
let mut schema = serde_json::to_value(settings.into_generator().into_root_schema_for::<T>())
|
.into_root_schema_for::<T>(),
|
||||||
.expect("JSON Schema serializes");
|
)
|
||||||
|
.expect("JSON Schema serializes");
|
||||||
strip_schema_titles(&mut schema);
|
strip_schema_titles(&mut schema);
|
||||||
if let Some(object) = schema.as_object_mut() {
|
if let Some(object) = schema.as_object_mut() {
|
||||||
object.insert(
|
object.insert(
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ fn nonempty_string_schema(_: &mut SchemaGenerator) -> Schema {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn true_only_boolean_schema(_: &mut SchemaGenerator) -> Schema {
|
fn true_only_boolean_schema(_: &mut SchemaGenerator) -> Schema {
|
||||||
json_schema!({ "type": "boolean", "enum": [true] })
|
json_schema!({ "type": "boolean", "description": "Must be true. Absence cannot be proven; false is rejected." })
|
||||||
}
|
}
|
||||||
|
|
||||||
fn nullable_string_schema(_: &mut SchemaGenerator) -> Schema {
|
fn nullable_string_schema(_: &mut SchemaGenerator) -> Schema {
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ uuid = { workspace = true }
|
|||||||
base64 = { workspace = true }
|
base64 = { workspace = true }
|
||||||
sha2 = "0.10"
|
sha2 = "0.10"
|
||||||
url = "2.5"
|
url = "2.5"
|
||||||
|
time = "0.3"
|
||||||
|
|
||||||
[target.'cfg(unix)'.dependencies]
|
[target.'cfg(unix)'.dependencies]
|
||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
|
|||||||
@@ -59,6 +59,8 @@ pub enum ActualDelivery {
|
|||||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||||
pub enum ActionTransport {
|
pub enum ActionTransport {
|
||||||
AgentCursorOverlay,
|
AgentCursorOverlay,
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
MacosAppleEvents,
|
||||||
MacosAxAction,
|
MacosAxAction,
|
||||||
MacosAxValue,
|
MacosAxValue,
|
||||||
MacosAxWindowFrame,
|
MacosAxWindowFrame,
|
||||||
@@ -81,11 +83,9 @@ pub enum ActionTransport {
|
|||||||
LinuxAtSpiAction,
|
LinuxAtSpiAction,
|
||||||
LinuxAtSpiValue,
|
LinuxAtSpiValue,
|
||||||
LinuxPty,
|
LinuxPty,
|
||||||
|
LinuxUinput,
|
||||||
LinuxXSendEvent,
|
LinuxXSendEvent,
|
||||||
LinuxXTest,
|
LinuxXTest,
|
||||||
LinuxLibei,
|
|
||||||
LinuxWaylandVirtualPointer,
|
|
||||||
LinuxCuaCompositorInject,
|
|
||||||
LinuxX11ConfigureWindow,
|
LinuxX11ConfigureWindow,
|
||||||
BrowserCdpInputMouse,
|
BrowserCdpInputMouse,
|
||||||
BrowserCdpInputKey,
|
BrowserCdpInputKey,
|
||||||
@@ -95,6 +95,8 @@ pub enum ActionTransport {
|
|||||||
impl ActionTransport {
|
impl ActionTransport {
|
||||||
pub const ALL: &'static [Self] = &[
|
pub const ALL: &'static [Self] = &[
|
||||||
Self::AgentCursorOverlay,
|
Self::AgentCursorOverlay,
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
Self::MacosAppleEvents,
|
||||||
Self::MacosAxAction,
|
Self::MacosAxAction,
|
||||||
Self::MacosAxValue,
|
Self::MacosAxValue,
|
||||||
Self::MacosAxWindowFrame,
|
Self::MacosAxWindowFrame,
|
||||||
@@ -117,11 +119,9 @@ impl ActionTransport {
|
|||||||
Self::LinuxAtSpiAction,
|
Self::LinuxAtSpiAction,
|
||||||
Self::LinuxAtSpiValue,
|
Self::LinuxAtSpiValue,
|
||||||
Self::LinuxPty,
|
Self::LinuxPty,
|
||||||
|
Self::LinuxUinput,
|
||||||
Self::LinuxXSendEvent,
|
Self::LinuxXSendEvent,
|
||||||
Self::LinuxXTest,
|
Self::LinuxXTest,
|
||||||
Self::LinuxLibei,
|
|
||||||
Self::LinuxWaylandVirtualPointer,
|
|
||||||
Self::LinuxCuaCompositorInject,
|
|
||||||
Self::LinuxX11ConfigureWindow,
|
Self::LinuxX11ConfigureWindow,
|
||||||
Self::BrowserCdpInputMouse,
|
Self::BrowserCdpInputMouse,
|
||||||
Self::BrowserCdpInputKey,
|
Self::BrowserCdpInputKey,
|
||||||
@@ -131,6 +131,8 @@ impl ActionTransport {
|
|||||||
pub const fn route(self) -> ActionRoute {
|
pub const fn route(self) -> ActionRoute {
|
||||||
match self {
|
match self {
|
||||||
Self::AgentCursorOverlay => ActionRoute::SyntheticEvents,
|
Self::AgentCursorOverlay => ActionRoute::SyntheticEvents,
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
Self::MacosAppleEvents => ActionRoute::SystemApi,
|
||||||
Self::MacosAxAction
|
Self::MacosAxAction
|
||||||
| Self::MacosAxValue
|
| Self::MacosAxValue
|
||||||
| Self::MacosAxWindowFrame
|
| Self::MacosAxWindowFrame
|
||||||
@@ -147,16 +149,14 @@ impl ActionTransport {
|
|||||||
Self::MacosCgEventPid
|
Self::MacosCgEventPid
|
||||||
| Self::WindowsPostMessage
|
| Self::WindowsPostMessage
|
||||||
| Self::LinuxPty
|
| Self::LinuxPty
|
||||||
|
| Self::LinuxUinput
|
||||||
| Self::LinuxXSendEvent => ActionRoute::SyntheticEvents,
|
| Self::LinuxXSendEvent => ActionRoute::SyntheticEvents,
|
||||||
Self::MacosCgEventHid
|
Self::MacosCgEventHid
|
||||||
| Self::WindowsTargetedInjection
|
| Self::WindowsTargetedInjection
|
||||||
| Self::WindowsSendInput
|
| Self::WindowsSendInput
|
||||||
| Self::WindowsSetCursorPos
|
| Self::WindowsSetCursorPos
|
||||||
| Self::WindowsShellExecute
|
| Self::WindowsShellExecute
|
||||||
| Self::LinuxXTest
|
| Self::LinuxXTest => ActionRoute::GlobalInput,
|
||||||
| Self::LinuxLibei
|
|
||||||
| Self::LinuxWaylandVirtualPointer
|
|
||||||
| Self::LinuxCuaCompositorInject => ActionRoute::GlobalInput,
|
|
||||||
Self::WindowsSetWindowPos | Self::LinuxX11ConfigureWindow => ActionRoute::SystemApi,
|
Self::WindowsSetWindowPos | Self::LinuxX11ConfigureWindow => ActionRoute::SystemApi,
|
||||||
Self::BrowserCdpRuntimeFunction => ActionRoute::Dom,
|
Self::BrowserCdpRuntimeFunction => ActionRoute::Dom,
|
||||||
Self::BrowserCdpInputMouse | Self::BrowserCdpInputKey => ActionRoute::TrustedInput,
|
Self::BrowserCdpInputMouse | Self::BrowserCdpInputKey => ActionRoute::TrustedInput,
|
||||||
@@ -452,15 +452,13 @@ impl ActionExecutionRecord {
|
|||||||
|
|
||||||
if legacy_has_publishable_readback(tool_name, structured) {
|
if legacy_has_publishable_readback(tool_name, structured) {
|
||||||
record.evidence.push(ActionEvidence {
|
record.evidence.push(ActionEvidence {
|
||||||
kind: if matches!(
|
kind: match transport {
|
||||||
transport,
|
#[cfg(target_os = "macos")]
|
||||||
|
ActionTransport::MacosAppleEvents => EvidenceKind::ValueReadback,
|
||||||
ActionTransport::BrowserCdpInputMouse
|
ActionTransport::BrowserCdpInputMouse
|
||||||
| ActionTransport::BrowserCdpInputKey
|
| ActionTransport::BrowserCdpInputKey
|
||||||
| ActionTransport::BrowserCdpRuntimeFunction
|
| ActionTransport::BrowserCdpRuntimeFunction => EvidenceKind::BrowserReadback,
|
||||||
) {
|
_ => EvidenceKind::AccessibilityReadback,
|
||||||
EvidenceKind::BrowserReadback
|
|
||||||
} else {
|
|
||||||
EvidenceKind::AccessibilityReadback
|
|
||||||
},
|
},
|
||||||
detail: structured
|
detail: structured
|
||||||
.get("verify")
|
.get("verify")
|
||||||
@@ -739,6 +737,8 @@ fn parse_actual_delivery_name(value: &str) -> Result<ActualDelivery, String> {
|
|||||||
fn parse_transport_name(value: &str) -> Result<ActionTransport, String> {
|
fn parse_transport_name(value: &str) -> Result<ActionTransport, String> {
|
||||||
let transport = match value {
|
let transport = match value {
|
||||||
"agent_cursor_overlay" => ActionTransport::AgentCursorOverlay,
|
"agent_cursor_overlay" => ActionTransport::AgentCursorOverlay,
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
"macos_apple_events" => ActionTransport::MacosAppleEvents,
|
||||||
"macos_ax_action" => ActionTransport::MacosAxAction,
|
"macos_ax_action" => ActionTransport::MacosAxAction,
|
||||||
"macos_ax_value" => ActionTransport::MacosAxValue,
|
"macos_ax_value" => ActionTransport::MacosAxValue,
|
||||||
"macos_ax_window_frame" => ActionTransport::MacosAxWindowFrame,
|
"macos_ax_window_frame" => ActionTransport::MacosAxWindowFrame,
|
||||||
@@ -761,11 +761,9 @@ fn parse_transport_name(value: &str) -> Result<ActionTransport, String> {
|
|||||||
"linux_at_spi_action" => ActionTransport::LinuxAtSpiAction,
|
"linux_at_spi_action" => ActionTransport::LinuxAtSpiAction,
|
||||||
"linux_at_spi_value" => ActionTransport::LinuxAtSpiValue,
|
"linux_at_spi_value" => ActionTransport::LinuxAtSpiValue,
|
||||||
"linux_pty" => ActionTransport::LinuxPty,
|
"linux_pty" => ActionTransport::LinuxPty,
|
||||||
|
"linux_uinput" => ActionTransport::LinuxUinput,
|
||||||
"linux_x_send_event" => ActionTransport::LinuxXSendEvent,
|
"linux_x_send_event" => ActionTransport::LinuxXSendEvent,
|
||||||
"linux_x_test" => ActionTransport::LinuxXTest,
|
"linux_x_test" => ActionTransport::LinuxXTest,
|
||||||
"linux_libei" => ActionTransport::LinuxLibei,
|
|
||||||
"linux_wayland_virtual_pointer" => ActionTransport::LinuxWaylandVirtualPointer,
|
|
||||||
"linux_cua_compositor_inject" => ActionTransport::LinuxCuaCompositorInject,
|
|
||||||
"linux_x11_configure_window" => ActionTransport::LinuxX11ConfigureWindow,
|
"linux_x11_configure_window" => ActionTransport::LinuxX11ConfigureWindow,
|
||||||
"browser_cdp_input_mouse" => ActionTransport::BrowserCdpInputMouse,
|
"browser_cdp_input_mouse" => ActionTransport::BrowserCdpInputMouse,
|
||||||
"browser_cdp_input_key" => ActionTransport::BrowserCdpInputKey,
|
"browser_cdp_input_key" => ActionTransport::BrowserCdpInputKey,
|
||||||
@@ -885,8 +883,13 @@ fn browser_refusal_escalation(code: &str) -> Option<ActionEscalation> {
|
|||||||
/// against the request. A click may do so only when the producer also declares
|
/// against the request. A click may do so only when the producer also declares
|
||||||
/// explicit accessibility read-back evidence, as the macOS collection-item
|
/// explicit accessibility read-back evidence, as the macOS collection-item
|
||||||
/// selection fallback does after observing `AXSelected`.
|
/// selection fallback does after observing `AXSelected`.
|
||||||
|
/// Native Apple Events save also compares the document's modified state.
|
||||||
fn legacy_has_publishable_readback(tool_name: &str, structured: &serde_json::Value) -> bool {
|
fn legacy_has_publishable_readback(tool_name: &str, structured: &serde_json::Value) -> bool {
|
||||||
let value_readback_tool = matches!(tool_name, "type_text" | "type_text_chars" | "set_value");
|
let value_readback_tool = matches!(tool_name, "type_text" | "type_text_chars" | "set_value");
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
let value_readback_tool = value_readback_tool
|
||||||
|
|| (tool_name == "hotkey"
|
||||||
|
&& structured.get("path").and_then(serde_json::Value::as_str) == Some("apple_events"));
|
||||||
let click_selection_readback = tool_name == "click"
|
let click_selection_readback = tool_name == "click"
|
||||||
&& structured
|
&& structured
|
||||||
.get("evidence")
|
.get("evidence")
|
||||||
@@ -913,6 +916,8 @@ fn transport_from_legacy(
|
|||||||
) -> Option<ActionTransport> {
|
) -> Option<ActionTransport> {
|
||||||
let path = raw_path.unwrap_or("");
|
let path = raw_path.unwrap_or("");
|
||||||
let transport = match path {
|
let transport = match path {
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
"apple_events" => ActionTransport::MacosAppleEvents,
|
||||||
"ax" | "ax_fg" => {
|
"ax" | "ax_fg" => {
|
||||||
if cfg!(target_os = "macos") {
|
if cfg!(target_os = "macos") {
|
||||||
if matches!(tool_name, "type_text" | "type_text_chars" | "set_value") {
|
if matches!(tool_name, "type_text" | "type_text_chars" | "set_value") {
|
||||||
@@ -941,16 +946,11 @@ fn transport_from_legacy(
|
|||||||
"post_message" | "PostMessage" => ActionTransport::WindowsPostMessage,
|
"post_message" | "PostMessage" => ActionTransport::WindowsPostMessage,
|
||||||
"SendInput" => ActionTransport::WindowsSendInput,
|
"SendInput" => ActionTransport::WindowsSendInput,
|
||||||
"SetCursorPos" => ActionTransport::WindowsSetCursorPos,
|
"SetCursorPos" => ActionTransport::WindowsSetCursorPos,
|
||||||
"atspi" | "wayland_atspi" | "x11_atspi" => ActionTransport::LinuxAtSpiAction,
|
"atspi" | "x11_atspi" => ActionTransport::LinuxAtSpiAction,
|
||||||
"pty" => ActionTransport::LinuxPty,
|
"pty" => ActionTransport::LinuxPty,
|
||||||
"x11_pixel" | "x11_pixel_fg" | "x11_xtest_fg" | "xtest" | "xtest_desktop" => {
|
"x11_pixel" | "x11_pixel_fg" | "x11_xtest_fg" | "xtest" | "xtest_desktop" => {
|
||||||
ActionTransport::LinuxXTest
|
ActionTransport::LinuxXTest
|
||||||
}
|
}
|
||||||
"wayland_activate" | "wayland_focused" => ActionTransport::LinuxLibei,
|
|
||||||
"wayland_desktop" => ActionTransport::LinuxWaylandVirtualPointer,
|
|
||||||
"cua_compositor_inject" | "wayland_cua_compositor" => {
|
|
||||||
ActionTransport::LinuxCuaCompositorInject
|
|
||||||
}
|
|
||||||
"hid" | "cgevent_hid" | "cgevent_fg" => ActionTransport::MacosCgEventHid,
|
"hid" | "cgevent_hid" | "cgevent_fg" => ActionTransport::MacosCgEventHid,
|
||||||
"cgevent" => {
|
"cgevent" => {
|
||||||
if args
|
if args
|
||||||
@@ -1138,9 +1138,7 @@ fn actual_delivery_from_legacy(
|
|||||||
}
|
}
|
||||||
match raw_path {
|
match raw_path {
|
||||||
Some(path) if path.ends_with("_fg") => Some(ActualDelivery::Foreground),
|
Some(path) if path.ends_with("_fg") => Some(ActualDelivery::Foreground),
|
||||||
Some("hid" | "cgevent_hid" | "SendInput" | "wayland_activate" | "wayland_focused") => {
|
Some("hid" | "cgevent_hid" | "SendInput") => Some(ActualDelivery::Foreground),
|
||||||
Some(ActualDelivery::Foreground)
|
|
||||||
}
|
|
||||||
Some(_) => Some(ActualDelivery::Background),
|
Some(_) => Some(ActualDelivery::Background),
|
||||||
None => Some(ActualDelivery::Unknown),
|
None => Some(ActualDelivery::Unknown),
|
||||||
}
|
}
|
||||||
@@ -1262,6 +1260,8 @@ fn escalation_kind_name(kind: EscalationKind) -> &'static str {
|
|||||||
fn transport_name(transport: ActionTransport) -> &'static str {
|
fn transport_name(transport: ActionTransport) -> &'static str {
|
||||||
match transport {
|
match transport {
|
||||||
ActionTransport::AgentCursorOverlay => "agent_cursor_overlay",
|
ActionTransport::AgentCursorOverlay => "agent_cursor_overlay",
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
ActionTransport::MacosAppleEvents => "macos_apple_events",
|
||||||
ActionTransport::MacosAxAction => "macos_ax_action",
|
ActionTransport::MacosAxAction => "macos_ax_action",
|
||||||
ActionTransport::MacosAxValue => "macos_ax_value",
|
ActionTransport::MacosAxValue => "macos_ax_value",
|
||||||
ActionTransport::MacosAxWindowFrame => "macos_ax_window_frame",
|
ActionTransport::MacosAxWindowFrame => "macos_ax_window_frame",
|
||||||
@@ -1284,11 +1284,9 @@ fn transport_name(transport: ActionTransport) -> &'static str {
|
|||||||
ActionTransport::LinuxAtSpiAction => "linux_at_spi_action",
|
ActionTransport::LinuxAtSpiAction => "linux_at_spi_action",
|
||||||
ActionTransport::LinuxAtSpiValue => "linux_at_spi_value",
|
ActionTransport::LinuxAtSpiValue => "linux_at_spi_value",
|
||||||
ActionTransport::LinuxPty => "linux_pty",
|
ActionTransport::LinuxPty => "linux_pty",
|
||||||
|
ActionTransport::LinuxUinput => "linux_uinput",
|
||||||
ActionTransport::LinuxXSendEvent => "linux_x_send_event",
|
ActionTransport::LinuxXSendEvent => "linux_x_send_event",
|
||||||
ActionTransport::LinuxXTest => "linux_x_test",
|
ActionTransport::LinuxXTest => "linux_x_test",
|
||||||
ActionTransport::LinuxLibei => "linux_libei",
|
|
||||||
ActionTransport::LinuxWaylandVirtualPointer => "linux_wayland_virtual_pointer",
|
|
||||||
ActionTransport::LinuxCuaCompositorInject => "linux_cua_compositor_inject",
|
|
||||||
ActionTransport::LinuxX11ConfigureWindow => "linux_x11_configure_window",
|
ActionTransport::LinuxX11ConfigureWindow => "linux_x11_configure_window",
|
||||||
ActionTransport::BrowserCdpInputMouse => "browser_cdp_input_mouse",
|
ActionTransport::BrowserCdpInputMouse => "browser_cdp_input_mouse",
|
||||||
ActionTransport::BrowserCdpInputKey => "browser_cdp_input_key",
|
ActionTransport::BrowserCdpInputKey => "browser_cdp_input_key",
|
||||||
@@ -1443,6 +1441,10 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let route = record.stable_projection().unwrap().route;
|
let route = record.stable_projection().unwrap().route;
|
||||||
assert_eq!(route, transport.route());
|
assert_eq!(route, transport.route());
|
||||||
|
assert_eq!(
|
||||||
|
ActionExecutionRecord::from_private_json(&record.debug_json()).unwrap(),
|
||||||
|
record
|
||||||
|
);
|
||||||
routes.push(route);
|
routes.push(route);
|
||||||
}
|
}
|
||||||
assert!(routes.contains(&ActionRoute::Accessibility));
|
assert!(routes.contains(&ActionRoute::Accessibility));
|
||||||
@@ -1565,7 +1567,7 @@ mod tests {
|
|||||||
fn projection_contains_only_outcome_information() {
|
fn projection_contains_only_outcome_information() {
|
||||||
let projection = ActionExecutionRecord::builder(
|
let projection = ActionExecutionRecord::builder(
|
||||||
ActionEffect::Confirmed,
|
ActionEffect::Confirmed,
|
||||||
ActionTransport::LinuxLibei,
|
ActionTransport::LinuxXTest,
|
||||||
RequestedDelivery::Foreground,
|
RequestedDelivery::Foreground,
|
||||||
)
|
)
|
||||||
.actual_delivery(ActualDelivery::Foreground)
|
.actual_delivery(ActualDelivery::Foreground)
|
||||||
@@ -1730,6 +1732,41 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
fn apple_event_native_readback_preserves_the_actual_background_system_route() {
|
||||||
|
for tool in ["hotkey", "set_value"] {
|
||||||
|
let args = serde_json::json!({"pid":101,"window_id":501,"delivery_mode":"foreground"});
|
||||||
|
let result = serde_json::json!({
|
||||||
|
"path":"apple_events", "verified":true, "effect":"confirmed",
|
||||||
|
"delivery_mode":"background"
|
||||||
|
});
|
||||||
|
let record = ActionExecutionRecord::from_legacy(tool, &args, &result)
|
||||||
|
.expect("the native save/rename producer has independent readback");
|
||||||
|
assert_eq!(record.requested_delivery, RequestedDelivery::Foreground);
|
||||||
|
assert_eq!(record.actual_delivery, Some(ActualDelivery::Background));
|
||||||
|
let public = record.public_result().unwrap();
|
||||||
|
public.validate_invariants().unwrap();
|
||||||
|
let public = serde_json::to_value(public).unwrap();
|
||||||
|
assert_eq!(public["effect"], "confirmed");
|
||||||
|
assert_eq!(public["route"], "system_api");
|
||||||
|
assert_eq!(public["delivery"]["mode"], "background");
|
||||||
|
assert_eq!(public["evidence"][0]["kind"], "value_readback");
|
||||||
|
|
||||||
|
let mut unverified = result.clone();
|
||||||
|
unverified["verified"] = serde_json::json!(false);
|
||||||
|
let record = ActionExecutionRecord::from_legacy(tool, &args, &unverified).unwrap();
|
||||||
|
assert_eq!(record.effect, ActionEffect::Unverifiable);
|
||||||
|
assert!(record.evidence.is_empty());
|
||||||
|
|
||||||
|
let mut acknowledgement = result.clone();
|
||||||
|
acknowledgement.as_object_mut().unwrap().remove("effect");
|
||||||
|
let record = ActionExecutionRecord::from_legacy(tool, &args, &acknowledgement).unwrap();
|
||||||
|
assert_eq!(record.effect, ActionEffect::Unverifiable);
|
||||||
|
assert!(record.evidence.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn system_api_value_readback_projects_without_transport_details() {
|
fn system_api_value_readback_projects_without_transport_details() {
|
||||||
let result = ActionExecutionRecord::builder(
|
let result = ActionExecutionRecord::builder(
|
||||||
@@ -2079,7 +2116,6 @@ mod tests {
|
|||||||
"SendInput",
|
"SendInput",
|
||||||
"SetCursorPos",
|
"SetCursorPos",
|
||||||
"atspi",
|
"atspi",
|
||||||
"wayland_atspi",
|
|
||||||
"x11_atspi",
|
"x11_atspi",
|
||||||
"pty",
|
"pty",
|
||||||
"x11_pixel",
|
"x11_pixel",
|
||||||
@@ -2087,11 +2123,6 @@ mod tests {
|
|||||||
"x11_xtest_fg",
|
"x11_xtest_fg",
|
||||||
"xtest",
|
"xtest",
|
||||||
"xtest_desktop",
|
"xtest_desktop",
|
||||||
"wayland_activate",
|
|
||||||
"wayland_focused",
|
|
||||||
"wayland_desktop",
|
|
||||||
"cua_compositor_inject",
|
|
||||||
"wayland_cua_compositor",
|
|
||||||
"hid",
|
"hid",
|
||||||
"cgevent_hid",
|
"cgevent_hid",
|
||||||
"cgevent",
|
"cgevent",
|
||||||
|
|||||||
@@ -270,6 +270,8 @@ const DESKTOP_INPUT_OPERATIONS: &[&str] = &[
|
|||||||
"type_text",
|
"type_text",
|
||||||
"type_text_chars",
|
"type_text_chars",
|
||||||
"press_key",
|
"press_key",
|
||||||
|
"key_down",
|
||||||
|
"key_up",
|
||||||
"hotkey",
|
"hotkey",
|
||||||
"set_value",
|
"set_value",
|
||||||
"paste",
|
"paste",
|
||||||
@@ -859,6 +861,8 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment {
|
|||||||
| "type_text"
|
| "type_text"
|
||||||
| "type_text_chars"
|
| "type_text_chars"
|
||||||
| "press_key"
|
| "press_key"
|
||||||
|
| "key_down"
|
||||||
|
| "key_up"
|
||||||
| "hotkey"
|
| "hotkey"
|
||||||
| "set_value"
|
| "set_value"
|
||||||
| "paste"
|
| "paste"
|
||||||
@@ -882,6 +886,11 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment {
|
|||||||
"zoom"
|
"zoom"
|
||||||
| "clipboard_read"
|
| "clipboard_read"
|
||||||
| "list_apps"
|
| "list_apps"
|
||||||
|
| "get_state"
|
||||||
|
| "get_app"
|
||||||
|
| "get_ax_state"
|
||||||
|
| "get_screenshot"
|
||||||
|
| "get_ax_state_and_screenshot"
|
||||||
| "list_windows"
|
| "list_windows"
|
||||||
| "debug_window_info"
|
| "debug_window_info"
|
||||||
| "check_permissions"
|
| "check_permissions"
|
||||||
@@ -1153,6 +1162,8 @@ fn enforce_hard_invariants(
|
|||||||
| "type_text"
|
| "type_text"
|
||||||
| "type_text_chars"
|
| "type_text_chars"
|
||||||
| "press_key"
|
| "press_key"
|
||||||
|
| "key_down"
|
||||||
|
| "key_up"
|
||||||
| "hotkey"
|
| "hotkey"
|
||||||
| "set_value"
|
| "set_value"
|
||||||
| "paste"
|
| "paste"
|
||||||
@@ -1474,6 +1485,19 @@ mod tests {
|
|||||||
assert_eq!(isolated.enforcement, RiskEnforcement::Active);
|
assert_eq!(isolated.enforcement, RiskEnforcement::Active);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn app_observation_aliases_have_reviewed_private_content_risk() {
|
||||||
|
for tool in [
|
||||||
|
"get_state",
|
||||||
|
"get_app",
|
||||||
|
"get_ax_state",
|
||||||
|
"get_screenshot",
|
||||||
|
"get_ax_state_and_screenshot",
|
||||||
|
] {
|
||||||
|
assert_eq!(advertised_risk_for(tool).class, RiskClass::R2, "{tool}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn unknown_tools_are_unclassified_and_denied() {
|
fn unknown_tools_are_unclassified_and_denied() {
|
||||||
let error = authorize_tool_call("new_unreviewed_tool", &serde_json::json!({}))
|
let error = authorize_tool_call("new_unreviewed_tool", &serde_json::json!({}))
|
||||||
|
|||||||
@@ -328,15 +328,9 @@ fn cleanup_created_profile(profile: &PreparedProfile) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
fn configure_linux_isolated_browser_command(
|
fn configure_linux_isolated_browser_command(command: &mut Command, no_sandbox: bool) {
|
||||||
command: &mut Command,
|
|
||||||
native_wayland: bool,
|
|
||||||
no_sandbox: bool,
|
|
||||||
) {
|
|
||||||
command.arg("--password-store=basic");
|
command.arg("--password-store=basic");
|
||||||
if native_wayland {
|
command.arg("--ozone-platform=x11");
|
||||||
command.arg("--ozone-platform=wayland");
|
|
||||||
}
|
|
||||||
if no_sandbox {
|
if no_sandbox {
|
||||||
command.arg("--no-sandbox");
|
command.arg("--no-sandbox");
|
||||||
}
|
}
|
||||||
@@ -364,12 +358,8 @@ fn isolated_browser_command(executable: &str, profile: &Path) -> Command {
|
|||||||
.arg("--window-size=900,640");
|
.arg("--window-size=900,640");
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
{
|
{
|
||||||
let native_wayland = std::env::var("XDG_SESSION_TYPE")
|
|
||||||
.is_ok_and(|session| session.eq_ignore_ascii_case("wayland"))
|
|
||||||
&& std::env::var_os("WAYLAND_DISPLAY").is_some()
|
|
||||||
&& std::env::var_os("DISPLAY").is_none();
|
|
||||||
let no_sandbox = std::env::var("CUA_E2E_BROWSER_NO_SANDBOX").as_deref() == Ok("1");
|
let no_sandbox = std::env::var("CUA_E2E_BROWSER_NO_SANDBOX").as_deref() == Ok("1");
|
||||||
configure_linux_isolated_browser_command(&mut command, native_wayland, no_sandbox);
|
configure_linux_isolated_browser_command(&mut command, no_sandbox);
|
||||||
}
|
}
|
||||||
let stderr = if std::env::var_os("CUA_E2E_BROWSER_STDERR").is_some() {
|
let stderr = if std::env::var_os("CUA_E2E_BROWSER_STDERR").is_some() {
|
||||||
Stdio::inherit()
|
Stdio::inherit()
|
||||||
@@ -1444,15 +1434,15 @@ mod tests {
|
|||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
#[test]
|
#[test]
|
||||||
fn isolated_launch_can_select_native_wayland_and_test_vm_sandbox_mode() {
|
fn isolated_launch_selects_x11_and_test_vm_sandbox_mode() {
|
||||||
let mut command = Command::new("chromium-under-test");
|
let mut command = Command::new("chromium-under-test");
|
||||||
configure_linux_isolated_browser_command(&mut command, true, true);
|
configure_linux_isolated_browser_command(&mut command, true);
|
||||||
let args = command
|
let args = command
|
||||||
.get_args()
|
.get_args()
|
||||||
.map(|arg| arg.to_string_lossy().into_owned())
|
.map(|arg| arg.to_string_lossy().into_owned())
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
assert!(args.iter().any(|arg| arg == "--password-store=basic"));
|
assert!(args.iter().any(|arg| arg == "--password-store=basic"));
|
||||||
assert!(args.iter().any(|arg| arg == "--ozone-platform=wayland"));
|
assert!(args.iter().any(|arg| arg == "--ozone-platform=x11"));
|
||||||
assert!(args.iter().any(|arg| arg == "--no-sandbox"));
|
assert!(args.iter().any(|arg| arg == "--no-sandbox"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -172,6 +172,8 @@ fn is_scoped_action(tool_name: &str) -> bool {
|
|||||||
| "type_text"
|
| "type_text"
|
||||||
| "type_text_chars"
|
| "type_text_chars"
|
||||||
| "press_key"
|
| "press_key"
|
||||||
|
| "key_down"
|
||||||
|
| "key_up"
|
||||||
| "hotkey"
|
| "hotkey"
|
||||||
| "set_value"
|
| "set_value"
|
||||||
| "paste"
|
| "paste"
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,274 @@
|
|||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex, OnceLock};
|
||||||
|
|
||||||
|
use crate::protocol::ToolResult;
|
||||||
|
use crate::tool::{
|
||||||
|
current_dispatch_session_identity, DispatchSessionIdentity, Tool, ToolDef, ToolRegistry,
|
||||||
|
};
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
pub trait DesktopKeyboard: Send + Sync {
|
||||||
|
fn resolve(&self, key: &str) -> Result<u32, String>;
|
||||||
|
fn transition(&self, code: u32, down: bool, held: &[u32]) -> Result<(), String>;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct HeldKey {
|
||||||
|
manager: u64,
|
||||||
|
owner: DispatchSessionIdentity,
|
||||||
|
code: u32,
|
||||||
|
keyboard: Arc<dyn DesktopKeyboard>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn held_keys() -> &'static Mutex<Vec<HeldKey>> {
|
||||||
|
static KEYS: OnceLock<Mutex<Vec<HeldKey>>> = OnceLock::new();
|
||||||
|
KEYS.get_or_init(|| Mutex::new(Vec::new()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn held_codes() -> Vec<u32> {
|
||||||
|
held_keys()
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|key| key.code)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn admit_input(name: &str, args: &Value) -> Result<(), ToolResult> {
|
||||||
|
if matches!(name, "key_down" | "key_up") {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let held = held_keys().lock().unwrap();
|
||||||
|
if held.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let owner = current_dispatch_session_identity();
|
||||||
|
if held.iter().any(|key| Some(&key.owner) != owner.as_ref()) {
|
||||||
|
return Err(crate::computer_use::error(
|
||||||
|
"input_busy",
|
||||||
|
"Another session holds desktop keys.",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if args["scope"] != "desktop"
|
||||||
|
|| !matches!(
|
||||||
|
name,
|
||||||
|
"click" | "double_click" | "right_click" | "scroll" | "drag"
|
||||||
|
)
|
||||||
|
|| args.get("modifier").is_some()
|
||||||
|
|| args.get("key").is_some()
|
||||||
|
{
|
||||||
|
return Err(crate::computer_use::error("held_keys_active", "Release held desktop keys before app input, text, chords or clicks with explicit modifiers. Use key_down/key_up for held chords.", false));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn transition(
|
||||||
|
manager: u64,
|
||||||
|
keyboard: Arc<dyn DesktopKeyboard>,
|
||||||
|
owner: DispatchSessionIdentity,
|
||||||
|
code: u32,
|
||||||
|
down: bool,
|
||||||
|
) -> Result<(), ToolResult> {
|
||||||
|
let mut held = held_keys().lock().unwrap();
|
||||||
|
if held.iter().any(|key| key.owner != owner) {
|
||||||
|
return Err(crate::computer_use::error(
|
||||||
|
"input_busy",
|
||||||
|
"Another session holds desktop keys.",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let index = held.iter().position(|key| key.code == code);
|
||||||
|
if down {
|
||||||
|
if index.is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
held.push(HeldKey {
|
||||||
|
manager,
|
||||||
|
owner,
|
||||||
|
code,
|
||||||
|
keyboard: keyboard.clone(),
|
||||||
|
});
|
||||||
|
let codes = held.iter().map(|key| key.code).collect::<Vec<_>>();
|
||||||
|
keyboard
|
||||||
|
.transition(code, true, &codes)
|
||||||
|
.map_err(|reason| crate::computer_use::error("operation_failed", reason, true))
|
||||||
|
} else {
|
||||||
|
let Some(index) = index else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
if held[index].manager != manager {
|
||||||
|
return Err(crate::computer_use::error(
|
||||||
|
"input_busy",
|
||||||
|
"The held key belongs to another runtime.",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let codes = held
|
||||||
|
.iter()
|
||||||
|
.filter(|key| key.code != code)
|
||||||
|
.map(|key| key.code)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
keyboard
|
||||||
|
.transition(code, false, &codes)
|
||||||
|
.map_err(|reason| crate::computer_use::error("operation_failed", reason, true))?;
|
||||||
|
held.remove(index);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_matching(manager: u64, session: Option<&str>) -> Result<(), String> {
|
||||||
|
let mut held = held_keys().lock().unwrap();
|
||||||
|
let mut failures = Vec::new();
|
||||||
|
for index in (0..held.len()).rev() {
|
||||||
|
let key = &held[index];
|
||||||
|
if key.manager != manager
|
||||||
|
|| session.is_some_and(|session| {
|
||||||
|
key.owner.session_id != session && key.owner.transport_session_id != session
|
||||||
|
})
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let codes = held
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter(|(i, _)| *i != index)
|
||||||
|
.map(|(_, key)| key.code)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
match key.keyboard.transition(key.code, false, &codes) {
|
||||||
|
Ok(()) => {
|
||||||
|
held.remove(index);
|
||||||
|
}
|
||||||
|
Err(reason) => failures.push(reason),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failures.is_empty() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(failures.join("; "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct KeyTool {
|
||||||
|
def: ToolDef,
|
||||||
|
manager: u64,
|
||||||
|
keyboard: Arc<dyn DesktopKeyboard>,
|
||||||
|
down: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl Tool for KeyTool {
|
||||||
|
fn def(&self) -> &ToolDef {
|
||||||
|
&self.def
|
||||||
|
}
|
||||||
|
async fn invoke(&self, args: Value) -> ToolResult {
|
||||||
|
let Some(owner) = current_dispatch_session_identity() else {
|
||||||
|
return crate::computer_use::error(
|
||||||
|
"session_unavailable",
|
||||||
|
"Holding keys requires a bound session.",
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
if args["scope"] != "desktop" {
|
||||||
|
return crate::computer_use::error(
|
||||||
|
"invalid_arguments",
|
||||||
|
"key_down/key_up require scope:desktop.",
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let Some(key) = args["key"].as_str() else {
|
||||||
|
return crate::computer_use::error("invalid_arguments", "Provide a key name.", false);
|
||||||
|
};
|
||||||
|
let code = match self.keyboard.resolve(key) {
|
||||||
|
Ok(code) => code,
|
||||||
|
Err(reason) => return crate::computer_use::error("invalid_arguments", reason, false),
|
||||||
|
};
|
||||||
|
match transition(self.manager, self.keyboard.clone(), owner, code, self.down) {
|
||||||
|
Ok(()) => ToolResult::text("Key state sent; observe before retrying.")
|
||||||
|
.with_structured(serde_json::json!({"computer_use":1,"sent":true})),
|
||||||
|
Err(result) => result,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn register(registry: &mut ToolRegistry, keyboard: Arc<dyn DesktopKeyboard>) {
|
||||||
|
static NEXT: AtomicU64 = AtomicU64::new(1);
|
||||||
|
let manager = NEXT.fetch_add(1, Ordering::Relaxed);
|
||||||
|
registry.retain_session_end_hook(crate::session::register_scoped_fallible_session_end_hook(
|
||||||
|
"desktop_keyboard",
|
||||||
|
move |session| release_matching(manager, Some(session)),
|
||||||
|
));
|
||||||
|
registry.retain_runtime_cleanup(move || {
|
||||||
|
if let Err(reason) = release_matching(manager, None) {
|
||||||
|
eprintln!("[cua-driver] key cleanup incomplete: {reason}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
for spec in cua_driver_contract::computer_use::tools()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|spec| matches!(spec.name, "key_down" | "key_up"))
|
||||||
|
{
|
||||||
|
let down = spec.name == "key_down";
|
||||||
|
registry.register(Box::new(KeyTool {
|
||||||
|
def: ToolDef {
|
||||||
|
name: spec.name.into(),
|
||||||
|
description: spec.description.into(),
|
||||||
|
input_schema: spec.input_schema,
|
||||||
|
read_only: false,
|
||||||
|
destructive: false,
|
||||||
|
idempotent: true,
|
||||||
|
open_world: true,
|
||||||
|
},
|
||||||
|
manager,
|
||||||
|
keyboard: keyboard.clone(),
|
||||||
|
down,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::atomic::AtomicBool;
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Keyboard {
|
||||||
|
events: Mutex<Vec<(u32, bool)>>,
|
||||||
|
fail: AtomicBool,
|
||||||
|
}
|
||||||
|
impl DesktopKeyboard for Keyboard {
|
||||||
|
fn resolve(&self, _: &str) -> Result<u32, String> {
|
||||||
|
Ok(1)
|
||||||
|
}
|
||||||
|
fn transition(&self, code: u32, down: bool, _: &[u32]) -> Result<(), String> {
|
||||||
|
self.events.lock().unwrap().push((code, down));
|
||||||
|
if self.fail.swap(false, Ordering::SeqCst) {
|
||||||
|
Err("uncertain dispatch".into())
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn uncertain_down_is_owned_until_successful_release_and_cleanup_retries() {
|
||||||
|
let keyboard = Arc::new(Keyboard::default());
|
||||||
|
let owner = DispatchSessionIdentity {
|
||||||
|
runtime_scope: "keyboard-test".into(),
|
||||||
|
session_id: "keyboard-session".into(),
|
||||||
|
transport_session_id: "keyboard-transport".into(),
|
||||||
|
};
|
||||||
|
let mut other = owner.clone();
|
||||||
|
other.session_id = "other".into();
|
||||||
|
keyboard.fail.store(true, Ordering::SeqCst);
|
||||||
|
assert!(transition(991, keyboard.clone(), owner.clone(), 10001, true).is_err());
|
||||||
|
assert!(transition(992, keyboard.clone(), other, 10001, false).is_err());
|
||||||
|
assert_eq!(*keyboard.events.lock().unwrap(), vec![(10001, true)]);
|
||||||
|
assert!(transition(991, keyboard.clone(), owner, 10001, true).is_ok());
|
||||||
|
keyboard.fail.store(true, Ordering::SeqCst);
|
||||||
|
assert!(release_matching(991, Some("keyboard-session")).is_err());
|
||||||
|
assert!(held_codes().contains(&10001));
|
||||||
|
release_matching(991, Some("keyboard-transport")).unwrap();
|
||||||
|
assert!(!held_codes().contains(&10001));
|
||||||
|
assert_eq!(
|
||||||
|
*keyboard.events.lock().unwrap(),
|
||||||
|
vec![(10001, true), (10001, false), (10001, false)]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,38 +1,8 @@
|
|||||||
//! Generic locked-HashMap plumbing for per-platform element caches.
|
//! Generic locked target directory for per-platform element caches.
|
||||||
//!
|
//!
|
||||||
//! Each platform crate stores a per-(pid, window) snapshot of the
|
//! Platform caches store weak references here. Native snapshots are owned by
|
||||||
//! actionable accessibility elements it last walked, so subsequent
|
//! the token registry and resolved actions, so replacing a directory entry
|
||||||
//! tool calls (`click`, `type_text`, etc.) can resolve an
|
//! cannot change the native object or metadata bound to an in-flight action.
|
||||||
//! `element_index` back to a native handle without re-walking the
|
|
||||||
//! tree. Before this module the three crates each owned a
|
|
||||||
//! near-identical `Mutex<HashMap<CacheKey, CachedSnapshot>>` plus
|
|
||||||
//! the same insert / lookup / count methods — see
|
|
||||||
//! `docs/dedup-audit.md` item #3.
|
|
||||||
//!
|
|
||||||
//! What lives here: the lock + HashMap, generic over the caller's
|
|
||||||
//! key type `K` and snapshot type `S`. What stays per-platform:
|
|
||||||
//!
|
|
||||||
//! - the concrete `CacheKey` (different pid/window-id widths)
|
|
||||||
//! - the concrete `CachedSnapshot` and its `Drop` impl, which on
|
|
||||||
//! macOS calls `CFRelease` on every cached AXUIElementRef and on
|
|
||||||
//! Windows calls COM `Release` on every IUIAutomationElement.
|
|
||||||
//! Those releases are load-bearing — dropping the wrong way leaks
|
|
||||||
//! the AX / UIA handles. The generic core just stores `S` by
|
|
||||||
//! value; when the entry is removed or replaced (via
|
|
||||||
//! `HashMap::insert`'s replace-return semantics) Rust runs `S`'s
|
|
||||||
//! destructor, so the platform `Drop` impl still fires exactly as
|
|
||||||
//! it did pre-refactor.
|
|
||||||
//!
|
|
||||||
//! Each platform's `ElementCache` is now a thin wrapper around an
|
|
||||||
//! `ElementCacheCore<CacheKey, CachedSnapshot>`. Specialised
|
|
||||||
//! accessors (`get_element_retained` on macOS and Windows,
|
|
||||||
//! `get_element_center` on Windows, `get_element_key` on Linux) call
|
|
||||||
//! `with_snapshot` and project the field they care about. The macOS and
|
|
||||||
//! Windows `get_element_retained` accessors retain the element under the
|
|
||||||
//! lock (CFRetain / COM AddRef) so a concurrent `insert` (snapshot replace)
|
|
||||||
//! can't free it mid-action — see
|
|
||||||
//! `platform-macos/src/ax/cache.rs::RetainedElement` and
|
|
||||||
//! `platform-windows/src/uia/cache.rs::RetainedElement`.
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::hash::Hash;
|
use std::hash::Hash;
|
||||||
@@ -54,10 +24,7 @@ impl<K: Eq + Hash, S> ElementCacheCore<K, S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replace the snapshot for `key`. If an entry already existed
|
/// Replace the entry for `key`, dropping the previous stored value.
|
||||||
/// its destructor runs here — for macOS that fires `CFRelease`
|
|
||||||
/// on every retained AXUIElementRef, for Windows it fires COM
|
|
||||||
/// `Release`, for Linux it's a no-op (just frees the `Vec<u64>`).
|
|
||||||
pub fn insert(&self, key: K, snapshot: S) {
|
pub fn insert(&self, key: K, snapshot: S) {
|
||||||
let mut inner = self.inner.lock().unwrap();
|
let mut inner = self.inner.lock().unwrap();
|
||||||
inner.insert(key, snapshot);
|
inner.insert(key, snapshot);
|
||||||
@@ -150,7 +117,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn insert_replaces_existing_and_runs_drop() {
|
fn insert_replaces_existing_and_runs_drop() {
|
||||||
// Smoke test that re-inserting under the same key replaces
|
// Smoke test that re-inserting under the same key replaces
|
||||||
// the prior snapshot — the platform Drop fires here in real
|
// the prior stored value — its Drop fires here in this
|
||||||
// code (CFRelease/COM Release). We can't exercise FFI in a
|
// code (CFRelease/COM Release). We can't exercise FFI in a
|
||||||
// unit test, but we can confirm the value was replaced.
|
// unit test, but we can confirm the value was replaced.
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|||||||
@@ -35,9 +35,8 @@
|
|||||||
//!
|
//!
|
||||||
//! ## Validity contract
|
//! ## Validity contract
|
||||||
//!
|
//!
|
||||||
//! - Snapshot IDs are minted in `register_snapshot` (called by every
|
//! - Native snapshots publish their ID and owned payload together in
|
||||||
//! platform's `get_window_state` implementation immediately after the
|
//! `register_snapshot_payload`. Resolution keeps that exact payload alive.
|
||||||
//! AX/UIA/AT-SPI walk lands in the per-platform element cache).
|
|
||||||
//! - A snapshot is valid until either (a) the LRU evicts it, or (b) a
|
//! - A snapshot is valid until either (a) the LRU evicts it, or (b) a
|
||||||
//! newer snapshot for the same `pid` pushes it past the LRU cap of
|
//! newer snapshot for the same `pid` pushes it past the LRU cap of
|
||||||
//! [`LRU_CAP_PER_PID`].
|
//! [`LRU_CAP_PER_PID`].
|
||||||
@@ -48,10 +47,11 @@
|
|||||||
//! The LRU is **per runtime and pid**, not global. Two snapshots in different
|
//! The LRU is **per runtime and pid**, not global. Two snapshots in different
|
||||||
//! lanes never collide even when their numeric counters match.
|
//! lanes never collide even when their numeric counters match.
|
||||||
|
|
||||||
|
use std::any::Any;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::sync::atomic::{AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicU32, Ordering};
|
||||||
use std::sync::Mutex;
|
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
/// LRU cap of valid snapshots retained per pid. Only the newest snapshot for
|
/// LRU cap of valid snapshots retained per pid. Only the newest snapshot for
|
||||||
/// each `(pid, window_id)` remains valid; the cap bounds snapshots retained for
|
/// each `(pid, window_id)` remains valid; the cap bounds snapshots retained for
|
||||||
@@ -82,6 +82,40 @@ struct SnapshotEntry {
|
|||||||
element_count: usize,
|
element_count: usize,
|
||||||
/// Linux window captures retain application-wide indices, including gaps.
|
/// Linux window captures retain application-wide indices, including gaps.
|
||||||
element_indices: Option<HashSet<usize>>,
|
element_indices: Option<HashSet<usize>>,
|
||||||
|
binding: Option<SnapshotBinding>,
|
||||||
|
owner_session: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An immutable native snapshot retained at the token-resolution boundary.
|
||||||
|
/// Type erasure keeps the core independent of platform native handle types.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct SnapshotBinding {
|
||||||
|
pid: i32,
|
||||||
|
window_id: u32,
|
||||||
|
payload: Arc<dyn Any + Send + Sync>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for SnapshotBinding {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("SnapshotBinding")
|
||||||
|
.field("pid", &self.pid)
|
||||||
|
.field("window_id", &self.window_id)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SnapshotBinding {
|
||||||
|
pub fn with_payload<S: Any, R>(
|
||||||
|
&self,
|
||||||
|
pid: i32,
|
||||||
|
window_id: u32,
|
||||||
|
read: impl FnOnce(&S) -> R,
|
||||||
|
) -> Option<R> {
|
||||||
|
if self.pid != pid || self.window_id != window_id {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
self.payload.downcast_ref::<S>().map(read)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Process-global token registry. Thread-safe; tools resolve from any
|
/// Process-global token registry. Thread-safe; tools resolve from any
|
||||||
@@ -114,7 +148,7 @@ impl TokenRegistry {
|
|||||||
/// in its lane, the oldest is evicted and any token that referenced
|
/// in its lane, the oldest is evicted and any token that referenced
|
||||||
/// it becomes stale — that's the contract.
|
/// it becomes stale — that's the contract.
|
||||||
pub fn register_snapshot(&self, pid: i32, window_id: u32, element_count: usize) -> u32 {
|
pub fn register_snapshot(&self, pid: i32, window_id: u32, element_count: usize) -> u32 {
|
||||||
self.register(pid, window_id, element_count, None)
|
self.register(pid, window_id, element_count, None, None, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn register_snapshot_indices(
|
pub fn register_snapshot_indices(
|
||||||
@@ -124,7 +158,40 @@ impl TokenRegistry {
|
|||||||
indices: impl IntoIterator<Item = usize>,
|
indices: impl IntoIterator<Item = usize>,
|
||||||
) -> u32 {
|
) -> u32 {
|
||||||
let indices: HashSet<usize> = indices.into_iter().collect();
|
let indices: HashSet<usize> = indices.into_iter().collect();
|
||||||
self.register(pid, window_id, indices.len(), Some(indices))
|
self.register(pid, window_id, indices.len(), Some(indices), None, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn register_snapshot_payload<S: Any + Send + Sync>(
|
||||||
|
&self,
|
||||||
|
pid: i32,
|
||||||
|
window_id: u32,
|
||||||
|
indices: impl IntoIterator<Item = usize>,
|
||||||
|
payload: Arc<S>,
|
||||||
|
) -> u32 {
|
||||||
|
self.register_snapshot_payload_for_session(pid, window_id, indices, payload, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn register_snapshot_payload_for_session<S: Any + Send + Sync>(
|
||||||
|
&self,
|
||||||
|
pid: i32,
|
||||||
|
window_id: u32,
|
||||||
|
indices: impl IntoIterator<Item = usize>,
|
||||||
|
payload: Arc<S>,
|
||||||
|
owner_session: Option<&str>,
|
||||||
|
) -> u32 {
|
||||||
|
let indices: HashSet<usize> = indices.into_iter().collect();
|
||||||
|
self.register(
|
||||||
|
pid,
|
||||||
|
window_id,
|
||||||
|
indices.len(),
|
||||||
|
Some(indices),
|
||||||
|
Some(SnapshotBinding {
|
||||||
|
pid,
|
||||||
|
window_id,
|
||||||
|
payload,
|
||||||
|
}),
|
||||||
|
owner_session.map(str::to_owned),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn register(
|
fn register(
|
||||||
@@ -133,6 +200,8 @@ impl TokenRegistry {
|
|||||||
window_id: u32,
|
window_id: u32,
|
||||||
element_count: usize,
|
element_count: usize,
|
||||||
element_indices: Option<HashSet<usize>>,
|
element_indices: Option<HashSet<usize>>,
|
||||||
|
binding: Option<SnapshotBinding>,
|
||||||
|
owner_session: Option<String>,
|
||||||
) -> u32 {
|
) -> u32 {
|
||||||
// Keep the full 32-bit counter. Truncating to 16 bits repeats an id
|
// Keep the full 32-bit counter. Truncating to 16 bits repeats an id
|
||||||
// every 65,536 process-global snapshots. A long-lived daemon can then
|
// every 65,536 process-global snapshots. A long-lived daemon can then
|
||||||
@@ -142,16 +211,16 @@ impl TokenRegistry {
|
|||||||
let runtime_scope = current_runtime_scope();
|
let runtime_scope = current_runtime_scope();
|
||||||
let mut entries = self.by_runtime_and_pid.lock().unwrap();
|
let mut entries = self.by_runtime_and_pid.lock().unwrap();
|
||||||
let lane = entries.entry((runtime_scope, pid)).or_default();
|
let lane = entries.entry((runtime_scope, pid)).or_default();
|
||||||
// Every platform replaces its per-window element cache on the next
|
// Publication invalidates future resolutions of the preceding window
|
||||||
// snapshot. Retaining an older token for the same window would resolve
|
// snapshot. Already-resolved actions retain their original payload.
|
||||||
// that token against the new cache and could target a different row.
|
|
||||||
// Invalidate it at the same boundary as the cache replacement.
|
|
||||||
lane.retain(|entry| entry.window_id != window_id);
|
lane.retain(|entry| entry.window_id != window_id);
|
||||||
lane.push(SnapshotEntry {
|
lane.push(SnapshotEntry {
|
||||||
snapshot_id: id,
|
snapshot_id: id,
|
||||||
window_id,
|
window_id,
|
||||||
element_count,
|
element_count,
|
||||||
element_indices,
|
element_indices,
|
||||||
|
binding,
|
||||||
|
owner_session,
|
||||||
});
|
});
|
||||||
// Evict oldest. The loop guards against pre-existing over-cap
|
// Evict oldest. The loop guards against pre-existing over-cap
|
||||||
// state from a previous version of the binary; in steady state
|
// state from a previous version of the binary; in steady state
|
||||||
@@ -174,6 +243,15 @@ impl TokenRegistry {
|
|||||||
/// - `"element_token element_index out of range"` — the index in
|
/// - `"element_token element_index out of range"` — the index in
|
||||||
/// the token is past the max recorded for the snapshot.
|
/// the token is past the max recorded for the snapshot.
|
||||||
pub fn resolve(&self, pid: i32, token: &str) -> Result<(u32, usize), String> {
|
pub fn resolve(&self, pid: i32, token: &str) -> Result<(u32, usize), String> {
|
||||||
|
self.resolve_bound(pid, token)
|
||||||
|
.map(|(window, index, _)| (window, index))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn resolve_bound(
|
||||||
|
&self,
|
||||||
|
pid: i32,
|
||||||
|
token: &str,
|
||||||
|
) -> Result<(u32, usize, Option<SnapshotBinding>), String> {
|
||||||
let (sid, idx) =
|
let (sid, idx) =
|
||||||
parse_token(token).ok_or_else(|| "element_token has invalid format".to_string())?;
|
parse_token(token).ok_or_else(|| "element_token has invalid format".to_string())?;
|
||||||
let runtime_scope = current_runtime_scope();
|
let runtime_scope = current_runtime_scope();
|
||||||
@@ -209,7 +287,17 @@ impl TokenRegistry {
|
|||||||
entry.element_count
|
entry.element_count
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
Ok((entry.window_id, idx))
|
Ok((entry.window_id, idx, entry.binding.clone()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn clear_session(&self, session_id: &str) -> usize {
|
||||||
|
let mut entries = self.by_runtime_and_pid.lock().unwrap();
|
||||||
|
let before = entries.values().map(Vec::len).sum::<usize>();
|
||||||
|
entries.retain(|_, snapshots| {
|
||||||
|
snapshots.retain(|entry| entry.owner_session.as_deref() != Some(session_id));
|
||||||
|
!snapshots.is_empty()
|
||||||
|
});
|
||||||
|
before - entries.values().map(Vec::len).sum::<usize>()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn clear_runtime_scope(&self, runtime_scope: &str) -> usize {
|
pub fn clear_runtime_scope(&self, runtime_scope: &str) -> usize {
|
||||||
@@ -229,6 +317,18 @@ impl TokenRegistry {
|
|||||||
before - entries.len()
|
before - entries.len()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Invalidate one observation target in the calling runtime only. A failed
|
||||||
|
/// observation is not proof that another runtime's target has disappeared.
|
||||||
|
pub fn clear_current_target(&self, pid: i32, window_id: u32) -> usize {
|
||||||
|
let mut entries = self.by_runtime_and_pid.lock().unwrap();
|
||||||
|
let Some(snapshots) = entries.get_mut(&(current_runtime_scope(), pid)) else {
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
let before = snapshots.len();
|
||||||
|
snapshots.retain(|entry| entry.window_id != window_id);
|
||||||
|
before - snapshots.len()
|
||||||
|
}
|
||||||
|
|
||||||
/// Invalidate snapshots for one exact process/window target across runtime
|
/// Invalidate snapshots for one exact process/window target across runtime
|
||||||
/// lanes while preserving snapshots for the process's other live windows.
|
/// lanes while preserving snapshots for the process's other live windows.
|
||||||
pub fn clear_target(&self, pid: i32, window_id: u32) -> usize {
|
pub fn clear_target(&self, pid: i32, window_id: u32) -> usize {
|
||||||
@@ -374,9 +474,19 @@ pub enum ResolvedElement {
|
|||||||
/// through the registry — informational, used by tools that
|
/// through the registry — informational, used by tools that
|
||||||
/// want to log "via token" in the success summary.
|
/// want to log "via token" in the success summary.
|
||||||
via_token: bool,
|
via_token: bool,
|
||||||
|
binding: Option<SnapshotBinding>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ResolvedElement {
|
||||||
|
pub fn binding(&self) -> Option<SnapshotBinding> {
|
||||||
|
match self {
|
||||||
|
Self::Element { binding, .. } => binding.clone(),
|
||||||
|
Self::None => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Validate tool args that accept `element_index`, `snapshot_id`, and
|
/// Validate tool args that accept `element_index`, `snapshot_id`, and
|
||||||
/// `element_token`. Returns either a structured refusal or the resolved
|
/// `element_token`. Returns either a structured refusal or the resolved
|
||||||
/// `(window_id, element_index)` pair.
|
/// `(window_id, element_index)` pair.
|
||||||
@@ -418,8 +528,8 @@ pub fn resolve_element_args(
|
|||||||
.resolve_current(pid, tok)
|
.resolve_current(pid, tok)
|
||||||
.map_err(|error| refusal(error.code(), error.to_string()))?;
|
.map_err(|error| refusal(error.code(), error.to_string()))?;
|
||||||
let snapshot_token = token_for(snapshot_id, element_index);
|
let snapshot_token = token_for(snapshot_id, element_index);
|
||||||
let (snapshot_window_id, snapshot_element_index) = global()
|
let (snapshot_window_id, snapshot_element_index, binding) = global()
|
||||||
.resolve(pid, &snapshot_token)
|
.resolve_bound(pid, &snapshot_token)
|
||||||
.map_err(|message| refusal("stale_element_token", message))?;
|
.map_err(|message| refusal("stale_element_token", message))?;
|
||||||
if snapshot_window_id != window_id || snapshot_element_index != element_index {
|
if snapshot_window_id != window_id || snapshot_element_index != element_index {
|
||||||
return Err(refusal(
|
return Err(refusal(
|
||||||
@@ -428,9 +538,9 @@ pub fn resolve_element_args(
|
|||||||
.to_owned(),
|
.to_owned(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
return Ok((window_id, element_index));
|
return Ok((window_id, element_index, binding));
|
||||||
}
|
}
|
||||||
let (wid, idx) = global().resolve(pid, tok).map_err(|message| {
|
let (wid, idx, binding) = global().resolve_bound(pid, tok).map_err(|message| {
|
||||||
let code = if message.contains("another runtime generation") {
|
let code = if message.contains("another runtime generation") {
|
||||||
"generation_mismatch"
|
"generation_mismatch"
|
||||||
} else if message == STALE_TOKEN_ERROR {
|
} else if message == STALE_TOKEN_ERROR {
|
||||||
@@ -440,7 +550,7 @@ pub fn resolve_element_args(
|
|||||||
};
|
};
|
||||||
refusal(code, message)
|
refusal(code, message)
|
||||||
})?;
|
})?;
|
||||||
Ok::<_, crate::protocol::ToolResult>((wid, idx))
|
Ok::<_, crate::protocol::ToolResult>((wid, idx, binding))
|
||||||
};
|
};
|
||||||
|
|
||||||
match (args_element_index, args_element_token, args_snapshot_id) {
|
match (args_element_index, args_element_token, args_snapshot_id) {
|
||||||
@@ -463,7 +573,7 @@ pub fn resolve_element_args(
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let token = format_token(snapshot_id, idx);
|
let token = format_token(snapshot_id, idx);
|
||||||
let (wid, resolved_idx) = resolve_token(&token)?;
|
let (wid, resolved_idx, binding) = resolve_token(&token)?;
|
||||||
if args_window_id.is_some_and(|arg_wid| arg_wid != wid) {
|
if args_window_id.is_some_and(|arg_wid| arg_wid != wid) {
|
||||||
return Err(refusal(
|
return Err(refusal(
|
||||||
"conflicting_element_target",
|
"conflicting_element_target",
|
||||||
@@ -477,10 +587,11 @@ pub fn resolve_element_args(
|
|||||||
window_id: Some(wid),
|
window_id: Some(wid),
|
||||||
element_index: resolved_idx,
|
element_index: resolved_idx,
|
||||||
via_token: false,
|
via_token: false,
|
||||||
|
binding,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
(idx_opt, Some(tok), snapshot_opt) => {
|
(idx_opt, Some(tok), snapshot_opt) => {
|
||||||
let (wid, idx) = resolve_token(tok)?;
|
let (wid, idx, binding) = resolve_token(tok)?;
|
||||||
if idx_opt.is_some_and(|arg_idx| arg_idx != idx)
|
if idx_opt.is_some_and(|arg_idx| arg_idx != idx)
|
||||||
|| args_window_id.is_some_and(|arg_wid| arg_wid != wid)
|
|| args_window_id.is_some_and(|arg_wid| arg_wid != wid)
|
||||||
|| snapshot_opt.is_some_and(|handle| {
|
|| snapshot_opt.is_some_and(|handle| {
|
||||||
@@ -499,6 +610,7 @@ pub fn resolve_element_args(
|
|||||||
window_id: Some(wid),
|
window_id: Some(wid),
|
||||||
element_index: idx,
|
element_index: idx,
|
||||||
via_token: true,
|
via_token: true,
|
||||||
|
binding,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -508,6 +620,118 @@ pub fn resolve_element_args(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
#[test]
|
||||||
|
fn resolved_payload_does_not_switch_to_a_replacement_snapshot() {
|
||||||
|
let registry = super::TokenRegistry::new();
|
||||||
|
let first = registry.register_snapshot_payload(42, 7, [130], std::sync::Arc::new("first"));
|
||||||
|
let (_, _, binding) = registry
|
||||||
|
.resolve_bound(42, &super::token_for(first, 130))
|
||||||
|
.unwrap();
|
||||||
|
let binding = binding.unwrap();
|
||||||
|
registry.register_snapshot_payload(42, 7, [130], std::sync::Arc::new("replacement"));
|
||||||
|
assert_eq!(
|
||||||
|
binding.with_payload(42, 7, |value: &&str| *value),
|
||||||
|
Some("first")
|
||||||
|
);
|
||||||
|
assert_eq!(binding.with_payload(42, 8, |value: &&str| *value), None);
|
||||||
|
assert_eq!(binding.with_payload(43, 7, |value: &&str| *value), None);
|
||||||
|
assert_eq!(
|
||||||
|
registry.resolve(42, &super::token_for(first, 130)),
|
||||||
|
Err(super::STALE_TOKEN_ERROR.into())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delayed_publication_owns_its_payload_instead_of_the_latest_window_cache() {
|
||||||
|
let registry = std::sync::Arc::new(super::TokenRegistry::new());
|
||||||
|
let (prepared_tx, prepared_rx) = std::sync::mpsc::channel();
|
||||||
|
let (publish_tx, publish_rx) = std::sync::mpsc::channel();
|
||||||
|
let delayed = registry.clone();
|
||||||
|
let first = std::thread::spawn(move || {
|
||||||
|
let pending = std::sync::Arc::new("first");
|
||||||
|
prepared_tx.send(()).unwrap();
|
||||||
|
publish_rx.recv().unwrap();
|
||||||
|
delayed.register_snapshot_payload(42, 7, [0], pending)
|
||||||
|
});
|
||||||
|
prepared_rx.recv().unwrap();
|
||||||
|
let second = registry.register_snapshot_payload(42, 7, [0], std::sync::Arc::new("second"));
|
||||||
|
publish_tx.send(()).unwrap();
|
||||||
|
let first = first.join().unwrap();
|
||||||
|
let (_, _, binding) = registry
|
||||||
|
.resolve_bound(42, &super::token_for(first, 0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
binding.unwrap().with_payload(42, 7, |value: &&str| *value),
|
||||||
|
Some("first")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
registry.resolve(42, &super::token_for(second, 0)),
|
||||||
|
Err(super::STALE_TOKEN_ERROR.into())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn snapshot_cleanup_releases_payload_after_the_last_resolved_action() {
|
||||||
|
struct Payload(std::sync::Arc<std::sync::atomic::AtomicUsize>);
|
||||||
|
impl Drop for Payload {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let drops = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||||
|
let registry = super::TokenRegistry::new();
|
||||||
|
let first = registry.register_snapshot_payload(
|
||||||
|
42,
|
||||||
|
7,
|
||||||
|
[0],
|
||||||
|
std::sync::Arc::new(Payload(drops.clone())),
|
||||||
|
);
|
||||||
|
let (_, _, binding) = registry
|
||||||
|
.resolve_bound(42, &super::token_for(first, 0))
|
||||||
|
.unwrap();
|
||||||
|
registry.clear_runtime_scope("legacy");
|
||||||
|
assert!(registry.resolve(42, &super::token_for(first, 0)).is_err());
|
||||||
|
assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||||
|
drop(binding);
|
||||||
|
assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||||
|
registry.register_snapshot_payload(42, 8, [0], std::sync::Arc::new(Payload(drops.clone())));
|
||||||
|
registry.clear_target(42, 8);
|
||||||
|
assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 2);
|
||||||
|
registry.register_snapshot_payload(42, 9, [0], std::sync::Arc::new(Payload(drops.clone())));
|
||||||
|
for window in 10..10 + super::LRU_CAP_PER_PID as u32 {
|
||||||
|
registry.register_snapshot(42, window, 1);
|
||||||
|
}
|
||||||
|
assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn observation_invalidation_preserves_peer_runtime_and_resolved_payload() {
|
||||||
|
use crate::tool::with_runtime_scope;
|
||||||
|
let registry = super::TokenRegistry::new();
|
||||||
|
let peer = with_runtime_scope("invalidation-peer".into(), || {
|
||||||
|
registry.register_snapshot(42, 7, 1)
|
||||||
|
});
|
||||||
|
with_runtime_scope("invalidation-owner".into(), || {
|
||||||
|
let sid = registry.register_snapshot_payload(42, 7, [0], Arc::new("original"));
|
||||||
|
let sibling = registry.register_snapshot(42, 8, 1);
|
||||||
|
let token = super::token_for(sid, 0);
|
||||||
|
let binding = registry.resolve_bound(42, &token).unwrap().2.unwrap();
|
||||||
|
assert_eq!(registry.clear_current_target(42, 7), 1);
|
||||||
|
assert!(registry.resolve(42, &token).is_err());
|
||||||
|
assert!(registry.resolve(42, &super::token_for(sibling, 0)).is_ok());
|
||||||
|
assert_eq!(binding.with_payload(42, 7, |s: &&str| *s), Some("original"));
|
||||||
|
assert_eq!(registry.clear_current_target(42, 7), 0);
|
||||||
|
});
|
||||||
|
with_runtime_scope("invalidation-peer".into(), || {
|
||||||
|
assert!(registry.resolve(42, &super::token_for(peer, 0)).is_ok());
|
||||||
|
});
|
||||||
|
// A confirmed target close still revokes every runtime lane.
|
||||||
|
assert_eq!(registry.clear_target(42, 7), 1);
|
||||||
|
with_runtime_scope("invalidation-peer".into(), || {
|
||||||
|
assert!(registry.resolve(42, &super::token_for(peer, 0)).is_err());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sparse_indices_are_membership_checked_and_window_scoped() {
|
fn sparse_indices_are_membership_checked_and_window_scoped() {
|
||||||
let registry = super::TokenRegistry::new();
|
let registry = super::TokenRegistry::new();
|
||||||
@@ -797,6 +1021,7 @@ mod tests {
|
|||||||
window_id,
|
window_id,
|
||||||
element_index,
|
element_index,
|
||||||
via_token,
|
via_token,
|
||||||
|
..
|
||||||
} => {
|
} => {
|
||||||
assert_eq!(window_id, Some(555), "window_id comes from the snapshot");
|
assert_eq!(window_id, Some(555), "window_id comes from the snapshot");
|
||||||
assert_eq!(element_index, 2);
|
assert_eq!(element_index, 2);
|
||||||
@@ -829,7 +1054,8 @@ mod tests {
|
|||||||
ResolvedElement::Element {
|
ResolvedElement::Element {
|
||||||
window_id: Some(888),
|
window_id: Some(888),
|
||||||
element_index: 2,
|
element_index: 2,
|
||||||
via_token: false
|
via_token: false,
|
||||||
|
..
|
||||||
}
|
}
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,11 +54,13 @@ pub mod capture_mode;
|
|||||||
pub mod capture_scope;
|
pub mod capture_scope;
|
||||||
pub mod cdp;
|
pub mod cdp;
|
||||||
pub mod clipboard;
|
pub mod clipboard;
|
||||||
|
mod computer_use;
|
||||||
pub mod consent;
|
pub mod consent;
|
||||||
pub mod coord_norm;
|
pub mod coord_norm;
|
||||||
pub mod cursor_events;
|
pub mod cursor_events;
|
||||||
pub mod cursor_sampler;
|
pub mod cursor_sampler;
|
||||||
pub mod daemon;
|
pub mod daemon;
|
||||||
|
pub mod desktop_keyboard;
|
||||||
pub mod element_cache;
|
pub mod element_cache;
|
||||||
pub mod element_query;
|
pub mod element_query;
|
||||||
pub mod element_token;
|
pub mod element_token;
|
||||||
|
|||||||
@@ -326,6 +326,18 @@ pub struct ToolResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ToolResult {
|
impl ToolResult {
|
||||||
|
pub fn computer_use_error(code: &str, reason: impl Into<String>, sent: bool) -> Self {
|
||||||
|
let reason = reason.into();
|
||||||
|
let next = if sent {
|
||||||
|
"Observe the app state before deciding whether to retry."
|
||||||
|
} else {
|
||||||
|
"Observe the app again and use its current target."
|
||||||
|
};
|
||||||
|
Self::error(format!("{code}: {reason} {next}")).with_structured(serde_json::json!({
|
||||||
|
"code":code,"reason":reason,"sent":sent,"next":next,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn text(msg: impl Into<String>) -> Self {
|
pub fn text(msg: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
content: vec![Content::text(msg)],
|
content: vec![Content::text(msg)],
|
||||||
@@ -378,6 +390,35 @@ pub fn initialize_result() -> Value {
|
|||||||
/// Goose uses in its `ComputerController` extension and Open
|
/// Goose uses in its `ComputerController` extension and Open
|
||||||
/// Interpreter uses for its system message.
|
/// Interpreter uses for its system message.
|
||||||
fn agent_instructions() -> String {
|
fn agent_instructions() -> String {
|
||||||
|
if std::env::var("CUA_DRIVER_EXTRA_TOOLS").as_deref() == Ok("1") {
|
||||||
|
legacy_agent_instructions()
|
||||||
|
} else {
|
||||||
|
computer_use_instructions()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn computer_use_instructions() -> String {
|
||||||
|
let platform = if cfg!(target_os = "macos") {
|
||||||
|
"macOS: application names, bundle IDs or absolute application paths."
|
||||||
|
} else if cfg!(target_os = "windows") {
|
||||||
|
"Windows: exact application names or executable paths."
|
||||||
|
} else {
|
||||||
|
"Linux: X11 applications only; inherit DISPLAY, XAUTHORITY and the accessibility session bus."
|
||||||
|
};
|
||||||
|
format!(
|
||||||
|
r#"cua-driver Computer Use. {platform}
|
||||||
|
|
||||||
|
Prefer client-provided APIs, CLI or filesystem operations for non-GUI outcomes. This server has no shell.
|
||||||
|
|
||||||
|
Use `get_state` for inventory, then `get_app(app)` for full AX state. `get_ax_state(app)` returns subsequent diffs; `force_full:true` requests a full tree. Act with `app` and a current short `element` ID. Observe again after actions; an acknowledgement does not prove the UI changed. Never replay uncertain input automatically.
|
||||||
|
|
||||||
|
Request `get_screenshot(app)` before point actions. Coordinates are window screenshot pixels, independent of legacy coordinate settings. For explicit desktop tasks, use `get_screenshot(scope:"desktop")`, then desktop input with `scope:"desktop"` and no `app`. Desktop coordinates are latest primary-display screenshot pixels. `key_down/key_up` are desktop-only; release held keys in finally and before app input or ordinary text/chords.
|
||||||
|
|
||||||
|
`start_session` is optional. For multi-call work, prefer a short `session` label and repeat it on every call that accepts it. Unnamed calls use the transport's implicit session. Only `start_session` revives an ended name; `end_session` explicitly cleans up."#
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn legacy_agent_instructions() -> String {
|
||||||
let (tree_kind, platform_skill_pointer) = if cfg!(target_os = "macos") {
|
let (tree_kind, platform_skill_pointer) = if cfg!(target_os = "macos") {
|
||||||
(
|
(
|
||||||
"AX (Accessibility)",
|
"AX (Accessibility)",
|
||||||
@@ -391,7 +432,7 @@ fn agent_instructions() -> String {
|
|||||||
} else {
|
} else {
|
||||||
(
|
(
|
||||||
"AT-SPI",
|
"AT-SPI",
|
||||||
"LINUX.md (X11/Wayland status, AT-SPI bus, BETA-level support)",
|
"LINUX.md (X11 support, AT-SPI bus, BETA-level support)",
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -399,9 +440,9 @@ fn agent_instructions() -> String {
|
|||||||
coordinate_terms(crate::coord_norm::default_normalized());
|
coordinate_terms(crate::coord_norm::default_normalized());
|
||||||
|
|
||||||
format!(
|
format!(
|
||||||
r#"cua-driver: cross-platform background computer-use automation.
|
r#"cua-driver: extra native tools. App/scope calls always use screenshot pixels; this guidance applies to raw tools.
|
||||||
|
|
||||||
Before starting UI work, classify the desired postcondition. For a non-GUI outcome, prefer a client-provided app API/SDK, headless/background interface, CLI, or filesystem operation and read the result back in that semantic domain. This server has no shell.
|
For a non-GUI outcome, prefer a client-provided app API/SDK, headless/background interface, CLI, or filesystem operation and read the result back in that semantic domain. This server has no shell.
|
||||||
|
|
||||||
For an app or window outcome, use the narrowest semantic Cua route first: `set_window_frame` plus `list_windows` readback for geometry, typed browser tools for supported page content, and clipboard tools for clipboard state. Then climb through background `element_index` ({tree_kind}), background {coordinate_term}, foreground delivery, and desktop fallback.{coordinate_note} Never advance on transport success alone.
|
For an app or window outcome, use the narrowest semantic Cua route first: `set_window_frame` plus `list_windows` readback for geometry, typed browser tools for supported page content, and clipboard tools for clipboard state. Then climb through background `element_index` ({tree_kind}), background {coordinate_term}, foreground delivery, and desktop fallback.{coordinate_note} Never advance on transport success alone.
|
||||||
|
|
||||||
@@ -559,11 +600,11 @@ mod action_record_wire_tests {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod agent_instruction_tests {
|
mod agent_instruction_tests {
|
||||||
use super::{agent_instructions, initialize_result};
|
use super::{computer_use_instructions, initialize_result, legacy_agent_instructions};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn instructions_route_structured_and_visual_verification_to_the_right_owner() {
|
fn instructions_route_structured_and_visual_verification_to_the_right_owner() {
|
||||||
let instructions = agent_instructions();
|
let instructions = legacy_agent_instructions();
|
||||||
assert!(instructions.contains("verify_state"));
|
assert!(instructions.contains("verify_state"));
|
||||||
assert!(instructions.contains("`unknown` is not success"));
|
assert!(instructions.contains("`unknown` is not success"));
|
||||||
assert!(instructions.contains("multimodal agent"));
|
assert!(instructions.contains("multimodal agent"));
|
||||||
@@ -585,6 +626,33 @@ mod agent_instruction_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compact_instructions_match_app_desktop_and_pixel_consumers() {
|
||||||
|
let instructions = computer_use_instructions();
|
||||||
|
for required in [
|
||||||
|
"get_app(app)",
|
||||||
|
"get_ax_state(app)",
|
||||||
|
"short `element` ID",
|
||||||
|
"window screenshot pixels",
|
||||||
|
"primary-display screenshot pixels",
|
||||||
|
"scope:\"desktop\"",
|
||||||
|
"key_down/key_up",
|
||||||
|
"Never replay uncertain input",
|
||||||
|
] {
|
||||||
|
assert!(instructions.contains(required), "missing {required}");
|
||||||
|
}
|
||||||
|
for obsolete in [
|
||||||
|
"element_index",
|
||||||
|
"get_window_state",
|
||||||
|
"verify_state",
|
||||||
|
"0-1000",
|
||||||
|
"SKILL.md",
|
||||||
|
] {
|
||||||
|
assert!(!instructions.contains(obsolete), "obsolete {obsolete}");
|
||||||
|
}
|
||||||
|
assert!(instructions.split_whitespace().count() <= 200);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn initialize_instructions_describe_implicit_session_lifecycle() {
|
fn initialize_instructions_describe_implicit_session_lifecycle() {
|
||||||
let result = initialize_result();
|
let result = initialize_result();
|
||||||
|
|||||||
@@ -9,9 +9,10 @@
|
|||||||
//! { tool, arguments, result_summary, result_error, timestamp,
|
//! { tool, arguments, result_summary, result_error, timestamp,
|
||||||
//! t_ms_from_session_start, t_start_ms_from_session_start }
|
//! t_ms_from_session_start, t_start_ms_from_session_start }
|
||||||
|
|
||||||
|
use std::collections::HashSet;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Mutex, OnceLock};
|
use std::sync::{Mutex, OnceLock};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
@@ -109,6 +110,69 @@ pub fn set_ax_snapshot_fn(
|
|||||||
let _ = AX_SNAPSHOT_FN.set(Box::new(f));
|
let _ = AX_SNAPSHOT_FN.set(Box::new(f));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cooperative native walk budget; the callback backstop also bounds providers
|
||||||
|
/// that do not return control to their walker.
|
||||||
|
pub const RECORDING_AX_WALK_BUDGET: Duration = Duration::from_secs(1);
|
||||||
|
const AX_CAPTURE_BACKSTOP: Duration = Duration::from_secs(2);
|
||||||
|
const MAX_AX_CAPTURE_WORKERS: usize = 4;
|
||||||
|
|
||||||
|
fn active_ax_captures() -> &'static Mutex<HashSet<Option<i64>>> {
|
||||||
|
static ACTIVE: OnceLock<Mutex<HashSet<Option<i64>>>> = OnceLock::new();
|
||||||
|
ACTIVE.get_or_init(|| Mutex::new(HashSet::new()))
|
||||||
|
}
|
||||||
|
|
||||||
|
struct AxCaptureSlot(Option<i64>);
|
||||||
|
|
||||||
|
impl AxCaptureSlot {
|
||||||
|
fn acquire(pid: Option<i64>) -> Option<Self> {
|
||||||
|
let mut active = active_ax_captures()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|p| p.into_inner());
|
||||||
|
if active.len() >= MAX_AX_CAPTURE_WORKERS || !active.insert(pid) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(Self(pid))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for AxCaptureSlot {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
active_ax_captures()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|p| p.into_inner())
|
||||||
|
.remove(&self.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capture_ax_state(
|
||||||
|
window_id: Option<u64>,
|
||||||
|
pid: Option<i64>,
|
||||||
|
) -> (Option<Vec<u8>>, Option<&'static str>) {
|
||||||
|
let Some(hook) = AX_SNAPSHOT_FN.get() else {
|
||||||
|
return (None, Some("capture_hook_unavailable"));
|
||||||
|
};
|
||||||
|
let Some(slot) = AxCaptureSlot::acquire(pid) else {
|
||||||
|
return (None, Some("state_capture_busy"));
|
||||||
|
};
|
||||||
|
let context = crate::tool::CapturedDispatchContext::capture();
|
||||||
|
let (sender, receiver) = std::sync::mpsc::sync_channel(1);
|
||||||
|
let worker = std::thread::Builder::new()
|
||||||
|
.name("cua-recording-state".into())
|
||||||
|
.spawn(move || {
|
||||||
|
let result = context.run_sync(|| hook(window_id, pid));
|
||||||
|
drop(slot);
|
||||||
|
let _ = sender.send(result);
|
||||||
|
});
|
||||||
|
if worker.is_err() {
|
||||||
|
return (None, Some("capture_failed"));
|
||||||
|
}
|
||||||
|
match receiver.recv_timeout(AX_CAPTURE_BACKSTOP) {
|
||||||
|
Ok(Some(bytes)) => (Some(bytes), None),
|
||||||
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => (None, Some("state_capture_timeout")),
|
||||||
|
_ => (None, Some("capture_failed")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Platform element-bounds callback ─────────────────────────────────────────
|
// ── Platform element-bounds callback ─────────────────────────────────────────
|
||||||
//
|
//
|
||||||
// Resolves an element_index to its center point in window-local screenshot
|
// Resolves an element_index to its center point in window-local screenshot
|
||||||
@@ -116,12 +180,19 @@ pub fn set_ax_snapshot_fn(
|
|||||||
// `CLICK_MARKER_FN`). Used so click.png is also written on element-indexed
|
// `CLICK_MARKER_FN`). Used so click.png is also written on element-indexed
|
||||||
// clicks, not just pixel-addressed ones.
|
// clicks, not just pixel-addressed ones.
|
||||||
|
|
||||||
type ElementBoundsFnBox = Box<dyn Fn(u64, i64, u32) -> Option<(f64, f64)> + Send + Sync>;
|
type ElementBoundsFnBox = Box<
|
||||||
|
dyn Fn(u64, i64, u32, &crate::element_token::SnapshotBinding) -> Option<(f64, f64)>
|
||||||
|
+ Send
|
||||||
|
+ Sync,
|
||||||
|
>;
|
||||||
static ELEMENT_BOUNDS_FN: OnceLock<ElementBoundsFnBox> = OnceLock::new();
|
static ELEMENT_BOUNDS_FN: OnceLock<ElementBoundsFnBox> = OnceLock::new();
|
||||||
|
|
||||||
/// Register the platform-specific element-bounds resolver. Args: (window_id, pid, element_index).
|
/// Register the platform-specific element-bounds resolver. Args: (window_id, pid, element_index, snapshot_binding).
|
||||||
pub fn set_element_bounds_fn(
|
pub fn set_element_bounds_fn(
|
||||||
f: impl Fn(u64, i64, u32) -> Option<(f64, f64)> + Send + Sync + 'static,
|
f: impl Fn(u64, i64, u32, &crate::element_token::SnapshotBinding) -> Option<(f64, f64)>
|
||||||
|
+ Send
|
||||||
|
+ Sync
|
||||||
|
+ 'static,
|
||||||
) {
|
) {
|
||||||
let _ = ELEMENT_BOUNDS_FN.set(Box::new(f));
|
let _ = ELEMENT_BOUNDS_FN.set(Box::new(f));
|
||||||
}
|
}
|
||||||
@@ -129,6 +200,7 @@ pub fn set_element_bounds_fn(
|
|||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct TurnCapture {
|
struct TurnCapture {
|
||||||
state: Option<Vec<u8>>,
|
state: Option<Vec<u8>>,
|
||||||
|
state_classification: Option<&'static str>,
|
||||||
screenshot: Option<Vec<u8>>,
|
screenshot: Option<Vec<u8>>,
|
||||||
screenshot_classification: Option<&'static str>,
|
screenshot_classification: Option<&'static str>,
|
||||||
}
|
}
|
||||||
@@ -138,6 +210,7 @@ struct TurnCapture {
|
|||||||
/// one stable `turn-NNNNN` directory even when calls complete out of order.
|
/// one stable `turn-NNNNN` directory even when calls complete out of order.
|
||||||
pub struct PendingTurn {
|
pub struct PendingTurn {
|
||||||
generation: u64,
|
generation: u64,
|
||||||
|
owner: Option<String>,
|
||||||
turn_dir: PathBuf,
|
turn_dir: PathBuf,
|
||||||
tool_name: String,
|
tool_name: String,
|
||||||
args: Value,
|
args: Value,
|
||||||
@@ -150,7 +223,13 @@ pub struct PendingTurn {
|
|||||||
before: TurnCapture,
|
before: TurnCapture,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persistent recording session state (singleton per process).
|
#[derive(Debug, Clone, Copy, Default)]
|
||||||
|
pub struct RecordingCaller<'a> {
|
||||||
|
pub session: Option<&'a str>,
|
||||||
|
pub transport: Option<&'a str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persistent recording session state owned by one tool registry.
|
||||||
pub struct RecordingSession {
|
pub struct RecordingSession {
|
||||||
inner: Mutex<RecordingInner>,
|
inner: Mutex<RecordingInner>,
|
||||||
}
|
}
|
||||||
@@ -332,9 +411,10 @@ impl RecordingSession {
|
|||||||
// Write initial session.json — final video metadata is rewritten on
|
// Write initial session.json — final video metadata is rewritten on
|
||||||
// stop. We mark `present` based on whether ffmpeg actually started,
|
// stop. We mark `present` based on whether ffmpeg actually started,
|
||||||
// not just whether the caller asked for video.
|
// not just whether the caller asked for video.
|
||||||
|
let started_at_ms = now_ms();
|
||||||
let session_payload = serde_json::json!({
|
let session_payload = serde_json::json!({
|
||||||
"schema_version": 1,
|
"schema_version": 1,
|
||||||
"started_at_monotonic_ms": now_ms(),
|
"started_at_monotonic_ms": started_at_ms,
|
||||||
"video": video_session_payload(video_present, video_error.as_deref(), None),
|
"video": video_session_payload(video_present, video_error.as_deref(), None),
|
||||||
"cursor": { "present": inner.cursor.is_some(), "sample_count": 0 }
|
"cursor": { "present": inner.cursor.is_some(), "sample_count": 0 }
|
||||||
});
|
});
|
||||||
@@ -350,7 +430,7 @@ impl RecordingSession {
|
|||||||
inner.enabled = true;
|
inner.enabled = true;
|
||||||
inner.output_dir = Some(dir);
|
inner.output_dir = Some(dir);
|
||||||
inner.next_turn = 1;
|
inner.next_turn = 1;
|
||||||
inner.session_start_ms = now_ms();
|
inner.session_start_ms = started_at_ms;
|
||||||
inner.session_monotonic_start = Some(monotonic_start);
|
inner.session_monotonic_start = Some(monotonic_start);
|
||||||
inner.last_error = video_error;
|
inner.last_error = video_error;
|
||||||
inner.last_video = None;
|
inner.last_video = None;
|
||||||
@@ -392,6 +472,7 @@ impl RecordingSession {
|
|||||||
}
|
}
|
||||||
inner.owner = None;
|
inner.owner = None;
|
||||||
let dir = inner.output_dir.clone();
|
let dir = inner.output_dir.clone();
|
||||||
|
let started_at_ms = inner.session_start_ms;
|
||||||
let (video_meta, stop_error) = match inner.video.take().map(|rec| rec.stop()) {
|
let (video_meta, stop_error) = match inner.video.take().map(|rec| rec.stop()) {
|
||||||
Some(Ok(meta)) => match validate_video_metadata(meta) {
|
Some(Ok(meta)) => match validate_video_metadata(meta) {
|
||||||
Ok(meta) => (Some(meta), None),
|
Ok(meta) => (Some(meta), None),
|
||||||
@@ -427,7 +508,7 @@ impl RecordingSession {
|
|||||||
};
|
};
|
||||||
let session_payload = serde_json::json!({
|
let session_payload = serde_json::json!({
|
||||||
"schema_version": 1,
|
"schema_version": 1,
|
||||||
"started_at_monotonic_ms": now_ms(),
|
"started_at_monotonic_ms": started_at_ms,
|
||||||
"video": video_block,
|
"video": video_block,
|
||||||
"cursor": { "present": cursor_samples > 0, "sample_count": cursor_samples }
|
"cursor": { "present": cursor_samples > 0, "sample_count": cursor_samples }
|
||||||
});
|
});
|
||||||
@@ -476,8 +557,14 @@ impl RecordingSession {
|
|||||||
|
|
||||||
/// Reserve a turn and capture its target immediately before tool dispatch.
|
/// Reserve a turn and capture its target immediately before tool dispatch.
|
||||||
/// No-op when recording is disabled.
|
/// No-op when recording is disabled.
|
||||||
pub fn begin_turn(&self, tool_name: &str, args: &Value, start_ms: u64) -> Option<PendingTurn> {
|
pub fn begin_turn(
|
||||||
self.begin_turn_with_capture(tool_name, args, start_ms, true)
|
&self,
|
||||||
|
tool_name: &str,
|
||||||
|
args: &Value,
|
||||||
|
start_ms: u64,
|
||||||
|
caller: RecordingCaller<'_>,
|
||||||
|
) -> Option<PendingTurn> {
|
||||||
|
self.begin_turn_with_capture(tool_name, args, start_ms, caller, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reserve a turn while deliberately suppressing visual and accessibility
|
/// Reserve a turn while deliberately suppressing visual and accessibility
|
||||||
@@ -489,8 +576,9 @@ impl RecordingSession {
|
|||||||
tool_name: &str,
|
tool_name: &str,
|
||||||
args: &Value,
|
args: &Value,
|
||||||
start_ms: u64,
|
start_ms: u64,
|
||||||
|
caller: RecordingCaller<'_>,
|
||||||
) -> Option<PendingTurn> {
|
) -> Option<PendingTurn> {
|
||||||
self.begin_turn_with_capture(tool_name, args, start_ms, false)
|
self.begin_turn_with_capture(tool_name, args, start_ms, caller, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn begin_turn_with_capture(
|
fn begin_turn_with_capture(
|
||||||
@@ -498,13 +586,19 @@ impl RecordingSession {
|
|||||||
tool_name: &str,
|
tool_name: &str,
|
||||||
args: &Value,
|
args: &Value,
|
||||||
start_ms: u64,
|
start_ms: u64,
|
||||||
|
caller: RecordingCaller<'_>,
|
||||||
capture_visual_state: bool,
|
capture_visual_state: bool,
|
||||||
) -> Option<PendingTurn> {
|
) -> Option<PendingTurn> {
|
||||||
let (turn_dir, session_start_ms, generation) = {
|
let (turn_dir, session_start_ms, generation, owner) = {
|
||||||
let mut inner = self.inner.lock().unwrap();
|
let mut inner = self.inner.lock().unwrap();
|
||||||
if !inner.enabled {
|
if !inner.enabled {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
if inner.owner.as_deref().is_some_and(|owner| {
|
||||||
|
caller.session != Some(owner) && caller.transport != Some(owner)
|
||||||
|
}) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
let out = inner.output_dir.clone()?;
|
let out = inner.output_dir.clone()?;
|
||||||
let idx = inner.next_turn;
|
let idx = inner.next_turn;
|
||||||
inner.next_turn += 1;
|
inner.next_turn += 1;
|
||||||
@@ -512,6 +606,7 @@ impl RecordingSession {
|
|||||||
out.join(format!("turn-{idx:05}")),
|
out.join(format!("turn-{idx:05}")),
|
||||||
inner.session_start_ms,
|
inner.session_start_ms,
|
||||||
inner.generation,
|
inner.generation,
|
||||||
|
inner.owner.clone(),
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -525,30 +620,47 @@ impl RecordingSession {
|
|||||||
let mut window_id = args.opt_u64("window_id");
|
let mut window_id = args.opt_u64("window_id");
|
||||||
let pid = args.opt_i64("pid");
|
let pid = args.opt_i64("pid");
|
||||||
let mut element_index = args.opt_u64("element_index");
|
let mut element_index = args.opt_u64("element_index");
|
||||||
if let (Some(pid), Some(token)) = (
|
let mut binding = None;
|
||||||
pid.and_then(|pid| i32::try_from(pid).ok()),
|
if let Some(pid) = pid.and_then(|pid| i32::try_from(pid).ok()) {
|
||||||
args.get("element_token").and_then(Value::as_str),
|
if let Ok(resolved) = crate::element_token::resolve_element_args(
|
||||||
) {
|
pid,
|
||||||
if let Ok((resolved_window, resolved_index)) =
|
element_index.and_then(|index| usize::try_from(index).ok()),
|
||||||
crate::element_token::global().resolve(pid, token)
|
args.get("element_token").and_then(Value::as_str),
|
||||||
{
|
args.get("snapshot_id").and_then(Value::as_str),
|
||||||
window_id = Some(u64::from(resolved_window));
|
window_id.and_then(|window| u32::try_from(window).ok()),
|
||||||
element_index = u64::try_from(resolved_index).ok();
|
tool_name,
|
||||||
|
) {
|
||||||
|
binding = resolved.binding();
|
||||||
|
if let crate::element_token::ResolvedElement::Element {
|
||||||
|
window_id: Some(window),
|
||||||
|
element_index: index,
|
||||||
|
..
|
||||||
|
} = resolved
|
||||||
|
{
|
||||||
|
window_id = Some(u64::from(window));
|
||||||
|
element_index = u64::try_from(index).ok();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let click_point = resolve_click_point(tool_name, &args, window_id, pid, element_index);
|
let click_point = resolve_click_point(
|
||||||
|
tool_name,
|
||||||
|
&args,
|
||||||
|
window_id,
|
||||||
|
pid,
|
||||||
|
element_index,
|
||||||
|
binding.as_ref(),
|
||||||
|
);
|
||||||
let before = if capture_visual_state {
|
let before = if capture_visual_state {
|
||||||
capture_turn(window_id, pid)
|
capture_turn(window_id, pid)
|
||||||
} else {
|
} else {
|
||||||
TurnCapture {
|
TurnCapture {
|
||||||
state: None,
|
|
||||||
screenshot: None,
|
|
||||||
screenshot_classification: Some("privacy_suppressed"),
|
screenshot_classification: Some("privacy_suppressed"),
|
||||||
|
..Default::default()
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut inner = self.inner.lock().unwrap();
|
let mut inner = self.inner.lock().unwrap();
|
||||||
if !inner.enabled || inner.generation != generation {
|
if !inner.enabled || inner.generation != generation || inner.owner != owner {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
if let Err(error) = write_phase_artifacts(&turn_dir, "before", &before) {
|
if let Err(error) = write_phase_artifacts(&turn_dir, "before", &before) {
|
||||||
@@ -558,6 +670,7 @@ impl RecordingSession {
|
|||||||
|
|
||||||
Some(PendingTurn {
|
Some(PendingTurn {
|
||||||
generation,
|
generation,
|
||||||
|
owner,
|
||||||
turn_dir,
|
turn_dir,
|
||||||
tool_name: tool_name.to_owned(),
|
tool_name: tool_name.to_owned(),
|
||||||
args,
|
args,
|
||||||
@@ -600,15 +713,27 @@ impl RecordingSession {
|
|||||||
action_record: Option<&crate::action_record::ActionExecutionRecord>,
|
action_record: Option<&crate::action_record::ActionExecutionRecord>,
|
||||||
result_is_error: bool,
|
result_is_error: bool,
|
||||||
) {
|
) {
|
||||||
let mut inner = self.inner.lock().unwrap();
|
let is_live = |inner: &RecordingInner| {
|
||||||
if !inner.enabled || inner.generation != pending.generation {
|
inner.enabled && inner.generation == pending.generation && inner.owner == pending.owner
|
||||||
tracing::warn!(
|
};
|
||||||
target: "recording",
|
if !is_live(&self.inner.lock().unwrap()) {
|
||||||
"discarding a turn from an inactive recording generation"
|
|
||||||
);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if let Err(error) = write_turn(pending, result_text, action_record, result_is_error) {
|
let after = if pending.capture_visual_state {
|
||||||
|
capture_turn(pending.window_id, pending.pid)
|
||||||
|
} else {
|
||||||
|
TurnCapture {
|
||||||
|
screenshot_classification: Some("privacy_suppressed"),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut inner = self.inner.lock().unwrap();
|
||||||
|
if !is_live(&inner) {
|
||||||
|
tracing::warn!(target: "recording", "discarding a turn from an inactive recording generation");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Err(error) = write_turn(pending, after, result_text, action_record, result_is_error)
|
||||||
|
{
|
||||||
inner.last_error = Some(error.to_string());
|
inner.last_error = Some(error.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -616,8 +741,15 @@ impl RecordingSession {
|
|||||||
/// Compatibility helper for callers that only report completed calls.
|
/// Compatibility helper for callers that only report completed calls.
|
||||||
/// New dispatch paths should use `begin_turn` and `finish_turn` so the
|
/// New dispatch paths should use `begin_turn` and `finish_turn` so the
|
||||||
/// before phase is captured before the action changes application state.
|
/// before phase is captured before the action changes application state.
|
||||||
pub fn record(&self, tool_name: &str, args: &Value, result_text: &str, start_ms: u64) {
|
pub fn record(
|
||||||
let Some(pending) = self.begin_turn(tool_name, args, start_ms) else {
|
&self,
|
||||||
|
tool_name: &str,
|
||||||
|
args: &Value,
|
||||||
|
result_text: &str,
|
||||||
|
start_ms: u64,
|
||||||
|
caller: RecordingCaller<'_>,
|
||||||
|
) {
|
||||||
|
let Some(pending) = self.begin_turn(tool_name, args, start_ms, caller) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
self.finish_turn(pending, result_text);
|
self.finish_turn(pending, result_text);
|
||||||
@@ -637,10 +769,10 @@ fn capture_turn(window_id: Option<u64>, pid: Option<i64>) -> TurnCapture {
|
|||||||
.get()
|
.get()
|
||||||
.map(|capture| capture(window_id, pid))
|
.map(|capture| capture(window_id, pid))
|
||||||
.unwrap_or_else(|| ScreenshotCapture::unavailable("capture_hook_unavailable"));
|
.unwrap_or_else(|| ScreenshotCapture::unavailable("capture_hook_unavailable"));
|
||||||
|
let (state, state_classification) = capture_ax_state(window_id, pid);
|
||||||
TurnCapture {
|
TurnCapture {
|
||||||
state: AX_SNAPSHOT_FN
|
state,
|
||||||
.get()
|
state_classification,
|
||||||
.and_then(|capture| capture(window_id, pid)),
|
|
||||||
screenshot: screenshot.png,
|
screenshot: screenshot.png,
|
||||||
screenshot_classification: screenshot.classification,
|
screenshot_classification: screenshot.classification,
|
||||||
}
|
}
|
||||||
@@ -652,6 +784,7 @@ fn resolve_click_point(
|
|||||||
window_id: Option<u64>,
|
window_id: Option<u64>,
|
||||||
pid: Option<i64>,
|
pid: Option<i64>,
|
||||||
element_index: Option<u64>,
|
element_index: Option<u64>,
|
||||||
|
binding: Option<&crate::element_token::SnapshotBinding>,
|
||||||
) -> Option<(f64, f64)> {
|
) -> Option<(f64, f64)> {
|
||||||
use crate::tool_args::ArgsExt;
|
use crate::tool_args::ArgsExt;
|
||||||
if !matches!(tool_name, "click" | "double_click" | "right_click") {
|
if !matches!(tool_name, "click" | "double_click" | "right_click") {
|
||||||
@@ -662,7 +795,7 @@ fn resolve_click_point(
|
|||||||
_ => match (window_id, pid, element_index, ELEMENT_BOUNDS_FN.get()) {
|
_ => match (window_id, pid, element_index, ELEMENT_BOUNDS_FN.get()) {
|
||||||
(Some(wid), Some(pid), Some(index), Some(resolve)) => u32::try_from(index)
|
(Some(wid), Some(pid), Some(index), Some(resolve)) => u32::try_from(index)
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|index| resolve(wid, pid, index)),
|
.and_then(|index| resolve(wid, pid, index, binding?)),
|
||||||
_ => None,
|
_ => None,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -711,7 +844,7 @@ fn write_evidence_manifest(
|
|||||||
let manifest = serde_json::json!({
|
let manifest = serde_json::json!({
|
||||||
"schema": "cua-turn-evidence/v1",
|
"schema": "cua-turn-evidence/v1",
|
||||||
"before": {
|
"before": {
|
||||||
"state": capture_status(before.state.is_some(), state_expected, None),
|
"state": capture_status(before.state.is_some(), state_expected, before.state_classification),
|
||||||
"screenshot": capture_status(
|
"screenshot": capture_status(
|
||||||
before.screenshot.is_some(),
|
before.screenshot.is_some(),
|
||||||
true,
|
true,
|
||||||
@@ -719,7 +852,7 @@ fn write_evidence_manifest(
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
"after": {
|
"after": {
|
||||||
"state": capture_status(after.state.is_some(), state_expected, None),
|
"state": capture_status(after.state.is_some(), state_expected, after.state_classification),
|
||||||
"screenshot": capture_status(
|
"screenshot": capture_status(
|
||||||
after.screenshot.is_some(),
|
after.screenshot.is_some(),
|
||||||
true,
|
true,
|
||||||
@@ -758,18 +891,20 @@ fn strip_internal_keys(args: &Value) -> std::borrow::Cow<'_, Value> {
|
|||||||
|
|
||||||
fn write_turn(
|
fn write_turn(
|
||||||
pending: PendingTurn,
|
pending: PendingTurn,
|
||||||
|
after: TurnCapture,
|
||||||
result_text: &str,
|
result_text: &str,
|
||||||
action_record: Option<&crate::action_record::ActionExecutionRecord>,
|
action_record: Option<&crate::action_record::ActionExecutionRecord>,
|
||||||
result_is_error: bool,
|
result_is_error: bool,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let PendingTurn {
|
let PendingTurn {
|
||||||
generation: _,
|
generation: _,
|
||||||
|
owner: _,
|
||||||
turn_dir,
|
turn_dir,
|
||||||
tool_name,
|
tool_name,
|
||||||
args,
|
args,
|
||||||
start_ms,
|
start_ms,
|
||||||
session_start_ms,
|
session_start_ms,
|
||||||
window_id,
|
window_id: _,
|
||||||
pid,
|
pid,
|
||||||
click_point,
|
click_point,
|
||||||
capture_visual_state,
|
capture_visual_state,
|
||||||
@@ -777,15 +912,6 @@ fn write_turn(
|
|||||||
} = pending;
|
} = pending;
|
||||||
std::fs::create_dir_all(&turn_dir)?;
|
std::fs::create_dir_all(&turn_dir)?;
|
||||||
let now = now_ms();
|
let now = now_ms();
|
||||||
let after = if capture_visual_state {
|
|
||||||
capture_turn(window_id, pid)
|
|
||||||
} else {
|
|
||||||
TurnCapture {
|
|
||||||
state: None,
|
|
||||||
screenshot: None,
|
|
||||||
screenshot_classification: Some("privacy_suppressed"),
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let click_family = matches!(tool_name.as_str(), "click" | "double_click" | "right_click");
|
let click_family = matches!(tool_name.as_str(), "click" | "double_click" | "right_click");
|
||||||
let action_refused = action_record
|
let action_refused = action_record
|
||||||
.is_some_and(|record| record.effect == crate::action_record::ActionEffect::Refused);
|
.is_some_and(|record| record.effect == crate::action_record::ActionEffect::Refused);
|
||||||
@@ -803,7 +929,7 @@ fn write_turn(
|
|||||||
"arguments": args,
|
"arguments": args,
|
||||||
"result_summary": result_text,
|
"result_summary": result_text,
|
||||||
"result_error": result_is_error,
|
"result_error": result_is_error,
|
||||||
"timestamp": iso_now(),
|
"timestamp": iso8601_utc_from_unix_ms(now),
|
||||||
"t_ms_from_session_start": now.saturating_sub(session_start_ms),
|
"t_ms_from_session_start": now.saturating_sub(session_start_ms),
|
||||||
"t_start_ms_from_session_start": start_ms.saturating_sub(session_start_ms),
|
"t_start_ms_from_session_start": start_ms.saturating_sub(session_start_ms),
|
||||||
});
|
});
|
||||||
@@ -874,12 +1000,19 @@ pub fn now_ms() -> u64 {
|
|||||||
.as_millis() as u64
|
.as_millis() as u64
|
||||||
}
|
}
|
||||||
|
|
||||||
fn iso_now() -> String {
|
fn iso8601_utc_from_unix_ms(ms: u64) -> String {
|
||||||
let d = SystemTime::now()
|
let t = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(ms) * 1_000_000)
|
||||||
.duration_since(UNIX_EPOCH)
|
.unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
|
||||||
.unwrap_or_default();
|
format!(
|
||||||
// Format as fractional Unix seconds (simple, unambiguous, machine-readable).
|
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z",
|
||||||
format!("{:.3}", d.as_secs_f64())
|
t.year(),
|
||||||
|
u8::from(t.month()),
|
||||||
|
t.day(),
|
||||||
|
t.hour(),
|
||||||
|
t.minute(),
|
||||||
|
t.second(),
|
||||||
|
t.millisecond()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the `session.json` `video` field. Three shapes:
|
/// Build the `session.json` `video` field. Three shapes:
|
||||||
@@ -940,6 +1073,93 @@ fn validate_video_metadata(meta: VideoMetadata) -> anyhow::Result<VideoMetadata>
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
#[test]
|
||||||
|
fn recording_turns_require_the_owner_session_or_connection() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let session = RecordingSession::new();
|
||||||
|
{
|
||||||
|
let mut inner = session.inner.lock().unwrap();
|
||||||
|
inner.enabled = true;
|
||||||
|
inner.output_dir = Some(root.path().to_owned());
|
||||||
|
inner.owner = Some("owner".to_owned());
|
||||||
|
}
|
||||||
|
for caller in [
|
||||||
|
RecordingCaller {
|
||||||
|
session: Some("foreign"),
|
||||||
|
transport: Some("other"),
|
||||||
|
},
|
||||||
|
RecordingCaller::default(),
|
||||||
|
] {
|
||||||
|
assert!(session
|
||||||
|
.begin_private_turn("type_text", &serde_json::json!({}), 0, caller)
|
||||||
|
.is_none());
|
||||||
|
assert_eq!(session.current_state().next_turn, 1);
|
||||||
|
}
|
||||||
|
for caller in [
|
||||||
|
RecordingCaller {
|
||||||
|
session: Some("owner"),
|
||||||
|
transport: Some("connection"),
|
||||||
|
},
|
||||||
|
RecordingCaller {
|
||||||
|
session: Some("named-on-owner-connection"),
|
||||||
|
transport: Some("owner"),
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
assert!(session
|
||||||
|
.begin_private_turn("type_text", &serde_json::json!({}), 0, caller)
|
||||||
|
.is_some());
|
||||||
|
}
|
||||||
|
assert_eq!(session.current_state().next_turn, 3);
|
||||||
|
session.inner.lock().unwrap().owner = None;
|
||||||
|
assert!(session
|
||||||
|
.begin_private_turn(
|
||||||
|
"type_text",
|
||||||
|
&serde_json::json!({}),
|
||||||
|
0,
|
||||||
|
RecordingCaller::default()
|
||||||
|
)
|
||||||
|
.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recording_timestamps_use_iso8601_and_keep_the_start_baseline() {
|
||||||
|
for (ms, expected) in [
|
||||||
|
(0, "1970-01-01T00:00:00.000Z"),
|
||||||
|
(951_782_400_123, "2000-02-29T00:00:00.123Z"),
|
||||||
|
(1_767_225_600_999, "2026-01-01T00:00:00.999Z"),
|
||||||
|
] {
|
||||||
|
assert_eq!(iso8601_utc_from_unix_ms(ms), expected);
|
||||||
|
}
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let session = RecordingSession::new();
|
||||||
|
session
|
||||||
|
.start(root.path().to_str().unwrap(), false, None)
|
||||||
|
.unwrap();
|
||||||
|
let manifest = || -> Value {
|
||||||
|
serde_json::from_slice(&std::fs::read(root.path().join("session.json")).unwrap())
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let started = manifest()["started_at_monotonic_ms"].as_u64().unwrap();
|
||||||
|
let pending = session
|
||||||
|
.begin_private_turn(
|
||||||
|
"type_text",
|
||||||
|
&serde_json::json!({}),
|
||||||
|
now_ms(),
|
||||||
|
RecordingCaller::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
session.finish_turn(pending, "fixture");
|
||||||
|
let action: Value = serde_json::from_slice(
|
||||||
|
&std::fs::read(root.path().join("turn-00001/action.json")).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let finished = started + action["t_ms_from_session_start"].as_u64().unwrap();
|
||||||
|
assert_eq!(action["timestamp"], iso8601_utc_from_unix_ms(finished));
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||||
|
session.stop_owner(None).unwrap();
|
||||||
|
assert_eq!(manifest()["started_at_monotonic_ms"], started);
|
||||||
|
}
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
@@ -974,7 +1194,7 @@ mod tests {
|
|||||||
Some(format!(r#"{{"phase":{phase}}}"#).into_bytes())
|
Some(format!(r#"{{"phase":{phase}}}"#).into_bytes())
|
||||||
});
|
});
|
||||||
set_click_marker_fn(|_, _, _| Some(b"click".to_vec()));
|
set_click_marker_fn(|_, _, _| Some(b"click".to_vec()));
|
||||||
set_element_bounds_fn(|window_id, pid, element_index| {
|
set_element_bounds_fn(|window_id, pid, element_index, _binding| {
|
||||||
Some((window_id as f64 + element_index as f64, pid as f64))
|
Some((window_id as f64 + element_index as f64, pid as f64))
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -995,6 +1215,7 @@ mod tests {
|
|||||||
"click",
|
"click",
|
||||||
&serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}),
|
&serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}),
|
||||||
now_ms(),
|
now_ms(),
|
||||||
|
RecordingCaller::default(),
|
||||||
)
|
)
|
||||||
.expect("recording should reserve a turn");
|
.expect("recording should reserve a turn");
|
||||||
let turn = output_dir.join("turn-00001");
|
let turn = output_dir.join("turn-00001");
|
||||||
@@ -1028,13 +1249,19 @@ mod tests {
|
|||||||
assert_eq!(action["action_truth"]["route"], "synthetic_events");
|
assert_eq!(action["action_truth"]["route"], "synthetic_events");
|
||||||
assert_eq!(action["action_truth"]["requested_delivery"], "background");
|
assert_eq!(action["action_truth"]["requested_delivery"], "background");
|
||||||
|
|
||||||
let snapshot_id = crate::element_token::global().register_snapshot(1, 77, 1);
|
let snapshot_id = crate::element_token::global().register_snapshot_payload(
|
||||||
|
1,
|
||||||
|
77,
|
||||||
|
[0],
|
||||||
|
std::sync::Arc::new(()),
|
||||||
|
);
|
||||||
let token = crate::element_token::token_for(snapshot_id, 0);
|
let token = crate::element_token::token_for(snapshot_id, 0);
|
||||||
let pending = session
|
let pending = session
|
||||||
.begin_turn(
|
.begin_turn(
|
||||||
"click",
|
"click",
|
||||||
&serde_json::json!({"pid": 1, "element_token": token}),
|
&serde_json::json!({"pid": 1, "element_token": token}),
|
||||||
now_ms(),
|
now_ms(),
|
||||||
|
RecordingCaller::default(),
|
||||||
)
|
)
|
||||||
.expect("token-only click should reserve a targeted turn");
|
.expect("token-only click should reserve a targeted turn");
|
||||||
session.finish_turn(pending, "token click");
|
session.finish_turn(pending, "token click");
|
||||||
@@ -1055,6 +1282,7 @@ mod tests {
|
|||||||
"click",
|
"click",
|
||||||
&serde_json::json!({"pid": 1, "element_token": stale_token}),
|
&serde_json::json!({"pid": 1, "element_token": stale_token}),
|
||||||
now_ms(),
|
now_ms(),
|
||||||
|
RecordingCaller::default(),
|
||||||
)
|
)
|
||||||
.expect("stale-token refusal should reserve an evidence turn");
|
.expect("stale-token refusal should reserve an evidence turn");
|
||||||
session.finish_turn_with_outcome(pending, "stale token", None, true);
|
session.finish_turn_with_outcome(pending, "stale token", None, true);
|
||||||
@@ -1081,6 +1309,7 @@ mod tests {
|
|||||||
"click",
|
"click",
|
||||||
&serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}),
|
&serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}),
|
||||||
now_ms(),
|
now_ms(),
|
||||||
|
RecordingCaller::default(),
|
||||||
)
|
)
|
||||||
.expect("resolved refusal should reserve an evidence turn");
|
.expect("resolved refusal should reserve an evidence turn");
|
||||||
let refusal_record = crate::action_record::ActionExecutionRecord::builder(
|
let refusal_record = crate::action_record::ActionExecutionRecord::builder(
|
||||||
@@ -1162,7 +1391,12 @@ mod tests {
|
|||||||
inner.session_start_ms = now_ms();
|
inner.session_start_ms = now_ms();
|
||||||
}
|
}
|
||||||
let pending = session
|
let pending = session
|
||||||
.begin_turn("click", &serde_json::json!({"x": 1, "y": 2}), now_ms())
|
.begin_turn(
|
||||||
|
"click",
|
||||||
|
&serde_json::json!({"x": 1, "y": 2}),
|
||||||
|
now_ms(),
|
||||||
|
RecordingCaller::default(),
|
||||||
|
)
|
||||||
.expect("reserve first generation turn");
|
.expect("reserve first generation turn");
|
||||||
session.inner.lock().unwrap().generation = 2;
|
session.inner.lock().unwrap().generation = 2;
|
||||||
session.finish_turn(pending, "must be discarded");
|
session.finish_turn(pending, "must be discarded");
|
||||||
@@ -1195,6 +1429,7 @@ mod tests {
|
|||||||
"browser_prepare",
|
"browser_prepare",
|
||||||
&serde_json::json!({"pid": 1, "window_id": 2}),
|
&serde_json::json!({"pid": 1, "window_id": 2}),
|
||||||
now_ms(),
|
now_ms(),
|
||||||
|
RecordingCaller::default(),
|
||||||
)
|
)
|
||||||
.expect("reserve private consent turn");
|
.expect("reserve private consent turn");
|
||||||
session.finish_turn(pending, "attached");
|
session.finish_turn(pending, "attached");
|
||||||
|
|||||||
@@ -45,9 +45,13 @@ impl Tool for StartRecordingTool {
|
|||||||
fn def(&self) -> &ToolDef {
|
fn def(&self) -> &ToolDef {
|
||||||
START_REC_DEF.get_or_init(|| ToolDef {
|
START_REC_DEF.get_or_init(|| ToolDef {
|
||||||
name: "start_recording".into(),
|
name: "start_recording".into(),
|
||||||
description: "Start trajectory recording. Every subsequent action-tool \
|
description: "Start trajectory recording for the calling session. Each action-tool \
|
||||||
invocation (click, right_click, scroll, type_text, press_key, hotkey, \
|
invocation (click, right_click, scroll, type_text, press_key, hotkey, \
|
||||||
set_value) writes a turn folder under `output_dir`:\n\n\
|
set_value) from that session writes a turn folder under `output_dir`. \
|
||||||
|
Without `session`, calls on the same connection are recorded, including \
|
||||||
|
named-session calls. Other connections and session lifecycle calls \
|
||||||
|
(`start_session` / `end_session`) are not recorded. CLI recordings started \
|
||||||
|
with `cua-driver recording start` are daemon-wide.\n\n\
|
||||||
- `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI \
|
- `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI \
|
||||||
state immediately before and after the action.\n\
|
state immediately before and after the action.\n\
|
||||||
- `before.png` / `after.png` — target-window screenshots immediately \
|
- `before.png` / `after.png` — target-window screenshots immediately \
|
||||||
@@ -91,6 +95,10 @@ impl Tool for StartRecordingTool {
|
|||||||
"description": "Absolute or ~-rooted directory where turn folders \
|
"description": "Absolute or ~-rooted directory where turn folders \
|
||||||
and (when enabled) the video file are written."
|
and (when enabled) the video file are written."
|
||||||
},
|
},
|
||||||
|
"session": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Public session label to record. Omit to record calls on the authenticated connection."
|
||||||
|
},
|
||||||
"record_video": {
|
"record_video": {
|
||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
"description": "Capture the main display to <output_dir>/recording.mp4. \
|
"description": "Capture the main display to <output_dir>/recording.mp4. \
|
||||||
@@ -117,10 +125,11 @@ impl Tool for StartRecordingTool {
|
|||||||
return ToolResult::error("`output_dir` is required.");
|
return ToolResult::error("`output_dir` is required.");
|
||||||
}
|
}
|
||||||
let record_video = args.bool_or("record_video", false);
|
let record_video = args.bool_or("record_video", false);
|
||||||
// Daemon-injected ownership key (absent for one-shot CLI / anonymous
|
// Use the trusted connection when no named lifecycle was selected.
|
||||||
// sessions). Stamps the recording so a session-scoped teardown
|
// CLI recording commands without either key remain daemon-wide.
|
||||||
// (session_end) only stops the recording its own session started.
|
let owner = args
|
||||||
let owner = args.opt_str("_session_id");
|
.opt_str("_session_id")
|
||||||
|
.or_else(|| args.opt_str("_transport_session_id"));
|
||||||
|
|
||||||
match self.session.start(
|
match self.session.start(
|
||||||
output_dir.as_deref().unwrap(),
|
output_dir.as_deref().unwrap(),
|
||||||
|
|||||||
@@ -457,15 +457,28 @@ fn begin_session_dispatch_inner(
|
|||||||
return Err("session has ended");
|
return Err("session has ended");
|
||||||
}
|
}
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
{
|
let recreated = loop {
|
||||||
// Keep tombstone admission and live-record insertion in one critical
|
// Match termination's lock order and recheck after cleanup: a transport
|
||||||
// section. Otherwise an end could land between the old pre-check and
|
// close or explicit end racing that work must stay terminal.
|
||||||
// insertion, leaving a tombstoned record that was silently recreated
|
let mut ended = ended_sessions().lock().unwrap();
|
||||||
// by a racing first action.
|
let recreated = if let Some(episode) = ended.get(session_id) {
|
||||||
let ended = ended_sessions().lock().unwrap();
|
if !implicit
|
||||||
if ended.contains_key(session_id) {
|
|| public_label.is_some()
|
||||||
return Err("session has ended");
|
|| !episode.recreates_for(session_id, owner_transport)
|
||||||
}
|
{
|
||||||
|
return Err("session has ended");
|
||||||
|
}
|
||||||
|
if cleanup_progress().lock().unwrap().contains_key(session_id) {
|
||||||
|
drop(ended);
|
||||||
|
if !retry_session_cleanup(session_id).complete {
|
||||||
|
return Err("session cleanup is incomplete; retry the call");
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
let mut records = lifecycle_records().lock().unwrap();
|
let mut records = lifecycle_records().lock().unwrap();
|
||||||
let record = records
|
let record = records
|
||||||
.entry(session_id.to_owned())
|
.entry(session_id.to_owned())
|
||||||
@@ -490,11 +503,18 @@ fn begin_session_dispatch_inner(
|
|||||||
record.idle_ttl = idle_ttl;
|
record.idle_ttl = idle_ttl;
|
||||||
}
|
}
|
||||||
record.in_flight += 1;
|
record.in_flight += 1;
|
||||||
}
|
if recreated {
|
||||||
|
ended.remove(session_id);
|
||||||
|
}
|
||||||
|
break recreated;
|
||||||
|
};
|
||||||
activity()
|
activity()
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.insert(session_id.to_owned(), now);
|
.insert(session_id.to_owned(), now);
|
||||||
|
if recreated {
|
||||||
|
fire_session_revive_for_owner(session_id, owner_transport);
|
||||||
|
}
|
||||||
Ok(SessionDispatchGuard {
|
Ok(SessionDispatchGuard {
|
||||||
session_id: session_id.to_owned(),
|
session_id: session_id.to_owned(),
|
||||||
})
|
})
|
||||||
@@ -612,7 +632,7 @@ pub fn activate_or_revive_session_for_owner(
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
loop {
|
loop {
|
||||||
let mut ended = ended_sessions().lock().unwrap();
|
let mut ended = ended_sessions().lock().unwrap();
|
||||||
let revived = match ended.get(session_id) {
|
let revived = match ended.get(session_id).map(|ended| ended.owner.as_deref()) {
|
||||||
Some(Some(owner)) if owner != owner_transport => {
|
Some(Some(owner)) if owner != owner_transport => {
|
||||||
return Err("session is not available to this transport");
|
return Err("session is not available to this transport");
|
||||||
}
|
}
|
||||||
@@ -707,7 +727,7 @@ fn session_owner_matches(session_id: &str, owner_transport: &str) -> bool {
|
|||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.get(session_id)
|
.get(session_id)
|
||||||
.is_some_and(|owner| owner.as_deref() == Some(owner_transport))
|
.is_some_and(|ended| ended.owner.as_deref() == Some(owner_transport))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// End a lifecycle episode only when it belongs to the authenticated
|
/// End a lifecycle episode only when it belongs to the authenticated
|
||||||
@@ -773,13 +793,25 @@ pub fn list_session_snapshots_with_prefix(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn end_sessions_for_owner(owner_transport: &str, reason: SessionEndReason) -> usize {
|
pub fn end_sessions_for_owner(owner_transport: &str, reason: SessionEndReason) -> usize {
|
||||||
let ids = lifecycle_records()
|
let mut ids = lifecycle_records()
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(_, record)| record.owner_transport == owner_transport)
|
.filter(|(_, record)| record.owner_transport == owner_transport)
|
||||||
.map(|(id, _)| id.clone())
|
.map(|(id, _)| id.clone())
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
ids.extend(
|
||||||
|
ended_sessions()
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, ended)| {
|
||||||
|
ended.recreate_after_idle && ended.owner.as_deref() == Some(owner_transport)
|
||||||
|
})
|
||||||
|
.map(|(id, _)| id.clone()),
|
||||||
|
);
|
||||||
|
ids.sort();
|
||||||
|
ids.dedup();
|
||||||
for id in &ids {
|
for id in &ids {
|
||||||
end_session_with_reason(id, reason);
|
end_session_with_reason(id, reason);
|
||||||
}
|
}
|
||||||
@@ -928,7 +960,20 @@ fn capture_modality_for(tool_name: &str, args: &serde_json::Value) -> Option<Cap
|
|||||||
/// be idempotent because the overlay Remove + recording stop must run exactly
|
/// be idempotent because the overlay Remove + recording stop must run exactly
|
||||||
/// once. Growth is bounded (one short string per ended session over the
|
/// once. Growth is bounded (one short string per ended session over the
|
||||||
/// daemon's lifetime); eviction is a deliberate non-blocking follow-up.
|
/// daemon's lifetime); eviction is a deliberate non-blocking follow-up.
|
||||||
static ENDED_SESSIONS: OnceLock<Mutex<HashMap<String, Option<String>>>> = OnceLock::new();
|
struct EndedSession {
|
||||||
|
owner: Option<String>,
|
||||||
|
recreate_after_idle: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EndedSession {
|
||||||
|
fn recreates_for(&self, session_id: &str, owner_transport: &str) -> bool {
|
||||||
|
self.recreate_after_idle
|
||||||
|
&& session_id == owner_transport
|
||||||
|
&& self.owner.as_deref() == Some(owner_transport)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static ENDED_SESSIONS: OnceLock<Mutex<HashMap<String, EndedSession>>> = OnceLock::new();
|
||||||
/// Runtime generations that have received terminal revoke-all.
|
/// Runtime generations that have received terminal revoke-all.
|
||||||
///
|
///
|
||||||
/// This latch is intentionally independent of grants and public session
|
/// This latch is intentionally independent of grants and public session
|
||||||
@@ -948,7 +993,7 @@ fn revive_hooks() -> &'static Mutex<HashMap<u64, SessionReviveHook>> {
|
|||||||
SESSION_REVIVE_HOOKS.get_or_init(|| Mutex::new(HashMap::new()))
|
SESSION_REVIVE_HOOKS.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ended_sessions() -> &'static Mutex<HashMap<String, Option<String>>> {
|
fn ended_sessions() -> &'static Mutex<HashMap<String, EndedSession>> {
|
||||||
ENDED_SESSIONS.get_or_init(|| Mutex::new(HashMap::new()))
|
ENDED_SESSIONS.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1127,7 +1172,7 @@ pub fn fire_session_end(session_id: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn fire_session_end_for_owner(session_id: &str, owner_transport: Option<&str>) -> bool {
|
fn fire_session_end_for_owner(session_id: &str, owner_transport: Option<&str>) -> bool {
|
||||||
let first_fire = mark_session_ended(session_id, owner_transport);
|
let first_fire = mark_session_ended(session_id, owner_transport, SessionEndReason::Unknown);
|
||||||
let _ = retry_session_cleanup(session_id);
|
let _ = retry_session_cleanup(session_id);
|
||||||
first_fire
|
first_fire
|
||||||
}
|
}
|
||||||
@@ -1137,15 +1182,22 @@ fn fire_session_end_for_owner(session_id: &str, owner_transport: Option<&str>) -
|
|||||||
/// Hooks run after this short critical section. Keeping this transition under
|
/// Hooks run after this short critical section. Keeping this transition under
|
||||||
/// the same lock order as dispatch admission prevents a racing first action
|
/// the same lock order as dispatch admission prevents a racing first action
|
||||||
/// from recreating a record immediately before or after termination.
|
/// from recreating a record immediately before or after termination.
|
||||||
fn mark_session_ended(session_id: &str, owner_transport: Option<&str>) -> bool {
|
fn mark_session_ended(
|
||||||
|
session_id: &str,
|
||||||
|
owner_transport: Option<&str>,
|
||||||
|
reason: SessionEndReason,
|
||||||
|
) -> bool {
|
||||||
activity().lock().unwrap().remove(session_id);
|
activity().lock().unwrap().remove(session_id);
|
||||||
let mut ended = ended_sessions().lock().unwrap();
|
let mut ended = ended_sessions().lock().unwrap();
|
||||||
let record_owner = lifecycle_records()
|
let record = lifecycle_records().lock().unwrap().remove(session_id);
|
||||||
.lock()
|
let recreate_after_idle = reason == SessionEndReason::IdleTimeout
|
||||||
.unwrap()
|
&& record
|
||||||
.remove(session_id)
|
.as_ref()
|
||||||
.map(|record| record.owner_transport);
|
.is_some_and(|record| record.implicit && record.owner_transport == session_id);
|
||||||
if ended.contains_key(session_id) {
|
if let Some(ended) = ended.get_mut(session_id) {
|
||||||
|
if reason != SessionEndReason::IdleTimeout {
|
||||||
|
ended.recreate_after_idle = false;
|
||||||
|
}
|
||||||
false
|
false
|
||||||
} else {
|
} else {
|
||||||
crate::capture_scope::clear_session(session_id);
|
crate::capture_scope::clear_session(session_id);
|
||||||
@@ -1167,7 +1219,12 @@ fn mark_session_ended(session_id: &str, owner_transport: Option<&str>) -> bool {
|
|||||||
);
|
);
|
||||||
ended.insert(
|
ended.insert(
|
||||||
session_id.to_owned(),
|
session_id.to_owned(),
|
||||||
owner_transport.map(str::to_owned).or(record_owner),
|
EndedSession {
|
||||||
|
owner: owner_transport
|
||||||
|
.map(str::to_owned)
|
||||||
|
.or(record.map(|record| record.owner_transport)),
|
||||||
|
recreate_after_idle,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
@@ -1268,7 +1325,17 @@ pub fn session_cleanup_status(session_id: &str) -> SessionCleanupReport {
|
|||||||
/// a local operator control path without requiring an authorization grant:
|
/// a local operator control path without requiring an authorization grant:
|
||||||
/// revocation can only remove authority, never create it.
|
/// revocation can only remove authority, never create it.
|
||||||
pub fn revoke_all_sessions() -> usize {
|
pub fn revoke_all_sessions() -> usize {
|
||||||
let sessions: Vec<String> = activity().lock().unwrap().keys().cloned().collect();
|
let mut sessions: Vec<String> = activity().lock().unwrap().keys().cloned().collect();
|
||||||
|
sessions.extend(
|
||||||
|
ended_sessions()
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, ended)| ended.recreate_after_idle)
|
||||||
|
.map(|(id, _)| id.clone()),
|
||||||
|
);
|
||||||
|
sessions.sort();
|
||||||
|
sessions.dedup();
|
||||||
for session in &sessions {
|
for session in &sessions {
|
||||||
end_session(session);
|
end_session(session);
|
||||||
}
|
}
|
||||||
@@ -1277,13 +1344,23 @@ pub fn revoke_all_sessions() -> usize {
|
|||||||
|
|
||||||
/// Revoke only sessions owned by one runtime-private namespace.
|
/// Revoke only sessions owned by one runtime-private namespace.
|
||||||
pub fn revoke_sessions_with_prefix(prefix: &str) -> usize {
|
pub fn revoke_sessions_with_prefix(prefix: &str) -> usize {
|
||||||
let sessions: Vec<String> = activity()
|
let mut sessions: Vec<String> = activity()
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.keys()
|
.keys()
|
||||||
.filter(|session| session.starts_with(prefix))
|
.filter(|session| session.starts_with(prefix))
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect();
|
.collect();
|
||||||
|
sessions.extend(
|
||||||
|
ended_sessions()
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.filter(|(id, ended)| ended.recreate_after_idle && id.starts_with(prefix))
|
||||||
|
.map(|(id, _)| id.clone()),
|
||||||
|
);
|
||||||
|
sessions.sort();
|
||||||
|
sessions.dedup();
|
||||||
for session in &sessions {
|
for session in &sessions {
|
||||||
end_session(session);
|
end_session(session);
|
||||||
}
|
}
|
||||||
@@ -1315,6 +1392,16 @@ pub fn is_session_ended(session_id: &str) -> bool {
|
|||||||
ended_sessions().lock().unwrap().contains_key(session_id)
|
ended_sessions().lock().unwrap().contains_key(session_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Only an idle-reclaimed implicit lifecycle may restart on its owner's call.
|
||||||
|
/// Admission rechecks this state and requires completed cleanup before revival.
|
||||||
|
pub fn recreates_on_next_call(session_id: &str, owner_transport: &str) -> bool {
|
||||||
|
ended_sessions()
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(session_id)
|
||||||
|
.is_some_and(|ended| ended.recreates_for(session_id, owner_transport))
|
||||||
|
}
|
||||||
|
|
||||||
/// Revive a previously-ended session id by clearing its tombstone, so a fresh
|
/// Revive a previously-ended session id by clearing its tombstone, so a fresh
|
||||||
/// `start_session` with a recycled id works as a caller would expect: the id
|
/// `start_session` with a recycled id works as a caller would expect: the id
|
||||||
/// becomes live again and its actions stop being rejected by the resurrection
|
/// becomes live again and its actions stop being rejected by the resurrection
|
||||||
@@ -1362,7 +1449,7 @@ pub fn revive_session_for_owner(
|
|||||||
let Some(ended_owner) = ended.get(session_id) else {
|
let Some(ended_owner) = ended.get(session_id) else {
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
};
|
};
|
||||||
match ended_owner.as_deref() {
|
match ended_owner.owner.as_deref() {
|
||||||
Some(owner) if owner != owner_transport => {
|
Some(owner) if owner != owner_transport => {
|
||||||
return Err("session is not available to this transport");
|
return Err("session is not available to this transport");
|
||||||
}
|
}
|
||||||
@@ -1445,7 +1532,7 @@ fn end_session_with_reason(session_id: &str, reason: SessionEndReason) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn finish_session_end(session_id: &str, reason: SessionEndReason) {
|
fn finish_session_end(session_id: &str, reason: SessionEndReason) {
|
||||||
let first_fire = mark_session_ended(session_id, None);
|
let first_fire = mark_session_ended(session_id, None, reason);
|
||||||
let mut cursor_readers = CURSOR_OUTCOME_READERS
|
let mut cursor_readers = CURSOR_OUTCOME_READERS
|
||||||
.get()
|
.get()
|
||||||
.map(|readers| {
|
.map(|readers| {
|
||||||
@@ -1546,6 +1633,164 @@ mod tests {
|
|||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, Barrier};
|
use std::sync::{Arc, Barrier};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn idle_recreation_requires_owner_implicit_episode_and_complete_cleanup() {
|
||||||
|
let base = format!("g08-owner-cleanup-{}", std::process::id());
|
||||||
|
let named = format!("{base}-named");
|
||||||
|
let begin = |id: &str, label: Option<&str>, owner: &str| {
|
||||||
|
begin_session_dispatch(
|
||||||
|
id,
|
||||||
|
label,
|
||||||
|
owner,
|
||||||
|
label.is_none(),
|
||||||
|
SessionTransport::McpStdio,
|
||||||
|
SessionClientKind::Mcp,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let failures = Arc::new(AtomicUsize::new(2));
|
||||||
|
let remaining = failures.clone();
|
||||||
|
let observed = base.clone();
|
||||||
|
let _hook = register_scoped_fallible_session_end_hook("g08-cleanup", move |id| {
|
||||||
|
if id == observed
|
||||||
|
&& remaining
|
||||||
|
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
return Err("cleanup fixture incomplete".into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
drop(begin(&base, None, &base).unwrap());
|
||||||
|
drop(begin(&named, Some("named"), &base).unwrap());
|
||||||
|
assert_eq!(evict_idle_with_prefix(Duration::ZERO, &base).len(), 2);
|
||||||
|
assert!(!recreates_on_next_call(&base, "foreign-owner"));
|
||||||
|
assert_eq!(
|
||||||
|
begin(&base, None, "foreign-owner").err(),
|
||||||
|
Some("session has ended")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
begin(&base, Some("explicit-label"), &base).err(),
|
||||||
|
Some("session has ended")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
begin(&named, Some("named"), &base).err(),
|
||||||
|
Some("session has ended")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
begin(&base, None, &base).err(),
|
||||||
|
Some("session cleanup is incomplete; retry the call")
|
||||||
|
);
|
||||||
|
assert!(is_session_ended(&base));
|
||||||
|
let guard = begin(&base, None, &base).unwrap();
|
||||||
|
assert!(!is_session_ended(&base));
|
||||||
|
assert!(evict_idle_with_prefix(Duration::ZERO, &base).is_empty());
|
||||||
|
drop(guard);
|
||||||
|
assert!(
|
||||||
|
session_snapshot(&base, &base, DEFAULT_SESSION_IDLE_TTL)
|
||||||
|
.unwrap()
|
||||||
|
.implicit
|
||||||
|
);
|
||||||
|
end_session(&base);
|
||||||
|
assert!(!recreates_on_next_call(&base, &base));
|
||||||
|
assert!(begin(&base, None, &base).is_err());
|
||||||
|
assert!(revive_session(&base));
|
||||||
|
assert!(revive_session(&named));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn transport_close_and_revoke_make_idle_tombstones_terminal() {
|
||||||
|
for (index, revoke) in [false, true].into_iter().enumerate() {
|
||||||
|
let id = format!("g08-terminal-{}-{index}", std::process::id());
|
||||||
|
drop(
|
||||||
|
begin_session_dispatch(
|
||||||
|
&id,
|
||||||
|
None,
|
||||||
|
&id,
|
||||||
|
true,
|
||||||
|
SessionTransport::McpStdio,
|
||||||
|
SessionClientKind::Mcp,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
assert_eq!(evict_idle_with_prefix(Duration::ZERO, &id), [id.clone()]);
|
||||||
|
assert!(recreates_on_next_call(&id, &id));
|
||||||
|
if revoke {
|
||||||
|
assert_eq!(revoke_sessions_with_prefix(&id), 1);
|
||||||
|
} else {
|
||||||
|
assert_eq!(
|
||||||
|
end_sessions_for_owner(&id, SessionEndReason::ProcessExit),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!recreates_on_next_call(&id, &id));
|
||||||
|
assert!(begin_session_dispatch(
|
||||||
|
&id,
|
||||||
|
None,
|
||||||
|
&id,
|
||||||
|
true,
|
||||||
|
SessionTransport::McpStdio,
|
||||||
|
SessionClientKind::Mcp
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
assert!(revive_session(&id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_end_during_idle_cleanup_cannot_be_revived_by_a_waiting_call() {
|
||||||
|
let id = format!("g08-cleanup-race-{}", std::process::id());
|
||||||
|
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
|
||||||
|
let (finish_tx, finish_rx) = std::sync::mpsc::channel();
|
||||||
|
let finish_rx = Mutex::new(finish_rx);
|
||||||
|
let observed = id.clone();
|
||||||
|
let _hook = register_scoped_session_end_hook(move |ended| {
|
||||||
|
if ended == observed {
|
||||||
|
entered_tx.send(()).unwrap();
|
||||||
|
finish_rx.lock().unwrap().recv().unwrap();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
drop(
|
||||||
|
begin_session_dispatch(
|
||||||
|
&id,
|
||||||
|
None,
|
||||||
|
&id,
|
||||||
|
true,
|
||||||
|
SessionTransport::McpStdio,
|
||||||
|
SessionClientKind::Mcp,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let swept = id.clone();
|
||||||
|
let sweep = std::thread::spawn(move || evict_idle_with_prefix(Duration::ZERO, &swept));
|
||||||
|
entered_rx.recv().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
begin_session_dispatch(
|
||||||
|
&id,
|
||||||
|
None,
|
||||||
|
&id,
|
||||||
|
true,
|
||||||
|
SessionTransport::McpStdio,
|
||||||
|
SessionClientKind::Mcp
|
||||||
|
)
|
||||||
|
.err(),
|
||||||
|
Some("session cleanup is incomplete; retry the call")
|
||||||
|
);
|
||||||
|
end_session(&id);
|
||||||
|
finish_tx.send(()).unwrap();
|
||||||
|
assert_eq!(sweep.join().unwrap(), [id.clone()]);
|
||||||
|
assert!(!recreates_on_next_call(&id, &id));
|
||||||
|
assert!(begin_session_dispatch(
|
||||||
|
&id,
|
||||||
|
None,
|
||||||
|
&id,
|
||||||
|
true,
|
||||||
|
SessionTransport::McpStdio,
|
||||||
|
SessionClientKind::Mcp
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
assert!(revive_session(&id));
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct ProbeObserver {
|
struct ProbeObserver {
|
||||||
starts: Mutex<Vec<(String, SessionStartObservation)>>,
|
starts: Mutex<Vec<(String, SessionStartObservation)>>,
|
||||||
@@ -2026,7 +2271,7 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
});
|
});
|
||||||
|
|
||||||
assert!(mark_session_ended(sid, None));
|
assert!(mark_session_ended(sid, None, SessionEndReason::Explicit));
|
||||||
let pending = session_cleanup_status(sid);
|
let pending = session_cleanup_status(sid);
|
||||||
assert!(
|
assert!(
|
||||||
!pending.complete,
|
!pending.complete,
|
||||||
|
|||||||
@@ -1223,6 +1223,9 @@ fn canonical_manifest_path(path: &Path) -> Result<String, String> {
|
|||||||
}
|
}
|
||||||
Ok(_) => return Err("path contains an unavailable filesystem entry".to_owned()),
|
Ok(_) => return Err("path contains an unavailable filesystem entry".to_owned()),
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotADirectory => {
|
||||||
|
return Err("an ancestor of the path is not a directory".to_owned())
|
||||||
|
}
|
||||||
Err(error) => return Err(error.to_string()),
|
Err(error) => return Err(error.to_string()),
|
||||||
}
|
}
|
||||||
let name = existing
|
let name = existing
|
||||||
@@ -1233,9 +1236,11 @@ fn canonical_manifest_path(path: &Path) -> Result<String, String> {
|
|||||||
.parent()
|
.parent()
|
||||||
.ok_or_else(|| "path has no existing ancestor".to_owned())?;
|
.ok_or_else(|| "path has no existing ancestor".to_owned())?;
|
||||||
}
|
}
|
||||||
let metadata = std::fs::symlink_metadata(existing).map_err(|error| error.to_string())?;
|
// Follow a symlinked deepest ancestor (macOS `/tmp` -> `/private/tmp`);
|
||||||
|
// `canonicalize` below records its resolved target.
|
||||||
|
let metadata = std::fs::metadata(existing).map_err(|error| error.to_string())?;
|
||||||
if !metadata.is_dir() {
|
if !metadata.is_dir() {
|
||||||
return Err("existing ancestor is not a directory".to_owned());
|
return Err("deepest existing ancestor is not a directory".to_owned());
|
||||||
}
|
}
|
||||||
let mut canonical = std::fs::canonicalize(existing).map_err(|error| error.to_string())?;
|
let mut canonical = std::fs::canonicalize(existing).map_err(|error| error.to_string())?;
|
||||||
for component in suffix.into_iter().rev() {
|
for component in suffix.into_iter().rev() {
|
||||||
@@ -1673,6 +1678,31 @@ allow:
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(all(feature = "yaml", unix))]
|
||||||
|
#[test]
|
||||||
|
fn manifest_paths_resolve_a_symlinked_deepest_ancestor() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let target = root.path().join("private-tmp");
|
||||||
|
std::fs::create_dir(&target).unwrap();
|
||||||
|
let link = root.path().join("tmp");
|
||||||
|
std::os::unix::fs::symlink(&target, &link).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
canonical_manifest_path(&link.join("out.png")).unwrap(),
|
||||||
|
std::fs::canonicalize(&target)
|
||||||
|
.unwrap()
|
||||||
|
.join("out.png")
|
||||||
|
.to_string_lossy()
|
||||||
|
);
|
||||||
|
let file = root.path().join("file");
|
||||||
|
std::fs::write(&file, b"").unwrap();
|
||||||
|
let file_link = root.path().join("file-link");
|
||||||
|
std::os::unix::fs::symlink(&file, &file_link).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
canonical_manifest_path(&file_link.join("out.png")).unwrap_err(),
|
||||||
|
"an ancestor of the path is not a directory"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(feature = "yaml")]
|
#[cfg(feature = "yaml")]
|
||||||
#[test]
|
#[test]
|
||||||
fn protected_resources_are_exactly_bounded_by_the_manifest() {
|
fn protected_resources_are_exactly_bounded_by_the_manifest() {
|
||||||
|
|||||||
@@ -36,6 +36,27 @@ tokio::task_local! {
|
|||||||
/// Opaque generation for runtime-owned mutable resources. Nested
|
/// Opaque generation for runtime-owned mutable resources. Nested
|
||||||
/// dispatches inherit this key, while public arguments can never select it.
|
/// dispatches inherit this key, while public arguments can never select it.
|
||||||
static DISPATCH_RUNTIME_SCOPE: String;
|
static DISPATCH_RUNTIME_SCOPE: String;
|
||||||
|
static DISPATCH_COMPUTER_USE_NATIVE: bool;
|
||||||
|
static DISPATCH_AX_READINESS_DEADLINE: Option<std::time::Instant>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[doc(hidden)]
|
||||||
|
pub const AX_READINESS_BUDGET: Duration = Duration::from_millis(500);
|
||||||
|
|
||||||
|
// Read before spawn_blocking, then pass the value explicitly to native code.
|
||||||
|
#[doc(hidden)]
|
||||||
|
pub fn current_ax_readiness_deadline() -> Option<std::time::Instant> {
|
||||||
|
DISPATCH_AX_READINESS_DEADLINE
|
||||||
|
.try_with(|value| *value)
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[doc(hidden)]
|
||||||
|
pub fn computer_use_native_dispatch() -> bool {
|
||||||
|
DISPATCH_COMPUTER_USE_NATIVE
|
||||||
|
.try_with(|native| *native)
|
||||||
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the immutable authorization context bound to the current dispatch.
|
/// Return the immutable authorization context bound to the current dispatch.
|
||||||
@@ -56,6 +77,48 @@ pub fn current_dispatch_runtime_scope() -> Option<String> {
|
|||||||
.or_else(|| CONSTRUCTION_RUNTIME_SCOPE.with(|scope| scope.borrow().clone()))
|
.or_else(|| CONSTRUCTION_RUNTIME_SCOPE.with(|scope| scope.borrow().clone()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Recording workers retain the dispatch that requested their read, even when
|
||||||
|
// the caller times out before the native provider returns.
|
||||||
|
pub(crate) struct CapturedDispatchContext {
|
||||||
|
runtime: Option<String>,
|
||||||
|
authorization: Option<Arc<crate::session_authorization::EffectiveAuthorizationContext>>,
|
||||||
|
evidence: Option<TrustedInvocationEvidence>,
|
||||||
|
native: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CapturedDispatchContext {
|
||||||
|
pub(crate) fn capture() -> Self {
|
||||||
|
Self {
|
||||||
|
runtime: current_dispatch_runtime_scope(),
|
||||||
|
authorization: current_dispatch_authorization_context(),
|
||||||
|
evidence: DISPATCH_TRUSTED_INVOCATION_EVIDENCE
|
||||||
|
.try_with(Clone::clone)
|
||||||
|
.ok(),
|
||||||
|
native: DISPATCH_COMPUTER_USE_NATIVE.try_with(|value| *value).ok(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn run_sync<R>(self, action: impl FnOnce() -> R) -> R {
|
||||||
|
fn enter<T: 'static, R>(
|
||||||
|
key: &'static tokio::task::LocalKey<T>,
|
||||||
|
value: Option<T>,
|
||||||
|
action: impl FnOnce() -> R,
|
||||||
|
) -> R {
|
||||||
|
match value {
|
||||||
|
Some(value) => key.sync_scope(value, action),
|
||||||
|
None => action(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
enter(&DISPATCH_RUNTIME_SCOPE, self.runtime, || {
|
||||||
|
enter(&DISPATCH_AUTHORIZATION_CONTEXT, self.authorization, || {
|
||||||
|
enter(&DISPATCH_TRUSTED_INVOCATION_EVIDENCE, self.evidence, || {
|
||||||
|
enter(&DISPATCH_COMPUTER_USE_NATIVE, self.native, action)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||||
pub(crate) struct DispatchSessionIdentity {
|
pub(crate) struct DispatchSessionIdentity {
|
||||||
pub runtime_scope: String,
|
pub runtime_scope: String,
|
||||||
@@ -245,6 +308,13 @@ impl ToolDef {
|
|||||||
|
|
||||||
fn advertised_runtime_input_schema(tool_name: &str, schema: &Value) -> Value {
|
fn advertised_runtime_input_schema(tool_name: &str, schema: &Value) -> Value {
|
||||||
let mut schema = schema.clone();
|
let mut schema = schema.clone();
|
||||||
|
cua_driver_contract::normalize_input_schema(&mut schema);
|
||||||
|
if tool_name == "set_config" {
|
||||||
|
if let Some(properties) = schema.get_mut("properties").and_then(Value::as_object_mut) {
|
||||||
|
properties.remove("key");
|
||||||
|
properties.remove("value");
|
||||||
|
}
|
||||||
|
}
|
||||||
if !crate::action_target::supports_typed_target(tool_name) {
|
if !crate::action_target::supports_typed_target(tool_name) {
|
||||||
return schema;
|
return schema;
|
||||||
}
|
}
|
||||||
@@ -598,6 +668,7 @@ impl TrustedInvocationEvidence {
|
|||||||
|
|
||||||
/// Thread-safe collection of all registered tools.
|
/// Thread-safe collection of all registered tools.
|
||||||
pub struct ToolRegistry {
|
pub struct ToolRegistry {
|
||||||
|
computer_use: Option<Arc<crate::computer_use::ComputerUseRuntime>>,
|
||||||
tools: HashMap<String, Box<dyn Tool>>,
|
tools: HashMap<String, Box<dyn Tool>>,
|
||||||
/// Ordered list of tool names for `tools/list`.
|
/// Ordered list of tool names for `tools/list`.
|
||||||
order: Vec<String>,
|
order: Vec<String>,
|
||||||
@@ -640,6 +711,7 @@ impl ToolRegistry {
|
|||||||
let weak_ownership = Arc::downgrade(&protected_resource_ownership);
|
let weak_ownership = Arc::downgrade(&protected_resource_ownership);
|
||||||
let session_end_hook =
|
let session_end_hook =
|
||||||
crate::session::register_scoped_session_end_hook(move |session_id| {
|
crate::session::register_scoped_session_end_hook(move |session_id| {
|
||||||
|
crate::element_token::global().clear_session(session_id);
|
||||||
if let Some(ownership) = weak_ownership.upgrade() {
|
if let Some(ownership) = weak_ownership.upgrade() {
|
||||||
ownership.remove_session(session_id);
|
ownership.remove_session(session_id);
|
||||||
}
|
}
|
||||||
@@ -668,6 +740,7 @@ impl ToolRegistry {
|
|||||||
})
|
})
|
||||||
}));
|
}));
|
||||||
Self {
|
Self {
|
||||||
|
computer_use: None,
|
||||||
tools: HashMap::new(),
|
tools: HashMap::new(),
|
||||||
order: Vec::new(),
|
order: Vec::new(),
|
||||||
recording,
|
recording,
|
||||||
@@ -776,6 +849,27 @@ impl ToolRegistry {
|
|||||||
self.tools.insert(name, tool);
|
self.tools.insert(name, tool);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn enable_computer_use(&mut self) {
|
||||||
|
if self.computer_use.is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let runtime = Arc::new(crate::computer_use::ComputerUseRuntime::default());
|
||||||
|
let weak = Arc::downgrade(&runtime);
|
||||||
|
self.retain_session_end_hook(crate::session::register_scoped_session_end_hook(
|
||||||
|
move |session| {
|
||||||
|
if let Some(runtime) = weak.upgrade() {
|
||||||
|
runtime.clear_session(session);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
));
|
||||||
|
for spec in cua_driver_contract::computer_use::tools() {
|
||||||
|
if !self.tools.contains_key(spec.name) {
|
||||||
|
self.register(Box::new(crate::computer_use::Entry::new(spec)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.computer_use = Some(runtime);
|
||||||
|
}
|
||||||
|
|
||||||
pub fn retain_session_end_hook(
|
pub fn retain_session_end_hook(
|
||||||
&mut self,
|
&mut self,
|
||||||
registration: crate::session::SessionEndHookRegistration,
|
registration: crate::session::SessionEndHookRegistration,
|
||||||
@@ -802,6 +896,25 @@ impl ToolRegistry {
|
|||||||
.push(RuntimeCleanup(Some(Box::new(cleanup))));
|
.push(RuntimeCleanup(Some(Box::new(cleanup))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn invoke_computer_use_native(&self, name: &str, args: Value) -> ToolResult {
|
||||||
|
self.invoke_computer_use_native_with_readiness(name, args, None)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn invoke_computer_use_native_with_readiness(
|
||||||
|
&self,
|
||||||
|
name: &str,
|
||||||
|
args: Value,
|
||||||
|
deadline: Option<std::time::Instant>,
|
||||||
|
) -> ToolResult {
|
||||||
|
DISPATCH_AX_READINESS_DEADLINE
|
||||||
|
.scope(
|
||||||
|
deadline,
|
||||||
|
DISPATCH_COMPUTER_USE_NATIVE.scope(true, Box::pin(self.invoke(name, args))),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
/// Register the four platform-independent recording/replay tools.
|
/// Register the four platform-independent recording/replay tools.
|
||||||
/// Call this after all platform tools have been registered.
|
/// Call this after all platform tools have been registered.
|
||||||
pub fn register_recording_tools(&mut self) {
|
pub fn register_recording_tools(&mut self) {
|
||||||
@@ -839,13 +952,52 @@ impl ToolRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn tools_list(&self) -> Value {
|
pub fn tools_list(&self) -> Value {
|
||||||
let list: Vec<Value> = self
|
let mut list: Vec<Value> = self
|
||||||
.order
|
.order
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|name| crate::policy::is_tool_listable(name))
|
.filter(|name| crate::policy::is_tool_listable(name))
|
||||||
.filter_map(|name| self.tools.get(name))
|
.filter_map(|name| self.tools.get(name))
|
||||||
.map(|tool| tool.def().to_list_entry())
|
.map(|tool| tool.def().to_list_entry())
|
||||||
.collect();
|
.collect();
|
||||||
|
if self.normalized_coordinates {
|
||||||
|
let mut native = serde_json::json!({"tools":list});
|
||||||
|
crate::coord_norm::rewrite_coord_desc(&mut native);
|
||||||
|
list = native["tools"].as_array().unwrap().clone();
|
||||||
|
}
|
||||||
|
if self.computer_use.is_some() {
|
||||||
|
let extra = std::env::var("CUA_DRIVER_EXTRA_TOOLS").as_deref() == Ok("1");
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
for spec in cua_driver_contract::computer_use::tools() {
|
||||||
|
if !crate::policy::is_tool_listable(spec.name) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut schema = spec.input_schema.clone();
|
||||||
|
if extra {
|
||||||
|
if let Some(native) = list.iter().find(|tool| tool["name"] == spec.name) {
|
||||||
|
schema = serde_json::json!({"type":"object","anyOf":[schema,native["inputSchema"]]});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let risk_tool = match spec.name {
|
||||||
|
"get_app" => "get_window_state",
|
||||||
|
"get_state" => "list_apps",
|
||||||
|
"get_ax_state" | "get_screenshot" | "get_ax_state_and_screenshot" => {
|
||||||
|
"get_window_state"
|
||||||
|
}
|
||||||
|
_ => spec.name,
|
||||||
|
};
|
||||||
|
entries.push(serde_json::json!({
|
||||||
|
"name":spec.name,"description":spec.description,"inputSchema":schema,
|
||||||
|
"annotations":{"readOnlyHint":spec.read_only,"destructiveHint":false,"idempotentHint":spec.read_only,"openWorldHint":!spec.read_only},
|
||||||
|
"capabilities":default_capabilities_for(risk_tool),
|
||||||
|
"risk":crate::authorization::risk_metadata_json(risk_tool),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
if extra {
|
||||||
|
list.retain(|entry| !entries.iter().any(|app| app["name"] == entry["name"]));
|
||||||
|
entries.extend(list);
|
||||||
|
}
|
||||||
|
list = entries;
|
||||||
|
}
|
||||||
// `capability_version` is the contract version for the
|
// `capability_version` is the contract version for the
|
||||||
// capability tokens claimed by each tool entry. Bumped on
|
// capability tokens claimed by each tool entry. Bumped on
|
||||||
// BREAKING vocabulary changes only; additive changes (new
|
// BREAKING vocabulary changes only; additive changes (new
|
||||||
@@ -860,16 +1012,13 @@ impl ToolRegistry {
|
|||||||
//
|
//
|
||||||
// Both fields are additive: existing consumers that read only
|
// Both fields are additive: existing consumers that read only
|
||||||
// `tools` keep working unchanged.
|
// `tools` keep working unchanged.
|
||||||
let mut result = serde_json::json!({
|
let result = serde_json::json!({
|
||||||
"tools": list,
|
"tools": list,
|
||||||
"platform": std::env::consts::OS,
|
"platform": std::env::consts::OS,
|
||||||
"capability_version": CAPABILITY_VERSION,
|
"capability_version": CAPABILITY_VERSION,
|
||||||
"schema_version": TOOLS_LIST_SCHEMA_VERSION,
|
"schema_version": TOOLS_LIST_SCHEMA_VERSION,
|
||||||
"enforcement_adapters": crate::authorization::enforcement_adapter_inventory_json(),
|
"enforcement_adapters": crate::authorization::enforcement_adapter_inventory_json(),
|
||||||
});
|
});
|
||||||
if self.normalized_coordinates {
|
|
||||||
crate::coord_norm::rewrite_coord_desc(&mut result);
|
|
||||||
}
|
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1071,6 +1220,16 @@ impl ToolRegistry {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Some(runtime) = &self.computer_use {
|
||||||
|
if !DISPATCH_COMPUTER_USE_NATIVE
|
||||||
|
.try_with(|native| *native)
|
||||||
|
.unwrap_or(false)
|
||||||
|
&& cua_driver_contract::computer_use::is_app_call(name, &args)
|
||||||
|
{
|
||||||
|
return runtime.invoke(self, name, args).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Deprecated alias: `type_text_chars` → `type_text`. Swift's
|
// Deprecated alias: `type_text_chars` → `type_text`. Swift's
|
||||||
// ToolRegistry.swift keeps the same alias (with stderr warning) for
|
// ToolRegistry.swift keeps the same alias (with stderr warning) for
|
||||||
// backwards compatibility with hermes-agent builds that still emit
|
// backwards compatibility with hermes-agent builds that still emit
|
||||||
@@ -1134,8 +1293,13 @@ impl ToolRegistry {
|
|||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(str::to_owned);
|
.map(str::to_owned);
|
||||||
if let Some(session) = runtime_session.as_deref() {
|
if let Some(session) = runtime_session.as_deref() {
|
||||||
|
let owner = args
|
||||||
|
.get("_transport_session_id")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or(session);
|
||||||
if !matches!(resolved_name, "start_session" | "end_session")
|
if !matches!(resolved_name, "start_session" | "end_session")
|
||||||
&& crate::session::is_session_ended(session)
|
&& crate::session::is_session_ended(session)
|
||||||
|
&& !crate::session::recreates_on_next_call(session, owner)
|
||||||
{
|
{
|
||||||
let mut result = protected_refusal(
|
let mut result = protected_refusal(
|
||||||
"session_ended",
|
"session_ended",
|
||||||
@@ -1149,7 +1313,7 @@ impl ToolRegistry {
|
|||||||
// Convert only the private dispatch copy. Authorization, consent,
|
// Convert only the private dispatch copy. Authorization, consent,
|
||||||
// manifests, recordings, and replay continue to use the caller's
|
// manifests, recordings, and replay continue to use the caller's
|
||||||
// normalized arguments, while platform workers receive pixels.
|
// normalized arguments, while platform workers receive pixels.
|
||||||
if self.normalized_coordinates {
|
if self.normalized_coordinates && !computer_use_native_dispatch() {
|
||||||
let pid = args.opt_i64("pid").unwrap_or(0);
|
let pid = args.opt_i64("pid").unwrap_or(0);
|
||||||
let window_id = args.opt_u64("window_id").unwrap_or(0);
|
let window_id = args.opt_u64("window_id").unwrap_or(0);
|
||||||
let (width, height) = crate::coord_norm::get_size(pid, window_id).unwrap_or((0, 0));
|
let (width, height) = crate::coord_norm::get_size(pid, window_id).unwrap_or((0, 0));
|
||||||
@@ -1166,8 +1330,9 @@ impl ToolRegistry {
|
|||||||
// rejected action has no before/after evidence and must not leave a
|
// rejected action has no before/after evidence and must not leave a
|
||||||
// pending recorder entry behind.
|
// pending recorder entry behind.
|
||||||
if let Err(violation) = crate::capture_scope::enforce_tool(resolved_name, &args) {
|
if let Err(violation) = crate::capture_scope::enforce_tool(resolved_name, &args) {
|
||||||
let structured =
|
let mut structured =
|
||||||
violation.as_json(args.get("session").and_then(Value::as_str).unwrap_or(""));
|
violation.as_json(args.get("session").and_then(Value::as_str).unwrap_or(""));
|
||||||
|
structured["sent"] = serde_json::json!(false);
|
||||||
let mut result = ToolResult::error(violation.message).with_structured(structured);
|
let mut result = ToolResult::error(violation.message).with_structured(structured);
|
||||||
restore_public_runtime_result(&mut result, &runtime_prefix);
|
restore_public_runtime_result(&mut result, &runtime_prefix);
|
||||||
return result;
|
return result;
|
||||||
@@ -1523,7 +1688,12 @@ impl ToolRegistry {
|
|||||||
let should_record = !tool.def().read_only
|
let should_record = !tool.def().read_only
|
||||||
&& !matches!(
|
&& !matches!(
|
||||||
resolved_name,
|
resolved_name,
|
||||||
"start_recording" | "stop_recording" | "get_recording_state" | "replay_trajectory"
|
"start_recording"
|
||||||
|
| "stop_recording"
|
||||||
|
| "get_recording_state"
|
||||||
|
| "replay_trajectory"
|
||||||
|
| "start_session"
|
||||||
|
| "end_session"
|
||||||
);
|
);
|
||||||
let private_consent_turn = is_existing_profile_prepare(resolved_name, &args);
|
let private_consent_turn = is_existing_profile_prepare(resolved_name, &args);
|
||||||
let _desktop_action = if is_physical_desktop_action(resolved_name) {
|
let _desktop_action = if is_physical_desktop_action(resolved_name) {
|
||||||
@@ -1540,14 +1710,27 @@ impl ToolRegistry {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
if is_physical_desktop_action(resolved_name) {
|
||||||
|
if let Err(result) = crate::desktop_keyboard::admit_input(resolved_name, &args) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
let pending_turn = should_record
|
let pending_turn = should_record
|
||||||
.then(|| {
|
.then(|| {
|
||||||
|
let caller = crate::recording::RecordingCaller {
|
||||||
|
session: runtime_session.as_deref(),
|
||||||
|
transport: args.get("_transport_session_id").and_then(Value::as_str),
|
||||||
|
};
|
||||||
if private_consent_turn {
|
if private_consent_turn {
|
||||||
self.recording
|
self.recording.begin_private_turn(
|
||||||
.begin_private_turn(resolved_name, &recording_args, start_ms)
|
resolved_name,
|
||||||
|
&recording_args,
|
||||||
|
start_ms,
|
||||||
|
caller,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
self.recording
|
self.recording
|
||||||
.begin_turn(resolved_name, &recording_args, start_ms)
|
.begin_turn(resolved_name, &recording_args, start_ms, caller)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.flatten();
|
.flatten();
|
||||||
@@ -1558,7 +1741,9 @@ impl ToolRegistry {
|
|||||||
crate::coord_norm::ingest_window_size(resolved_name, &args, &result);
|
crate::coord_norm::ingest_window_size(resolved_name, &args, &result);
|
||||||
crate::coord_norm::ingest_screen_size(resolved_name, &result);
|
crate::coord_norm::ingest_screen_size(resolved_name, &result);
|
||||||
crate::coord_norm::ingest_zoom_size(resolved_name, &args, &result);
|
crate::coord_norm::ingest_zoom_size(resolved_name, &args, &result);
|
||||||
crate::coord_norm::normalize_result(resolved_name, &mut result);
|
if !computer_use_native_dispatch() {
|
||||||
|
crate::coord_norm::normalize_result(resolved_name, &mut result);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// The platform worker has exited, so another text operation for this
|
// The platform worker has exited, so another text operation for this
|
||||||
// pid may now start even while result projection and evidence capture
|
// pid may now start even while result projection and evidence capture
|
||||||
@@ -1858,7 +2043,7 @@ impl ToolRegistry {
|
|||||||
// checked against its approved manifest.
|
// checked against its approved manifest.
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let delivery_mode = if tool_name == "bring_to_front" {
|
let delivery_mode = if tool_name == "bring_to_front" || args["scope"] == "desktop" {
|
||||||
"foreground"
|
"foreground"
|
||||||
} else {
|
} else {
|
||||||
args.get("delivery_mode")
|
args.get("delivery_mode")
|
||||||
@@ -2445,7 +2630,12 @@ fn canonical_existing_file(raw: &str) -> Result<String, ToolResult> {
|
|||||||
/// The deepest existing ancestor is canonicalized first, so symlinked parents
|
/// The deepest existing ancestor is canonicalized first, so symlinked parents
|
||||||
/// are captured in the approved identity. Only normal path components may be
|
/// are captured in the approved identity. Only normal path components may be
|
||||||
/// appended after that ancestor; lexical parent traversal never enters a
|
/// appended after that ancestor; lexical parent traversal never enters a
|
||||||
/// protected-resource digest.
|
/// protected-resource digest. The caller replaces the raw argument with this
|
||||||
|
/// canonical path, so the tool writes exactly the approved location.
|
||||||
|
///
|
||||||
|
/// A deepest existing ancestor that is itself a symbolic link to a directory
|
||||||
|
/// (macOS `/tmp` -> `/private/tmp`) is resolved like any other symlinked
|
||||||
|
/// parent. A link to a non-directory is refused.
|
||||||
fn canonical_proposed_path(raw: &str) -> Result<String, ToolResult> {
|
fn canonical_proposed_path(raw: &str) -> Result<String, ToolResult> {
|
||||||
let path = expanded_path(raw)?;
|
let path = expanded_path(raw)?;
|
||||||
if path.exists() {
|
if path.exists() {
|
||||||
@@ -2469,6 +2659,11 @@ fn canonical_proposed_path(raw: &str) -> Result<String, ToolResult> {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotADirectory => {
|
||||||
|
return Err(protected_scope_refusal(
|
||||||
|
"an ancestor of the output path is not a directory",
|
||||||
|
))
|
||||||
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
return Err(protected_scope_refusal(
|
return Err(protected_scope_refusal(
|
||||||
"the output path could not be inspected safely",
|
"the output path could not be inspected safely",
|
||||||
@@ -2483,11 +2678,13 @@ fn canonical_proposed_path(raw: &str) -> Result<String, ToolResult> {
|
|||||||
.parent()
|
.parent()
|
||||||
.ok_or_else(|| protected_scope_refusal("the output path has no existing ancestor"))?;
|
.ok_or_else(|| protected_scope_refusal("the output path has no existing ancestor"))?;
|
||||||
}
|
}
|
||||||
let metadata = std::fs::symlink_metadata(existing)
|
// Follow a symlinked ancestor: `exists()` above already resolved it, and
|
||||||
|
// `canonicalize` below records its target in the approved identity.
|
||||||
|
let metadata = std::fs::metadata(existing)
|
||||||
.map_err(|_| protected_scope_refusal("the output ancestor is unavailable"))?;
|
.map_err(|_| protected_scope_refusal("the output ancestor is unavailable"))?;
|
||||||
if !metadata.is_dir() {
|
if !metadata.is_dir() {
|
||||||
return Err(protected_scope_refusal(
|
return Err(protected_scope_refusal(
|
||||||
"the output path's existing ancestor is not a directory",
|
"the output path's deepest existing ancestor is not a directory",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let mut canonical = std::fs::canonicalize(existing)
|
let mut canonical = std::fs::canonicalize(existing)
|
||||||
@@ -2523,6 +2720,8 @@ fn is_physical_desktop_action(tool: &str) -> bool {
|
|||||||
| "mouse_button_up"
|
| "mouse_button_up"
|
||||||
| "type_text"
|
| "type_text"
|
||||||
| "press_key"
|
| "press_key"
|
||||||
|
| "key_down"
|
||||||
|
| "key_up"
|
||||||
| "hotkey"
|
| "hotkey"
|
||||||
| "set_value"
|
| "set_value"
|
||||||
| "paste"
|
| "paste"
|
||||||
@@ -2732,6 +2931,37 @@ fn restore_public_runtime_value(value: &mut Value, runtime_prefix: &str) -> bool
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod runtime_isolation_tests {
|
mod runtime_isolation_tests {
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn readiness_deadline_follows_the_task_and_is_explicitly_captured_for_workers() {
|
||||||
|
let first = std::time::Instant::now();
|
||||||
|
let second = first + std::time::Duration::from_secs(1);
|
||||||
|
let run = |deadline| {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
super::DISPATCH_AX_READINESS_DEADLINE
|
||||||
|
.scope(Some(deadline), async move {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(super::current_ax_readiness_deadline(), Some(deadline));
|
||||||
|
let captured = super::current_ax_readiness_deadline();
|
||||||
|
let worker = tokio::task::spawn_blocking(move || {
|
||||||
|
assert!(super::current_ax_readiness_deadline().is_none());
|
||||||
|
captured
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(worker, Some(deadline));
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(super::current_ax_readiness_deadline(), Some(deadline));
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(super::current_ax_readiness_deadline().is_none());
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let (a, b) = tokio::join!(run(first), run(second));
|
||||||
|
a.unwrap();
|
||||||
|
b.unwrap();
|
||||||
|
assert!(super::current_ax_readiness_deadline().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
canonical_proposed_path, desktop_action_coordinator, namespace_runtime_args,
|
canonical_proposed_path, desktop_action_coordinator, namespace_runtime_args,
|
||||||
publish_action_result, restore_public_runtime_result, try_admit_text_input,
|
publish_action_result, restore_public_runtime_result, try_admit_text_input,
|
||||||
@@ -2763,6 +2993,74 @@ mod runtime_isolation_tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn captured_dispatch_context_preserves_task_scope_and_restores_worker_scope() {
|
||||||
|
let auth = standard_context();
|
||||||
|
let evidence = TrustedInvocationEvidence {
|
||||||
|
session_id: Some("session-a".into()),
|
||||||
|
transport_session_id: Some("transport-a".into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
DISPATCH_RUNTIME_SCOPE.scope("runtime-a".into(), async {
|
||||||
|
super::DISPATCH_AUTHORIZATION_CONTEXT.scope(auth.clone(), async {
|
||||||
|
super::DISPATCH_TRUSTED_INVOCATION_EVIDENCE.scope(evidence, async {
|
||||||
|
super::DISPATCH_COMPUTER_USE_NATIVE.scope(true, async {
|
||||||
|
let context = super::with_runtime_scope("construction-fallback".into(), || {
|
||||||
|
super::CapturedDispatchContext::capture()
|
||||||
|
});
|
||||||
|
let worker_auth = auth.clone();
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
super::with_runtime_scope("worker-fallback".into(), || {
|
||||||
|
DISPATCH_RUNTIME_SCOPE.sync_scope("worker-runtime".into(), || {
|
||||||
|
context.run_sync(|| {
|
||||||
|
assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("runtime-a"));
|
||||||
|
assert!(Arc::ptr_eq(&super::current_dispatch_authorization_context().unwrap(), &worker_auth));
|
||||||
|
assert!(super::computer_use_native_dispatch());
|
||||||
|
assert_eq!(super::current_dispatch_session_identity(), Some(super::DispatchSessionIdentity {
|
||||||
|
runtime_scope: "runtime-a".into(),
|
||||||
|
session_id: "__cua_runtime_runtime-a:session-a".into(),
|
||||||
|
transport_session_id: "__cua_runtime_runtime-a:transport-a".into(),
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("worker-runtime"));
|
||||||
|
assert!(super::current_dispatch_authorization_context().is_none());
|
||||||
|
assert!(super::current_dispatch_session_identity().is_none());
|
||||||
|
assert!(!super::computer_use_native_dispatch());
|
||||||
|
});
|
||||||
|
assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("worker-fallback"));
|
||||||
|
});
|
||||||
|
assert!(super::current_dispatch_runtime_scope().is_none());
|
||||||
|
}).join().unwrap();
|
||||||
|
assert_eq!(super::current_dispatch_runtime_scope().as_deref(), Some("runtime-a"));
|
||||||
|
assert!(super::computer_use_native_dispatch());
|
||||||
|
}).await;
|
||||||
|
}).await;
|
||||||
|
}).await;
|
||||||
|
}).await;
|
||||||
|
assert!(super::current_dispatch_runtime_scope().is_none());
|
||||||
|
assert!(super::current_dispatch_authorization_context().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn captured_dispatch_context_preserves_explicit_thread_runtime() {
|
||||||
|
let context = super::with_runtime_scope("construction-runtime".into(), || {
|
||||||
|
super::CapturedDispatchContext::capture()
|
||||||
|
});
|
||||||
|
assert!(super::current_dispatch_runtime_scope().is_none());
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
context.run_sync(|| {
|
||||||
|
assert_eq!(
|
||||||
|
super::current_dispatch_runtime_scope().as_deref(),
|
||||||
|
Some("construction-runtime")
|
||||||
|
);
|
||||||
|
assert!(super::current_dispatch_session_identity().is_none());
|
||||||
|
});
|
||||||
|
assert!(super::current_dispatch_runtime_scope().is_none());
|
||||||
|
})
|
||||||
|
.join()
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
fn unrestricted_context() -> Arc<crate::session_authorization::EffectiveAuthorizationContext> {
|
fn unrestricted_context() -> Arc<crate::session_authorization::EffectiveAuthorizationContext> {
|
||||||
let ceiling = SessionModeCeiling::for_trusted_sessions(
|
let ceiling = SessionModeCeiling::for_trusted_sessions(
|
||||||
[PermissionMode::Unrestricted],
|
[PermissionMode::Unrestricted],
|
||||||
@@ -3124,6 +3422,139 @@ mod runtime_isolation_tests {
|
|||||||
registry
|
registry
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn recording_scope_registry(hits: Arc<AtomicUsize>) -> Arc<super::ToolRegistry> {
|
||||||
|
let mut registry = super::ToolRegistry::new();
|
||||||
|
registry.register(Box::new(ReplayProbe {
|
||||||
|
hits,
|
||||||
|
def: super::ToolDef {
|
||||||
|
name: "probe".into(),
|
||||||
|
description: "runtime-local recording-scope probe".into(),
|
||||||
|
input_schema: serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": { "session": { "type": "string" } },
|
||||||
|
}),
|
||||||
|
read_only: false,
|
||||||
|
destructive: false,
|
||||||
|
idempotent: true,
|
||||||
|
open_world: false,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
registry.register_recording_tools();
|
||||||
|
registry.register_session_tools();
|
||||||
|
let registry = Arc::new(registry);
|
||||||
|
registry.init_self_weak();
|
||||||
|
registry
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn recording_keeps_only_the_owning_session_and_never_lifecycle_calls() {
|
||||||
|
let registry = recording_scope_registry(Arc::new(AtomicUsize::new(0)));
|
||||||
|
let context = unrestricted_context();
|
||||||
|
let output = tempfile::tempdir().expect("temp dir");
|
||||||
|
|
||||||
|
let started = registry
|
||||||
|
.invoke_with_context(
|
||||||
|
"start_recording",
|
||||||
|
serde_json::json!({
|
||||||
|
"output_dir": output.path(),
|
||||||
|
"record_video": false,
|
||||||
|
"session": "owner",
|
||||||
|
}),
|
||||||
|
context.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_ne!(started.is_error, Some(true), "{started:?}");
|
||||||
|
|
||||||
|
for (tool, session) in [
|
||||||
|
("probe", "other"),
|
||||||
|
("end_session", "other"),
|
||||||
|
("end_session", "owner-lifecycle"),
|
||||||
|
] {
|
||||||
|
let result = registry
|
||||||
|
.invoke_with_context(
|
||||||
|
tool,
|
||||||
|
serde_json::json!({ "session": session }),
|
||||||
|
context.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_ne!(result.is_error, Some(true), "{tool} {session}: {result:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let owned = registry
|
||||||
|
.invoke_with_context(
|
||||||
|
"probe",
|
||||||
|
serde_json::json!({ "session": "owner" }),
|
||||||
|
context.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_ne!(owned.is_error, Some(true), "{owned:?}");
|
||||||
|
registry
|
||||||
|
.invoke_with_context("stop_recording", serde_json::json!({}), context)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let action: serde_json::Value = serde_json::from_slice(
|
||||||
|
&std::fs::read(output.path().join("turn-00001").join("action.json"))
|
||||||
|
.expect("the owning session's action is turn-00001"),
|
||||||
|
)
|
||||||
|
.expect("parse action.json");
|
||||||
|
assert_eq!(action["tool"], "probe");
|
||||||
|
assert_eq!(action["arguments"]["session"], "owner");
|
||||||
|
assert!(
|
||||||
|
!output.path().join("turn-00002").exists(),
|
||||||
|
"foreign and lifecycle calls must not leave turn folders"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn implicit_recording_keeps_only_calls_on_the_owning_connection() {
|
||||||
|
let registry = recording_scope_registry(Arc::new(AtomicUsize::new(0)));
|
||||||
|
let context = unrestricted_context();
|
||||||
|
let output = tempfile::tempdir().expect("temp dir");
|
||||||
|
let evidence = |transport| {
|
||||||
|
super::TrustedInvocationEvidence::extract_from_adapter_args(
|
||||||
|
&mut serde_json::json!({"_transport_session_id": transport}),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let started = registry
|
||||||
|
.invoke_with_context_and_evidence(
|
||||||
|
"start_recording",
|
||||||
|
serde_json::json!({"output_dir": output.path(), "record_video": false}),
|
||||||
|
context.clone(),
|
||||||
|
evidence("owner-connection"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_ne!(started.is_error, Some(true), "{started:?}");
|
||||||
|
assert!(registry.recording.current_state().owner.is_some());
|
||||||
|
for (transport, session) in [
|
||||||
|
("owner-connection", "named-on-owner"),
|
||||||
|
("foreign-connection", "foreign"),
|
||||||
|
] {
|
||||||
|
let result = registry
|
||||||
|
.invoke_with_context_and_evidence(
|
||||||
|
"probe",
|
||||||
|
serde_json::json!({"session": session}),
|
||||||
|
context.clone(),
|
||||||
|
evidence(transport),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_ne!(result.is_error, Some(true), "{result:?}");
|
||||||
|
}
|
||||||
|
registry
|
||||||
|
.invoke_with_context_and_evidence(
|
||||||
|
"stop_recording",
|
||||||
|
serde_json::json!({}),
|
||||||
|
context,
|
||||||
|
evidence("owner-connection"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let action: serde_json::Value = serde_json::from_slice(
|
||||||
|
&std::fs::read(output.path().join("turn-00001/action.json")).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(action["arguments"]["session"], "named-on-owner");
|
||||||
|
assert!(!output.path().join("turn-00002").exists());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn replay_dispatches_only_through_the_owning_registry() {
|
async fn replay_dispatches_only_through_the_owning_registry() {
|
||||||
let hits_a = Arc::new(AtomicUsize::new(0));
|
let hits_a = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -3172,6 +3603,122 @@ mod runtime_isolation_tests {
|
|||||||
assert_ne!(result.is_error, Some(true));
|
assert_ne!(result.is_error, Some(true));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct IdleSnapshotProbe(super::ToolDef);
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl super::Tool for IdleSnapshotProbe {
|
||||||
|
fn def(&self) -> &super::ToolDef {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
async fn invoke(&self, args: serde_json::Value) -> ToolResult {
|
||||||
|
let owner = args["_session_id"].as_str().expect("admitted session");
|
||||||
|
let window = args["window_id"].as_u64().unwrap() as u32;
|
||||||
|
let id = crate::element_token::global().register_snapshot_payload_for_session(
|
||||||
|
42,
|
||||||
|
window,
|
||||||
|
[0],
|
||||||
|
Arc::new(owner.to_owned()),
|
||||||
|
Some(owner),
|
||||||
|
);
|
||||||
|
ToolResult::text("snapshot").with_structured(serde_json::json!({
|
||||||
|
"token": crate::element_token::token_for(id, 0),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn idle_implicit_recreation_retires_only_its_owned_raw_snapshots() {
|
||||||
|
let mut registry = super::ToolRegistry::new_with_protected_consent_provider(None);
|
||||||
|
registry.register(Box::new(IdleSnapshotProbe(super::ToolDef {
|
||||||
|
name: "get_window_state".into(),
|
||||||
|
description: "snapshot owner probe".into(),
|
||||||
|
input_schema: serde_json::json!({"type":"object"}),
|
||||||
|
read_only: true,
|
||||||
|
destructive: false,
|
||||||
|
idempotent: true,
|
||||||
|
open_world: false,
|
||||||
|
})));
|
||||||
|
registry.register_session_tools();
|
||||||
|
let context = standard_context();
|
||||||
|
let call = |session: Option<&str>, window: u32| {
|
||||||
|
let mut args = serde_json::json!({"pid":42,"window_id":window});
|
||||||
|
if let Some(session) = session {
|
||||||
|
args["session"] = session.into();
|
||||||
|
}
|
||||||
|
registry.invoke_with_context("get_window_state", args, context.clone())
|
||||||
|
};
|
||||||
|
let first = call(None, 7).await;
|
||||||
|
let peer = call(Some("peer"), 8).await;
|
||||||
|
assert_ne!(first.is_error, Some(true), "{first:?}");
|
||||||
|
assert_ne!(peer.is_error, Some(true), "{peer:?}");
|
||||||
|
let token = first.structured_content.unwrap()["token"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.to_owned();
|
||||||
|
let peer_token = peer.structured_content.unwrap()["token"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.to_owned();
|
||||||
|
let implicit = context.runtime_session_key("implicit-direct");
|
||||||
|
assert_eq!(
|
||||||
|
crate::session::evict_idle_with_prefix(Duration::ZERO, &implicit),
|
||||||
|
[implicit.clone()]
|
||||||
|
);
|
||||||
|
super::DISPATCH_RUNTIME_SCOPE
|
||||||
|
.scope(context.runtime_scope_key(), async {
|
||||||
|
assert!(
|
||||||
|
crate::element_token::global().resolve(42, &token).is_err(),
|
||||||
|
"idle cleanup must retire the old snapshot before any new observation"
|
||||||
|
);
|
||||||
|
assert!(crate::element_token::global()
|
||||||
|
.resolve(42, &peer_token)
|
||||||
|
.is_ok());
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let fresh = call(None, 7).await;
|
||||||
|
assert_ne!(fresh.is_error, Some(true), "{fresh:?}");
|
||||||
|
let new_token = fresh.structured_content.unwrap()["token"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.to_owned();
|
||||||
|
super::DISPATCH_RUNTIME_SCOPE
|
||||||
|
.scope(context.runtime_scope_key(), async {
|
||||||
|
assert!(crate::element_token::global().resolve(42, &token).is_err());
|
||||||
|
assert!(crate::element_token::global()
|
||||||
|
.resolve(42, &new_token)
|
||||||
|
.is_ok());
|
||||||
|
assert!(crate::element_token::global()
|
||||||
|
.resolve(42, &peer_token)
|
||||||
|
.is_ok());
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let peer_id = context.runtime_session_key("peer");
|
||||||
|
assert_eq!(
|
||||||
|
crate::session::evict_idle_with_prefix(Duration::ZERO, &peer_id),
|
||||||
|
[peer_id]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
call(Some("peer"), 8).await.structured_content.unwrap()["refusal"]["code"],
|
||||||
|
"session_ended"
|
||||||
|
);
|
||||||
|
let ended = registry
|
||||||
|
.invoke_with_context("end_session", serde_json::json!({}), context.clone())
|
||||||
|
.await;
|
||||||
|
assert_ne!(ended.is_error, Some(true), "{ended:?}");
|
||||||
|
assert_eq!(
|
||||||
|
call(None, 7).await.structured_content.unwrap()["refusal"]["code"],
|
||||||
|
"session_ended"
|
||||||
|
);
|
||||||
|
let restarted = registry
|
||||||
|
.invoke_with_context("start_session", serde_json::json!({}), context.clone())
|
||||||
|
.await;
|
||||||
|
assert_ne!(restarted.is_error, Some(true), "{restarted:?}");
|
||||||
|
assert_ne!(call(None, 7).await.is_error, Some(true));
|
||||||
|
let prefix = context.runtime_session_key("");
|
||||||
|
crate::session::revoke_sessions_with_prefix(&prefix);
|
||||||
|
crate::session::forget_ended_sessions_with_prefix(&prefix);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn default_public_session_uses_an_implicit_runtime_lifecycle() {
|
async fn default_public_session_uses_an_implicit_runtime_lifecycle() {
|
||||||
let hits = Arc::new(AtomicUsize::new(0));
|
let hits = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -4243,6 +4790,55 @@ resources:
|
|||||||
assert_eq!(provider.requests.load(Ordering::SeqCst), 0);
|
assert_eq!(provider.requests.load(Ordering::SeqCst), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn proposed_output_scope_resolves_a_symlinked_deepest_ancestor() {
|
||||||
|
// Mirrors macOS `/tmp/x.png`, where `/tmp` links to `/private/tmp`.
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let target = root.path().join("private-tmp");
|
||||||
|
std::fs::create_dir(&target).unwrap();
|
||||||
|
let link = root.path().join("tmp");
|
||||||
|
std::os::unix::fs::symlink(&target, &link).unwrap();
|
||||||
|
let proposed = link.join("x.png");
|
||||||
|
|
||||||
|
let canonical = canonical_proposed_path(proposed.to_str().unwrap()).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
canonical,
|
||||||
|
std::fs::canonicalize(&target)
|
||||||
|
.unwrap()
|
||||||
|
.join("x.png")
|
||||||
|
.to_string_lossy()
|
||||||
|
);
|
||||||
|
assert!(!proposed.exists());
|
||||||
|
// The same file already existing was always resolved; both agree.
|
||||||
|
std::fs::write(target.join("x.png"), b"").unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
canonical_proposed_path(proposed.to_str().unwrap()).unwrap(),
|
||||||
|
canonical
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn proposed_output_scope_refuses_a_symlinked_ancestor_that_is_not_a_directory() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let file = root.path().join("file");
|
||||||
|
std::fs::write(&file, b"").unwrap();
|
||||||
|
let link = root.path().join("link");
|
||||||
|
std::os::unix::fs::symlink(&file, &link).unwrap();
|
||||||
|
let proposed = link.join("x.png");
|
||||||
|
|
||||||
|
let refusal = canonical_proposed_path(proposed.to_str().unwrap()).unwrap_err();
|
||||||
|
|
||||||
|
let text = serde_json::to_string(&refusal).unwrap();
|
||||||
|
assert!(
|
||||||
|
text.contains("an ancestor of the output path is not a directory"),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
assert!(!file.with_file_name("x.png").exists());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn proposed_output_scope_canonicalizes_the_existing_ancestor_without_creating_output() {
|
fn proposed_output_scope_canonicalizes_the_existing_ancestor_without_creating_output() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let root = tempfile::tempdir().unwrap();
|
||||||
@@ -4585,6 +5181,7 @@ resources:
|
|||||||
fn permission_denied_result(message: String) -> ToolResult {
|
fn permission_denied_result(message: String) -> ToolResult {
|
||||||
ToolResult::error(message.clone()).with_structured(serde_json::json!({
|
ToolResult::error(message.clone()).with_structured(serde_json::json!({
|
||||||
"status": "refused",
|
"status": "refused",
|
||||||
|
"sent": false,
|
||||||
"refusal": {
|
"refusal": {
|
||||||
"code": "permission_denied",
|
"code": "permission_denied",
|
||||||
"message": message,
|
"message": message,
|
||||||
@@ -4614,6 +5211,7 @@ fn protected_consent_refusal(error: crate::consent::ConsentError) -> ToolResult
|
|||||||
let message = error.to_string();
|
let message = error.to_string();
|
||||||
ToolResult::error(message.clone()).with_structured(serde_json::json!({
|
ToolResult::error(message.clone()).with_structured(serde_json::json!({
|
||||||
"status": "refused",
|
"status": "refused",
|
||||||
|
"sent": false,
|
||||||
"refusal": {
|
"refusal": {
|
||||||
"code": code,
|
"code": code,
|
||||||
"message": message,
|
"message": message,
|
||||||
@@ -4628,6 +5226,7 @@ fn protected_scope_refusal(message: &str) -> ToolResult {
|
|||||||
fn protected_refusal(code: &str, message: &str) -> ToolResult {
|
fn protected_refusal(code: &str, message: &str) -> ToolResult {
|
||||||
ToolResult::error(message).with_structured(serde_json::json!({
|
ToolResult::error(message).with_structured(serde_json::json!({
|
||||||
"status": "refused",
|
"status": "refused",
|
||||||
|
"sent": false,
|
||||||
"refusal": {
|
"refusal": {
|
||||||
"code": code,
|
"code": code,
|
||||||
"message": message,
|
"message": message,
|
||||||
|
|||||||
@@ -138,51 +138,47 @@ impl FfmpegVideoBackend {
|
|||||||
impl VideoBackend for FfmpegVideoBackend {
|
impl VideoBackend for FfmpegVideoBackend {
|
||||||
fn stop(mut self: Box<Self>) -> anyhow::Result<VideoMetadata> {
|
fn stop(mut self: Box<Self>) -> anyhow::Result<VideoMetadata> {
|
||||||
let elapsed = self.started_at.elapsed();
|
let elapsed = self.started_at.elapsed();
|
||||||
let finalized;
|
|
||||||
|
|
||||||
if let Some(mut stdin) = self.child.stdin.take() {
|
if let Some(mut stdin) = self.child.stdin.take() {
|
||||||
let _ = stdin.write_all(b"q\n");
|
let _ = stdin.write_all(b"q\n");
|
||||||
let _ = stdin.flush();
|
let _ = stdin.flush();
|
||||||
}
|
}
|
||||||
|
|
||||||
let deadline = Instant::now() + Duration::from_millis(3000);
|
let shutdown_timeout = Duration::from_millis(3000);
|
||||||
loop {
|
let deadline = Instant::now() + shutdown_timeout;
|
||||||
|
let result = loop {
|
||||||
match self.child.try_wait()? {
|
match self.child.try_wait()? {
|
||||||
|
Some(status) if status.success() => break Ok(()),
|
||||||
Some(status) => {
|
Some(status) => {
|
||||||
finalized = status.success();
|
let cause = status
|
||||||
break;
|
.code()
|
||||||
|
.map_or_else(|| status.to_string(), |code| format!("code {code}"));
|
||||||
|
break Err(anyhow::anyhow!("ffmpeg exited with {cause}"));
|
||||||
}
|
}
|
||||||
None => {
|
None if Instant::now() > deadline => {
|
||||||
if Instant::now() > deadline {
|
let _ = self.child.kill();
|
||||||
// Polite shutdown stalled — force kill. mp4 will lack
|
let _ = self.child.wait();
|
||||||
// a moov atom and won't be playable; `finalized:
|
break Err(anyhow::anyhow!(
|
||||||
// false` tells the caller.
|
"ffmpeg shutdown timed out after {} ms",
|
||||||
let _ = self.child.kill();
|
shutdown_timeout.as_millis()
|
||||||
let _ = self.child.wait();
|
));
|
||||||
finalized = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
std::thread::sleep(Duration::from_millis(80));
|
|
||||||
}
|
}
|
||||||
|
None => std::thread::sleep(Duration::from_millis(80)),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some(handle) = self.stderr_thread.take() {
|
||||||
|
let stderr = handle.join().unwrap_or_default();
|
||||||
|
if let Err(error) = &result {
|
||||||
|
tracing::warn!(target: "recording",
|
||||||
|
%error,
|
||||||
|
stderr_tail = %String::from_utf8_lossy(&stderr),
|
||||||
|
"ffmpeg shutdown failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
result?;
|
||||||
if !finalized {
|
|
||||||
if let Some(handle) = self.stderr_thread.take() {
|
|
||||||
if let Ok(buf) = handle.join() {
|
|
||||||
let tail = String::from_utf8_lossy(&buf);
|
|
||||||
tracing::warn!(target: "recording",
|
|
||||||
"ffmpeg did not finalize cleanly. Last stderr tail:\n{tail}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if let Some(handle) = self.stderr_thread.take() {
|
|
||||||
let _ = handle.join();
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(VideoMetadata {
|
Ok(VideoMetadata {
|
||||||
path: self.output_path,
|
path: self.output_path,
|
||||||
duration_ms: elapsed.as_millis() as u64,
|
duration_ms: elapsed.as_millis() as u64,
|
||||||
finalized,
|
finalized: true,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -333,3 +329,66 @@ fn platform_input_args(cmd: &mut Command) {
|
|||||||
let _ = cmd;
|
let _ = cmd;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod shutdown_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn encoder_child() {
|
||||||
|
let Ok(mode) = std::env::var("CUA_P7_ENCODER_CHILD_MODE") else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let mut line = String::new();
|
||||||
|
std::io::stdin().read_line(&mut line).unwrap();
|
||||||
|
assert_eq!(line, "q\n");
|
||||||
|
eprintln!("owned encoder fixture received shutdown");
|
||||||
|
match mode.as_str() {
|
||||||
|
"exit23" => std::process::exit(23),
|
||||||
|
"timeout" => loop {
|
||||||
|
std::thread::sleep(Duration::from_secs(1));
|
||||||
|
},
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stop_reports_encoder_exit_and_timeout_after_reaping_child() {
|
||||||
|
for (mode, expected) in [
|
||||||
|
("ok", None),
|
||||||
|
("exit23", Some("ffmpeg exited with code 23")),
|
||||||
|
("timeout", Some("ffmpeg shutdown timed out after 3000 ms")),
|
||||||
|
] {
|
||||||
|
let mut child = Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"video_ffmpeg::shutdown_tests::encoder_child",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env("CUA_P7_ENCODER_CHILD_MODE", mode)
|
||||||
|
.stdin(Stdio::piped())
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
let mut stderr = child.stderr.take().unwrap();
|
||||||
|
let stderr_thread = std::thread::spawn(move || {
|
||||||
|
use std::io::Read;
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
stderr.read_to_end(&mut bytes).unwrap();
|
||||||
|
bytes
|
||||||
|
});
|
||||||
|
let backend = FfmpegVideoBackend {
|
||||||
|
child,
|
||||||
|
output_path: PathBuf::from("unused-fixture.mp4"),
|
||||||
|
started_at: Instant::now(),
|
||||||
|
stderr_thread: Some(stderr_thread),
|
||||||
|
};
|
||||||
|
let result = Box::new(backend).stop();
|
||||||
|
match expected {
|
||||||
|
Some(message) => assert_eq!(result.unwrap_err().to_string(), message),
|
||||||
|
None => assert!(result.unwrap().finalized),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
//! The real recording API with a controlled native provider; no desktop is opened.
|
||||||
|
use cua_driver_core::recording::{
|
||||||
|
now_ms, set_ax_snapshot_fn, PendingTurn, RecordingCaller, RecordingSession,
|
||||||
|
};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Arc, Condvar, Mutex};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Gate {
|
||||||
|
state: Mutex<(usize, bool)>,
|
||||||
|
changed: Condvar,
|
||||||
|
block_call: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Gate {
|
||||||
|
fn new(block_call: usize) -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
block_call,
|
||||||
|
..Self::default()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capture(&self) -> Option<Vec<u8>> {
|
||||||
|
let mut state = self.state.lock().unwrap();
|
||||||
|
state.0 += 1;
|
||||||
|
let call = state.0;
|
||||||
|
self.changed.notify_all();
|
||||||
|
if call == self.block_call {
|
||||||
|
let (next, timeout) = self
|
||||||
|
.changed
|
||||||
|
.wait_timeout_while(state, Duration::from_secs(10), |state| !state.1)
|
||||||
|
.unwrap();
|
||||||
|
state = next;
|
||||||
|
assert!(
|
||||||
|
!timeout.timed_out(),
|
||||||
|
"test failed to release controlled provider"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(state);
|
||||||
|
Some(serde_json::to_vec(&json!({"call": call})).unwrap())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_for_call(&self, expected: usize) {
|
||||||
|
let (state, timeout) = self
|
||||||
|
.changed
|
||||||
|
.wait_timeout_while(
|
||||||
|
self.state.lock().unwrap(),
|
||||||
|
Duration::from_secs(3),
|
||||||
|
|state| state.0 < expected,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!timeout.timed_out(),
|
||||||
|
"provider did not receive call {expected}: {}",
|
||||||
|
state.0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release(&self) {
|
||||||
|
self.state.lock().unwrap().1 = true;
|
||||||
|
self.changed.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ReleaseAll(Vec<Arc<Gate>>);
|
||||||
|
impl Drop for ReleaseAll {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
for gate in &self.0 {
|
||||||
|
gate.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn begin(session: &RecordingSession, pid: i64, owner: Option<&str>) -> Option<PendingTurn> {
|
||||||
|
session.begin_turn(
|
||||||
|
"budget_probe",
|
||||||
|
&json!({"pid": pid}),
|
||||||
|
now_ms(),
|
||||||
|
RecordingCaller {
|
||||||
|
session: owner,
|
||||||
|
transport: owner,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evidence(dir: &std::path::Path, turn: usize) -> Value {
|
||||||
|
serde_json::from_slice(
|
||||||
|
&std::fs::read(dir.join(format!("turn-{turn:05}/evidence.json"))).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bounded_native_capture_releases_locks_and_rejects_late_generations() {
|
||||||
|
let gates = Arc::new(Mutex::new(HashMap::<i64, Arc<Gate>>::new()));
|
||||||
|
let provider_gates = gates.clone();
|
||||||
|
set_ax_snapshot_fn(move |_, pid| {
|
||||||
|
let gate = provider_gates.lock().unwrap().get(&pid?).cloned()?;
|
||||||
|
gate.capture()
|
||||||
|
});
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let mut releases = ReleaseAll(Vec::new());
|
||||||
|
|
||||||
|
// Four slow providers time out for their callers but retain all four slots.
|
||||||
|
let mut captures = Vec::new();
|
||||||
|
for pid in 1..=4 {
|
||||||
|
let gate = Gate::new(1);
|
||||||
|
releases.0.push(gate.clone());
|
||||||
|
gates.lock().unwrap().insert(pid, gate.clone());
|
||||||
|
let session = Arc::new(RecordingSession::new());
|
||||||
|
let dir = root.path().join(format!("timeout-{pid}"));
|
||||||
|
session.start(dir.to_str().unwrap(), false, None).unwrap();
|
||||||
|
let worker_session = session.clone();
|
||||||
|
let worker = std::thread::spawn(move || {
|
||||||
|
let start = Instant::now();
|
||||||
|
let pending = begin(&worker_session, pid, None).unwrap();
|
||||||
|
(pending, start.elapsed())
|
||||||
|
});
|
||||||
|
gate.wait_for_call(1);
|
||||||
|
captures.push((session, dir, worker, gate));
|
||||||
|
}
|
||||||
|
let fifth = Gate::new(0);
|
||||||
|
gates.lock().unwrap().insert(5, fifth.clone());
|
||||||
|
let spare = RecordingSession::new();
|
||||||
|
let spare_dir = root.path().join("spare");
|
||||||
|
spare
|
||||||
|
.start(spare_dir.to_str().unwrap(), false, None)
|
||||||
|
.unwrap();
|
||||||
|
let started = Instant::now();
|
||||||
|
let pending = begin(&spare, 5, None).unwrap();
|
||||||
|
spare.finish_turn(pending, "busy");
|
||||||
|
assert!(started.elapsed() < Duration::from_secs(1));
|
||||||
|
assert_eq!(fifth.state.lock().unwrap().0, 0);
|
||||||
|
assert_eq!(
|
||||||
|
evidence(&spare_dir, 1)["before"]["state"]["classification"],
|
||||||
|
"state_capture_busy"
|
||||||
|
);
|
||||||
|
|
||||||
|
for (session, dir, worker, gate) in captures {
|
||||||
|
let (pending, elapsed) = worker.join().unwrap();
|
||||||
|
assert!(elapsed >= Duration::from_millis(1900));
|
||||||
|
assert!(elapsed < Duration::from_secs(4));
|
||||||
|
session.finish_turn(pending, "timeout");
|
||||||
|
let result = evidence(&dir, 1);
|
||||||
|
assert_eq!(
|
||||||
|
result["before"]["state"]["classification"],
|
||||||
|
"state_capture_timeout"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
result["after"]["state"]["classification"],
|
||||||
|
"state_capture_busy"
|
||||||
|
);
|
||||||
|
assert_eq!(gate.state.lock().unwrap().0, 1);
|
||||||
|
assert!(!dir.join("turn-00001/before_state.json").exists());
|
||||||
|
gate.release();
|
||||||
|
session.stop_owner(None).unwrap();
|
||||||
|
}
|
||||||
|
// Wait for a slot to be released by the worker, not by its timed-out caller.
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(2);
|
||||||
|
loop {
|
||||||
|
let pending = begin(&spare, 5, None).unwrap();
|
||||||
|
spare.finish_turn(pending, "recovered");
|
||||||
|
if fifth.state.lock().unwrap().0 > 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
Instant::now() < deadline,
|
||||||
|
"workers did not release their slots"
|
||||||
|
);
|
||||||
|
std::thread::yield_now();
|
||||||
|
}
|
||||||
|
spare.stop_owner(None).unwrap();
|
||||||
|
for pid in 1..=4 {
|
||||||
|
let turn = root.path().join(format!("timeout-{pid}/turn-00001"));
|
||||||
|
assert!(!turn.join("before_state.json").exists());
|
||||||
|
assert!(!turn.join("after_state.json").exists());
|
||||||
|
assert!(!turn.join("app_state.json").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
// An after capture must neither hold the session lock nor outlive its owner/generation.
|
||||||
|
for (pid, replacement_owner) in [(10, "owner-a"), (11, "owner-b")] {
|
||||||
|
let gate = Gate::new(2);
|
||||||
|
releases.0.push(gate.clone());
|
||||||
|
gates.lock().unwrap().insert(pid, gate.clone());
|
||||||
|
let session = Arc::new(RecordingSession::new());
|
||||||
|
let old_dir = root.path().join(format!("old-{pid}"));
|
||||||
|
let new_dir = root.path().join(format!("new-{pid}"));
|
||||||
|
session
|
||||||
|
.start(old_dir.to_str().unwrap(), false, Some("owner-a"))
|
||||||
|
.unwrap();
|
||||||
|
let pending = begin(&session, pid, Some("owner-a")).unwrap();
|
||||||
|
let worker_session = session.clone();
|
||||||
|
let worker = std::thread::spawn(move || worker_session.finish_turn(pending, "old result"));
|
||||||
|
gate.wait_for_call(2);
|
||||||
|
let started = Instant::now();
|
||||||
|
assert!(session.current_state().enabled);
|
||||||
|
session.stop_owner(Some("owner-a")).unwrap();
|
||||||
|
session
|
||||||
|
.start(new_dir.to_str().unwrap(), false, Some(replacement_owner))
|
||||||
|
.unwrap();
|
||||||
|
assert!(started.elapsed() < Duration::from_secs(1));
|
||||||
|
gate.release();
|
||||||
|
worker.join().unwrap();
|
||||||
|
for dir in [&old_dir, &new_dir] {
|
||||||
|
for file in [
|
||||||
|
"action.json",
|
||||||
|
"after_state.json",
|
||||||
|
"app_state.json",
|
||||||
|
"evidence.json",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!dir.join("turn-00001").join(file).exists(),
|
||||||
|
"stale write: {dir:?}/{file}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
session.stop_owner(Some(replacement_owner)).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,9 +36,6 @@ platform-linux = { path = "../platform-linux" }
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
portal-input = ["platform-linux/portal-input"]
|
|
||||||
portal-capture = ["platform-linux/portal-capture"]
|
|
||||||
portal-libei = ["portal-input", "portal-capture"]
|
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
|
|||||||
@@ -435,6 +435,8 @@ impl EmbeddedCuaDriverHost {
|
|||||||
Stdio::null()
|
Stdio::null()
|
||||||
})
|
})
|
||||||
.kill_on_drop(true);
|
.kill_on_drop(true);
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
prevent_console_window(&mut command);
|
||||||
for variable in safe_environment(&self.options.environment) {
|
for variable in safe_environment(&self.options.environment) {
|
||||||
command.env(variable.name, variable.value);
|
command.env(variable.name, variable.value);
|
||||||
}
|
}
|
||||||
@@ -846,12 +848,14 @@ pub(crate) fn allowed_environment_name(name: &str) -> bool {
|
|||||||
| "LOCALAPPDATA"
|
| "LOCALAPPDATA"
|
||||||
| "PROGRAMDATA"
|
| "PROGRAMDATA"
|
||||||
| "DISPLAY"
|
| "DISPLAY"
|
||||||
| "WAYLAND_DISPLAY"
|
|
||||||
| "XDG_RUNTIME_DIR"
|
| "XDG_RUNTIME_DIR"
|
||||||
| "XDG_SESSION_TYPE"
|
| "XDG_SESSION_TYPE"
|
||||||
| "DBUS_SESSION_BUS_ADDRESS"
|
| "DBUS_SESSION_BUS_ADDRESS"
|
||||||
| "XAUTHORITY"
|
| "XAUTHORITY"
|
||||||
| "CUA_LOG"
|
| "CUA_LOG"
|
||||||
|
| "CUA_DRIVER_RS_TELEMETRY_ENABLED"
|
||||||
|
| "CUA_TELEMETRY_ENABLED"
|
||||||
|
| "CUA_DRIVER_RS_UPDATE_CHECK"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1127,6 +1131,18 @@ async fn terminate_startup_child(child: &mut Child, liveness: &mut ChildStdin) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
fn prevent_console_window(command: &mut Command) {
|
||||||
|
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||||
|
#[link(name = "kernel32")]
|
||||||
|
unsafe extern "system" {
|
||||||
|
fn GetConsoleCP() -> u32;
|
||||||
|
}
|
||||||
|
if unsafe { GetConsoleCP() } == 0 {
|
||||||
|
command.creation_flags(CREATE_NO_WINDOW);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -1194,12 +1210,37 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn environment_is_allowlisted_and_driver_controls_are_reserved() {
|
fn environment_is_allowlisted_and_driver_controls_are_reserved() {
|
||||||
assert!(allowed_environment_name("PATH"));
|
assert!(allowed_environment_name("PATH"));
|
||||||
assert!(allowed_environment_name("WAYLAND_DISPLAY"));
|
assert!(!allowed_environment_name("WAYLAND_DISPLAY"));
|
||||||
assert!(!allowed_environment_name("CUA_DRIVER_PERMISSION_MODE"));
|
assert!(!allowed_environment_name("CUA_DRIVER_PERMISSION_MODE"));
|
||||||
assert!(!allowed_environment_name("LD_PRELOAD"));
|
assert!(!allowed_environment_name("LD_PRELOAD"));
|
||||||
assert!(!allowed_environment_name("NODE_OPTIONS"));
|
assert!(!allowed_environment_name("NODE_OPTIONS"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn embedded_environment_preserves_explicit_telemetry_and_update_preferences() {
|
||||||
|
for name in [
|
||||||
|
"CUA_DRIVER_RS_TELEMETRY_ENABLED",
|
||||||
|
"CUA_TELEMETRY_ENABLED",
|
||||||
|
"CUA_DRIVER_RS_UPDATE_CHECK",
|
||||||
|
] {
|
||||||
|
let inherited = [(name.to_owned(), "0".to_owned())];
|
||||||
|
let values = merge_safe_environment(inherited.clone(), &[]);
|
||||||
|
assert!(values
|
||||||
|
.iter()
|
||||||
|
.any(|variable| variable.name == name && variable.value == "0"));
|
||||||
|
|
||||||
|
let values = merge_safe_environment(
|
||||||
|
inherited,
|
||||||
|
&[EmbeddedEnvironmentVariable {
|
||||||
|
name: name.to_owned(),
|
||||||
|
value: "false".to_owned(),
|
||||||
|
}],
|
||||||
|
);
|
||||||
|
assert_eq!(values.len(), 1);
|
||||||
|
assert_eq!(values[0].value, "false");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn managed_environment_is_inherited_case_insensitively_but_never_overridden() {
|
fn managed_environment_is_inherited_case_insensitively_but_never_overridden() {
|
||||||
assert!(inherited_managed_environment_name(
|
assert!(inherited_managed_environment_name(
|
||||||
@@ -1239,6 +1280,8 @@ mod tests {
|
|||||||
fn interactive_linux_session_environment_is_inherited() {
|
fn interactive_linux_session_environment_is_inherited() {
|
||||||
let values = merge_safe_environment(
|
let values = merge_safe_environment(
|
||||||
[
|
[
|
||||||
|
("DISPLAY".into(), ":7".into()),
|
||||||
|
("XAUTHORITY".into(), "/tmp/cua-test-cookie".into()),
|
||||||
("WAYLAND_DISPLAY".into(), "wayland-7".into()),
|
("WAYLAND_DISPLAY".into(), "wayland-7".into()),
|
||||||
("XDG_RUNTIME_DIR".into(), "/run/user/1000".into()),
|
("XDG_RUNTIME_DIR".into(), "/run/user/1000".into()),
|
||||||
("XDG_SESSION_TYPE".into(), "wayland".into()),
|
("XDG_SESSION_TYPE".into(), "wayland".into()),
|
||||||
@@ -1252,7 +1295,8 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
for (name, value) in [
|
for (name, value) in [
|
||||||
("WAYLAND_DISPLAY", "wayland-7"),
|
("DISPLAY", ":7"),
|
||||||
|
("XAUTHORITY", "/tmp/cua-test-cookie"),
|
||||||
("XDG_RUNTIME_DIR", "/run/user/1000"),
|
("XDG_RUNTIME_DIR", "/run/user/1000"),
|
||||||
("XDG_SESSION_TYPE", "wayland"),
|
("XDG_SESSION_TYPE", "wayland"),
|
||||||
("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"),
|
("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"),
|
||||||
@@ -1263,7 +1307,8 @@ mod tests {
|
|||||||
}
|
}
|
||||||
assert!(!values
|
assert!(!values
|
||||||
.iter()
|
.iter()
|
||||||
.any(|variable| variable.name == "AT_SPI_BUS_ADDRESS"));
|
.any(|variable| variable.name == "AT_SPI_BUS_ADDRESS"
|
||||||
|
|| variable.name == "WAYLAND_DISPLAY"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1362,4 +1407,76 @@ mod tests {
|
|||||||
assert_eq!(host.state(), EmbeddedDriverHostState::Stopped);
|
assert_eq!(host.state(), EmbeddedDriverHostState::Stopped);
|
||||||
assert!(!socket_path.exists());
|
assert!(!socket_path.exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn daemon_console_window_follows_the_host_console() {
|
||||||
|
use std::os::windows::process::CommandExt as _;
|
||||||
|
|
||||||
|
const ROLE: &str = "CUA_DRIVER_SDK_TEST_CONSOLE_ROLE";
|
||||||
|
const HOST_PID: &str = "CUA_DRIVER_SDK_TEST_CONSOLE_HOST_PID";
|
||||||
|
const DETACHED_PROCESS: u32 = 0x0000_0008;
|
||||||
|
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||||
|
#[link(name = "kernel32")]
|
||||||
|
unsafe extern "system" {
|
||||||
|
fn GetConsoleWindow() -> *mut std::ffi::c_void;
|
||||||
|
fn GetConsoleProcessList(list: *mut u32, count: u32) -> u32;
|
||||||
|
}
|
||||||
|
let rerun = [
|
||||||
|
"--exact",
|
||||||
|
"embedded::tests::daemon_console_window_follows_the_host_console",
|
||||||
|
"--nocapture",
|
||||||
|
];
|
||||||
|
let assert_probe_passed = |output: std::process::Output| {
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
|
assert!(
|
||||||
|
output.status.success() && stdout.contains("1 passed"),
|
||||||
|
"{stdout}{}",
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
match std::env::var(ROLE).as_deref() {
|
||||||
|
Ok("probe-without-console") => {
|
||||||
|
let window = unsafe { GetConsoleWindow() };
|
||||||
|
assert!(
|
||||||
|
window.is_null(),
|
||||||
|
"a console window was opened for the daemon"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok("probe-with-console") => {
|
||||||
|
let host: u32 = std::env::var(HOST_PID).unwrap().parse().unwrap();
|
||||||
|
let mut attached = [0u32; 64];
|
||||||
|
let count = unsafe { GetConsoleProcessList(attached.as_mut_ptr(), 64) } as usize;
|
||||||
|
assert!(
|
||||||
|
attached[..count.min(64)].contains(&host),
|
||||||
|
"the daemon left its host's console"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(host @ ("host-without-console" | "host-with-console")) => {
|
||||||
|
let mut probe = Command::new(std::env::current_exe().unwrap());
|
||||||
|
probe
|
||||||
|
.args(rerun)
|
||||||
|
.env(ROLE, host.replace("host-", "probe-"))
|
||||||
|
.env(HOST_PID, std::process::id().to_string())
|
||||||
|
.stdin(Stdio::null());
|
||||||
|
prevent_console_window(&mut probe);
|
||||||
|
assert_probe_passed(probe.output().await.unwrap());
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
for (host, flags) in [
|
||||||
|
("host-without-console", DETACHED_PROCESS),
|
||||||
|
("host-with-console", CREATE_NO_WINDOW),
|
||||||
|
] {
|
||||||
|
let output = std::process::Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args(rerun)
|
||||||
|
.env(ROLE, host)
|
||||||
|
.creation_flags(flags)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert_probe_passed(output);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1799,7 +1799,19 @@ fn normalize_result(tool: &str, raw: Value) -> Result<ToolResult, DriverError> {
|
|||||||
.get("isError")
|
.get("isError")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
let action = if !is_error && cua_driver_core::action_record::is_action_tool(tool) {
|
let app_action =
|
||||||
|
structured.is_some_and(|value| value.get("computer_use") == Some(&serde_json::json!(1)));
|
||||||
|
if !is_error && app_action {
|
||||||
|
serde_json::from_value::<cua_driver_contract::computer_use::AppActionOutput>(
|
||||||
|
structured.unwrap().clone(),
|
||||||
|
)
|
||||||
|
.and_then(|value| value.validate().map_err(serde::de::Error::custom))
|
||||||
|
.map_err(|error| DriverError::Protocol {
|
||||||
|
reason: format!("{tool} returned an invalid Computer Use acknowledgement: {error}"),
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
let action = if !is_error && !app_action && cua_driver_core::action_record::is_action_tool(tool)
|
||||||
|
{
|
||||||
let structured = structured.ok_or_else(|| DriverError::Protocol {
|
let structured = structured.ok_or_else(|| DriverError::Protocol {
|
||||||
reason: format!("{tool} response omitted ActionResult"),
|
reason: format!("{tool} response omitted ActionResult"),
|
||||||
})?;
|
})?;
|
||||||
@@ -2084,9 +2096,11 @@ mod tests {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter_map(|tool| tool["name"].as_str())
|
.filter_map(|tool| tool["name"].as_str())
|
||||||
.collect::<std::collections::BTreeSet<_>>();
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
for published in EXPORTED_TOOL_NAMES {
|
for spec in cua_driver_contract::computer_use::tools() {
|
||||||
assert!(names.contains(published), "missing {published}");
|
assert!(names.contains(spec.name), "missing {}", spec.name);
|
||||||
}
|
}
|
||||||
|
assert!(!names.contains("start_recording"));
|
||||||
|
assert!(!names.contains("get_browser_state"));
|
||||||
|
|
||||||
driver.shutdown().await.unwrap();
|
driver.shutdown().await.unwrap();
|
||||||
driver.shutdown().await.unwrap();
|
driver.shutdown().await.unwrap();
|
||||||
@@ -2850,6 +2864,13 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn generic_app_action_acknowledgement_does_not_claim_a_legacy_action_effect() {
|
||||||
|
let result = normalize_result("click", serde_json::json!({"content":[{"type":"text","text":"Sent but not confirmed."}],"structuredContent":{"computer_use":1,"sent":true}})).unwrap();
|
||||||
|
assert!(result.action.is_none());
|
||||||
|
assert_eq!(result.text, "Sent but not confirmed.");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
async fn tool_discovery_uses_the_shared_direct_daemon_protocol() {
|
async fn tool_discovery_uses_the_shared_direct_daemon_protocol() {
|
||||||
|
|||||||
@@ -559,6 +559,7 @@ fn build_registry(options: &RuntimeOptions) -> ToolRegistry {
|
|||||||
if let Some(register_host_tools) = options.register_host_tools {
|
if let Some(register_host_tools) = options.register_host_tools {
|
||||||
register_host_tools(&mut registry);
|
register_host_tools(&mut registry);
|
||||||
}
|
}
|
||||||
|
registry.enable_computer_use();
|
||||||
let recording = Arc::downgrade(®istry.recording);
|
let recording = Arc::downgrade(®istry.recording);
|
||||||
let recording_session_end = cua_driver_core::session::register_scoped_fallible_session_end_hook(
|
let recording_session_end = cua_driver_core::session::register_scoped_fallible_session_end_hook(
|
||||||
"recording",
|
"recording",
|
||||||
@@ -596,8 +597,8 @@ fn configure_macos_runtime() {
|
|||||||
cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
|
cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
|
||||||
platform_macos::recording_hooks::app_state_json_for(window_id, pid)
|
platform_macos::recording_hooks::app_state_json_for(window_id, pid)
|
||||||
});
|
});
|
||||||
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
|
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index, binding| {
|
||||||
platform_macos::recording_hooks::element_window_local_xy(window_id, pid, index)
|
platform_macos::recording_hooks::element_window_local_xy(window_id, pid, index, binding)
|
||||||
});
|
});
|
||||||
cua_driver_core::video::set_video_backend_factory(Box::new(
|
cua_driver_core::video::set_video_backend_factory(Box::new(
|
||||||
platform_macos::video_sckit::SckitVideoBackendFactory,
|
platform_macos::video_sckit::SckitVideoBackendFactory,
|
||||||
@@ -615,8 +616,8 @@ fn configure_windows_runtime() {
|
|||||||
cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
|
cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
|
||||||
platform_windows::recording_hooks::app_state_json_for(window_id, pid)
|
platform_windows::recording_hooks::app_state_json_for(window_id, pid)
|
||||||
});
|
});
|
||||||
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
|
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index, binding| {
|
||||||
platform_windows::recording_hooks::element_window_local_xy(window_id, pid, index)
|
platform_windows::recording_hooks::element_window_local_xy(window_id, pid, index, binding)
|
||||||
});
|
});
|
||||||
cua_driver_core::video::set_video_backend_factory(Box::new(
|
cua_driver_core::video::set_video_backend_factory(Box::new(
|
||||||
cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
|
cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
|
||||||
@@ -642,14 +643,10 @@ fn configure_linux_runtime(prepare_desktop_environment: bool) {
|
|||||||
cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
|
cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
|
||||||
platform_linux::recording_hooks::app_state_json_for(window_id, pid)
|
platform_linux::recording_hooks::app_state_json_for(window_id, pid)
|
||||||
});
|
});
|
||||||
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
|
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index, binding| {
|
||||||
platform_linux::recording_hooks::element_window_local_xy(window_id, pid, index)
|
platform_linux::recording_hooks::element_window_local_xy(window_id, pid, index, binding)
|
||||||
});
|
});
|
||||||
if platform_linux::wayland::is_wayland() {
|
{
|
||||||
cua_driver_core::video::set_video_backend_factory(Box::new(
|
|
||||||
platform_linux::video_wayland::WfRecorderVideoBackendFactory,
|
|
||||||
));
|
|
||||||
} else {
|
|
||||||
cua_driver_core::video::set_video_backend_factory(Box::new(
|
cua_driver_core::video::set_video_backend_factory(Box::new(
|
||||||
cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
|
cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
|
||||||
));
|
));
|
||||||
|
|||||||
@@ -50,7 +50,6 @@ pub enum DisplayServer {
|
|||||||
Win32,
|
Win32,
|
||||||
Quartz,
|
Quartz,
|
||||||
X11,
|
X11,
|
||||||
Wayland,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DisplayServer {
|
impl DisplayServer {
|
||||||
@@ -65,14 +64,7 @@ impl DisplayServer {
|
|||||||
}
|
}
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
{
|
{
|
||||||
match std::env::var("XDG_SESSION_TYPE")
|
Self::X11
|
||||||
.unwrap_or_else(|_| "x11".to_owned())
|
|
||||||
.to_ascii_lowercase()
|
|
||||||
.as_str()
|
|
||||||
{
|
|
||||||
"wayland" => Self::Wayland,
|
|
||||||
_ => Self::X11,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
#[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))]
|
#[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))]
|
||||||
{
|
{
|
||||||
@@ -133,9 +125,7 @@ pub enum DriverRoute {
|
|||||||
LinuxAtSpiValue,
|
LinuxAtSpiValue,
|
||||||
LinuxXSendEvent,
|
LinuxXSendEvent,
|
||||||
LinuxXTest,
|
LinuxXTest,
|
||||||
LinuxLibei,
|
|
||||||
LinuxWaylandVirtualPointer,
|
|
||||||
LinuxCuaCompositorInject,
|
|
||||||
Cdp,
|
Cdp,
|
||||||
Composite,
|
Composite,
|
||||||
}
|
}
|
||||||
@@ -146,24 +136,6 @@ pub fn shared_web_route(
|
|||||||
action: &str,
|
action: &str,
|
||||||
targeting: Targeting,
|
targeting: Targeting,
|
||||||
delivery: Delivery,
|
delivery: Delivery,
|
||||||
) -> Result<DriverRoute, String> {
|
|
||||||
shared_web_route_for_environment(
|
|
||||||
platform,
|
|
||||||
display_server,
|
|
||||||
action,
|
|
||||||
targeting,
|
|
||||||
delivery,
|
|
||||||
nested_inject_from_env(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn shared_web_route_for_environment(
|
|
||||||
platform: Platform,
|
|
||||||
display_server: DisplayServer,
|
|
||||||
action: &str,
|
|
||||||
targeting: Targeting,
|
|
||||||
delivery: Delivery,
|
|
||||||
nested_inject: bool,
|
|
||||||
) -> Result<DriverRoute, String> {
|
) -> Result<DriverRoute, String> {
|
||||||
use DriverRoute as Route;
|
use DriverRoute as Route;
|
||||||
|
|
||||||
@@ -226,30 +198,19 @@ fn shared_web_route_for_environment(
|
|||||||
Ok(Route::Composite)
|
Ok(Route::Composite)
|
||||||
}
|
}
|
||||||
|
|
||||||
(Platform::Linux, DisplayServer::Wayland, Targeting::Px, _) if nested_inject =>
|
|
||||||
{
|
|
||||||
Ok(Route::LinuxCuaCompositorInject)
|
|
||||||
}
|
|
||||||
(Platform::Linux, DisplayServer::Wayland, Targeting::Px, _) => {
|
|
||||||
Ok(Route::LinuxWaylandVirtualPointer)
|
|
||||||
}
|
|
||||||
(Platform::Linux, DisplayServer::X11, Targeting::Px, _) => {
|
(Platform::Linux, DisplayServer::X11, Targeting::Px, _) => {
|
||||||
pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest)
|
pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest)
|
||||||
}
|
}
|
||||||
(Platform::Linux, DisplayServer::Wayland, Targeting::Ax, "type_text")
|
|
||||||
if nested_inject =>
|
|
||||||
{
|
|
||||||
Ok(Route::LinuxCuaCompositorInject)
|
|
||||||
}
|
|
||||||
(
|
(
|
||||||
Platform::Linux,
|
Platform::Linux,
|
||||||
DisplayServer::X11 | DisplayServer::Wayland,
|
DisplayServer::X11,
|
||||||
Targeting::Ax,
|
Targeting::Ax,
|
||||||
"left_click" | "child_window" | "scroll",
|
"left_click" | "child_window" | "scroll",
|
||||||
) => Ok(Route::LinuxAtSpiAction),
|
) => Ok(Route::LinuxAtSpiAction),
|
||||||
(
|
(
|
||||||
Platform::Linux,
|
Platform::Linux,
|
||||||
DisplayServer::X11 | DisplayServer::Wayland,
|
DisplayServer::X11,
|
||||||
Targeting::Ax,
|
Targeting::Ax,
|
||||||
"type_text",
|
"type_text",
|
||||||
) => Ok(Route::LinuxAtSpiValue),
|
) => Ok(Route::LinuxAtSpiValue),
|
||||||
@@ -259,21 +220,10 @@ fn shared_web_route_for_environment(
|
|||||||
Targeting::Ax,
|
Targeting::Ax,
|
||||||
"right_click" | "double_click" | "press_key" | "hotkey",
|
"right_click" | "double_click" | "press_key" | "hotkey",
|
||||||
) => pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest),
|
) => pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest),
|
||||||
|
|
||||||
(
|
(
|
||||||
Platform::Linux,
|
Platform::Linux,
|
||||||
DisplayServer::Wayland,
|
DisplayServer::X11,
|
||||||
Targeting::Ax,
|
|
||||||
"right_click" | "double_click" | "press_key" | "hotkey",
|
|
||||||
) if nested_inject => Ok(Route::LinuxCuaCompositorInject),
|
|
||||||
(
|
|
||||||
Platform::Linux,
|
|
||||||
DisplayServer::Wayland,
|
|
||||||
Targeting::Ax,
|
|
||||||
"right_click" | "double_click" | "press_key" | "hotkey",
|
|
||||||
) => Ok(Route::LinuxWaylandVirtualPointer),
|
|
||||||
(
|
|
||||||
Platform::Linux,
|
|
||||||
DisplayServer::X11 | DisplayServer::Wayland,
|
|
||||||
Targeting::Ax,
|
Targeting::Ax,
|
||||||
"editor_save",
|
"editor_save",
|
||||||
) => Ok(Route::Composite),
|
) => Ok(Route::Composite),
|
||||||
@@ -416,10 +366,6 @@ pub fn environment_schema_supported(schema: &str) -> bool {
|
|||||||
matches!(schema, ENVIRONMENT_SCHEMA | ENVIRONMENT_SCHEMA_V2)
|
matches!(schema, ENVIRONMENT_SCHEMA | ENVIRONMENT_SCHEMA_V2)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn nested_inject_from_env() -> bool {
|
|
||||||
std::env::var_os("CUA_INJECT_SOCKET").is_some()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn compositor_from_env() -> Option<String> {
|
fn compositor_from_env() -> Option<String> {
|
||||||
if let Ok(value) = std::env::var("CUA_E2E_COMPOSITOR") {
|
if let Ok(value) = std::env::var("CUA_E2E_COMPOSITOR") {
|
||||||
if !value.trim().is_empty() {
|
if !value.trim().is_empty() {
|
||||||
@@ -430,23 +376,6 @@ fn compositor_from_env() -> Option<String> {
|
|||||||
(Platform::Windows, DisplayServer::Win32) => Some("windows-desktop".to_owned()),
|
(Platform::Windows, DisplayServer::Win32) => Some("windows-desktop".to_owned()),
|
||||||
(Platform::Macos, DisplayServer::Quartz) => Some("windowserver".to_owned()),
|
(Platform::Macos, DisplayServer::Quartz) => Some("windowserver".to_owned()),
|
||||||
(Platform::Linux, DisplayServer::X11) => Some("openbox-x11".to_owned()),
|
(Platform::Linux, DisplayServer::X11) => Some("openbox-x11".to_owned()),
|
||||||
(Platform::Linux, DisplayServer::Wayland) if nested_inject_from_env() => {
|
|
||||||
Some("cua-compositor-nested".to_owned())
|
|
||||||
}
|
|
||||||
(Platform::Linux, DisplayServer::Wayland) => {
|
|
||||||
let desktop = std::env::var("XDG_CURRENT_DESKTOP")
|
|
||||||
.unwrap_or_default()
|
|
||||||
.to_ascii_lowercase();
|
|
||||||
if desktop.contains("sway") {
|
|
||||||
Some("sway".to_owned())
|
|
||||||
} else if desktop.contains("gnome") {
|
|
||||||
Some("gnome-mutter".to_owned())
|
|
||||||
} else if desktop.contains("kde") || desktop.contains("plasma") {
|
|
||||||
Some("kwin".to_owned())
|
|
||||||
} else {
|
|
||||||
Some("wayland-unknown".to_owned())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -473,10 +402,7 @@ fn input_backends_from_env() -> Vec<String> {
|
|||||||
"xsend-event".to_owned(),
|
"xsend-event".to_owned(),
|
||||||
"xtest".to_owned(),
|
"xtest".to_owned(),
|
||||||
],
|
],
|
||||||
(Platform::Linux, DisplayServer::Wayland) if nested_inject_from_env() => {
|
|
||||||
vec!["atspi".to_owned(), "cua-compositor-inject".to_owned()]
|
|
||||||
}
|
|
||||||
(Platform::Linux, DisplayServer::Wayland) => vec!["atspi".to_owned()],
|
|
||||||
_ => Vec::new(),
|
_ => Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -629,9 +555,7 @@ pub fn native_background_case(
|
|||||||
OracleKind::ZOrder,
|
OracleKind::ZOrder,
|
||||||
OracleKind::NoLeakedInput,
|
OracleKind::NoLeakedInput,
|
||||||
];
|
];
|
||||||
if DisplayServer::current() != DisplayServer::Wayland {
|
oracles.push(OracleKind::Cursor);
|
||||||
oracles.push(OracleKind::Cursor);
|
|
||||||
}
|
|
||||||
CaseSpec::delivered(
|
CaseSpec::delivered(
|
||||||
native_cell_id(toolkit, action, targeting, Delivery::Background),
|
native_cell_id(toolkit, action, targeting, Delivery::Background),
|
||||||
toolkit,
|
toolkit,
|
||||||
@@ -2617,7 +2541,6 @@ mod tests {
|
|||||||
(Platform::Windows, DisplayServer::Win32),
|
(Platform::Windows, DisplayServer::Win32),
|
||||||
(Platform::Macos, DisplayServer::Quartz),
|
(Platform::Macos, DisplayServer::Quartz),
|
||||||
(Platform::Linux, DisplayServer::X11),
|
(Platform::Linux, DisplayServer::X11),
|
||||||
(Platform::Linux, DisplayServer::Wayland),
|
|
||||||
] {
|
] {
|
||||||
for (action, targeting, delivery) in cells.iter().copied() {
|
for (action, targeting, delivery) in cells.iter().copied() {
|
||||||
shared_web_route(platform, display_server, action, targeting, delivery)
|
shared_web_route(platform, display_server, action, targeting, delivery)
|
||||||
@@ -2640,44 +2563,12 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn nested_wayland_pixel_route_is_distinct_from_stock_wayland() {
|
|
||||||
let stock = shared_web_route_for_environment(
|
|
||||||
Platform::Linux,
|
|
||||||
DisplayServer::Wayland,
|
|
||||||
"left_click",
|
|
||||||
Targeting::Px,
|
|
||||||
Delivery::Background,
|
|
||||||
false,
|
|
||||||
);
|
|
||||||
let nested = shared_web_route_for_environment(
|
|
||||||
Platform::Linux,
|
|
||||||
DisplayServer::Wayland,
|
|
||||||
"left_click",
|
|
||||||
Targeting::Px,
|
|
||||||
Delivery::Background,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
assert_eq!(stock, Ok(DriverRoute::LinuxWaylandVirtualPointer));
|
|
||||||
assert_eq!(nested, Ok(DriverRoute::LinuxCuaCompositorInject));
|
|
||||||
|
|
||||||
let nested_text = shared_web_route_for_environment(
|
|
||||||
Platform::Linux,
|
|
||||||
DisplayServer::Wayland,
|
|
||||||
"type_text",
|
|
||||||
Targeting::Ax,
|
|
||||||
Delivery::Background,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
assert_eq!(nested_text, Ok(DriverRoute::LinuxCuaCompositorInject));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn environment_v2_artifacts_remain_readable() {
|
fn environment_v2_artifacts_remain_readable() {
|
||||||
let record: EnvironmentRecord = serde_json::from_value(serde_json::json!({
|
let record: EnvironmentRecord = serde_json::from_value(serde_json::json!({
|
||||||
"schema": ENVIRONMENT_SCHEMA_V2,
|
"schema": ENVIRONMENT_SCHEMA_V2,
|
||||||
"platform": "linux",
|
"platform": "linux",
|
||||||
"display_server": "wayland",
|
"display_server": "x11",
|
||||||
"source_sha": null,
|
"source_sha": null,
|
||||||
"status": "ready",
|
"status": "ready",
|
||||||
"duration_ms": 1,
|
"duration_ms": 1,
|
||||||
|
|||||||
@@ -893,13 +893,11 @@ pub mod macos {
|
|||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
pub mod linux {
|
pub mod linux {
|
||||||
use std::process::{Command, Stdio};
|
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::thread::JoinHandle;
|
use std::thread::JoinHandle;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use serde::Deserialize;
|
|
||||||
use x11rb::connection::Connection;
|
use x11rb::connection::Connection;
|
||||||
use x11rb::protocol::xproto::{
|
use x11rb::protocol::xproto::{
|
||||||
AtomEnum, ConnectionExt, MapState, Rectangle, Window, WindowClass,
|
AtomEnum, ConnectionExt, MapState, Rectangle, Window, WindowClass,
|
||||||
@@ -914,9 +912,6 @@ pub mod linux {
|
|||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
enum SessionKind {
|
enum SessionKind {
|
||||||
X11,
|
X11,
|
||||||
Sway,
|
|
||||||
Gnome,
|
|
||||||
CuaCompositor,
|
|
||||||
Missing,
|
Missing,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -929,21 +924,7 @@ pub mod linux {
|
|||||||
|
|
||||||
impl LinuxObserver {
|
impl LinuxObserver {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
let explicit_wayland = std::env::var("XDG_SESSION_TYPE")
|
let session = if std::env::var_os("DISPLAY").is_some() {
|
||||||
.map(|value| value.eq_ignore_ascii_case("wayland"))
|
|
||||||
.unwrap_or(false)
|
|
||||||
|| std::env::var_os("WAYLAND_DISPLAY").is_some();
|
|
||||||
let session = if explicit_wayland {
|
|
||||||
if cua_compositor_available() {
|
|
||||||
SessionKind::CuaCompositor
|
|
||||||
} else if sway_available() {
|
|
||||||
SessionKind::Sway
|
|
||||||
} else if gnome_windows().is_ok() {
|
|
||||||
SessionKind::Gnome
|
|
||||||
} else {
|
|
||||||
SessionKind::Missing
|
|
||||||
}
|
|
||||||
} else if std::env::var_os("DISPLAY").is_some() {
|
|
||||||
if x11_window_manager_ready() {
|
if x11_window_manager_ready() {
|
||||||
SessionKind::X11
|
SessionKind::X11
|
||||||
} else {
|
} else {
|
||||||
@@ -976,14 +957,7 @@ pub mod linux {
|
|||||||
cursor: true,
|
cursor: true,
|
||||||
leaked_input: false,
|
leaked_input: false,
|
||||||
},
|
},
|
||||||
SessionKind::Sway | SessionKind::Gnome | SessionKind::CuaCompositor => {
|
|
||||||
ObserverCapabilities {
|
|
||||||
focus: true,
|
|
||||||
z_order: true,
|
|
||||||
cursor: false,
|
|
||||||
leaked_input: false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
SessionKind::Missing => ObserverCapabilities::default(),
|
SessionKind::Missing => ObserverCapabilities::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -991,9 +965,7 @@ pub mod linux {
|
|||||||
fn snapshot(&self, target: TargetWindow) -> Result<DesktopSnapshot, ObserverError> {
|
fn snapshot(&self, target: TargetWindow) -> Result<DesktopSnapshot, ObserverError> {
|
||||||
match self.session {
|
match self.session {
|
||||||
SessionKind::X11 => x11_snapshot(target),
|
SessionKind::X11 => x11_snapshot(target),
|
||||||
SessionKind::Sway => sway_snapshot(target),
|
|
||||||
SessionKind::Gnome => gnome_snapshot(target),
|
|
||||||
SessionKind::CuaCompositor => cua_compositor_snapshot(target),
|
|
||||||
SessionKind::Missing => Ok(DesktopSnapshot {
|
SessionKind::Missing => Ok(DesktopSnapshot {
|
||||||
foreground: None,
|
foreground: None,
|
||||||
input_focus: None,
|
input_focus: None,
|
||||||
@@ -1018,9 +990,7 @@ pub mod linux {
|
|||||||
self.sampler = Some(std::thread::spawn(move || {
|
self.sampler = Some(std::thread::spawn(move || {
|
||||||
let focus_identity = || match session {
|
let focus_identity = || match session {
|
||||||
SessionKind::X11 => x11_focus_identity(),
|
SessionKind::X11 => x11_focus_identity(),
|
||||||
SessionKind::Sway => sway_focus_identity(),
|
|
||||||
SessionKind::Gnome => gnome_focus_identity(),
|
|
||||||
SessionKind::CuaCompositor => cua_compositor_focus_identity(),
|
|
||||||
SessionKind::Missing => Ok(None),
|
SessionKind::Missing => Ok(None),
|
||||||
};
|
};
|
||||||
let mut previous = focus_identity().ok().flatten();
|
let mut previous = focus_identity().ok().flatten();
|
||||||
@@ -1068,366 +1038,6 @@ pub mod linux {
|
|||||||
.map_err(|error| ObserverError::new(format!("X11 connect failed: {error}")))
|
.map_err(|error| ObserverError::new(format!("X11 connect failed: {error}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
struct SwayTreeState {
|
|
||||||
focused: Option<u64>,
|
|
||||||
focused_workspace: Option<u64>,
|
|
||||||
focused_fullscreen: Option<u64>,
|
|
||||||
target: Option<(u64, bool, Option<u64>)>,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sway_tree() -> Result<serde_json::Value, ObserverError> {
|
|
||||||
let output = Command::new("swaymsg")
|
|
||||||
.args(["-r", "-t", "get_tree"])
|
|
||||||
.output()
|
|
||||||
.map_err(|error| ObserverError::new(format!("swaymsg get_tree failed: {error}")))?;
|
|
||||||
if !output.status.success() {
|
|
||||||
return Err(ObserverError::new(format!(
|
|
||||||
"swaymsg get_tree exited with {}: {}",
|
|
||||||
output.status,
|
|
||||||
String::from_utf8_lossy(&output.stderr)
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
serde_json::from_slice(&output.stdout)
|
|
||||||
.map_err(|error| ObserverError::new(format!("invalid sway tree JSON: {error}")))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sway_available() -> bool {
|
|
||||||
std::env::var_os("SWAYSOCK").is_some_and(|value| !value.is_empty()) && sway_tree().is_ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
||||||
struct CuaCompositorState {
|
|
||||||
focused_pid: Option<u64>,
|
|
||||||
target_z: TargetZ,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_cua_compositor_state(line: &str) -> Result<CuaCompositorState, ObserverError> {
|
|
||||||
let mut fields = line.split_whitespace();
|
|
||||||
if fields.next() != Some("state") {
|
|
||||||
return Err(ObserverError::new(format!(
|
|
||||||
"invalid cua-compositor observer response: {line:?}"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let focused_pid = fields
|
|
||||||
.next()
|
|
||||||
.ok_or_else(|| ObserverError::new("cua-compositor state omitted focused pid"))?
|
|
||||||
.parse::<u64>()
|
|
||||||
.map_err(|error| ObserverError::new(format!("invalid focused pid: {error}")))?;
|
|
||||||
let target_z = match fields.next() {
|
|
||||||
Some("foreground") => TargetZ::Foreground,
|
|
||||||
Some("background_occluded") => TargetZ::BackgroundOccluded,
|
|
||||||
Some("background_visible") => TargetZ::BackgroundVisible,
|
|
||||||
Some("not_found") => TargetZ::NotFound,
|
|
||||||
Some(value) => {
|
|
||||||
return Err(ObserverError::new(format!(
|
|
||||||
"invalid cua-compositor target state: {value}"
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
return Err(ObserverError::new(
|
|
||||||
"cua-compositor state omitted target state",
|
|
||||||
))
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if fields.next().is_some() {
|
|
||||||
return Err(ObserverError::new(
|
|
||||||
"cua-compositor state contained trailing fields",
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(CuaCompositorState {
|
|
||||||
focused_pid: (focused_pid != 0).then_some(focused_pid),
|
|
||||||
target_z,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn cua_compositor_state(target_pid: u32) -> Result<CuaCompositorState, ObserverError> {
|
|
||||||
use std::io::{BufRead, BufReader, Write};
|
|
||||||
use std::os::unix::net::UnixStream;
|
|
||||||
|
|
||||||
let socket = std::env::var("CUA_INJECT_SOCKET")
|
|
||||||
.map_err(|_| ObserverError::new("CUA_INJECT_SOCKET is not set"))?;
|
|
||||||
let stream = UnixStream::connect(&socket).map_err(|error| {
|
|
||||||
ObserverError::new(format!("connect cua-compositor observer socket: {error}"))
|
|
||||||
})?;
|
|
||||||
stream
|
|
||||||
.set_read_timeout(Some(Duration::from_secs(2)))
|
|
||||||
.map_err(|error| ObserverError::new(format!("set observer timeout: {error}")))?;
|
|
||||||
let mut writer = stream
|
|
||||||
.try_clone()
|
|
||||||
.map_err(|error| ObserverError::new(format!("clone observer socket: {error}")))?;
|
|
||||||
let mut reader = BufReader::new(stream);
|
|
||||||
|
|
||||||
writeln!(writer, "cua-inject v1")
|
|
||||||
.and_then(|_| writer.flush())
|
|
||||||
.map_err(|error| ObserverError::new(format!("write observer handshake: {error}")))?;
|
|
||||||
let mut line = String::new();
|
|
||||||
reader
|
|
||||||
.read_line(&mut line)
|
|
||||||
.map_err(|error| ObserverError::new(format!("read observer handshake: {error}")))?;
|
|
||||||
if line.trim() != "cua-inject v1" {
|
|
||||||
return Err(ObserverError::new(format!(
|
|
||||||
"cua-compositor observer handshake mismatch: {:?}",
|
|
||||||
line.trim()
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
writeln!(writer, "q {target_pid}")
|
|
||||||
.and_then(|_| writer.flush())
|
|
||||||
.map_err(|error| ObserverError::new(format!("write observer query: {error}")))?;
|
|
||||||
line.clear();
|
|
||||||
let read = reader
|
|
||||||
.read_line(&mut line)
|
|
||||||
.map_err(|error| ObserverError::new(format!("read observer query: {error}")))?;
|
|
||||||
if read == 0 {
|
|
||||||
return Err(ObserverError::new(
|
|
||||||
"cua-compositor closed the observer query",
|
|
||||||
));
|
|
||||||
}
|
|
||||||
parse_cua_compositor_state(&line)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn cua_compositor_available() -> bool {
|
|
||||||
std::env::var_os("CUA_INJECT_SOCKET").is_some() && cua_compositor_state(0).is_ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn cua_compositor_snapshot(target: TargetWindow) -> Result<DesktopSnapshot, ObserverError> {
|
|
||||||
let state = cua_compositor_state(target.pid)?;
|
|
||||||
Ok(DesktopSnapshot {
|
|
||||||
foreground: state.focused_pid,
|
|
||||||
input_focus: state.focused_pid,
|
|
||||||
target_z: state.target_z,
|
|
||||||
cursor_pos: None,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn cua_compositor_focus_identity() -> Result<Option<u64>, ObserverError> {
|
|
||||||
Ok(cua_compositor_state(0)?.focused_pid)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
|
|
||||||
struct GnomeWindow {
|
|
||||||
id: u64,
|
|
||||||
pid: u32,
|
|
||||||
title: String,
|
|
||||||
x: i64,
|
|
||||||
y: i64,
|
|
||||||
w: i64,
|
|
||||||
h: i64,
|
|
||||||
focused: bool,
|
|
||||||
minimized: bool,
|
|
||||||
visible: bool,
|
|
||||||
stacking: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn gnome_windows() -> Result<Vec<GnomeWindow>, ObserverError> {
|
|
||||||
let mut child = Command::new("gdbus")
|
|
||||||
.args([
|
|
||||||
"call",
|
|
||||||
"--session",
|
|
||||||
"--dest",
|
|
||||||
"org.cua.WinRects",
|
|
||||||
"--object-path",
|
|
||||||
"/org/cua/WinRects",
|
|
||||||
"--method",
|
|
||||||
"org.cua.WinRects.GetRects",
|
|
||||||
])
|
|
||||||
.stdin(Stdio::null())
|
|
||||||
.stdout(Stdio::piped())
|
|
||||||
.stderr(Stdio::piped())
|
|
||||||
.spawn()
|
|
||||||
.map_err(|error| ObserverError::new(format!("gdbus GetRects failed: {error}")))?;
|
|
||||||
let deadline = std::time::Instant::now() + Duration::from_millis(800);
|
|
||||||
loop {
|
|
||||||
match child.try_wait() {
|
|
||||||
Ok(Some(_)) => {
|
|
||||||
let output = child.wait_with_output().map_err(|error| {
|
|
||||||
ObserverError::new(format!("gdbus GetRects output failed: {error}"))
|
|
||||||
})?;
|
|
||||||
if !output.status.success() {
|
|
||||||
return Err(ObserverError::new(format!(
|
|
||||||
"gdbus GetRects exited with {}: {}",
|
|
||||||
output.status,
|
|
||||||
String::from_utf8_lossy(&output.stderr)
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
return parse_gnome_windows(&String::from_utf8_lossy(&output.stdout));
|
|
||||||
}
|
|
||||||
Ok(None) if std::time::Instant::now() < deadline => {
|
|
||||||
std::thread::sleep(Duration::from_millis(15))
|
|
||||||
}
|
|
||||||
Ok(None) => {
|
|
||||||
let _ = child.kill();
|
|
||||||
let _ = child.wait();
|
|
||||||
return Err(ObserverError::new("gdbus GetRects timed out"));
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
let _ = child.kill();
|
|
||||||
let _ = child.wait();
|
|
||||||
return Err(ObserverError::new(format!(
|
|
||||||
"gdbus GetRects wait failed: {error}"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_gnome_windows(raw: &str) -> Result<Vec<GnomeWindow>, ObserverError> {
|
|
||||||
let start = raw
|
|
||||||
.find('[')
|
|
||||||
.ok_or_else(|| ObserverError::new("GetRects response had no JSON array"))?;
|
|
||||||
let end = raw
|
|
||||||
.rfind(']')
|
|
||||||
.filter(|end| *end >= start)
|
|
||||||
.ok_or_else(|| ObserverError::new("GetRects response had no complete JSON array"))?;
|
|
||||||
serde_json::from_str(&raw[start..=end])
|
|
||||||
.map_err(|error| ObserverError::new(format!("invalid GetRects JSON: {error}")))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn gnome_target<'a>(
|
|
||||||
windows: &'a [GnomeWindow],
|
|
||||||
target: TargetWindow,
|
|
||||||
) -> Option<&'a GnomeWindow> {
|
|
||||||
let matching = windows.iter().filter(|window| window.pid == target.pid);
|
|
||||||
matching
|
|
||||||
.clone()
|
|
||||||
.find(|window| window.id == target.native_id)
|
|
||||||
.or_else(|| matching.max_by_key(|window| window.stacking))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn fully_covers(cover: &GnomeWindow, target: &GnomeWindow) -> bool {
|
|
||||||
let valid_rects = target.w > 0 && target.h > 0 && cover.w > 0 && cover.h > 0;
|
|
||||||
let covers_declared_frame = cover.x <= target.x
|
|
||||||
&& cover.y <= target.y
|
|
||||||
&& cover.x.saturating_add(cover.w) >= target.x.saturating_add(target.w)
|
|
||||||
&& cover.y.saturating_add(cover.h) >= target.y.saturating_add(target.h);
|
|
||||||
let test_sentinel_covers_visible_frame = cover.title.starts_with("CuaTestHarness Sentinel")
|
|
||||||
&& cover.x == 0
|
|
||||||
&& cover.y == 0
|
|
||||||
&& target.x < cover.w
|
|
||||||
&& target.y < cover.h
|
|
||||||
&& target.x.saturating_add(target.w) > 0
|
|
||||||
&& target.y.saturating_add(target.h) > 0;
|
|
||||||
valid_rects && (covers_declared_frame || test_sentinel_covers_visible_frame)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn classify_gnome_target(windows: &[GnomeWindow], target: TargetWindow) -> TargetZ {
|
|
||||||
let Some(target) = gnome_target(windows, target) else {
|
|
||||||
return TargetZ::NotFound;
|
|
||||||
};
|
|
||||||
if target.focused {
|
|
||||||
TargetZ::Foreground
|
|
||||||
} else if target.minimized || !target.visible {
|
|
||||||
TargetZ::Minimized
|
|
||||||
} else if windows.iter().any(|window| {
|
|
||||||
window.stacking > target.stacking
|
|
||||||
&& window.visible
|
|
||||||
&& !window.minimized
|
|
||||||
&& fully_covers(window, target)
|
|
||||||
}) {
|
|
||||||
TargetZ::BackgroundOccluded
|
|
||||||
} else {
|
|
||||||
TargetZ::BackgroundVisible
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn gnome_focus_identity_from(windows: &[GnomeWindow]) -> Option<u64> {
|
|
||||||
windows
|
|
||||||
.iter()
|
|
||||||
.find(|window| window.focused)
|
|
||||||
.map(|window| window.id)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn gnome_snapshot(target: TargetWindow) -> Result<DesktopSnapshot, ObserverError> {
|
|
||||||
let windows = gnome_windows()?;
|
|
||||||
let focused = gnome_focus_identity_from(&windows);
|
|
||||||
Ok(DesktopSnapshot {
|
|
||||||
foreground: focused,
|
|
||||||
input_focus: focused,
|
|
||||||
target_z: classify_gnome_target(&windows, target),
|
|
||||||
cursor_pos: None,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn gnome_focus_identity() -> Result<Option<u64>, ObserverError> {
|
|
||||||
Ok(gnome_focus_identity_from(&gnome_windows()?))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn walk_sway_tree(
|
|
||||||
node: &serde_json::Value,
|
|
||||||
target_pid: u32,
|
|
||||||
workspace: Option<u64>,
|
|
||||||
state: &mut SwayTreeState,
|
|
||||||
) -> bool {
|
|
||||||
let id = node["id"].as_u64();
|
|
||||||
let workspace = if node["type"].as_str() == Some("workspace") {
|
|
||||||
id
|
|
||||||
} else {
|
|
||||||
workspace
|
|
||||||
};
|
|
||||||
let focused = node["focused"].as_bool().unwrap_or(false);
|
|
||||||
if focused {
|
|
||||||
state.focused = id;
|
|
||||||
state.focused_workspace = workspace;
|
|
||||||
}
|
|
||||||
if node["pid"].as_u64() == Some(u64::from(target_pid)) {
|
|
||||||
if let Some(id) = id {
|
|
||||||
state.target = Some((id, node["visible"].as_bool().unwrap_or(true), workspace));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let mut subtree_focused = focused;
|
|
||||||
for child in ["nodes", "floating_nodes"]
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|key| node[key].as_array().into_iter().flatten())
|
|
||||||
{
|
|
||||||
subtree_focused |= walk_sway_tree(child, target_pid, workspace, state);
|
|
||||||
}
|
|
||||||
// Sway commonly puts fullscreen_mode on a container while focus belongs
|
|
||||||
// to a descendant surface. Treat the focused subtree as fullscreen so a
|
|
||||||
// sibling hidden behind that container is classified as occluded.
|
|
||||||
if subtree_focused && node["fullscreen_mode"].as_i64().unwrap_or(0) != 0 {
|
|
||||||
state.focused_fullscreen = id;
|
|
||||||
}
|
|
||||||
subtree_focused
|
|
||||||
}
|
|
||||||
|
|
||||||
fn classify_sway_target(state: &SwayTreeState) -> TargetZ {
|
|
||||||
match state.target {
|
|
||||||
None => TargetZ::NotFound,
|
|
||||||
Some((id, _, _)) if state.focused == Some(id) => TargetZ::Foreground,
|
|
||||||
Some((_, _, target_workspace))
|
|
||||||
if state.focused_fullscreen.is_some()
|
|
||||||
&& target_workspace == state.focused_workspace =>
|
|
||||||
{
|
|
||||||
// Sway reports windows hidden behind a fullscreen sibling as
|
|
||||||
// visible=false. They are occluded, not minimized.
|
|
||||||
TargetZ::BackgroundOccluded
|
|
||||||
}
|
|
||||||
Some((_, false, _)) => TargetZ::Minimized,
|
|
||||||
Some(_) => TargetZ::BackgroundVisible,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sway_snapshot(target: TargetWindow) -> Result<DesktopSnapshot, ObserverError> {
|
|
||||||
let tree = sway_tree()?;
|
|
||||||
let mut state = SwayTreeState::default();
|
|
||||||
let _ = walk_sway_tree(&tree, target.pid, None, &mut state);
|
|
||||||
let target_z = classify_sway_target(&state);
|
|
||||||
Ok(DesktopSnapshot {
|
|
||||||
foreground: state.focused,
|
|
||||||
input_focus: state.focused,
|
|
||||||
target_z,
|
|
||||||
cursor_pos: None,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sway_focus_identity() -> Result<Option<u64>, ObserverError> {
|
|
||||||
let tree = sway_tree()?;
|
|
||||||
let mut state = SwayTreeState::default();
|
|
||||||
let _ = walk_sway_tree(&tree, 0, None, &mut state);
|
|
||||||
Ok(state.focused)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn x11_snapshot(target: TargetWindow) -> Result<DesktopSnapshot, ObserverError> {
|
fn x11_snapshot(target: TargetWindow) -> Result<DesktopSnapshot, ObserverError> {
|
||||||
let (connection, screen_index) = x11_connection()?;
|
let (connection, screen_index) = x11_connection()?;
|
||||||
let root = connection.setup().roots[screen_index].root;
|
let root = connection.setup().roots[screen_index].root;
|
||||||
@@ -1685,10 +1295,6 @@ pub mod linux {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
fn gnome_payload() -> &'static str {
|
|
||||||
r#"('[{"id":10,"pid":100,"title":"target","x":10,"y":20,"w":300,"h":200,"focused":false,"minimized":false,"visible":true,"stacking":0},{"id":20,"pid":200,"title":"sentinel","x":0,"y":0,"w":800,"h":600,"focused":true,"minimized":false,"visible":true,"stacking":1}]',)"#
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn occlusion_samples_corners_and_center() {
|
fn occlusion_samples_corners_and_center() {
|
||||||
let bounds = Rectangle {
|
let bounds = Rectangle {
|
||||||
@@ -1726,159 +1332,6 @@ pub mod linux {
|
|||||||
assert!(!rectangle_fully_contains(partial, target, 2));
|
assert!(!rectangle_fully_contains(partial, target, 2));
|
||||||
assert!(rectangle_fully_contains(full, target, 2));
|
assert!(rectangle_fully_contains(full, target, 2));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn sway_hidden_target_behind_fullscreen_sibling_is_occluded() {
|
|
||||||
let state = SwayTreeState {
|
|
||||||
focused: Some(30),
|
|
||||||
focused_workspace: Some(10),
|
|
||||||
focused_fullscreen: Some(30),
|
|
||||||
target: Some((20, false, Some(10))),
|
|
||||||
};
|
|
||||||
assert_eq!(classify_sway_target(&state), TargetZ::BackgroundOccluded);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn sway_fullscreen_ancestor_of_focused_surface_occludes_target() {
|
|
||||||
let tree = serde_json::json!({
|
|
||||||
"id": 1,
|
|
||||||
"type": "root",
|
|
||||||
"nodes": [{
|
|
||||||
"id": 10,
|
|
||||||
"type": "workspace",
|
|
||||||
"nodes": [
|
|
||||||
{
|
|
||||||
"id": 20,
|
|
||||||
"pid": 100,
|
|
||||||
"visible": false,
|
|
||||||
"focused": false,
|
|
||||||
"fullscreen_mode": 0,
|
|
||||||
"nodes": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 30,
|
|
||||||
"focused": false,
|
|
||||||
"fullscreen_mode": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 31,
|
|
||||||
"pid": 200,
|
|
||||||
"visible": true,
|
|
||||||
"focused": true,
|
|
||||||
"fullscreen_mode": 0,
|
|
||||||
"nodes": []
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}]
|
|
||||||
});
|
|
||||||
let mut state = SwayTreeState::default();
|
|
||||||
assert!(walk_sway_tree(&tree, 100, None, &mut state));
|
|
||||||
assert_eq!(state.focused, Some(31));
|
|
||||||
assert_eq!(state.focused_fullscreen, Some(30));
|
|
||||||
assert_eq!(classify_sway_target(&state), TargetZ::BackgroundOccluded);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn sway_hidden_target_on_another_workspace_is_not_occluded() {
|
|
||||||
let state = SwayTreeState {
|
|
||||||
focused: Some(30),
|
|
||||||
focused_workspace: Some(10),
|
|
||||||
focused_fullscreen: Some(30),
|
|
||||||
target: Some((20, false, Some(11))),
|
|
||||||
};
|
|
||||||
assert_eq!(classify_sway_target(&state), TargetZ::Minimized);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn cua_compositor_state_is_strict_and_pid_based() {
|
|
||||||
assert_eq!(
|
|
||||||
parse_cua_compositor_state("state 200 background_occluded\n")
|
|
||||||
.expect("valid compositor state"),
|
|
||||||
CuaCompositorState {
|
|
||||||
focused_pid: Some(200),
|
|
||||||
target_z: TargetZ::BackgroundOccluded,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
parse_cua_compositor_state("state 0 not_found")
|
|
||||||
.expect("zero means no focused client"),
|
|
||||||
CuaCompositorState {
|
|
||||||
focused_pid: None,
|
|
||||||
target_z: TargetZ::NotFound,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
assert!(parse_cua_compositor_state("ok").is_err());
|
|
||||||
assert!(parse_cua_compositor_state("state 1 unknown").is_err());
|
|
||||||
assert!(parse_cua_compositor_state("state 1 foreground trailing").is_err());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn gnome_json_and_focus_identity_are_compositor_derived() {
|
|
||||||
let windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON");
|
|
||||||
assert_eq!(windows.len(), 2);
|
|
||||||
assert_eq!(gnome_focus_identity_from(&windows), Some(20));
|
|
||||||
assert!(parse_gnome_windows(r#"('[{"pid":100}]',)"#).is_err());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn gnome_only_full_higher_cover_occludes() {
|
|
||||||
let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON");
|
|
||||||
let target = TargetWindow {
|
|
||||||
pid: 100,
|
|
||||||
native_id: 10,
|
|
||||||
};
|
|
||||||
assert_eq!(
|
|
||||||
classify_gnome_target(&windows, target),
|
|
||||||
TargetZ::BackgroundOccluded
|
|
||||||
);
|
|
||||||
windows[1].w = 200;
|
|
||||||
assert_eq!(
|
|
||||||
classify_gnome_target(&windows, target),
|
|
||||||
TargetZ::BackgroundVisible
|
|
||||||
);
|
|
||||||
windows[1].w = 800;
|
|
||||||
windows[1].visible = false;
|
|
||||||
assert_eq!(
|
|
||||||
classify_gnome_target(&windows, target),
|
|
||||||
TargetZ::BackgroundVisible
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn gnome_fullscreen_test_sentinel_covers_the_visible_part_of_an_oversized_target() {
|
|
||||||
let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON");
|
|
||||||
let target = TargetWindow {
|
|
||||||
pid: 100,
|
|
||||||
native_id: 10,
|
|
||||||
};
|
|
||||||
windows[0].x = 66;
|
|
||||||
windows[0].w = 1020;
|
|
||||||
windows[1].title = "CuaTestHarness Sentinel [cdp=12345]".to_owned();
|
|
||||||
assert_eq!(
|
|
||||||
classify_gnome_target(&windows, target),
|
|
||||||
TargetZ::BackgroundOccluded
|
|
||||||
);
|
|
||||||
|
|
||||||
windows[1].title = "ordinary window".to_owned();
|
|
||||||
assert_eq!(
|
|
||||||
classify_gnome_target(&windows, target),
|
|
||||||
TargetZ::BackgroundVisible
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn gnome_focused_and_minimized_are_direct() {
|
|
||||||
let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON");
|
|
||||||
let target = TargetWindow {
|
|
||||||
pid: 100,
|
|
||||||
native_id: 10,
|
|
||||||
};
|
|
||||||
windows[0].focused = true;
|
|
||||||
assert_eq!(classify_gnome_target(&windows, target), TargetZ::Foreground);
|
|
||||||
windows[0].focused = false;
|
|
||||||
windows[0].minimized = true;
|
|
||||||
assert_eq!(classify_gnome_target(&windows, target), TargetZ::Minimized);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -106,13 +106,7 @@ impl ForegroundSentinel {
|
|||||||
reaper.track_pid(target.pid);
|
reaper.track_pid(target.pid);
|
||||||
|
|
||||||
let focus_deadline = Instant::now() + Duration::from_secs(10);
|
let focus_deadline = Instant::now() + Duration::from_secs(10);
|
||||||
if is_wayland_session() {
|
{
|
||||||
wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready");
|
|
||||||
try_activate_native_foreground(driver, target)?;
|
|
||||||
// Electron may already be focused before its preload listener is ready.
|
|
||||||
// The compositor observation is the authoritative Wayland focus gate.
|
|
||||||
wait_for_native_focus_stable(target);
|
|
||||||
} else {
|
|
||||||
wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready");
|
wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready");
|
||||||
try_activate_native_foreground(driver, target)?;
|
try_activate_native_foreground(driver, target)?;
|
||||||
wait_for_native_focus_stable(target);
|
wait_for_native_focus_stable(target);
|
||||||
@@ -161,7 +155,7 @@ impl ForegroundSentinel {
|
|||||||
{
|
{
|
||||||
violations.push("foreground sentinel heartbeat stopped".to_owned());
|
violations.push("foreground sentinel heartbeat stopped".to_owned());
|
||||||
}
|
}
|
||||||
if !is_wayland_session() {
|
{
|
||||||
if events.iter().any(|event| {
|
if events.iter().any(|event| {
|
||||||
event_kind(event) == Some("blur")
|
event_kind(event) == Some("blur")
|
||||||
|| (event_kind(event) == Some("visibility")
|
|| (event_kind(event) == Some("visibility")
|
||||||
@@ -211,22 +205,12 @@ impl ForegroundSentinel {
|
|||||||
wait_for_event(&self.journal_path, "heartbeat", Duration::from_secs(2))?;
|
wait_for_event(&self.journal_path, "heartbeat", Duration::from_secs(2))?;
|
||||||
reset_journal(&self.journal_path)?;
|
reset_journal(&self.journal_path)?;
|
||||||
|
|
||||||
let canary_key = if std::env::var("CUA_E2E_WAYLAND_SESSION")
|
|
||||||
.is_ok_and(|session| session == "cua-compositor")
|
|
||||||
{
|
|
||||||
// The intentionally small injection protocol exposes navigation
|
|
||||||
// and control keys, not printable letters. Space still exercises
|
|
||||||
// the renderer keydown leak detector without broadening it.
|
|
||||||
"space"
|
|
||||||
} else {
|
|
||||||
"a"
|
|
||||||
};
|
|
||||||
let leaked_key = driver.call(
|
let leaked_key = driver.call(
|
||||||
"press_key",
|
"press_key",
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"pid": self.target.pid,
|
"pid": self.target.pid,
|
||||||
"window_id": self.target.native_id,
|
"window_id": self.target.native_id,
|
||||||
"key": canary_key,
|
"key": "a",
|
||||||
"delivery_mode": "foreground",
|
"delivery_mode": "foreground",
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
@@ -248,8 +232,6 @@ impl ForegroundSentinel {
|
|||||||
}
|
}
|
||||||
reset_journal(&self.journal_path)?;
|
reset_journal(&self.journal_path)?;
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
set_sway_fullscreen(driver, self.target, false)?;
|
|
||||||
let raised = driver.call(
|
let raised = driver.call(
|
||||||
"bring_to_front",
|
"bring_to_front",
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
@@ -263,11 +245,7 @@ impl ForegroundSentinel {
|
|||||||
raised.text()
|
raised.text()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
#[cfg(target_os = "linux")]
|
{
|
||||||
focus_sway_target(driver, background_target)?;
|
|
||||||
if is_wayland_session() {
|
|
||||||
wait_for_native_focus_lost(self.target)?;
|
|
||||||
} else {
|
|
||||||
wait_for_event(&self.journal_path, "blur", Duration::from_secs(3))?;
|
wait_for_event(&self.journal_path, "blur", Duration::from_secs(3))?;
|
||||||
let (_, focus_violations) = self.observe();
|
let (_, focus_violations) = self.observe();
|
||||||
if !focus_violations
|
if !focus_violations
|
||||||
@@ -281,8 +259,6 @@ impl ForegroundSentinel {
|
|||||||
}
|
}
|
||||||
|
|
||||||
activate_native_foreground(driver, self.target);
|
activate_native_foreground(driver, self.target);
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
set_sway_fullscreen(driver, self.target, true)?;
|
|
||||||
wait_for_native_focus_stable(self.target);
|
wait_for_native_focus_stable(self.target);
|
||||||
std::thread::sleep(Duration::from_millis(150));
|
std::thread::sleep(Duration::from_millis(150));
|
||||||
reset_journal(&self.journal_path)?;
|
reset_journal(&self.journal_path)?;
|
||||||
@@ -377,13 +353,7 @@ impl ForegroundSentinel {
|
|||||||
before.target_z
|
before.target_z
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let mut native_oracles = vec![OracleKind::Focus, OracleKind::ZOrder];
|
let native_oracles = [OracleKind::Focus, OracleKind::ZOrder, OracleKind::Cursor];
|
||||||
if std::env::var("XDG_SESSION_TYPE")
|
|
||||||
.map(|session| !session.eq_ignore_ascii_case("wayland"))
|
|
||||||
.unwrap_or(true)
|
|
||||||
{
|
|
||||||
native_oracles.push(OracleKind::Cursor);
|
|
||||||
}
|
|
||||||
let (result, delta) = observer
|
let (result, delta) = observer
|
||||||
.observe(&native_oracles, action)
|
.observe(&native_oracles, action)
|
||||||
.map_err(|error| error.to_string())?;
|
.map_err(|error| error.to_string())?;
|
||||||
@@ -465,12 +435,6 @@ fn wait_for_event(path: &std::path::Path, kind: &str, timeout: Duration) -> Resu
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_wayland_session() -> bool {
|
|
||||||
cfg!(target_os = "linux")
|
|
||||||
&& std::env::var("XDG_SESSION_TYPE")
|
|
||||||
.is_ok_and(|session| session.eq_ignore_ascii_case("wayland"))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn activate_native_foreground(driver: &mut impl Driver, target: TargetWindow) {
|
fn activate_native_foreground(driver: &mut impl Driver, target: TargetWindow) {
|
||||||
try_activate_native_foreground(driver, target)
|
try_activate_native_foreground(driver, target)
|
||||||
.unwrap_or_else(|error| panic!("could not activate foreground sentinel: {error}"));
|
.unwrap_or_else(|error| panic!("could not activate foreground sentinel: {error}"));
|
||||||
@@ -490,10 +454,6 @@ fn try_activate_native_foreground(
|
|||||||
if response.is_error() {
|
if response.is_error() {
|
||||||
return Err(response.text().to_owned());
|
return Err(response.text().to_owned());
|
||||||
}
|
}
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
focus_sway_target(driver, target).map_err(|error| {
|
|
||||||
format!("could not focus foreground sentinel through Sway IPC: {error}")
|
|
||||||
})?;
|
|
||||||
#[cfg(target_os = "windows")]
|
#[cfg(target_os = "windows")]
|
||||||
physically_focus_windows_sentinel(target);
|
physically_focus_windows_sentinel(target);
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
@@ -529,385 +489,6 @@ fn focus_macos_sentinel_contents(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn focus_sway_target(driver: &mut impl Driver, target: TargetWindow) -> Result<(), String> {
|
|
||||||
let is_sway = is_wayland_session()
|
|
||||||
&& std::env::var("CUA_E2E_WAYLAND_SESSION").is_ok_and(|session| session == "sway");
|
|
||||||
if !is_sway {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
let (_, con_id) = sway_tree_and_container_for_target(driver, target)?;
|
|
||||||
run_sway_container_command(con_id, &["focus"], "focus canary")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn set_sway_fullscreen(
|
|
||||||
driver: &mut impl Driver,
|
|
||||||
target: TargetWindow,
|
|
||||||
enabled: bool,
|
|
||||||
) -> Result<(), String> {
|
|
||||||
let is_sway = is_wayland_session()
|
|
||||||
&& std::env::var("CUA_E2E_WAYLAND_SESSION").is_ok_and(|session| session == "sway");
|
|
||||||
if !is_sway {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
|
|
||||||
let (tree, view_id) = sway_tree_and_container_for_target(driver, target)?;
|
|
||||||
let con_id = if enabled {
|
|
||||||
view_id
|
|
||||||
} else {
|
|
||||||
sway_fullscreen_holder_id(&tree, view_id).unwrap_or(view_id)
|
|
||||||
};
|
|
||||||
let state = if enabled { "enable" } else { "disable" };
|
|
||||||
run_sway_container_command(con_id, &["fullscreen", state], "fullscreen canary")?;
|
|
||||||
if !enabled {
|
|
||||||
wait_for_sway_fullscreen_cleared(view_id)?;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn sway_tree_and_container_for_target(
|
|
||||||
driver: &mut impl Driver,
|
|
||||||
target: TargetWindow,
|
|
||||||
) -> Result<(serde_json::Value, i64), String> {
|
|
||||||
let tree = read_sway_tree()?;
|
|
||||||
|
|
||||||
let windows = driver.call("list_windows", serde_json::json!({}));
|
|
||||||
if windows.is_error() {
|
|
||||||
return Err(format!(
|
|
||||||
"list windows before Sway focus canary: {}",
|
|
||||||
windows.text()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let title = sway_target_title(windows.structured(), target);
|
|
||||||
let con_id = title
|
|
||||||
.and_then(|title| sway_container_id(&tree, title))
|
|
||||||
.or_else(|| sway_container_id_by_unique_pid(&tree, target.pid))
|
|
||||||
.ok_or_else(|| {
|
|
||||||
format!(
|
|
||||||
"Sway focus canary could not resolve container for pid {} window {}",
|
|
||||||
target.pid, target.native_id
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
Ok((tree, con_id))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn read_sway_tree() -> Result<serde_json::Value, String> {
|
|
||||||
let tree_output = Command::new("swaymsg")
|
|
||||||
.args(["-r", "-t", "get_tree"])
|
|
||||||
.output()
|
|
||||||
.map_err(|error| format!("read Sway tree for focus canary: {error}"))?;
|
|
||||||
if !tree_output.status.success() {
|
|
||||||
return Err(format!(
|
|
||||||
"read Sway tree for focus canary: {}",
|
|
||||||
String::from_utf8_lossy(&tree_output.stderr).trim()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
serde_json::from_slice(&tree_output.stdout)
|
|
||||||
.map_err(|error| format!("parse Sway tree for focus canary: {error}"))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn sway_path_to_container<'a>(
|
|
||||||
node: &'a serde_json::Value,
|
|
||||||
con_id: i64,
|
|
||||||
path: &mut Vec<&'a serde_json::Value>,
|
|
||||||
) -> bool {
|
|
||||||
path.push(node);
|
|
||||||
if node["id"].as_i64() == Some(con_id) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
for key in ["nodes", "floating_nodes"] {
|
|
||||||
if let Some(children) = node[key].as_array() {
|
|
||||||
for child in children {
|
|
||||||
if sway_path_to_container(child, con_id, path) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
path.pop();
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn sway_fullscreen_holder_id(tree: &serde_json::Value, view_id: i64) -> Option<i64> {
|
|
||||||
let mut path = Vec::new();
|
|
||||||
sway_path_to_container(tree, view_id, &mut path).then_some(())?;
|
|
||||||
path.into_iter()
|
|
||||||
.rev()
|
|
||||||
.find(|node| node["fullscreen_mode"].as_i64().unwrap_or(0) != 0)
|
|
||||||
.and_then(|node| node["id"].as_i64())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn sway_container_fullscreen_mode(tree: &serde_json::Value, con_id: i64) -> Option<i64> {
|
|
||||||
if tree["id"].as_i64() == Some(con_id) {
|
|
||||||
return Some(tree["fullscreen_mode"].as_i64().unwrap_or(0));
|
|
||||||
}
|
|
||||||
["nodes", "floating_nodes"].into_iter().find_map(|key| {
|
|
||||||
tree[key].as_array().and_then(|children| {
|
|
||||||
children
|
|
||||||
.iter()
|
|
||||||
.find_map(|child| sway_container_fullscreen_mode(child, con_id))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn wait_for_sway_fullscreen_cleared(view_id: i64) -> Result<(), String> {
|
|
||||||
let deadline = Instant::now() + Duration::from_secs(1);
|
|
||||||
loop {
|
|
||||||
let tree = read_sway_tree()?;
|
|
||||||
let fullscreen_mode = sway_container_fullscreen_mode(&tree, view_id)
|
|
||||||
.ok_or_else(|| format!("Sway fullscreen canary lost view {view_id}"))?;
|
|
||||||
if fullscreen_mode == 0 {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if Instant::now() >= deadline {
|
|
||||||
return Err(format!(
|
|
||||||
"Sway fullscreen canary did not clear fullscreen mode {fullscreen_mode} for view {view_id}"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
std::thread::sleep(Duration::from_millis(25));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn run_sway_container_command(con_id: i64, command: &[&str], purpose: &str) -> Result<(), String> {
|
|
||||||
let criterion = format!("[con_id={con_id}]");
|
|
||||||
let output = Command::new("swaymsg")
|
|
||||||
.arg("-r")
|
|
||||||
.arg(criterion.as_str())
|
|
||||||
.args(command)
|
|
||||||
.output()
|
|
||||||
.map_err(|error| format!("run Sway {purpose} for con_id {con_id}: {error}"))?;
|
|
||||||
let result: serde_json::Value = serde_json::from_slice(&output.stdout)
|
|
||||||
.map_err(|error| format!("parse Sway {purpose} result for con_id {con_id}: {error}"))?;
|
|
||||||
let succeeded = output.status.success()
|
|
||||||
&& result.as_array().is_some_and(|results| {
|
|
||||||
!results.is_empty() && results.iter().all(|item| item["success"] == true)
|
|
||||||
});
|
|
||||||
if !succeeded {
|
|
||||||
Err(format!(
|
|
||||||
"Sway {purpose} for con_id {con_id} failed: stdout={} stderr={}",
|
|
||||||
String::from_utf8_lossy(&output.stdout).trim(),
|
|
||||||
String::from_utf8_lossy(&output.stderr).trim(),
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
if std::env::var_os("CUA_E2E_DEBUG_SWAY").is_some() {
|
|
||||||
let tree = read_sway_tree()?;
|
|
||||||
let mut rows = Vec::new();
|
|
||||||
collect_sway_debug_rows(&tree, &mut rows);
|
|
||||||
eprintln!(
|
|
||||||
"[testkit] Sway {purpose}: con_id={con_id} command={command:?} tree={rows:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn collect_sway_debug_rows(node: &serde_json::Value, rows: &mut Vec<String>) {
|
|
||||||
let focused = node["focused"].as_bool().unwrap_or(false);
|
|
||||||
let fullscreen = node["fullscreen_mode"].as_i64().unwrap_or(0);
|
|
||||||
if focused || fullscreen != 0 || node["pid"].as_u64().is_some() {
|
|
||||||
rows.push(format!(
|
|
||||||
"id={} pid={} name={:?} focused={focused} fullscreen={fullscreen}",
|
|
||||||
node["id"].as_i64().unwrap_or_default(),
|
|
||||||
node["pid"].as_u64().unwrap_or_default(),
|
|
||||||
node["name"].as_str().unwrap_or("")
|
|
||||||
));
|
|
||||||
}
|
|
||||||
for key in ["nodes", "floating_nodes"] {
|
|
||||||
if let Some(children) = node[key].as_array() {
|
|
||||||
for child in children {
|
|
||||||
collect_sway_debug_rows(child, rows);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn sway_target_title(windows: &serde_json::Value, target: TargetWindow) -> Option<&str> {
|
|
||||||
let windows = windows["windows"].as_array()?;
|
|
||||||
windows
|
|
||||||
.iter()
|
|
||||||
.find(|window| {
|
|
||||||
window["pid"].as_u64() == Some(target.pid as u64)
|
|
||||||
&& window["window_id"].as_u64() == Some(target.native_id)
|
|
||||||
})
|
|
||||||
.and_then(|window| window["title"].as_str())
|
|
||||||
.or_else(|| {
|
|
||||||
let mut titles = windows
|
|
||||||
.iter()
|
|
||||||
.filter(|window| window["pid"].as_u64() == Some(target.pid as u64))
|
|
||||||
.filter_map(|window| window["title"].as_str())
|
|
||||||
.filter(|title| !title.is_empty());
|
|
||||||
let title = titles.next()?;
|
|
||||||
titles.next().is_none().then_some(title)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn collect_sway_container_ids_by_pid(node: &serde_json::Value, pid: u32, ids: &mut Vec<i64>) {
|
|
||||||
if node["pid"].as_u64() == Some(pid as u64) {
|
|
||||||
if let Some(id) = node["id"].as_i64() {
|
|
||||||
ids.push(id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for key in ["nodes", "floating_nodes"] {
|
|
||||||
if let Some(children) = node[key].as_array() {
|
|
||||||
for child in children {
|
|
||||||
collect_sway_container_ids_by_pid(child, pid, ids);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn sway_container_id_by_unique_pid(node: &serde_json::Value, pid: u32) -> Option<i64> {
|
|
||||||
let mut ids = Vec::new();
|
|
||||||
collect_sway_container_ids_by_pid(node, pid, &mut ids);
|
|
||||||
(ids.len() == 1).then(|| ids[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn sway_container_id(node: &serde_json::Value, title: &str) -> Option<i64> {
|
|
||||||
let raw_title = title
|
|
||||||
.rsplit_once(" [")
|
|
||||||
.map(|(candidate, _)| candidate)
|
|
||||||
.unwrap_or(title);
|
|
||||||
if node["name"]
|
|
||||||
.as_str()
|
|
||||||
.is_some_and(|name| name == title || name == raw_title)
|
|
||||||
{
|
|
||||||
return node["id"].as_i64();
|
|
||||||
}
|
|
||||||
["nodes", "floating_nodes"].into_iter().find_map(|key| {
|
|
||||||
node[key].as_array().and_then(|children| {
|
|
||||||
children
|
|
||||||
.iter()
|
|
||||||
.find_map(|child| sway_container_id(child, title))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(all(test, target_os = "linux"))]
|
|
||||||
mod sway_tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fullscreen_command_targets_the_holder_ancestor() {
|
|
||||||
let tree = serde_json::json!({
|
|
||||||
"id": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 30,
|
|
||||||
"fullscreen_mode": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 31,
|
|
||||||
"pid": 1234,
|
|
||||||
"name": "CuaTestHarness Sentinel",
|
|
||||||
"focused": true,
|
|
||||||
"fullscreen_mode": 0
|
|
||||||
}]
|
|
||||||
}]
|
|
||||||
});
|
|
||||||
assert_eq!(sway_fullscreen_holder_id(&tree, 31), Some(30));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fullscreen_command_prefers_the_actionable_view() {
|
|
||||||
let tree = serde_json::json!({
|
|
||||||
"id": 1,
|
|
||||||
"fullscreen_mode": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 30,
|
|
||||||
"fullscreen_mode": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 31,
|
|
||||||
"pid": 1234,
|
|
||||||
"name": "CuaTestHarness Sentinel",
|
|
||||||
"focused": true,
|
|
||||||
"fullscreen_mode": 1
|
|
||||||
}]
|
|
||||||
}]
|
|
||||||
});
|
|
||||||
assert_eq!(sway_fullscreen_holder_id(&tree, 31), Some(31));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fullscreen_clear_checks_the_view_not_inherited_workspace_state() {
|
|
||||||
let tree = serde_json::json!({
|
|
||||||
"id": 1,
|
|
||||||
"fullscreen_mode": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 30,
|
|
||||||
"fullscreen_mode": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 31,
|
|
||||||
"pid": 1234,
|
|
||||||
"name": "CuaTestHarness Sentinel",
|
|
||||||
"focused": true,
|
|
||||||
"fullscreen_mode": 0
|
|
||||||
}]
|
|
||||||
}]
|
|
||||||
});
|
|
||||||
assert_eq!(sway_container_fullscreen_mode(&tree, 31), Some(0));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn focus_target_uses_unique_sway_pid_when_title_is_unavailable() {
|
|
||||||
let tree = serde_json::json!({
|
|
||||||
"id": 1,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 9,
|
|
||||||
"nodes": [{
|
|
||||||
"id": 42,
|
|
||||||
"pid": 1234,
|
|
||||||
"name": "CuaTestHarness Electron"
|
|
||||||
}]
|
|
||||||
}]
|
|
||||||
});
|
|
||||||
assert_eq!(sway_container_id_by_unique_pid(&tree, 1234), Some(42));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn focus_target_does_not_guess_when_pid_owns_multiple_windows() {
|
|
||||||
let tree = serde_json::json!({
|
|
||||||
"id": 1,
|
|
||||||
"nodes": [
|
|
||||||
{"id": 41, "pid": 1234, "name": "First"},
|
|
||||||
{"id": 42, "pid": 1234, "name": "Second"}
|
|
||||||
]
|
|
||||||
});
|
|
||||||
assert_eq!(sway_container_id_by_unique_pid(&tree, 1234), None);
|
|
||||||
assert_eq!(sway_container_id(&tree, "Second"), Some(42));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn focus_target_uses_unique_same_process_title_when_window_id_changes() {
|
|
||||||
let windows = serde_json::json!({
|
|
||||||
"windows": [{
|
|
||||||
"pid": 1234,
|
|
||||||
"window_id": 88,
|
|
||||||
"title": "CuaTestHarness Electron [cdp=9223]"
|
|
||||||
}]
|
|
||||||
});
|
|
||||||
let target = TargetWindow {
|
|
||||||
pid: 1234,
|
|
||||||
native_id: 99,
|
|
||||||
};
|
|
||||||
assert_eq!(
|
|
||||||
sway_target_title(&windows, target),
|
|
||||||
Some("CuaTestHarness Electron [cdp=9223]")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "windows")]
|
#[cfg(target_os = "windows")]
|
||||||
fn physically_focus_windows_sentinel(target: TargetWindow) {
|
fn physically_focus_windows_sentinel(target: TargetWindow) {
|
||||||
use windows::Win32::Foundation::{HWND, POINT, RECT};
|
use windows::Win32::Foundation::{HWND, POINT, RECT};
|
||||||
@@ -988,34 +569,6 @@ fn wait_for_native_focus_stable(target: TargetWindow) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
|
||||||
fn wait_for_native_focus_lost(target: TargetWindow) -> Result<(), String> {
|
|
||||||
use crate::observer::{ObserverBackend, TargetZ};
|
|
||||||
|
|
||||||
let backend = NativeObserver::new();
|
|
||||||
let deadline = Instant::now() + Duration::from_secs(3);
|
|
||||||
loop {
|
|
||||||
let snapshot = backend.snapshot(target).ok();
|
|
||||||
let lost = snapshot
|
|
||||||
.as_ref()
|
|
||||||
.is_some_and(|snapshot| snapshot.target_z != TargetZ::Foreground);
|
|
||||||
if lost {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if Instant::now() >= deadline {
|
|
||||||
return Err(format!(
|
|
||||||
"native Wayland focus-loss canary was not detected; last snapshot: {snapshot:?}"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
std::thread::sleep(Duration::from_millis(25));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(target_os = "linux"))]
|
|
||||||
fn wait_for_native_focus_lost(_target: TargetWindow) -> Result<(), String> {
|
|
||||||
Err("native Wayland focus observation is only available on Linux".to_owned())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(any(target_os = "windows", target_os = "linux")))]
|
#[cfg(not(any(target_os = "windows", target_os = "linux")))]
|
||||||
fn wait_for_native_focus_stable(_target: TargetWindow) {}
|
fn wait_for_native_focus_stable(_target: TargetWindow) {}
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ anyhow = { workspace = true }
|
|||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
cua-driver-core = { path = "../cua-driver-core" }
|
cua-driver-core = { path = "../cua-driver-core" }
|
||||||
|
cua-driver-contract = { path = "../cua-driver-contract" }
|
||||||
cua-driver-sdk = { path = "../cua-driver-sdk" }
|
cua-driver-sdk = { path = "../cua-driver-sdk" }
|
||||||
cursor-overlay = { path = "../cursor-overlay" }
|
cursor-overlay = { path = "../cursor-overlay" }
|
||||||
pip-preview = { path = "../pip-preview" }
|
pip-preview = { path = "../pip-preview" }
|
||||||
@@ -57,12 +58,6 @@ platform-linux = { path = "../platform-linux" }
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
# GNOME/KDE portal input ships in ordinary Linux releases; PipeWire capture
|
|
||||||
# remains a separate modern-desktop feature. The combined name stays as a
|
|
||||||
# compatibility alias for Nix and downstream builds.
|
|
||||||
portal-input = ["platform-linux/portal-input"]
|
|
||||||
portal-capture = ["platform-linux/portal-capture"]
|
|
||||||
portal-libei = ["platform-linux/portal-libei"]
|
|
||||||
|
|
||||||
[target.'cfg(target_os = "windows")'.build-dependencies]
|
[target.'cfg(target_os = "windows")'.build-dependencies]
|
||||||
# Embed the Windows manifest for DPI awareness (fixes coordinate mismatches
|
# Embed the Windows manifest for DPI awareness (fixes coordinate mismatches
|
||||||
@@ -73,7 +68,6 @@ embed-resource = "2"
|
|||||||
# Shared integration-test harness (MCP/CLI transports, child reaping, paths,
|
# Shared integration-test harness (MCP/CLI transports, child reaping, paths,
|
||||||
# response accessors). Dev-only — never linked into the shipped driver.
|
# response accessors). Dev-only — never linked into the shipped driver.
|
||||||
cua-driver-testkit = { path = "../cua-driver-testkit" }
|
cua-driver-testkit = { path = "../cua-driver-testkit" }
|
||||||
cua-driver-contract = { path = "../cua-driver-contract" }
|
|
||||||
tokio = { workspace = true, features = ["full"] }
|
tokio = { workspace = true, features = ["full"] }
|
||||||
image = { workspace = true }
|
image = { workspace = true }
|
||||||
base64 = { workspace = true }
|
base64 = { workspace = true }
|
||||||
|
|||||||
@@ -184,6 +184,31 @@ fn starts_with_drive_letter(path: &str) -> bool {
|
|||||||
matches!(path.as_bytes(), [drive, b':', ..] if drive.is_ascii_alphabetic())
|
matches!(path.as_bytes(), [drive, b':', ..] if drive.is_ascii_alphabetic())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(any(target_os = "windows", test))]
|
||||||
|
const ELEVATE_PS: &str = r#"
|
||||||
|
$p = $null
|
||||||
|
try {
|
||||||
|
$p = Start-Process -FilePath 'powershell.exe' -ArgumentList @('-NoProfile','-NonInteractive','-Command',__INNER_COMMAND__) -Verb RunAs -Wait -PassThru -ErrorAction Stop
|
||||||
|
} catch {
|
||||||
|
[Console]::Error.WriteLine("qwen-cua-driver: autostart elevation failed: " + $_.Exception.Message)
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
if ($null -eq $p -or $null -eq $p.ExitCode) {
|
||||||
|
[Console]::Error.WriteLine("qwen-cua-driver: autostart elevation returned no exit code.")
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
exit $p.ExitCode
|
||||||
|
"#;
|
||||||
|
|
||||||
|
#[cfg(any(target_os = "windows", test))]
|
||||||
|
fn elevation_script(exe: &str) -> String {
|
||||||
|
let inner = format!("& '{}' autostart enable", exe.replace('\'', "''"));
|
||||||
|
ELEVATE_PS.replace(
|
||||||
|
"__INNER_COMMAND__",
|
||||||
|
&format!("'{}'", inner.replace('\'', "''")),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// ── Windows impl ──────────────────────────────────────────────────────────
|
// ── Windows impl ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
#[cfg(target_os = "windows")]
|
#[cfg(target_os = "windows")]
|
||||||
@@ -250,8 +275,7 @@ $action = New-ScheduledTaskAction `
|
|||||||
$trigger = New-ScheduledTaskTrigger -AtLogOn -User $user
|
$trigger = New-ScheduledTaskTrigger -AtLogOn -User $user
|
||||||
$principal = New-ScheduledTaskPrincipal -UserId $user -LogonType Interactive -RunLevel Highest
|
$principal = New-ScheduledTaskPrincipal -UserId $user -LogonType Interactive -RunLevel Highest
|
||||||
$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1) -ExecutionTimeLimit (New-TimeSpan -Hours 0)
|
$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1) -ExecutionTimeLimit (New-TimeSpan -Hours 0)
|
||||||
Unregister-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Confirm:$false -ErrorAction SilentlyContinue
|
Register-ScheduledTask -Force -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Description "${env:CUA_DRIVER_AS_CLI}: serve daemon, auto-start at interactive logon, RunLevel=Highest for UWP/AppContainer support" | Out-Null
|
||||||
Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Description "${env:CUA_DRIVER_AS_CLI}: serve daemon, auto-start at interactive logon, RunLevel=Highest for UWP/AppContainer support" | Out-Null
|
|
||||||
|
|
||||||
# The uiAccess worker (`qwen-cua-driver-uia.exe`) has no public client route and is
|
# The uiAccess worker (`qwen-cua-driver-uia.exe`) has no public client route and is
|
||||||
# not started by this task. The High-IL daemon is the supported path for
|
# not started by this task. The High-IL daemon is the supported path for
|
||||||
@@ -273,7 +297,7 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge
|
|||||||
.output()
|
.output()
|
||||||
.map_err(|e| anyhow!("failed to invoke powershell: {e}"))?;
|
.map_err(|e| anyhow!("failed to invoke powershell: {e}"))?;
|
||||||
if out.status.success() {
|
if out.status.success() {
|
||||||
return Ok(());
|
return confirm_registered();
|
||||||
}
|
}
|
||||||
|
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||||
@@ -311,14 +335,7 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge
|
|||||||
// happens INSIDE the elevated process (where it'll succeed on
|
// happens INSIDE the elevated process (where it'll succeed on
|
||||||
// the first attempt and not re-enter this branch). -Wait so we
|
// the first attempt and not re-enter this branch). -Wait so we
|
||||||
// can capture the child's exit code.
|
// can capture the child's exit code.
|
||||||
let inner = format!("& \"{}\" autostart enable", exe.replace('"', "`\""));
|
let outer = elevation_script(exe);
|
||||||
let outer = format!(
|
|
||||||
"$p = Start-Process -FilePath '{}' -ArgumentList @('-NoProfile','-NonInteractive','-Command',{}) -Verb RunAs -Wait -PassThru; exit $p.ExitCode",
|
|
||||||
"powershell.exe",
|
|
||||||
// Embed the inner command as a single-quoted PowerShell string
|
|
||||||
// (PS escapes ' as '' inside ''-strings).
|
|
||||||
format!("'{}'", inner.replace('\'', "''"))
|
|
||||||
);
|
|
||||||
Command::new("powershell")
|
Command::new("powershell")
|
||||||
.args(["-NoProfile", "-NonInteractive", "-Command", &outer])
|
.args(["-NoProfile", "-NonInteractive", "-Command", &outer])
|
||||||
.creation_flags(CREATE_NO_WINDOW)
|
.creation_flags(CREATE_NO_WINDOW)
|
||||||
@@ -327,7 +344,7 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge
|
|||||||
};
|
};
|
||||||
|
|
||||||
if elevated_status.success() {
|
if elevated_status.success() {
|
||||||
Ok(())
|
confirm_registered()
|
||||||
} else {
|
} else {
|
||||||
Err(anyhow!(
|
Err(anyhow!(
|
||||||
"self-elevation for autostart registration failed (exit {}). The UAC \
|
"self-elevation for autostart registration failed (exit {}). The UAC \
|
||||||
@@ -339,6 +356,19 @@ Register-ScheduledTask -TaskName $env:CUA_DRIVER_AS_TASK -Action $action -Trigge
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn confirm_registered() -> Result<()> {
|
||||||
|
match status() {
|
||||||
|
Ok(Status::NotRegistered) => Err(anyhow!(
|
||||||
|
"autostart registration reported success but task '{}' does not exist",
|
||||||
|
task_name()
|
||||||
|
)),
|
||||||
|
Ok(Status::RegisteredIdle | Status::RegisteredRunning) => Ok(()),
|
||||||
|
Err(error) => Err(anyhow!(
|
||||||
|
"autostart registration may have completed, but could not be verified: {error}"
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn disable() -> Result<()> {
|
pub fn disable() -> Result<()> {
|
||||||
// schtasks /Delete returns 0 on success, 1 on "task not found"
|
// schtasks /Delete returns 0 on success, 1 on "task not found"
|
||||||
// (which we treat as success: the goal is "no task registered"
|
// (which we treat as success: the goal is "no task registered"
|
||||||
@@ -523,6 +553,36 @@ pub fn run_autostart_cmd(subcommand: &str) {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn elevation_quotes_the_executable_as_a_literal_argument() {
|
||||||
|
let script = elevation_script(r"C:\o'brien\$data`files\qwen-cua-driver.exe");
|
||||||
|
assert!(script
|
||||||
|
.contains("'& ''C:\\o''''brien\\$data`files\\qwen-cua-driver.exe'' autostart enable'"));
|
||||||
|
assert!(!script.contains("__INNER_COMMAND__"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
#[test]
|
||||||
|
fn elevation_returns_failure_or_the_actual_child_exit_code() {
|
||||||
|
for (body, expected) in [
|
||||||
|
("Write-Error 'UAC cancelled'", 1),
|
||||||
|
("return $null", 1),
|
||||||
|
("return [pscustomobject]@{ ExitCode = $null }", 1),
|
||||||
|
("return [pscustomobject]@{ ExitCode = 0 }", 0),
|
||||||
|
("return [pscustomobject]@{ ExitCode = 7 }", 7),
|
||||||
|
] {
|
||||||
|
let script = format!(
|
||||||
|
"function Start-Process {{ [CmdletBinding()] param([string]$FilePath, [string[]]$ArgumentList, [string]$Verb, [switch]$Wait, [switch]$PassThru)\n{body}\n}}\n{}",
|
||||||
|
elevation_script(r"C:\Qwen\qwen-cua-driver.exe")
|
||||||
|
);
|
||||||
|
let status = std::process::Command::new("powershell")
|
||||||
|
.args(["-NoProfile", "-NonInteractive", "-Command", &script])
|
||||||
|
.status()
|
||||||
|
.expect("run the elevation payload with a stubbed Start-Process");
|
||||||
|
assert_eq!(status.code(), Some(expected), "{body}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn windows_task_path_strips_short_extended_drive_prefix() {
|
fn windows_task_path_strips_short_extended_drive_prefix() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -17,7 +17,7 @@
|
|||||||
//! telemetry state.
|
//! telemetry state.
|
||||||
//! - **Windows**: interactive desktop session detection (Session 0 warning),
|
//! - **Windows**: interactive desktop session detection (Session 0 warning),
|
||||||
//! UI Automation COM availability, top-level window enumeration count.
|
//! UI Automation COM availability, top-level window enumeration count.
|
||||||
//! - **Linux**: `DISPLAY` / `WAYLAND_DISPLAY` presence, X11 connection
|
//! - **Linux**: `DISPLAY` presence, X11 connection
|
||||||
//! reachability, AT-SPI bus availability hint.
|
//! reachability, AT-SPI bus availability hint.
|
||||||
//! - **macOS**: existing legacy-cleanup steps (LaunchAgent plist + update
|
//! - **macOS**: existing legacy-cleanup steps (LaunchAgent plist + update
|
||||||
//! script), plus a hint to run `qwen-cua-driver diagnose` for a full
|
//! script), plus a hint to run `qwen-cua-driver diagnose` for a full
|
||||||
@@ -431,40 +431,16 @@ fn wait_for_child(
|
|||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
fn append_platform_probes(report: &mut Report) {
|
fn append_platform_probes(report: &mut Report) {
|
||||||
// Display server probe. Order matters: Wayland wins when both are set
|
|
||||||
// (XWayland leaves DISPLAY pointing at the X server XWayland exposes,
|
|
||||||
// but the actual session is still Wayland).
|
|
||||||
let display = std::env::var("DISPLAY").ok().filter(|v| !v.is_empty());
|
let display = std::env::var("DISPLAY").ok().filter(|v| !v.is_empty());
|
||||||
let wayland = std::env::var("WAYLAND_DISPLAY")
|
match display.as_deref() {
|
||||||
.ok()
|
Some(display) => report.push(Probe::ok(
|
||||||
.filter(|v| !v.is_empty());
|
|
||||||
match (display.as_deref(), wayland.as_deref()) {
|
|
||||||
(None, None) => report.push(
|
|
||||||
Probe::warn(
|
|
||||||
"display server",
|
|
||||||
"neither DISPLAY nor WAYLAND_DISPLAY set — window-driving tools will fail",
|
|
||||||
)
|
|
||||||
.with_detail(
|
|
||||||
"run from an interactive desktop session (X11 / Wayland with XWayland) or set DISPLAY explicitly.",
|
|
||||||
),
|
|
||||||
),
|
|
||||||
(Some(d), None) => report.push(Probe::ok(
|
|
||||||
"display server",
|
"display server",
|
||||||
format!("X11 (DISPLAY={d})"),
|
format!("X11 (DISPLAY={display})"),
|
||||||
)),
|
)),
|
||||||
(None, Some(w)) => report.push(
|
None => report.push(
|
||||||
Probe::warn(
|
Probe::warn("display server", "only X11 is supported; DISPLAY is unset")
|
||||||
"display server",
|
.with_detail("Set DISPLAY and XAUTHORITY for an X server, including XWayland."),
|
||||||
format!("Wayland only (WAYLAND_DISPLAY={w}, DISPLAY unset)"),
|
|
||||||
)
|
|
||||||
.with_detail(
|
|
||||||
"X11 tools (list_windows, screenshot) need XWayland — start your session with XWayland enabled.",
|
|
||||||
),
|
|
||||||
),
|
),
|
||||||
(Some(d), Some(w)) => report.push(Probe::ok(
|
|
||||||
"display server",
|
|
||||||
format!("Wayland+XWayland (WAYLAND_DISPLAY={w}, DISPLAY={d})"),
|
|
||||||
)),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// X11 window enumeration probe. An empty result could mean either an
|
// X11 window enumeration probe. An empty result could mean either an
|
||||||
|
|||||||
@@ -109,6 +109,9 @@ fn maybe_wrap_finite_command() {
|
|||||||
let Some(command_name) = cli::finite_command_name_from_argv() else {
|
let Some(command_name) = cli::finite_command_name_from_argv() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
if !telemetry::is_enabled() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
let tool_name = cli::finite_tool_name_from_argv();
|
let tool_name = cli::finite_tool_name_from_argv();
|
||||||
let computer_action = cli::finite_computer_action_from_argv();
|
let computer_action = cli::finite_computer_action_from_argv();
|
||||||
let operation = cli::finite_operation_from_argv();
|
let operation = cli::finite_operation_from_argv();
|
||||||
@@ -817,6 +820,12 @@ fn main() {
|
|||||||
|
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
fn main() -> anyhow::Result<()> {
|
fn main() -> anyhow::Result<()> {
|
||||||
|
#[cfg(windows)]
|
||||||
|
if let Some(code) =
|
||||||
|
platform_windows::standard_user_launch::run_shell_launch_helper_if_requested()
|
||||||
|
{
|
||||||
|
std::process::exit(code);
|
||||||
|
}
|
||||||
init_logging();
|
init_logging();
|
||||||
if let Some(generation) = private_worker::requested_generation() {
|
if let Some(generation) = private_worker::requested_generation() {
|
||||||
return private_worker::run(generation, None);
|
return private_worker::run(generation, None);
|
||||||
|
|||||||
@@ -698,6 +698,19 @@ async fn handle_proxy_request(
|
|||||||
Err(e) => Response::error(id, -32602, format!("Invalid params: {e}")),
|
Err(e) => Response::error(id, -32602, format!("Invalid params: {e}")),
|
||||||
Ok(call) => {
|
Ok(call) => {
|
||||||
if let Err(error) = authorize_tool_call(&call.name, &call.args) {
|
if let Err(error) = authorize_tool_call(&call.name, &call.args) {
|
||||||
|
if cua_driver_contract::computer_use::is_app_call(&call.name, &call.args) {
|
||||||
|
return Response::ok(
|
||||||
|
id,
|
||||||
|
serde_json::to_value(
|
||||||
|
cua_driver_core::protocol::ToolResult::computer_use_error(
|
||||||
|
"permission_denied",
|
||||||
|
error.to_string(),
|
||||||
|
false,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.expect("serializable tool result"),
|
||||||
|
);
|
||||||
|
}
|
||||||
return Response::error(id, -32603, error.to_string());
|
return Response::error(id, -32603, error.to_string());
|
||||||
}
|
}
|
||||||
forward_tool_call(
|
forward_tool_call(
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ pub struct SdkAdapter {
|
|||||||
runtime_prefix: String,
|
runtime_prefix: String,
|
||||||
runtime_scope: String,
|
runtime_scope: String,
|
||||||
_session_end_hook: cua_driver_core::session::SessionEndHookRegistration,
|
_session_end_hook: cua_driver_core::session::SessionEndHookRegistration,
|
||||||
|
_session_revive_hook: cua_driver_core::session::SessionReviveHookRegistration,
|
||||||
session_lifecycle: tokio::sync::Mutex<()>,
|
session_lifecycle: tokio::sync::Mutex<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +92,14 @@ impl SdkAdapter {
|
|||||||
sessions.scopes.entry(public.to_owned()).or_default();
|
sessions.scopes.entry(public.to_owned()).or_default();
|
||||||
sessions.mark_ended(public);
|
sessions.mark_ended(public);
|
||||||
});
|
});
|
||||||
|
let revive_sessions = public_sessions.clone();
|
||||||
|
let revive_prefix = runtime_prefix.clone();
|
||||||
|
let session_revive_hook =
|
||||||
|
cua_driver_core::session::register_scoped_session_revive_hook(move |session| {
|
||||||
|
if let Some(public) = session.strip_prefix(&revive_prefix) {
|
||||||
|
revive_sessions.lock().unwrap().ended.remove(public);
|
||||||
|
}
|
||||||
|
});
|
||||||
Ok(Arc::new(Self {
|
Ok(Arc::new(Self {
|
||||||
driver,
|
driver,
|
||||||
tools_list,
|
tools_list,
|
||||||
@@ -98,6 +107,7 @@ impl SdkAdapter {
|
|||||||
runtime_prefix,
|
runtime_prefix,
|
||||||
runtime_scope,
|
runtime_scope,
|
||||||
_session_end_hook: session_end_hook,
|
_session_end_hook: session_end_hook,
|
||||||
|
_session_revive_hook: session_revive_hook,
|
||||||
session_lifecycle: tokio::sync::Mutex::new(()),
|
session_lifecycle: tokio::sync::Mutex::new(()),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
@@ -107,7 +117,7 @@ impl SdkAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_known_tool(&self, name: &str) -> bool {
|
pub fn is_known_tool(&self, name: &str) -> bool {
|
||||||
name == "type_text_chars"
|
matches!(name, "type_text_chars" | "start_session" | "end_session")
|
||||||
|| self
|
|| self
|
||||||
.tools_list
|
.tools_list
|
||||||
.get("tools")
|
.get("tools")
|
||||||
@@ -228,11 +238,17 @@ impl SdkAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_session_ended(&self, session: &str) -> bool {
|
pub fn is_session_ended(&self, session: &str) -> bool {
|
||||||
self.public_sessions
|
if !self
|
||||||
|
.public_sessions
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.ended
|
.ended
|
||||||
.contains_key(session)
|
.contains_key(session)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let internal = format!("{}{session}", self.runtime_prefix);
|
||||||
|
!cua_driver_core::session::recreates_on_next_call(&internal, &internal)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn mark_all_sessions_ended(&self) {
|
pub fn mark_all_sessions_ended(&self) {
|
||||||
@@ -553,6 +569,38 @@ mod tests {
|
|||||||
sdk.shutdown().await.expect("shutdown");
|
sdk.shutdown().await.expect("shutdown");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn idle_implicit_recreation_updates_the_daemon_mirror() {
|
||||||
|
let _runtime_guard = crate::test_runtime_lock().lock().await;
|
||||||
|
let sdk = SdkAdapter::load(host_driver()).await.expect("SDK adapter");
|
||||||
|
let transport = "g08-adapter-idle-transport";
|
||||||
|
let args = || json!({"_session_id": transport, "_transport_session_id": transport});
|
||||||
|
let first = sdk.invoke_raw("get_config", args()).await.unwrap();
|
||||||
|
assert_ne!(first["isError"], true, "{first}");
|
||||||
|
let internal = format!("{}{transport}", sdk.runtime_prefix);
|
||||||
|
assert_eq!(
|
||||||
|
cua_driver_core::session::evict_idle_with_prefix(std::time::Duration::ZERO, &internal),
|
||||||
|
[internal]
|
||||||
|
);
|
||||||
|
assert!(sdk.public_session_observation_state(transport).ended);
|
||||||
|
assert!(
|
||||||
|
!sdk.is_session_ended(transport),
|
||||||
|
"early daemon guard permits core recreation"
|
||||||
|
);
|
||||||
|
let next = sdk.invoke_raw("get_config", args()).await.unwrap();
|
||||||
|
assert_ne!(next["isError"], true, "{next}");
|
||||||
|
assert!(!sdk.public_session_observation_state(transport).ended);
|
||||||
|
let ended = sdk.invoke_raw("end_session", args()).await.unwrap();
|
||||||
|
assert_ne!(ended["isError"], true, "{ended}");
|
||||||
|
assert!(sdk.is_session_ended(transport));
|
||||||
|
let refused = sdk.invoke_raw("get_config", args()).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
refused["structuredContent"]["refusal"]["code"],
|
||||||
|
"session_ended"
|
||||||
|
);
|
||||||
|
sdk.shutdown().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn runtime_owned_end_hook_updates_only_its_adapter() {
|
async fn runtime_owned_end_hook_updates_only_its_adapter() {
|
||||||
let _runtime_guard = crate::test_runtime_lock().lock().await;
|
let _runtime_guard = crate::test_runtime_lock().lock().await;
|
||||||
|
|||||||
@@ -360,6 +360,16 @@ async fn invoke_daemon_tool(
|
|||||||
// the MCP policy contract now that every call passes through the daemon.
|
// the MCP policy contract now that every call passes through the daemon.
|
||||||
if let Err(error) = cua_driver_core::authorization::authorize_tool_call(&tool_name, &args) {
|
if let Err(error) = cua_driver_core::authorization::authorize_tool_call(&tool_name, &args) {
|
||||||
observe_daemon_error(observation, 1);
|
observe_daemon_error(observation, 1);
|
||||||
|
if cua_driver_contract::computer_use::is_app_call(&tool_name, &args) {
|
||||||
|
return DaemonResponse::ok(
|
||||||
|
serde_json::to_value(cua_driver_core::protocol::ToolResult::computer_use_error(
|
||||||
|
"permission_denied",
|
||||||
|
error.to_string(),
|
||||||
|
false,
|
||||||
|
))
|
||||||
|
.expect("serializable tool result"),
|
||||||
|
);
|
||||||
|
}
|
||||||
return DaemonResponse::err(error.to_string(), 1);
|
return DaemonResponse::err(error.to_string(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1318,8 +1328,6 @@ pub async fn run_serve(
|
|||||||
|
|
||||||
cua_driver_core::authorization::validate_startup_authorization()?;
|
cua_driver_core::authorization::validate_startup_authorization()?;
|
||||||
|
|
||||||
eprintln!("Qwen Cua Driver daemon listening on {socket_path}");
|
|
||||||
|
|
||||||
// Build the descriptor once and reuse it for every pipe instance. An
|
// Build the descriptor once and reuse it for every pipe instance. An
|
||||||
// embedded host may explicitly delegate its trusted session to one exact
|
// embedded host may explicitly delegate its trusted session to one exact
|
||||||
// sandbox account SID; ordinary daemons remain current-user-only.
|
// sandbox account SID; ordinary daemons remain current-user-only.
|
||||||
@@ -1366,28 +1374,72 @@ pub async fn run_serve(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut first_pipe = true;
|
let mut first_pipe = true;
|
||||||
|
let mut consecutive_failures = 0;
|
||||||
|
let mut fatal_error = None;
|
||||||
loop {
|
loop {
|
||||||
|
if consecutive_failures > 0 {
|
||||||
|
if consecutive_failures >= PIPE_INSTANCE_FAILURE_LIMIT {
|
||||||
|
fatal_error = Some(anyhow::anyhow!(
|
||||||
|
"named pipe {socket_path}: {consecutive_failures} consecutive pipe instance failures"
|
||||||
|
));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
_ = tokio::time::sleep(PIPE_INSTANCE_RETRY_DELAY) => {}
|
||||||
|
_ = &mut shutdown_rx => {
|
||||||
|
eprintln!("Qwen Cua Driver daemon shutting down.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_ = &mut parent_liveness => {
|
||||||
|
eprintln!("Qwen Cua Driver embedded host closed its lifetime pipe; shutting down.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// All daemons use the current-user descriptor. Embedded daemons also
|
// All daemons use the current-user descriptor. Embedded daemons also
|
||||||
// reserve the pipe name with their first instance.
|
// reserve the pipe name with their first instance.
|
||||||
let first_pipe_instance = embedded && first_pipe;
|
let first_pipe_instance = embedded && first_pipe;
|
||||||
let server = if sec_attrs_ptr.is_null() {
|
let created = if sec_attrs_ptr.is_null() {
|
||||||
ServerOptions::new()
|
ServerOptions::new()
|
||||||
.first_pipe_instance(first_pipe_instance)
|
.first_pipe_instance(first_pipe_instance)
|
||||||
.create(socket_path)
|
.create(socket_path)
|
||||||
.map_err(|e| anyhow::anyhow!("create named pipe {socket_path}: {e}"))?
|
|
||||||
} else {
|
} else {
|
||||||
unsafe {
|
unsafe {
|
||||||
ServerOptions::new()
|
ServerOptions::new()
|
||||||
.first_pipe_instance(first_pipe_instance)
|
.first_pipe_instance(first_pipe_instance)
|
||||||
.create_with_security_attributes_raw(socket_path, sec_attrs_ptr)
|
.create_with_security_attributes_raw(socket_path, sec_attrs_ptr)
|
||||||
.map_err(|e| anyhow::anyhow!("create named pipe {socket_path}: {e}"))?
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
first_pipe = false;
|
let server = match created {
|
||||||
|
Ok(server) => server,
|
||||||
|
Err(error) if first_pipe => {
|
||||||
|
fatal_error = Some(anyhow::anyhow!("create named pipe {socket_path}: {error}"));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
consecutive_failures += 1;
|
||||||
|
eprintln!(
|
||||||
|
"Qwen Cua Driver: create named pipe {socket_path} failed ({consecutive_failures} consecutive): {error}; retrying"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if first_pipe {
|
||||||
|
first_pipe = false;
|
||||||
|
eprintln!("Qwen Cua Driver daemon listening on {socket_path}");
|
||||||
|
}
|
||||||
|
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
result = server.connect() => {
|
result = server.connect() => {
|
||||||
result.map_err(|e| anyhow::anyhow!("named pipe connect: {e}"))?;
|
if let Err(error) = result {
|
||||||
|
consecutive_failures += 1;
|
||||||
|
eprintln!(
|
||||||
|
"Qwen Cua Driver: named pipe connect on {socket_path} failed ({consecutive_failures} consecutive): {error}; recreating the instance"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
consecutive_failures = 0;
|
||||||
let client_process_id =
|
let client_process_id =
|
||||||
unsafe { named_pipe_client_process_id(server.as_raw_handle().cast()) };
|
unsafe { named_pipe_client_process_id(server.as_raw_handle().cast()) };
|
||||||
let client_sid = unsafe {
|
let client_sid = unsafe {
|
||||||
@@ -1743,9 +1795,18 @@ pub async fn run_serve(
|
|||||||
sdk.shutdown()
|
sdk.shutdown()
|
||||||
.await
|
.await
|
||||||
.map_err(|error| anyhow::anyhow!("shut down SDK runtime: {error}"))?;
|
.map_err(|error| anyhow::anyhow!("shut down SDK runtime: {error}"))?;
|
||||||
Ok(())
|
match fatal_error {
|
||||||
|
Some(error) => Err(error),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
const PIPE_INSTANCE_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(500);
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
const PIPE_INSTANCE_FAILURE_LIMIT: u32 = 120;
|
||||||
|
|
||||||
#[cfg(all(test, target_os = "windows"))]
|
#[cfg(all(test, target_os = "windows"))]
|
||||||
mod named_pipe_authentication_tests {
|
mod named_pipe_authentication_tests {
|
||||||
use super::{
|
use super::{
|
||||||
|
|||||||
@@ -142,8 +142,7 @@ desktop state outside the repo-local fixtures:
|
|||||||
through the browser endpoint instead of relying on browser command-line
|
through the browser endpoint instead of relying on browser command-line
|
||||||
handoff behavior. Run it through the cross-platform
|
handoff behavior. Run it through the cross-platform
|
||||||
`scripts/ci/run-rust-standalone-browser-e2e.sh`, which stages the Electron
|
`scripts/ci/run-rust-standalone-browser-e2e.sh`, which stages the Electron
|
||||||
foreground sentinel when needed and opts pure Wayland runs into the native
|
foreground sentinel when needed. Linux requires an authenticated X11 display. Non-macOS runs prove that standard mode refuses existing-profile
|
||||||
backend. Non-macOS runs prove that standard mode refuses existing-profile
|
|
||||||
attachment before running the authorized success rows with a disposable
|
attachment before running the authorized success rows with a disposable
|
||||||
unrestricted daemon. macOS Lume maintainers can add the success rows to the
|
unrestricted daemon. macOS Lume maintainers can add the success rows to the
|
||||||
VM acceptance run with `run-all.sh --standalone-browser`.
|
VM acceptance run with `run-all.sh --standalone-browser`.
|
||||||
Loaded 100 of 303 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user