Preserve reviewed Computer Use element identity through native clicks

Vendor CU 4ffebcc67073529a57070ac00d5ac2cbdfaddce5. Exact observed element presses retain app, window, path, role and label and refuse a stale helper or identity without coordinate fallback. Explicit event input retains the ownership guard. Dispatch remains distinct from verified UI effect.

Validation: source CU macOS backend suite 33 passed, 0 failed, including all six native tests. Bundled direct and wire target suite 19 passed, 0 failed. All seven vendored files match the reviewed CU source. Full Core compile and live UI replay remain pending.

crates/tui/plugins/computer-use/mcp/server.mjs

@@ -103,14 +103,12 @@ async function normalizeTarget(computer, target, kind, resolve, sink) {
         throw new ServerError("element_stale", `element ${target.index} of ${target.state_id} no longer resolves (${res?.reason ?? "not_found"}) — call get_app_state again`);
       }
       fresh = res.element;
-      if (fresh.role && element.role && fresh.role !== element.role) {
+      if (fresh.role !== element.role) {
         throw new ServerError("element_stale", `element ${target.index} of ${target.state_id} changed role (${element.role} → ${fresh.role}) — call get_app_state again`);
       }
       // In-place replacement: same role and geometry but a different label is
       // still a different element (e.g. "Load" → "Confirm").
-      if (typeof fresh.label === "string" && fresh.label.length > 0 &&
-          typeof element.label === "string" && element.label.length > 0 &&
-          fresh.label !== element.label) {
+      if (fresh.label !== element.label) {
         throw new ServerError("element_stale", `element ${target.index} of ${target.state_id} changed label (${element.label} → ${fresh.label}) — call get_app_state again`);
       }
     }
@@ -127,9 +125,11 @@ async function normalizeTarget(computer, target, kind, resolve, sink) {
     const sz = fresh?.size ?? element.size;
     if (!pos || !sz) throw new ServerError("element_no_geometry", `element ${target.index} has no cached geometry — use a coordinate target`);
     if (moved && sink) sink.reacquired = true;
-    // Pointer tools on element targets: aim at the (fresh) element center.
+    // Keep the element identity as well as geometry: semantic clicks must not
+    // substitute whichever element happens to occupy an oversized AX center.
     const c = { x: Math.round(pos.x + sz.w / 2), y: Math.round(pos.y + sz.h / 2) };
-    return { ...c, strategy: "a11y-center", role: element.role, label: element.label, app_ref: state.app_ref, reacquired: moved };
+    return { ...c, strategy: "a11y-center", role: element.role, label: element.label, app_ref: state.app_ref,
+      windowIndex: element.windowIndex ?? 0, path: element.path, reacquired: moved };
   }
   throw new ServerError("bad_target", "target must be {type:'coordinate',x,y} or {type:'element',state_id,index}");
 }

crates/tui/plugins/computer-use/src/backends/darwin-accessibility.m

@@ -139,6 +139,13 @@ static NSDictionary *info(AXUIElementRef el, NSInteger index, NSInteger win, NSA
   else d[@"actions"]=@[];
   return d;
 }
+static void cuValidateElementIdentity(AXUIElementRef el, NSDictionary *target) {
+  NSDictionary *current=info(el,0,0,@[]);
+  for(NSString *key in @[@"role",@"label"]) {
+    id expected=target[key]?:NSNull.null, actual=current[key]?:NSNull.null;
+    if(![expected isEqual:actual]) @throw [NSException exceptionWithName:@"stale" reason:[NSString stringWithFormat:@"element changed %@; observe again before acting",key] userInfo:nil];
+  }
+}
 static void walk(AXUIElementRef el, NSInteger win, NSArray *path, NSInteger depth, NSInteger limit, NSInteger max, BOOL depthIsTruncation, NSMutableArray *out, BOOL *truncated) {
   // Breadth first keeps a long file listing from hiding its dialog buttons.
   NSMutableArray *queue=[NSMutableArray arrayWithObject:@{@"el":(__bridge id)el,@"path":path,@"depth":@(depth)}];
@@ -367,10 +374,14 @@ static id execute(NSDictionary *p) {
     return cuPostKey(args,[args[@"input_app_ref"][@"pid"] intValue]);
   }
   cuCheckCancelled();
-  if([tool isEqual:@"input_capabilities"]) return @{@"input_lease":@1,@"owner_pipe":@YES,@"record_owner_pipe":@1,@"window_ocr":@1};
+  if([tool isEqual:@"input_capabilities"]) return @{@"input_lease":@1,@"owner_pipe":@YES,@"record_owner_pipe":@1,@"window_ocr":@1,@"element_identity":@1};
   if([tool isEqual:@"record"]) return cuRecord(args);
   if([tool isEqual:@"recognize_text"]) return cuRecognizeText(args[@"file"]);
 #ifdef CU_TEST
+  if([tool isEqual:@"inspect_element_identity"]) {
+    cuValidateElementIdentity((__bridge AXUIElementRef)args[@"element"],args[@"target"]);
+    return @{@"identity_matches":@YES};
+  }
   if([tool isEqual:@"inspect_window_match"]) {
     NSDictionary *b=args[@"bounds"];
     CGRect bounds=CGRectMake([b[@"x"] doubleValue],[b[@"y"] doubleValue],[b[@"w"] doubleValue],[b[@"h"] doubleValue]);
@@ -699,6 +710,7 @@ static id execute(NSDictionary *p) {
       }
     }
     cuCheckCancelled();
+    if([t[@"type"] isEqual:@"element"]) cuValidateElementIdentity((__bridge AXUIElementRef)el,t);
     AXError e=kAXErrorFailure;
     if([tool isEqual:@"set_value"]) e=AXUIElementSetAttributeValue((__bridge AXUIElementRef)el,kAXValueAttribute,(__bridge CFTypeRef)args[@"value"]);
     else if([tool isEqual:@"select_text"]){ NSArray *r=args[@"text_range"]?:@[@0,@0]; if(r.count!=2 || [r[0] longValue]<0 || [r[1] longValue]<0) @throw [NSException exceptionWithName:@"range" reason:@"text_range must be [start, length], both nonnegative" userInfo:nil]; CFRange range=CFRangeMake([r[0] longValue],[r[1] longValue]); AXValueRef v=AXValueCreate(kAXValueCFRangeType,&range); e=AXUIElementSetAttributeValue((__bridge AXUIElementRef)el,kAXSelectedTextRangeAttribute,v); CFRelease(v); }

crates/tui/plugins/computer-use/src/backends/darwin.mjs

@@ -74,7 +74,7 @@ export function create({ exec }) {
       if (r.aborted) error.code = "cancelled";
       // A deterministic native refusal sent no input. A killed/timed-out
       // helper may have posted the press before losing its response.
-      const postsPress = (tool === "key_event" && args.down) || (tool === "pointer_sequence" && args.steps?.some((step) => [1, 3, 25].includes(step.type)));
+      const postsPress = (tool === "key_event" && args.down) || (tool === "perform_action" && args.action === "AXPress") || (tool === "pointer_sequence" && args.steps?.some((step) => [1, 3, 25].includes(step.type)));
       error.inputMayHaveBeenSent = postsPress && r.spawned === true && (r.aborted || r.timedOut);
       throw error;
     }
@@ -528,7 +528,24 @@ export function create({ exec }) {
     },
     screenshot,
     zoom,
-    left_click: ({ target, strategy }) => pointerClick("left", target.x, target.y, 1, strategy ?? "auto"),
+    left_click: async ({ target, strategy = "auto" }) => {
+      if (target.type !== "element" || strategy === "event") return pointerClick("left", target.x, target.y, 1, strategy);
+      if (!["auto", "a11y"].includes(strategy)) throw new ExecError(`strategy must be auto, a11y or event (got ${JSON.stringify(strategy)})`);
+      try {
+        if (!state.inputApp || target.app_ref?.pid !== state.inputApp.pid) throw new ExecError("element does not belong to the bound application — open_application and observe again");
+        if (!Array.isArray(target.path) || !Number.isInteger(target.windowIndex) || !target.role) throw new ExecError("element has no resolved accessibility identity");
+        if ((await native("input_capabilities"))?.element_identity !== 1) throw new ExecError("native helper needs an update for element identity validation");
+        const receipt = await native("perform_action", { target, action: "AXPress" });
+        if (!receipt?.action_sent) throw new ExecError("element press was not acknowledged");
+        return { ...receipt, action: "AXPress", strategy: "a11y", pointer_moved: false,
+          element: { role: target.role, label: target.label ?? null }, verified: false, verification_required: "screenshot" };
+      } catch (error) {
+        // An AX frame can cover other controls. Never turn a refused or
+        // ambiguous element press into another element's press or a raw click.
+        error.message += ' — no coordinate fallback was sent; take a fresh screenshot or OCR observation before choosing a coordinate with strategy "event"';
+        throw error;
+      }
+    },
     double_click: ({ target }) => pointerClick("left", target.x, target.y, 2),
     triple_click: ({ target }) => pointerClick("left", target.x, target.y, 3),
     right_click: ({ target }) => pointerClick("right", target.x, target.y, 1),

crates/tui/plugins/computer-use/src/tools.mjs

@@ -8,7 +8,7 @@ const computerParam = {
 
 const strategyParam = {
   enum: ["auto", "a11y", "event"],
-  description: "How the point is dispatched. auto (default): macOS resolves the point against the bound application's accessibility tree first and performs the element's press action, falling back to a raw pointer event; other platforms always use raw events. a11y: require an accessibility press and fail closed otherwise. event: force the raw pointer event.",
+  description: "macOS auto (default): element targets press that exact revalidated element and fail closed, with no coordinate fallback; coordinate targets hit-test the point for an accessibility press, falling back to a guarded raw event. a11y: require an accessibility press and fail closed otherwise. event: force the guarded raw pointer event. Other platforms use raw events. action_sent confirms dispatch, not the effect; observe again before deciding another action.",
 };
 
 const targetSchema = {

crates/tui/plugins/computer-use/tests/darwin.test.mjs

@@ -63,6 +63,14 @@ test('native summary keeps text and top-level menus without spending the UI budg
   assert.equal(full.elements.find(e=>e.label==='Command 149').windowIndex,-1);
   assert.deepEqual(full.elements.find(e=>e.label==='Command 149').path,[0,0,149]);
   assert.ok(full.elements.some(e=>e.label==='Field 349'));
+  const identity=(element,target)=>spawnSync(binary,[JSON.stringify({tool:'inspect_element_identity',args:{element,target}})],{encoding:'utf8'});
+  const target={role:'AXMenuItem',label:'Files'};
+  assert.equal(identity(node('AXMenuItem','Files'),target).status,0);
+  for(const element of [node('AXMenuItem','Delete'),node('AXButton','Files'),node('AXMenuItem','')]) {
+    const refused=identity(element,target);
+    assert.equal(refused.status,1);assert.match(refused.stderr,/element changed (role|label)/);
+  }
+  assert.equal(identity(node('AXMenuItem','Files'),{role:'AXMenuItem'}).status,1,'an unlabeled element replaced by a labeled one is stale too');
 });
 
 test('native Unicode encoding round-trips through the actual CoreGraphics event', {skip:process.platform!=='darwin'}, t=>{
@@ -192,6 +200,64 @@ function stubBackend(t, reply) {
 
 const PRESSABLE = { found: true, element: { role: 'AXButton', label: 'Tab B', actions: ['AXPress'] }, action: 'AXPress', action_sent: true };
 const NOT_PRESSABLE = { found: false, reason: 'no_pressable_element' };
+const FILES_TARGET = { type:'element', app_ref:{pid:321,bundle_id:'test.app'}, windowIndex:0, path:[0,4,2],
+  role:'AXMenuItem', label:'Files', x:1607, y:692 };
+
+test('macOS element click preserves the observed path despite an oversized frame center', async t => {
+  const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,element_identity:1}:r.tool==='hit_test'?PRESSABLE:null);
+  await backend.open_application({name:'Fixture'});
+  const receipt=await backend.left_click({target:FILES_TARGET});
+  assert.equal(receipt.action_sent,true);
+  assert.equal(receipt.element.label,'Files');
+  assert.equal(receipt.verified,false);
+  assert.equal(receipt.verification_required,'screenshot');
+  const presses=calls.filter(r=>r.tool==='perform_action');
+  assert.equal(presses.length,1,'one dispatch, even when the app does not report a changed state');
+  assert.deepEqual(presses[0].args.target,FILES_TARGET);
+  assert.equal(presses[0].args.action,'AXPress');
+  assert.ok(!calls.some(r=>['hit_test','pointer_sequence','window_at_point'].includes(r.tool)),'the center never selects another control');
+});
+
+for(const reason of ['action is not advertised by this element','element changed label; observe again','window blocked by modal sheet'])
+test(`macOS element click does not fall back after ${reason}`, async t => {
+  const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,element_identity:1}
+    :r.tool==='perform_action'?{nativeResult:{code:1,stdout:'',stderr:reason}}:null);
+  await backend.open_application({name:'Fixture'});
+  await assert.rejects(backend.left_click({target:FILES_TARGET,strategy:'a11y'}),error=>error.message.includes(reason)&&/fresh screenshot or OCR/.test(error.message));
+  assert.equal(calls.filter(r=>r.tool==='perform_action').length,1);
+  assert.ok(!calls.some(r=>['hit_test','pointer_sequence'].includes(r.tool)));
+});
+
+test('macOS ambiguous element press is never retried or converted to pointer input', async t => {
+  const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,element_identity:1}
+    :r.tool==='perform_action'?{nativeResult:{code:null,spawned:true,timedOut:true,stdout:'',stderr:''}}:null);
+  await backend.open_application({name:'Fixture'});
+  await assert.rejects(backend.left_click({target:FILES_TARGET}),error=>error.inputMayHaveBeenSent===true&&/timed out/.test(error.message));
+  assert.equal(calls.filter(r=>r.tool==='perform_action').length,1);
+  assert.ok(!calls.some(r=>['hit_test','pointer_sequence'].includes(r.tool)));
+});
+
+test('macOS element clicks refuse missing identity, another bound app and an old helper before dispatch', async t => {
+  const {backend,calls}=stubBackend(t,()=>null);
+  await backend.open_application({name:'Fixture'});
+  await assert.rejects(backend.left_click({target:{...FILES_TARGET,path:undefined}}),/no resolved accessibility identity/);
+  await assert.rejects(backend.left_click({target:{...FILES_TARGET,app_ref:{pid:999}}}),/bound application/);
+  await assert.rejects(backend.left_click({target:FILES_TARGET}),/helper needs an update/);
+  assert.ok(!calls.some(r=>['perform_action','hit_test','pointer_sequence'].includes(r.tool)));
+});
+
+test('macOS explicit event selection remains usable and retains the app ownership guard', async t => {
+  let covered=false;
+  const {backend,calls}=stubBackend(t,r=>r.tool==='window_at_point'&&covered?{found:true,owner_pid:999,owner_name:'Mail'}:null);
+  await backend.open_application({name:'Fixture'});
+  assert.equal((await backend.left_click({target:FILES_TARGET,strategy:'event'})).strategy,'event');
+  const seq=calls.find(r=>r.tool==='pointer_sequence');
+  assert.deepEqual([seq.args.steps[1].x,seq.args.steps[1].y],[1607,692]);
+  covered=true;
+  await assert.rejects(backend.left_click({target:FILES_TARGET,strategy:'event'}),/covered by a window owned by Mail/);
+  assert.equal(calls.filter(r=>r.tool==='pointer_sequence').length,1);
+  assert.ok(!calls.some(r=>['perform_action','hit_test'].includes(r.tool)));
+});
 
 test('macOS refuses an old native helper before any held input is dispatched', async t => {
   const {backend,calls}=stubBackend(t,request=>request.tool==='input_capabilities'?{input_lease:0}:null);

crates/tui/plugins/computer-use/tests/server-targets.test.mjs

@@ -215,6 +215,9 @@ test("element targets are revalidated; moved geometry re-aims and marks the rece
     assert.equal(r.target_reacquired, true);
     const last = calls("left_click").at(-1);
     assert.deepEqual({ x: last.args.target.x, y: last.args.target.y }, { x: 130, y: 215 }); // fresh center
+    assert.deepEqual(last.args.target.path, [0, 1], "pointer dispatch retains the original element path");
+    assert.equal(last.args.target.windowIndex, 0);
+    assert.equal(last.args.target.label, "OK");
   } finally {
     setControl(null);
   }
@@ -260,6 +263,20 @@ test("in-place replacement (same geometry, different label) fails element_stale"
   }
 });
 
+test("an element losing its label or role is stale even if the geometry matches", async () => {
+  const st = await freshState();
+  const before = calls("left_click").length;
+  try {
+    for (const identity of [{ role: "AXButton", label: "" }, { role: "AXButton" }, { label: "OK" }]) {
+      setControl({ found: true, element: { ...identity, position: { x: 10, y: 20 }, size: { w: 60, h: 30 } } });
+      const r = await tool("left_click", { target: { type: "element", state_id: st.state_id, index: 1 } });
+      assert.equal(r.ok, false);
+      assert.equal(r.error.code, "element_stale");
+    }
+    assert.equal(calls("left_click").length, before);
+  } finally { setControl(null); }
+});
+
 test("a state_id issued on another computer fails state_wrong_computer", async () => {
   const st = await freshState(); // bound to "local"
   const reg = await tool("computer_register", { computer: "pad", transport: "hdc" });

crates/tui/plugins/computer-use/tests/server-wire-targets.test.mjs

@@ -109,7 +109,7 @@ test("out-of-process coordinate actions keep the raster refusals", async () => {
     "refusals never reach the backend; the valid click does");
 });
 
-test("element targets travel as a center for pointer tools and an AX path for semantic ones", async () => {
+test("element targets retain their identity and AX path on the out-of-process pointer route", async () => {
   const state = await tool("get_app_state", {});
   assert.equal(state.ok, true);
   assert.equal(state.detail, "summary");
@@ -122,6 +122,10 @@ test("element targets travel as a center for pointer tools and an AX path for se
   assert.equal(click.ok, true, JSON.stringify(click));
   const clicked = calls().filter((c) => c.method === "left_click").at(-1);
   assert.deepEqual({ x: clicked.args.target.x, y: clicked.args.target.y }, { x: 40, y: 35 }, "element center in points");
+  assert.deepEqual(clicked.args.target.path, [0, 1]);
+  assert.equal(clicked.args.target.windowIndex, 0);
+  assert.equal(clicked.args.target.role, "AXButton");
+  assert.equal(clicked.args.target.label, "OK");
 
   const pressed = await tool("perform_action", { target, action: "AXPress" });
   assert.equal(pressed.ok, true, JSON.stringify(pressed));