explore/test/node/keyboard.test.mjs

import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
import { tsImport } from "tsx/esm/api";

const { snapshotFromTree } = await tsImport("../../src/capture/argent-driver.ts", import.meta.url);
const { focusedTextInput, keyboardBounds, keyboardNodes, keyboardSettle } = await tsImport("../../src/capture/keyboard.ts", import.meta.url);
const { parseControllerOutput } = await tsImport("../../src/capture/controller-output.ts", import.meta.url);
const { normalizeSnapshot } = await tsImport("../../src/map/store.ts", import.meta.url);

const fixture = name => JSON.parse(readFileSync(new URL(`../fixtures/argent-0.29/${name}`, import.meta.url), "utf8"));
// euxy's rename sheet on iOS 26 under Argent 0.29, its field focused and the keyboard up: the head of brentvatne/euxy#172
// (review run 39362642343). Its root holds three windows: the app's, the input view placeholder's, and the keyboard's.
const RENAME_KEYBOARD_UP = fixture("fixture-ios-ui-tree-rename.json");
// The same sheet a moment earlier, as the base was captured: the field focused and its cursor showing, no keyboard yet.
const RENAME_KEYBOARD_DOWN = { ...RENAME_KEYBOARD_UP, keyboardVisible: false,
  roots: [{ ...RENAME_KEYBOARD_UP.roots[0], children: RENAME_KEYBOARD_UP.roots[0].children.slice(0, 1) }] };
const ANDROID_RENAME = fixture("fixture-android-ui-tree-rename.json");

const nodesOf = tree => snapshotFromTree(tree, "ios", 1).nodes;
const labelsOf = nodes => parseControllerOutput("agent-device", JSON.stringify({ success: true, data: { nodes } })).elements.map(element => element.label);
const nameOf = node => node.label ?? node.identifier;
const rect = (x, y, width, height) => ({ x, y, width, height });

test("the iOS keyboard Argent reports is its own windows: the input view placeholder's and the keyboard's, Emoji and Dictate included", () => {
  const nodes = nodesOf(RENAME_KEYBOARD_UP);
  assert.deepEqual(keyboardBounds(nodes), rect(0, 539, 402, 335), "the input view and prediction bar down to the bottom edge");
  const owned = [...keyboardNodes(nodes)];
  const names = owned.map(nameOf).filter(Boolean);
  for (const name of ["inputView", "SystemInputAssistantView", "Typing Predictions", "Q", "shift", "Emoji", "Dictate"]) assert.ok(names.includes(name), name);
  assert.ok(owned.every(node => node.systemKeyboard === true), "only nodes of the keyboard's windows");
  // Every element of the app's window stays, those behind the keyboard's band among them.
  const app = nodesOf(RENAME_KEYBOARD_DOWN).map(nameOf).filter(Boolean);
  for (const name of app) assert.ok(nodes.some(node => nameOf(node) === name && !keyboardNodes(nodes).has(node)), name);
  assert.ok(app.includes("AdditionalDimmingOverlay") && app.includes("Tab Bar") && app.includes("Clear name"));
});

test("a keyboard up on one side only leaves the elements and the accessibility fingerprint unchanged", () => {
  // In euxy#172 the review led with "Added button “Dictate”", “Emoji” and “inputView” over the label change.
  const up = nodesOf(RENAME_KEYBOARD_UP), down = nodesOf(RENAME_KEYBOARD_DOWN);
  assert.deepEqual(labelsOf(up), labelsOf(down));
  for (const name of ["Dictate", "Emoji", "inputView", "Q"]) assert.equal(labelsOf(up).includes(name), false, name);
  // The bridge numbers nodes in order, so the app's own nodes keep their indices when the keyboard's windows follow them.
  const snapshot = nodes => JSON.stringify({ success: true, data: { nodes: nodes.map(({ ref: _ref, ...node }) => node) } });
  assert.equal(normalizeSnapshot(snapshot(up)), normalizeSnapshot(snapshot(down)));
});

test("the text field with the focus is passed on from Argent", () => {
  const up = nodesOf(RENAME_KEYBOARD_UP), down = nodesOf(RENAME_KEYBOARD_DOWN);
  assert.equal(up.filter(node => node.focused === true).map(node => node.type).join(), "TextField");
  assert.equal(focusedTextInput(up), true);
  assert.equal(focusedTextInput(down), true);
  assert.equal(keyboardBounds(down), undefined);
  assert.equal(focusedTextInput(nodesOf(fixture("fixture-ios-ui-tree-home.json"))), false);
});

test("an element the app identifies as inputView in its own window is the app's", () => {
  // A chat composer with testID="inputView", no keyboard up.
  const app = RENAME_KEYBOARD_DOWN.roots[0].children[0];
  const composer = { role: "AXGroup", type: "Other", identifier: "inputView", frame: rect(0, 0.8, 1, 0.1),
    children: [{ role: "AXButton", type: "Button", label: "Send", frame: rect(0.8, 0.82, 0.15, 0.05), children: [] }] };
  const tree = { ...RENAME_KEYBOARD_DOWN, roots: [{ ...RENAME_KEYBOARD_DOWN.roots[0], children: [{ ...app, children: [...app.children, composer] }] }] };
  const nodes = nodesOf(tree);
  assert.equal(keyboardNodes(nodes).size, 0);
  assert.equal(keyboardBounds(nodes), undefined);
  assert.ok(labelsOf(nodes).includes("inputView") && labelsOf(nodes).includes("Send"));
});

test("a second window of the app's own, its testID inputView included, is the app's; the keyboard's windows need a Keyboard", () => {
  // The app's second UIWindow (a toast or a floating composer) with a view identified as inputView and a Save button.
  const window = { role: "AXGroup", type: "Application", frame: rect(0, 0, 1, 1), children: [
    { role: "AXGroup", type: "Other", identifier: "inputView", frame: rect(0, 0.8, 1, 0.1),
      children: [{ role: "AXButton", type: "Button", label: "Save", frame: rect(0.8, 0.82, 0.15, 0.05), children: [] }] }] };
  const [app, placeholder, keyboard] = RENAME_KEYBOARD_UP.roots[0].children;
  const withWindows = windows => nodesOf({ ...RENAME_KEYBOARD_UP, roots: [{ ...RENAME_KEYBOARD_UP.roots[0], children: [app, ...windows] }] });
  for (const windows of [[window], [window, keyboard], [window, placeholder, keyboard]]) {
    const labels = labelsOf(withWindows(windows));
    assert.ok(labels.includes("Save") && labels.includes("inputView"), `kept beside ${windows.length - 1} keyboard windows`);
  }
  // Without the keyboard's own window, an input view placeholder window proves nothing.
  assert.ok(labelsOf(withWindows([placeholder])).includes("inputView"));
  assert.equal(keyboardNodes(withWindows([placeholder])).size, 0);
  // With it, the placeholder is the keyboard's too, as in the real tree.
  assert.equal(labelsOf(withWindows([placeholder, keyboard])).includes("inputView"), false);
});

test("agent-device's keyboard is its Keyboard element and what is under it, and its host bridge's keyboard view classes", () => {
  // XCTest: the Keyboard directly under the Application, beside the app's own controls in the same band.
  const xctest = [
    { index: 0, type: "Application", label: "Example", rect: rect(0, 0, 402, 874) },
    { index: 1, parentIndex: 0, type: "TextField", label: "Pattern name", focused: true, rect: rect(23, 386, 355, 50) },
    { index: 2, parentIndex: 0, type: "Keyboard", rect: rect(0, 583, 402, 233) },
    { index: 3, parentIndex: 2, type: "Key", label: "Q", rect: rect(5, 590, 39, 54) },
    { index: 4, parentIndex: 0, type: "Button", label: "Home tab", rect: rect(0, 800, 134, 74) },
    { index: 5, parentIndex: 0, type: "Other", label: "Formatting toolbar", rect: rect(0, 540, 402, 43) },
    { index: 6, parentIndex: 0, type: "Button", label: "Emoji", rect: rect(8, 300, 68, 40) },
  ];
  assert.deepEqual(keyboardBounds(xctest), rect(0, 583, 402, 233));
  assert.deepEqual(labelsOf(xctest), ["Example", "Pattern name", "Home tab", "Formatting toolbar", "Emoji"]);
  // The host bridge names the dock's buttons by class, wherever they sit.
  const hostBridge = [
    { type: "TextField", role: "RCTUITextField", label: "Pattern name", focused: true, rect: rect(23, 386, 355, 50) },
    { type: "Button", role: "UIKeyboardDockItemButton", label: "Dictate", rect: rect(325, 806, 68, 68) },
    { type: "Other", role: "UIAccessibilityElementKBKey", label: "Q", rect: rect(5, 590, 39, 54) },
    { type: "Button", role: "RCTViewComponentView", label: "Profile tab", rect: rect(268, 800, 134, 74) },
  ];
  assert.ok(keyboardBounds(hostBridge));
  assert.deepEqual(labelsOf(hostBridge), ["Pattern name", "Profile tab"]);
});

test("nothing without a parent link is taken for the keyboard's, and keys alone are no keyboard", () => {
  // agent-device's flat shape: no parentIndex. The Keyboard element goes; nothing can be proven to be under it.
  const flat = [
    { type: "Keyboard", rect: rect(0, 583, 402, 233) },
    { type: "Key", label: "Q", rect: rect(5, 590, 39, 54) },
    { type: "Button", label: "Profile tab", rect: rect(268, 800, 134, 74) },
  ];
  assert.deepEqual(labelsOf(flat), ["Q", "Profile tab"]);
  // An app's own PIN pad (accessibilityRole="keyboardkey") and its footer action, with no system keyboard up.
  const pinPad = [
    { index: 0, type: "Application", label: "Example", rect: rect(0, 0, 402, 874) },
    { index: 1, parentIndex: 0, type: "SecureTextField", label: "PIN", focused: true, rect: rect(23, 200, 355, 50) },
    ...["1", "2", "3"].map((digit, offset) => ({ index: 2 + offset, parentIndex: 0, type: "Key", label: digit, rect: rect(offset * 134, 600, 134, 70) })),
    { index: 5, parentIndex: 0, type: "Button", label: "App footer action", rect: rect(0, 800, 402, 74) },
  ];
  assert.equal(keyboardNodes(pinPad).size, 0);
  assert.equal(keyboardBounds(pinPad), undefined);
  assert.deepEqual(labelsOf(pinPad), ["Example", "PIN", "1", "2", "3", "App footer action"]);
});

test("in landscape the keyboard is told apart the same way, the app's bottom bar kept", () => {
  const landscape = [
    { index: 0, type: "Application", label: "Example", rect: rect(0, 0, 874, 402) },
    { index: 1, parentIndex: 0, type: "TextField", label: "Search", focused: true, rect: rect(100, 60, 674, 40) },
    { index: 2, parentIndex: 0, type: "Keyboard", rect: rect(0, 230, 874, 172) },
    { index: 3, parentIndex: 2, type: "Key", label: "Q", rect: rect(60, 236, 70, 38) },
    { index: 4, parentIndex: 0, type: "TabBar", label: "Tab Bar", rect: rect(0, 352, 874, 50) },
  ];
  assert.deepEqual(keyboardBounds(landscape), rect(0, 230, 874, 172));
  assert.deepEqual(labelsOf(landscape), ["Example", "Search", "Tab Bar"]);
});

test("an Android input method's window is the keyboard under Argent, AOSP's included, and Argent says when one shows", () => {
  const ime = pkg => ({ role: "FrameLayout", type: "android.widget.FrameLayout", bundleId: pkg, frame: rect(0, 0.6, 1, 0.4),
    children: [{ role: "Button", type: "android.widget.Button", contentDescription: "q", frame: rect(0.01, 0.65, 0.09, 0.06), children: [] }] });
  const before = snapshotFromTree(ANDROID_RENAME, "android", 1, "dev.brent.euxy").nodes;
  assert.equal(focusedTextInput(before), true, "Argent marks the focused EditText");
  assert.equal(keyboardBounds(before), undefined);
  for (const pkg of ["com.google.android.inputmethod.latin", "com.android.inputmethod.latin"]) {
    const after = snapshotFromTree({ ...ANDROID_RENAME, keyboardVisible: true, roots: [...ANDROID_RENAME.roots, ime(pkg)] }, "android", 1, "dev.brent.euxy").nodes;
    assert.deepEqual(keyboardBounds(after), rect(0, 960, 720, 640), pkg);
    assert.deepEqual(labelsOf(after), labelsOf(before), pkg);
  }
});

test("the wait reads until the keyboard is in one place twice, at least twice, at most six times", () => {
  const field = { type: "TextField", label: "Pattern name", focused: true, rect: rect(23, 386, 355, 50) };
  const keyboardAt = y => [field, { index: 9, type: "Keyboard", rect: rect(0, y, 402, 874 - y) }];
  // Already up when the screen was read: one more read must find it in the same place.
  const up = keyboardSettle({ nodes: keyboardAt(583) });
  assert.equal(up.more(0), true, "a keyboard already up may still be rising");
  up.read({ nodes: keyboardAt(583) });
  assert.deepEqual([up.outcome, up.reads, up.more(250)], ["shown", 1, false]);
  // Caught rising: it moves between reads, then holds.
  const rising = keyboardSettle({ nodes: [field] });
  for (const y of [700, 583, 583]) rising.read({ nodes: keyboardAt(y) });
  assert.deepEqual([rising.outcome, rising.reads], ["shown", 3]);
  // A slow controller: the second read is owed even after a second has passed.
  const slow = keyboardSettle({ nodes: [field] });
  slow.read({ nodes: [field] });
  assert.equal(slow.more(2_500), true);
  slow.read({ nodes: [field] });
  assert.deepEqual([slow.outcome, slow.more(5_000)], ["absent", false], "no keyboard came: captured without one");
  // A slow first read without a keyboard, then the keyboard first seen still moving: read on until it holds.
  const late = keyboardSettle({ nodes: [field] });
  late.read({ nodes: [field] });
  late.read({ nodes: keyboardAt(700) });
  assert.deepEqual([late.outcome, late.more(2_500)], ["moving", true], "a moving keyboard is not given up on after a second");
  late.read({ nodes: keyboardAt(583) });
  late.read({ nodes: keyboardAt(583) });
  assert.deepEqual([late.outcome, late.reads], ["shown", 4]);
  // But not past four seconds, however few reads that was.
  const stalled = keyboardSettle({ nodes: [field] });
  stalled.read({ nodes: [field] });
  stalled.read({ nodes: keyboardAt(700) });
  assert.equal(stalled.more(4_000), false);
  // Six reads at most, however fast they come.
  const busy = keyboardSettle({ nodes: [field] });
  for (let read = 0; busy.more(0); read++) busy.read({ nodes: keyboardAt(read % 2 ? 583 : 700) });
  assert.deepEqual([busy.outcome, busy.reads], ["moving", 6]);
  // The field let go and no keyboard shows: nothing more to wait for.
  const released = keyboardSettle({ nodes: [field] });
  released.read({ nodes: [{ ...field, focused: false }] });
  assert.deepEqual([released.outcome, released.more(0)], ["unfocused", false]);
  // Argent on Android says a keyboard shows without placing it: it shows twice running.
  const android = keyboardSettle({ nodes: [field] });
  android.read({ nodes: [field], keyboardVisible: true });
  android.read({ nodes: [field], keyboardVisible: true });
  assert.equal(android.outcome, "shown");
  assert.equal(keyboardSettle({ nodes: [{ ...field, focused: false }] }), undefined, "a screen without a focused field is not waited on");
});