Compare commits

...
Author SHA1 Message Date
yhh 50f10f9419 chore(tools): tighten the computer tool descriptions to fit the schema budget
The window parameter on computer_read and computer_act put the built-in schemas 14
tokens over their budget. Shorten wording in the two descriptions instead of
widening the budget, and regenerate docs/TOOL_CONTRACT.md.
2026-10-11 16:49:56 -07:00
yhh 68c1f9d551 Merge studio 2026-10-11 16:47:10 -07:00
YHH a9dde35b70 Merge branch 'studio' into feat/computer-ambiguous-window-target 2026-10-11 15:27:45 -07:00
yhh 85a1cbdd8a feat(computer): refuse an ambiguous window target and let a call name its window
An application showing two or more independent top-level windows left the
front one to be guessed: a screenshot, typing, keys and coordinate steps went
to whichever was in front. computer_read and computer_act now take an optional
window id from the application list; without one, such a call is refused
before anything is sent with computer.ambiguous_window_target and the
candidates. A dialog another window owns is not a second window.
2026-10-10 08:35:45 -07:00
17 changed files with 546 additions and 115 deletions

No files matched your search

+1
View File
@@ -27,6 +27,7 @@ Json listApps() {
windows.push(Json::object()
.set("id", static_cast<long long>(reinterpret_cast<uintptr_t>(hwnd)))
.set("title", narrow(std::wstring(title, n > 0 ? n : 0)))
.set("owned", GetWindow(hwnd, GW_OWNER) != nullptr && !(GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_APPWINDOW))
.set("bounds", rectJson(windowBounds(hwnd))));
}
apps.push(Json::object()
+3
View File
@@ -48,6 +48,9 @@ bool ownsPoint(DWORD pid, POINT at);
bool isFront(DWORD pid);
HWND heldOwner(HWND window, DWORD pid);
void front(DWORD pid);
// targetWindow aims the request being served at one of the application's
// windows, which appWindows then lists first; null leaves the front one.
void targetWindow(DWORD pid, HWND window);
RECT windowBounds(HWND hwnd);
Json rectJson(const RECT& r);
+7
View File
@@ -109,6 +109,13 @@ static Json handle(const std::string& method, const Json& params) {
if (!perApp.count(method)) throw Failure{"computer.bad_request", "unknown method " + method};
DWORD pid = pidParam(params);
requireOperable(pid);
struct Aim {
explicit Aim(DWORD pid, const Json& params) {
const Json* w = params.get("window");
if (w && w->isNumber() && w->number() > 0 && w->number() <= 9007199254740992.0) targetWindow(pid, reinterpret_cast<HWND>(static_cast<uintptr_t>(w->number())));
}
~Aim() { targetWindow(0, nullptr); }
} aim(pid, params);
if (method == "snapshot") return snapshot(pid);
if (method == "screenshot") return capture(pid);
if (method == "press") return press(pid, stringParam(params, "ref"));
+30 -6
View File
@@ -153,6 +153,30 @@ bool appListed(HWND top) {
return GetWindow(top, GW_OWNER) == nullptr && (ex & WS_EX_NOACTIVATE) == 0;
}
static HWND aimed = nullptr;
// ownsForeground is whether the foreground is the aimed window or a dialog it
// owns; with no window aimed any foreground of the process will do.
static bool ownsForeground(HWND window) {
if (!window) return true;
for (HWND w = GetForegroundWindow(); w; w = GetWindow(w, GW_OWNER)) {
if (w == window) return true;
}
return false;
}
void targetWindow(DWORD pid, HWND window) {
aimed = nullptr;
if (!window) return;
for (HWND w : appWindows(pid, true)) {
if (w == window) {
aimed = window;
return;
}
}
throw Failure{"computer.no_window", "that window is not one of the application's windows on screen"};
}
// appWindows lists a process's top-level windows front to back, dialogs it
// owns included, with the one in front of the person first.
std::vector<HWND> appWindows(DWORD pid, bool includeMinimized) {
@@ -168,7 +192,7 @@ std::vector<HWND> appWindows(DWORD pid, bool includeMinimized) {
if (r.right - r.left > 1 && r.bottom - r.top > 1) w->out.push_back(hwnd);
return TRUE;
}, reinterpret_cast<LPARAM>(&w));
HWND fg = GetForegroundWindow();
HWND fg = aimed ? aimed : GetForegroundWindow();
for (size_t i = 1; i < w.out.size(); i++) {
if (w.out[i] == fg) {
w.out.erase(w.out.begin() + i);
@@ -223,7 +247,7 @@ HWND heldOwner(HWND window, DWORD pid) {
// foreground is. Windows grants the foreground only to a process that received
// the last input, so an input with no effect is injected first.
void front(DWORD pid) {
if (isFront(pid)) return;
if (isFront(pid) && ownsForeground(aimed)) return;
std::vector<HWND> wins = appWindows(pid, true);
if (wins.empty()) throw Failure{"computer.no_window", "the application has no window to bring to the front"};
HWND target = wins[0];
@@ -233,8 +257,8 @@ void front(DWORD pid) {
nudge.mi.dwFlags = MOUSEEVENTF_MOVE;
SendInput(1, &nudge, sizeof nudge);
SetForegroundWindow(target);
for (int i = 0; i < 25 && !isFront(pid); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
if (!isFront(pid)) {
for (int i = 0; i < 25 && !(isFront(pid) && ownsForeground(aimed)); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
if (!(isFront(pid) && ownsForeground(aimed))) {
DWORD fgThread = GetWindowThreadProcessId(GetForegroundWindow(), nullptr);
DWORD me = GetCurrentThreadId();
if (fgThread && fgThread != me && AttachThreadInput(me, fgThread, TRUE)) {
@@ -242,9 +266,9 @@ void front(DWORD pid) {
SetForegroundWindow(target);
AttachThreadInput(me, fgThread, FALSE);
}
for (int i = 0; i < 25 && !isFront(pid); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
for (int i = 0; i < 25 && !(isFront(pid) && ownsForeground(aimed)); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
}
if (!isFront(pid)) {
if (!(isFront(pid) && ownsForeground(aimed))) {
throw Failure{"computer.needs_front",
"Windows kept this application from coming to the front; ask the person to click it once, then try again"};
}
+2 -2
View File
@@ -19,8 +19,8 @@ This document records the provider-visible contract for Reasonix compile-time bu
| `code_index` | true | Lightweight built-in code symbol index. Prefer lsp_* for language semantics and installed code graph MCP tools for call graph, impact, and architecture relationships; use this as the local fallback for file outlines and symbol definition candidates, then verify with read_file or grep. |
| `complete_step` | true | Record the evidence-backed completion of ONE step of an approved plan. Call it as you finish each step instead of silently moving on: a completion with no evidence is REJECTED, so don't claim a step is done until you can show why. The host advances the list for you — it marks this step completed and moves the next to in_progress, so a todo_write that marks such an item completed on its own is REJECTED too. Sign off one step per tool-call ROUND, not one per call: the sign-off promotes the next item, and a second sign-off sent beside it signs against the list as it stood before. |
| `compress` | true | Compress a selected part of the current model-visible conversation without deleting visible history. Use only when the user explicitly asks for context compression. Choose `before` to summarize everything before the uniquely matched user turn while keeping that turn and later context, or `after` to summarize from that turn through the last completed turn while keeping the active turn. The anchor must be an exact, unique excerpt from a real user message; use a longer excerpt if the tool reports multiple matches. |
| `computer_act` | false | Operate another application. Steps run in order and stop at the first failure. Through its accessibility actions, leaving the pointer alone: click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is borrowed and put back), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. Paste needs the application in front on macOS; on Windows type, paste and key bring it forward. scroll brings a ref into view or turns the wheel by amount lines, wait pauses ms. pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, and the pointer goes back where it was. Returns a snapshot afterwards. |
| `computer_read` | false | See another application on this computer, named as what=apps lists it. what=apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels, and the person is shown it too). |
| `computer_act` | false | Operate another application. Steps run in order and stop at the first failure. Through accessibility actions, pointer untouched: click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is restored), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. Paste needs the application in front on macOS; on Windows type, paste and key bring it forward. scroll brings a ref into view or turns the wheel by amount lines, wait pauses ms. pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, then the pointer is restored. Returns a snapshot afterwards. |
| `computer_read` | false | See another application, named as what=apps lists it. apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels; the person sees it too). |
| `context_budget` | true | Report how much context room is left before this conversation is automatically compacted. Nothing reports it unprompted: check it at the start of a long task, before a large change, and before output you may not finish reading (a broad search, a large file, a long build log), so you can narrow the command. `tokens_remaining` counts down to the compaction trigger, not the physical window. |
| `delete_range` | false | Delete a contiguous text range from a file using exact start/end text anchors. Each anchor must match exactly one line. Returns unified diff on success. Use for large deletions — smaller changes should use edit_file. |
| `delete_symbol` | false | Delete a named symbol (function, method, type, interface, const, var) from a Go source file using AST parsing. For non-Go files, use delete_range with manual anchors. |
+49 -1
View File
@@ -33,9 +33,16 @@ func runFakeComputerHelper(in io.Reader, out io.Writer) {
var result any = map[string]any{}
switch req.Method {
case "apps":
bounds := map[string]any{"x": 0, "y": 0, "width": 800, "height": 600}
result = map[string]any{"apps": []any{map[string]any{
"pid": 42, "bundle": "com.example.Notes", "name": "Notes", "active": true,
"windows": []any{map[string]any{"id": 1, "title": "Draft", "bounds": map[string]any{"x": 0, "y": 0, "width": 800, "height": 600}}},
"windows": []any{map[string]any{"id": 1, "title": "Draft", "bounds": bounds}},
}, map[string]any{
"pid": 43, "bundle": "com.example.Pair", "name": "Pair",
"windows": []any{
map[string]any{"id": 11, "title": "Left", "owned": false, "bounds": bounds},
map[string]any{"id": 12, "title": "Right", "owned": false, "bounds": bounds},
},
}}}
case "snapshot":
result = map[string]any{
@@ -46,6 +53,13 @@ func runFakeComputerHelper(in io.Reader, out io.Writer) {
value, _ = req.Params["text"].(string)
result = map[string]any{"effect": map[string]any{"class": "confirmed", "evidence": []string{"value_readback"}}}
case "type":
if req.Params["pid"] == float64(43) {
if req.Params["window"] == nil {
_ = enc.Encode(map[string]any{"id": req.ID, "error": map[string]any{"code": "computer.failed", "message": "type reached the helper with no window named"}})
continue
}
break
}
_ = enc.Encode(map[string]any{"id": req.ID, "error": map[string]any{
"code": "computer.blocked", "message": "the keys would land in a dialog whose focused element takes no text",
"blocked_by": map[string]any{"ref": "a9", "title": "Unsaved changes", "blocks": "Draft"},
@@ -135,3 +149,37 @@ func TestComputerToolIdentityAgreesAcrossTheKernel(t *testing.T) {
t.Errorf("operating an application is classified %q, want unknown: it can write where the application may", got)
}
}
// Through the real assembly: an application showing two windows is refused
// before the helper is asked anything, the candidates reach the model, and
// naming one of them lets the same call through.
func TestEffectComputerUseRefusesAnAmbiguousWindowThroughTheRealAssembly(t *testing.T) {
exe, err := os.Executable()
if err != nil {
t.Fatal(err)
}
t.Setenv(fakeComputerHelperEnv, "1")
SetComputerHelper(exe)
t.Cleanup(func() { SetComputerHelper("") })
reqs := buildBrowserEffect(t, "", []func(string) *provider.ToolCall{
func(string) *provider.ToolCall {
return browserCall("act-1", "computer_act", map[string]any{"app": "com.example.Pair", "steps": []any{map[string]any{"action": "type", "text": "hi"}}})
},
func(string) *provider.ToolCall {
return browserCall("act-2", "computer_act", map[string]any{"app": "com.example.Pair", "window": 12, "steps": []any{map[string]any{"action": "type", "text": "hi"}}})
},
})
results := effectToolResults(reqs[len(reqs)-1])
if len(results) != 2 {
t.Fatalf("tool results = %q", results)
}
for _, want := range []string{"computer.ambiguous_window_target", `window 11 "Left" (front)`, `window 12 "Right"`, "Completed 0 of 1 step(s)."} {
if !strings.Contains(results[0], want) {
t.Fatalf("the refusal is missing %q:\n%s", want, results[0])
}
}
if !strings.Contains(results[1], "Completed 1 of 1 step(s).") {
t.Fatalf("naming the window did not let the call through:\n%s", results[1])
}
}
+2 -2
View File
@@ -12,7 +12,7 @@ import (
// it fails with an identity, and the failure names the modal from structure.
func TestAStepAModalHoldsFailsAndNamesTheModal(t *testing.T) {
s := fakeSession(t)
res, err := s.Act(context.Background(), "com.example.Held", []Step{{Action: "type", Text: "hello"}})
res, err := s.Act(context.Background(), "com.example.Held", 0, []Step{{Action: "type", Text: "hello"}})
if CodeOf(err) != CodeBlocked || res.Done != 0 || res.FailedAt != 0 {
t.Fatalf("typing into a held application = %+v, %v; want %s at step 1 with nothing done", res, err, CodeBlocked)
}
@@ -32,7 +32,7 @@ func TestAStepAModalHoldsFailsAndNamesTheModal(t *testing.T) {
// carries. A step that sends nothing has no effect at all.
func TestAnEffectIsOnlyAsStrongAsItsEvidence(t *testing.T) {
s := fakeSession(t)
res, err := s.Act(context.Background(), "com.example.Notes", []Step{
res, err := s.Act(context.Background(), "com.example.Notes", 0, []Step{
{Action: "set_value", Ref: "a2", Text: "hi"},
{Action: "type", Text: "hi"},
{Action: "focus", Ref: "a2"},
+19 -4
View File
@@ -3,6 +3,7 @@ package computer
import (
"errors"
"fmt"
"strings"
)
// Code is the identity of a computer-use failure, shared with the helper that
@@ -15,6 +16,7 @@ const (
CodePermissionMissing Code = "computer.permission_missing"
CodeNoApp Code = "computer.no_app"
CodeNoWindow Code = "computer.no_window"
CodeAmbiguousWindow Code = "computer.ambiguous_window_target"
CodeAppRefused Code = "computer.app_refused"
CodeUnknownRef Code = "computer.unknown_ref"
CodeStaleRef Code = "computer.stale_ref"
@@ -36,17 +38,30 @@ type Failure struct {
Detail string
// BlockedBy is the modal that would have taken the input instead.
BlockedBy *Modal
// Candidates are the windows an ambiguous target could have meant.
Candidates []Window
}
func (f *Failure) Error() string {
s := string(f.Code)
var b strings.Builder
b.WriteString(string(f.Code))
if f.Detail != "" {
s += ": " + f.Detail
b.WriteString(": " + f.Detail)
}
if f.BlockedBy != nil {
s += "; held by " + f.BlockedBy.String()
b.WriteString("; held by " + f.BlockedBy.String())
}
return s
for i, w := range f.Candidates {
if i == maxCandidates {
fmt.Fprintf(&b, "; and %d more", len(f.Candidates)-i)
break
}
fmt.Fprintf(&b, "; window %d %s", w.ID, ShownName(w.Title))
if i == 0 {
b.WriteString(" (front)")
}
}
return b.String()
}
// Is matches another Failure by code alone.
+18 -18
View File
@@ -75,7 +75,7 @@ func TestLiveOperatesAnApplicationWithoutItsPointer(t *testing.T) {
field := lineRef(t, snap.Lines, `textField "Probe field"`)
button := lineRef(t, snap.Lines, `button "Probe button"`)
res, err := s.Act(ctx, targetBundle, []Step{
res, err := s.Act(ctx, targetBundle, 0, []Step{
{Action: "set_value", Ref: field, Text: "from-ax"},
{Action: "click", Ref: button},
{Action: "focus", Ref: field},
@@ -87,7 +87,7 @@ func TestLiveOperatesAnApplicationWithoutItsPointer(t *testing.T) {
waitLog(t, log, "button pressed")
waitLog(t, log, "from-ax 李雷")
shot, app, err := s.Screenshot(ctx, targetBundle)
shot, app, err := s.Screenshot(ctx, targetBundle, 0)
if err != nil {
t.Fatalf("Screenshot: %v", err)
}
@@ -99,11 +99,11 @@ func TestLiveOperatesAnApplicationWithoutItsPointer(t *testing.T) {
geometry := s.shots[targetBundle]
s.mu.Unlock()
x, y := 345/geometry.scale, (geometry.bounds.Height-162)/geometry.scale
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "click", X: &x, Y: &y}}); err != nil {
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "click", X: &x, Y: &y}}); err != nil {
t.Fatalf("click at a screenshot point: %v", err)
}
cx, cy := 100/geometry.scale, (geometry.bounds.Height-70)/geometry.scale
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "click", X: &cx, Y: &cy}}); CodeOf(err) != CodeNoAction {
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "click", X: &cx, Y: &cy}}); CodeOf(err) != CodeNoAction {
t.Fatalf("clicking a view with no accessibility action = %v, want %s", err, CodeNoAction)
}
if _, err := s.Snapshot(ctx, "com.apple.Terminal"); CodeOf(err) != CodeAppRefused {
@@ -125,7 +125,7 @@ func TestLiveContextMenuScrollAndWait(t *testing.T) {
}
t.Logf("snapshot:\n%s", strings.Join(snap.Lines, "\n"))
menu := lineRef(t, snap.Lines, `"Probe menu area"`)
res, err := s.Act(ctx, targetBundle, []Step{
res, err := s.Act(ctx, targetBundle, 0, []Step{
{Action: "right_click", Ref: menu},
{Action: "wait", Ms: 400},
})
@@ -151,7 +151,7 @@ func TestLiveContextMenuScrollAndWait(t *testing.T) {
}
before := position()
deep := lineRef(t, snap.Lines, `"Deep row"`)
res, err = s.Act(ctx, targetBundle, []Step{{Action: "scroll", Ref: deep}})
res, err = s.Act(ctx, targetBundle, 0, []Step{{Action: "scroll", Ref: deep}})
if err != nil || len(res.Steps) == 0 || !strings.Contains(res.Steps[0].Note, "into view") {
t.Fatalf("scroll to a ref: %v %v", err, res.Steps)
}
@@ -159,10 +159,10 @@ func TestLiveContextMenuScrollAndWait(t *testing.T) {
t.Fatalf("the scroll area did not move: %q", after)
}
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "scroll", Amount: -3}}); err != nil {
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "scroll", Amount: -3}}); err != nil {
t.Fatalf("scroll by lines: %v", err)
}
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
t.Fatalf("a right_click with no ref = %v, want %s", err, CodeBadStep)
}
}
@@ -191,22 +191,22 @@ func TestLivePasteLeavesTheClipboardAsItFoundIt(t *testing.T) {
paste := []Step{{Action: "focus", Ref: field}, {Action: "paste", Text: pasted}}
// Behind, the application never handles the keystroke, and this says so
// rather than reporting a paste that put nothing anywhere.
if _, err := s.Act(ctx, targetBundle, paste); CodeOf(err) != CodeNeedsFront {
if _, err := s.Act(ctx, targetBundle, 0, paste); CodeOf(err) != CodeNeedsFront {
t.Fatalf("pasting into an application that is behind = %v, want %s", err, CodeNeedsFront)
}
// Bringing it forward is a pointer step, which is answered for on its own.
if _, _, err := s.Screenshot(ctx, targetBundle); err != nil {
if _, _, err := s.Screenshot(ctx, targetBundle, 0); err != nil {
t.Fatalf("Screenshot: %v", err)
}
s.mu.Lock()
geometry := s.shots[targetBundle]
s.mu.Unlock()
x, y := 345/geometry.scale, (geometry.bounds.Height-162)/geometry.scale
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
t.Fatalf("bring it forward: %v", err)
}
if _, err := s.Act(ctx, targetBundle, paste); err != nil {
if _, err := s.Act(ctx, targetBundle, 0, paste); err != nil {
t.Fatalf("paste: %v", err)
}
waitLog(t, log, "text "+pasted)
@@ -237,7 +237,7 @@ func TestLiveThePointerReachesWhatAccessibilityCannot(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
shot, app, err := s.Screenshot(ctx, targetBundle)
shot, app, err := s.Screenshot(ctx, targetBundle, 0)
if err != nil {
t.Fatalf("Screenshot: %v", err)
}
@@ -251,22 +251,22 @@ func TestLiveThePointerReachesWhatAccessibilityCannot(t *testing.T) {
// the screenshot the model was shown.
x, y := 100/geometry.scale, (geometry.bounds.Height-70)/geometry.scale
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "click", X: &x, Y: &y}}); CodeOf(err) != CodeNoAction {
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "click", X: &x, Y: &y}}); CodeOf(err) != CodeNoAction {
t.Fatalf("an accessibility click on a drawn view = %v, want %s", err, CodeNoAction)
}
// Where the person's pointer is, in the pixels of the screenshot the model
// was shown, before and after the agent borrows it.
home, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_position"}})
home, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_position"}})
if err != nil {
t.Fatalf("pointer_position: %v", err)
}
res, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_click", X: &x, Y: &y}})
res, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_click", X: &x, Y: &y}})
if err != nil {
t.Fatalf("pointer_click: %v (%v)", err, res.Steps)
}
waitLog(t, log, "mouseDown")
back, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_position"}})
back, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_position"}})
if err != nil {
t.Fatalf("pointer_position: %v", err)
}
@@ -275,7 +275,7 @@ func TestLiveThePointerReachesWhatAccessibilityCannot(t *testing.T) {
}
toX, toY := x+40, y-20
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}}); err != nil {
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}}); err != nil {
t.Fatalf("pointer_drag: %v", err)
}
waitLog(t, log, "mouseDragged")
+88 -15
View File
@@ -93,7 +93,7 @@ func TestLiveWindowsOperatesAnApplication(t *testing.T) {
field := lineRef(t, snap.Lines, `edit "Probe field"`)
button := lineRef(t, snap.Lines, `button "Probe button"`)
res, err := s.Act(ctx, windowsTarget, []Step{
res, err := s.Act(ctx, windowsTarget, 0, []Step{
{Action: "set_value", Ref: field, Text: "from-ax"},
{Action: "click", Ref: button},
{Action: "focus", Ref: field},
@@ -112,7 +112,7 @@ func TestLiveWindowsOperatesAnApplication(t *testing.T) {
waitLog(t, p.log, "button pressed 1")
waitLog(t, p.log, "text from-ax 李雷3\ntext \n")
shot, app, err := s.Screenshot(ctx, windowsTarget)
shot, app, err := s.Screenshot(ctx, windowsTarget, 0)
if err != nil {
t.Fatalf("Screenshot: %v", err)
}
@@ -120,18 +120,18 @@ func TestLiveWindowsOperatesAnApplication(t *testing.T) {
t.Fatalf("screenshot = %.40q for %+v", shot, app)
}
x, y := inShot(s, p.button)
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "click", X: x, Y: y}}); err != nil {
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "click", X: x, Y: y}}); err != nil {
t.Fatalf("click at a screenshot point: %v", err)
}
waitLog(t, p.log, "button pressed 2")
cx, cy := inShot(s, p.canvas)
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "click", X: cx, Y: cy}}); CodeOf(err) != CodeNoAction {
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "click", X: cx, Y: cy}}); CodeOf(err) != CodeNoAction {
t.Fatalf("clicking a window with no accessibility action = %v, want %s", err, CodeNoAction)
}
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "key", Key: "meta+s"}}); CodeOf(err) != CodeBadStep {
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "key", Key: "meta+s"}}); CodeOf(err) != CodeBadStep {
t.Fatalf("the Windows key = %v, want %s", err, CodeBadStep)
}
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "scroll", Amount: -3}}); err != nil {
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "scroll", Amount: -3}}); err != nil {
t.Fatalf("scroll by lines: %v", err)
}
for _, name := range []string{"cmd.exe", "PowerShell.exe"} {
@@ -168,7 +168,7 @@ func TestLiveWindowsPasteGivesTheClipboardBack(t *testing.T) {
}
field := lineRef(t, snap.Lines, `edit "Probe field"`)
pasted := "粘贴 without typing"
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "focus", Ref: field}, {Action: "paste", Text: pasted}}); err != nil {
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "focus", Ref: field}, {Action: "paste", Text: pasted}}); err != nil {
t.Fatalf("paste: %v", err)
}
waitLog(t, p.log, "text "+pasted)
@@ -192,19 +192,19 @@ func TestLiveWindowsThePointerReachesWhatAccessibilityCannot(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
if _, _, err := s.Screenshot(ctx, windowsTarget); err != nil {
if _, _, err := s.Screenshot(ctx, windowsTarget, 0); err != nil {
t.Fatalf("Screenshot: %v", err)
}
x, y := inShot(s, p.canvas)
home, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_position"}})
home, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_position"}})
if err != nil {
t.Fatalf("pointer_position: %v", err)
}
if res, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_click", X: x, Y: y}}); err != nil {
if res, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_click", X: x, Y: y}}); err != nil {
t.Fatalf("pointer_click: %v (%v)", err, res.Steps)
}
waitLog(t, p.log, "mouseDown")
back, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_position"}})
back, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_position"}})
if err != nil {
t.Fatalf("pointer_position: %v", err)
}
@@ -212,7 +212,7 @@ func TestLiveWindowsThePointerReachesWhatAccessibilityCannot(t *testing.T) {
t.Fatalf("the pointer was left at %q, not where the person had it (%q)", back.Steps[0].Note, home.Steps[0].Note)
}
toX, toY := *x+40, *y-20
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_drag", X: x, Y: y, ToX: &toX, ToY: &toY}}); err != nil {
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_drag", X: x, Y: y, ToX: &toX, ToY: &toY}}); err != nil {
t.Fatalf("pointer_drag: %v", err)
}
waitLog(t, p.log, "mouseDragged")
@@ -241,7 +241,7 @@ func heldSnapshot(t *testing.T, ctx context.Context, s *Session) Snapshot {
func actor(t *testing.T, ctx context.Context, s *Session) func(...Step) Effect {
return func(steps ...Step) Effect {
t.Helper()
res, err := s.Act(ctx, windowsTarget, steps)
res, err := s.Act(ctx, windowsTarget, 0, steps)
if err != nil || len(res.Steps) == 0 {
t.Fatalf("Act = %+v, %v", res, err)
}
@@ -264,14 +264,14 @@ func TestLiveWindowsAModalHoldsTheInput(t *testing.T) {
t.Fatalf("modal = %+v", m)
}
button := lineRef(t, snap.Lines, `button "Probe button"`)
res, err := s.Act(ctx, windowsTarget, []Step{{Action: "type", Text: "hello"}})
res, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "type", Text: "hello"}})
if CodeOf(err) != CodeBlocked || res.Done != 0 {
t.Fatalf("typing while a message box holds the input = %+v, %v; want %s", res, err, CodeBlocked)
}
if f, _ := errors.AsType[*Failure](err); f.BlockedBy == nil || f.BlockedBy.Title != "Probe modal" {
t.Fatalf("the failure does not name the modal: %v", err)
}
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "click", Ref: button}}); CodeOf(err) != CodeBlocked {
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "click", Ref: button}}); CodeOf(err) != CodeBlocked {
t.Fatalf("pressing a button behind the modal = %v, want %s", err, CodeBlocked)
}
if e := act(Step{Action: "key", Key: "escape"}); e.Class != EffectUnverifiable || e.BlockedBy == nil {
@@ -318,3 +318,76 @@ func TestLiveWindowsTypingIntoAModalsFieldIsReadBack(t *testing.T) {
}
waitLog(t, p.log, "form text xyz")
}
// Two top-level windows of one application leave the target ambiguous: input
// is refused before it is sent, and naming a window aims it there.
func TestLiveWindowsTwoWindowsAreRefusedUntilOneIsNamed(t *testing.T) {
s := liveSession(t)
p := launchWindowsTarget(t, "second")
waitLog(t, p.log, "second open")
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
apps, err := s.Apps(ctx)
if err != nil {
t.Fatal(err)
}
var app App
for _, a := range apps {
if a.Bundle == windowsTarget {
app = a
}
}
main, second := 0, 0
for _, w := range app.Windows {
switch w.Title {
case "Computer Target":
main = w.ID
case "Second Window":
second = w.ID
}
if w.Owned == nil || *w.Owned {
t.Fatalf("window %+v is not reported as an independent window", w)
}
}
if main == 0 || second == 0 {
t.Fatalf("windows = %+v", app.Windows)
}
res, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "type", Text: "hello"}})
f, _ := errors.AsType[*Failure](err)
if f == nil || f.Code != CodeAmbiguousWindow || res.Done != 0 || len(f.Candidates) != 2 {
t.Fatalf("typing at two windows = %+v, %v; want %s with both candidates", res, err, CodeAmbiguousWindow)
}
if _, _, err := s.Screenshot(ctx, windowsTarget, 0); CodeOf(err) != CodeAmbiguousWindow {
t.Fatalf("a screenshot of two windows = %v, want %s", err, CodeAmbiguousWindow)
}
if body, _ := os.ReadFile(p.log); strings.Contains(string(body), "hello") {
t.Fatalf("input reached the application despite the refusal:\n%s", body)
}
if _, _, err := s.Screenshot(ctx, windowsTarget, second); err != nil {
t.Fatalf("screenshot of the second window: %v", err)
}
s.mu.Lock()
shot := s.shots[windowsTarget]
s.mu.Unlock()
if shot.window != second || shot.bounds.Width < 250 || shot.bounds.Width > 400 {
t.Fatalf("the capture is %+v, want the 300-wide second window %d", shot, second)
}
snap, err := s.Snapshot(ctx, windowsTarget)
if err != nil {
t.Fatal(err)
}
field := lineRef(t, snap.Lines, `edit "Probe field"`)
if _, err := s.Act(ctx, windowsTarget, second, []Step{{Action: "key", Key: "home"}}); err != nil {
t.Fatalf("raising the second window: %v", err)
}
if res, err := s.Act(ctx, windowsTarget, main, []Step{{Action: "focus", Ref: field}, {Action: "type", Text: "hello"}}); err != nil || res.Done != 2 {
t.Fatalf("typing into the main window = %+v, %v", res, err)
}
waitLog(t, p.log, "hello")
if _, err := s.Act(ctx, windowsTarget, 1, []Step{{Action: "type", Text: "x"}}); CodeOf(err) != CodeNoWindow {
t.Fatalf("a window the application does not have = %v, want %s", err, CodeNoWindow)
}
}
+44 -23
View File
@@ -102,6 +102,9 @@ type Window struct {
ID int `json:"id"`
Title string `json:"title"`
Bounds Rect `json:"bounds"`
// Owned is true for a dialog or tool window another window owns. Nil means
// the helper does not classify windows, which leaves a target unguarded.
Owned *bool `json:"owned,omitempty"`
}
// Rect is a rectangle in global points, origin top-left.
@@ -122,6 +125,7 @@ type Session struct {
}
type shotGeometry struct {
window int
bounds Rect
scale float64 // screen points per pixel of the image the model was shown
}
@@ -196,19 +200,26 @@ func (s *Session) Snapshot(ctx context.Context, bundle string) (Snapshot, error)
// Screenshot captures an application's front window, sized for a vision model.
// x and y in later steps are read in this image's pixels.
func (s *Session) Screenshot(ctx context.Context, bundle string) (string, App, error) {
func (s *Session) Screenshot(ctx context.Context, bundle string, window int) (string, App, error) {
app, err := s.App(ctx, bundle)
if err != nil {
return "", App{}, err
}
if err := app.target(window, true); err != nil {
return "", App{}, err
}
var r struct {
Data string `json:"data"`
Mime string `json:"mime"`
Window int `json:"window"`
Bounds Rect `json:"bounds"`
}
if err := s.call(ctx, "screenshot", map[string]any{"pid": app.PID}, &r); err != nil {
if err := s.call(ctx, "screenshot", withWindow(map[string]any{"pid": app.PID}, window), &r); err != nil {
return "", App{}, err
}
if window != 0 && r.Window != window {
return "", App{}, fail(CodeNoWindow, "window %d could not be captured; the capture is of window %d", window, r.Window)
}
raw, err := base64.StdEncoding.DecodeString(r.Data)
if err != nil {
return "", App{}, fail(CodeCaptureFailed, "the capture was not valid base64")
@@ -222,7 +233,7 @@ func (s *Session) Screenshot(ctx context.Context, bundle string) (string, App, e
return "", App{}, fail(CodeCaptureFailed, "the capture could not be measured")
}
s.mu.Lock()
s.shots[app.Bundle] = shotGeometry{bounds: r.Bounds, scale: r.Bounds.Width / float64(cfg.Width)}
s.shots[app.Bundle] = shotGeometry{window: r.Window, bounds: r.Bounds, scale: r.Bounds.Width / float64(cfg.Width)}
s.mu.Unlock()
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(fitted), app, nil
}
@@ -282,7 +293,7 @@ type StepResult struct {
// Act runs steps in order on one application and stops at the first that
// fails, or when the person presses Escape while the agent's cursor is showing.
func (s *Session) Act(ctx context.Context, bundle string, steps []Step) (ActResult, error) {
func (s *Session) Act(ctx context.Context, bundle string, window int, steps []Step) (ActResult, error) {
res := ActResult{FailedAt: -1}
app, err := s.App(ctx, bundle)
if err != nil {
@@ -290,6 +301,10 @@ func (s *Session) Act(ctx context.Context, bundle string, steps []Step) (ActResu
return res, err
}
res.App = app
if err := app.target(window, needsWindow(steps)); err != nil {
res.FailedAt = 0
return res, err
}
// The pointer goes back where the person left it when the steps are done,
// whether they finished or stopped.
if PointerSteps(steps) {
@@ -305,7 +320,7 @@ func (s *Session) Act(ctx context.Context, bundle string, steps []Step) (ActResu
res.FailedAt = i
return res, fail(CodeStopped, "the person pressed Escape; ask before continuing")
}
done, err := s.step(ctx, app, step)
done, err := s.step(ctx, app, window, step)
if err != nil {
res.FailedAt = i
return res, err
@@ -335,36 +350,39 @@ func (s *Session) deliver(ctx context.Context, method string, params map[string]
func noted(note string) func(actReply) string { return func(actReply) string { return note } }
func (s *Session) step(ctx context.Context, app App, step Step) (StepResult, error) {
func (s *Session) step(ctx context.Context, app App, window int, step Step) (StepResult, error) {
pid := app.PID
deliver := func(method string, params map[string]any, note func(actReply) string) (StepResult, error) {
return s.deliver(ctx, method, withWindow(params, window), note)
}
switch strings.ToLower(strings.TrimSpace(step.Action)) {
case "click":
if step.Ref != "" {
return s.deliver(ctx, "press", map[string]any{"pid": pid, "ref": step.Ref}, noted("press "+step.Ref))
return deliver("press", map[string]any{"pid": pid, "ref": step.Ref}, noted("press "+step.Ref))
}
x, y, err := s.screenPoint(app, step)
x, y, err := s.screenPoint(app, window, step)
if err != nil {
return StepResult{}, err
}
return s.deliver(ctx, "click", map[string]any{"pid": pid, "x": x, "y": y}, func(r actReply) string {
return deliver("click", map[string]any{"pid": pid, "x": x, "y": y}, func(r actReply) string {
return fmt.Sprintf("click the %s at (%v,%v)", r.Role, *step.X, *step.Y)
})
case "focus":
return s.deliver(ctx, "focus", map[string]any{"pid": pid, "ref": step.Ref}, noted("focus "+step.Ref))
return deliver("focus", map[string]any{"pid": pid, "ref": step.Ref}, noted("focus "+step.Ref))
case "set_value":
return s.deliver(ctx, "set_value", map[string]any{"pid": pid, "ref": step.Ref, "text": step.Text},
return deliver("set_value", map[string]any{"pid": pid, "ref": step.Ref, "text": step.Text},
noted(fmt.Sprintf("set %s to %d characters", step.Ref, len([]rune(step.Text)))))
case "type":
return s.deliver(ctx, "type", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("type %d characters", len([]rune(step.Text)))))
return deliver("type", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("type %d characters", len([]rune(step.Text)))))
case "key":
return s.deliver(ctx, "key", map[string]any{"pid": pid, "key": step.Key, "times": max(step.Times, 1)}, noted("press "+step.Key))
return deliver("key", map[string]any{"pid": pid, "key": step.Key, "times": max(step.Times, 1)}, noted("press "+step.Key))
case "paste":
return s.deliver(ctx, "paste", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("paste %d characters", len([]rune(step.Text)))))
return deliver("paste", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("paste %d characters", len([]rune(step.Text)))))
case "hold_key":
return s.deliver(ctx, "hold_key", map[string]any{"pid": pid, "key": step.Key, "seconds": step.Seconds},
return deliver("hold_key", map[string]any{"pid": pid, "key": step.Key, "seconds": step.Seconds},
noted(fmt.Sprintf("hold %s for %vs", step.Key, step.Seconds)))
case "pointer_move", "pointer_click", "pointer_drag":
return s.pointerStep(ctx, app, step)
return s.pointerStep(ctx, app, window, step)
case "pointer_position":
var at struct{ X, Y float64 }
if err := s.call(ctx, "pointer_position", nil, &at); err != nil {
@@ -375,9 +393,9 @@ func (s *Session) step(ctx context.Context, app App, step Step) (StepResult, err
if step.Ref == "" {
return StepResult{}, fail(CodeBadStep, "a right_click needs a ref; the menu belongs to the element")
}
return s.deliver(ctx, "menu", map[string]any{"pid": pid, "ref": step.Ref}, noted("open the menu of "+step.Ref))
return deliver("menu", map[string]any{"pid": pid, "ref": step.Ref}, noted("open the menu of "+step.Ref))
case "scroll":
return s.deliver(ctx, "scroll", map[string]any{"pid": pid, "ref": step.Ref, "amount": step.Amount}, func(r actReply) string {
return deliver("scroll", map[string]any{"pid": pid, "ref": step.Ref, "amount": step.Amount}, func(r actReply) string {
if r.How == "revealed" {
return "bring " + step.Ref + " into view"
}
@@ -398,12 +416,12 @@ func (s *Session) step(ctx context.Context, app App, step Step) (StepResult, err
// pointerStep takes the person's pointer to the point a screenshot named. The
// helper refuses a point whose window belongs to another application, so an
// approval for one application cannot reach into the next.
func (s *Session) pointerStep(ctx context.Context, app App, step Step) (StepResult, error) {
x, y, err := s.screenPoint(app, step)
func (s *Session) pointerStep(ctx context.Context, app App, window int, step Step) (StepResult, error) {
x, y, err := s.screenPoint(app, window, step)
if err != nil {
return StepResult{}, err
}
args := map[string]any{"pid": app.PID, "x": x, "y": y}
args := withWindow(map[string]any{"pid": app.PID, "x": x, "y": y}, window)
switch strings.ToLower(strings.TrimSpace(step.Action)) {
case "pointer_move":
return s.deliver(ctx, "pointer_move", args, noted(fmt.Sprintf("move the pointer to (%v,%v)", *step.X, *step.Y)))
@@ -412,7 +430,7 @@ func (s *Session) pointerStep(ctx context.Context, app App, step Step) (StepResu
if to.X == nil || to.Y == nil {
return StepResult{}, fail(CodeBadStep, "a pointer_drag needs where it ends: to_x and to_y")
}
toX, toY, err := s.screenPoint(app, to)
toX, toY, err := s.screenPoint(app, window, to)
if err != nil {
return StepResult{}, err
}
@@ -442,7 +460,7 @@ func (s *Session) describePoint(app App, x, y float64) string {
// screenPoint converts a point in the latest screenshot's pixels to global
// screen points.
func (s *Session) screenPoint(app App, step Step) (float64, float64, error) {
func (s *Session) screenPoint(app App, window int, step Step) (float64, float64, error) {
if step.X == nil || step.Y == nil {
return 0, 0, fail(CodeBadStep, "a click needs a ref, or x and y from a screenshot")
}
@@ -452,6 +470,9 @@ func (s *Session) screenPoint(app App, step Step) (float64, float64, error) {
if !ok {
return 0, 0, fail(CodeNeedsScreenshot, "take a screenshot of %s first; x and y are read in its pixels", app.Bundle)
}
if window != 0 && shot.window != window {
return 0, 0, fail(CodeNeedsScreenshot, "the latest screenshot of %s is of another window; take one of window %d first", app.Bundle, window)
}
return shot.bounds.X + *step.X*shot.scale, shot.bounds.Y + *step.Y*shot.scale, nil
}
+55 -30
View File
@@ -42,6 +42,8 @@ func runFakeHelper(in io.Reader, out io.Writer) {
var lastCall map[string]any
released := 0
asked := 0
delivered := 0
var lastSent map[string]any
sc := bufio.NewScanner(in)
for sc.Scan() {
var req struct {
@@ -60,13 +62,25 @@ func runFakeHelper(in io.Reader, out io.Writer) {
delete(reply, "result")
reply["error"] = map[string]any{"code": code, "message": message}
}
if m := req.Method; m != "apps" && m != "request_permission" && !strings.HasPrefix(m, "_") {
delivered++
lastSent = map[string]any{"method": m, "params": req.Params}
}
switch req.Method {
case "_delivered":
reply["result"] = map[string]any{"count": delivered}
case "_last_sent":
reply["result"] = lastSent
case "apps":
reply["result"] = map[string]any{"apps": []any{
map[string]any{"pid": 7, "bundle": "com.example.Notes", "name": "Notes", "windows": []any{}},
map[string]any{"pid": 8, "bundle": "com.example.Locked", "name": "Locked", "windows": []any{}},
map[string]any{"pid": 9, "bundle": "Vendor.Shell_abc123", "exe": "PWSH.exe", "name": "Shell", "windows": []any{}},
map[string]any{"pid": 10, "bundle": "com.example.Held", "name": "Held", "windows": []any{}},
map[string]any{"pid": 11, "bundle": "com.example.Two", "name": "Two", "windows": []any{fakeWindow(101, "Main", false), fakeWindow(102, "Other", false)}},
map[string]any{"pid": 12, "bundle": "com.example.Dialog", "name": "Dialog", "windows": []any{fakeWindow(202, "Save as", true), fakeWindow(201, "Main", false)}},
map[string]any{"pid": 14, "bundle": "com.example.Drift", "name": "Drift", "windows": []any{fakeWindow(401, "A", false), fakeWindow(402, "B", false)}},
map[string]any{"pid": 13, "bundle": "com.example.Flat", "name": "Flat", "windows": []any{map[string]any{"id": 301, "title": "A"}, map[string]any{"id": 302, "title": "A"}}},
}}
case "snapshot":
if req.Params["pid"] == float64(8) {
@@ -85,10 +99,17 @@ func runFakeHelper(in io.Reader, out io.Writer) {
case "request_permission":
asked++
case "screenshot":
reply["result"] = map[string]any{
shot := map[string]any{
"data": base64.StdEncoding.EncodeToString(fakeScreenshot()), "mime": "image/png",
"bounds": map[string]any{"x": 100, "y": 50, "width": 2000, "height": 1000},
}
if w, ok := req.Params["window"]; ok {
shot["window"] = w
}
if req.Params["pid"] == float64(14) {
shot["window"] = 401
}
reply["result"] = shot
case "click":
lastClick = req.Params
reply["result"] = map[string]any{"role": "button"}
@@ -154,7 +175,7 @@ func fakeSession(t *testing.T) *Session {
func TestARefusedApplicationIsRefusedBeforeTheHelperIsAsked(t *testing.T) {
s := NewSession(NewHelper("/nonexistent/helper"))
for _, bundle := range []string{"com.apple.Terminal", " io.reasonix.studio ", "com.apple.systempreferences"} {
if _, err := s.Act(context.Background(), bundle, []Step{{Action: "key", Key: "Enter"}}); CodeOf(err) != CodeAppRefused {
if _, err := s.Act(context.Background(), bundle, 0, []Step{{Action: "key", Key: "Enter"}}); CodeOf(err) != CodeAppRefused {
t.Errorf("%q = %v, want %s", bundle, err, CodeAppRefused)
}
}
@@ -177,10 +198,10 @@ func TestAClickAtAPointIsReadInTheLatestScreenshotsPixels(t *testing.T) {
ctx := context.Background()
x, y := 100.0, 40.0
click := []Step{{Action: "click", X: &x, Y: &y}}
if _, err := s.Act(ctx, "com.example.Notes", click); CodeOf(err) != CodeNeedsScreenshot {
if _, err := s.Act(ctx, "com.example.Notes", 0, click); CodeOf(err) != CodeNeedsScreenshot {
t.Fatalf("a click before any screenshot = %v, want %s", err, CodeNeedsScreenshot)
}
shot, app, err := s.Screenshot(ctx, "com.example.Notes")
shot, app, err := s.Screenshot(ctx, "com.example.Notes", 0)
if err != nil || app.PID != 7 {
t.Fatalf("Screenshot = %v for %+v", err, app)
}
@@ -192,7 +213,7 @@ func TestAClickAtAPointIsReadInTheLatestScreenshotsPixels(t *testing.T) {
if _, _, err := visionimage.Fit(raw, "image/png"); err != nil || fitted.Width >= 4000 {
t.Fatalf("the screenshot was not fitted for a vision model: %dpx, %v", fitted.Width, err)
}
res, err := s.Act(ctx, "com.example.Notes", click)
res, err := s.Act(ctx, "com.example.Notes", 0, click)
if err != nil || res.Steps[0].Note != "click the button at (100,40)" {
t.Fatalf("click = %+v, %v", res, err)
}
@@ -224,18 +245,18 @@ func TestAMissingPermissionIsRequestedOnceAndSaysWhatThePersonDoes(t *testing.T)
func TestStepsStopAtTheFirstFailureAndWhenThePersonPressesEscape(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Enter"}, {Action: "click", Ref: "a9"}, {Action: "key", Key: "Enter"}})
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Enter"}, {Action: "click", Ref: "a9"}, {Action: "key", Key: "Enter"}})
if CodeOf(err) != CodeStaleRef || res.Done != 1 || res.FailedAt != 1 {
t.Fatalf("a stale ref = %+v, %v", res, err)
}
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Escape"}, {Action: "key", Key: "Enter"}})
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Escape"}, {Action: "key", Key: "Enter"}})
if CodeOf(err) != CodeStopped || res.Done != 1 || res.FailedAt != 1 {
t.Fatalf("Escape = %+v, %v", res, err)
}
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "drag"}}); CodeOf(err) != CodeBadStep {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "drag"}}); CodeOf(err) != CodeBadStep {
t.Fatalf("an unknown action = %v, want %s", err, CodeBadStep)
}
if _, err := s.Act(ctx, "com.example.Missing", nil); CodeOf(err) != CodeNoApp {
if _, err := s.Act(ctx, "com.example.Missing", 0, nil); CodeOf(err) != CodeNoApp {
t.Fatalf("an application that is not running = %v, want %s", err, CodeNoApp)
}
}
@@ -252,7 +273,7 @@ func TestAHelperThatExitsFailsWhatWasPendingAndStartsAgain(t *testing.T) {
deadline := time.Now().Add(5 * time.Second)
for {
apps, err := s.Apps(ctx)
if err == nil && len(apps) == 4 {
if err == nil && len(apps) == 8 {
return
}
if time.Now().After(deadline) {
@@ -266,7 +287,7 @@ func TestAHelperThatExitsFailsWhatWasPendingAndStartsAgain(t *testing.T) {
// was given covers, which is what every point in a step is read in.
func fittedScale(t *testing.T, s *Session, ctx context.Context) float64 {
t.Helper()
shot, _, err := s.Screenshot(ctx, "com.example.Notes")
shot, _, err := s.Screenshot(ctx, "com.example.Notes", 0)
if err != nil {
t.Fatal(err)
}
@@ -297,7 +318,7 @@ func TestEveryPointerStepIsAimedInTheScreenshotsPixels(t *testing.T) {
x, y, toX, toY := 100.0, 40.0, 300.0, 120.0
wantX, wantY := 100+x*scale, 50+y*scale
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_move", X: &x, Y: &y}})
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_move", X: &x, Y: &y}})
if err != nil || res.Steps[0].Note != "move the pointer to (100,40)" {
t.Fatalf("pointer_move = %+v, %v", res, err)
}
@@ -306,7 +327,7 @@ func TestEveryPointerStepIsAimedInTheScreenshotsPixels(t *testing.T) {
t.Fatalf("the helper was sent %s %v, want pointer_move at (%v,%v)", method, params, wantX, wantY)
}
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_click", X: &x, Y: &y, Button: "right", Times: 2}})
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_click", X: &x, Y: &y, Button: "right", Times: 2}})
if err != nil || res.Steps[0].Note != "right click at (100,40)" {
t.Fatalf("pointer_click = %+v, %v", res, err)
}
@@ -314,14 +335,14 @@ func TestEveryPointerStepIsAimedInTheScreenshotsPixels(t *testing.T) {
t.Fatalf("the helper was sent %s %v, want a right double click", method, params)
}
// A click that says neither button nor count is one left click.
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
t.Fatal(err)
}
if _, params = lastCall(t, s); params["button"] != "left" || params["clicks"] != 1.0 {
t.Fatalf("a bare pointer_click sent %v, want one left click", params)
}
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}})
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}})
if err != nil || res.Steps[0].Note != "drag the pointer from (100,40) to (300,120)" {
t.Fatalf("pointer_drag = %+v, %v", res, err)
}
@@ -342,12 +363,12 @@ func TestAPointerStepThatCannotSayWhereIsRefused(t *testing.T) {
{Action: "pointer_drag", X: &x, Y: &x},
{Action: "pointer_drag", X: &x, Y: &x, ToX: &x},
} {
if _, err := s.Act(ctx, "com.example.Notes", []Step{step}); CodeOf(err) != CodeNeedsScreenshot && CodeOf(err) != CodeBadStep {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{step}); CodeOf(err) != CodeNeedsScreenshot && CodeOf(err) != CodeBadStep {
t.Fatalf("%+v = %v", step, err)
}
}
_ = fittedScale(t, s, ctx)
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_drag", X: &x, Y: &x}}); CodeOf(err) != CodeBadStep {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_drag", X: &x, Y: &x}}); CodeOf(err) != CodeBadStep {
t.Fatalf("a drag with no destination = %v, want %s", err, CodeBadStep)
}
}
@@ -365,20 +386,20 @@ func TestThePointerIsHandedBackWhicheverWayTheStepsEnd(t *testing.T) {
}
return r.Count
}
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Enter"}}); err != nil {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Enter"}}); err != nil {
t.Fatal(err)
}
if released() != 0 {
t.Fatal("steps that never took the pointer gave one back")
}
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_move", X: &x, Y: &y}}); err != nil {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_move", X: &x, Y: &y}}); err != nil {
t.Fatal(err)
}
if released() != 1 {
t.Fatal("the pointer was not given back after it was taken")
}
// A run that fails part way took it just the same.
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_move", X: &x, Y: &y}, {Action: "click", Ref: "a9"}}); CodeOf(err) != CodeStaleRef {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_move", X: &x, Y: &y}, {Action: "click", Ref: "a9"}}); CodeOf(err) != CodeStaleRef {
t.Fatalf("want %s, got %v", CodeStaleRef, err)
}
if released() != 2 {
@@ -393,12 +414,12 @@ func TestThePointerIsHandedBackWhicheverWayTheStepsEnd(t *testing.T) {
func TestThePointerSaysWhereItIsInTheFrameTheModelHas(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_position"}})
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_position"}})
if err != nil || !strings.Contains(res.Steps[0].Note, "take a screenshot") {
t.Fatalf("pointer_position with no screenshot = %+v, %v", res, err)
}
scale := fittedScale(t, s, ctx)
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_position"}})
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_position"}})
if err != nil {
t.Fatal(err)
}
@@ -411,10 +432,10 @@ func TestThePointerSaysWhereItIsInTheFrameTheModelHas(t *testing.T) {
func TestTheMenuBelongsToAnElementAndScrollingSaysWhichItWas(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
t.Fatalf("a right_click with no ref = %v, want %s", err, CodeBadStep)
}
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "right_click", Ref: "a4"}})
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "right_click", Ref: "a4"}})
if err != nil || res.Steps[0].Note != "open the menu of a4" {
t.Fatalf("right_click = %+v, %v", res, err)
}
@@ -423,7 +444,7 @@ func TestTheMenuBelongsToAnElementAndScrollingSaysWhichItWas(t *testing.T) {
}
// Bringing an element into view and turning the wheel are one step and two
// outcomes, and the helper says which one it did.
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "scroll", Ref: "a2"}, {Action: "scroll", Amount: -3}})
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "scroll", Ref: "a2"}, {Action: "scroll", Amount: -3}})
if err != nil || res.Steps[0].Note != "bring a2 into view" || res.Steps[1].Note != "scroll -3 lines" {
t.Fatalf("scroll = %+v, %v", res, err)
}
@@ -432,27 +453,27 @@ func TestTheMenuBelongsToAnElementAndScrollingSaysWhichItWas(t *testing.T) {
func TestHoldingAKeyAndRepeatingOneCarryTheirCount(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "hold_key", Key: "shift+a", Seconds: 1.5}})
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "hold_key", Key: "shift+a", Seconds: 1.5}})
if err != nil || res.Steps[0].Note != "hold shift+a for 1.5s" {
t.Fatalf("hold_key = %+v, %v", res, err)
}
if method, params := lastCall(t, s); method != "hold_key" || params["seconds"] != 1.5 {
t.Fatalf("the helper was sent %s %v, want the seconds it holds for", method, params)
}
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Tab", Times: 4}}); err != nil {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Tab", Times: 4}}); err != nil {
t.Fatal(err)
}
}
func TestWaitingIsBoundedAndTheContextEndsIt(t *testing.T) {
s := fakeSession(t)
res, err := s.Act(context.Background(), "com.example.Notes", []Step{{Action: "wait", Ms: 5}})
res, err := s.Act(context.Background(), "com.example.Notes", 0, []Step{{Action: "wait", Ms: 5}})
if err != nil || res.Steps[0].Note != "wait 5ms" {
t.Fatalf("wait = %+v, %v", res, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "wait", Ms: 60000}}); err == nil {
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "wait", Ms: 60000}}); err == nil {
t.Fatal("a wait outlived the context that was cancelled under it")
}
// The same cancellation reaches a call already sent to the helper.
@@ -466,7 +487,7 @@ func TestWaitingIsBoundedAndTheContextEndsIt(t *testing.T) {
func TestPastePutsTheWholeTextInAtOnce(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "paste", Text: "从剪贴板 pasted"}})
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "paste", Text: "从剪贴板 pasted"}})
if err != nil || res.Steps[0].Note != "paste 11 characters" {
t.Fatalf("paste = %+v, %v", res, err)
}
@@ -474,3 +495,7 @@ func TestPastePutsTheWholeTextInAtOnce(t *testing.T) {
t.Fatalf("the helper was sent %s %v, want the text to paste", method, params)
}
}
func fakeWindow(id int, title string, owned bool) map[string]any {
return map[string]any{"id": id, "title": title, "owned": owned, "bounds": map[string]any{"x": 0, "y": 0, "width": 800, "height": 600}}
}
+4
View File
@@ -210,6 +210,10 @@ func main() {
switch mode {
case "modal":
postMessage.Call(top, wmApp, 0, 0)
case "second":
other := create("ReasonixProbe", "Second Window", 0x00CF0000, 620, 120, 300, 200, 0, 0)
showWindow.Call(other, 4)
logf("second open")
case "form":
form := create("ProbeForm", "Probe form", 0x10CF0000, 160, 160, 320, 140, top, 0) // owned, WS_VISIBLE
create("STATIC", "Form field", child|visible, 10, 12, 80, 22, form, 0)
+66
View File
@@ -0,0 +1,66 @@
package computer
import (
"slices"
"strings"
)
const maxCandidates = 10
// withWindow adds the window a request is aimed at; zero leaves the helper to
// its default, the application's front window.
func withWindow(params map[string]any, window int) map[string]any {
if window != 0 {
params["window"] = window
}
return params
}
// needsWindow reports whether any step goes to one window of the application
// rather than to an element a ref already names.
func needsWindow(steps []Step) bool {
return slices.ContainsFunc(steps, func(s Step) bool {
switch strings.ToLower(strings.TrimSpace(s.Action)) {
case "type", "paste", "key", "hold_key", "pointer_move", "pointer_click", "pointer_drag":
return true
case "click", "scroll":
return s.Ref == ""
}
return false
})
}
// target settles which window an operation is aimed at: one the model names
// must be the application's own and a helper that classifies windows must be
// able to aim at it; with none named, two or more independent
// windows are refused before anything is sent. A dialog another window owns is
// not a second window, and a helper that does not classify windows judges none.
func (a App) target(window int, scoped bool) error {
if window != 0 {
if slices.ContainsFunc(a.Windows, func(w Window) bool { return w.Owned == nil }) {
return fail(CodeUnsupported, "%s cannot be aimed at one of its windows on this platform; leave window out", ShownBundle(a.Bundle))
}
if !slices.ContainsFunc(a.Windows, func(w Window) bool { return w.ID == window }) {
return fail(CodeNoWindow, "window %d is not a window of %s; list the applications for its windows", window, ShownBundle(a.Bundle))
}
return nil
}
if !scoped {
return nil
}
var independent []Window
for _, w := range a.Windows {
if w.Owned == nil {
return nil
}
if !*w.Owned {
independent = append(independent, w)
}
}
if len(independent) < 2 {
return nil
}
f := fail(CodeAmbiguousWindow, "%s has %d windows and this would go to one of them; repeat the call with window set to the one meant", ShownBundle(a.Bundle), len(independent))
f.Candidates = independent
return f
}
+138
View File
@@ -0,0 +1,138 @@
package computer
import (
"context"
"errors"
"testing"
)
func delivered(t *testing.T, s *Session) int {
t.Helper()
var r struct{ Count int }
if err := s.helper.Call(context.Background(), "_delivered", nil, &r); err != nil {
t.Fatal(err)
}
return r.Count
}
// Two independent windows make the target ambiguous: the call is refused before
// anything is sent, and the candidates come back with their ids and the front.
func TestAnApplicationWithTwoWindowsIsRefusedBeforeAnythingIsSent(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
for name, run := range map[string]func() error{
"type": func() error {
_, err := s.Act(ctx, "com.example.Two", 0, []Step{{Action: "type", Text: "x"}})
return err
},
"key": func() error {
_, err := s.Act(ctx, "com.example.Two", 0, []Step{{Action: "key", Key: "Enter"}})
return err
},
"shot": func() error { _, _, err := s.Screenshot(ctx, "com.example.Two", 0); return err },
} {
err := run()
f, ok := errors.AsType[*Failure](err)
if !ok || f.Code != CodeAmbiguousWindow {
t.Fatalf("%s = %v, want %s", name, err, CodeAmbiguousWindow)
}
if len(f.Candidates) != 2 || f.Candidates[0].ID != 101 || f.Candidates[1].ID != 102 {
t.Errorf("%s: candidates = %+v", name, f.Candidates)
}
}
if n := delivered(t, s); n != 0 {
t.Fatalf("%d requests reached the helper before the refusal", n)
}
}
// An element a ref names needs no window, and a window the model names settles
// the target and travels to the helper.
func TestANamedWindowOrARefSettlesTheTarget(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
if _, err := s.Act(ctx, "com.example.Two", 0, []Step{{Action: "focus", Ref: "a1"}}); err != nil {
t.Fatalf("a ref step on two windows = %v", err)
}
if res, err := s.Act(ctx, "com.example.Two", 102, []Step{{Action: "type", Text: "x"}}); err != nil || res.Done != 1 {
t.Fatalf("type into window 102 = %+v, %v", res, err)
}
var sent struct {
Method string
Params map[string]any
}
if err := s.helper.Call(ctx, "_last_sent", nil, &sent); err != nil || sent.Method != "type" || sent.Params["window"] != float64(102) {
t.Fatalf("the helper was sent %+v, %v", sent, err)
}
if _, _, err := s.Screenshot(ctx, "com.example.Two", 101); err != nil {
t.Fatalf("screenshot of window 101 = %v", err)
}
if _, err := s.Act(ctx, "com.example.Two", 999, []Step{{Action: "type", Text: "x"}}); CodeOf(err) != CodeNoWindow {
t.Fatalf("a window the application does not have = %v, want %s", err, CodeNoWindow)
}
}
// A dialog another window owns is not a second window, and a helper that does
// not classify windows leaves the target unjudged.
func TestADialogOrAnUnclassifiedWindowIsNotAmbiguity(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
for _, bundle := range []string{"com.example.Dialog", "com.example.Flat"} {
if _, err := s.Act(ctx, bundle, 0, []Step{{Action: "type", Text: "x"}}); err != nil {
t.Errorf("%s: %v", bundle, err)
}
}
}
// A point read in one window's screenshot is not a point in another's.
func TestAPointNeedsTheScreenshotOfTheWindowItIsAimedAt(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
if _, _, err := s.Screenshot(ctx, "com.example.Two", 101); err != nil {
t.Fatal(err)
}
x, y := 10.0, 10.0
if _, err := s.Act(ctx, "com.example.Two", 102, []Step{{Action: "click", X: &x, Y: &y}}); CodeOf(err) != CodeNeedsScreenshot {
t.Fatalf("a click aimed at a window never shot = %v, want %s", err, CodeNeedsScreenshot)
}
if _, err := s.Act(ctx, "com.example.Two", 101, []Step{{Action: "click", X: &x, Y: &y}}); err != nil {
t.Fatalf("a click in the window that was shot = %v", err)
}
}
// A capture of another window than the one asked for is not handed back as if
// it were: the helper may have aimed elsewhere, or the window may be gone.
func TestACaptureOfAnotherWindowIsRefused(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
if _, _, err := s.Screenshot(ctx, "com.example.Drift", 401); err != nil {
t.Fatalf("the window that was captured = %v", err)
}
if _, _, err := s.Screenshot(ctx, "com.example.Drift", 402); CodeOf(err) != CodeNoWindow {
t.Fatalf("a capture of window 401 for window 402 = %v, want %s", err, CodeNoWindow)
}
}
// A helper that does not classify windows cannot aim at one, so naming a
// window there is refused rather than silently sent to the front one.
func TestANamedWindowIsRefusedWhereTheHelperCannotAimAtIt(t *testing.T) {
s := fakeSession(t)
ctx := context.Background()
if _, err := s.Act(ctx, "com.example.Flat", 301, []Step{{Action: "type", Text: "x"}}); CodeOf(err) != CodeUnsupported {
t.Fatalf("a window on an unclassified helper = %v, want %s", err, CodeUnsupported)
}
if _, _, err := s.Screenshot(ctx, "com.example.Flat", 301); CodeOf(err) != CodeUnsupported {
t.Fatalf("a screenshot of a window on an unclassified helper = %v, want %s", err, CodeUnsupported)
}
if n := delivered(t, s); n != 0 {
t.Fatalf("%d requests reached the helper", n)
}
}
func TestTheAmbiguityNamesItsCandidatesBounded(t *testing.T) {
f := &Failure{Code: CodeAmbiguousWindow, Detail: "d", Candidates: []Window{{ID: 5, Title: "A\nB"}, {ID: 6, Title: "C"}}}
got := f.Error()
want := `computer.ambiguous_window_target: d; window 5 "A\u{a}B" (front); window 6 "C"`
if got != want {
t.Fatalf("Error() = %q, want %q", got, want)
}
}
+19 -13
View File
@@ -45,11 +45,11 @@ type computerRead struct{ session *computer.Session }
func (computerRead) Name() string { return "computer_read" }
func (computerRead) Description() string {
return "See another application on this computer, named as what=apps lists it. what=apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels, and the person is shown it too)."
return "See another application, named as what=apps lists it. apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels; the person sees it too)."
}
func (computerRead) Schema() json.RawMessage {
return json.RawMessage(`{"type":"object","properties":{"what":{"type":"string","enum":["apps","snapshot","screenshot"]},"app":{"type":"string"}},"required":["what"]}`)
return json.RawMessage(`{"type":"object","properties":{"what":{"type":"string","enum":["apps","snapshot","screenshot"]},"app":{"type":"string"},"window":{"type":"integer"}},"required":["what"]}`)
}
func (computerRead) ReadOnly() bool { return false }
@@ -75,8 +75,9 @@ func (c computerRead) Execute(ctx context.Context, args json.RawMessage) (string
func (c computerRead) ExecuteWithImages(ctx context.Context, args json.RawMessage) (string, []string, error) {
var p struct {
What string `json:"what"`
App string `json:"app"`
What string `json:"what"`
App string `json:"app"`
Window int `json:"window"`
}
if err := json.Unmarshal(args, &p); err != nil {
return "", nil, fmt.Errorf("invalid args: %w", err)
@@ -98,7 +99,7 @@ func (c computerRead) ExecuteWithImages(ctx context.Context, args json.RawMessag
}
return renderComputerSnapshot(snap), nil, nil
case "screenshot":
shot, app, err := c.session.Screenshot(ctx, p.App)
shot, app, err := c.session.Screenshot(ctx, p.App, p.Window)
if err != nil {
return "", nil, err
}
@@ -116,15 +117,15 @@ type computerAct struct{ session *computer.Session }
func (computerAct) Name() string { return "computer_act" }
func (computerAct) Description() string {
return "Operate another application. Steps run in order and stop at the first failure. Through its accessibility actions, leaving the pointer alone: " +
"click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is borrowed and put back), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. " +
return "Operate another application. Steps run in order and stop at the first failure. Through accessibility actions, pointer untouched: " +
"click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is restored), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. " +
"Paste needs the application in front on macOS; on Windows type, paste and key bring it forward. scroll brings a ref into view or turns the wheel by amount lines, wait pauses ms. " +
"pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, and the pointer goes back where it was. " +
"pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, then the pointer is restored. " +
"Returns a snapshot afterwards."
}
func (computerAct) Schema() json.RawMessage {
return json.RawMessage(`{"type":"object","properties":{"app":{"type":"string"},"steps":{"type":"array","items":{"type":"object","properties":{"action":{"type":"string","enum":["click","right_click","focus","set_value","type","paste","key","hold_key","scroll","wait","pointer_move","pointer_click","pointer_drag","pointer_position"]},"ref":{"type":"string"},"text":{"type":"string"},"key":{"type":"string"},"x":{"type":"number"},"y":{"type":"number"},"amount":{"type":"number"},"ms":{"type":"integer"},"to_x":{"type":"number"},"to_y":{"type":"number"},"button":{"type":"string","enum":["left","right","middle"]},"times":{"type":"integer"},"seconds":{"type":"number"}},"required":["action"]}}},"required":["app","steps"]}`)
return json.RawMessage(`{"type":"object","properties":{"app":{"type":"string"},"window":{"type":"integer"},"steps":{"type":"array","items":{"type":"object","properties":{"action":{"type":"string","enum":["click","right_click","focus","set_value","type","paste","key","hold_key","scroll","wait","pointer_move","pointer_click","pointer_drag","pointer_position"]},"ref":{"type":"string"},"text":{"type":"string"},"key":{"type":"string"},"x":{"type":"number"},"y":{"type":"number"},"amount":{"type":"number"},"ms":{"type":"integer"},"to_x":{"type":"number"},"to_y":{"type":"number"},"button":{"type":"string","enum":["left","right","middle"]},"times":{"type":"integer"},"seconds":{"type":"number"}},"required":["action"]}}},"required":["app","steps"]}`)
}
func (computerAct) ReadOnly() bool { return false }
@@ -146,8 +147,9 @@ func (c computerAct) PermissionArgs(_ context.Context, args json.RawMessage) jso
func (c computerAct) Execute(ctx context.Context, args json.RawMessage) (string, error) {
var p struct {
App string `json:"app"`
Steps []computer.Step `json:"steps"`
App string `json:"app"`
Window int `json:"window"`
Steps []computer.Step `json:"steps"`
}
if err := json.Unmarshal(args, &p); err != nil {
return "", fmt.Errorf("invalid args: %w", err)
@@ -158,7 +160,7 @@ func (c computerAct) Execute(ctx context.Context, args json.RawMessage) (string,
if len(p.Steps) == 0 {
return "", &computer.Failure{Code: computer.CodeBadStep, Detail: "steps is empty"}
}
res, stepErr := c.session.Act(ctx, p.App, p.Steps)
res, stepErr := c.session.Act(ctx, p.App, p.Window, p.Steps)
var out strings.Builder
fmt.Fprintf(&out, "Completed %d of %d step(s).\n", res.Done, len(p.Steps))
for i, step := range res.Steps {
@@ -196,7 +198,11 @@ func renderApps(apps []computer.App) string {
if w.Title != "" {
title = computer.ShownName(w.Title)
}
fmt.Fprintf(&b, " window %s %.0f×%.0f\n", title, w.Bounds.Width, w.Bounds.Height)
fmt.Fprintf(&b, " window %d %s %.0f×%.0f", w.ID, title, w.Bounds.Width, w.Bounds.Height)
if w.Owned != nil && *w.Owned {
b.WriteString(" (dialog)")
}
b.WriteString("\n")
}
}
if b.Len() == 0 {
+1 -1
View File
@@ -30,7 +30,7 @@ func TestApplicationListMarksTheOnesNeverOperated(t *testing.T) {
{Bundle: "com.apple.Notes", Name: "Notes", Active: true, Windows: []computer.Window{{Title: "", Bounds: computer.Rect{Width: 800, Height: 600}}}},
{Bundle: "com.apple.Terminal", Name: "Terminal"},
})
if !strings.Contains(out, `* com.apple.Notes — "Notes"`+"\n") || !strings.Contains(out, `window "(untitled)" 800×600`) {
if !strings.Contains(out, `* com.apple.Notes — "Notes"`+"\n") || !strings.Contains(out, `window 0 "(untitled)" 800×600`) {
t.Fatalf("listing = %q", out)
}
if !strings.Contains(out, `com.apple.Terminal — "Terminal" (never operated: a terminal)`) {