mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-08 01:48:26 -10:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
50f10f9419 | ||
|
|
68c1f9d551 | ||
|
|
a9dde35b70 | ||
|
|
85a1cbdd8a |
No files matched your search
@@ -27,6 +27,7 @@ Json listApps() {
|
||||
windows.push(Json::object()
|
||||
.set("id", static_cast<long long>(reinterpret_cast<uintptr_t>(hwnd)))
|
||||
.set("title", narrow(std::wstring(title, n > 0 ? n : 0)))
|
||||
.set("owned", GetWindow(hwnd, GW_OWNER) != nullptr && !(GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_APPWINDOW))
|
||||
.set("bounds", rectJson(windowBounds(hwnd))));
|
||||
}
|
||||
apps.push(Json::object()
|
||||
|
||||
@@ -48,6 +48,9 @@ bool ownsPoint(DWORD pid, POINT at);
|
||||
bool isFront(DWORD pid);
|
||||
HWND heldOwner(HWND window, DWORD pid);
|
||||
void front(DWORD pid);
|
||||
// targetWindow aims the request being served at one of the application's
|
||||
// windows, which appWindows then lists first; null leaves the front one.
|
||||
void targetWindow(DWORD pid, HWND window);
|
||||
RECT windowBounds(HWND hwnd);
|
||||
Json rectJson(const RECT& r);
|
||||
|
||||
|
||||
@@ -109,6 +109,13 @@ static Json handle(const std::string& method, const Json& params) {
|
||||
if (!perApp.count(method)) throw Failure{"computer.bad_request", "unknown method " + method};
|
||||
DWORD pid = pidParam(params);
|
||||
requireOperable(pid);
|
||||
struct Aim {
|
||||
explicit Aim(DWORD pid, const Json& params) {
|
||||
const Json* w = params.get("window");
|
||||
if (w && w->isNumber() && w->number() > 0 && w->number() <= 9007199254740992.0) targetWindow(pid, reinterpret_cast<HWND>(static_cast<uintptr_t>(w->number())));
|
||||
}
|
||||
~Aim() { targetWindow(0, nullptr); }
|
||||
} aim(pid, params);
|
||||
if (method == "snapshot") return snapshot(pid);
|
||||
if (method == "screenshot") return capture(pid);
|
||||
if (method == "press") return press(pid, stringParam(params, "ref"));
|
||||
|
||||
@@ -153,6 +153,30 @@ bool appListed(HWND top) {
|
||||
return GetWindow(top, GW_OWNER) == nullptr && (ex & WS_EX_NOACTIVATE) == 0;
|
||||
}
|
||||
|
||||
static HWND aimed = nullptr;
|
||||
|
||||
// ownsForeground is whether the foreground is the aimed window or a dialog it
|
||||
// owns; with no window aimed any foreground of the process will do.
|
||||
static bool ownsForeground(HWND window) {
|
||||
if (!window) return true;
|
||||
for (HWND w = GetForegroundWindow(); w; w = GetWindow(w, GW_OWNER)) {
|
||||
if (w == window) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void targetWindow(DWORD pid, HWND window) {
|
||||
aimed = nullptr;
|
||||
if (!window) return;
|
||||
for (HWND w : appWindows(pid, true)) {
|
||||
if (w == window) {
|
||||
aimed = window;
|
||||
return;
|
||||
}
|
||||
}
|
||||
throw Failure{"computer.no_window", "that window is not one of the application's windows on screen"};
|
||||
}
|
||||
|
||||
// appWindows lists a process's top-level windows front to back, dialogs it
|
||||
// owns included, with the one in front of the person first.
|
||||
std::vector<HWND> appWindows(DWORD pid, bool includeMinimized) {
|
||||
@@ -168,7 +192,7 @@ std::vector<HWND> appWindows(DWORD pid, bool includeMinimized) {
|
||||
if (r.right - r.left > 1 && r.bottom - r.top > 1) w->out.push_back(hwnd);
|
||||
return TRUE;
|
||||
}, reinterpret_cast<LPARAM>(&w));
|
||||
HWND fg = GetForegroundWindow();
|
||||
HWND fg = aimed ? aimed : GetForegroundWindow();
|
||||
for (size_t i = 1; i < w.out.size(); i++) {
|
||||
if (w.out[i] == fg) {
|
||||
w.out.erase(w.out.begin() + i);
|
||||
@@ -223,7 +247,7 @@ HWND heldOwner(HWND window, DWORD pid) {
|
||||
// foreground is. Windows grants the foreground only to a process that received
|
||||
// the last input, so an input with no effect is injected first.
|
||||
void front(DWORD pid) {
|
||||
if (isFront(pid)) return;
|
||||
if (isFront(pid) && ownsForeground(aimed)) return;
|
||||
std::vector<HWND> wins = appWindows(pid, true);
|
||||
if (wins.empty()) throw Failure{"computer.no_window", "the application has no window to bring to the front"};
|
||||
HWND target = wins[0];
|
||||
@@ -233,8 +257,8 @@ void front(DWORD pid) {
|
||||
nudge.mi.dwFlags = MOUSEEVENTF_MOVE;
|
||||
SendInput(1, &nudge, sizeof nudge);
|
||||
SetForegroundWindow(target);
|
||||
for (int i = 0; i < 25 && !isFront(pid); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
|
||||
if (!isFront(pid)) {
|
||||
for (int i = 0; i < 25 && !(isFront(pid) && ownsForeground(aimed)); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
|
||||
if (!(isFront(pid) && ownsForeground(aimed))) {
|
||||
DWORD fgThread = GetWindowThreadProcessId(GetForegroundWindow(), nullptr);
|
||||
DWORD me = GetCurrentThreadId();
|
||||
if (fgThread && fgThread != me && AttachThreadInput(me, fgThread, TRUE)) {
|
||||
@@ -242,9 +266,9 @@ void front(DWORD pid) {
|
||||
SetForegroundWindow(target);
|
||||
AttachThreadInput(me, fgThread, FALSE);
|
||||
}
|
||||
for (int i = 0; i < 25 && !isFront(pid); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
|
||||
for (int i = 0; i < 25 && !(isFront(pid) && ownsForeground(aimed)); i++) std::this_thread::sleep_for(std::chrono::milliseconds(20));
|
||||
}
|
||||
if (!isFront(pid)) {
|
||||
if (!(isFront(pid) && ownsForeground(aimed))) {
|
||||
throw Failure{"computer.needs_front",
|
||||
"Windows kept this application from coming to the front; ask the person to click it once, then try again"};
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ This document records the provider-visible contract for Reasonix compile-time bu
|
||||
| `code_index` | true | Lightweight built-in code symbol index. Prefer lsp_* for language semantics and installed code graph MCP tools for call graph, impact, and architecture relationships; use this as the local fallback for file outlines and symbol definition candidates, then verify with read_file or grep. |
|
||||
| `complete_step` | true | Record the evidence-backed completion of ONE step of an approved plan. Call it as you finish each step instead of silently moving on: a completion with no evidence is REJECTED, so don't claim a step is done until you can show why. The host advances the list for you — it marks this step completed and moves the next to in_progress, so a todo_write that marks such an item completed on its own is REJECTED too. Sign off one step per tool-call ROUND, not one per call: the sign-off promotes the next item, and a second sign-off sent beside it signs against the list as it stood before. |
|
||||
| `compress` | true | Compress a selected part of the current model-visible conversation without deleting visible history. Use only when the user explicitly asks for context compression. Choose `before` to summarize everything before the uniquely matched user turn while keeping that turn and later context, or `after` to summarize from that turn through the last completed turn while keeping the active turn. The anchor must be an exact, unique excerpt from a real user message; use a longer excerpt if the tool reports multiple matches. |
|
||||
| `computer_act` | false | Operate another application. Steps run in order and stop at the first failure. Through its accessibility actions, leaving the pointer alone: click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is borrowed and put back), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. Paste needs the application in front on macOS; on Windows type, paste and key bring it forward. scroll brings a ref into view or turns the wheel by amount lines, wait pauses ms. pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, and the pointer goes back where it was. Returns a snapshot afterwards. |
|
||||
| `computer_read` | false | See another application on this computer, named as what=apps lists it. what=apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels, and the person is shown it too). |
|
||||
| `computer_act` | false | Operate another application. Steps run in order and stop at the first failure. Through accessibility actions, pointer untouched: click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is restored), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. Paste needs the application in front on macOS; on Windows type, paste and key bring it forward. scroll brings a ref into view or turns the wheel by amount lines, wait pauses ms. pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, then the pointer is restored. Returns a snapshot afterwards. |
|
||||
| `computer_read` | false | See another application, named as what=apps lists it. apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels; the person sees it too). |
|
||||
| `context_budget` | true | Report how much context room is left before this conversation is automatically compacted. Nothing reports it unprompted: check it at the start of a long task, before a large change, and before output you may not finish reading (a broad search, a large file, a long build log), so you can narrow the command. `tokens_remaining` counts down to the compaction trigger, not the physical window. |
|
||||
| `delete_range` | false | Delete a contiguous text range from a file using exact start/end text anchors. Each anchor must match exactly one line. Returns unified diff on success. Use for large deletions — smaller changes should use edit_file. |
|
||||
| `delete_symbol` | false | Delete a named symbol (function, method, type, interface, const, var) from a Go source file using AST parsing. For non-Go files, use delete_range with manual anchors. |
|
||||
|
||||
@@ -33,9 +33,16 @@ func runFakeComputerHelper(in io.Reader, out io.Writer) {
|
||||
var result any = map[string]any{}
|
||||
switch req.Method {
|
||||
case "apps":
|
||||
bounds := map[string]any{"x": 0, "y": 0, "width": 800, "height": 600}
|
||||
result = map[string]any{"apps": []any{map[string]any{
|
||||
"pid": 42, "bundle": "com.example.Notes", "name": "Notes", "active": true,
|
||||
"windows": []any{map[string]any{"id": 1, "title": "Draft", "bounds": map[string]any{"x": 0, "y": 0, "width": 800, "height": 600}}},
|
||||
"windows": []any{map[string]any{"id": 1, "title": "Draft", "bounds": bounds}},
|
||||
}, map[string]any{
|
||||
"pid": 43, "bundle": "com.example.Pair", "name": "Pair",
|
||||
"windows": []any{
|
||||
map[string]any{"id": 11, "title": "Left", "owned": false, "bounds": bounds},
|
||||
map[string]any{"id": 12, "title": "Right", "owned": false, "bounds": bounds},
|
||||
},
|
||||
}}}
|
||||
case "snapshot":
|
||||
result = map[string]any{
|
||||
@@ -46,6 +53,13 @@ func runFakeComputerHelper(in io.Reader, out io.Writer) {
|
||||
value, _ = req.Params["text"].(string)
|
||||
result = map[string]any{"effect": map[string]any{"class": "confirmed", "evidence": []string{"value_readback"}}}
|
||||
case "type":
|
||||
if req.Params["pid"] == float64(43) {
|
||||
if req.Params["window"] == nil {
|
||||
_ = enc.Encode(map[string]any{"id": req.ID, "error": map[string]any{"code": "computer.failed", "message": "type reached the helper with no window named"}})
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
_ = enc.Encode(map[string]any{"id": req.ID, "error": map[string]any{
|
||||
"code": "computer.blocked", "message": "the keys would land in a dialog whose focused element takes no text",
|
||||
"blocked_by": map[string]any{"ref": "a9", "title": "Unsaved changes", "blocks": "Draft"},
|
||||
@@ -135,3 +149,37 @@ func TestComputerToolIdentityAgreesAcrossTheKernel(t *testing.T) {
|
||||
t.Errorf("operating an application is classified %q, want unknown: it can write where the application may", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Through the real assembly: an application showing two windows is refused
|
||||
// before the helper is asked anything, the candidates reach the model, and
|
||||
// naming one of them lets the same call through.
|
||||
func TestEffectComputerUseRefusesAnAmbiguousWindowThroughTheRealAssembly(t *testing.T) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(fakeComputerHelperEnv, "1")
|
||||
SetComputerHelper(exe)
|
||||
t.Cleanup(func() { SetComputerHelper("") })
|
||||
|
||||
reqs := buildBrowserEffect(t, "", []func(string) *provider.ToolCall{
|
||||
func(string) *provider.ToolCall {
|
||||
return browserCall("act-1", "computer_act", map[string]any{"app": "com.example.Pair", "steps": []any{map[string]any{"action": "type", "text": "hi"}}})
|
||||
},
|
||||
func(string) *provider.ToolCall {
|
||||
return browserCall("act-2", "computer_act", map[string]any{"app": "com.example.Pair", "window": 12, "steps": []any{map[string]any{"action": "type", "text": "hi"}}})
|
||||
},
|
||||
})
|
||||
results := effectToolResults(reqs[len(reqs)-1])
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("tool results = %q", results)
|
||||
}
|
||||
for _, want := range []string{"computer.ambiguous_window_target", `window 11 "Left" (front)`, `window 12 "Right"`, "Completed 0 of 1 step(s)."} {
|
||||
if !strings.Contains(results[0], want) {
|
||||
t.Fatalf("the refusal is missing %q:\n%s", want, results[0])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(results[1], "Completed 1 of 1 step(s).") {
|
||||
t.Fatalf("naming the window did not let the call through:\n%s", results[1])
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
// it fails with an identity, and the failure names the modal from structure.
|
||||
func TestAStepAModalHoldsFailsAndNamesTheModal(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
res, err := s.Act(context.Background(), "com.example.Held", []Step{{Action: "type", Text: "hello"}})
|
||||
res, err := s.Act(context.Background(), "com.example.Held", 0, []Step{{Action: "type", Text: "hello"}})
|
||||
if CodeOf(err) != CodeBlocked || res.Done != 0 || res.FailedAt != 0 {
|
||||
t.Fatalf("typing into a held application = %+v, %v; want %s at step 1 with nothing done", res, err, CodeBlocked)
|
||||
}
|
||||
@@ -32,7 +32,7 @@ func TestAStepAModalHoldsFailsAndNamesTheModal(t *testing.T) {
|
||||
// carries. A step that sends nothing has no effect at all.
|
||||
func TestAnEffectIsOnlyAsStrongAsItsEvidence(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
res, err := s.Act(context.Background(), "com.example.Notes", []Step{
|
||||
res, err := s.Act(context.Background(), "com.example.Notes", 0, []Step{
|
||||
{Action: "set_value", Ref: "a2", Text: "hi"},
|
||||
{Action: "type", Text: "hi"},
|
||||
{Action: "focus", Ref: "a2"},
|
||||
|
||||
@@ -3,6 +3,7 @@ package computer
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Code is the identity of a computer-use failure, shared with the helper that
|
||||
@@ -15,6 +16,7 @@ const (
|
||||
CodePermissionMissing Code = "computer.permission_missing"
|
||||
CodeNoApp Code = "computer.no_app"
|
||||
CodeNoWindow Code = "computer.no_window"
|
||||
CodeAmbiguousWindow Code = "computer.ambiguous_window_target"
|
||||
CodeAppRefused Code = "computer.app_refused"
|
||||
CodeUnknownRef Code = "computer.unknown_ref"
|
||||
CodeStaleRef Code = "computer.stale_ref"
|
||||
@@ -36,17 +38,30 @@ type Failure struct {
|
||||
Detail string
|
||||
// BlockedBy is the modal that would have taken the input instead.
|
||||
BlockedBy *Modal
|
||||
// Candidates are the windows an ambiguous target could have meant.
|
||||
Candidates []Window
|
||||
}
|
||||
|
||||
func (f *Failure) Error() string {
|
||||
s := string(f.Code)
|
||||
var b strings.Builder
|
||||
b.WriteString(string(f.Code))
|
||||
if f.Detail != "" {
|
||||
s += ": " + f.Detail
|
||||
b.WriteString(": " + f.Detail)
|
||||
}
|
||||
if f.BlockedBy != nil {
|
||||
s += "; held by " + f.BlockedBy.String()
|
||||
b.WriteString("; held by " + f.BlockedBy.String())
|
||||
}
|
||||
return s
|
||||
for i, w := range f.Candidates {
|
||||
if i == maxCandidates {
|
||||
fmt.Fprintf(&b, "; and %d more", len(f.Candidates)-i)
|
||||
break
|
||||
}
|
||||
fmt.Fprintf(&b, "; window %d %s", w.ID, ShownName(w.Title))
|
||||
if i == 0 {
|
||||
b.WriteString(" (front)")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Is matches another Failure by code alone.
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestLiveOperatesAnApplicationWithoutItsPointer(t *testing.T) {
|
||||
field := lineRef(t, snap.Lines, `textField "Probe field"`)
|
||||
button := lineRef(t, snap.Lines, `button "Probe button"`)
|
||||
|
||||
res, err := s.Act(ctx, targetBundle, []Step{
|
||||
res, err := s.Act(ctx, targetBundle, 0, []Step{
|
||||
{Action: "set_value", Ref: field, Text: "from-ax"},
|
||||
{Action: "click", Ref: button},
|
||||
{Action: "focus", Ref: field},
|
||||
@@ -87,7 +87,7 @@ func TestLiveOperatesAnApplicationWithoutItsPointer(t *testing.T) {
|
||||
waitLog(t, log, "button pressed")
|
||||
waitLog(t, log, "from-ax 李雷")
|
||||
|
||||
shot, app, err := s.Screenshot(ctx, targetBundle)
|
||||
shot, app, err := s.Screenshot(ctx, targetBundle, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Screenshot: %v", err)
|
||||
}
|
||||
@@ -99,11 +99,11 @@ func TestLiveOperatesAnApplicationWithoutItsPointer(t *testing.T) {
|
||||
geometry := s.shots[targetBundle]
|
||||
s.mu.Unlock()
|
||||
x, y := 345/geometry.scale, (geometry.bounds.Height-162)/geometry.scale
|
||||
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "click", X: &x, Y: &y}}); err != nil {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "click", X: &x, Y: &y}}); err != nil {
|
||||
t.Fatalf("click at a screenshot point: %v", err)
|
||||
}
|
||||
cx, cy := 100/geometry.scale, (geometry.bounds.Height-70)/geometry.scale
|
||||
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "click", X: &cx, Y: &cy}}); CodeOf(err) != CodeNoAction {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "click", X: &cx, Y: &cy}}); CodeOf(err) != CodeNoAction {
|
||||
t.Fatalf("clicking a view with no accessibility action = %v, want %s", err, CodeNoAction)
|
||||
}
|
||||
if _, err := s.Snapshot(ctx, "com.apple.Terminal"); CodeOf(err) != CodeAppRefused {
|
||||
@@ -125,7 +125,7 @@ func TestLiveContextMenuScrollAndWait(t *testing.T) {
|
||||
}
|
||||
t.Logf("snapshot:\n%s", strings.Join(snap.Lines, "\n"))
|
||||
menu := lineRef(t, snap.Lines, `"Probe menu area"`)
|
||||
res, err := s.Act(ctx, targetBundle, []Step{
|
||||
res, err := s.Act(ctx, targetBundle, 0, []Step{
|
||||
{Action: "right_click", Ref: menu},
|
||||
{Action: "wait", Ms: 400},
|
||||
})
|
||||
@@ -151,7 +151,7 @@ func TestLiveContextMenuScrollAndWait(t *testing.T) {
|
||||
}
|
||||
before := position()
|
||||
deep := lineRef(t, snap.Lines, `"Deep row"`)
|
||||
res, err = s.Act(ctx, targetBundle, []Step{{Action: "scroll", Ref: deep}})
|
||||
res, err = s.Act(ctx, targetBundle, 0, []Step{{Action: "scroll", Ref: deep}})
|
||||
if err != nil || len(res.Steps) == 0 || !strings.Contains(res.Steps[0].Note, "into view") {
|
||||
t.Fatalf("scroll to a ref: %v %v", err, res.Steps)
|
||||
}
|
||||
@@ -159,10 +159,10 @@ func TestLiveContextMenuScrollAndWait(t *testing.T) {
|
||||
t.Fatalf("the scroll area did not move: %q", after)
|
||||
}
|
||||
|
||||
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "scroll", Amount: -3}}); err != nil {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "scroll", Amount: -3}}); err != nil {
|
||||
t.Fatalf("scroll by lines: %v", err)
|
||||
}
|
||||
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
|
||||
t.Fatalf("a right_click with no ref = %v, want %s", err, CodeBadStep)
|
||||
}
|
||||
}
|
||||
@@ -191,22 +191,22 @@ func TestLivePasteLeavesTheClipboardAsItFoundIt(t *testing.T) {
|
||||
paste := []Step{{Action: "focus", Ref: field}, {Action: "paste", Text: pasted}}
|
||||
// Behind, the application never handles the keystroke, and this says so
|
||||
// rather than reporting a paste that put nothing anywhere.
|
||||
if _, err := s.Act(ctx, targetBundle, paste); CodeOf(err) != CodeNeedsFront {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, paste); CodeOf(err) != CodeNeedsFront {
|
||||
t.Fatalf("pasting into an application that is behind = %v, want %s", err, CodeNeedsFront)
|
||||
}
|
||||
|
||||
// Bringing it forward is a pointer step, which is answered for on its own.
|
||||
if _, _, err := s.Screenshot(ctx, targetBundle); err != nil {
|
||||
if _, _, err := s.Screenshot(ctx, targetBundle, 0); err != nil {
|
||||
t.Fatalf("Screenshot: %v", err)
|
||||
}
|
||||
s.mu.Lock()
|
||||
geometry := s.shots[targetBundle]
|
||||
s.mu.Unlock()
|
||||
x, y := 345/geometry.scale, (geometry.bounds.Height-162)/geometry.scale
|
||||
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
|
||||
t.Fatalf("bring it forward: %v", err)
|
||||
}
|
||||
if _, err := s.Act(ctx, targetBundle, paste); err != nil {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, paste); err != nil {
|
||||
t.Fatalf("paste: %v", err)
|
||||
}
|
||||
waitLog(t, log, "text "+pasted)
|
||||
@@ -237,7 +237,7 @@ func TestLiveThePointerReachesWhatAccessibilityCannot(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
shot, app, err := s.Screenshot(ctx, targetBundle)
|
||||
shot, app, err := s.Screenshot(ctx, targetBundle, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Screenshot: %v", err)
|
||||
}
|
||||
@@ -251,22 +251,22 @@ func TestLiveThePointerReachesWhatAccessibilityCannot(t *testing.T) {
|
||||
// the screenshot the model was shown.
|
||||
x, y := 100/geometry.scale, (geometry.bounds.Height-70)/geometry.scale
|
||||
|
||||
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "click", X: &x, Y: &y}}); CodeOf(err) != CodeNoAction {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "click", X: &x, Y: &y}}); CodeOf(err) != CodeNoAction {
|
||||
t.Fatalf("an accessibility click on a drawn view = %v, want %s", err, CodeNoAction)
|
||||
}
|
||||
|
||||
// Where the person's pointer is, in the pixels of the screenshot the model
|
||||
// was shown, before and after the agent borrows it.
|
||||
home, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_position"}})
|
||||
home, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_position"}})
|
||||
if err != nil {
|
||||
t.Fatalf("pointer_position: %v", err)
|
||||
}
|
||||
res, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_click", X: &x, Y: &y}})
|
||||
res, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_click", X: &x, Y: &y}})
|
||||
if err != nil {
|
||||
t.Fatalf("pointer_click: %v (%v)", err, res.Steps)
|
||||
}
|
||||
waitLog(t, log, "mouseDown")
|
||||
back, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_position"}})
|
||||
back, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_position"}})
|
||||
if err != nil {
|
||||
t.Fatalf("pointer_position: %v", err)
|
||||
}
|
||||
@@ -275,7 +275,7 @@ func TestLiveThePointerReachesWhatAccessibilityCannot(t *testing.T) {
|
||||
}
|
||||
|
||||
toX, toY := x+40, y-20
|
||||
if _, err := s.Act(ctx, targetBundle, []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}}); err != nil {
|
||||
if _, err := s.Act(ctx, targetBundle, 0, []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}}); err != nil {
|
||||
t.Fatalf("pointer_drag: %v", err)
|
||||
}
|
||||
waitLog(t, log, "mouseDragged")
|
||||
|
||||
@@ -93,7 +93,7 @@ func TestLiveWindowsOperatesAnApplication(t *testing.T) {
|
||||
field := lineRef(t, snap.Lines, `edit "Probe field"`)
|
||||
button := lineRef(t, snap.Lines, `button "Probe button"`)
|
||||
|
||||
res, err := s.Act(ctx, windowsTarget, []Step{
|
||||
res, err := s.Act(ctx, windowsTarget, 0, []Step{
|
||||
{Action: "set_value", Ref: field, Text: "from-ax"},
|
||||
{Action: "click", Ref: button},
|
||||
{Action: "focus", Ref: field},
|
||||
@@ -112,7 +112,7 @@ func TestLiveWindowsOperatesAnApplication(t *testing.T) {
|
||||
waitLog(t, p.log, "button pressed 1")
|
||||
waitLog(t, p.log, "text from-ax 李雷3\ntext \n")
|
||||
|
||||
shot, app, err := s.Screenshot(ctx, windowsTarget)
|
||||
shot, app, err := s.Screenshot(ctx, windowsTarget, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Screenshot: %v", err)
|
||||
}
|
||||
@@ -120,18 +120,18 @@ func TestLiveWindowsOperatesAnApplication(t *testing.T) {
|
||||
t.Fatalf("screenshot = %.40q for %+v", shot, app)
|
||||
}
|
||||
x, y := inShot(s, p.button)
|
||||
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "click", X: x, Y: y}}); err != nil {
|
||||
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "click", X: x, Y: y}}); err != nil {
|
||||
t.Fatalf("click at a screenshot point: %v", err)
|
||||
}
|
||||
waitLog(t, p.log, "button pressed 2")
|
||||
cx, cy := inShot(s, p.canvas)
|
||||
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "click", X: cx, Y: cy}}); CodeOf(err) != CodeNoAction {
|
||||
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "click", X: cx, Y: cy}}); CodeOf(err) != CodeNoAction {
|
||||
t.Fatalf("clicking a window with no accessibility action = %v, want %s", err, CodeNoAction)
|
||||
}
|
||||
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "key", Key: "meta+s"}}); CodeOf(err) != CodeBadStep {
|
||||
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "key", Key: "meta+s"}}); CodeOf(err) != CodeBadStep {
|
||||
t.Fatalf("the Windows key = %v, want %s", err, CodeBadStep)
|
||||
}
|
||||
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "scroll", Amount: -3}}); err != nil {
|
||||
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "scroll", Amount: -3}}); err != nil {
|
||||
t.Fatalf("scroll by lines: %v", err)
|
||||
}
|
||||
for _, name := range []string{"cmd.exe", "PowerShell.exe"} {
|
||||
@@ -168,7 +168,7 @@ func TestLiveWindowsPasteGivesTheClipboardBack(t *testing.T) {
|
||||
}
|
||||
field := lineRef(t, snap.Lines, `edit "Probe field"`)
|
||||
pasted := "粘贴 without typing"
|
||||
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "focus", Ref: field}, {Action: "paste", Text: pasted}}); err != nil {
|
||||
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "focus", Ref: field}, {Action: "paste", Text: pasted}}); err != nil {
|
||||
t.Fatalf("paste: %v", err)
|
||||
}
|
||||
waitLog(t, p.log, "text "+pasted)
|
||||
@@ -192,19 +192,19 @@ func TestLiveWindowsThePointerReachesWhatAccessibilityCannot(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if _, _, err := s.Screenshot(ctx, windowsTarget); err != nil {
|
||||
if _, _, err := s.Screenshot(ctx, windowsTarget, 0); err != nil {
|
||||
t.Fatalf("Screenshot: %v", err)
|
||||
}
|
||||
x, y := inShot(s, p.canvas)
|
||||
home, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_position"}})
|
||||
home, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_position"}})
|
||||
if err != nil {
|
||||
t.Fatalf("pointer_position: %v", err)
|
||||
}
|
||||
if res, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_click", X: x, Y: y}}); err != nil {
|
||||
if res, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_click", X: x, Y: y}}); err != nil {
|
||||
t.Fatalf("pointer_click: %v (%v)", err, res.Steps)
|
||||
}
|
||||
waitLog(t, p.log, "mouseDown")
|
||||
back, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_position"}})
|
||||
back, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_position"}})
|
||||
if err != nil {
|
||||
t.Fatalf("pointer_position: %v", err)
|
||||
}
|
||||
@@ -212,7 +212,7 @@ func TestLiveWindowsThePointerReachesWhatAccessibilityCannot(t *testing.T) {
|
||||
t.Fatalf("the pointer was left at %q, not where the person had it (%q)", back.Steps[0].Note, home.Steps[0].Note)
|
||||
}
|
||||
toX, toY := *x+40, *y-20
|
||||
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "pointer_drag", X: x, Y: y, ToX: &toX, ToY: &toY}}); err != nil {
|
||||
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "pointer_drag", X: x, Y: y, ToX: &toX, ToY: &toY}}); err != nil {
|
||||
t.Fatalf("pointer_drag: %v", err)
|
||||
}
|
||||
waitLog(t, p.log, "mouseDragged")
|
||||
@@ -241,7 +241,7 @@ func heldSnapshot(t *testing.T, ctx context.Context, s *Session) Snapshot {
|
||||
func actor(t *testing.T, ctx context.Context, s *Session) func(...Step) Effect {
|
||||
return func(steps ...Step) Effect {
|
||||
t.Helper()
|
||||
res, err := s.Act(ctx, windowsTarget, steps)
|
||||
res, err := s.Act(ctx, windowsTarget, 0, steps)
|
||||
if err != nil || len(res.Steps) == 0 {
|
||||
t.Fatalf("Act = %+v, %v", res, err)
|
||||
}
|
||||
@@ -264,14 +264,14 @@ func TestLiveWindowsAModalHoldsTheInput(t *testing.T) {
|
||||
t.Fatalf("modal = %+v", m)
|
||||
}
|
||||
button := lineRef(t, snap.Lines, `button "Probe button"`)
|
||||
res, err := s.Act(ctx, windowsTarget, []Step{{Action: "type", Text: "hello"}})
|
||||
res, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "type", Text: "hello"}})
|
||||
if CodeOf(err) != CodeBlocked || res.Done != 0 {
|
||||
t.Fatalf("typing while a message box holds the input = %+v, %v; want %s", res, err, CodeBlocked)
|
||||
}
|
||||
if f, _ := errors.AsType[*Failure](err); f.BlockedBy == nil || f.BlockedBy.Title != "Probe modal" {
|
||||
t.Fatalf("the failure does not name the modal: %v", err)
|
||||
}
|
||||
if _, err := s.Act(ctx, windowsTarget, []Step{{Action: "click", Ref: button}}); CodeOf(err) != CodeBlocked {
|
||||
if _, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "click", Ref: button}}); CodeOf(err) != CodeBlocked {
|
||||
t.Fatalf("pressing a button behind the modal = %v, want %s", err, CodeBlocked)
|
||||
}
|
||||
if e := act(Step{Action: "key", Key: "escape"}); e.Class != EffectUnverifiable || e.BlockedBy == nil {
|
||||
@@ -318,3 +318,76 @@ func TestLiveWindowsTypingIntoAModalsFieldIsReadBack(t *testing.T) {
|
||||
}
|
||||
waitLog(t, p.log, "form text xyz")
|
||||
}
|
||||
|
||||
// Two top-level windows of one application leave the target ambiguous: input
|
||||
// is refused before it is sent, and naming a window aims it there.
|
||||
func TestLiveWindowsTwoWindowsAreRefusedUntilOneIsNamed(t *testing.T) {
|
||||
s := liveSession(t)
|
||||
p := launchWindowsTarget(t, "second")
|
||||
waitLog(t, p.log, "second open")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
apps, err := s.Apps(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var app App
|
||||
for _, a := range apps {
|
||||
if a.Bundle == windowsTarget {
|
||||
app = a
|
||||
}
|
||||
}
|
||||
main, second := 0, 0
|
||||
for _, w := range app.Windows {
|
||||
switch w.Title {
|
||||
case "Computer Target":
|
||||
main = w.ID
|
||||
case "Second Window":
|
||||
second = w.ID
|
||||
}
|
||||
if w.Owned == nil || *w.Owned {
|
||||
t.Fatalf("window %+v is not reported as an independent window", w)
|
||||
}
|
||||
}
|
||||
if main == 0 || second == 0 {
|
||||
t.Fatalf("windows = %+v", app.Windows)
|
||||
}
|
||||
|
||||
res, err := s.Act(ctx, windowsTarget, 0, []Step{{Action: "type", Text: "hello"}})
|
||||
f, _ := errors.AsType[*Failure](err)
|
||||
if f == nil || f.Code != CodeAmbiguousWindow || res.Done != 0 || len(f.Candidates) != 2 {
|
||||
t.Fatalf("typing at two windows = %+v, %v; want %s with both candidates", res, err, CodeAmbiguousWindow)
|
||||
}
|
||||
if _, _, err := s.Screenshot(ctx, windowsTarget, 0); CodeOf(err) != CodeAmbiguousWindow {
|
||||
t.Fatalf("a screenshot of two windows = %v, want %s", err, CodeAmbiguousWindow)
|
||||
}
|
||||
if body, _ := os.ReadFile(p.log); strings.Contains(string(body), "hello") {
|
||||
t.Fatalf("input reached the application despite the refusal:\n%s", body)
|
||||
}
|
||||
|
||||
if _, _, err := s.Screenshot(ctx, windowsTarget, second); err != nil {
|
||||
t.Fatalf("screenshot of the second window: %v", err)
|
||||
}
|
||||
s.mu.Lock()
|
||||
shot := s.shots[windowsTarget]
|
||||
s.mu.Unlock()
|
||||
if shot.window != second || shot.bounds.Width < 250 || shot.bounds.Width > 400 {
|
||||
t.Fatalf("the capture is %+v, want the 300-wide second window %d", shot, second)
|
||||
}
|
||||
snap, err := s.Snapshot(ctx, windowsTarget)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
field := lineRef(t, snap.Lines, `edit "Probe field"`)
|
||||
if _, err := s.Act(ctx, windowsTarget, second, []Step{{Action: "key", Key: "home"}}); err != nil {
|
||||
t.Fatalf("raising the second window: %v", err)
|
||||
}
|
||||
if res, err := s.Act(ctx, windowsTarget, main, []Step{{Action: "focus", Ref: field}, {Action: "type", Text: "hello"}}); err != nil || res.Done != 2 {
|
||||
t.Fatalf("typing into the main window = %+v, %v", res, err)
|
||||
}
|
||||
waitLog(t, p.log, "hello")
|
||||
if _, err := s.Act(ctx, windowsTarget, 1, []Step{{Action: "type", Text: "x"}}); CodeOf(err) != CodeNoWindow {
|
||||
t.Fatalf("a window the application does not have = %v, want %s", err, CodeNoWindow)
|
||||
}
|
||||
}
|
||||
@@ -102,6 +102,9 @@ type Window struct {
|
||||
ID int `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Bounds Rect `json:"bounds"`
|
||||
// Owned is true for a dialog or tool window another window owns. Nil means
|
||||
// the helper does not classify windows, which leaves a target unguarded.
|
||||
Owned *bool `json:"owned,omitempty"`
|
||||
}
|
||||
|
||||
// Rect is a rectangle in global points, origin top-left.
|
||||
@@ -122,6 +125,7 @@ type Session struct {
|
||||
}
|
||||
|
||||
type shotGeometry struct {
|
||||
window int
|
||||
bounds Rect
|
||||
scale float64 // screen points per pixel of the image the model was shown
|
||||
}
|
||||
@@ -196,19 +200,26 @@ func (s *Session) Snapshot(ctx context.Context, bundle string) (Snapshot, error)
|
||||
|
||||
// Screenshot captures an application's front window, sized for a vision model.
|
||||
// x and y in later steps are read in this image's pixels.
|
||||
func (s *Session) Screenshot(ctx context.Context, bundle string) (string, App, error) {
|
||||
func (s *Session) Screenshot(ctx context.Context, bundle string, window int) (string, App, error) {
|
||||
app, err := s.App(ctx, bundle)
|
||||
if err != nil {
|
||||
return "", App{}, err
|
||||
}
|
||||
if err := app.target(window, true); err != nil {
|
||||
return "", App{}, err
|
||||
}
|
||||
var r struct {
|
||||
Data string `json:"data"`
|
||||
Mime string `json:"mime"`
|
||||
Window int `json:"window"`
|
||||
Bounds Rect `json:"bounds"`
|
||||
}
|
||||
if err := s.call(ctx, "screenshot", map[string]any{"pid": app.PID}, &r); err != nil {
|
||||
if err := s.call(ctx, "screenshot", withWindow(map[string]any{"pid": app.PID}, window), &r); err != nil {
|
||||
return "", App{}, err
|
||||
}
|
||||
if window != 0 && r.Window != window {
|
||||
return "", App{}, fail(CodeNoWindow, "window %d could not be captured; the capture is of window %d", window, r.Window)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(r.Data)
|
||||
if err != nil {
|
||||
return "", App{}, fail(CodeCaptureFailed, "the capture was not valid base64")
|
||||
@@ -222,7 +233,7 @@ func (s *Session) Screenshot(ctx context.Context, bundle string) (string, App, e
|
||||
return "", App{}, fail(CodeCaptureFailed, "the capture could not be measured")
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.shots[app.Bundle] = shotGeometry{bounds: r.Bounds, scale: r.Bounds.Width / float64(cfg.Width)}
|
||||
s.shots[app.Bundle] = shotGeometry{window: r.Window, bounds: r.Bounds, scale: r.Bounds.Width / float64(cfg.Width)}
|
||||
s.mu.Unlock()
|
||||
return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(fitted), app, nil
|
||||
}
|
||||
@@ -282,7 +293,7 @@ type StepResult struct {
|
||||
|
||||
// Act runs steps in order on one application and stops at the first that
|
||||
// fails, or when the person presses Escape while the agent's cursor is showing.
|
||||
func (s *Session) Act(ctx context.Context, bundle string, steps []Step) (ActResult, error) {
|
||||
func (s *Session) Act(ctx context.Context, bundle string, window int, steps []Step) (ActResult, error) {
|
||||
res := ActResult{FailedAt: -1}
|
||||
app, err := s.App(ctx, bundle)
|
||||
if err != nil {
|
||||
@@ -290,6 +301,10 @@ func (s *Session) Act(ctx context.Context, bundle string, steps []Step) (ActResu
|
||||
return res, err
|
||||
}
|
||||
res.App = app
|
||||
if err := app.target(window, needsWindow(steps)); err != nil {
|
||||
res.FailedAt = 0
|
||||
return res, err
|
||||
}
|
||||
// The pointer goes back where the person left it when the steps are done,
|
||||
// whether they finished or stopped.
|
||||
if PointerSteps(steps) {
|
||||
@@ -305,7 +320,7 @@ func (s *Session) Act(ctx context.Context, bundle string, steps []Step) (ActResu
|
||||
res.FailedAt = i
|
||||
return res, fail(CodeStopped, "the person pressed Escape; ask before continuing")
|
||||
}
|
||||
done, err := s.step(ctx, app, step)
|
||||
done, err := s.step(ctx, app, window, step)
|
||||
if err != nil {
|
||||
res.FailedAt = i
|
||||
return res, err
|
||||
@@ -335,36 +350,39 @@ func (s *Session) deliver(ctx context.Context, method string, params map[string]
|
||||
|
||||
func noted(note string) func(actReply) string { return func(actReply) string { return note } }
|
||||
|
||||
func (s *Session) step(ctx context.Context, app App, step Step) (StepResult, error) {
|
||||
func (s *Session) step(ctx context.Context, app App, window int, step Step) (StepResult, error) {
|
||||
pid := app.PID
|
||||
deliver := func(method string, params map[string]any, note func(actReply) string) (StepResult, error) {
|
||||
return s.deliver(ctx, method, withWindow(params, window), note)
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(step.Action)) {
|
||||
case "click":
|
||||
if step.Ref != "" {
|
||||
return s.deliver(ctx, "press", map[string]any{"pid": pid, "ref": step.Ref}, noted("press "+step.Ref))
|
||||
return deliver("press", map[string]any{"pid": pid, "ref": step.Ref}, noted("press "+step.Ref))
|
||||
}
|
||||
x, y, err := s.screenPoint(app, step)
|
||||
x, y, err := s.screenPoint(app, window, step)
|
||||
if err != nil {
|
||||
return StepResult{}, err
|
||||
}
|
||||
return s.deliver(ctx, "click", map[string]any{"pid": pid, "x": x, "y": y}, func(r actReply) string {
|
||||
return deliver("click", map[string]any{"pid": pid, "x": x, "y": y}, func(r actReply) string {
|
||||
return fmt.Sprintf("click the %s at (%v,%v)", r.Role, *step.X, *step.Y)
|
||||
})
|
||||
case "focus":
|
||||
return s.deliver(ctx, "focus", map[string]any{"pid": pid, "ref": step.Ref}, noted("focus "+step.Ref))
|
||||
return deliver("focus", map[string]any{"pid": pid, "ref": step.Ref}, noted("focus "+step.Ref))
|
||||
case "set_value":
|
||||
return s.deliver(ctx, "set_value", map[string]any{"pid": pid, "ref": step.Ref, "text": step.Text},
|
||||
return deliver("set_value", map[string]any{"pid": pid, "ref": step.Ref, "text": step.Text},
|
||||
noted(fmt.Sprintf("set %s to %d characters", step.Ref, len([]rune(step.Text)))))
|
||||
case "type":
|
||||
return s.deliver(ctx, "type", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("type %d characters", len([]rune(step.Text)))))
|
||||
return deliver("type", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("type %d characters", len([]rune(step.Text)))))
|
||||
case "key":
|
||||
return s.deliver(ctx, "key", map[string]any{"pid": pid, "key": step.Key, "times": max(step.Times, 1)}, noted("press "+step.Key))
|
||||
return deliver("key", map[string]any{"pid": pid, "key": step.Key, "times": max(step.Times, 1)}, noted("press "+step.Key))
|
||||
case "paste":
|
||||
return s.deliver(ctx, "paste", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("paste %d characters", len([]rune(step.Text)))))
|
||||
return deliver("paste", map[string]any{"pid": pid, "text": step.Text}, noted(fmt.Sprintf("paste %d characters", len([]rune(step.Text)))))
|
||||
case "hold_key":
|
||||
return s.deliver(ctx, "hold_key", map[string]any{"pid": pid, "key": step.Key, "seconds": step.Seconds},
|
||||
return deliver("hold_key", map[string]any{"pid": pid, "key": step.Key, "seconds": step.Seconds},
|
||||
noted(fmt.Sprintf("hold %s for %vs", step.Key, step.Seconds)))
|
||||
case "pointer_move", "pointer_click", "pointer_drag":
|
||||
return s.pointerStep(ctx, app, step)
|
||||
return s.pointerStep(ctx, app, window, step)
|
||||
case "pointer_position":
|
||||
var at struct{ X, Y float64 }
|
||||
if err := s.call(ctx, "pointer_position", nil, &at); err != nil {
|
||||
@@ -375,9 +393,9 @@ func (s *Session) step(ctx context.Context, app App, step Step) (StepResult, err
|
||||
if step.Ref == "" {
|
||||
return StepResult{}, fail(CodeBadStep, "a right_click needs a ref; the menu belongs to the element")
|
||||
}
|
||||
return s.deliver(ctx, "menu", map[string]any{"pid": pid, "ref": step.Ref}, noted("open the menu of "+step.Ref))
|
||||
return deliver("menu", map[string]any{"pid": pid, "ref": step.Ref}, noted("open the menu of "+step.Ref))
|
||||
case "scroll":
|
||||
return s.deliver(ctx, "scroll", map[string]any{"pid": pid, "ref": step.Ref, "amount": step.Amount}, func(r actReply) string {
|
||||
return deliver("scroll", map[string]any{"pid": pid, "ref": step.Ref, "amount": step.Amount}, func(r actReply) string {
|
||||
if r.How == "revealed" {
|
||||
return "bring " + step.Ref + " into view"
|
||||
}
|
||||
@@ -398,12 +416,12 @@ func (s *Session) step(ctx context.Context, app App, step Step) (StepResult, err
|
||||
// pointerStep takes the person's pointer to the point a screenshot named. The
|
||||
// helper refuses a point whose window belongs to another application, so an
|
||||
// approval for one application cannot reach into the next.
|
||||
func (s *Session) pointerStep(ctx context.Context, app App, step Step) (StepResult, error) {
|
||||
x, y, err := s.screenPoint(app, step)
|
||||
func (s *Session) pointerStep(ctx context.Context, app App, window int, step Step) (StepResult, error) {
|
||||
x, y, err := s.screenPoint(app, window, step)
|
||||
if err != nil {
|
||||
return StepResult{}, err
|
||||
}
|
||||
args := map[string]any{"pid": app.PID, "x": x, "y": y}
|
||||
args := withWindow(map[string]any{"pid": app.PID, "x": x, "y": y}, window)
|
||||
switch strings.ToLower(strings.TrimSpace(step.Action)) {
|
||||
case "pointer_move":
|
||||
return s.deliver(ctx, "pointer_move", args, noted(fmt.Sprintf("move the pointer to (%v,%v)", *step.X, *step.Y)))
|
||||
@@ -412,7 +430,7 @@ func (s *Session) pointerStep(ctx context.Context, app App, step Step) (StepResu
|
||||
if to.X == nil || to.Y == nil {
|
||||
return StepResult{}, fail(CodeBadStep, "a pointer_drag needs where it ends: to_x and to_y")
|
||||
}
|
||||
toX, toY, err := s.screenPoint(app, to)
|
||||
toX, toY, err := s.screenPoint(app, window, to)
|
||||
if err != nil {
|
||||
return StepResult{}, err
|
||||
}
|
||||
@@ -442,7 +460,7 @@ func (s *Session) describePoint(app App, x, y float64) string {
|
||||
|
||||
// screenPoint converts a point in the latest screenshot's pixels to global
|
||||
// screen points.
|
||||
func (s *Session) screenPoint(app App, step Step) (float64, float64, error) {
|
||||
func (s *Session) screenPoint(app App, window int, step Step) (float64, float64, error) {
|
||||
if step.X == nil || step.Y == nil {
|
||||
return 0, 0, fail(CodeBadStep, "a click needs a ref, or x and y from a screenshot")
|
||||
}
|
||||
@@ -452,6 +470,9 @@ func (s *Session) screenPoint(app App, step Step) (float64, float64, error) {
|
||||
if !ok {
|
||||
return 0, 0, fail(CodeNeedsScreenshot, "take a screenshot of %s first; x and y are read in its pixels", app.Bundle)
|
||||
}
|
||||
if window != 0 && shot.window != window {
|
||||
return 0, 0, fail(CodeNeedsScreenshot, "the latest screenshot of %s is of another window; take one of window %d first", app.Bundle, window)
|
||||
}
|
||||
return shot.bounds.X + *step.X*shot.scale, shot.bounds.Y + *step.Y*shot.scale, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,8 @@ func runFakeHelper(in io.Reader, out io.Writer) {
|
||||
var lastCall map[string]any
|
||||
released := 0
|
||||
asked := 0
|
||||
delivered := 0
|
||||
var lastSent map[string]any
|
||||
sc := bufio.NewScanner(in)
|
||||
for sc.Scan() {
|
||||
var req struct {
|
||||
@@ -60,13 +62,25 @@ func runFakeHelper(in io.Reader, out io.Writer) {
|
||||
delete(reply, "result")
|
||||
reply["error"] = map[string]any{"code": code, "message": message}
|
||||
}
|
||||
if m := req.Method; m != "apps" && m != "request_permission" && !strings.HasPrefix(m, "_") {
|
||||
delivered++
|
||||
lastSent = map[string]any{"method": m, "params": req.Params}
|
||||
}
|
||||
switch req.Method {
|
||||
case "_delivered":
|
||||
reply["result"] = map[string]any{"count": delivered}
|
||||
case "_last_sent":
|
||||
reply["result"] = lastSent
|
||||
case "apps":
|
||||
reply["result"] = map[string]any{"apps": []any{
|
||||
map[string]any{"pid": 7, "bundle": "com.example.Notes", "name": "Notes", "windows": []any{}},
|
||||
map[string]any{"pid": 8, "bundle": "com.example.Locked", "name": "Locked", "windows": []any{}},
|
||||
map[string]any{"pid": 9, "bundle": "Vendor.Shell_abc123", "exe": "PWSH.exe", "name": "Shell", "windows": []any{}},
|
||||
map[string]any{"pid": 10, "bundle": "com.example.Held", "name": "Held", "windows": []any{}},
|
||||
map[string]any{"pid": 11, "bundle": "com.example.Two", "name": "Two", "windows": []any{fakeWindow(101, "Main", false), fakeWindow(102, "Other", false)}},
|
||||
map[string]any{"pid": 12, "bundle": "com.example.Dialog", "name": "Dialog", "windows": []any{fakeWindow(202, "Save as", true), fakeWindow(201, "Main", false)}},
|
||||
map[string]any{"pid": 14, "bundle": "com.example.Drift", "name": "Drift", "windows": []any{fakeWindow(401, "A", false), fakeWindow(402, "B", false)}},
|
||||
map[string]any{"pid": 13, "bundle": "com.example.Flat", "name": "Flat", "windows": []any{map[string]any{"id": 301, "title": "A"}, map[string]any{"id": 302, "title": "A"}}},
|
||||
}}
|
||||
case "snapshot":
|
||||
if req.Params["pid"] == float64(8) {
|
||||
@@ -85,10 +99,17 @@ func runFakeHelper(in io.Reader, out io.Writer) {
|
||||
case "request_permission":
|
||||
asked++
|
||||
case "screenshot":
|
||||
reply["result"] = map[string]any{
|
||||
shot := map[string]any{
|
||||
"data": base64.StdEncoding.EncodeToString(fakeScreenshot()), "mime": "image/png",
|
||||
"bounds": map[string]any{"x": 100, "y": 50, "width": 2000, "height": 1000},
|
||||
}
|
||||
if w, ok := req.Params["window"]; ok {
|
||||
shot["window"] = w
|
||||
}
|
||||
if req.Params["pid"] == float64(14) {
|
||||
shot["window"] = 401
|
||||
}
|
||||
reply["result"] = shot
|
||||
case "click":
|
||||
lastClick = req.Params
|
||||
reply["result"] = map[string]any{"role": "button"}
|
||||
@@ -154,7 +175,7 @@ func fakeSession(t *testing.T) *Session {
|
||||
func TestARefusedApplicationIsRefusedBeforeTheHelperIsAsked(t *testing.T) {
|
||||
s := NewSession(NewHelper("/nonexistent/helper"))
|
||||
for _, bundle := range []string{"com.apple.Terminal", " io.reasonix.studio ", "com.apple.systempreferences"} {
|
||||
if _, err := s.Act(context.Background(), bundle, []Step{{Action: "key", Key: "Enter"}}); CodeOf(err) != CodeAppRefused {
|
||||
if _, err := s.Act(context.Background(), bundle, 0, []Step{{Action: "key", Key: "Enter"}}); CodeOf(err) != CodeAppRefused {
|
||||
t.Errorf("%q = %v, want %s", bundle, err, CodeAppRefused)
|
||||
}
|
||||
}
|
||||
@@ -177,10 +198,10 @@ func TestAClickAtAPointIsReadInTheLatestScreenshotsPixels(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
x, y := 100.0, 40.0
|
||||
click := []Step{{Action: "click", X: &x, Y: &y}}
|
||||
if _, err := s.Act(ctx, "com.example.Notes", click); CodeOf(err) != CodeNeedsScreenshot {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, click); CodeOf(err) != CodeNeedsScreenshot {
|
||||
t.Fatalf("a click before any screenshot = %v, want %s", err, CodeNeedsScreenshot)
|
||||
}
|
||||
shot, app, err := s.Screenshot(ctx, "com.example.Notes")
|
||||
shot, app, err := s.Screenshot(ctx, "com.example.Notes", 0)
|
||||
if err != nil || app.PID != 7 {
|
||||
t.Fatalf("Screenshot = %v for %+v", err, app)
|
||||
}
|
||||
@@ -192,7 +213,7 @@ func TestAClickAtAPointIsReadInTheLatestScreenshotsPixels(t *testing.T) {
|
||||
if _, _, err := visionimage.Fit(raw, "image/png"); err != nil || fitted.Width >= 4000 {
|
||||
t.Fatalf("the screenshot was not fitted for a vision model: %dpx, %v", fitted.Width, err)
|
||||
}
|
||||
res, err := s.Act(ctx, "com.example.Notes", click)
|
||||
res, err := s.Act(ctx, "com.example.Notes", 0, click)
|
||||
if err != nil || res.Steps[0].Note != "click the button at (100,40)" {
|
||||
t.Fatalf("click = %+v, %v", res, err)
|
||||
}
|
||||
@@ -224,18 +245,18 @@ func TestAMissingPermissionIsRequestedOnceAndSaysWhatThePersonDoes(t *testing.T)
|
||||
func TestStepsStopAtTheFirstFailureAndWhenThePersonPressesEscape(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Enter"}, {Action: "click", Ref: "a9"}, {Action: "key", Key: "Enter"}})
|
||||
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Enter"}, {Action: "click", Ref: "a9"}, {Action: "key", Key: "Enter"}})
|
||||
if CodeOf(err) != CodeStaleRef || res.Done != 1 || res.FailedAt != 1 {
|
||||
t.Fatalf("a stale ref = %+v, %v", res, err)
|
||||
}
|
||||
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Escape"}, {Action: "key", Key: "Enter"}})
|
||||
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Escape"}, {Action: "key", Key: "Enter"}})
|
||||
if CodeOf(err) != CodeStopped || res.Done != 1 || res.FailedAt != 1 {
|
||||
t.Fatalf("Escape = %+v, %v", res, err)
|
||||
}
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "drag"}}); CodeOf(err) != CodeBadStep {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "drag"}}); CodeOf(err) != CodeBadStep {
|
||||
t.Fatalf("an unknown action = %v, want %s", err, CodeBadStep)
|
||||
}
|
||||
if _, err := s.Act(ctx, "com.example.Missing", nil); CodeOf(err) != CodeNoApp {
|
||||
if _, err := s.Act(ctx, "com.example.Missing", 0, nil); CodeOf(err) != CodeNoApp {
|
||||
t.Fatalf("an application that is not running = %v, want %s", err, CodeNoApp)
|
||||
}
|
||||
}
|
||||
@@ -252,7 +273,7 @@ func TestAHelperThatExitsFailsWhatWasPendingAndStartsAgain(t *testing.T) {
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
apps, err := s.Apps(ctx)
|
||||
if err == nil && len(apps) == 4 {
|
||||
if err == nil && len(apps) == 8 {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
@@ -266,7 +287,7 @@ func TestAHelperThatExitsFailsWhatWasPendingAndStartsAgain(t *testing.T) {
|
||||
// was given covers, which is what every point in a step is read in.
|
||||
func fittedScale(t *testing.T, s *Session, ctx context.Context) float64 {
|
||||
t.Helper()
|
||||
shot, _, err := s.Screenshot(ctx, "com.example.Notes")
|
||||
shot, _, err := s.Screenshot(ctx, "com.example.Notes", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -297,7 +318,7 @@ func TestEveryPointerStepIsAimedInTheScreenshotsPixels(t *testing.T) {
|
||||
x, y, toX, toY := 100.0, 40.0, 300.0, 120.0
|
||||
wantX, wantY := 100+x*scale, 50+y*scale
|
||||
|
||||
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_move", X: &x, Y: &y}})
|
||||
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_move", X: &x, Y: &y}})
|
||||
if err != nil || res.Steps[0].Note != "move the pointer to (100,40)" {
|
||||
t.Fatalf("pointer_move = %+v, %v", res, err)
|
||||
}
|
||||
@@ -306,7 +327,7 @@ func TestEveryPointerStepIsAimedInTheScreenshotsPixels(t *testing.T) {
|
||||
t.Fatalf("the helper was sent %s %v, want pointer_move at (%v,%v)", method, params, wantX, wantY)
|
||||
}
|
||||
|
||||
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_click", X: &x, Y: &y, Button: "right", Times: 2}})
|
||||
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_click", X: &x, Y: &y, Button: "right", Times: 2}})
|
||||
if err != nil || res.Steps[0].Note != "right click at (100,40)" {
|
||||
t.Fatalf("pointer_click = %+v, %v", res, err)
|
||||
}
|
||||
@@ -314,14 +335,14 @@ func TestEveryPointerStepIsAimedInTheScreenshotsPixels(t *testing.T) {
|
||||
t.Fatalf("the helper was sent %s %v, want a right double click", method, params)
|
||||
}
|
||||
// A click that says neither button nor count is one left click.
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_click", X: &x, Y: &y}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, params = lastCall(t, s); params["button"] != "left" || params["clicks"] != 1.0 {
|
||||
t.Fatalf("a bare pointer_click sent %v, want one left click", params)
|
||||
}
|
||||
|
||||
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}})
|
||||
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_drag", X: &x, Y: &y, ToX: &toX, ToY: &toY}})
|
||||
if err != nil || res.Steps[0].Note != "drag the pointer from (100,40) to (300,120)" {
|
||||
t.Fatalf("pointer_drag = %+v, %v", res, err)
|
||||
}
|
||||
@@ -342,12 +363,12 @@ func TestAPointerStepThatCannotSayWhereIsRefused(t *testing.T) {
|
||||
{Action: "pointer_drag", X: &x, Y: &x},
|
||||
{Action: "pointer_drag", X: &x, Y: &x, ToX: &x},
|
||||
} {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{step}); CodeOf(err) != CodeNeedsScreenshot && CodeOf(err) != CodeBadStep {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{step}); CodeOf(err) != CodeNeedsScreenshot && CodeOf(err) != CodeBadStep {
|
||||
t.Fatalf("%+v = %v", step, err)
|
||||
}
|
||||
}
|
||||
_ = fittedScale(t, s, ctx)
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_drag", X: &x, Y: &x}}); CodeOf(err) != CodeBadStep {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_drag", X: &x, Y: &x}}); CodeOf(err) != CodeBadStep {
|
||||
t.Fatalf("a drag with no destination = %v, want %s", err, CodeBadStep)
|
||||
}
|
||||
}
|
||||
@@ -365,20 +386,20 @@ func TestThePointerIsHandedBackWhicheverWayTheStepsEnd(t *testing.T) {
|
||||
}
|
||||
return r.Count
|
||||
}
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Enter"}}); err != nil {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Enter"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if released() != 0 {
|
||||
t.Fatal("steps that never took the pointer gave one back")
|
||||
}
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_move", X: &x, Y: &y}}); err != nil {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_move", X: &x, Y: &y}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if released() != 1 {
|
||||
t.Fatal("the pointer was not given back after it was taken")
|
||||
}
|
||||
// A run that fails part way took it just the same.
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_move", X: &x, Y: &y}, {Action: "click", Ref: "a9"}}); CodeOf(err) != CodeStaleRef {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_move", X: &x, Y: &y}, {Action: "click", Ref: "a9"}}); CodeOf(err) != CodeStaleRef {
|
||||
t.Fatalf("want %s, got %v", CodeStaleRef, err)
|
||||
}
|
||||
if released() != 2 {
|
||||
@@ -393,12 +414,12 @@ func TestThePointerIsHandedBackWhicheverWayTheStepsEnd(t *testing.T) {
|
||||
func TestThePointerSaysWhereItIsInTheFrameTheModelHas(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_position"}})
|
||||
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_position"}})
|
||||
if err != nil || !strings.Contains(res.Steps[0].Note, "take a screenshot") {
|
||||
t.Fatalf("pointer_position with no screenshot = %+v, %v", res, err)
|
||||
}
|
||||
scale := fittedScale(t, s, ctx)
|
||||
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "pointer_position"}})
|
||||
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "pointer_position"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -411,10 +432,10 @@ func TestThePointerSaysWhereItIsInTheFrameTheModelHas(t *testing.T) {
|
||||
func TestTheMenuBelongsToAnElementAndScrollingSaysWhichItWas(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "right_click"}}); CodeOf(err) != CodeBadStep {
|
||||
t.Fatalf("a right_click with no ref = %v, want %s", err, CodeBadStep)
|
||||
}
|
||||
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "right_click", Ref: "a4"}})
|
||||
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "right_click", Ref: "a4"}})
|
||||
if err != nil || res.Steps[0].Note != "open the menu of a4" {
|
||||
t.Fatalf("right_click = %+v, %v", res, err)
|
||||
}
|
||||
@@ -423,7 +444,7 @@ func TestTheMenuBelongsToAnElementAndScrollingSaysWhichItWas(t *testing.T) {
|
||||
}
|
||||
// Bringing an element into view and turning the wheel are one step and two
|
||||
// outcomes, and the helper says which one it did.
|
||||
res, err = s.Act(ctx, "com.example.Notes", []Step{{Action: "scroll", Ref: "a2"}, {Action: "scroll", Amount: -3}})
|
||||
res, err = s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "scroll", Ref: "a2"}, {Action: "scroll", Amount: -3}})
|
||||
if err != nil || res.Steps[0].Note != "bring a2 into view" || res.Steps[1].Note != "scroll -3 lines" {
|
||||
t.Fatalf("scroll = %+v, %v", res, err)
|
||||
}
|
||||
@@ -432,27 +453,27 @@ func TestTheMenuBelongsToAnElementAndScrollingSaysWhichItWas(t *testing.T) {
|
||||
func TestHoldingAKeyAndRepeatingOneCarryTheirCount(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "hold_key", Key: "shift+a", Seconds: 1.5}})
|
||||
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "hold_key", Key: "shift+a", Seconds: 1.5}})
|
||||
if err != nil || res.Steps[0].Note != "hold shift+a for 1.5s" {
|
||||
t.Fatalf("hold_key = %+v, %v", res, err)
|
||||
}
|
||||
if method, params := lastCall(t, s); method != "hold_key" || params["seconds"] != 1.5 {
|
||||
t.Fatalf("the helper was sent %s %v, want the seconds it holds for", method, params)
|
||||
}
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "key", Key: "Tab", Times: 4}}); err != nil {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "key", Key: "Tab", Times: 4}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitingIsBoundedAndTheContextEndsIt(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
res, err := s.Act(context.Background(), "com.example.Notes", []Step{{Action: "wait", Ms: 5}})
|
||||
res, err := s.Act(context.Background(), "com.example.Notes", 0, []Step{{Action: "wait", Ms: 5}})
|
||||
if err != nil || res.Steps[0].Note != "wait 5ms" {
|
||||
t.Fatalf("wait = %+v, %v", res, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "wait", Ms: 60000}}); err == nil {
|
||||
if _, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "wait", Ms: 60000}}); err == nil {
|
||||
t.Fatal("a wait outlived the context that was cancelled under it")
|
||||
}
|
||||
// The same cancellation reaches a call already sent to the helper.
|
||||
@@ -466,7 +487,7 @@ func TestWaitingIsBoundedAndTheContextEndsIt(t *testing.T) {
|
||||
func TestPastePutsTheWholeTextInAtOnce(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
res, err := s.Act(ctx, "com.example.Notes", []Step{{Action: "paste", Text: "从剪贴板 pasted"}})
|
||||
res, err := s.Act(ctx, "com.example.Notes", 0, []Step{{Action: "paste", Text: "从剪贴板 pasted"}})
|
||||
if err != nil || res.Steps[0].Note != "paste 11 characters" {
|
||||
t.Fatalf("paste = %+v, %v", res, err)
|
||||
}
|
||||
@@ -474,3 +495,7 @@ func TestPastePutsTheWholeTextInAtOnce(t *testing.T) {
|
||||
t.Fatalf("the helper was sent %s %v, want the text to paste", method, params)
|
||||
}
|
||||
}
|
||||
|
||||
func fakeWindow(id int, title string, owned bool) map[string]any {
|
||||
return map[string]any{"id": id, "title": title, "owned": owned, "bounds": map[string]any{"x": 0, "y": 0, "width": 800, "height": 600}}
|
||||
}
|
||||
@@ -210,6 +210,10 @@ func main() {
|
||||
switch mode {
|
||||
case "modal":
|
||||
postMessage.Call(top, wmApp, 0, 0)
|
||||
case "second":
|
||||
other := create("ReasonixProbe", "Second Window", 0x00CF0000, 620, 120, 300, 200, 0, 0)
|
||||
showWindow.Call(other, 4)
|
||||
logf("second open")
|
||||
case "form":
|
||||
form := create("ProbeForm", "Probe form", 0x10CF0000, 160, 160, 320, 140, top, 0) // owned, WS_VISIBLE
|
||||
create("STATIC", "Form field", child|visible, 10, 12, 80, 22, form, 0)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package computer
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const maxCandidates = 10
|
||||
|
||||
// withWindow adds the window a request is aimed at; zero leaves the helper to
|
||||
// its default, the application's front window.
|
||||
func withWindow(params map[string]any, window int) map[string]any {
|
||||
if window != 0 {
|
||||
params["window"] = window
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
// needsWindow reports whether any step goes to one window of the application
|
||||
// rather than to an element a ref already names.
|
||||
func needsWindow(steps []Step) bool {
|
||||
return slices.ContainsFunc(steps, func(s Step) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(s.Action)) {
|
||||
case "type", "paste", "key", "hold_key", "pointer_move", "pointer_click", "pointer_drag":
|
||||
return true
|
||||
case "click", "scroll":
|
||||
return s.Ref == ""
|
||||
}
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
// target settles which window an operation is aimed at: one the model names
|
||||
// must be the application's own and a helper that classifies windows must be
|
||||
// able to aim at it; with none named, two or more independent
|
||||
// windows are refused before anything is sent. A dialog another window owns is
|
||||
// not a second window, and a helper that does not classify windows judges none.
|
||||
func (a App) target(window int, scoped bool) error {
|
||||
if window != 0 {
|
||||
if slices.ContainsFunc(a.Windows, func(w Window) bool { return w.Owned == nil }) {
|
||||
return fail(CodeUnsupported, "%s cannot be aimed at one of its windows on this platform; leave window out", ShownBundle(a.Bundle))
|
||||
}
|
||||
if !slices.ContainsFunc(a.Windows, func(w Window) bool { return w.ID == window }) {
|
||||
return fail(CodeNoWindow, "window %d is not a window of %s; list the applications for its windows", window, ShownBundle(a.Bundle))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !scoped {
|
||||
return nil
|
||||
}
|
||||
var independent []Window
|
||||
for _, w := range a.Windows {
|
||||
if w.Owned == nil {
|
||||
return nil
|
||||
}
|
||||
if !*w.Owned {
|
||||
independent = append(independent, w)
|
||||
}
|
||||
}
|
||||
if len(independent) < 2 {
|
||||
return nil
|
||||
}
|
||||
f := fail(CodeAmbiguousWindow, "%s has %d windows and this would go to one of them; repeat the call with window set to the one meant", ShownBundle(a.Bundle), len(independent))
|
||||
f.Candidates = independent
|
||||
return f
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package computer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func delivered(t *testing.T, s *Session) int {
|
||||
t.Helper()
|
||||
var r struct{ Count int }
|
||||
if err := s.helper.Call(context.Background(), "_delivered", nil, &r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r.Count
|
||||
}
|
||||
|
||||
// Two independent windows make the target ambiguous: the call is refused before
|
||||
// anything is sent, and the candidates come back with their ids and the front.
|
||||
func TestAnApplicationWithTwoWindowsIsRefusedBeforeAnythingIsSent(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
for name, run := range map[string]func() error{
|
||||
"type": func() error {
|
||||
_, err := s.Act(ctx, "com.example.Two", 0, []Step{{Action: "type", Text: "x"}})
|
||||
return err
|
||||
},
|
||||
"key": func() error {
|
||||
_, err := s.Act(ctx, "com.example.Two", 0, []Step{{Action: "key", Key: "Enter"}})
|
||||
return err
|
||||
},
|
||||
"shot": func() error { _, _, err := s.Screenshot(ctx, "com.example.Two", 0); return err },
|
||||
} {
|
||||
err := run()
|
||||
f, ok := errors.AsType[*Failure](err)
|
||||
if !ok || f.Code != CodeAmbiguousWindow {
|
||||
t.Fatalf("%s = %v, want %s", name, err, CodeAmbiguousWindow)
|
||||
}
|
||||
if len(f.Candidates) != 2 || f.Candidates[0].ID != 101 || f.Candidates[1].ID != 102 {
|
||||
t.Errorf("%s: candidates = %+v", name, f.Candidates)
|
||||
}
|
||||
}
|
||||
if n := delivered(t, s); n != 0 {
|
||||
t.Fatalf("%d requests reached the helper before the refusal", n)
|
||||
}
|
||||
}
|
||||
|
||||
// An element a ref names needs no window, and a window the model names settles
|
||||
// the target and travels to the helper.
|
||||
func TestANamedWindowOrARefSettlesTheTarget(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
if _, err := s.Act(ctx, "com.example.Two", 0, []Step{{Action: "focus", Ref: "a1"}}); err != nil {
|
||||
t.Fatalf("a ref step on two windows = %v", err)
|
||||
}
|
||||
if res, err := s.Act(ctx, "com.example.Two", 102, []Step{{Action: "type", Text: "x"}}); err != nil || res.Done != 1 {
|
||||
t.Fatalf("type into window 102 = %+v, %v", res, err)
|
||||
}
|
||||
var sent struct {
|
||||
Method string
|
||||
Params map[string]any
|
||||
}
|
||||
if err := s.helper.Call(ctx, "_last_sent", nil, &sent); err != nil || sent.Method != "type" || sent.Params["window"] != float64(102) {
|
||||
t.Fatalf("the helper was sent %+v, %v", sent, err)
|
||||
}
|
||||
if _, _, err := s.Screenshot(ctx, "com.example.Two", 101); err != nil {
|
||||
t.Fatalf("screenshot of window 101 = %v", err)
|
||||
}
|
||||
if _, err := s.Act(ctx, "com.example.Two", 999, []Step{{Action: "type", Text: "x"}}); CodeOf(err) != CodeNoWindow {
|
||||
t.Fatalf("a window the application does not have = %v, want %s", err, CodeNoWindow)
|
||||
}
|
||||
}
|
||||
|
||||
// A dialog another window owns is not a second window, and a helper that does
|
||||
// not classify windows leaves the target unjudged.
|
||||
func TestADialogOrAnUnclassifiedWindowIsNotAmbiguity(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
for _, bundle := range []string{"com.example.Dialog", "com.example.Flat"} {
|
||||
if _, err := s.Act(ctx, bundle, 0, []Step{{Action: "type", Text: "x"}}); err != nil {
|
||||
t.Errorf("%s: %v", bundle, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A point read in one window's screenshot is not a point in another's.
|
||||
func TestAPointNeedsTheScreenshotOfTheWindowItIsAimedAt(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
if _, _, err := s.Screenshot(ctx, "com.example.Two", 101); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
x, y := 10.0, 10.0
|
||||
if _, err := s.Act(ctx, "com.example.Two", 102, []Step{{Action: "click", X: &x, Y: &y}}); CodeOf(err) != CodeNeedsScreenshot {
|
||||
t.Fatalf("a click aimed at a window never shot = %v, want %s", err, CodeNeedsScreenshot)
|
||||
}
|
||||
if _, err := s.Act(ctx, "com.example.Two", 101, []Step{{Action: "click", X: &x, Y: &y}}); err != nil {
|
||||
t.Fatalf("a click in the window that was shot = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A capture of another window than the one asked for is not handed back as if
|
||||
// it were: the helper may have aimed elsewhere, or the window may be gone.
|
||||
func TestACaptureOfAnotherWindowIsRefused(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
if _, _, err := s.Screenshot(ctx, "com.example.Drift", 401); err != nil {
|
||||
t.Fatalf("the window that was captured = %v", err)
|
||||
}
|
||||
if _, _, err := s.Screenshot(ctx, "com.example.Drift", 402); CodeOf(err) != CodeNoWindow {
|
||||
t.Fatalf("a capture of window 401 for window 402 = %v, want %s", err, CodeNoWindow)
|
||||
}
|
||||
}
|
||||
|
||||
// A helper that does not classify windows cannot aim at one, so naming a
|
||||
// window there is refused rather than silently sent to the front one.
|
||||
func TestANamedWindowIsRefusedWhereTheHelperCannotAimAtIt(t *testing.T) {
|
||||
s := fakeSession(t)
|
||||
ctx := context.Background()
|
||||
if _, err := s.Act(ctx, "com.example.Flat", 301, []Step{{Action: "type", Text: "x"}}); CodeOf(err) != CodeUnsupported {
|
||||
t.Fatalf("a window on an unclassified helper = %v, want %s", err, CodeUnsupported)
|
||||
}
|
||||
if _, _, err := s.Screenshot(ctx, "com.example.Flat", 301); CodeOf(err) != CodeUnsupported {
|
||||
t.Fatalf("a screenshot of a window on an unclassified helper = %v, want %s", err, CodeUnsupported)
|
||||
}
|
||||
if n := delivered(t, s); n != 0 {
|
||||
t.Fatalf("%d requests reached the helper", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAmbiguityNamesItsCandidatesBounded(t *testing.T) {
|
||||
f := &Failure{Code: CodeAmbiguousWindow, Detail: "d", Candidates: []Window{{ID: 5, Title: "A\nB"}, {ID: 6, Title: "C"}}}
|
||||
got := f.Error()
|
||||
want := `computer.ambiguous_window_target: d; window 5 "A\u{a}B" (front); window 6 "C"`
|
||||
if got != want {
|
||||
t.Fatalf("Error() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -45,11 +45,11 @@ type computerRead struct{ session *computer.Session }
|
||||
func (computerRead) Name() string { return "computer_read" }
|
||||
|
||||
func (computerRead) Description() string {
|
||||
return "See another application on this computer, named as what=apps lists it. what=apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels, and the person is shown it too)."
|
||||
return "See another application, named as what=apps lists it. apps lists applications with windows; snapshot returns its accessibility tree with [aN] refs for computer_act; screenshot captures its front window (computer_act x/y use its pixels; the person sees it too)."
|
||||
}
|
||||
|
||||
func (computerRead) Schema() json.RawMessage {
|
||||
return json.RawMessage(`{"type":"object","properties":{"what":{"type":"string","enum":["apps","snapshot","screenshot"]},"app":{"type":"string"}},"required":["what"]}`)
|
||||
return json.RawMessage(`{"type":"object","properties":{"what":{"type":"string","enum":["apps","snapshot","screenshot"]},"app":{"type":"string"},"window":{"type":"integer"}},"required":["what"]}`)
|
||||
}
|
||||
|
||||
func (computerRead) ReadOnly() bool { return false }
|
||||
@@ -75,8 +75,9 @@ func (c computerRead) Execute(ctx context.Context, args json.RawMessage) (string
|
||||
|
||||
func (c computerRead) ExecuteWithImages(ctx context.Context, args json.RawMessage) (string, []string, error) {
|
||||
var p struct {
|
||||
What string `json:"what"`
|
||||
App string `json:"app"`
|
||||
What string `json:"what"`
|
||||
App string `json:"app"`
|
||||
Window int `json:"window"`
|
||||
}
|
||||
if err := json.Unmarshal(args, &p); err != nil {
|
||||
return "", nil, fmt.Errorf("invalid args: %w", err)
|
||||
@@ -98,7 +99,7 @@ func (c computerRead) ExecuteWithImages(ctx context.Context, args json.RawMessag
|
||||
}
|
||||
return renderComputerSnapshot(snap), nil, nil
|
||||
case "screenshot":
|
||||
shot, app, err := c.session.Screenshot(ctx, p.App)
|
||||
shot, app, err := c.session.Screenshot(ctx, p.App, p.Window)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -116,15 +117,15 @@ type computerAct struct{ session *computer.Session }
|
||||
func (computerAct) Name() string { return "computer_act" }
|
||||
|
||||
func (computerAct) Description() string {
|
||||
return "Operate another application. Steps run in order and stop at the first failure. Through its accessibility actions, leaving the pointer alone: " +
|
||||
"click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is borrowed and put back), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. " +
|
||||
return "Operate another application. Steps run in order and stop at the first failure. Through accessibility actions, pointer untouched: " +
|
||||
"click (ref, or x/y from a screenshot), right_click opens a ref's context menu, focus and set_value take a ref, type enters text where focus is and paste puts it there at once (the clipboard is restored), key presses keys like Enter or Meta+s (control+s on Windows; times repeats), hold_key holds one for seconds. " +
|
||||
"Paste needs the application in front on macOS; on Windows type, paste and key bring it forward. scroll brings a ref into view or turns the wheel by amount lines, wait pauses ms. " +
|
||||
"pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, and the pointer goes back where it was. " +
|
||||
"pointer_move, pointer_click (button, times), pointer_drag (to_x/to_y) and pointer_position instead take the person's pointer and bring the application forward, approved separately; use them for what has no accessibility action, then the pointer is restored. " +
|
||||
"Returns a snapshot afterwards."
|
||||
}
|
||||
|
||||
func (computerAct) Schema() json.RawMessage {
|
||||
return json.RawMessage(`{"type":"object","properties":{"app":{"type":"string"},"steps":{"type":"array","items":{"type":"object","properties":{"action":{"type":"string","enum":["click","right_click","focus","set_value","type","paste","key","hold_key","scroll","wait","pointer_move","pointer_click","pointer_drag","pointer_position"]},"ref":{"type":"string"},"text":{"type":"string"},"key":{"type":"string"},"x":{"type":"number"},"y":{"type":"number"},"amount":{"type":"number"},"ms":{"type":"integer"},"to_x":{"type":"number"},"to_y":{"type":"number"},"button":{"type":"string","enum":["left","right","middle"]},"times":{"type":"integer"},"seconds":{"type":"number"}},"required":["action"]}}},"required":["app","steps"]}`)
|
||||
return json.RawMessage(`{"type":"object","properties":{"app":{"type":"string"},"window":{"type":"integer"},"steps":{"type":"array","items":{"type":"object","properties":{"action":{"type":"string","enum":["click","right_click","focus","set_value","type","paste","key","hold_key","scroll","wait","pointer_move","pointer_click","pointer_drag","pointer_position"]},"ref":{"type":"string"},"text":{"type":"string"},"key":{"type":"string"},"x":{"type":"number"},"y":{"type":"number"},"amount":{"type":"number"},"ms":{"type":"integer"},"to_x":{"type":"number"},"to_y":{"type":"number"},"button":{"type":"string","enum":["left","right","middle"]},"times":{"type":"integer"},"seconds":{"type":"number"}},"required":["action"]}}},"required":["app","steps"]}`)
|
||||
}
|
||||
|
||||
func (computerAct) ReadOnly() bool { return false }
|
||||
@@ -146,8 +147,9 @@ func (c computerAct) PermissionArgs(_ context.Context, args json.RawMessage) jso
|
||||
|
||||
func (c computerAct) Execute(ctx context.Context, args json.RawMessage) (string, error) {
|
||||
var p struct {
|
||||
App string `json:"app"`
|
||||
Steps []computer.Step `json:"steps"`
|
||||
App string `json:"app"`
|
||||
Window int `json:"window"`
|
||||
Steps []computer.Step `json:"steps"`
|
||||
}
|
||||
if err := json.Unmarshal(args, &p); err != nil {
|
||||
return "", fmt.Errorf("invalid args: %w", err)
|
||||
@@ -158,7 +160,7 @@ func (c computerAct) Execute(ctx context.Context, args json.RawMessage) (string,
|
||||
if len(p.Steps) == 0 {
|
||||
return "", &computer.Failure{Code: computer.CodeBadStep, Detail: "steps is empty"}
|
||||
}
|
||||
res, stepErr := c.session.Act(ctx, p.App, p.Steps)
|
||||
res, stepErr := c.session.Act(ctx, p.App, p.Window, p.Steps)
|
||||
var out strings.Builder
|
||||
fmt.Fprintf(&out, "Completed %d of %d step(s).\n", res.Done, len(p.Steps))
|
||||
for i, step := range res.Steps {
|
||||
@@ -196,7 +198,11 @@ func renderApps(apps []computer.App) string {
|
||||
if w.Title != "" {
|
||||
title = computer.ShownName(w.Title)
|
||||
}
|
||||
fmt.Fprintf(&b, " window %s %.0f×%.0f\n", title, w.Bounds.Width, w.Bounds.Height)
|
||||
fmt.Fprintf(&b, " window %d %s %.0f×%.0f", w.ID, title, w.Bounds.Width, w.Bounds.Height)
|
||||
if w.Owned != nil && *w.Owned {
|
||||
b.WriteString(" (dialog)")
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestApplicationListMarksTheOnesNeverOperated(t *testing.T) {
|
||||
{Bundle: "com.apple.Notes", Name: "Notes", Active: true, Windows: []computer.Window{{Title: "", Bounds: computer.Rect{Width: 800, Height: 600}}}},
|
||||
{Bundle: "com.apple.Terminal", Name: "Terminal"},
|
||||
})
|
||||
if !strings.Contains(out, `* com.apple.Notes — "Notes"`+"\n") || !strings.Contains(out, `window "(untitled)" 800×600`) {
|
||||
if !strings.Contains(out, `* com.apple.Notes — "Notes"`+"\n") || !strings.Contains(out, `window 0 "(untitled)" 800×600`) {
|
||||
t.Fatalf("listing = %q", out)
|
||||
}
|
||||
if !strings.Contains(out, `com.apple.Terminal — "Terminal" (never operated: a terminal)`) {
|
||||
|
||||
Reference in new issue
Block a user