diff --git a/REASONIX.md b/REASONIX.md index 29289a4ffd..40f0bee613 100644 --- a/REASONIX.md +++ b/REASONIX.md @@ -339,6 +339,7 @@ out of how a path is spelled, which cannot tell `internal/auth` from - sensitive: internal/pluginpkg/** - sensitive: internal/netclient/** - sensitive: internal/redirectguard/** +- sensitive: internal/browser/** One declaration, two effects: the same list is also the coverage gate's subject. `make coverage-gate` holds each path to the coverage it already has, because diff --git a/go.mod b/go.mod index 7b5cae6428..a28bae5628 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/charmbracelet/colorprofile v0.4.3 github.com/charmbracelet/x/ansi v0.11.8 github.com/godbus/dbus/v5 v5.2.2 + github.com/gorilla/websocket v1.5.3 github.com/joho/godotenv v1.5.1 github.com/kevinburke/ssh_config v1.6.0 github.com/larksuite/oapi-sdk-go/v3 v3.11.0 @@ -60,7 +61,6 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/kr/fs v0.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect github.com/mattn/go-isatty v0.0.24 // indirect diff --git a/internal/browser/act.go b/internal/browser/act.go new file mode 100644 index 0000000000..54ffe9a82d --- /dev/null +++ b/internal/browser/act.go @@ -0,0 +1,549 @@ +package browser + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +const ( + objectGroup = "reasonix" + maxWait = 30 * time.Second + defaultWaitFor = 5 * time.Second +) + +// Step is one input. X and Y are in the pixels of the tab's latest screenshot. +type Step struct { + Action string `json:"action"` + Ref string `json:"ref,omitempty"` + Text string `json:"text,omitempty"` + Key string `json:"key,omitempty"` + Values []string `json:"values,omitempty"` + X *float64 `json:"x,omitempty"` + Y *float64 `json:"y,omitempty"` + DeltaY float64 `json:"delta_y,omitempty"` + Ms int `json:"ms,omitempty"` + Accept *bool `json:"accept,omitempty"` +} + +// ActResult is what a run of steps did before it finished or stopped. +type ActResult struct { + Tab TabInfo + Done int // steps that completed + Notes []string // one per completed step + Logs []LogEntry // errors and warnings the page reported meanwhile + Opened []string // tabs the page opened meanwhile + Dialog *Dialog + FailedAt int // index of the step that failed, or -1 +} + +// Act runs steps in order on a tab and stops at the first that fails. The +// returned error is that step's failure; the result says what came before it. +func (s *Session) Act(ctx context.Context, tabID string, steps []Step) (ActResult, error) { + t, err := s.tab(tabID) + if err != nil { + return ActResult{FailedAt: 0}, err + } + if err := t.checkCurrentURL(); err != nil { + return ActResult{FailedAt: 0}, err + } + _, logMark := t.logsAfter(1 << 62) + t.mu.Lock() + popupMark := len(t.popups) + t.mu.Unlock() + res := ActResult{FailedAt: -1} + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), closeTimeout) + _ = t.call(releaseCtx, "Runtime.releaseObjectGroup", map[string]any{"objectGroup": objectGroup}, nil) + cancel() + }() + var stepErr error + for i, step := range steps { + note, err := s.runStep(ctx, t, step) + if err != nil { + res.FailedAt, stepErr = i, err + break + } + res.Done++ + res.Notes = append(res.Notes, note) + } + entries, _ := t.logsAfter(logMark) + for _, e := range entries { + if e.Level != "info" { + res.Logs = append(res.Logs, e) + } + } + t.mu.Lock() + res.Opened = append(res.Opened, t.popups[popupMark:]...) + t.mu.Unlock() + active := s.activeTab() + if active == nil { + active = t + } + res.Tab = active.info(true) + res.Dialog = active.currentDialog() + return res, stepErr +} + +func (s *Session) runStep(ctx context.Context, t *tab, step Step) (string, error) { + action := strings.ToLower(strings.TrimSpace(step.Action)) + if d := t.currentDialog(); d != nil && action != "dialog" { + return "", dialogFailure(d) + } + before := t.navigationCount() + switch action { + case "click", "double_click": + x, y, err := s.point(ctx, t, step) + if err != nil { + return "", err + } + clicks := 1 + if action == "double_click" { + clicks = 2 + } + if err := t.click(ctx, x, y, clicks); err != nil { + return "", err + } + t.settle(ctx, before) + return fmt.Sprintf("%s %s", action, target(step)), nil + case "hover": + x, y, err := s.point(ctx, t, step) + if err != nil { + return "", err + } + if err := t.mouse(ctx, "mouseMoved", x, y, 0); err != nil { + return "", err + } + return "hover " + target(step), nil + case "type", "fill": + if step.Ref != "" { + if err := s.focus(ctx, t, step.Ref, action == "fill"); err != nil { + return "", err + } + } + if step.Text == "" && action == "fill" { + if err := t.press(ctx, "Backspace"); err != nil { + return "", err + } + } else if err := t.call(ctx, "Input.insertText", map[string]any{"text": step.Text}, nil); err != nil { + return "", engineFailure(err) + } + return fmt.Sprintf("%s %d characters into %s", action, len([]rune(step.Text)), target(step)), nil + case "press": + if err := t.press(ctx, step.Key); err != nil { + return "", err + } + t.settle(ctx, before) + return "press " + step.Key, nil + case "select": + return s.selectOptions(ctx, t, step) + case "scroll": + x, y, err := s.scrollPoint(ctx, t, step) + if err != nil { + return "", err + } + if err := t.call(ctx, "Input.dispatchMouseEvent", map[string]any{"type": "mouseWheel", "x": x, "y": y, "deltaX": 0, "deltaY": step.DeltaY}, nil); err != nil { + return "", engineFailure(err) + } + return fmt.Sprintf("scroll %v px", step.DeltaY), nil + case "wait": + d := min(time.Duration(step.Ms)*time.Millisecond, maxWait) + select { + case <-time.After(d): + case <-ctx.Done(): + return "", ctx.Err() + } + return fmt.Sprintf("wait %s", d), nil + case "wait_for": + return t.waitForText(ctx, step) + case "dialog": + return t.answerDialog(ctx, step) + case "back": + return t.back(ctx) + } + return "", fail(CodeBadStep, "unknown action %q", step.Action) +} + +func target(step Step) string { + if step.Ref != "" { + return step.Ref + } + if step.X != nil && step.Y != nil { + return fmt.Sprintf("(%v,%v)", *step.X, *step.Y) + } + return "the focused element" +} + +// point is where an input for step lands, in CSS pixels. For a ref it is the +// middle of the element's visible part, and the element must be what a real +// pointer there would reach. +func (s *Session) point(ctx context.Context, t *tab, step Step) (float64, float64, error) { + if step.Ref == "" { + if step.X == nil || step.Y == nil { + return 0, 0, fail(CodeBadStep, "a %s step needs a ref, or x and y from a screenshot", step.Action) + } + scale := t.screenshotScale() + return *step.X * scale, *step.Y * scale, nil + } + node, err := s.node(ctx, t, step.Ref) + if err != nil { + return 0, 0, err + } + if err := t.call(ctx, "DOM.scrollIntoViewIfNeeded", map[string]any{"backendNodeId": node}, nil); err != nil { + if isProtocolError(err) { + return 0, 0, &Failure{Code: CodeNotVisible, Ref: step.Ref, Detail: fmt.Sprintf("%s has no box on the page (hidden or not rendered)", step.Ref)} + } + return 0, 0, engineFailure(err) + } + x, y, err := t.visibleCenter(ctx, node, step.Ref) + if err != nil { + return 0, 0, err + } + if err := s.checkHit(ctx, t, node, step.Ref, x, y); err != nil { + return 0, 0, err + } + return x, y, nil +} + +// node resolves a ref on t to a backend node that still exists. +func (s *Session) node(ctx context.Context, t *tab, ref string) (int64, error) { + target, err := s.refs.resolve(ref) + if err != nil { + return 0, err + } + if target.tab != t.id { + return 0, &Failure{Code: CodeUnknownRef, Ref: ref, Detail: fmt.Sprintf("%s is on tab %s; switch to it first", ref, target.tab)} + } + if err := t.call(ctx, "DOM.describeNode", map[string]any{"backendNodeId": target.node}, nil); err != nil { + if isProtocolError(err) { + return 0, &Failure{Code: CodeStaleRef, Ref: ref, Detail: fmt.Sprintf("%s is no longer on the page; take a new snapshot", ref)} + } + return 0, engineFailure(err) + } + return target.node, nil +} + +func (t *tab) visibleCenter(ctx context.Context, node int64, ref string) (float64, float64, error) { + var quads struct { + Quads [][]float64 `json:"quads"` + } + if err := t.call(ctx, "DOM.getContentQuads", map[string]any{"backendNodeId": node}, &quads); err != nil && !isProtocolError(err) { + return 0, 0, engineFailure(err) + } + var metrics struct { + Viewport struct { + Width float64 `json:"clientWidth"` + Height float64 `json:"clientHeight"` + } `json:"cssLayoutViewport"` + } + if err := t.call(ctx, "Page.getLayoutMetrics", nil, &metrics); err != nil { + return 0, 0, engineFailure(err) + } + for _, q := range quads.Quads { + if len(q) != 8 { + continue + } + minX, maxX := min(q[0], q[2], q[4], q[6]), max(q[0], q[2], q[4], q[6]) + minY, maxY := min(q[1], q[3], q[5], q[7]), max(q[1], q[3], q[5], q[7]) + minX, minY = max(minX, 0), max(minY, 0) + maxX, maxY = min(maxX, metrics.Viewport.Width), min(maxY, metrics.Viewport.Height) + if maxX-minX >= 1 && maxY-minY >= 1 { + return (minX + maxX) / 2, (minY + maxY) / 2, nil + } + } + return 0, 0, &Failure{Code: CodeNotVisible, Ref: ref, Detail: fmt.Sprintf("%s has no visible area in the viewport", ref)} +} + +// checkHit refuses an input whose point would land on another element, such +// as a banner drawn over the target. +func (s *Session) checkHit(ctx context.Context, t *tab, node int64, ref string, x, y float64) error { + var hit struct { + BackendNodeID int64 `json:"backendNodeId"` + } + if err := t.call(ctx, "DOM.getNodeForLocation", map[string]any{"x": int(x), "y": int(y), "includeUserAgentShadowDOM": false, "ignorePointerEventsNone": true}, &hit); err != nil { + if isProtocolError(err) { + return nil + } + return engineFailure(err) + } + if hit.BackendNodeID == 0 || hit.BackendNodeID == node { + return nil + } + targetObj, err := t.resolveObject(ctx, node) + if err != nil { + return err + } + hitObj, err := t.resolveObject(ctx, hit.BackendNodeID) + if err != nil { + return err + } + var within struct { + Result struct { + Value bool `json:"value"` + } `json:"result"` + } + if err := t.call(ctx, "Runtime.callFunctionOn", map[string]any{ + "objectId": targetObj, + "functionDeclaration": "function(n){for(;n;n=n.parentNode||n.host){if(n===this)return true}return false}", + "arguments": []map[string]any{{"objectId": hitObj}}, + "returnByValue": true, + }, &within); err != nil { + return engineFailure(err) + } + if within.Result.Value { + return nil + } + cover := s.refs.refFor(t.id, hit.BackendNodeID) + return &Failure{Code: CodeCovered, Ref: ref, Detail: fmt.Sprintf("%s is covered by %s %s at that point; deal with that element first", ref, t.describe(ctx, hit.BackendNodeID), cover)} +} + +func (t *tab) resolveObject(ctx context.Context, node int64) (string, error) { + var r struct { + Object struct { + ObjectID string `json:"objectId"` + } `json:"object"` + } + if err := t.call(ctx, "DOM.resolveNode", map[string]any{"backendNodeId": node, "objectGroup": objectGroup}, &r); err != nil { + if isProtocolError(err) { + return "", fail(CodeStaleRef, "the element is no longer on the page") + } + return "", engineFailure(err) + } + return r.Object.ObjectID, nil +} + +// describe names a DOM element by its tag and identifying attributes. +func (t *tab) describe(ctx context.Context, node int64) string { + var d struct { + Node struct { + LocalName string `json:"localName"` + Attributes []string `json:"attributes"` + } `json:"node"` + } + if t.call(ctx, "DOM.describeNode", map[string]any{"backendNodeId": node}, &d) != nil { + return "another element" + } + b := strings.Builder{} + b.WriteString("<" + d.Node.LocalName) + for i := 0; i+1 < len(d.Node.Attributes); i += 2 { + switch d.Node.Attributes[i] { + case "id", "class", "role", "aria-label": + fmt.Fprintf(&b, " %s=%q", d.Node.Attributes[i], clip(d.Node.Attributes[i+1], 60)) + } + } + b.WriteString(">") + return b.String() +} + +func (t *tab) mouse(ctx context.Context, kind string, x, y float64, clicks int) error { + params := map[string]any{"type": kind, "x": x, "y": y} + if clicks > 0 { + params["button"], params["clickCount"] = "left", clicks + } + return engineFailure(t.call(ctx, "Input.dispatchMouseEvent", params, nil)) +} + +func (t *tab) click(ctx context.Context, x, y float64, clicks int) error { + if err := t.mouse(ctx, "mouseMoved", x, y, 0); err != nil { + return err + } + for c := 1; c <= clicks; c++ { + if err := t.mouse(ctx, "mousePressed", x, y, c); err != nil { + return err + } + if err := t.mouse(ctx, "mouseReleased", x, y, c); err != nil { + return err + } + } + return nil +} + +func (s *Session) focus(ctx context.Context, t *tab, ref string, selectAll bool) error { + node, err := s.node(ctx, t, ref) + if err != nil { + return err + } + if err := t.call(ctx, "DOM.focus", map[string]any{"backendNodeId": node}, nil); err != nil { + if isProtocolError(err) { + return &Failure{Code: CodeNotVisible, Ref: ref, Detail: fmt.Sprintf("%s cannot take focus", ref)} + } + return engineFailure(err) + } + if !selectAll { + return nil + } + obj, err := t.resolveObject(ctx, node) + if err != nil { + return err + } + return engineFailure(t.call(ctx, "Runtime.callFunctionOn", map[string]any{ + "objectId": obj, + "functionDeclaration": `function(){ + if (typeof this.select === "function") { this.select(); return; } + if (this.isContentEditable) { + const r = document.createRange(); r.selectNodeContents(this); + const s = getSelection(); s.removeAllRanges(); s.addRange(r); + } + }`, + }, nil)) +} + +func (t *tab) press(ctx context.Context, chord string) error { + def, modifiers, ok := parseKey(chord) + if !ok { + return fail(CodeBadStep, "%q is not a key this can press; use a name like Enter, Tab, ArrowDown, or a single character, with modifiers joined by +", chord) + } + down := map[string]any{ + "type": "keyDown", "key": def.key, "code": def.code, "modifiers": modifiers, + "windowsVirtualKeyCode": def.vk, "nativeVirtualKeyCode": def.vk, + } + if def.text != "" { + down["text"], down["unmodifiedText"] = def.text, def.text + } else { + down["type"] = "rawKeyDown" + } + if err := t.call(ctx, "Input.dispatchKeyEvent", down, nil); err != nil { + return engineFailure(err) + } + return engineFailure(t.call(ctx, "Input.dispatchKeyEvent", map[string]any{ + "type": "keyUp", "key": def.key, "code": def.code, "modifiers": modifiers, + "windowsVirtualKeyCode": def.vk, "nativeVirtualKeyCode": def.vk, + }, nil)) +} + +func (s *Session) selectOptions(ctx context.Context, t *tab, step Step) (string, error) { + node, err := s.node(ctx, t, step.Ref) + if err != nil { + return "", err + } + obj, err := t.resolveObject(ctx, node) + if err != nil { + return "", err + } + var r struct { + Result struct { + Value json.RawMessage `json:"value"` + } `json:"result"` + } + if err := t.call(ctx, "Runtime.callFunctionOn", map[string]any{ + "objectId": obj, + "functionDeclaration": `function(values){ + if (!(this instanceof HTMLSelectElement)) return -1; + const want = new Set(values); let hit = 0; + for (const o of this.options) { + const on = want.has(o.value) || want.has(o.label); + if (on || !this.multiple) o.selected = on; + if (on) hit++; + } + if (hit) { this.dispatchEvent(new Event("input", {bubbles: true})); this.dispatchEvent(new Event("change", {bubbles: true})); } + return hit; + }`, + "arguments": []map[string]any{{"value": step.Values}}, + "returnByValue": true, + }, &r); err != nil { + return "", engineFailure(err) + } + var hit int + _ = json.Unmarshal(r.Result.Value, &hit) + switch { + case hit < 0: + return "", &Failure{Code: CodeNotSelectable, Ref: step.Ref, Detail: fmt.Sprintf("%s is not a + + +

+
+Next page + +` + +func TestLiveSnapshotAndAct(t *testing.T) { + s := liveSession(t, true) + srv := liveServer(t, map[string]string{"/": formPage, "/next": `Next

Arrived

`}) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + if _, err := s.Open(ctx, srv.URL+"/", "", false); err != nil { + t.Fatalf("Open: %v", err) + } + snap, err := s.Snapshot(ctx, "", "") + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + t.Logf("snapshot:\n%s", strings.Join(snap.Lines, "\n")) + email := findRef(t, snap, `textbox "Email"`) + plan := findRef(t, snap, `combobox "Plan"`) + submit := findRef(t, snap, `button "Submit"`) + + res, err := s.Act(ctx, "", []Step{ + {Action: "fill", Ref: email, Text: "李雷@example.com"}, + {Action: "select", Ref: plan, Values: []string{"Pro"}}, + {Action: "click", Ref: submit}, + {Action: "wait_for", Text: "hello 李雷@example.com pro"}, + }) + if err != nil { + t.Fatalf("Act: %v (done %d, notes %v)", err, res.Done, res.Notes) + } + if len(res.Logs) == 0 || !strings.Contains(res.Logs[0].Text, "boom") { + t.Fatalf("the console error did not come back with the act: %+v", res.Logs) + } + + hidden := findRef(t, snap, `button "Hidden"`) + _, err = s.Act(ctx, "", []Step{{Action: "click", Ref: hidden}}) + if CodeOf(err) != CodeCovered || !strings.Contains(err.Error(), "veil") { + t.Fatalf("clicking under a veil = %v, want %s naming the veil", err, CodeCovered) + } + + ask := findRef(t, snap, `button "Ask"`) + res, err = s.Act(ctx, "", []Step{{Action: "click", Ref: ask}}) + if err != nil || res.Dialog == nil || res.Dialog.Type != "confirm" { + t.Fatalf("confirm dialog: err=%v dialog=%+v", err, res.Dialog) + } + _, err = s.Act(ctx, "", []Step{{Action: "click", Ref: submit}}) + if CodeOf(err) != CodeDialogOpen { + t.Fatalf("acting under an open dialog = %v, want %s", err, CodeDialogOpen) + } + no := false + if _, err = s.Act(ctx, "", []Step{{Action: "dialog", Accept: &no}, {Action: "wait_for", Text: "no"}}); err != nil { + t.Fatalf("dismiss dialog: %v", err) + } + + pop := findRef(t, snap, `button "Pop"`) + res, err = s.Act(ctx, "", []Step{{Action: "click", Ref: pop}, {Action: "wait", Ms: 500}}) + if err != nil || len(res.Opened) != 1 { + t.Fatalf("popup: err=%v opened=%v", err, res.Opened) + } + if err := s.CloseTab(ctx, res.Opened[0]); err != nil { + t.Fatalf("close popup: %v", err) + } + + next := findRef(t, snap, `link "Next page"`) + res, err = s.Act(ctx, "t1", []Step{{Action: "click", Ref: next}}) + if err != nil || !strings.HasSuffix(res.Tab.URL, "/next") || res.Tab.Title != "Next" { + t.Fatalf("link: err=%v tab=%+v", err, res.Tab) + } + _, err = s.Act(ctx, "t1", []Step{{Action: "click", Ref: submit}}) + if CodeOf(err) != CodeStaleRef { + t.Fatalf("a ref from the previous page = %v, want %s", err, CodeStaleRef) + } + res, err = s.Act(ctx, "t1", []Step{{Action: "back"}}) + if err != nil || strings.HasSuffix(res.Tab.URL, "/next") { + t.Fatalf("back: err=%v tab=%+v", err, res.Tab) + } +} + +func TestLiveScreenshotCoordinatesLandOnTheElement(t *testing.T) { + s := liveSession(t, true) + srv := liveServer(t, map[string]string{"/": `Canvas +
`}) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + if _, err := s.Open(ctx, srv.URL+"/", "", false); err != nil { + t.Fatalf("Open: %v", err) + } + url, _, err := s.Screenshot(ctx, "") + if err != nil || !strings.HasPrefix(url, "data:image/") { + t.Fatalf("Screenshot: %v", err) + } + scale := s.activeTab().screenshotScale() + x, y := 640/scale, 430/scale + res, err := s.Act(ctx, "", []Step{{Action: "click", X: &x, Y: &y}}) + if err != nil || res.Tab.Title != "hit" { + t.Fatalf("coordinate click: err=%v title=%q scale=%v", err, res.Tab.Title, scale) + } +} + +func TestLiveRefusesFilesOutsideTheWorkspace(t *testing.T) { + s := liveSession(t, true) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err := s.Open(ctx, "file:///etc/hosts", "", false); CodeOf(err) != CodeURLRefused { + t.Fatalf("file outside the workspace = %v, want %s", err, CodeURLRefused) + } + if _, err := s.Open(ctx, "chrome://settings", "", false); CodeOf(err) != CodeURLRefused { + t.Fatalf("browser page = %v, want %s", err, CodeURLRefused) + } +} + +// The port endpoint is what Windows uses; driving it here keeps that path +// exercised on machines that can run the pipe. +func TestLiveWebSocketTransport(t *testing.T) { + was := usePipe + usePipe = false + t.Cleanup(func() { usePipe = was }) + s := liveSession(t, true) + srv := liveServer(t, map[string]string{"/": `Socket`}) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + info, err := s.Open(ctx, srv.URL+"/", "", false) + if err != nil || info.Title != "Socket" { + t.Fatalf("Open over the port endpoint: info=%+v err=%v", info, err) + } + snap, err := s.Snapshot(ctx, "", "") + if err != nil { + t.Fatalf("Snapshot: %v", err) + } + findRef(t, snap, `button "Go"`) +} diff --git a/internal/browser/policy.go b/internal/browser/policy.go new file mode 100644 index 0000000000..7e20df7928 --- /dev/null +++ b/internal/browser/policy.go @@ -0,0 +1,81 @@ +package browser + +import ( + "net/url" + "path/filepath" + "runtime" + "strings" + + "reasonix/internal/fileutil" +) + +// checkURL decides whether a page may be loaded. http and https go anywhere +// the network does; about:blank is the empty page; a file must lie inside one +// of roots once symlinks resolve. Every other scheme — the browser's own +// pages, script and data URLs — is refused. +func checkURL(raw string, roots []string) (string, error) { + raw = strings.TrimSpace(raw) + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" { + return "", fail(CodeURLRefused, "%q is not an absolute URL; include the scheme, e.g. https://", raw) + } + switch strings.ToLower(u.Scheme) { + case "http", "https": + if u.Host == "" { + return "", fail(CodeURLRefused, "%q names no host", raw) + } + return u.String(), nil + case "about": + if u.Opaque == "blank" { + return "about:blank", nil + } + case "file": + path, err := url.PathUnescape(u.Path) + if err != nil || !fileWithin(path, roots) { + return "", fail(CodeURLRefused, "%q is outside the workspace; a page may only be a local file inside it", raw) + } + return u.String(), nil + } + return "", fail(CodeURLRefused, "the %s: scheme is not a page the agent may open", u.Scheme) +} + +func fileWithin(path string, roots []string) bool { + if path == "" { + return false + } + if runtime.GOOS == "windows" && len(path) >= 3 && path[0] == '/' && path[2] == ':' { + path = path[1:] + } + path = filepath.FromSlash(path) + if resolved, err := filepath.EvalSymlinks(path); err == nil { + path = resolved + } + for _, root := range roots { + if resolved, err := filepath.EvalSymlinks(root); err == nil { + root = resolved + } + if fileutil.AtOrUnder(path, root) { + return true + } + } + return false +} + +// OriginOf reduces a URL to what a site grant names. It answers "" for the +// empty page and anything that is not a page an agent may open. +func OriginOf(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return "" + } + switch strings.ToLower(u.Scheme) { + case "http", "https": + if u.Host == "" { + return "" + } + return strings.ToLower(u.Scheme) + "://" + strings.ToLower(u.Host) + case "file": + return "file://" + } + return "" +} diff --git a/internal/browser/refs.go b/internal/browser/refs.go new file mode 100644 index 0000000000..9a56ab5dce --- /dev/null +++ b/internal/browser/refs.go @@ -0,0 +1,74 @@ +package browser + +import ( + "fmt" + "strings" + "sync" +) + +// refTable names elements for the model. A ref is issued once and never +// reused, so one the model read earlier cannot come to mean a different +// element; the same element keeps its ref across snapshots of one document. +type refTable struct { + mu sync.Mutex + next int + byRef map[string]refTarget + byNode map[refKey]string +} + +type refKey struct { + tab string + node int64 +} + +type refTarget struct { + tab string + node int64 + retired bool +} + +func (r *refTable) refFor(tab string, node int64) string { + r.mu.Lock() + defer r.mu.Unlock() + if r.byRef == nil { + r.byRef, r.byNode = map[string]refTarget{}, map[refKey]string{} + } + key := refKey{tab, node} + if ref, ok := r.byNode[key]; ok { + return ref + } + r.next++ + ref := fmt.Sprintf("e%d", r.next) + r.byNode[key] = ref + r.byRef[ref] = refTarget{tab: tab, node: node} + return ref +} + +func (r *refTable) resolve(ref string) (refTarget, error) { + ref = strings.TrimSpace(ref) + r.mu.Lock() + defer r.mu.Unlock() + target, ok := r.byRef[ref] + switch { + case !ok: + return refTarget{}, &Failure{Code: CodeUnknownRef, Ref: ref, Detail: fmt.Sprintf("%s was never issued; refs come from a snapshot", ref)} + case target.retired: + return refTarget{}, &Failure{Code: CodeStaleRef, Ref: ref, Detail: fmt.Sprintf("%s belonged to a page that has since navigated away; take a new snapshot", ref)} + } + return target, nil +} + +// retireTab ends every ref issued on a tab's current document. +func (r *refTable) retireTab(tab string) { + r.mu.Lock() + defer r.mu.Unlock() + for key, ref := range r.byNode { + if key.tab != tab { + continue + } + delete(r.byNode, key) + target := r.byRef[ref] + target.retired = true + r.byRef[ref] = target + } +} diff --git a/internal/browser/session.go b/internal/browser/session.go new file mode 100644 index 0000000000..2ce9a1292d --- /dev/null +++ b/internal/browser/session.go @@ -0,0 +1,351 @@ +package browser + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" +) + +const navigationTimeout = 30 * time.Second + +// Config is what a Session needs from its host. Launch.Executable is the +// configured browser, or empty to find one when the browser is first needed; +// Roots bound which local files a page may be. +type Config struct { + Launch LaunchSpec + Roots []string + Pool *Pool +} + +// Session is one agent's view of the browser: the tabs it opened and the refs +// it has been shown. Its browser starts on first use, is shared with other +// sessions on the same profile, and is released by Close. +type Session struct { + cfg Config + + mu sync.Mutex + eng *engine + stopEng func() + tabs []*tab + active *tab + nextTab int + refs refTable + closed bool + openedBy map[string]string // popup target id → opener tab id +} + +func NewSession(cfg Config) *Session { + if cfg.Pool == nil { + cfg.Pool = &Pool{} + } + return &Session{cfg: cfg, openedBy: map[string]string{}} +} + +// TabInfo is what a tab is showing. +type TabInfo struct { + ID string `json:"id"` + URL string `json:"url"` + Title string `json:"title"` + Active bool `json:"active"` +} + +// ensureEngine returns a live browser, starting one when this session has none +// or the one it had is gone. Tabs on a browser that went away go with it. +func (s *Session) ensureEngine(ctx context.Context) (*engine, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return nil, fail(CodeEngineFailed, "the browser session is closed") + } + if s.eng != nil && s.eng.alive() { + return s.eng, nil + } + if s.eng != nil { + s.dropEngineLocked() + } + spec := s.cfg.Launch + exe, err := Discover(spec.Executable) + if err != nil { + return nil, err + } + spec.Executable = exe + eng, err := s.cfg.Pool.acquire(ctx, spec) + if err != nil { + return nil, err + } + s.eng = eng + s.stopEng = eng.listen(s.onBrowserEvent) + return eng, nil +} + +func (s *Session) dropEngineLocked() { + if s.stopEng != nil { + s.stopEng() + s.stopEng = nil + } + for _, t := range s.tabs { + t.detach() + } + s.tabs, s.active = nil, nil + eng := s.eng + s.eng = nil + go s.cfg.Pool.release(eng) +} + +// Close closes this session's tabs and releases its browser. +func (s *Session) Close() { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.closed = true + eng, tabs := s.eng, s.tabs + s.mu.Unlock() + if eng == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), closeTimeout) + defer cancel() + for _, t := range tabs { + _ = eng.conn.call(ctx, "", "Target.closeTarget", map[string]any{"targetId": t.targetID}, nil) + t.detach() + } + s.mu.Lock() + if s.stopEng != nil { + s.stopEng() + } + s.eng = nil + s.mu.Unlock() + s.cfg.Pool.release(eng) +} + +// Origin is the scheme, host and port of the page a tab shows ("" is the +// active tab), or "" when it shows no page. A local file's origin is file://. +func (s *Session) Origin(tabID string) string { + t, err := s.tab(tabID) + if err != nil { + return "" + } + raw, _ := t.location() + return OriginOf(raw) +} + +// Tabs lists this session's tabs in the order they opened. +func (s *Session) Tabs() []TabInfo { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]TabInfo, 0, len(s.tabs)) + for _, t := range s.tabs { + url, title := t.location() + out = append(out, TabInfo{ID: t.id, URL: url, Title: title, Active: t == s.active}) + } + return out +} + +// tab resolves a tab id, "" meaning the active tab. +func (s *Session) tab(id string) (*tab, error) { + s.mu.Lock() + defer s.mu.Unlock() + if id == "" { + if s.active == nil { + return nil, fail(CodeNoTab, "no page is open; open one first") + } + return s.active, nil + } + for _, t := range s.tabs { + if t.id == id { + return t, nil + } + } + return nil, fail(CodeNoTab, "no tab %s in this session", id) +} + +// Open navigates a tab to rawURL and waits for it to load. newTab opens a fresh +// tab; otherwise the named tab, or the active one, navigates. +func (s *Session) Open(ctx context.Context, rawURL, tabID string, newTab bool) (TabInfo, error) { + target, err := checkURL(rawURL, s.cfg.Roots) + if err != nil { + return TabInfo{}, err + } + eng, err := s.ensureEngine(ctx) + if err != nil { + return TabInfo{}, err + } + var t *tab + if newTab || (tabID == "" && s.activeTab() == nil) { + t, err = s.createTab(ctx, eng) + } else { + t, err = s.tab(tabID) + } + if err != nil { + return TabInfo{}, err + } + s.activate(t) + if err := t.navigate(ctx, target); err != nil { + return t.info(true), err + } + return t.info(true), nil +} + +func (s *Session) activeTab() *tab { + s.mu.Lock() + defer s.mu.Unlock() + return s.active +} + +func (s *Session) activate(t *tab) { + s.mu.Lock() + s.active = t + s.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), closeTimeout) + defer cancel() + _ = t.call(ctx, "Page.bringToFront", nil, nil) +} + +// Switch makes a tab the active one. +func (s *Session) Switch(tabID string) (TabInfo, error) { + t, err := s.tab(tabID) + if err != nil { + return TabInfo{}, err + } + s.activate(t) + return t.info(true), nil +} + +// CloseTab closes one tab; the most recently opened remaining tab becomes active. +func (s *Session) CloseTab(ctx context.Context, tabID string) error { + t, err := s.tab(tabID) + if err != nil { + return err + } + _ = t.eng.conn.call(ctx, "", "Target.closeTarget", map[string]any{"targetId": t.targetID}, nil) + s.removeTab(t) + return nil +} + +func (s *Session) createTab(ctx context.Context, eng *engine) (*tab, error) { + var created struct { + TargetID string `json:"targetId"` + } + if err := eng.conn.call(ctx, "", "Target.createTarget", map[string]any{"url": "about:blank"}, &created); err != nil { + return nil, engineFailure(err) + } + return s.attach(ctx, eng, created.TargetID) +} + +func (s *Session) attach(ctx context.Context, eng *engine, targetID string) (*tab, error) { + var attached struct { + SessionID string `json:"sessionId"` + } + if err := eng.conn.call(ctx, "", "Target.attachToTarget", map[string]any{"targetId": targetID, "flatten": true}, &attached); err != nil { + return nil, engineFailure(err) + } + s.mu.Lock() + s.nextTab++ + t := newTab(s, eng, fmt.Sprintf("t%d", s.nextTab), targetID, attached.SessionID) + s.tabs = append(s.tabs, t) + s.mu.Unlock() + if err := t.enable(ctx); err != nil { + s.removeTab(t) + return nil, engineFailure(err) + } + return t, nil +} + +func (s *Session) removeTab(t *tab) { + t.detach() + s.mu.Lock() + defer s.mu.Unlock() + for i, candidate := range s.tabs { + if candidate == t { + s.tabs = append(s.tabs[:i], s.tabs[i+1:]...) + break + } + } + if s.active == t { + s.active = nil + if n := len(s.tabs); n > 0 { + s.active = s.tabs[n-1] + } + } + s.refs.retireTab(t.id) +} + +// onBrowserEvent follows tabs this session did not open itself: a page one of +// its tabs opened, and any of its tabs closing. +func (s *Session) onBrowserEvent(ev event) { + switch ev.Method { + case "Target.targetCreated": + var p struct { + TargetInfo struct { + TargetID string `json:"targetId"` + Type string `json:"type"` + OpenerID string `json:"openerId"` + } `json:"targetInfo"` + } + if json.Unmarshal(ev.Params, &p) != nil || p.TargetInfo.Type != "page" || p.TargetInfo.OpenerID == "" { + return + } + opener := s.tabByTarget(p.TargetInfo.OpenerID) + if opener == nil { + return + } + s.mu.Lock() + eng := s.eng + s.mu.Unlock() + if eng == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), launchTimeout) + defer cancel() + if popup, err := s.attach(ctx, eng, p.TargetInfo.TargetID); err == nil { + s.mu.Lock() + s.openedBy[popup.targetID] = opener.id + s.active = popup + s.mu.Unlock() + opener.notePopup(popup.id) + } + case "Target.targetDestroyed": + var p struct { + TargetID string `json:"targetId"` + } + if json.Unmarshal(ev.Params, &p) != nil { + return + } + if t := s.tabByTarget(p.TargetID); t != nil { + s.removeTab(t) + } + } +} + +func (s *Session) tabByTarget(targetID string) *tab { + s.mu.Lock() + defer s.mu.Unlock() + for _, t := range s.tabs { + if t.targetID == targetID { + return t + } + } + return nil +} + +// engineFailure attributes an error from the connection itself. A protocol +// error is the browser refusing a command and stays as it is. +func engineFailure(err error) error { + switch { + case err == nil: + return nil + case CodeOf(err) != "": + return err + case errors.Is(err, errConnClosed): + return fail(CodeEngineFailed, "the browser closed; open a page again to restart it") + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + return err + default: + return fail(CodeEngineFailed, "%v", err) + } +} diff --git a/internal/browser/snapshot.go b/internal/browser/snapshot.go new file mode 100644 index 0000000000..29ab11706c --- /dev/null +++ b/internal/browser/snapshot.go @@ -0,0 +1,272 @@ +package browser + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "unicode/utf8" +) + +const ( + maxNameRunes = 160 + maxTextRunes = 300 +) + +// Snapshot is a page as its accessibility tree presents it: one line per +// element, indented by nesting, each actionable element carrying a ref. +type Snapshot struct { + Tab TabInfo + Lines []string + Dialog *Dialog +} + +type axValue struct { + Value json.RawMessage `json:"value"` +} + +func (v *axValue) String() string { + if v == nil || len(v.Value) == 0 { + return "" + } + var s string + if json.Unmarshal(v.Value, &s) == nil { + return s + } + return string(v.Value) +} + +type axNode struct { + NodeID string `json:"nodeId"` + Ignored bool `json:"ignored"` + Role *axValue `json:"role"` + Name *axValue `json:"name"` + Value *axValue `json:"value"` + ChildIDs []string `json:"childIds"` + ParentID string `json:"parentId"` + BackendDOMNodeID int64 `json:"backendDOMNodeId"` + Properties []struct { + Name string `json:"name"` + Value axValue `json:"value"` + } `json:"properties"` +} + +// Snapshot renders a tab's page. scope, when set, is a ref whose subtree alone +// is rendered. +func (s *Session) Snapshot(ctx context.Context, tabID, scope string) (Snapshot, error) { + t, err := s.tab(tabID) + if err != nil { + return Snapshot{}, err + } + if err := t.checkCurrentURL(); err != nil { + return Snapshot{}, err + } + var tree struct { + Nodes []axNode `json:"nodes"` + } + if err := t.call(ctx, "Accessibility.getFullAXTree", nil, &tree); err != nil { + return Snapshot{}, engineFailure(err) + } + r := renderer{s: s, tab: t.id, nodes: make(map[string]*axNode, len(tree.Nodes))} + var root *axNode + for i := range tree.Nodes { + n := &tree.Nodes[i] + r.nodes[n.NodeID] = n + if n.ParentID == "" && root == nil { + root = n + } + } + if scope != "" { + target, err := s.refs.resolve(scope) + if err != nil { + return Snapshot{}, err + } + root = nil + for i := range tree.Nodes { + if tree.Nodes[i].BackendDOMNodeID == target.node { + root = &tree.Nodes[i] + break + } + } + if root == nil { + return Snapshot{}, &Failure{Code: CodeStaleRef, Ref: scope, Detail: fmt.Sprintf("%s is no longer on the page", scope)} + } + r.node(root, 0, "") + } else if root != nil { + r.children(root, 0, root.Name.String()) + t.mu.Lock() + t.seen = seenSnapshot{lines: r.lines, document: t.navigated, valid: true} + t.mu.Unlock() + } + return Snapshot{Tab: t.info(true), Lines: r.lines, Dialog: t.currentDialog()}, nil +} + +// Changes is how a page differs from the last snapshot the agent saw of it. +// NewDocument means there is nothing to compare: the tab navigated since, or +// the agent never saw it whole. +type Changes struct { + NewDocument bool + Added []string + Removed []string +} + +// Refresh snapshots a tab and reports what changed since the agent's last look. +func (s *Session) Refresh(ctx context.Context, tabID string) (Snapshot, Changes, error) { + t, err := s.tab(tabID) + if err != nil { + return Snapshot{}, Changes{}, err + } + t.mu.Lock() + prev := t.seen + t.mu.Unlock() + snap, err := s.Snapshot(ctx, t.id, "") + if err != nil { + return Snapshot{}, Changes{}, err + } + t.mu.Lock() + document := t.navigated + t.mu.Unlock() + if !prev.valid || prev.document != document { + return snap, Changes{NewDocument: true}, nil + } + return snap, diffLines(prev.lines, snap.Lines), nil +} + +// diffLines lists lines that appeared and disappeared, each in page order. A +// line that moved is neither. +func diffLines(before, after []string) Changes { + remaining := make(map[string]int, len(before)) + for _, l := range before { + remaining[l]++ + } + var c Changes + for _, l := range after { + if remaining[l] > 0 { + remaining[l]-- + continue + } + c.Added = append(c.Added, l) + } + for _, l := range before { + if remaining[l] > 0 { + remaining[l]-- + c.Removed = append(c.Removed, l) + } + } + return c +} + +type renderer struct { + s *Session + tab string + nodes map[string]*axNode + lines []string +} + +func role(n *axNode) string { + if n.Role == nil { + return "" + } + return n.Role.String() +} + +// transparent roles carry no meaning of their own when unnamed: their children +// are rendered in their place. +var transparentRoles = map[string]bool{"generic": true, "none": true, "presentation": true, "": true} + +func (r *renderer) children(n *axNode, depth int, parentName string) { + var text []string + flush := func() { + if len(text) == 0 { + return + } + joined := strings.Join(text, " ") + text = nil + if joined == parentName { + return + } + r.lines = append(r.lines, indent(depth)+"- text: "+strconv.Quote(clip(joined, maxTextRunes))) + } + for _, id := range n.ChildIDs { + child := r.nodes[id] + if child == nil { + continue + } + if !child.Ignored && role(child) == "StaticText" { + if s := strings.TrimSpace(child.Name.String()); s != "" { + text = append(text, s) + } + continue + } + flush() + r.node(child, depth, parentName) + } + flush() +} + +func (r *renderer) node(n *axNode, depth int, parentName string) { + kind := role(n) + name := strings.TrimSpace(n.Name.String()) + switch { + case kind == "InlineTextBox" || kind == "LineBreak": + return + case n.Ignored || (transparentRoles[kind] && name == ""): + r.children(n, depth, parentName) + return + } + line := indent(depth) + "- " + kind + if name != "" { + line += " " + strconv.Quote(clip(name, maxNameRunes)) + } + if n.BackendDOMNodeID > 0 { + line += " [" + r.s.refs.refFor(r.tab, n.BackendDOMNodeID) + "]" + } + if v := strings.TrimSpace(n.Value.String()); v != "" && v != name { + line += " value=" + strconv.Quote(clip(v, maxNameRunes)) + } + line += properties(n) + if kind == "Iframe" { + line += " (frame content is not included)" + } + r.lines = append(r.lines, line) + r.children(n, depth+1, name) +} + +func properties(n *axNode) string { + var b strings.Builder + for _, p := range n.Properties { + v := p.Value.String() + switch p.Name { + case "focused", "disabled", "required", "readonly", "multiselectable": + if v == "true" { + b.WriteString(" " + p.Name) + } + case "checked", "pressed", "selected", "expanded": + if v == "true" || v == "false" || v == "mixed" { + b.WriteString(" " + p.Name + "=" + v) + } + case "invalid": + if v != "" && v != "false" { + b.WriteString(" invalid") + } + case "level": + b.WriteString(" level=" + v) + case "url": + if v != "" { + b.WriteString(" → " + clip(v, maxNameRunes)) + } + } + } + return b.String() +} + +func indent(depth int) string { return strings.Repeat(" ", depth) } + +func clip(s string, limit int) string { + if utf8.RuneCountInString(s) <= limit { + return s + } + runes := []rune(s) + return string(runes[:limit]) + "…" +} diff --git a/internal/browser/tab.go b/internal/browser/tab.go new file mode 100644 index 0000000000..c72a7ec544 --- /dev/null +++ b/internal/browser/tab.go @@ -0,0 +1,540 @@ +package browser + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "time" +) + +const ( + maxLogEntries = 300 + maxOpenRequests = 2000 +) + +// LogEntry is one thing a page reported: console output, an uncaught +// exception, or a request that failed. +type LogEntry struct { + Seq int64 `json:"seq"` + Kind string `json:"kind"` // console | exception | network + Level string `json:"level"` // error | warning | info + Text string `json:"text"` +} + +// Dialog is a JavaScript dialog blocking the page. +type Dialog struct { + Type string `json:"type"` + Message string `json:"message"` +} + +type tab struct { + s *Session + eng *engine + id string + targetID string + sessionID string + + mu sync.Mutex + unsub func() + url string + title string + mainFrame string + loaded map[string]bool // loader ids whose load event has fired + changed chan struct{} // closed and replaced on every load or navigation + dialog *Dialog + logs []LogEntry + logSeq int64 + requests map[string]string + popups []string + navigated int64 // main-frame cross-document navigations so far + logRead int64 // newest log seq a Logs call has returned + shotScale float64 // CSS pixels per pixel of the latest screenshot + seen seenSnapshot +} + +// seenSnapshot is the last whole-page snapshot the agent was given, and which +// document it was of. +type seenSnapshot struct { + lines []string + document int64 + valid bool +} + +func newTab(s *Session, eng *engine, id, targetID, sessionID string) *tab { + t := &tab{ + s: s, eng: eng, id: id, targetID: targetID, sessionID: sessionID, + loaded: map[string]bool{}, changed: make(chan struct{}), requests: map[string]string{}, + } + t.unsub = eng.conn.subscribe(sessionID, t.onEvent) + return t +} + +// call sends a command to this tab. A JavaScript dialog stops the page from +// answering anything that runs on it, so such a command fails at once while a +// dialog is open and is abandoned when one opens while it waits. +func (t *tab) call(ctx context.Context, method string, params, out any) error { + input := strings.HasPrefix(method, "Input.") + if !input && !waitsOnPage(method) { + return t.eng.conn.call(ctx, t.sessionID, method, params, out) + } + t.mu.Lock() + dialog, changed := t.dialog, t.changed + t.mu.Unlock() + if dialog != nil { + return dialogFailure(dialog) + } + callCtx, cancel := context.WithCancel(ctx) + defer cancel() + watching := make(chan struct{}) + defer close(watching) + go func() { + for { + select { + case <-changed: + case <-watching: + return + } + t.mu.Lock() + open := t.dialog != nil + changed = t.changed + t.mu.Unlock() + if open { + cancel() + return + } + } + }() + err := t.eng.conn.call(callCtx, t.sessionID, method, params, out) + if err != nil && ctx.Err() == nil && callCtx.Err() != nil { + if d := t.currentDialog(); d != nil { + if input { + return nil // the input was delivered; the dialog is what it did + } + return dialogFailure(d) + } + } + return err +} + +// waitsOnPage reports whether a command is answered by the page itself rather +// than by the browser around it. +func waitsOnPage(method string) bool { + for _, prefix := range []string{"Runtime.", "DOM.", "Accessibility.", "Page.captureScreenshot", "Page.getLayoutMetrics"} { + if strings.HasPrefix(method, prefix) { + return true + } + } + return false +} + +func dialogFailure(d *Dialog) *Failure { + return &Failure{Code: CodeDialogOpen, Detail: fmt.Sprintf("a %s dialog is open (%q); answer it with a dialog step first", d.Type, d.Message)} +} + +func (t *tab) enable(ctx context.Context) error { + for _, method := range []string{"Page.enable", "Runtime.enable", "Network.enable"} { + if err := t.call(ctx, method, nil, nil); err != nil { + return err + } + } + if err := t.call(ctx, "Page.setLifecycleEventsEnabled", map[string]any{"enabled": true}, nil); err != nil { + return err + } + var tree struct { + FrameTree struct { + Frame struct { + ID string `json:"id"` + URL string `json:"url"` + } `json:"frame"` + } `json:"frameTree"` + } + if err := t.call(ctx, "Page.getFrameTree", nil, &tree); err != nil { + return err + } + t.mu.Lock() + t.mainFrame, t.url = tree.FrameTree.Frame.ID, tree.FrameTree.Frame.URL + t.mu.Unlock() + return nil +} + +func (t *tab) detach() { + t.mu.Lock() + unsub := t.unsub + t.unsub = nil + t.mu.Unlock() + if unsub != nil { + unsub() + } +} + +func (t *tab) location() (url, title string) { + t.mu.Lock() + defer t.mu.Unlock() + return t.url, t.title +} + +// info reads the title fresh when asked to, since pages set it after load. +func (t *tab) info(refreshTitle bool) TabInfo { + if refreshTitle { + ctx, cancel := context.WithTimeout(context.Background(), closeTimeout) + var r struct { + Result struct { + Value string `json:"value"` + } `json:"result"` + } + if t.call(ctx, "Runtime.evaluate", map[string]any{"expression": "document.title", "returnByValue": true}, &r) == nil { + t.mu.Lock() + t.title = r.Result.Value + t.mu.Unlock() + } + cancel() + } + url, title := t.location() + return TabInfo{ID: t.id, URL: url, Title: title, Active: t.s.activeTab() == t} +} + +func (t *tab) signalLocked() { + close(t.changed) + t.changed = make(chan struct{}) +} + +func (t *tab) onEvent(ev event) { + switch ev.Method { + case "Page.frameNavigated": + var p struct { + Frame struct { + ID string `json:"id"` + ParentID string `json:"parentId"` + URL string `json:"url"` + URLFragment string `json:"urlFragment"` + } `json:"frame"` + } + if json.Unmarshal(ev.Params, &p) != nil || p.Frame.ParentID != "" { + return + } + t.mu.Lock() + t.mainFrame, t.url = p.Frame.ID, p.Frame.URL+p.Frame.URLFragment + t.navigated++ + t.signalLocked() + t.mu.Unlock() + t.s.refs.retireTab(t.id) + case "Page.navigatedWithinDocument": + var p struct { + FrameID string `json:"frameId"` + URL string `json:"url"` + } + if json.Unmarshal(ev.Params, &p) != nil { + return + } + t.mu.Lock() + if p.FrameID == t.mainFrame { + t.url = p.URL + t.signalLocked() + } + t.mu.Unlock() + case "Page.lifecycleEvent": + var p struct { + FrameID string `json:"frameId"` + LoaderID string `json:"loaderId"` + Name string `json:"name"` + } + if json.Unmarshal(ev.Params, &p) != nil || p.Name != "load" { + return + } + t.mu.Lock() + if p.FrameID == t.mainFrame { + t.loaded[p.LoaderID] = true + t.signalLocked() + } + t.mu.Unlock() + case "Page.javascriptDialogOpening": + var d Dialog + if json.Unmarshal(ev.Params, &d) != nil { + return + } + t.mu.Lock() + t.dialog = &d + t.signalLocked() + t.mu.Unlock() + case "Page.javascriptDialogClosed": + t.mu.Lock() + t.dialog = nil + t.signalLocked() + t.mu.Unlock() + case "Runtime.consoleAPICalled": + t.onConsole(ev.Params) + case "Runtime.exceptionThrown": + var p struct { + ExceptionDetails struct { + Text string `json:"text"` + Exception struct { + Description string `json:"description"` + } `json:"exception"` + } `json:"exceptionDetails"` + } + if json.Unmarshal(ev.Params, &p) != nil { + return + } + text := p.ExceptionDetails.Exception.Description + if text == "" { + text = p.ExceptionDetails.Text + } + t.log("exception", "error", text) + default: + t.onNetwork(ev) + } +} + +// onNetwork records requests that failed or answered with an error status. +func (t *tab) onNetwork(ev event) { + switch ev.Method { + case "Network.requestWillBeSent": + var p struct { + RequestID string `json:"requestId"` + Request struct { + Method string `json:"method"` + URL string `json:"url"` + } `json:"request"` + } + if json.Unmarshal(ev.Params, &p) != nil { + return + } + t.mu.Lock() + if len(t.requests) < maxOpenRequests { + t.requests[p.RequestID] = p.Request.Method + " " + p.Request.URL + } + t.mu.Unlock() + case "Network.responseReceived": + var p struct { + RequestID string `json:"requestId"` + Response struct { + URL string `json:"url"` + Status int `json:"status"` + } `json:"response"` + } + if json.Unmarshal(ev.Params, &p) != nil || p.Response.Status < 400 { + return + } + t.log("network", "warning", fmt.Sprintf("%s → HTTP %d", t.requestName(p.RequestID, p.Response.URL), p.Response.Status)) + case "Network.loadingFailed": + var p struct { + RequestID string `json:"requestId"` + ErrorText string `json:"errorText"` + Canceled bool `json:"canceled"` + } + if json.Unmarshal(ev.Params, &p) != nil { + return + } + name := t.requestName(p.RequestID, "") + t.forgetRequest(p.RequestID) + if !p.Canceled { + t.log("network", "error", fmt.Sprintf("%s failed: %s", name, p.ErrorText)) + } + case "Network.loadingFinished": + var p struct { + RequestID string `json:"requestId"` + } + if json.Unmarshal(ev.Params, &p) == nil { + t.forgetRequest(p.RequestID) + } + } +} + +func (t *tab) onConsole(params json.RawMessage) { + var p struct { + Type string `json:"type"` + Args []struct { + Type string `json:"type"` + Value json.RawMessage `json:"value"` + Description string `json:"description"` + } `json:"args"` + } + if json.Unmarshal(params, &p) != nil { + return + } + parts := make([]string, 0, len(p.Args)) + for _, a := range p.Args { + var s string + switch { + case a.Type == "string" && json.Unmarshal(a.Value, &s) == nil: + parts = append(parts, s) + case len(a.Value) > 0: + parts = append(parts, string(a.Value)) + default: + parts = append(parts, a.Description) + } + } + level := "info" + switch p.Type { + case "error", "assert": + level = "error" + case "warning": + level = "warning" + } + t.log("console", level, strings.Join(parts, " ")) +} + +func (t *tab) requestName(id, fallback string) string { + t.mu.Lock() + defer t.mu.Unlock() + if name, ok := t.requests[id]; ok { + return name + } + return fallback +} + +func (t *tab) forgetRequest(id string) { + t.mu.Lock() + delete(t.requests, id) + t.mu.Unlock() +} + +func (t *tab) log(kind, level, text string) { + t.mu.Lock() + defer t.mu.Unlock() + t.logSeq++ + t.logs = append(t.logs, LogEntry{Seq: t.logSeq, Kind: kind, Level: level, Text: text}) + if n := len(t.logs); n > maxLogEntries { + t.logs = append([]LogEntry(nil), t.logs[n-maxLogEntries:]...) + } +} + +// logsAfter returns entries newer than seq, and the newest seq it has seen. +func (t *tab) logsAfter(seq int64) ([]LogEntry, int64) { + t.mu.Lock() + defer t.mu.Unlock() + var out []LogEntry + for _, e := range t.logs { + if e.Seq > seq { + out = append(out, e) + } + } + return out, t.logSeq +} + +func (t *tab) notePopup(id string) { + t.mu.Lock() + t.popups = append(t.popups, id) + t.mu.Unlock() +} + +func (t *tab) currentDialog() *Dialog { + t.mu.Lock() + defer t.mu.Unlock() + if t.dialog == nil { + return nil + } + d := *t.dialog + return &d +} + +// navigate loads url in this tab and waits for its load event. +func (t *tab) navigate(ctx context.Context, url string) error { + var r struct { + LoaderID string `json:"loaderId"` + ErrorText string `json:"errorText"` + } + if err := t.call(ctx, "Page.navigate", map[string]any{"url": url}, &r); err != nil { + return engineFailure(err) + } + if r.ErrorText != "" { + return fail(CodeNavigationFailed, "%s: %s", url, r.ErrorText) + } + if r.LoaderID == "" { + return nil + } + return t.waitLoaded(ctx, r.LoaderID, navigationTimeout) +} + +func (t *tab) waitLoaded(ctx context.Context, loaderID string, limit time.Duration) error { + deadline := time.NewTimer(limit) + defer deadline.Stop() + for { + t.mu.Lock() + done, changed, dialog := t.loaded[loaderID], t.changed, t.dialog + t.mu.Unlock() + if done { + return nil + } + if dialog != nil { + return dialogFailure(dialog) + } + select { + case <-changed: + case <-deadline.C: + return fail(CodeNavigationTimeout, "the page did not finish loading within %s; it may still be usable", limit) + case <-ctx.Done(): + return ctx.Err() + case <-t.eng.conn.closed(): + return engineFailure(errConnClosed) + } + } +} + +// settle waits for a navigation an input may have started. Nothing starting +// within the grace period is the common case and not a failure. +func (t *tab) settle(ctx context.Context, before int64) { + grace := time.NewTimer(150 * time.Millisecond) + defer grace.Stop() + for { + t.mu.Lock() + navigated, changed := t.navigated > before, t.changed + t.mu.Unlock() + if navigated { + _ = t.waitAnyLoad(ctx, navigationTimeout) + return + } + select { + case <-changed: + case <-grace.C: + return + case <-ctx.Done(): + return + } + } +} + +// waitAnyLoad waits until the document the main frame is now showing has loaded. +func (t *tab) waitAnyLoad(ctx context.Context, limit time.Duration) error { + var r struct { + Result struct { + Value string `json:"value"` + } `json:"result"` + } + poll := time.NewTicker(100 * time.Millisecond) + defer poll.Stop() + deadline := time.NewTimer(limit) + defer deadline.Stop() + for { + if t.call(ctx, "Runtime.evaluate", map[string]any{"expression": "document.readyState", "returnByValue": true}, &r) == nil && r.Result.Value == "complete" { + return nil + } + select { + case <-poll.C: + case <-deadline.C: + return fail(CodeNavigationTimeout, "the page did not finish loading within %s", limit) + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (t *tab) navigationCount() int64 { + t.mu.Lock() + defer t.mu.Unlock() + return t.navigated +} + +// checkCurrentURL refuses a page the agent could not have opened itself, which +// a link from an allowed local file can still reach. The browser's own error +// page is what a failed load leaves behind and stays readable. +func (t *tab) checkCurrentURL() error { + url, _ := t.location() + if url == "" || strings.HasPrefix(url, "about:") || strings.HasPrefix(url, "chrome-error:") { + return nil + } + _, err := checkURL(url, t.s.cfg.Roots) + return err +} diff --git a/internal/browser/transport.go b/internal/browser/transport.go new file mode 100644 index 0000000000..e72bc6c12d --- /dev/null +++ b/internal/browser/transport.go @@ -0,0 +1,87 @@ +package browser + +import ( + "bufio" + "bytes" + "context" + "io" + "sync" + + "github.com/gorilla/websocket" +) + +// transport carries whole CDP messages. Chrome speaks the same JSON over a +// pair of pipes or a WebSocket; only the framing differs. +type transport interface { + read() ([]byte, error) + write([]byte) error + close() error +} + +// pipeTransport frames messages with a trailing NUL, the --remote-debugging-pipe +// protocol. Nothing listens on a port, so no other local process can drive the +// browser. +type pipeTransport struct { + r *bufio.Reader + w io.Writer + closers []io.Closer + wmu sync.Mutex +} + +func newPipeTransport(r io.Reader, w io.Writer, closers ...io.Closer) *pipeTransport { + return &pipeTransport{r: bufio.NewReaderSize(r, 1<<20), w: w, closers: closers} +} + +func (p *pipeTransport) read() ([]byte, error) { + msg, err := p.r.ReadBytes(0) + if err != nil { + return nil, err + } + return bytes.TrimSuffix(msg, []byte{0}), nil +} + +func (p *pipeTransport) write(msg []byte) error { + p.wmu.Lock() + defer p.wmu.Unlock() + _, err := p.w.Write(append(msg, 0)) + return err +} + +func (p *pipeTransport) close() error { + var first error + for _, c := range p.closers { + if err := c.Close(); err != nil && first == nil { + first = err + } + } + return first +} + +// wsTransport is the port-based endpoint, used where a child cannot inherit +// extra pipe descriptors. +type wsTransport struct { + c *websocket.Conn + wmu sync.Mutex +} + +func dialWS(ctx context.Context, url string) (*wsTransport, error) { + dialer := websocket.Dialer{ReadBufferSize: 1 << 16, WriteBufferSize: 1 << 16} + c, _, err := dialer.DialContext(ctx, url, nil) + if err != nil { + return nil, err + } + return &wsTransport{c: c}, nil +} + +func (t *wsTransport) read() ([]byte, error) { + _, msg, err := t.c.ReadMessage() + return msg, err +} + +func (t *wsTransport) write(msg []byte) error { + t.wmu.Lock() + defer t.wmu.Unlock() + return t.c.WriteMessage(websocket.TextMessage, msg) +} + +func (t *wsTransport) close() error { return t.c.Close() } diff --git a/tools/covergate/baseline.json b/tools/covergate/baseline.json index 77245780df..0176bce8c9 100644 --- a/tools/covergate/baseline.json +++ b/tools/covergate/baseline.json @@ -1,4 +1,5 @@ { + "internal/browser/**": 21.7, "internal/control/approval.go": 94.3, "internal/control/approval_bridge.go": 77, "internal/control/approval_orchestration.go": 90.7, diff --git a/tools/repolint/layers.go b/tools/repolint/layers.go index a0f04ebc9f..5612aa3f63 100644 --- a/tools/repolint/layers.go +++ b/tools/repolint/layers.go @@ -26,6 +26,7 @@ var leaves = []string{ "internal/agentgraph", "internal/agentpreset", "internal/billing", + "internal/browser", "internal/diff", "internal/extension/rpcwire", "internal/execgraph",