fix(studio): input that carries a point is scaled with a fitted page or refused

desktop/electron/package.json

@@ -15,6 +15,7 @@
     "build:dist": "electron-builder",
     "smoke": "electron test/smoke.js",
     "browser-live": "electron test/browser_live.js",
+    "browser-fit-live": "electron test/browser_fit_live.js",
     "image-live": "electron test/image_live.js",
     "image-pane-live": "electron test/image_pane_live.js",
     "sidepanel-live": "electron test/sidepanel_live.js",

desktop/electron/src/browserviews.js

@@ -116,7 +116,7 @@ class BrowserViews {
       }
       onEvent(method, params);
     });
-    const entry = { view, onDownload, typed: 0, synthUntil: 0, attached: false };
+    const entry = { view, onDownload, typed: 0, synthUntil: 0, attached: false, panel: null, override: null, scale: 1 };
     this.keyboard.watch(contents, () => Date.now() < entry.synthUntil);
     this.watchLoads(targetId, entry);
     let closed = false;
@@ -140,6 +140,13 @@ class BrowserViews {
     return {
       targetId,
       send: (method, params) => {
+        if (method === "Emulation.setDeviceMetricsOverride") return this.emulate(entry, params);
+        if (method === "Emulation.clearDeviceMetricsOverride") return this.emulate(entry, null);
+        try {
+          params = scaleInput(method, params, entry.scale);
+        } catch (e) {
+          return Promise.reject(e);
+        }
         if (!method.startsWith("Input.")) return contents.debugger.sendCommand(method, params);
         entry.synthUntil = Date.now() + 250;
         return this.wake.run(() => contents.debugger.sendCommand(method, params));
@@ -164,7 +171,7 @@ class BrowserViews {
     // which during a turn is constantly, and nearly always the same rectangle:
     // re-adding the view and setting the bounds it already has is work for
     // every one of those.
-    if (this.shown === targetId && sameRect(entry.at, at)) return;
+    if (this.shown === targetId && sameRect(entry.panel, at)) return;
     this.placed++;
     for (const [id, other] of this.entries) {
       if (id !== targetId) this.putAway(other.view, other);
@@ -176,9 +183,48 @@ class BrowserViews {
     if (entry.attached && !entry.at) this.win.contentView.removeChildView(entry.view);
     this.win.contentView.addChildView(entry.view);
     entry.attached = true;
+    entry.panel = at;
+    this.fit(entry);
+    this.keepKeyboard();
+  }
+
+  // fit puts the view where its page is drawn whole and tells the page how it
+  // is scaled; a page parked out of sight is not scaled.
+  fit(entry) {
+    const { at, scale } = placement(entry.panel, entry.override && { width: entry.override.width, height: entry.override.height });
     entry.view.setBounds(at);
     entry.at = at;
-    this.keepKeyboard();
+    return this.scaled(entry, scale);
+  }
+
+  scaled(entry, scale) {
+    if (!entry.override || entry.scale === scale) return Promise.resolve();
+    entry.scale = scale;
+    return this.override(entry).catch(() => {});
+  }
+
+  override(entry) {
+    const params = entry.scale === 1 ? entry.override : { ...entry.override, scale: entry.scale };
+    return entry.view.webContents.debugger.sendCommand("Emulation.setDeviceMetricsOverride", params);
+  }
+
+  // emulate records the viewport a page was told to lay out in, which the view
+  // is then fitted to; null is the page going back to the size of its view.
+  async emulate(entry, params) {
+    entry.override = params ? { ...params } : null;
+    if (!params) {
+      entry.scale = 1;
+      if (entry.panel) this.fit(entry);
+      return entry.view.webContents.debugger.sendCommand("Emulation.clearDeviceMetricsOverride", {});
+    }
+    entry.scale = 1;
+    if (entry.panel) {
+      const { at, scale } = placement(entry.panel, entry.override);
+      entry.view.setBounds(at);
+      entry.at = at;
+      entry.scale = scale;
+    }
+    return this.override(entry);
   }
 
   // Attaching a view can leave nobody holding the keyboard, and the platform
@@ -253,7 +299,11 @@ class BrowserViews {
   putAway(view, entry) {
     const { height } = this.win.getContentBounds();
     view.setBounds({ x: 1 - AGENT_VIEWPORT.width, y: height - 1, ...AGENT_VIEWPORT });
-    if (entry) entry.at = null;
+    if (entry) {
+      entry.at = null;
+      entry.panel = null;
+      void this.scaled(entry, 1);
+    }
   }
 
   control(targetId, action) {
@@ -420,6 +470,42 @@ async function loadResult(contents, url) {
   }
 }
 
+// Where a page's view sits in the rectangle the panel gave it. A page laid out
+// in a viewport of its own is shown whole: scaled down to fit and never up,
+// centred, with the view sized to exactly what it draws.
+function placement(panel, emulated) {
+  if (!emulated) return { at: panel, scale: 1 };
+  const scale = Math.min(1, panel.width / emulated.width, panel.height / emulated.height);
+  const width = Math.max(1, Math.round(emulated.width * scale));
+  const height = Math.max(1, Math.round(emulated.height * scale));
+  const x = panel.x + Math.round((panel.width - width) / 2);
+  const y = panel.y + Math.round((panel.height - height) / 2);
+  return { at: { x, y, width, height }, scale };
+}
+
+// Input that carries a point. The page reads it in the viewport it was told it
+// has, the view draws that viewport scaled, and a point meant for the page lands
+// where it is drawn only after the same scaling. The mouse is measured to; the
+// rest are not, and a point that silently lands elsewhere is worse than a
+// refusal, so they are refused while the page is scaled.
+const POINTED_INPUT = new Set([
+  "Input.dispatchTouchEvent",
+  "Input.dispatchDragEvent",
+  "Input.emulateTouchFromMouseEvent",
+  "Input.synthesizeTapGesture",
+  "Input.synthesizeScrollGesture",
+  "Input.synthesizePinchGesture",
+]);
+
+function scaleInput(method, params, scale) {
+  if (scale === 1) return params;
+  if (method === "Input.dispatchMouseEvent") return { ...params, x: params.x * scale, y: params.y * scale };
+  if (POINTED_INPUT.has(method)) {
+    throw Object.assign(new Error(`${method} is not available while the page is scaled to fit the panel`), { code: "input_unscaled" });
+  }
+  return params;
+}
+
 function sameRect(a, b) {
   return !!a && a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height;
 }
@@ -428,4 +514,4 @@ function validRect(rect) {
   return rect && [rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) && rect.width > 0 && rect.height > 0;
 }
 
-module.exports = { BrowserViews, AGENT_VIEWPORT, LEAVE_LIMIT_MS };
+module.exports = { BrowserViews, AGENT_VIEWPORT, LEAVE_LIMIT_MS, placement, scaleInput };

desktop/electron/test/browser_fit_live.js

@@ -0,0 +1,82 @@
+"use strict";
+// A page the agent lays out in a viewport of its own (a resize) is shown whole in the panel, not cropped to its
+// top-left corner: scaled down to fit and centred, with the agent's clicks still landing where they were aimed.
+// Run with `pnpm browser-fit-live`; it needs no network. SHOTS=<dir> also writes pictures of the window.
+const fs = require("node:fs");
+const http = require("node:http");
+const { execFileSync } = require("node:child_process");
+const { app, BrowserWindow } = require("electron");
+const { BrowserViews } = require("../src/browserviews");
+const { checker } = require("./livekit");
+
+const PAGE = `<!doctype html><title>wide</title><style>body{margin:0;font:20px sans-serif}.bar{display:flex;justify-content:space-between;background:#1a56db;color:#fff;padding:14px;width:1280px;box-sizing:border-box}.c{width:1280px;height:600px;background:linear-gradient(90deg,#fde68a,#a7f3d0);position:relative}.r{position:absolute;right:10px;top:10px;background:#111;color:#fff;padding:8px}</style><div class=bar><b>LEFT edge</b><b>RIGHT edge</b></div><div class=c><span class=r id=rc onclick="document.title='hit'">right column</span></div>`;
+const { check, failures } = checker();
+const wait = (ms) => new Promise((r) => setTimeout(r, ms));
+
+app.whenReady().then(async () => {
+  const server = http.createServer((_q, s) => { s.writeHead(200, { "Content-Type": "text/html" }); s.end(PAGE); });
+  await new Promise((r) => server.listen(0, "127.0.0.1", r));
+  const win = new BrowserWindow({ width: 1000, height: 760, show: true });
+  await win.loadURL("data:text/html,<body style='margin:0;background:%23ddd;font:14px sans-serif'><div style='position:absolute;left:20px;top:20px'>app chrome</div>");
+  const views = new BrowserViews({ win, kernelOrigin: "http://127.0.0.1:1" });
+  const page = views.create({ partition: "fit", url: `http://127.0.0.1:${server.address().port}/`, onEvent() {}, onClosed() {}, onPopup() {}, onDownload() {} });
+  const view = views.entries.get(page.targetId).view;
+  await wait(1500);
+  const evalIn = (expression) => page.send("Runtime.evaluate", { expression, returnByValue: true }).then((r) => r.result.value);
+  const shot = (name) => {
+    if (!process.env.SHOTS) return;
+    fs.mkdirSync(process.env.SHOTS, { recursive: true });
+    win.focus();
+    const b = win.getBounds();
+    execFileSync("screencapture", ["-x", "-R", `${b.x},${b.y},${b.width},${b.height}`, `${process.env.SHOTS}/${name}.png`]);
+  };
+  const click = async () => {
+    await evalIn("document.title='reset'");
+    const r = JSON.parse(await evalIn("JSON.stringify(document.getElementById('rc').getBoundingClientRect())"));
+    for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) {
+      await page.send("Input.dispatchMouseEvent", { type, x: r.x + r.width / 2, y: r.y + r.height / 2, button: "left", clickCount: 1 });
+    }
+    await wait(250);
+    return evalIn("document.title");
+  };
+  const panel = { x: 380, y: 60, width: 560, height: 620 };
+  views.show(page.targetId, panel);
+  await wait(600);
+  check("no resize: the page fills the panel as before", JSON.stringify(view.getBounds()) === JSON.stringify(panel));
+
+  await page.send("Emulation.setDeviceMetricsOverride", { width: 1280, height: 900, deviceScaleFactor: 0, mobile: false });
+  await wait(600);
+  const fit = view.getBounds();
+  check("a 1280x900 viewport in a 560x620 panel: the view is scaled to 560x394 and centred", fit.width === 560 && fit.height === 394 && fit.x === 380 && fit.y === 173, JSON.stringify(fit));
+  check("the page still lays out at the viewport it was given", (await evalIn("innerWidth")) === 1280);
+  check("an agent click aimed at the right column lands on it", (await click()) === "hit");
+  shot("fit-in-panel");
+
+  views.show(page.targetId, { x: 380, y: 60, width: 400, height: 500 });
+  await wait(600);
+  const narrower = view.getBounds();
+  check("a narrower panel re-fits the view", narrower.width === 400 && narrower.height === 281, JSON.stringify(narrower));
+  check("and the click still lands", (await click()) === "hit");
+
+  views.show(page.targetId, { x: 20, y: 20, width: 1400, height: 1000 });
+  await wait(600);
+  const roomy = view.getBounds();
+  check("a panel larger than the viewport shows it at full size, centred, never enlarged", roomy.width === 1280 && roomy.height === 900 && roomy.x === 80 && roomy.y === 70, JSON.stringify(roomy));
+  check("and the click lands at full size", (await click()) === "hit");
+
+  views.hide();
+  await wait(600);
+  check("parked, the page is not scaled and the click lands", (await click()) === "hit");
+
+  views.show(page.targetId, panel);
+  await wait(600);
+  check("shown again it is fitted again", view.getBounds().width === 560 && (await click()) === "hit");
+
+  await page.send("Emulation.clearDeviceMetricsOverride", {});
+  await wait(600);
+  check("clearing the viewport gives the page its panel back", JSON.stringify(view.getBounds()) === JSON.stringify(panel) && (await evalIn("innerWidth")) === 560);
+  shot("cleared");
+
+  console.log(failures.length ? `\n${failures.length} failed` : "\nall passed");
+  app.exit(failures.length ? 1 : 0);
+});

desktop/electron/test/unit.mjs

@@ -2281,6 +2281,55 @@ test("a parked page is attached again when it is shown, and a page already on sc
   assert.deepEqual(calls, ["remove", "add", "bounds 10"], "a page put away and shown again is attached again");
 });
 
+test("a page given its own viewport is fitted whole into its panel, centred, and never enlarged", () => {
+  const Module = require("node:module");
+  const load = Module._load;
+  Module._load = function (request, ...rest) {
+    if (request === "electron") return { WebContentsView: class {}, session: { fromPartition: () => ({}) } };
+    return load.call(this, request, ...rest);
+  };
+  let placement;
+  try {
+    ({ placement } = require("../src/browserviews.js"));
+  } finally {
+    Module._load = load;
+  }
+  const panel = { x: 380, y: 60, width: 560, height: 620 };
+  assert.deepEqual(placement(panel, null), { at: panel, scale: 1 }, "a page without a viewport of its own fills the panel");
+  const wide = placement(panel, { width: 1280, height: 900 });
+  assert.equal(wide.scale, 0.4375);
+  assert.deepEqual(wide.at, { x: 380, y: 173, width: 560, height: 394 });
+  const phone = placement(panel, { width: 390, height: 844 });
+  assert.ok(phone.scale < 1 && phone.at.height <= panel.height && phone.at.width <= panel.width);
+  assert.deepEqual(placement({ x: 20, y: 20, width: 1400, height: 1000 }, { width: 1280, height: 900 }), { at: { x: 80, y: 70, width: 1280, height: 900 }, scale: 1 });
+});
+
+test("input that carries a point is scaled with the page or refused, never sent to the wrong place", () => {
+  const Module = require("node:module");
+  const load = Module._load;
+  Module._load = function (request, ...rest) {
+    if (request === "electron") return { WebContentsView: class {}, session: { fromPartition: () => ({}) } };
+    return load.call(this, request, ...rest);
+  };
+  let scaleInput;
+  try {
+    ({ scaleInput } = require("../src/browserviews.js"));
+  } finally {
+    Module._load = load;
+  }
+  const mouse = { type: "mousePressed", x: 1200, y: 800, button: "left" };
+  assert.equal(scaleInput("Input.dispatchMouseEvent", mouse, 1), mouse, "an unscaled page gets the point as it is");
+  assert.deepEqual(scaleInput("Input.dispatchMouseEvent", mouse, 0.5), { type: "mousePressed", x: 600, y: 400, button: "left" });
+  for (const method of ["Input.dispatchTouchEvent", "Input.dispatchDragEvent", "Input.emulateTouchFromMouseEvent", "Input.synthesizeTapGesture", "Input.synthesizeScrollGesture", "Input.synthesizePinchGesture"]) {
+    const params = { x: 10, y: 20, touchPoints: [{ x: 10, y: 20 }] };
+    assert.equal(scaleInput(method, params, 1), params, `${method} passes on an unscaled page`);
+    assert.throws(() => scaleInput(method, params, 0.5), { code: "input_unscaled" }, `${method} must not reach a scaled page unscaled`);
+  }
+  const key = { type: "keyDown", key: "a" };
+  assert.equal(scaleInput("Input.dispatchKeyEvent", key, 0.5), key, "keys carry no point");
+  assert.equal(scaleInput("Input.insertText", { text: "x" }, 0.5).text, "x");
+});
+
 test("rewind notice dismissal survives the desktop preference disk round trip", (t) => {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rewind-notice-prefs-"));
   const file = path.join(dir, "prefs.json");

release-notes/studio/unreleased/13811-browser-fit-panel.md

@@ -0,0 +1,2 @@
+- 修复：智能体把内置浏览器页面调成固定视口时，页面不再只露出左上角，而是整页缩小居中显示在面板里，点击位置也保持准确。关联 #13811。
+- Fixed: A built-in browser page the agent gave its own viewport is now shown whole, scaled down and centred in the panel, and its clicks still land. Refs #13811.