fix(ci): close storage verifier sockets and trace browser verifier stages

Since 8b4c3de3b added the storage end-to-end check, the real-browser lane
has hit its 10-minute job limit on main and on every PR. The verifier
printed nothing before cancellation, so the stalled step was unknown.
Locally, on macOS with Chrome for Testing 155, it passes in about 70 s.

The storage site was the only verifier site closed without
`closeAllConnections()`. The external browser outlives the site, so a
socket it keeps open can hold `server.close()` and its `close` event
indefinitely. The popup, network and metadata sites already close
connections first.

Write per-stage markers to stderr so a cancelled job shows which step
stalled.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

scripts/lib/browser-scenario-storage-e2e.mjs

@@ -99,6 +99,12 @@ const scenarioFor = (browser, origin) =>
     capture: { after_each_step: ["dom"], at_end: ["dom"] },
   });
 
+const storageStart = Date.now();
+const stage = (name) =>
+  process.stderr.write(
+    `[verify:browser:storage] ${name} +${String(Date.now() - storageStart)}ms\n`,
+  );
+
 const raceWithTimeout = (operation, timeoutMs, onTimeout = () => undefined) => {
   const controller = new AbortController();
   return Promise.race([
@@ -175,6 +181,8 @@ const startStorageSite = async () => {
   return {
     origin: `http://127.0.0.1:${port}`,
     close: async () => {
+      // The external browser outlives this site and can keep sockets open.
+      server.closeAllConnections();
       server.close();
       await once(server, "close");
     },
@@ -200,8 +208,11 @@ export async function verifyScenarioStorage({
 }) {
   const site = await startStorageSite();
   try {
+    stage("launch");
     await verifyLaunchStorage(executable, site.origin);
+    stage("connect");
     await verifyConnectedStorage(endpoint, targetId, site.origin);
+    stage("close");
     return {
       mocked: false,
       launch: true,
@@ -224,6 +235,7 @@ const verifyLaunchStorage = async (executable, origin) => {
     origin,
   );
   const profilesBefore = await scenarioProfiles();
+  stage("launch: scenario CLI");
   const result = await runScenarioCli(scenario);
   const capture = result.normalized_result;
   assert.ok(capture, "Launch-mode storage scenario returned no result");
@@ -236,6 +248,7 @@ const verifyLaunchStorage = async (executable, origin) => {
     ...scenario,
     actions: [scenario.actions[0]],
   });
+  stage("launch: fresh in-process scenario");
   const fresh = await createBrowserScenarioProvider(
     process.env,
   ).captureScenario(freshScenario);
@@ -297,7 +310,9 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
     "capture_browser_scenario",
   );
   try {
+    stage("connect: storage partition");
     await assertStorageKeyPartition(observer, origin);
+    stage("connect: scenario CLI");
     const result = await runScenarioCli(scenario);
     const capture = result.normalized_result;
     assert.ok(capture, "Connect-mode storage scenario returned no result");
@@ -308,9 +323,12 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
       empty: "main-empty-seed",
     });
     assert.equal(capture.browser.cleanup, "disconnected-external");
+    stage("connect: caller debugger pause");
     await verifyCallerDebuggerPause(observer, origin, scenario);
+    stage("connect: cancellation");
     await verifyCancelledStorage(observer, endpoint, targetId, scenario);
   } finally {
+    stage("connect: restore target");
     await raceWithTimeout(restoreTarget(observer, target.url), 1_000).catch(
       () => undefined,
     );

scripts/verify-real-browser.mjs

@@ -53,6 +53,12 @@ const SCENARIO_URL_SECRET_VALUE = "rea-browser-url-verifier-secret";
 process.env.REA_BROWSER_VERIFIER_SECRET = SCENARIO_SECRET_VALUE;
 process.env.REA_BROWSER_VERIFIER_URL = SCENARIO_URL_SECRET_VALUE;
 const verifierRun = createVerifierRun();
+const verifierStart = Date.now();
+// Stage markers locate a stalled step when CI cancels the job at its limit.
+const stage = (name) =>
+  process.stderr.write(
+    `[verify:browser] ${name} +${String(Date.now() - verifierStart)}ms\n`,
+  );
 
 const executable = await browserExecutable();
 const profile = await mkdtemp(join(tmpdir(), "rea-real-browser-"));
@@ -109,6 +115,7 @@ try {
   );
   if (!observed.ok) throw observed.error;
   assertObservation(observed.value, site.origin);
+  stage("verifyBrowserDomDestinations");
   const domDestinations = await verifyBrowserDomDestinations({
     cdp_endpoint: endpoint,
     target_id: target,
@@ -168,6 +175,7 @@ try {
     );
   assertSensitiveShapes(withSource.value);
 
+  stage("verifyPublicBundle");
   const bundle = await verifyPublicBundle(endpoint, target, site.origin);
 
   const captureDiff = await provider.compareCaptures(
@@ -231,6 +239,7 @@ try {
     throw new Error("Real Chrome same-origin SPA timeline was missing");
 
   pageProxy = await startPageCdpProxy(endpoint);
+  stage("verifyPageScopedTransport");
   await verifyPageScopedTransport(provider, pageProxy, site.origin);
   const attachedScenarioInput = browserScenario(
     {
@@ -277,16 +286,19 @@ try {
     throw new Error("Scenario attachment terminated its external browser");
 
   const profilesBefore = await scenarioProfiles();
+  stage("verifyScenarioEnvironment");
   const scenarioEnvironment = await verifyScenarioEnvironment(
     endpoint,
     target,
     site.origin,
   );
+  stage("verifyScenarioStorage");
   const scenarioStorage = await verifyScenarioStorage({
     executable,
     endpoint,
     targetId: target,
   });
+  stage("launchedScenario");
   const launchedScenario = await createBrowserScenarioProvider(
     process.env,
   ).captureScenario(
@@ -308,6 +320,7 @@ try {
   if ([...profilesAfter].some((entry) => !profilesBefore.has(entry)))
     throw new Error("Scenario launch retained a temporary browser profile");
   assertScenarioCapture(launchedScenario.value);
+  stage("verifyScenarioFailureEvidence");
   const scenarioFailure = await verifyScenarioFailureEvidence(
     executable,
     site.origin,
@@ -339,18 +352,24 @@ try {
         `Browser scenario included unselected structured value: ${unselectedValue}`,
       );
 
+  stage("verifyLargeScreenshotE2e");
   const largeScreenshot = await verifyLargeScreenshotE2e(endpoint, site.origin);
+  stage("verifyPopupEventCoverage");
   const popupEvents = await verifyPopupEventCoverage(executable);
+  stage("verifyBrowserNetworkEvidence");
   const networkEvidence = await verifyBrowserNetworkEvidence(
     executable,
     undefined,
     { cdp_endpoint: endpoint, target_id: target },
   );
+  stage("verifyBrowserScriptExport");
   const scriptExport = await verifyBrowserScriptExport(executable, undefined, {
     cdp_endpoint: endpoint,
     target_id: target,
   });
+  stage("verifyBrowserModules");
   const moduleTrace = await verifyBrowserModules(executable);
+  stage("verifyBrowserCaptureMetadataBudget");
   const captureMetadata = await verifyBrowserCaptureMetadataBudget(endpoint);
   report = {
     browser: observed.value.browser.product,
@@ -393,6 +412,7 @@ try {
   process.stderr.write("Browser verifier cleanup: stopping owned Chrome\n");
   if (browser !== undefined) await stopProcess(browser);
   process.stderr.write("Browser verifier cleanup: closing fixture site\n");
+  stage("cleanup");
   await site.close();
   await rm(profile, {
     recursive: true,