diff --git a/toolkit/components/prompts/src/PromptUtils.sys.mjs b/toolkit/components/prompts/src/PromptUtils.sys.mjs
index 7a15802b848d..8c1b0ff992d5 100644
--- a/toolkit/components/prompts/src/PromptUtils.sys.mjs
+++ b/toolkit/components/prompts/src/PromptUtils.sys.mjs
@@ -76,6 +76,12 @@ export var EnableDelayHelper = function ({
   this.focusTarget.addEventListener("keydown", this, true);
   this.focusTarget.document.addEventListener("unload", this);
 
+  // If we're not part of the active window, don't even start the timer yet.
+  let topWin = focusTarget.browsingContext.top.window;
+  if (topWin != Services.focus.activeWindow) {
+    return;
+  }
+
   this.startOnFocusDelay();
 };