Bug 1321472, when the capturing content is inside a popup, use the popup as the root frame when searching for a mouse target rather than the root frame, otherwise the capturing content's frame won't be found, r=smaug

layout/base/PresShell.cpp

@@ -7257,25 +7257,31 @@ PresShell::HandleEvent(nsIFrame* aFrame,
     nsPresContext* rootPresContext = framePresContext->GetRootPresContext();
     NS_ASSERTION(rootPresContext == mPresContext->GetRootPresContext(),
                  "How did we end up outside the connected prescontext/viewmanager hierarchy?");
-    // If we aren't starting our event dispatch from the root frame of the root prescontext,
-    // then someone must be capturing the mouse. In that case we don't want to search the popup
-    // list.
-    if (framePresContext == rootPresContext &&
-        frame == mFrameConstructor->GetRootFrame()) {
-      nsIFrame* popupFrame =
-        nsLayoutUtils::GetPopupFrameForEventCoordinates(rootPresContext, aEvent);
-      // If a remote browser is currently capturing input break out if we
-      // detect a chrome generated popup.
-      if (popupFrame && capturingContent &&
-          EventStateManager::IsRemoteTarget(capturingContent)) {
-        capturingContent = nullptr;
-      }
-      // If the popupFrame is an ancestor of the 'frame', the frame should
-      // handle the event, otherwise, the popup should handle it.
-      if (popupFrame &&
-          !nsContentUtils::ContentIsCrossDocDescendantOf(
-             framePresContext->GetPresShell()->GetDocument(),
-             popupFrame->GetContent())) {
+    nsIFrame* popupFrame =
+      nsLayoutUtils::GetPopupFrameForEventCoordinates(rootPresContext, aEvent);
+    // If a remote browser is currently capturing input break out if we
+    // detect a chrome generated popup.
+    if (popupFrame && capturingContent &&
+        EventStateManager::IsRemoteTarget(capturingContent)) {
+      capturingContent = nullptr;
+    }
+    // If the popupFrame is an ancestor of the 'frame', the frame should
+    // handle the event, otherwise, the popup should handle it.
+    if (popupFrame &&
+        !nsContentUtils::ContentIsCrossDocDescendantOf(
+           framePresContext->GetPresShell()->GetDocument(),
+           popupFrame->GetContent())) {
+
+      // If we aren't starting our event dispatch from the root frame of the
+      // root prescontext, then someone must be capturing the mouse. In that
+      // case we only want to use the popup list if the capture is
+      // inside the popup.
+      if (framePresContext == rootPresContext &&
+          frame == mFrameConstructor->GetRootFrame()) {
+        frame = popupFrame;
+      } else if (capturingContent &&
+                 nsContentUtils::ContentIsDescendantOf(
+                   capturingContent, popupFrame->GetContent())) {
         frame = popupFrame;
       }
     }

toolkit/content/tests/chrome/chrome.ini

@@ -112,6 +112,7 @@ subsuite = clipboard
 skip-if = os == 'linux' && !debug #Bug 1207174
 [test_menu.xul]
 [test_menu_anchored.xul]
+[test_menu_withcapture.xul]
 [test_menu_hide.xul]
 [test_menuchecks.xul]
 [test_menuitem_blink.xul]

toolkit/content/tests/chrome/test_menu_withcapture.xul

@@ -0,0 +1,59 @@
+<?xml version="1.0"?>
+<?xml-stylesheet href="chrome://global/skin" type="text/css"?>
+<?xml-stylesheet href="chrome://mochikit/content/tests/SimpleTest/test.css" type="text/css"?>
+
+<window title="Menu with Mouse Capture"
+        xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+
+  <script type="application/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>      
+  <script type="application/javascript" src="chrome://mochikit/content/tests/SimpleTest/EventUtils.js"></script>      
+
+  <menulist id="menulist">
+    <menupopup onpopupshown="shown(this)" onpopuphidden="done();">
+      <menuitem id="menuitem" label="Menu Item"
+                onmousemove="moveHappened = true;"
+                onmouseup="upHappened = true;"/>
+    </menupopup>
+  </menulist>
+
+<script class="testbody" type="application/javascript">
+<![CDATA[
+
+SimpleTest.waitForExplicitFinish();
+SimpleTest.waitForFocus(startTest);
+
+let moveHappened = false, upHappened = false;
+
+function startTest() {
+  disableNonTestMouseEvents(true);
+  document.getElementById("menulist"). open = true;
+}
+
+function shown(popup)
+{
+  popup.setCaptureAlways();
+  synthesizeMouseAtCenter(document.getElementById("menuitem"), { type: "mousemove" });
+  synthesizeMouseAtCenter(document.getElementById("menuitem"), { type: "mouseup" });
+}
+
+function done()
+{
+  ok(moveHappened, "mousemove happened");
+  ok(upHappened, "mouseup happened");
+  disableNonTestMouseEvents(false);
+  SimpleTest.finish();
+}
+
+]]>
+</script>
+
+<body xmlns="http://www.w3.org/1999/xhtml">
+<p id="display">
+</p>
+<div id="content" style="display: none">
+</div>
+<pre id="test">
+</pre>
+</body>
+
+</window>