diff --git a/browser/components/customizableui/src/CustomizeMode.jsm b/browser/components/customizableui/src/CustomizeMode.jsm
index 2eaf9e07f1cb..f8ae392b86c2 100644
--- a/browser/components/customizableui/src/CustomizeMode.jsm
+++ b/browser/components/customizableui/src/CustomizeMode.jsm
@@ -1688,7 +1688,15 @@ CustomizeMode.prototype = {
     let dragY = aEvent.clientY - this._dragOffset.y;
 
     // Ensure this is within the container
-    let bounds = expectedParent.getBoundingClientRect();
+    let boundsContainer = expectedParent;
+    // NB: because the panel UI itself is inside a scrolling container, we need
+    // to use the parent bounds (otherwise, if the panel UI is scrolled down,
+    // the numbers we get are in window coordinates which leads to various kinds
+    // of weirdness)
+    if (boundsContainer == this.panelUIContents) {
+      boundsContainer = boundsContainer.parentNode;
+    }
+    let bounds = boundsContainer.getBoundingClientRect();
     dragX = Math.min(bounds.right, Math.max(dragX, bounds.left));
     dragY = Math.min(bounds.bottom, Math.max(dragY, bounds.top));
 
@@ -1700,6 +1708,16 @@ CustomizeMode.prototype = {
       }
     } else {
       let positionManager = DragPositionManager.getManagerForArea(aAreaElement);
+      // Make it relative to the container:
+      dragX -= bounds.left;
+      // NB: but if we're in the panel UI, we need to use the actual panel
+      // contents instead of the scrolling container to determine our origin
+      // offset against:
+      if (expectedParent == this.panelUIContents) {
+        dragY -= this.panelUIContents.getBoundingClientRect().top;
+      } else {
+        dragY -= bounds.top;
+      }
       // Find the closest node:
       targetNode = positionManager.find(aAreaElement, dragX, dragY, aDraggedItemId);
     }
diff --git a/browser/components/customizableui/src/DragPositionManager.jsm b/browser/components/customizableui/src/DragPositionManager.jsm
index af6f63f5706e..945fce6f330f 100644
--- a/browser/components/customizableui/src/DragPositionManager.jsm
+++ b/browser/components/customizableui/src/DragPositionManager.jsm
@@ -21,6 +21,7 @@ function AreaPositionManager(aContainer) {
   this._containerInfo = {
     left: containerRect.left,
     right: containerRect.right,
+    top: containerRect.top,
     width: containerRect.width
   };
   this._inPanel = aContainer.id == CustomizableUI.AREA_PANEL;
@@ -76,10 +77,14 @@ AreaPositionManager.prototype = {
   find: function(aContainer, aX, aY, aDraggedItemId) {
     let closest = null;
     let minCartesian = Number.MAX_VALUE;
+    let containerX = this._containerInfo.left;
+    let containerY = this._containerInfo.top;
     for (let node of aContainer.children) {
       let coordinates = this._lazyStoreGet(node);
-      let hDiff = coordinates.x - aX;
-      let vDiff = coordinates.y - aY;
+      let offsetX = coordinates.x - containerX;
+      let offsetY = coordinates.y - containerY;
+      let hDiff = offsetX - aX;
+      let vDiff = offsetY - aY;
       // For wide widgets, we're always going to be further from the center
       // horizontally. Compensate:
       if (this.isWide(node)) {