diff --git a/layout/base/nsCSSRendering.cpp b/layout/base/nsCSSRendering.cpp
index 89562e7088e7..5fa2faaa7097 100644
--- a/layout/base/nsCSSRendering.cpp
+++ b/layout/base/nsCSSRendering.cpp
@@ -575,10 +575,8 @@ GetOutlineInnerRect(nsIFrame* aFrame)
     (aFrame->Properties().Get(nsIFrame::OutlineInnerRectProperty()));
   if (savedOutlineInnerRect)
     return *savedOutlineInnerRect;
-  // FIXME (bug 599652): We probably want something narrower than either
-  // overflow rect here, but for now use the visual overflow in order to
-  // be consistent with ComputeEffectsRect in nsFrame.cpp.
-  return aFrame->GetVisualOverflowRect();
+  NS_NOTREACHED("we should have saved a frame property");
+  return nsRect(nsPoint(0, 0), aFrame->GetSize());
 }
 
 void
@@ -608,37 +606,22 @@ nsCSSRendering::PaintOutline(nsPresContext* aPresContext,
   nscolor bgColor =
     bgContext->GetVisitedDependentColor(eCSSProperty_background_color);
 
-  // When the outline property is set on :-moz-anonymous-block or
-  // :-moz-anonyomus-positioned-block pseudo-elements, it inherited that
-  // outline from the inline that was broken because it contained a
-  // block.  In that case, we don't want a really wide outline if the
-  // block inside the inline is narrow, so union the actual contents of
-  // the anonymous blocks.
-  nsIFrame *frameForArea = aForFrame;
-  do {
-    nsIAtom *pseudoType = frameForArea->StyleContext()->GetPseudo();
-    if (pseudoType != nsCSSAnonBoxes::mozAnonymousBlock &&
-        pseudoType != nsCSSAnonBoxes::mozAnonymousPositionedBlock)
-      break;
-    // If we're done, we really want it and all its later siblings.
-    frameForArea = frameForArea->GetFirstPrincipalChild();
-    NS_ASSERTION(frameForArea, "anonymous block with no children?");
-  } while (frameForArea);
-  nsRect innerRect; // relative to aBorderArea.TopLeft()
-  if (frameForArea == aForFrame) {
-    innerRect = GetOutlineInnerRect(aForFrame);
+  nsRect innerRect;
+  if (
+#ifdef MOZ_XUL
+      aStyleContext->GetPseudoType() == nsCSSPseudoElements::ePseudo_XULTree
+#else
+      false
+#endif
+     ) {
+    // FIXME: This behavior doesn't make sense; we should switch back to
+    // using aBorderArea.  But since this has been broken since bug
+    // 133165 in August of 2004, that switch should be made in its own
+    // patch changing only that behavior.
+    innerRect = aForFrame->GetVisualOverflowRect();
   } else {
-    for (; frameForArea; frameForArea = frameForArea->GetNextSibling()) {
-      // The outline has already been included in aForFrame's overflow
-      // area, but not in those of its descendants, so we have to
-      // include it.  Otherwise we'll end up drawing the outline inside
-      // the border.
-      nsRect r(GetOutlineInnerRect(frameForArea) +
-               frameForArea->GetOffsetTo(aForFrame));
-      innerRect.UnionRect(innerRect, r);
-    }
+    innerRect = GetOutlineInnerRect(aForFrame);
   }
-
   innerRect += aBorderArea.TopLeft();
   nscoord offset = ourOutline->mOutlineOffset;
   innerRect.Inflate(offset, offset);
diff --git a/layout/generic/nsFrame.cpp b/layout/generic/nsFrame.cpp
index f3251bf55afc..5461cec87b39 100644
--- a/layout/generic/nsFrame.cpp
+++ b/layout/generic/nsFrame.cpp
@@ -4976,27 +4976,6 @@ ComputeEffectsRect(nsIFrame* aFrame, const nsRect& aOverflowRect,
   // box-shadow
   r.UnionRect(r, nsLayoutUtils::GetBoxShadowRectForFrame(aFrame, aNewSize));
 
-  const nsStyleOutline* outline = aFrame->StyleOutline();
-  uint8_t outlineStyle = outline->GetOutlineStyle();
-  if (outlineStyle != NS_STYLE_BORDER_STYLE_NONE) {
-    nscoord width;
-    DebugOnly<bool> result = outline->GetOutlineWidth(width);
-    NS_ASSERTION(result, "GetOutlineWidth had no cached outline width");
-    if (width > 0) {
-      aFrame->Properties().
-        Set(nsIFrame::OutlineInnerRectProperty(), new nsRect(r));
-
-      nscoord offset = outline->mOutlineOffset;
-      nscoord inflateBy = std::max(width + offset, 0);
-      // FIXME (bug 599652): We probably want outline to be drawn around
-      // something smaller than the visual overflow rect (perhaps the
-      // scrollable overflow rect is correct).  When we change that, we
-      // need to keep this code (and the storing of properties just
-      // above) in sync with GetOutlineInnerRect in nsCSSRendering.cpp.
-      r.Inflate(inflateBy, inflateBy);
-    }
-  }
-
   // border-image-outset.
   // We need to include border-image-outset because it can cause the
   // border image to be drawn beyond the border box.
@@ -6854,6 +6833,185 @@ IsInlineFrame(nsIFrame *aFrame)
   return type == nsGkAtoms::inlineFrame;
 }
 
+/**
+ * Compute the union of the border boxes of aFrame and its descendants,
+ * in aFrame's coordinate space (if aApplyTransform is false) or its
+ * post-transform coordinate space (if aApplyTransform is true).
+ */
+static nsRect
+UnionBorderBoxes(nsIFrame* aFrame, bool aApplyTransform,
+                 const nsSize* aSizeOverride = nullptr,
+                 const nsOverflowAreas* aOverflowOverride = nullptr)
+{
+  const nsRect bounds(nsPoint(0, 0),
+                      aSizeOverride ? *aSizeOverride : aFrame->GetSize());
+
+  // Start from our border-box, transformed.  See comment below about
+  // transform of children.
+  nsRect u;
+  bool doTransform = aApplyTransform && aFrame->IsTransformed();
+  if (doTransform) {
+    u = nsDisplayTransform::TransformRect(bounds, aFrame,
+                                          nsPoint(0, 0), &bounds);
+  } else {
+    u = bounds;
+  }
+
+  // Only iterate through the children if the overflow areas suggest
+  // that we might need to, and if the frame doesn't clip its overflow
+  // anyway.
+  if (aOverflowOverride) {
+    if (!doTransform &&
+        bounds.IsEqualEdges(aOverflowOverride->VisualOverflow()) &&
+        bounds.IsEqualEdges(aOverflowOverride->ScrollableOverflow())) {
+      return u;
+    }
+  } else {
+    if (!doTransform &&
+        bounds.IsEqualEdges(aFrame->GetVisualOverflowRect()) &&
+        bounds.IsEqualEdges(aFrame->GetScrollableOverflowRect())) {
+      return u;
+    }
+  }
+  const nsStyleDisplay* disp = aFrame->StyleDisplay();
+  nsIAtom* fType = aFrame->GetType();
+  if (nsFrame::ShouldApplyOverflowClipping(aFrame, disp) ||
+      fType == nsGkAtoms::scrollFrame ||
+      fType == nsGkAtoms::svgOuterSVGFrame) {
+    return u;
+  }
+
+  nsRect clipPropClipRect;
+  bool hasClipPropClip =
+    aFrame->GetClipPropClipRect(disp, &clipPropClipRect, bounds.Size());
+
+  // Iterate over all children except pop-ups.
+  const nsIFrame::ChildListIDs skip(nsIFrame::kPopupList |
+                                    nsIFrame::kSelectPopupList);
+  for (nsIFrame::ChildListIterator childLists(aFrame);
+       !childLists.IsDone(); childLists.Next()) {
+    if (skip.Contains(childLists.CurrentID())) {
+      continue;
+    }
+
+    nsFrameList children = childLists.CurrentList();
+    for (nsFrameList::Enumerator e(children); !e.AtEnd(); e.Next()) {
+      nsIFrame* child = e.get();
+      // Note that passing |true| for aApplyTransform when
+      // child->Preserves3D() is incorrect if our aApplyTransform is
+      // false... but the opposite would be as well.  This is because
+      // elements within a preserve-3d scene are always transformed up
+      // to the top of the scene.  This means we don't have a
+      // mechanism for getting a transform up to an intermediate point
+      // within the scene.  We choose to over-transform rather than
+      // under-transform because this is consistent with other
+      // overflow areas.
+      nsRect childRect = UnionBorderBoxes(child, true) +
+                         child->GetPosition();
+
+      if (hasClipPropClip) {
+        // Intersect with the clip before transforming.
+        childRect.IntersectRect(childRect, clipPropClipRect);
+      }
+
+      // Note that we transform each child separately according to
+      // aFrame's transform, and then union, which gives a different
+      // (smaller) result from unioning and then transforming the
+      // union.  This doesn't match the way we handle overflow areas
+      // with 2-D transforms, though it does match the way we handle
+      // overflow areas in preserve-3d 3-D scenes.
+      if (doTransform && !child->Preserves3D()) {
+        childRect = nsDisplayTransform::TransformRect(childRect, aFrame,
+                                                      nsPoint(0, 0), &bounds);
+      }
+      u.UnionRectEdges(u, childRect);
+    }
+  }
+
+  return u;
+}
+
+static void
+ComputeAndIncludeOutlineArea(nsIFrame* aFrame, nsOverflowAreas& aOverflowAreas,
+                             const nsSize& aNewSize)
+{
+  const nsStyleOutline* outline = aFrame->StyleOutline();
+  if (outline->GetOutlineStyle() == NS_STYLE_BORDER_STYLE_NONE) {
+    return;
+  }
+
+  nscoord width;
+  DebugOnly<bool> result = outline->GetOutlineWidth(width);
+  NS_ASSERTION(result, "GetOutlineWidth had no cached outline width");
+  if (width <= 0) {
+    return;
+  }
+
+  // When the outline property is set on :-moz-anonymous-block or
+  // :-moz-anonymous-positioned-block pseudo-elements, it inherited
+  // that outline from the inline that was broken because it
+  // contained a block.  In that case, we don't want a really wide
+  // outline if the block inside the inline is narrow, so union the
+  // actual contents of the anonymous blocks.
+  nsIFrame *frameForArea = aFrame;
+  do {
+    nsIAtom *pseudoType = frameForArea->StyleContext()->GetPseudo();
+    if (pseudoType != nsCSSAnonBoxes::mozAnonymousBlock &&
+        pseudoType != nsCSSAnonBoxes::mozAnonymousPositionedBlock)
+      break;
+    // If we're done, we really want it and all its later siblings.
+    frameForArea = frameForArea->GetFirstPrincipalChild();
+    NS_ASSERTION(frameForArea, "anonymous block with no children?");
+  } while (frameForArea);
+
+  // Find the union of the border boxes of all descendants, or in
+  // the block-in-inline case, all descendants we care about.
+  //
+  // Note that the interesting perspective-related cases are taken
+  // care of by the code that handles those issues for overflow
+  // calling FinishAndStoreOverflow again, which in turn calls this
+  // function again.  We still need to deal with preserve-3d a bit.
+  nsRect innerRect;
+  if (frameForArea == aFrame) {
+    innerRect = UnionBorderBoxes(aFrame, false, &aNewSize, &aOverflowAreas);
+  } else {
+    for (; frameForArea; frameForArea = frameForArea->GetNextSibling()) {
+      nsRect r(UnionBorderBoxes(frameForArea, true));
+
+      // Adjust for offsets transforms up to aFrame's pre-transform
+      // (i.e., normal) coordinate space; see comments in
+      // UnionBorderBoxes for some of the subtlety here.
+      for (nsIFrame *f = frameForArea, *parent = f->GetParent();
+           /* see middle of loop */;
+           f = parent, parent = f->GetParent()) {
+        r += f->GetPosition();
+        if (parent == aFrame) {
+          break;
+        }
+        if (parent->IsTransformed() && !f->Preserves3D()) {
+          r = nsDisplayTransform::TransformRect(r, parent, nsPoint(0, 0));
+        }
+      }
+
+      innerRect.UnionRect(innerRect, r);
+    }
+  }
+
+  aFrame->Properties().Set(nsIFrame::OutlineInnerRectProperty(),
+                           new nsRect(innerRect));
+
+  nscoord offset = outline->mOutlineOffset;
+  nscoord inflateBy = std::max(width + offset, 0);
+
+  // Keep this code (and the storing of properties just above) in
+  // sync with GetOutlineInnerRect in nsCSSRendering.cpp.
+  nsRect outerRect(innerRect);
+  outerRect.Inflate(inflateBy, inflateBy);
+
+  nsRect& vo = aOverflowAreas.VisualOverflow();
+  vo.UnionRectEdges(vo, outerRect);
+}
+
 bool
 nsIFrame::FinishAndStoreOverflow(nsOverflowAreas& aOverflowAreas,
                                  nsSize aNewSize, nsSize* aOldSize)
@@ -6933,6 +7091,8 @@ nsIFrame::FinishAndStoreOverflow(nsOverflowAreas& aOverflowAreas,
     }
   }
 
+  ComputeAndIncludeOutlineArea(this, aOverflowAreas, aNewSize);
+
   // Nothing in here should affect scrollable overflow.
   aOverflowAreas.VisualOverflow() =
     ComputeEffectsRect(this, aOverflowAreas.VisualOverflow(), aNewSize);
diff --git a/layout/reftests/outline/outline-and-3d-transform-1-ref.html b/layout/reftests/outline/outline-and-3d-transform-1-ref.html
new file mode 100644
index 000000000000..30fa7eabec01
--- /dev/null
+++ b/layout/reftests/outline/outline-and-3d-transform-1-ref.html
@@ -0,0 +1,28 @@
+<!DOCTYPE HTML>
+<title>Testcase for outline around 3-D transform</title>
+<style>
+
+html, body { margin: 0; padding: 0; border: none }
+
+div {
+  width: 100px;
+  height: 100px;
+}
+
+body > div {
+  margin-top: 200px;
+  margin-left: 200px;
+  transform-style: flat;
+  position: relative;
+}
+
+body > div > div {
+  position: absolute; top: 0; left: 0;
+  height: 150px; width: 150px; top: -25px; left: -25px;
+  background: rgba(255, 255, 0, 0.4);
+  outline: 2px dashed blue;
+}
+
+</style>
+
+<div><div></div></div>
diff --git a/layout/reftests/outline/outline-and-3d-transform-1a.html b/layout/reftests/outline/outline-and-3d-transform-1a.html
new file mode 100644
index 000000000000..6b7926c2dac7
--- /dev/null
+++ b/layout/reftests/outline/outline-and-3d-transform-1a.html
@@ -0,0 +1,40 @@
+<!DOCTYPE HTML>
+<title>Testcase for outline around 3-D transform</title>
+<style>
+
+html, body { margin: 0; padding: 0; border: none }
+
+div {
+  width: 100px;
+  height: 100px;
+}
+
+body > div {
+  margin-top: 200px;
+  margin-left: 200px;
+  transform-style: flat;
+  outline: 2px dashed blue;
+}
+
+body > div > div {
+  transform: rotateX(30deg);
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+}
+
+body > div > div > div {
+  transform: rotateX(30deg);
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+}
+
+body > div > div > div > div {
+  transform: scale(1.5, 3);
+  transform-origin: 50% 50%;
+  background: rgba(255, 255, 0, 0.4);
+}
+
+
+</style>
+
+<div><div><div><div></div></div></div></div>
diff --git a/layout/reftests/outline/outline-and-3d-transform-1b.html b/layout/reftests/outline/outline-and-3d-transform-1b.html
new file mode 100644
index 000000000000..d2e2316de691
--- /dev/null
+++ b/layout/reftests/outline/outline-and-3d-transform-1b.html
@@ -0,0 +1,40 @@
+<!DOCTYPE HTML>
+<title>Testcase for outline around 3-D transform</title>
+<style>
+
+html, body { margin: 0; padding: 0; border: none }
+
+div {
+  width: 100px;
+  height: 100px;
+}
+
+body > div {
+  margin-top: 200px;
+  margin-left: 200px;
+  transform-style: flat;
+  outline: 2px dashed blue;
+}
+
+body > div > div {
+  transform: rotateX(90deg);
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+}
+
+body > div > div > div {
+  transform: rotateX(-30deg);
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+}
+
+body > div > div > div > div {
+  transform: scale(1.5, 3);
+  transform-origin: 50% 50%;
+  background: rgba(255, 255, 0, 0.4);
+}
+
+
+</style>
+
+<div><div><div><div></div></div></div></div>
diff --git a/layout/reftests/outline/outline-and-3d-transform-2-ref.html b/layout/reftests/outline/outline-and-3d-transform-2-ref.html
new file mode 100644
index 000000000000..93a45d8f80eb
--- /dev/null
+++ b/layout/reftests/outline/outline-and-3d-transform-2-ref.html
@@ -0,0 +1,36 @@
+<!DOCTYPE HTML>
+<title>Testcase for outline around 3-D transform</title>
+<style>
+
+html, body { margin: 0; padding: 0; border: none }
+
+div {
+  width: 100px;
+  height: 100px;
+}
+
+body > div {
+  margin-top: 200px;
+  margin-left: 200px;
+  transform-style: flat;
+}
+
+body > div > div {
+  transform: rotateX(30deg);
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+  border: 5px solid green;
+  margin: -5px;
+}
+
+body > div > div > div {
+  transform: rotateX(30deg);
+  width: 50px; margin-left: 20px; margin-top: -5px;
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+  border: 5px solid blue;
+}
+
+</style>
+
+<div><div><div></div></div></div>
diff --git a/layout/reftests/outline/outline-and-3d-transform-2.html b/layout/reftests/outline/outline-and-3d-transform-2.html
new file mode 100644
index 000000000000..72436697171a
--- /dev/null
+++ b/layout/reftests/outline/outline-and-3d-transform-2.html
@@ -0,0 +1,35 @@
+<!DOCTYPE HTML>
+<title>Testcase for outline around 3-D transform</title>
+<style>
+
+html, body { margin: 0; padding: 0; border: none }
+
+div {
+  width: 100px;
+  height: 100px;
+}
+
+body > div {
+  margin-top: 200px;
+  margin-left: 200px;
+  transform-style: flat;
+}
+
+body > div > div {
+  transform: rotateX(30deg);
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+  outline: 5px solid green;
+}
+
+body > div > div > div {
+  transform: rotateX(30deg);
+  width: 50px; margin-left: 25px;
+  transform-origin: 50% 50%;
+  transform-style: preserve-3d;
+  outline: 5px solid blue;
+}
+
+</style>
+
+<div><div><div></div></div></div>
diff --git a/layout/reftests/outline/outline-and-box-shadow-ref.html b/layout/reftests/outline/outline-and-box-shadow-ref.html
new file mode 100644
index 000000000000..0d4a9ffcf692
--- /dev/null
+++ b/layout/reftests/outline/outline-and-box-shadow-ref.html
@@ -0,0 +1,13 @@
+<!DOCTYPE HTML>
+<title>outline and box-shadow</title>
+<style>
+html, body { margin: 0; padding: 0 }
+p {
+  margin: 48px;
+  border: 2px solid blue;
+  padding: 5px; /* ensure no font overhang */
+  background: yellow; color: black;
+  box-shadow: 10px 10px 10px 0px black;
+}
+</style>
+<p>The outline should be adjacent to the background.</p>
diff --git a/layout/reftests/outline/outline-and-box-shadow.html b/layout/reftests/outline/outline-and-box-shadow.html
new file mode 100644
index 000000000000..3bb717200b49
--- /dev/null
+++ b/layout/reftests/outline/outline-and-box-shadow.html
@@ -0,0 +1,13 @@
+<!DOCTYPE HTML>
+<title>outline and box-shadow</title>
+<style>
+html, body { margin: 0; padding: 0 }
+p {
+  margin: 50px;
+  outline: 2px solid blue;
+  padding: 5px; /* ensure no font overhang */
+  background: yellow; color: black;
+  box-shadow: 10px 10px 10px 2px black;
+}
+</style>
+<p>The outline should be adjacent to the background.</p>
diff --git a/layout/reftests/outline/reftest.list b/layout/reftests/outline/reftest.list
new file mode 100644
index 000000000000..7384bd4893c7
--- /dev/null
+++ b/layout/reftests/outline/reftest.list
@@ -0,0 +1,4 @@
+== outline-and-box-shadow.html outline-and-box-shadow-ref.html
+== outline-and-3d-transform-1a.html outline-and-3d-transform-1-ref.html
+== outline-and-3d-transform-1b.html outline-and-3d-transform-1-ref.html
+== outline-and-3d-transform-2.html outline-and-3d-transform-2-ref.html
diff --git a/layout/reftests/reftest.list b/layout/reftests/reftest.list
index d31429ad7ec2..971e76b8b83d 100644
--- a/layout/reftests/reftest.list
+++ b/layout/reftests/reftest.list
@@ -219,6 +219,8 @@ skip-if(Android||B2G) include native-theme/reftest.list
 # netwerk/
 skip-if(B2G) include ../../netwerk/test/reftest/reftest.list
 
+include outline/reftest.list
+
 # object/
 skip-if(B2G) include object/reftest.list