Bug 1156238. Fix the computation of animated geometry roots for transform items. r=mattwoodrow

Removing the "stop at ancestor" parameter from functions that compute AGR meant that nsLayoutUtils::GetAnimatedGeometryRootFor could no longer pass the display item's reference frame as the "stop at ancestor" which meant that the AGR could cross the reference frame for the item, which we don't want. So we make transformed frames into AGRs.

This makes the computation of display items whose frames are transformed tricky. We need the AGR of the transform item to be the ancestor AGR, not the underlying frame of the transform item (which is now an AGR). So we modify nsLayoutUtils::GetAnimatedGeometryRootFor to handle this. (The patch from bug 1205087 didn't suffer from this problem because it special cased the computation of the AGR of transform items. Leaving anybody who called nsLayoutUtils::GetAnimatedGeometryRootFor to get the wrong result.)

The computation of the AGR for scroll metadata in ContainerState::ProcessDisplayItems specifically bypassed nsLayoutUtils::GetAnimatedGeometryRootFor to avoid it's special processing of fixed background items. However we do want the AGR for scroll metadata to do this special processing of transform items. So we add a flag to bypass the fixed background behaviour and use it for the scroll metadata AGR.

layout/base/FrameLayerBuilder.cpp

@@ -3903,8 +3903,8 @@ ContainerState::ProcessDisplayItems(nsDisplayList* aList)
         // Unlike GetAnimatedGeometryRootFor(), GetAnimatedGeometryRootForFrame() does not
         // take ShouldFixToViewport() into account, so it will return something different
         // for fixed background items.
-        animatedGeometryRootForScrollMetadata = nsLayoutUtils::GetAnimatedGeometryRootForFrame(
-            mBuilder, item->Frame());
+        animatedGeometryRootForScrollMetadata = nsLayoutUtils::GetAnimatedGeometryRootFor(
+          item, mBuilder, nsLayoutUtils::AGR_IGNORE_BACKGROUND_ATTACHMENT_FIXED);
       } else {
         // For inactive layer subtrees, splitting content into PaintedLayers
         // based on animated geometry roots is pointless. It's more efficient

layout/base/nsDisplayList.cpp

@@ -1041,6 +1041,9 @@ nsDisplayListBuilder::IsAnimatedGeometryRoot(nsIFrame* aFrame, nsIFrame** aParen
     // for background-attachment:fixed elements.
     return true;
   }
+  if (aFrame->IsTransformed()) {
+    return true;
+  }
 
   nsIFrame* parent = nsLayoutUtils::GetCrossDocParentFrame(aFrame);
   if (!parent)
@@ -4776,6 +4779,9 @@ nsDisplayTransform::nsDisplayTransform(nsDisplayListBuilder* aBuilder,
 void
 nsDisplayTransform::SetReferenceFrameToAncestor(nsDisplayListBuilder* aBuilder)
 {
+  if (mFrame == aBuilder->RootReferenceFrame()) {
+    return;
+  }
   nsIFrame *outerFrame = nsLayoutUtils::GetCrossDocParentFrame(mFrame);
   mReferenceFrame =
     aBuilder->FindReferenceFrameFor(outerFrame);

layout/base/nsDisplayList.h

@@ -3939,6 +3939,15 @@ public:
              mFrame->Combines3DTransformWithAncestors()));
   }
 
+  /**
+   * Whether this transform item forms a reference frame boundary.
+   * In other words, the reference frame of the contained items is our frame,
+   * and the reference frame of this item is some ancestor of our frame.
+   */
+  bool IsReferenceFrameBoundary() {
+    return !mTransformGetter && !mIsTransformSeparator;
+  }
+
 private:
   void ComputeBounds(nsDisplayListBuilder* aBuilder);
   void SetReferenceFrameToAncestor(nsDisplayListBuilder* aBuilder);

layout/base/nsLayoutUtils.cpp

@@ -1878,10 +1878,12 @@ nsLayoutUtils::GetAnimatedGeometryRootForFrame(nsDisplayListBuilder* aBuilder,
 
 nsIFrame*
 nsLayoutUtils::GetAnimatedGeometryRootFor(nsDisplayItem* aItem,
-                                          nsDisplayListBuilder* aBuilder)
+                                          nsDisplayListBuilder* aBuilder,
+                                          uint32_t aFlags)
 {
   nsIFrame* f = aItem->Frame();
-  if (aItem->ShouldFixToViewport(aBuilder)) {
+  if (!(aFlags & AGR_IGNORE_BACKGROUND_ATTACHMENT_FIXED) &&
+      aItem->ShouldFixToViewport(aBuilder)) {
     // Make its active scrolled root be the active scrolled root of
     // the enclosing viewport, since it shouldn't be scrolled by scrolled
     // frames in its document. InvalidateFixedBackgroundFramesFromList in
@@ -1892,6 +1894,14 @@ nsLayoutUtils::GetAnimatedGeometryRootFor(nsDisplayItem* aItem,
       return GetAnimatedGeometryRootForFrame(aBuilder, viewportFrame);
     }
   }
+  if (aItem->GetType() == nsDisplayItem::TYPE_TRANSFORM &&
+      static_cast<nsDisplayTransform*>(aItem)->IsReferenceFrameBoundary() &&
+      f != aBuilder->RootReferenceFrame()) {
+    nsIFrame* parent = nsLayoutUtils::GetCrossDocParentFrame(f);
+    if (parent) {
+      return GetAnimatedGeometryRootForFrame(aBuilder, parent);
+    }
+  }
   return GetAnimatedGeometryRootForFrame(aBuilder, f);
 }
 

layout/base/nsLayoutUtils.h

@@ -552,8 +552,17 @@ public:
    * returning aItem->ReferenceFrame() when we can't find another animated
    * geometry root.
    */
+  enum {
+    /**
+     * If the AGR_IGNORE_BACKGROUND_ATTACHMENT_FIXED flag is set, then we
+     * do not do any special processing for background attachment fixed items,
+     * instead treating them like any other frame.
+     */
+    AGR_IGNORE_BACKGROUND_ATTACHMENT_FIXED = 0x01
+  };
   static nsIFrame* GetAnimatedGeometryRootFor(nsDisplayItem* aItem,
-                                              nsDisplayListBuilder* aBuilder);
+                                              nsDisplayListBuilder* aBuilder,
+                                              uint32_t aFlags = 0);
 
   /**
    * Finds the nearest ancestor frame to aFrame that is considered to have (or