Bug 1381753 - Recompute visible regions for ContainerLayers without intermediate surfaces when needed for invalidation. r=mstange

gfx/layers/Layers.cpp

@@ -689,6 +689,18 @@ Layer::GetEffectiveMixBlendMode()
   return mSimpleAttrs.MixBlendMode();
 }
 
+Matrix4x4
+Layer::ComputeTransformToPreserve3DRoot()
+{
+  Matrix4x4 transform = GetLocalTransform();
+  for (Layer* layer = GetParent();
+       layer && layer->Extend3DContext();
+       layer = layer->GetParent()) {
+    transform = transform * layer->GetLocalTransform();
+  }
+  return transform;
+}
+
 void
 Layer::ComputeEffectiveTransformForMaskLayers(const gfx::Matrix4x4& aTransformToSurface)
 {

gfx/layers/Layers.h

@@ -1682,6 +1682,15 @@ public:
     return mEffectiveTransform;
   }
 
+  /**
+   * If the current layers participates in a preserve-3d
+   * context (returns true for Combines3DTransformWithAncestors),
+   * returns the combined transform up to the preserve-3d (nearest
+   * ancestor that doesn't Extend3DContext()). Otherwise returns
+   * the local transform.
+   */
+  gfx::Matrix4x4 ComputeTransformToPreserve3DRoot();
+
   /**
    * @param aTransformToSurface the composition of the transforms
    * from the parent layer (if any) to the destination pixel grid.

gfx/layers/composite/ContainerLayerComposite.cpp

@@ -723,6 +723,74 @@ ContainerLayerComposite::CleanupResources()
   }
 }
 
+static LayerIntRect
+TransformRect(const LayerIntRect& aRect, const Matrix4x4& aTransform)
+{
+  if (aRect.IsEmpty()) {
+    return LayerIntRect();
+  }
+
+  Rect rect(aRect.x, aRect.y, aRect.width, aRect.height);
+  rect = aTransform.TransformAndClipBounds(rect, Rect::MaxIntRect());
+  rect.RoundOut();
+
+  IntRect intRect;
+  if (!gfxUtils::GfxRectToIntRect(ThebesRect(rect), &intRect)) {
+    return LayerIntRect();
+  }
+
+  return ViewAs<LayerPixel>(intRect);
+}
+
+static void
+AddTransformedRegion(LayerIntRegion& aDest, const LayerIntRegion& aSource, const Matrix4x4& aTransform)
+{
+  for (auto iter = aSource.RectIter(); !iter.Done(); iter.Next()) {
+    aDest.Or(aDest, TransformRect(iter.Get(), aTransform));
+  }
+  aDest.SimplifyOutward(20);
+}
+
+// Async animations can move child layers without updating our visible region.
+// PostProcessLayers will recompute visible regions for layers with an intermediate
+// surface, but otherwise we need to do it now.
+void
+ComputeVisibleRegionForChildren(ContainerLayer* aContainer, LayerIntRegion& aResult)
+{
+  for (Layer* l = aContainer->GetFirstChild(); l; l = l->GetNextSibling()) {
+    if (l->Extend3DContext()) {
+      MOZ_ASSERT(l->AsContainerLayer());
+      ComputeVisibleRegionForChildren(l->AsContainerLayer(), aResult);
+    } else {
+      AddTransformedRegion(aResult,
+                           l->GetLocalVisibleRegion(),
+                           l->ComputeTransformToPreserve3DRoot());
+    }
+  }
+}
+
+const LayerIntRegion&
+ContainerLayerComposite::GetShadowVisibleRegion()
+{
+  if (!UseIntermediateSurface()) {
+    mShadowVisibleRegion.SetEmpty();
+    ComputeVisibleRegionForChildren(this, mShadowVisibleRegion);
+  }
+
+  return mShadowVisibleRegion;
+}
+
+const LayerIntRegion&
+RefLayerComposite::GetShadowVisibleRegion()
+{
+  if (!UseIntermediateSurface()) {
+    mShadowVisibleRegion.SetEmpty();
+    ComputeVisibleRegionForChildren(this, mShadowVisibleRegion);
+  }
+
+  return mShadowVisibleRegion;
+}
+
 RefLayerComposite::RefLayerComposite(LayerManagerComposite* aManager)
   : RefLayer(aManager, nullptr)
   , LayerComposite(aManager)

gfx/layers/composite/ContainerLayerComposite.h

@@ -88,6 +88,8 @@ public:
     DefaultComputeEffectiveTransforms(aTransformToSurface);
   }
 
+  virtual const LayerIntRegion& GetShadowVisibleRegion() override;
+
   virtual void CleanupResources() override;
 
   virtual HostLayer* AsHostLayer() override { return this; }
@@ -183,6 +185,8 @@ public:
     DefaultComputeEffectiveTransforms(aTransformToSurface);
   }
 
+  virtual const LayerIntRegion& GetShadowVisibleRegion() override;
+
   virtual void Cleanup() override;
 
   virtual void CleanupResources() override;

gfx/layers/composite/LayerManagerComposite.cpp

@@ -262,21 +262,6 @@ bool ShouldProcessLayer(Layer* aLayer)
   return aLayer->AsContainerLayer()->UseIntermediateSurface();
 }
 
-/**
- * Get accumulated transform of from the context creating layer to the
- * given layer.
- */
-static Matrix4x4
-GetAccTransformIn3DContext(Layer* aLayer) {
-  Matrix4x4 transform = aLayer->GetLocalTransform();
-  for (Layer* layer = aLayer->GetParent();
-       layer && layer->Extend3DContext();
-       layer = layer->GetParent()) {
-    transform = transform * layer->GetLocalTransform();
-  }
-  return transform;
-}
-
 void
 LayerManagerComposite::PostProcessLayers(Layer* aLayer,
                                          nsIntRegion& aOpaqueRegion,
@@ -307,7 +292,7 @@ LayerManagerComposite::PostProcessLayers(Layer* aLayer,
     // that it can later be intersected with our visible region.
     // If our transform is a perspective, there's no meaningful insideClip rect
     // we can compute (it would need to be a cone).
-    Matrix4x4 localTransform = GetAccTransformIn3DContext(aLayer);
+    Matrix4x4 localTransform = aLayer->ComputeTransformToPreserve3DRoot();
     if (!localTransform.HasPerspectiveComponent() && localTransform.Invert()) {
       LayerRect insideClipFloat =
         UntransformBy(ViewAs<ParentLayerToLayerMatrix4x4>(localTransform),
@@ -327,6 +312,13 @@ LayerManagerComposite::PostProcessLayers(Layer* aLayer,
       Some(ViewAs<ParentLayerPixel>(*insideClip, PixelCastJustification::MovingDownToChildren));
   }
 
+  nsIntRegion dummy;
+  nsIntRegion& opaqueRegion = aOpaqueRegion;
+  if (aLayer->Extend3DContext() ||
+      aLayer->Combines3DTransformWithAncestors()) {
+   opaqueRegion = dummy;
+  }
+
   if (!ShouldProcessLayer(aLayer)) {
     MOZ_ASSERT(aLayer->AsContainerLayer() && !aLayer->AsContainerLayer()->UseIntermediateSurface());
     // For layers participating 3D rendering context, their visible
@@ -345,7 +337,7 @@ LayerManagerComposite::PostProcessLayers(Layer* aLayer,
         renderTargetClip = IntersectMaybeRects(renderTargetClip, Some(clip));
       }
 
-      PostProcessLayers(child, aOpaqueRegion, aVisibleRegion,
+      PostProcessLayers(child, opaqueRegion, aVisibleRegion,
                         renderTargetClip, ancestorClipForChildren);
     }
     return;
@@ -363,7 +355,7 @@ LayerManagerComposite::PostProcessLayers(Layer* aLayer,
   if (transform.Is2D(&transform2d)) {
     if (transform2d.IsIntegerTranslation()) {
       integerTranslation = Some(IntPoint::Truncate(transform2d.GetTranslation()));
-      localOpaque = aOpaqueRegion;
+      localOpaque = opaqueRegion;
       localOpaque.MoveBy(-*integerTranslation);
     }
   }
@@ -431,7 +423,7 @@ LayerManagerComposite::PostProcessLayers(Layer* aLayer,
     if (aRenderTargetClip) {
       localOpaque.AndWith(aRenderTargetClip->ToUnknownRect());
     }
-    aOpaqueRegion.OrWith(localOpaque);
+    opaqueRegion.OrWith(localOpaque);
   }
 }
 

gfx/layers/composite/LayerManagerComposite.h

@@ -607,12 +607,11 @@ public:
   // These getters can be used anytime.
   float GetShadowOpacity() { return mShadowOpacity; }
   const Maybe<ParentLayerIntRect>& GetShadowClipRect() { return mShadowClipRect; }
-  const LayerIntRegion& GetShadowVisibleRegion() const { return mShadowVisibleRegion; }
+  virtual const LayerIntRegion& GetShadowVisibleRegion() { return mShadowVisibleRegion; }
   const gfx::Matrix4x4& GetShadowBaseTransform() { return mShadowTransform; }
   gfx::Matrix4x4 GetShadowTransform();
   bool GetShadowTransformSetByAnimation() { return mShadowTransformSetByAnimation; }
   bool GetShadowOpacitySetByAnimation() { return mShadowOpacitySetByAnimation; }
-  LayerIntRegion&& GetShadowVisibleRegion() { return Move(mShadowVisibleRegion); }
 
 protected:
   HostLayerManager* mCompositorManager;

testing/web-platform/meta/css/css-transforms-2/transform3d-sorting-001.html.ini

@@ -1,3 +0,0 @@
-[transform3d-sorting-001.html]
-  type: reftest
-  expected: FAIL