Bug 1359206 - Fix CompositionTime and CompositeUntil handling of WebRenderImageHost r=nical

gfx/layers/composite/ImageComposite.cpp

@@ -5,15 +5,14 @@
 
 #include "ImageComposite.h"
 
-// this is also defined in ImageHost.cpp
-#define BIAS_TIME_MS 1.0
-
 namespace mozilla {
 
 using namespace gfx;
 
 namespace layers {
 
+/* static */ const float ImageComposite::BIAS_TIME_MS = 1.0f;
+
 ImageComposite::ImageComposite()
   : mLastFrameID(-1)
   , mLastProducerID(-1)
@@ -24,8 +23,8 @@ ImageComposite::~ImageComposite()
 {
 }
 
-static TimeStamp
-GetBiasedTime(const TimeStamp& aInput, ImageComposite::Bias aBias)
+/* static */ TimeStamp
+ImageComposite::GetBiasedTime(const TimeStamp& aInput, ImageComposite::Bias aBias)
 {
   switch (aBias) {
   case ImageComposite::BIAS_NEGATIVE:
@@ -120,5 +119,3 @@ ImageComposite::TimedImage* ImageComposite::ChooseImage()
 
 } // namespace layers
 } // namespace mozilla
-
-#undef BIAS_TIME_MS

gfx/layers/composite/ImageComposite.h

@@ -20,6 +20,8 @@ namespace layers {
 class ImageComposite
 {
 public:
+  static const float BIAS_TIME_MS;
+
   explicit ImageComposite();
   ~ImageComposite();
 
@@ -47,6 +49,8 @@ public:
     BIAS_POSITIVE,
   };
 
+  static TimeStamp GetBiasedTime(const TimeStamp& aInput, ImageComposite::Bias aBias);
+
 protected:
   static Bias UpdateBias(const TimeStamp& aCompositionTime,
                          const TimeStamp& aCompositedImageTime,

gfx/layers/composite/ImageHost.cpp

@@ -16,9 +16,6 @@
 #include "nsPrintfCString.h"            // for nsPrintfCString
 #include "nsString.h"                   // for nsAutoCString
 
-// this is also defined in ImageComposite.cpp
-#define BIAS_TIME_MS 1.0
-
 namespace mozilla {
 
 using namespace gfx;
@@ -476,5 +473,3 @@ ImageHost::GenEffect(const gfx::SamplingFilter aSamplingFilter)
 
 } // namespace layers
 } // namespace mozilla
-
-#undef BIAS_TIME_MS

gfx/layers/wr/WebRenderBridgeParent.cpp

@@ -21,7 +21,9 @@
 #include "mozilla/layers/ImageDataSerializer.h"
 #include "mozilla/layers/TextureHost.h"
 #include "mozilla/layers/WebRenderCompositableHolder.h"
+#include "mozilla/layers/WebRenderImageHost.h"
 #include "mozilla/layers/WebRenderTextureHost.h"
+#include "mozilla/TimeStamp.h"
 #include "mozilla/webrender/RenderThread.h"
 #include "mozilla/widget/CompositorWidget.h"
 
@@ -401,6 +403,7 @@ WebRenderBridgeParent::ProcessWebRenderCommands(const gfx::IntSize &aSize,
                                                 const ByteBuffer& aux,
                                                 const WrAuxiliaryListsDescriptor& auxDesc)
 {
+  mCompositableHolder->SetCompositionTime(TimeStamp::Now());
 
   for (InfallibleTArray<WebRenderParentCommand>::index_type i = 0; i < aCommands.Length(); ++i) {
     const WebRenderParentCommand& cmd = aCommands[i];
@@ -412,13 +415,13 @@ WebRenderBridgeParent::ProcessWebRenderCommands(const gfx::IntSize &aSize,
         MOZ_ASSERT(!mActiveKeys.Get(wr::AsUint64(key), nullptr));
         mActiveKeys.Put(wr::AsUint64(key), key);
 
-        RefPtr<CompositableHost> host = mExternalImageIds.Get(wr::AsUint64(op.externalImageId()));
+        RefPtr<WebRenderImageHost> host = mExternalImageIds.Get(wr::AsUint64(op.externalImageId()));
         if (!host) {
           NS_ERROR("CompositableHost does not exist");
           break;
         }
         // XXX select Texture for video in CompositeToTarget().
-        TextureHost* texture = host->GetAsTextureHost();
+        TextureHost* texture = host->GetAsTextureHostForComposite();
         if (!texture) {
           NS_ERROR("TextureHost does not exist");
           break;
@@ -587,12 +590,14 @@ WebRenderBridgeParent::RecvAddExternalImageId(const ExternalImageId& aImageId,
     return IPC_FAIL_NO_REASON(this);
   }
   MOZ_ASSERT(host->AsWebRenderImageHost());
-  if (!host->AsWebRenderImageHost()) {
+  WebRenderImageHost* wrHost = host->AsWebRenderImageHost();
+  if (!wrHost) {
     NS_ERROR("Incompatible CompositableHost");
     return IPC_OK();
   }
 
-  mExternalImageIds.Put(wr::AsUint64(aImageId), host);
+  wrHost->SetWrCompositableHolder(mCompositableHolder);
+  mExternalImageIds.Put(wr::AsUint64(aImageId), wrHost);
 
   return IPC_OK();
 }
@@ -608,12 +613,14 @@ WebRenderBridgeParent::RecvAddExternalImageIdForCompositable(const ExternalImage
 
   RefPtr<CompositableHost> host = FindCompositable(aHandle);
   MOZ_ASSERT(host->AsWebRenderImageHost());
-  if (!host->AsWebRenderImageHost()) {
+  WebRenderImageHost* wrHost = host->AsWebRenderImageHost();
+  if (!wrHost) {
     NS_ERROR("Incompatible CompositableHost");
     return IPC_OK();
   }
 
-  mExternalImageIds.Put(wr::AsUint64(aImageId), host);
+  wrHost->SetWrCompositableHolder(mCompositableHolder);
+  mExternalImageIds.Put(wr::AsUint64(aImageId), wrHost);
 
   return IPC_OK();
 }
@@ -625,6 +632,10 @@ WebRenderBridgeParent::RecvRemoveExternalImageId(const ExternalImageId& aImageId
     return IPC_OK();
   }
   MOZ_ASSERT(mExternalImageIds.Get(wr::AsUint64(aImageId)).get());
+  WebRenderImageHost* wrHost = mExternalImageIds.Get(wr::AsUint64(aImageId)).get();
+  if (wrHost) {
+    wrHost->SetWrCompositableHolder(nullptr);
+  }
   mExternalImageIds.Remove(wr::AsUint64(aImageId));
 
   return IPC_OK();
@@ -708,6 +719,11 @@ WebRenderBridgeParent::CompositeToTarget(gfx::DrawTarget* aTarget, const gfx::In
   }
 
   mApi->GenerateFrame();
+
+  // XXX Enable it when async video is supported.
+  // if (!mCompositableHolder->GetCompositeUntilTime().IsNull()) {
+  //   ScheduleComposition();
+  // }
 }
 
 void
@@ -828,6 +844,9 @@ WebRenderBridgeParent::ClearResources()
     }
   }
   mCompositableHolder->RemovePipeline(mPipelineId);
+  for (auto iter = mExternalImageIds.Iter(); !iter.Done(); iter.Next()) {
+    iter.Data()->SetWrCompositableHolder(nullptr);
+  }
   mExternalImageIds.Clear();
 
   if (mWidget && mCompositorScheduler) {

gfx/layers/wr/WebRenderBridgeParent.h

@@ -37,6 +37,7 @@ class Compositor;
 class CompositorBridgeParentBase;
 class CompositorVsyncScheduler;
 class WebRenderCompositableHolder;
+class WebRenderImageHost;
 
 class WebRenderBridgeParent final : public PWebRenderBridgeParent
                                   , public CompositorVsyncSchedulerOwner
@@ -209,7 +210,7 @@ private:
   std::vector<wr::ImageKey> mKeysToDelete;
   // XXX How to handle active keys of non-ExternalImages?
   nsDataHashtable<nsUint64HashKey, wr::ImageKey> mActiveKeys;
-  nsDataHashtable<nsUint64HashKey, RefPtr<CompositableHost>> mExternalImageIds;
+  nsDataHashtable<nsUint64HashKey, RefPtr<WebRenderImageHost>> mExternalImageIds;
   nsTArray<ImageCompositeNotificationInfo> mImageCompositeNotifications;
 
   // These fields keep track of the latest layer observer epoch values in the child and the

gfx/layers/wr/WebRenderCompositableHolder.h

@@ -39,6 +39,26 @@ public:
   void HoldExternalImage(const wr::PipelineId& aPipelineId, const wr::Epoch& aEpoch, WebRenderTextureHost* aTexture);
   void Update(const wr::PipelineId& aPipelineId, const wr::Epoch& aEpoch);
 
+  TimeStamp GetCompositionTime() const {
+    return mCompositionTime;
+  }
+  void SetCompositionTime(TimeStamp aTimeStamp) {
+    mCompositionTime = aTimeStamp;
+    if (!mCompositionTime.IsNull() && !mCompositeUntilTime.IsNull() &&
+        mCompositionTime >= mCompositeUntilTime) {
+      mCompositeUntilTime = TimeStamp();
+    }
+  }
+  void CompositeUntil(TimeStamp aTimeStamp) {
+    if (mCompositeUntilTime.IsNull() ||
+        mCompositeUntilTime < aTimeStamp) {
+      mCompositeUntilTime = aTimeStamp;
+    }
+  }
+  TimeStamp GetCompositeUntilTime() const {
+    return mCompositeUntilTime;
+  }
+
 private:
 
   struct ForwardingTextureHost {
@@ -56,6 +76,14 @@ private:
   };
 
   nsClassHashtable<nsUint64HashKey, PipelineTexturesHolder> mPipelineTexturesHolders;
+
+  // Render time for the current composition.
+  TimeStamp mCompositionTime;
+
+  // When nonnull, during rendering, some compositable indicated that it will
+  // change its rendering at this time. In order not to miss it, we composite
+  // on every vsync until this time occurs (this is the latest such time).
+  TimeStamp mCompositeUntilTime;
 };
 
 } // namespace layers

gfx/layers/wr/WebRenderImageHost.cpp

@@ -9,6 +9,7 @@
 #include "mozilla/layers/Compositor.h"  // for Compositor
 #include "mozilla/layers/Effects.h"     // for TexturedEffect, Effect, etc
 #include "mozilla/layers/LayerManagerComposite.h"     // for TexturedEffect, Effect, etc
+#include "mozilla/layers/WebRenderCompositableHolder.h"
 #include "nsAString.h"
 #include "nsDebug.h"                    // for NS_WARNING, NS_ASSERTION
 #include "nsPrintfCString.h"            // for nsPrintfCString
@@ -25,10 +26,12 @@ class ISurfaceAllocator;
 WebRenderImageHost::WebRenderImageHost(const TextureInfo& aTextureInfo)
   : CompositableHost(aTextureInfo)
   , ImageComposite()
+  , mWrCompositableHolder(nullptr)
 {}
 
 WebRenderImageHost::~WebRenderImageHost()
 {
+  MOZ_ASSERT(!mWrCompositableHolder);
 }
 
 void
@@ -62,6 +65,22 @@ WebRenderImageHost::UseTextureHost(const nsTArray<TimedTexture>& aTextures)
 
   mImages.SwapElements(newImages);
   newImages.Clear();
+
+  // Video producers generally send replacement images with the same frameID but
+  // slightly different timestamps in order to sync with the audio clock. This
+  // means that any CompositeUntil() call we made in Composite() may no longer
+  // guarantee that we'll composite until the next frame is ready. Fix that here.
+  if (mWrCompositableHolder && mLastFrameID >= 0) {
+    for (size_t i = 0; i < mImages.Length(); ++i) {
+      bool frameComesAfter = mImages[i].mFrameID > mLastFrameID ||
+                             mImages[i].mProducerID != mLastProducerID;
+      if (frameComesAfter && !mImages[i].mTimeStamp.IsNull()) {
+        mWrCompositableHolder->CompositeUntil(mImages[i].mTimeStamp +
+                           TimeDuration::FromMilliseconds(BIAS_TIME_MS));
+        break;
+      }
+    }
+  }
 }
 
 void
@@ -95,8 +114,11 @@ WebRenderImageHost::RemoveTextureHost(TextureHost* aTexture)
 TimeStamp
 WebRenderImageHost::GetCompositionTime() const
 {
-  // XXX temporary workaround
-  return TimeStamp::Now();
+  TimeStamp time;
+  if (mWrCompositableHolder) {
+    time = mWrCompositableHolder->GetCompositionTime();
+  }
+  return time;
 }
 
 TextureHost*
@@ -109,6 +131,27 @@ WebRenderImageHost::GetAsTextureHost(IntRect* aPictureRect)
   return nullptr;
 }
 
+TextureHost*
+WebRenderImageHost::GetAsTextureHostForComposite()
+{
+  int imageIndex = ChooseImageIndex();
+  if (imageIndex < 0) {
+    return nullptr;
+  }
+
+  if (mWrCompositableHolder && uint32_t(imageIndex) + 1 < mImages.Length()) {
+    mWrCompositableHolder->CompositeUntil(mImages[imageIndex + 1].mTimeStamp + TimeDuration::FromMilliseconds(BIAS_TIME_MS));
+  }
+
+  TimedImage* img = &mImages[imageIndex];
+
+  if (mLastFrameID != img->mFrameID || mLastProducerID != img->mProducerID) {
+    mLastFrameID = img->mFrameID;
+    mLastProducerID = img->mProducerID;
+  }
+  return img->mTextureHost;
+}
+
 void WebRenderImageHost::Attach(Layer* aLayer,
                        TextureSourceProvider* aProvider,
                        AttachFlags aFlags)

gfx/layers/wr/WebRenderImageHost.h

@@ -12,6 +12,8 @@
 namespace mozilla {
 namespace layers {
 
+class WebRenderCompositableHolder;
+
 /**
  * ImageHost. Works with ImageClientSingle and ImageClientBuffered
  */
@@ -65,9 +67,18 @@ public:
 
   virtual WebRenderImageHost* AsWebRenderImageHost() override { return this; }
 
+  TextureHost* GetAsTextureHostForComposite();
+
+  void SetWrCompositableHolder(WebRenderCompositableHolder* aWrCompositableHolder)
+  {
+    mWrCompositableHolder = aWrCompositableHolder;
+  }
+
 protected:
   // ImageComposite
   virtual TimeStamp GetCompositionTime() const override;
+
+  WebRenderCompositableHolder* MOZ_NON_OWNING_REF mWrCompositableHolder;
 };
 
 } // namespace layers