Bug 1254011 - Avoid allocating RGB buffer for YUV data everytime r=nical

gfx/layers/BufferTexture.cpp

@@ -97,13 +97,12 @@ static bool UsingX11Compositor()
   return false;
 }
 
-static bool ComputeHasIntermediateBuffer(gfx::SurfaceFormat aFormat,
-                                         LayersBackend aLayersBackend)
+bool ComputeHasIntermediateBuffer(gfx::SurfaceFormat aFormat,
+                                  LayersBackend aLayersBackend)
 {
   return aLayersBackend != LayersBackend::LAYERS_BASIC
       || UsingX11Compositor()
-      || aFormat == gfx::SurfaceFormat::UNKNOWN
-      || aFormat == gfx::SurfaceFormat::YUV;
+      || aFormat == gfx::SurfaceFormat::UNKNOWN;
 }
 
 BufferTextureData*
@@ -158,10 +157,16 @@ BufferTextureData::CreateForYCbCrWithBufferSize(ClientIPCAllocator* aAllocator,
     return nullptr;
   }
 
+  auto fwd = aAllocator->AsCompositableForwarder();
+  bool hasIntermediateBuffer = fwd ? ComputeHasIntermediateBuffer(gfx::SurfaceFormat::YUV,
+                                                                  fwd->GetCompositorBackendType())
+                                   : true;
+
   // Initialize the metadata with something, even if it will have to be rewritten
   // afterwards since we don't know the dimensions of the texture at this point.
   BufferDescriptor desc = YCbCrDescriptor(gfx::IntSize(), gfx::IntSize(),
-                                          0, 0, 0, StereoMode::MONO);
+                                          0, 0, 0, StereoMode::MONO,
+                                          hasIntermediateBuffer);
 
   return CreateInternal(aAllocator, desc, gfx::BackendType::NONE, aBufferSize,
                         aTextureFlags);
@@ -186,8 +191,14 @@ BufferTextureData::CreateForYCbCr(ClientIPCAllocator* aAllocator,
                                           aCbCrSize.width, aCbCrSize.height,
                                           yOffset, cbOffset, crOffset);
 
+  auto fwd = aAllocator ? aAllocator->AsCompositableForwarder() : nullptr;
+  bool hasIntermediateBuffer = fwd ? ComputeHasIntermediateBuffer(gfx::SurfaceFormat::YUV,
+                                                                  fwd->GetCompositorBackendType())
+                                   : true;
+
   YCbCrDescriptor descriptor = YCbCrDescriptor(aYSize, aCbCrSize, yOffset, cbOffset,
-                                               crOffset, aStereoMode);
+                                               crOffset, aStereoMode,
+                                               hasIntermediateBuffer);
 
  return CreateInternal(aAllocator, descriptor, gfx::BackendType::NONE, bufSize,
                        aTextureFlags);
@@ -202,7 +213,7 @@ BufferTextureData::FillInfo(TextureData::Info& aInfo) const
   aInfo.canExposeMappedData = true;
 
   if (mDescriptor.type() == BufferDescriptor::TYCbCrDescriptor) {
-    aInfo.hasIntermediateBuffer = true;
+    aInfo.hasIntermediateBuffer = mDescriptor.get_YCbCrDescriptor().hasIntermediateBuffer();
   } else {
     aInfo.hasIntermediateBuffer = mDescriptor.get_RGBDescriptor().hasIntermediateBuffer();
   }

gfx/layers/BufferTexture.h

@@ -15,6 +15,9 @@
 namespace mozilla {
 namespace layers {
 
+bool ComputeHasIntermediateBuffer(gfx::SurfaceFormat aFormat,
+                                  LayersBackend aLayersBackend);
+
 class BufferTextureData : public TextureData
 {
 public:

gfx/layers/Compositor.h

@@ -200,6 +200,9 @@ public:
   virtual already_AddRefed<DataTextureSource>
   CreateDataTextureSourceAround(gfx::DataSourceSurface* aSurface) { return nullptr; }
 
+  virtual already_AddRefed<DataTextureSource>
+  CreateDataTextureSourceAroundYCbCr(TextureHost* aTexture) { return nullptr; }
+
   virtual bool Initialize(nsCString* const out_failureReason) = 0;
   virtual void Destroy();
   bool IsDestroyed() const { return mIsDestroyed; }

gfx/layers/ImageDataSerializer.cpp

@@ -168,15 +168,27 @@ uint8_t* GetCrChannel(uint8_t* aBuffer, const YCbCrDescriptor& aDescriptor)
 }
 
 already_AddRefed<DataSourceSurface>
-DataSourceSurfaceFromYCbCrDescriptor(uint8_t* aBuffer, const YCbCrDescriptor& aDescriptor)
+DataSourceSurfaceFromYCbCrDescriptor(uint8_t* aBuffer, const YCbCrDescriptor& aDescriptor, gfx::DataSourceSurface* aSurface)
 {
   gfx::IntSize ySize = aDescriptor.ySize();
   gfx::IntSize cbCrSize = aDescriptor.cbCrSize();
   int32_t yStride = ySize.width;
   int32_t cbCrStride = cbCrSize.width;
 
-  RefPtr<DataSourceSurface> result =
-    Factory::CreateDataSourceSurface(ySize, gfx::SurfaceFormat::B8G8R8X8);
+  RefPtr<DataSourceSurface> result;
+  if (aSurface) {
+    MOZ_ASSERT(aSurface->GetSize() == ySize);
+    MOZ_ASSERT(aSurface->GetFormat() == gfx::SurfaceFormat::B8G8R8X8);
+    if (aSurface->GetSize() == ySize &&
+        aSurface->GetFormat() == gfx::SurfaceFormat::B8G8R8X8) {
+      result = aSurface;
+    }
+  }
+
+  if (!result) {
+    result =
+      Factory::CreateDataSourceSurface(ySize, gfx::SurfaceFormat::B8G8R8X8);
+  }
   if (NS_WARN_IF(!result)) {
     return nullptr;
   }

gfx/layers/ImageDataSerializer.h

@@ -69,7 +69,7 @@ uint8_t* GetCbChannel(uint8_t* aBuffer, const YCbCrDescriptor& aDescriptor);
 uint8_t* GetCrChannel(uint8_t* aBuffer, const YCbCrDescriptor& aDescriptor);
 
 already_AddRefed<gfx::DataSourceSurface>
-DataSourceSurfaceFromYCbCrDescriptor(uint8_t* aBuffer, const YCbCrDescriptor& aDescriptor);
+DataSourceSurfaceFromYCbCrDescriptor(uint8_t* aBuffer, const YCbCrDescriptor& aDescriptor, gfx::DataSourceSurface* aSurface = nullptr);
 
 } // ImageDataSerializer
 

gfx/layers/basic/BasicCompositor.cpp

@@ -13,6 +13,7 @@
 #include "mozilla/gfx/Helpers.h"
 #include "mozilla/gfx/Tools.h"
 #include "mozilla/gfx/ssse3-scaler.h"
+#include "mozilla/layers/ImageDataSerializer.h"
 #include "mozilla/SSE.h"
 #include "gfxUtils.h"
 #include "YCbCrUtils.h"
@@ -80,6 +81,103 @@ public:
   bool mWrappingExistingData;
 };
 
+/**
+ * WrappingTextureSourceYCbCrBasic wraps YUV format BufferTextureHost to defer
+ * yuv->rgb conversion. The conversion happens when GetSurface is called.
+ */
+class WrappingTextureSourceYCbCrBasic : public DataTextureSource
+                                      , public TextureSourceBasic
+{
+public:
+  virtual const char* Name() const override { return "WrappingTextureSourceYCbCrBasic"; }
+
+  explicit WrappingTextureSourceYCbCrBasic(BufferTextureHost* aTexture)
+  : mTexture(aTexture)
+  , mSize(aTexture->GetSize())
+  , mNeedsUpdate(true)
+  {
+    mFromYCBCR = true;
+  }
+
+  virtual DataTextureSource* AsDataTextureSource() override
+  {
+    return this;
+  }
+
+  virtual TextureSourceBasic* AsSourceBasic() override { return this; }
+
+  virtual WrappingTextureSourceYCbCrBasic* AsWrappingTextureSourceYCbCrBasic() override { return this; }
+
+  virtual gfx::SourceSurface* GetSurface(DrawTarget* aTarget) override
+  {
+    if (mSurface && !mNeedsUpdate) {
+      return mSurface;
+    }
+    MOZ_ASSERT(mTexture);
+    if (!mTexture) {
+      return nullptr;
+    }
+
+    if (!mSurface) {
+      mSurface = Factory::CreateDataSourceSurface(mSize, gfx::SurfaceFormat::B8G8R8X8);
+    }
+    if (!mSurface) {
+      return nullptr;
+    }
+    MOZ_ASSERT(mTexture->GetBufferDescriptor().type() == BufferDescriptor::TYCbCrDescriptor);
+    MOZ_ASSERT(mTexture->GetSize() == mSize);
+
+    mSurface =
+      ImageDataSerializer::DataSourceSurfaceFromYCbCrDescriptor(
+        mTexture->GetBuffer(),
+        mTexture->GetBufferDescriptor().get_YCbCrDescriptor(),
+        mSurface);
+    mNeedsUpdate = false;
+    return mSurface;
+  }
+
+  SurfaceFormat GetFormat() const override
+  {
+    return gfx::SurfaceFormat::B8G8R8X8;
+  }
+
+  virtual IntSize GetSize() const override
+  {
+    return mSize;
+  }
+
+  virtual bool Update(gfx::DataSourceSurface* aSurface,
+                      nsIntRegion* aDestRegion = nullptr,
+                      gfx::IntPoint* aSrcOffset = nullptr) override
+  {
+    return false;
+  }
+
+  virtual void DeallocateDeviceData() override
+  {
+    mTexture = nullptr;
+    mSurface = nullptr;
+    SetUpdateSerial(0);
+  }
+
+  virtual void Unbind() override
+  {
+    mNeedsUpdate = true;
+  }
+
+  void SetBufferTextureHost(BufferTextureHost* aTexture) override
+  {
+    mTexture = aTexture;
+    mNeedsUpdate = true;
+  }
+
+public:
+  BufferTextureHost* mTexture;
+  const gfx::IntSize mSize;
+  RefPtr<gfx::DataSourceSurface> mSurface;
+  bool mNeedsUpdate;
+};
+
 BasicCompositor::BasicCompositor(CompositorBridgeParent* aParent, widget::CompositorWidget* aWidget)
   : Compositor(aWidget, aParent)
   , mDidExternalComposition(false)
@@ -215,6 +313,19 @@ BasicCompositor::CreateDataTextureSourceAround(DataSourceSurface* aSurface)
   return result.forget();
 }
 
+already_AddRefed<DataTextureSource>
+BasicCompositor::CreateDataTextureSourceAroundYCbCr(TextureHost* aTexture)
+{
+  BufferTextureHost* bufferTexture = aTexture->AsBufferTextureHost();
+  MOZ_ASSERT(bufferTexture);
+
+  if (!bufferTexture) {
+    return nullptr;
+  }
+  RefPtr<DataTextureSource> result = new WrappingTextureSourceYCbCrBasic(bufferTexture);
+  return result.forget();
+}
+
 bool
 BasicCompositor::SupportsEffect(EffectTypes aEffect)
 {

gfx/layers/basic/BasicCompositor.h

@@ -75,6 +75,9 @@ public:
   virtual already_AddRefed<DataTextureSource>
   CreateDataTextureSourceAround(gfx::DataSourceSurface* aSurface) override;
 
+  virtual already_AddRefed<DataTextureSource>
+  CreateDataTextureSourceAroundYCbCr(TextureHost* aTexture) override;
+
   virtual bool SupportsEffect(EffectTypes aEffect) override;
 
   virtual void SetRenderTarget(CompositingRenderTarget *aSource) override

gfx/layers/basic/TextureHostBasic.h

@@ -24,6 +24,7 @@ public:
   TextureSourceBasic() : mFromYCBCR(false) {}
   virtual ~TextureSourceBasic() {}
   virtual gfx::SourceSurface* GetSurface(gfx::DrawTarget* aTarget) = 0;
+  virtual void SetBufferTextureHost(BufferTextureHost* aTexture) {}
   bool mFromYCBCR; // we to track sources from YCBCR so we can use a less accurate fast path for video
 };
 

gfx/layers/composite/TextureHost.cpp

@@ -15,6 +15,7 @@
 #include "mozilla/layers/Compositor.h"  // for Compositor
 #include "mozilla/layers/ISurfaceAllocator.h"  // for ISurfaceAllocator
 #include "mozilla/layers/LayersSurfaces.h"  // for SurfaceDescriptor, etc
+#include "mozilla/layers/TextureHostBasic.h"
 #include "mozilla/layers/TextureHostOGL.h"  // for TextureHostOGL
 #include "mozilla/layers/ImageDataSerializer.h"
 #include "mozilla/layers/TextureClient.h"
@@ -474,7 +475,7 @@ BufferTextureHost::BufferTextureHost(const BufferDescriptor& aDesc,
       const YCbCrDescriptor& ycbcr = mDescriptor.get_YCbCrDescriptor();
       mSize = ycbcr.ySize();
       mFormat = gfx::SurfaceFormat::YUV;
-      mHasIntermediateBuffer = true;
+      mHasIntermediateBuffer = ycbcr.hasIntermediateBuffer();
       break;
     }
     case BufferDescriptor::TRGBDescriptor: {
@@ -605,7 +606,6 @@ bool
 BufferTextureHost::EnsureWrappingTextureSource()
 {
   MOZ_ASSERT(!mHasIntermediateBuffer);
-  MOZ_ASSERT(mFormat != gfx::SurfaceFormat::YUV);
 
   if (mFirstSource) {
     return true;
@@ -615,15 +615,18 @@ BufferTextureHost::EnsureWrappingTextureSource()
     return false;
   }
 
-  RefPtr<gfx::DataSourceSurface> surf =
-    gfx::Factory::CreateWrappingDataSourceSurface(GetBuffer(),
-      ImageDataSerializer::ComputeRGBStride(mFormat, mSize.width), mSize, mFormat);
-
-  if (!surf) {
-    return false;
+  if (mFormat == gfx::SurfaceFormat::YUV) {
+    mFirstSource = mCompositor->CreateDataTextureSourceAroundYCbCr(this);
+  } else {
+    RefPtr<gfx::DataSourceSurface> surf =
+      gfx::Factory::CreateWrappingDataSourceSurface(GetBuffer(),
+        ImageDataSerializer::ComputeRGBStride(mFormat, mSize.width), mSize, mFormat);
+    if (!surf) {
+      return false;
+    }
+    mFirstSource = mCompositor->CreateDataTextureSourceAround(surf);
   }
 
-  mFirstSource = mCompositor->CreateDataTextureSourceAround(surf);
   if (!mFirstSource) {
     // BasicCompositor::CreateDataTextureSourceAround never returns null
     // and we don't expect to take this branch if we are using another backend.
@@ -693,6 +696,18 @@ bool IsCompatibleTextureSource(TextureSource* aTexture,
 void
 BufferTextureHost::PrepareTextureSource(CompositableTextureSourceRef& aTexture)
 {
+  // Reuse WrappingTextureSourceYCbCrBasic to reduce memory consumption.
+  if (mFormat == gfx::SurfaceFormat::YUV &&
+      !mHasIntermediateBuffer &&
+      aTexture.get() &&
+      aTexture->AsWrappingTextureSourceYCbCrBasic() &&
+      aTexture->NumCompositableRefs() <= 1 &&
+      aTexture->GetSize() == GetSize()) {
+    aTexture->AsSourceBasic()->SetBufferTextureHost(this);
+    aTexture->AsDataTextureSource()->SetOwner(this);
+    mFirstSource = aTexture->AsDataTextureSource();
+  }
+
   if (!mHasIntermediateBuffer) {
     EnsureWrappingTextureSource();
   }
@@ -745,6 +760,9 @@ BufferTextureHost::BindTextureSource(CompositableTextureSourceRef& aTexture)
 void
 BufferTextureHost::UnbindTextureSource()
 {
+  if (mFirstSource && mFirstSource->IsOwnedBy(this)) {
+    mFirstSource->Unbind();
+  }
   // This texture is not used by any layer anymore.
   // If the texture doesn't have an intermediate buffer, it means we are
   // compositing synchronously on the CPU, so we don't need to wait until

gfx/layers/composite/TextureHost.h

@@ -39,6 +39,7 @@ class Shmem;
 namespace layers {
 
 class BufferDescriptor;
+class BufferTextureHost;
 class Compositor;
 class CompositableParentManager;
 class ReadLockDescriptor;
@@ -56,6 +57,7 @@ class TextureSourceBasic;
 class DataTextureSource;
 class PTextureParent;
 class TextureParent;
+class WrappingTextureSourceYCbCrBasic;
 
 /**
  * A view on a TextureHost where the texture is internally represented as tiles
@@ -127,6 +129,7 @@ public:
    * Cast to a DataTextureSurce.
    */
   virtual DataTextureSource* AsDataTextureSource() { return nullptr; }
+  virtual WrappingTextureSourceYCbCrBasic* AsWrappingTextureSourceYCbCrBasic() { return nullptr; }
 
   /**
    * Overload this if the TextureSource supports big textures that don't fit in
@@ -136,6 +139,8 @@ public:
 
   virtual void SetCompositor(Compositor* aCompositor) {}
 
+  virtual void Unbind() {}
+
   void SetNextSibling(TextureSource* aTexture) { mNextSibling = aTexture; }
 
   TextureSource* GetNextSibling() const { return mNextSibling; }
@@ -604,6 +609,8 @@ public:
 
   virtual Compositor* GetCompositor() = 0;
 
+  virtual BufferTextureHost* AsBufferTextureHost() { return nullptr; }
+
   virtual GrallocTextureHostOGL* AsGrallocTextureHostOGL() { return nullptr; }
 
 protected:
@@ -691,6 +698,10 @@ public:
 
   virtual bool HasIntermediateBuffer() const override { return mHasIntermediateBuffer; }
 
+  virtual BufferTextureHost* AsBufferTextureHost() override { return this; }
+
+  const BufferDescriptor& GetBufferDescriptor() const { return mDescriptor; }
+
 protected:
   bool Upload(nsIntRegion *aRegion = nullptr);
   bool MaybeUpload(nsIntRegion *aRegion = nullptr);
@@ -708,6 +719,8 @@ protected:
   bool mLocked;
   bool mNeedsFullUpdate;
   bool mHasIntermediateBuffer;
+
+  class DataTextureSourceYCbCrBasic;
 };
 
 /**

gfx/layers/ipc/LayersSurfaces.ipdlh

@@ -111,6 +111,7 @@ struct YCbCrDescriptor {
   uint32_t cbOffset;
   uint32_t crOffset;
   StereoMode stereoMode;
+  bool hasIntermediateBuffer;
 };
 
 union BufferDescriptor {

gfx/layers/ipc/SharedPlanarYCbCrImage.cpp

@@ -159,9 +159,14 @@ SharedPlanarYCbCrImage::AdoptData(const Data &aData)
   uint32_t cbOffset = aData.mCbChannel - base;
   uint32_t crOffset = aData.mCrChannel - base;
 
+  auto fwd = mCompositable->GetForwarder()->AsCompositableForwarder();
+  bool hasIntermediateBuffer = fwd ? ComputeHasIntermediateBuffer(gfx::SurfaceFormat::YUV,
+                                                                  fwd->GetCompositorBackendType())
+                                   : true;
+
   static_cast<BufferTextureData*>(mTextureClient->GetInternalData())->SetDesciptor(
     YCbCrDescriptor(aData.mYSize, aData.mCbCrSize, yOffset, cbOffset, crOffset,
-                    aData.mStereoMode)
+                    aData.mStereoMode, hasIntermediateBuffer)
   );
 
   return true;