Bug 1863367 - Part 1. Add Image::BuildSurfaceDescriptorBuffer and use for RDD readback. r=gfx-reviewers,lsalzman

This patch adds a new method for Image subclasses to override in order
to avoid extra copies when the ultimate destination of a readback is
into an IPDL Shmem. It also makes RemoteDecoderManagerParent's
RecvReadback take advantage of it.

Differential Revision: https://phabricator.services.mozilla.com/D192883

dom/media/ipc/RemoteDecoderManagerParent.cpp

@@ -289,6 +289,32 @@ mozilla::ipc::IPCResult RemoteDecoderManagerParent::RecvReadback(
     return IPC_OK();
   }
 
+  // Let's try reading directly into the shmem first to avoid extra copies.
+  SurfaceDescriptorBuffer sdb;
+  nsresult rv = image->BuildSurfaceDescriptorBuffer(
+      sdb, Image::BuildSdbFlags::RgbOnly, [&](uint32_t aBufferSize) {
+        Shmem buffer;
+        if (!AllocShmem(aBufferSize, &buffer)) {
+          return MemoryOrShmem();
+        }
+        return MemoryOrShmem(std::move(buffer));
+      });
+
+  if (NS_SUCCEEDED(rv)) {
+    *aResult = std::move(sdb);
+    return IPC_OK();
+  }
+
+  if (sdb.data().type() == MemoryOrShmem::TShmem) {
+    DeallocShmem(sdb.data().get_Shmem());
+  }
+
+  if (rv != NS_ERROR_NOT_IMPLEMENTED) {
+    *aResult = null_t();
+    return IPC_OK();
+  }
+
+  // Fallback to reading to a SourceSurface and copying that into a shmem.
   RefPtr<SourceSurface> source = image->GetAsSourceSurface();
   if (!source) {
     *aResult = null_t();

dom/media/ipc/RemoteVideoDecoder.cpp

@@ -252,7 +252,7 @@ MediaResult RemoteVideoDecoderParent::ProcessDecodedData(
 
       SurfaceDescriptorBuffer sdBuffer;
       nsresult rv = image->BuildSurfaceDescriptorBuffer(
-          sdBuffer, [&](uint32_t aBufferSize) {
+          sdBuffer, Image::BuildSdbFlags::Default, [&](uint32_t aBufferSize) {
             ShmemBuffer buffer = AllocateBuffer(aBufferSize);
             if (buffer.Valid()) {
               return MemoryOrShmem(std::move(buffer.Get()));

gfx/layers/ImageContainer.cpp

@@ -20,6 +20,7 @@
 #include "mozilla/StaticPrefs_layers.h"
 #include "mozilla/gfx/2D.h"
 #include "mozilla/gfx/gfxVars.h"
+#include "mozilla/gfx/Swizzle.h"
 #include "mozilla/ipc/CrossProcessMutex.h"  // for CrossProcessMutex, etc
 #include "mozilla/layers/CompositorTypes.h"
 #include "mozilla/layers/ImageBridgeChild.h"     // for ImageBridgeChild
@@ -214,6 +215,42 @@ ImageContainer::~ImageContainer() {
   }
 }
 
+/* static */ nsresult Image::AllocateSurfaceDescriptorBufferRgb(
+    const gfx::IntSize& aSize, gfx::SurfaceFormat aFormat, uint8_t*& aOutBuffer,
+    SurfaceDescriptorBuffer& aSdBuffer, int32_t& aStride,
+    const std::function<layers::MemoryOrShmem(uint32_t)>& aAllocate) {
+  aStride = ImageDataSerializer::ComputeRGBStride(aFormat, aSize.width);
+  size_t length = ImageDataSerializer::ComputeRGBBufferSize(aSize, aFormat);
+
+  if (aStride <= 0 || length == 0) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  aSdBuffer.desc() = RGBDescriptor(aSize, aFormat);
+  aSdBuffer.data() = aAllocate(length);
+
+  const layers::MemoryOrShmem& memOrShmem = aSdBuffer.data();
+  switch (memOrShmem.type()) {
+    case layers::MemoryOrShmem::Tuintptr_t:
+      aOutBuffer = reinterpret_cast<uint8_t*>(memOrShmem.get_uintptr_t());
+      break;
+    case layers::MemoryOrShmem::TShmem:
+      aOutBuffer = memOrShmem.get_Shmem().get<uint8_t>();
+      break;
+    default:
+      return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  MOZ_ASSERT(aOutBuffer);
+  return NS_OK;
+}
+
+nsresult Image::BuildSurfaceDescriptorBuffer(
+    SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
+    const std::function<MemoryOrShmem(uint32_t)>& aAllocate) {
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
 Maybe<SurfaceDescriptor> Image::GetDesc() { return GetDescFromTexClient(); }
 
 Maybe<SurfaceDescriptor> Image::GetDescFromTexClient(
@@ -619,13 +656,41 @@ PlanarYCbCrImage::PlanarYCbCrImage()
       mBufferSize(0) {}
 
 nsresult PlanarYCbCrImage::BuildSurfaceDescriptorBuffer(
-    SurfaceDescriptorBuffer& aSdBuffer,
+    SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
     const std::function<MemoryOrShmem(uint32_t)>& aAllocate) {
   const PlanarYCbCrData* pdata = GetData();
   MOZ_ASSERT(pdata, "must have PlanarYCbCrData");
   MOZ_ASSERT(pdata->mYSkip == 0 && pdata->mCbSkip == 0 && pdata->mCrSkip == 0,
              "YCbCrDescriptor doesn't hold skip values");
 
+  if (aFlags & BuildSdbFlags::RgbOnly) {
+    gfx::IntSize size(mSize);
+    auto format = gfx::ImageFormatToSurfaceFormat(GetOffscreenFormat());
+    gfx::GetYCbCrToRGBDestFormatAndSize(mData, format, size);
+
+    uint8_t* buffer = nullptr;
+    int32_t stride = 0;
+    nsresult rv = AllocateSurfaceDescriptorBufferRgb(
+        size, format, buffer, aSdBuffer, stride, aAllocate);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    // If we can copy directly from the surface, let's do that to avoid the YUV
+    // to RGB conversion.
+    if (mSourceSurface && mSourceSurface->GetSize() == size) {
+      DataSourceSurface::ScopedMap map(mSourceSurface, DataSourceSurface::READ);
+      if (map.IsMapped() && SwizzleData(map.GetData(), map.GetStride(),
+                                        mSourceSurface->GetFormat(), buffer,
+                                        stride, format, size)) {
+        return NS_OK;
+      }
+    }
+
+    gfx::ConvertYCbCrToRGB(mData, format, size, buffer, stride);
+    return NS_OK;
+  }
+
   auto ySize = pdata->YDataSize();
   auto cbcrSize = pdata->CbCrDataSize();
   uint32_t yOffset;

gfx/layers/ImageContainer.h

@@ -21,6 +21,7 @@
 #include "mozilla/layers/LayersTypes.h"  // for LayersBackend, etc
 #include "mozilla/layers/CompositorTypes.h"
 #include "mozilla/mozalloc.h"  // for operator delete, etc
+#include "mozilla/TypedEnumBits.h"
 #include "nsDebug.h"           // for NS_ASSERTION
 #include "nsISupportsImpl.h"   // for Image::Release, etc
 #include "nsTArray.h"          // for nsTArray
@@ -129,6 +130,15 @@ class Image {
 
   virtual already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() = 0;
 
+  enum class BuildSdbFlags : uint8_t {
+    Default = 0,
+    RgbOnly = 1 << 0,
+  };
+
+  virtual nsresult BuildSurfaceDescriptorBuffer(
+      SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
+      const std::function<MemoryOrShmem(uint32_t)>& aAllocate);
+
   virtual bool IsValid() const { return true; }
 
   /**
@@ -157,6 +167,12 @@ class Image {
 
   virtual Maybe<SurfaceDescriptor> GetDesc();
 
+  static nsresult AllocateSurfaceDescriptorBufferRgb(
+      const gfx::IntSize& aSize, gfx::SurfaceFormat aFormat,
+      uint8_t*& aOutBuffer, SurfaceDescriptorBuffer& aSdBuffer,
+      int32_t& aStride,
+      const std::function<layers::MemoryOrShmem(uint32_t)>& aAllocate);
+
  protected:
   Maybe<SurfaceDescriptor> GetDescFromTexClient(
       TextureClient* tcOverride = nullptr);
@@ -183,6 +199,8 @@ class Image {
   static mozilla::Atomic<int32_t> sSerialCounter;
 };
 
+MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(Image::BuildSdbFlags)
+
 /**
  * A RecycleBin is owned by an ImageContainer. We store buffers in it that we
  * want to recycle from one image to the next.It's a separate object from
@@ -829,9 +847,9 @@ class PlanarYCbCrImage : public Image {
    * Build a SurfaceDescriptorBuffer with this image.  A function to allocate
    * a MemoryOrShmem with the given capacity must be provided.
    */
-  virtual nsresult BuildSurfaceDescriptorBuffer(
-      SurfaceDescriptorBuffer& aSdBuffer,
-      const std::function<MemoryOrShmem(uint32_t)>& aAllocate);
+  nsresult BuildSurfaceDescriptorBuffer(
+      SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
+      const std::function<MemoryOrShmem(uint32_t)>& aAllocate) override;
 
  protected:
   already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;
@@ -845,7 +863,7 @@ class PlanarYCbCrImage : public Image {
   gfx::IntPoint mOrigin;
   gfx::IntSize mSize;
   gfxImageFormat mOffscreenFormat;
-  RefPtr<gfx::SourceSurface> mSourceSurface;
+  RefPtr<gfx::DataSourceSurface> mSourceSurface;
   uint32_t mBufferSize;
 };