Bug 1889275 - Prevent to call GPUVideoImage::GetAsSourceSurface() in CanvasRenderingContext2D::DrawImage() for SurfaceDescriptorD3D10 if CanvasTranslator is used r=gfx-reviewers,lsalzman

Fast path for RemoteDecoderVideoSubDescriptor::Tnull_t was added by Bug 1887818. The change is going to extend it to RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorD3D10.

D3D11Texture from GpuProcessD3D11TextureMap::GetSharedHandleOfCopiedTexture() sometimes did not work with D3D11DXVA2Manager::CopyToBGRATexture(). The GetSharedHandleOfCopiedTexture() need to be used when D3D11Device is not compositor device.

When D3D11Device is compositor device, the GetSharedHandleOfCopiedTexture() does not needed to be used. In this case, D3D11DXVA2Manager::CopyToBGRATexture() worked well.

After the CopyToBGRATexture() call, D3D11Texture is opened with D3D11Device(CanvasDevice).

To avoid readback to CPU, DXGITextureHostD3D11::GetAsSurfaceWithDevice() is used to get DataSourceSurface with aDevice without readback to CPU.

Differential Revision: https://phabricator.services.mozilla.com/D206446

dom/canvas/CanvasRenderingContext2D.cpp

@@ -5429,12 +5429,12 @@ MaybeGetSurfaceDescriptorForRemoteCanvas(
     return Nothing();
   }
 
-  const auto& sdv = sd.ref().get_SurfaceDescriptorGPUVideo();
+  auto& sdv = sd.ref().get_SurfaceDescriptorGPUVideo();
   const auto& sdvType = sdv.type();
   if (sdvType ==
       layers::SurfaceDescriptorGPUVideo::TSurfaceDescriptorRemoteDecoder) {
-    const auto& sdrd = sdv.get_SurfaceDescriptorRemoteDecoder();
-    const auto& subdesc = sdrd.subdesc();
+    auto& sdrd = sdv.get_SurfaceDescriptorRemoteDecoder();
+    auto& subdesc = sdrd.subdesc();
     const auto& subdescType = subdesc.type();
     if (subdescType == layers::RemoteDecoderVideoSubDescriptor::Tnull_t) {
       return sd;
@@ -5443,6 +5443,17 @@ MaybeGetSurfaceDescriptorForRemoteCanvas(
                            TSurfaceDescriptorMacIOSurface) {
       return sd;
     }
+    if (subdescType ==
+        layers::RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorD3D10) {
+      auto& descD3D10 = subdesc.get_SurfaceDescriptorD3D10();
+      // Clear FileHandleWrapper, since FileHandleWrapper::mHandle could not be
+      // cross process delivered by using Shmem. Cross-process delivery of
+      // FileHandleWrapper::mHandle is not possible simply by using shmen. When
+      // it is tried, parent side process just causes crash during destroying
+      // FileHandleWrapper.
+      descD3D10.handle() = nullptr;
+      return sd;
+    }
   }
 
   return Nothing();

dom/media/platforms/wmf/DXVA2Manager.cpp

@@ -1032,8 +1032,11 @@ D3D11DXVA2Manager::CopyToBGRATexture(ID3D11Texture2D* aInTexture,
     inTexture = newTexture;
   }
 
-  desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
-  desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
+  desc = CD3D11_TEXTURE2D_DESC(
+      DXGI_FORMAT_B8G8R8A8_UNORM, desc.Width, desc.Height, 1, 1,
+      D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE);
+  desc.MiscFlags =
+      D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED;
 
   hr = mDevice->CreateTexture2D(&desc, nullptr, getter_AddRefs(texture));
   NS_ENSURE_TRUE(SUCCEEDED(hr) && texture, E_FAIL);

gfx/layers/composite/TextureHost.h

@@ -62,6 +62,7 @@ class Compositor;
 class CompositableParentManager;
 class ReadLockDescriptor;
 class CompositorBridgeParent;
+class DXGITextureHostD3D11;
 class SurfaceDescriptor;
 class HostIPCAllocator;
 class ISurfaceAllocator;
@@ -623,6 +624,8 @@ class TextureHost : public AtomicRefCountedWithFinalize<TextureHost> {
     return nullptr;
   }
 
+  virtual DXGITextureHostD3D11* AsDXGITextureHostD3D11() { return nullptr; }
+
   virtual bool IsWrappingSurfaceTextureHost() { return false; }
 
   // Create the corresponding RenderTextureHost type of this texture, and

gfx/layers/d3d11/TextureD3D11.cpp

@@ -7,6 +7,7 @@
 #include "TextureD3D11.h"
 
 #include "CompositorD3D11.h"
+#include "DXVA2Manager.h"
 #include "Effects.h"
 #include "MainThreadUtils.h"
 #include "gfx2DGlue.h"
@@ -865,11 +866,7 @@ static RefPtr<ID3D11Texture2D> OpenSharedD3D11Texture(
   if (gpuProcessTextureId.isSome()) {
     auto* textureMap = layers::GpuProcessD3D11TextureMap::Get();
     if (textureMap) {
-      Maybe<HANDLE> handle =
-          textureMap->GetSharedHandleOfCopiedTexture(gpuProcessTextureId.ref());
-      if (handle.isSome()) {
-        texture = OpenSharedD3D11Texture(aDevice, handle.ref());
-      }
+      texture = textureMap->GetTexture(gpuProcessTextureId.ref());
     }
   } else if (handle) {
     texture = OpenSharedD3D11Texture(aDevice, handle->GetHandle());
@@ -901,7 +898,72 @@ already_AddRefed<gfx::DataSourceSurface> DXGITextureHostD3D11::GetAsSurface(
     gfx::DataSourceSurface* aSurface) {
   RefPtr<ID3D11Device> d3d11Device =
       DeviceManagerDx::Get()->GetCompositorDevice();
-  return GetAsSurfaceWithDevice(d3d11Device);
+  if (!d3d11Device) {
+    return nullptr;
+  }
+
+  RefPtr<ID3D11Texture2D> d3dTexture =
+      OpenSharedD3D11Texture(this, d3d11Device);
+  if (!d3dTexture) {
+    return nullptr;
+  }
+
+  bool isLocked = LockD3DTexture(d3dTexture.get());
+  if (!isLocked) {
+    return nullptr;
+  }
+
+  const auto onExit =
+      mozilla::MakeScopeExit([&]() { UnlockD3DTexture(d3dTexture.get()); });
+
+  bool isRGB = [&]() {
+    switch (mFormat) {
+      case gfx::SurfaceFormat::R8G8B8X8:
+      case gfx::SurfaceFormat::R8G8B8A8:
+      case gfx::SurfaceFormat::B8G8R8A8:
+      case gfx::SurfaceFormat::B8G8R8X8:
+        return true;
+      default:
+        break;
+    }
+    return false;
+  }();
+
+  if (!isRGB) {
+    return nullptr;
+  }
+
+  D3D11_TEXTURE2D_DESC textureDesc = {0};
+  d3dTexture->GetDesc(&textureDesc);
+
+  RefPtr<ID3D11DeviceContext> context;
+  d3d11Device->GetImmediateContext(getter_AddRefs(context));
+
+  textureDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
+  textureDesc.Usage = D3D11_USAGE_STAGING;
+  textureDesc.BindFlags = 0;
+  textureDesc.MiscFlags = 0;
+  textureDesc.MipLevels = 1;
+  RefPtr<ID3D11Texture2D> cpuTexture;
+  HRESULT hr = d3d11Device->CreateTexture2D(&textureDesc, nullptr,
+                                            getter_AddRefs(cpuTexture));
+  if (FAILED(hr)) {
+    return nullptr;
+  }
+
+  context->CopyResource(cpuTexture, d3dTexture);
+
+  D3D11_MAPPED_SUBRESOURCE mappedSubresource;
+  hr = context->Map(cpuTexture, 0, D3D11_MAP_READ, 0, &mappedSubresource);
+  if (FAILED(hr)) {
+    return nullptr;
+  }
+
+  RefPtr<DataSourceSurface> surf = gfx::CreateDataSourceSurfaceFromData(
+      IntSize(textureDesc.Width, textureDesc.Height), GetFormat(),
+      (uint8_t*)mappedSubresource.pData, mappedSubresource.RowPitch);
+  context->Unmap(cpuTexture, 0);
+  return surf.forget();
 }
 
 already_AddRefed<gfx::DataSourceSurface>
@@ -936,41 +998,94 @@ DXGITextureHostD3D11::GetAsSurfaceWithDevice(ID3D11Device* const aDevice) {
     return false;
   }();
 
-  if (!isRGB) {
+  if (isRGB) {
+    RefPtr<gfx::DrawTarget> dt =
+        gfx::Factory::CreateDrawTargetForD3D11Texture(d3dTexture, mFormat);
+    if (!dt) {
+      return nullptr;
+    }
+    RefPtr<gfx::SourceSurface> surface = dt->Snapshot();
+    if (!surface) {
+      return nullptr;
+    }
+    RefPtr<DataSourceSurface> dataSurface = surface->GetDataSurface();
+    if (!dataSurface) {
+      return nullptr;
+    }
+    return dataSurface.forget();
+  }
+
+  if (mFormat != gfx::SurfaceFormat::NV12 &&
+      mFormat != gfx::SurfaceFormat::P010 &&
+      mFormat != gfx::SurfaceFormat::P016) {
+    MOZ_ASSERT_UNREACHABLE("unexpected to be called");
     return nullptr;
   }
 
-  D3D11_TEXTURE2D_DESC textureDesc = {0};
-  d3dTexture->GetDesc(&textureDesc);
+  RefPtr<ID3D11Device> device;
+  d3dTexture->GetDevice(getter_AddRefs(device));
+  if (!device) {
+    gfxCriticalNoteOnce << "Failed to get D3D11 device from source texture";
+    return nullptr;
+  }
 
-  RefPtr<ID3D11DeviceContext> context;
-  aDevice->GetImmediateContext(getter_AddRefs(context));
+  nsAutoCString error;
+  std::unique_ptr<DXVA2Manager> manager(
+      DXVA2Manager::CreateD3D11DXVA(nullptr, error, device));
+  if (!manager) {
+    gfxCriticalNoteOnce << "Failed to create DXVA2 manager!";
+    return nullptr;
+  }
 
-  textureDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
-  textureDesc.Usage = D3D11_USAGE_STAGING;
-  textureDesc.BindFlags = 0;
-  textureDesc.MiscFlags = 0;
-  textureDesc.MipLevels = 1;
-  RefPtr<ID3D11Texture2D> cpuTexture;
-  HRESULT hr = aDevice->CreateTexture2D(&textureDesc, nullptr,
-                                        getter_AddRefs(cpuTexture));
+  RefPtr<ID3D11Texture2D> copiedTexture;
+  HRESULT hr = manager->CopyToBGRATexture(d3dTexture, mArrayIndex,
+                                          getter_AddRefs(copiedTexture));
   if (FAILED(hr)) {
+    gfxCriticalNoteOnce << "Failed to copy to BGRA texture: " << gfx::hexa(hr);
     return nullptr;
   }
 
-  context->CopyResource(cpuTexture, d3dTexture);
+  RefPtr<IDXGIResource1> resource;
+  copiedTexture->QueryInterface((IDXGIResource1**)getter_AddRefs(resource));
+  if (!resource) {
+    gfxCriticalNoteOnce << "Failed to get IDXGIResource";
+    return nullptr;
+  }
 
-  D3D11_MAPPED_SUBRESOURCE mappedSubresource;
-  hr = context->Map(cpuTexture, 0, D3D11_MAP_READ, 0, &mappedSubresource);
+  HANDLE sharedHandle;
+  hr = resource->CreateSharedHandle(
+      nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, nullptr,
+      &sharedHandle);
   if (FAILED(hr)) {
+    gfxCriticalNoteOnce << "GetSharedHandle failed: " << gfx::hexa(hr);
     return nullptr;
   }
 
-  RefPtr<DataSourceSurface> surf = gfx::CreateDataSourceSurfaceFromData(
-      IntSize(textureDesc.Width, textureDesc.Height), GetFormat(),
-      (uint8_t*)mappedSubresource.pData, mappedSubresource.RowPitch);
-  context->Unmap(cpuTexture, 0);
-  return surf.forget();
+  RefPtr<gfx::FileHandleWrapper> handle =
+      new gfx::FileHandleWrapper(UniqueFileHandle(sharedHandle));
+
+  d3dTexture = OpenSharedD3D11Texture(aDevice, handle->GetHandle());
+  if (!d3dTexture) {
+    gfxCriticalNoteOnce << "Failed to open copied texture handle";
+    return nullptr;
+  }
+
+  RefPtr<gfx::DrawTarget> dt = gfx::Factory::CreateDrawTargetForD3D11Texture(
+      d3dTexture, gfx::SurfaceFormat::B8G8R8A8);
+  if (!dt) {
+    gfxCriticalNote << "Failed to create DrawTarget (D3D11)";
+    return nullptr;
+  }
+  RefPtr<gfx::SourceSurface> surface = dt->Snapshot();
+  if (!surface) {
+    return nullptr;
+  }
+  RefPtr<DataSourceSurface> dataSurface = surface->GetDataSurface();
+  if (!dataSurface) {
+    return nullptr;
+  }
+
+  return dataSurface.forget();
 }
 
 void DXGITextureHostD3D11::CreateRenderTexture(

gfx/layers/d3d11/TextureD3D11.h

@@ -358,6 +358,7 @@ class DXGITextureHostD3D11 : public TextureHost {
   already_AddRefed<gfx::DataSourceSurface> GetAsSurface(
       gfx::DataSourceSurface* aSurface) override;
 
+  // Return DataSourceSurface using aDevice withou readback to CPU.
   already_AddRefed<gfx::DataSourceSurface> GetAsSurfaceWithDevice(
       ID3D11Device* const aDevice);
 
@@ -379,6 +380,8 @@ class DXGITextureHostD3D11 : public TextureHost {
 
   bool SupportsExternalCompositing(WebRenderBackend aBackend) override;
 
+  DXGITextureHostD3D11* AsDXGITextureHostD3D11() override { return this; }
+
   const RefPtr<gfx::FileHandleWrapper> mHandle;
   const Maybe<GpuProcessTextureId> mGpuProcessTextureId;
   const uint32_t mArrayIndex;

gfx/layers/ipc/CanvasTranslator.cpp

@@ -1378,7 +1378,8 @@ static bool SDIsSupportedRemoteDecoder(const SurfaceDescriptor& sd) {
 
   if (subdescType == RemoteDecoderVideoSubDescriptor::Tnull_t ||
       subdescType ==
-          RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorMacIOSurface) {
+          RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorMacIOSurface ||
+      subdescType == RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorD3D10) {
     return true;
   }
 
@@ -1468,6 +1469,37 @@ CanvasTranslator::LookupSourceSurfaceFromSurfaceDescriptor(
     return nullptr;
   }
 
+#if defined(XP_WIN)
+  if (subdescType == RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorD3D10) {
+    auto* textureHostD3D11 = texture->AsDXGITextureHostD3D11();
+    if (!textureHostD3D11) {
+      MOZ_ASSERT_UNREACHABLE("unexpected to be called");
+      return nullptr;
+    }
+    auto& usedSurf = mUsedDataSurfaceForSurfaceDescriptor;
+    auto& usedDescriptor = mUsedSurfaceDescriptorForSurfaceDescriptor;
+
+    if (usedDescriptor.isSome() && usedDescriptor.ref() == sdrd) {
+      MOZ_ASSERT(usedSurf);
+      MOZ_ASSERT(texture->GetSize() == usedSurf->GetSize());
+
+      // Since the data is the same as before, the DataSourceSurfaceWrapper can
+      // be reused.
+      return do_AddRef(usedSurf);
+    }
+
+    usedSurf = textureHostD3D11->GetAsSurfaceWithDevice(mDevice);
+    if (!usedSurf) {
+      MOZ_ASSERT_UNREACHABLE("unexpected to be called");
+      usedDescriptor = Nothing();
+      return nullptr;
+    }
+    usedDescriptor = Some(sdrd);
+
+    return do_AddRef(usedSurf);
+  }
+#endif
+
   if (subdescType ==
       RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorMacIOSurface) {
     MOZ_ASSERT(texture->AsMacIOSurfaceTextureHost());