Bug 1632357 - Use compositor window only when it is necessary r=nical

With current gecko, there are cases that compositor window is used even when it is not necessary. For example, fallback from DirectComposite with WebRender and fallback from double buffering with Compositor. With some gpu drivers, the fallbacks with compositor window causes rendering problem.

Differential Revision: https://phabricator.services.mozilla.com/D76470

gfx/layers/d3d11/CompositorD3D11.cpp

@@ -240,6 +240,10 @@ bool CompositorD3D11::Initialize(nsCString* const out_failureReason) {
         swapChain->SetBackgroundColor(&color);
 
         mSwapChain = swapChain;
+      } else if (mWidget->AsWindows()->GetCompositorHwnd()) {
+        // Destroy compositor window.
+        mWidget->AsWindows()->DestroyCompositorWindow();
+        mHwnd = mWidget->AsWindows()->GetHwnd();
       }
     }
 

gfx/layers/d3d11/MLGDeviceD3D11.cpp

@@ -268,6 +268,10 @@ bool MLGSwapChainD3D11::Initialize(CompositorWidget* aWidget) {
       mSwapChain1->SetBackgroundColor(&color);
       mSwapChain = mSwapChain1;
       mIsDoubleBuffered = true;
+    } else if (aWidget->AsWindows()->GetCompositorHwnd()) {
+      // Destroy compositor window.
+      aWidget->AsWindows()->DestroyCompositorWindow();
+      hwnd = aWidget->AsWindows()->GetHwnd();
     }
   }
 

gfx/webrender_bindings/RenderCompositorANGLE.cpp

@@ -166,8 +166,13 @@ bool RenderCompositorANGLE::Initialize() {
     if (compositorHwnd) {
       mDCLayerTree =
           DCLayerTree::Create(gl, mEGLConfig, mDevice, compositorHwnd);
+      if (!mDCLayerTree) {
+        gfxCriticalNote << "Failed to create DCLayerTree";
+        return false;
+      }
     } else {
       gfxCriticalNote << "Compositor window was not created";
+      return false;
     }
   }
 
@@ -236,6 +241,11 @@ bool RenderCompositorANGLE::CreateSwapChain() {
   }
 
   CreateSwapChainForDCompIfPossible(dxgiFactory2);
+  if (gfx::gfxVars::UseWebRenderDCompWin() && !mSwapChain) {
+    MOZ_ASSERT(GetCompositorHwnd());
+    gfxCriticalNote << "Failed to create SwapChain for DComp";
+    return false;
+  }
 
   if (!mSwapChain && dxgiFactory2 && IsWin8OrLater()) {
     RefPtr<IDXGISwapChain1> swapChain1;
@@ -272,6 +282,10 @@ bool RenderCompositorANGLE::CreateSwapChain() {
       mSwapChain = swapChain1;
       mSwapChain1 = swapChain1;
       mUseTripleBuffering = useTripleBuffering;
+    } else if (gfx::gfxVars::UseWebRenderFlipSequentialWin()) {
+      MOZ_ASSERT(GetCompositorHwnd());
+      gfxCriticalNote << "Failed to create flip mode SwapChain";
+      return false;
     }
   }
 

widget/nsBaseWidget.cpp

@@ -1229,6 +1229,7 @@ already_AddRefed<LayerManager> nsBaseWidget::CreateCompositorSession(
       if (textureFactoryIdentifier.mParentBackend != LayersBackend::LAYERS_WR) {
         retry = true;
         DestroyCompositor();
+        // gfxVars::UseDoubleBufferingWithCompositor() is also disabled.
         gfx::GPUProcessManager::Get()->DisableWebRender(
             wr::WebRenderError::INITIALIZE);
       }