Bug 1616000 - Fix broken push notifications r=handyman

Bug 1604412 enabled "remote backbuffer" compositing when using non-accelerated
layers. However, due to my misunderstanding of how nWindow and compositing
handles layered windows, I broke the usecase where a child window is
transparent.

In this case, it is the top-level anscestor window that has the
WS_EX_LAYERED attribute and not the child window. The ancestor is the one
that needs to be repainted when a present is requested.

Differential Revision: https://phabricator.services.mozilla.com/D63377

--HG--
extra : moz-landing-system : lando

widget/windows/CompositorWidgetChild.cpp

@@ -36,7 +36,8 @@ CompositorWidgetChild::~CompositorWidgetChild() {}
 
 bool CompositorWidgetChild::Initialize() {
   mRemoteBackbufferProvider = std::make_unique<remote_backbuffer::Provider>();
-  if (!mRemoteBackbufferProvider->Initialize(mWnd, OtherPid())) {
+  if (!mRemoteBackbufferProvider->Initialize(mWnd, OtherPid(),
+                                             mTransparencyMode)) {
     return false;
   }
 
@@ -67,6 +68,8 @@ bool CompositorWidgetChild::OnWindowResize(const LayoutDeviceIntSize& aSize) {
 void CompositorWidgetChild::OnWindowModeChange(nsSizeMode aSizeMode) {}
 
 void CompositorWidgetChild::UpdateTransparency(nsTransparencyMode aMode) {
+  mTransparencyMode = aMode;
+  mRemoteBackbufferProvider->UpdateTransparencyMode(aMode);
   Unused << SendUpdateTransparency(aMode);
 }
 

widget/windows/RemoteBackbuffer.cpp

@@ -206,16 +206,20 @@ class PresentableSharedImage {
     return true;
   }
 
-  bool PresentToWindow(HWND aWindowHandle) {
-    bool isLayered =
-        ::GetWindowLongPtrW(aWindowHandle, GWL_EXSTYLE) & WS_EX_LAYERED;
+  bool PresentToWindow(HWND aWindowHandle,
+                       nsTransparencyMode aTransparencyMode) {
+    if (aTransparencyMode == eTransparencyTransparent) {
+      // If our window is a child window or a child-of-a-child, the window
+      // that needs to be updated is the top level ancestor of the tree
+      HWND topLevelWindow = WinUtils::GetTopLevelHWND(aWindowHandle, true);
+      MOZ_ASSERT(::GetWindowLongPtr(topLevelWindow, GWL_EXSTYLE) &
+                 WS_EX_LAYERED);
 
-    if (isLayered) {
       BLENDFUNCTION bf = {AC_SRC_OVER, 0, 255, AC_SRC_ALPHA};
       SIZE winSize = {mSharedImage.GetWidth(), mSharedImage.GetHeight()};
       POINT srcPos = {0, 0};
       return !!::UpdateLayeredWindow(
-          aWindowHandle, nullptr /*paletteDC*/, nullptr /*newPos*/, &winSize,
+          topLevelWindow, nullptr /*paletteDC*/, nullptr /*newPos*/, &winSize,
           mDeviceContext, &srcPos, 0 /*colorKey*/, &bf, ULW_ALPHA);
     }
 
@@ -294,7 +298,8 @@ Provider::~Provider() {
   }
 }
 
-bool Provider::Initialize(HWND aWindowHandle, DWORD aTargetProcessId) {
+bool Provider::Initialize(HWND aWindowHandle, DWORD aTargetProcessId,
+                          nsTransparencyMode aTransparencyMode) {
   MOZ_ASSERT(aWindowHandle);
   MOZ_ASSERT(aTargetProcessId);
 
@@ -335,6 +340,8 @@ bool Provider::Initialize(HWND aWindowHandle, DWORD aTargetProcessId) {
 
   mServiceThread = std::thread([this] { this->ThreadMain(); });
 
+  mTransparencyMode = aTransparencyMode;
+
   return true;
 }
 
@@ -365,6 +372,10 @@ Maybe<RemoteBackbufferHandles> Provider::CreateRemoteHandles() {
       reinterpret_cast<WindowsHandle>(responseReadyEvent)));
 }
 
+void Provider::UpdateTransparencyMode(nsTransparencyMode aTransparencyMode) {
+  mTransparencyMode = aTransparencyMode;
+}
+
 void Provider::ThreadMain() {
   while (true) {
     MOZ_ALWAYS_TRUE(::WaitForSingleObject(mRequestReadyEvent, INFINITE) ==
@@ -457,7 +468,7 @@ void Provider::HandlePresentRequest(PresentResponseData* aResponseData) {
     return;
   }
 
-  if (!mBackbuffer->PresentToWindow(mWindowHandle)) {
+  if (!mBackbuffer->PresentToWindow(mWindowHandle, mTransparencyMode)) {
     return;
   }
 

widget/windows/RemoteBackbuffer.h

@@ -25,10 +25,13 @@ class Provider {
   Provider();
   ~Provider();
 
-  bool Initialize(HWND aWindowHandle, DWORD aTargetProcessId);
+  bool Initialize(HWND aWindowHandle, DWORD aTargetProcessId,
+                  nsTransparencyMode aTransparencyMode);
 
   Maybe<RemoteBackbufferHandles> CreateRemoteHandles();
 
+  void UpdateTransparencyMode(nsTransparencyMode aTransparencyMode);
+
   Provider(const Provider&) = delete;
   Provider(Provider&&) = delete;
   Provider& operator=(const Provider&) = delete;
@@ -49,6 +52,8 @@ class Provider {
   bool mStopServiceThread;
   std::thread mServiceThread;
   std::unique_ptr<PresentableSharedImage> mBackbuffer;
+  mozilla::Atomic<nsTransparencyMode, MemoryOrdering::Relaxed>
+      mTransparencyMode;
 };
 
 class Client {