Bug 1891063 - Improve invalidation of top-level transparent windows. r=win-reviewers,rkraesig,handyman

This makes sure that the transparent area is cleared properly.

We move the clear to a point where Gecko has already informed us of
the opaque region for simplicity.

Differential Revision: https://phabricator.services.mozilla.com/D207302

widget/windows/WinUtils.cpp

@@ -1189,6 +1189,15 @@ LayoutDeviceIntRect WinUtils::ToIntRect(const RECT& aRect) {
                              aRect.bottom - aRect.top);
 }
 
+RECT WinUtils::ToWinRect(const LayoutDeviceIntRect& aRect) {
+  return {
+      .left = aRect.x,
+      .top = aRect.y,
+      .right = aRect.XMost(),
+      .bottom = aRect.YMost(),
+  };
+}
+
 /* static */
 bool WinUtils::IsIMEEnabled(const InputContext& aInputContext) {
   return IsIMEEnabled(aInputContext.mIMEState.mEnabled);

widget/windows/WinUtils.h

@@ -424,6 +424,10 @@ class WinUtils {
    * returns the LayoutDeviceIntRect.
    */
   static LayoutDeviceIntRect ToIntRect(const RECT& aRect);
+  /**
+   * Performs the inverse conversion of ToIntRect
+   */
+  static RECT ToWinRect(const LayoutDeviceIntRect& aRect);
 
   /**
    * Returns true if the context or IME state is enabled.  Otherwise, false.

widget/windows/nsWindow.cpp

@@ -2639,8 +2639,7 @@ bool nsWindow::UpdateNonClientMargins(bool aReflowWindow) {
     mNonClientOffset.left = 0;
     mNonClientOffset.right = 0;
 
-    mozilla::Maybe<UINT> maybeEdge = GetHiddenTaskbarEdge();
-    if (maybeEdge) {
+    if (mozilla::Maybe<UINT> maybeEdge = GetHiddenTaskbarEdge()) {
       auto edge = maybeEdge.value();
       if (ABE_LEFT == edge) {
         mNonClientOffset.left -= kHiddenTaskbarSize;
@@ -2649,19 +2648,13 @@ bool nsWindow::UpdateNonClientMargins(bool aReflowWindow) {
       } else if (ABE_BOTTOM == edge || ABE_TOP == edge) {
         mNonClientOffset.bottom -= kHiddenTaskbarSize;
       }
-
-      // When we are drawing the non-client region, we need
-      // to clear the portion of the NC region that is exposed by the
-      // hidden taskbar.  As above, we clear the bottom of the NC region
-      // when the taskbar is at the top of the screen.
-      UINT clearEdge = (edge == ABE_TOP) ? ABE_BOTTOM : edge;
-      mClearNCEdge = Some(clearEdge);
     }
   } else {
     mNonClientOffset = NormalWindowNonClientOffset();
   }
 
   UpdateOpaqueRegionInternal();
+  mNeedsNCAreaClear = true;
 
   if (aReflowWindow) {
     // Force a reflow of content based on the new client
@@ -2717,7 +2710,7 @@ void nsWindow::SetResizeMargin(mozilla::LayoutDeviceIntCoord aResizeMargin) {
   UpdateNonClientMargins();
 }
 
-void nsWindow::InvalidateNonClientRegion() {
+HRGN nsWindow::ComputeNonClientHRGN() {
   // +-+-----------------------+-+
   // | | app non-client chrome | |
   // | +-----------------------+ |
@@ -2747,7 +2740,11 @@ void nsWindow::InvalidateNonClientRegion() {
   HRGN clientRgn = CreateRectRgnIndirect(&rect);
   CombineRgn(winRgn, winRgn, clientRgn, RGN_DIFF);
   DeleteObject(clientRgn);
+  return winRgn;
+}
 
+void nsWindow::InvalidateNonClientRegion() {
+  HRGN winRgn = ComputeNonClientHRGN();
   // triggers ncpaint and paint events for the two areas
   RedrawWindow(mWnd, nullptr, winRgn, RDW_FRAME | RDW_INVALIDATE);
   DeleteObject(winRgn);
@@ -5044,12 +5041,7 @@ bool nsWindow::ProcessMessageInternal(UINT msg, WPARAM& wParam, LPARAM& lParam,
       auto GeckoClientToWinScreenRect =
           [&origin](LayoutDeviceIntRect aRect) -> RECT {
         aRect.MoveBy(origin);
-        return {
-            .left = aRect.x,
-            .top = aRect.y,
-            .right = aRect.XMost(),
-            .bottom = aRect.YMost(),
-        };
+        return WinUtils::ToWinRect(aRect);
       };
       auto SetButton = [&](size_t aIndex, WindowButtonType aType) {
         info->rgrect[aIndex] =

widget/windows/nsWindow.h

@@ -519,6 +519,8 @@ class nsWindow final : public nsBaseWidget {
   bool UpdateNonClientMargins(bool aReflowWindow = true);
   void UpdateDarkModeToolbar();
   void ResetLayout();
+  // Returns an HRGN object which needs to be released with ::DeleteObject().
+  HRGN ComputeNonClientHRGN();
   void InvalidateNonClientRegion();
   static const wchar_t* GetMainWindowClass();
   HWND GetOwnerWnd() const { return ::GetWindow(mWnd, GW_OWNER); }
@@ -635,8 +637,7 @@ class nsWindow final : public nsBaseWidget {
   void StopFlashing();
   static HWND WindowAtMouse();
   static bool IsTopLevelMouseExit(HWND aWnd);
-  LayoutDeviceIntRegion GetRegionToPaint(bool aForceFullRepaint, PAINTSTRUCT ps,
-                                         HDC aDC);
+  LayoutDeviceIntRegion GetRegionToPaint(const PAINTSTRUCT& ps, HDC aDC) const;
   nsIWidgetListener* GetPaintListener();
 
   void CreateCompositor() override;
@@ -899,9 +900,9 @@ class nsWindow final : public nsBaseWidget {
 
   mozilla::DataMutex<Desktop> mDesktopId;
 
-  // If set, indicates the edge of the NC region we should clear to black
-  // on next paint.  One of: ABE_TOP, ABE_BOTTOM, ABE_LEFT or ABE_RIGHT.
-  mozilla::Maybe<UINT> mClearNCEdge;
+  // If set, indicates the non-client-area region must be cleared to black on
+  // next paint.
+  bool mNeedsNCAreaClear = false;
 
   friend class nsWindowGfx;
 

widget/windows/nsWindowGfx.cpp

@@ -99,27 +99,20 @@ static IconMetrics sIconMetrics[] = {
  **************************************************************/
 
 // GetRegionToPaint returns the invalidated region that needs to be painted
-LayoutDeviceIntRegion nsWindow::GetRegionToPaint(bool aForceFullRepaint,
-                                                 PAINTSTRUCT ps, HDC aDC) {
-  if (aForceFullRepaint) {
-    RECT paintRect;
-    ::GetClientRect(mWnd, &paintRect);
-    return LayoutDeviceIntRegion(WinUtils::ToIntRect(paintRect));
-  }
-
+LayoutDeviceIntRegion nsWindow::GetRegionToPaint(const PAINTSTRUCT& ps,
+                                                 HDC aDC) const {
+  LayoutDeviceIntRegion fullRegion(WinUtils::ToIntRect(ps.rcPaint));
   HRGN paintRgn = ::CreateRectRgn(0, 0, 0, 0);
-  if (paintRgn != nullptr) {
-    int result = GetRandomRgn(aDC, paintRgn, SYSRGN);
-    if (result == 1) {
+  if (paintRgn) {
+    if (GetRandomRgn(aDC, paintRgn, SYSRGN) == 1) {
       POINT pt = {0, 0};
       ::MapWindowPoints(nullptr, mWnd, &pt, 1);
       ::OffsetRgn(paintRgn, pt.x, pt.y);
+      fullRegion.AndWith(WinUtils::ConvertHRGNToRegion(paintRgn));
     }
-    LayoutDeviceIntRegion rgn(WinUtils::ConvertHRGNToRegion(paintRgn));
     ::DeleteObject(paintRgn);
-    return rgn;
   }
-  return LayoutDeviceIntRegion(WinUtils::ToIntRect(ps.rcPaint));
+  return fullRegion;
 }
 
 nsIWidgetListener* nsWindow::GetPaintListener() {
@@ -175,39 +168,9 @@ bool nsWindow::OnPaint(uint32_t aNestingLevel) {
   KnowsCompositor* knowsCompositor = renderer->AsKnowsCompositor();
   WebRenderLayerManager* layerManager = renderer->AsWebRender();
 
-  if (mClearNCEdge) {
-    // We need to clear this edge of the non-client region to black (once).
-    HDC hdc;
-    RECT rect;
-    hdc = ::GetWindowDC(mWnd);
-    ::GetWindowRect(mWnd, &rect);
-    ::MapWindowPoints(nullptr, mWnd, (LPPOINT)&rect, 2);
-    switch (mClearNCEdge.value()) {
-      case ABE_TOP:
-        rect.bottom = rect.top + kHiddenTaskbarSize;
-        break;
-      case ABE_LEFT:
-        rect.right = rect.left + kHiddenTaskbarSize;
-        break;
-      case ABE_BOTTOM:
-        rect.top = rect.bottom - kHiddenTaskbarSize;
-        break;
-      case ABE_RIGHT:
-        rect.left = rect.right - kHiddenTaskbarSize;
-        break;
-      default:
-        MOZ_ASSERT_UNREACHABLE("Invalid edge value");
-        break;
-    }
-    ::FillRect(hdc, &rect,
-               reinterpret_cast<HBRUSH>(::GetStockObject(BLACK_BRUSH)));
-    ::ReleaseDC(mWnd, hdc);
+  const bool didResize = !mBounds.IsEqualEdges(mLastPaintBounds);
 
-    mClearNCEdge.reset();
-  }
-
-  if (knowsCompositor && layerManager &&
-      !mBounds.IsEqualEdges(mLastPaintBounds)) {
+  if (didResize && knowsCompositor && layerManager) {
     // Do an early async composite so that we at least have something on the
     // screen in the right place, even if the content is out of date.
     layerManager->ScheduleComposite(wr::RenderReasons::WIDGET);
@@ -237,37 +200,51 @@ bool nsWindow::OnPaint(uint32_t aNestingLevel) {
     hDC = ::BeginPaint(mWnd, &ps);
   }
 
-  const bool forceRepaint = mTransparencyMode == TransparencyMode::Transparent;
-  const LayoutDeviceIntRegion region = GetRegionToPaint(forceRepaint, ps, hDC);
-
-  if (knowsCompositor && layerManager) {
-    // We need to paint to the screen even if nothing changed, since if we
-    // don't have a compositing window manager, our pixels could be stale.
-    layerManager->SetNeedsComposite(true);
-    layerManager->SendInvalidRegion(region.ToUnknownRegion());
-  }
-
+  const LayoutDeviceIntRegion region = GetRegionToPaint(ps, hDC);
   RefPtr<nsWindow> strongThis(this);
-
-  nsIWidgetListener* listener = GetPaintListener();
-  if (listener) {
+  if (nsIWidgetListener* listener = GetPaintListener()) {
     listener->WillPaintWindow(this);
   }
+
+  if (mNeedsNCAreaClear || didResize) {
+    // We need to clear the non-client-area region, and the transparent parts
+    // of the window to black (once). WillPaintWindow updates the opaque region.
+    HDC hdc = ::GetWindowDC(mWnd);
+    auto black = reinterpret_cast<HBRUSH>(::GetStockObject(BLACK_BRUSH));
+    {
+      HRGN ncRegion = ComputeNonClientHRGN();
+      ::FillRgn(hdc, ncRegion, black);
+      ::DeleteObject(ncRegion);
+    }
+    if (mTransparencyMode == TransparencyMode::Transparent) {
+      RECT winRect;
+      GetWindowRect(mWnd, &winRect);
+      MapWindowPoints(nullptr, mWnd, (LPPOINT)&winRect, 2);
+      LayoutDeviceIntRegion translucent(WinUtils::ToIntRect(winRect));
+      translucent.SubOut(mOpaqueRegion);
+      for (auto iter = translucent.RectIter(); !iter.Done(); iter.Next()) {
+        RECT rect = WinUtils::ToWinRect(iter.Get());
+        ::FillRect(hdc, &rect, black);
+      }
+    }
+    ::ReleaseDC(mWnd, hdc);
+    mNeedsNCAreaClear = false;
+  }
+
   // Re-get the listener since the will paint notification may have killed it.
-  listener = GetPaintListener();
+  nsIWidgetListener* listener = GetPaintListener();
   if (!listener) {
     return false;
   }
 
-  if (knowsCompositor && layerManager && layerManager->NeedsComposite()) {
-    layerManager->ScheduleComposite(wr::RenderReasons::WIDGET);
-    layerManager->SetNeedsComposite(false);
-  }
-
   bool result = true;
   if (!region.IsEmpty() && listener) {
+    if (knowsCompositor && layerManager) {
+      layerManager->SendInvalidRegion(region.ToUnknownRegion());
+      layerManager->ScheduleComposite(wr::RenderReasons::WIDGET);
+    }
     // Should probably pass in a real region here, using GetRandomRgn
-    // http://msdn.microsoft.com/library/default.asp?url=/library/en-us/gdi/clipping_4q0e.asp
+    // https://learn.microsoft.com/en-us/windows/win32/api/wingdi/nf-wingdi-getrandomrgn
 
 #ifdef WIDGET_DEBUG_OUTPUT
     debug_DumpPaintEvent(stdout, this, region.ToUnknownRegion(), "noname",