Bug 1529713 - [Titlebar] Use Window manager decorations on GNOME again, r=lsalzman

This patch enables the shape mask in CSD and Window manager decorations mode
when we're runnin on composited screen and mozilla.widget.use-argb-visuals is not set.

Also don't use shape mask with Wayland and GL backend. When shape mask is set,
advertise toplevel window transparency but don't advertise it
as alpha to GtkCompositorWidget.

Differential Revision: https://phabricator.services.mozilla.com/D20726

--HG--
extra : moz-landing-system : lando

widget/gtk/nsLookAndFeel.cpp

@@ -689,9 +689,15 @@ nsresult nsLookAndFeel::GetIntImpl(IntID aID, int32_t& aResult) {
       EnsureInit();
       aResult = mCSDCloseButton;
       break;
-    case eIntID_GTKCSDTransparentBackground:
-      aResult = nsWindow::TopLevelWindowUseARGBVisual();
+    case eIntID_GTKCSDTransparentBackground: {
+      // Enable transparent titlebar corners for titlebar mode.
+      GdkScreen* screen = gdk_screen_get_default();
+      aResult = gdk_screen_is_composited(screen)
+                    ? (nsWindow::GetSystemCSDSupportLevel() !=
+                       nsWindow::CSD_SUPPORT_NONE)
+                    : false;
       break;
+    }
     case eIntID_GTKCSDReversedPlacement:
       EnsureInit();
       aResult = mCSDReversedPlacement;

widget/gtk/nsWindow.cpp

@@ -3287,28 +3287,31 @@ nsresult nsWindow::Create(nsIWidget *aParent, nsNativeWidget aNativeParent,
         // We enable titlebar rendering for toplevel windows only.
         mCSDSupportLevel = GetSystemCSDSupportLevel();
 
-        // Some Gtk+ themes use non-rectangular toplevel windows. To fully
-        // support such themes we need to make toplevel window transparent
-        // with ARGB visual.
-        // It may cause performanance issue so make it configurable
-        // and enable it by default for selected window managers.
-        needsAlphaVisual = TopLevelWindowUseARGBVisual();
-        if (needsAlphaVisual && mIsX11Display && !shouldAccelerate) {
-          // We want to draw a transparent titlebar but we can't use
-          // ARGB visual due to Bug 1516224.
-          // We use ARGB visual for mShell only and shape mask
-          // for mContainer where is all our content drawn.
-          mTransparencyBitmapForTitlebar = true;
-        }
-
-        // When mozilla.widget.use-argb-visuals is set don't use shape mask.
-        if (mTransparencyBitmapForTitlebar &&
-            Preferences::GetBool("mozilla.widget.use-argb-visuals", false)) {
-          mTransparencyBitmapForTitlebar = false;
-        }
-
-        if (mTransparencyBitmapForTitlebar) {
-          mCSDSupportLevel = CSD_SUPPORT_CLIENT;
+        // There's no point to configure transparency
+        // on non-composited screens.
+        GdkScreen *screen = gdk_screen_get_default();
+        if (gdk_screen_is_composited(screen)) {
+          // Some Gtk+ themes use non-rectangular toplevel windows. To fully
+          // support such themes we need to make toplevel window transparent
+          // with ARGB visual.
+          // It may cause performanance issue so make it configurable
+          // and enable it by default for selected window managers.
+          if (Preferences::HasUserValue("mozilla.widget.use-argb-visuals")) {
+            // argb visual is explicitly required so use it
+            needsAlphaVisual =
+                Preferences::GetBool("mozilla.widget.use-argb-visuals");
+          } else if (!mIsX11Display) {
+            // Wayland uses ARGB visual by default
+            needsAlphaVisual = true;
+          } else if (mCSDSupportLevel != CSD_SUPPORT_NONE) {
+            if (shouldAccelerate) {
+              needsAlphaVisual = true;
+            } else {
+              // We want to draw a transparent titlebar but we can't use
+              // ARGB visual due to Bug 1516224.
+              mTransparencyBitmapForTitlebar = true;
+            }
+          }
         }
       }
 
@@ -3356,7 +3359,8 @@ nsresult nsWindow::Create(nsIWidget *aParent, nsNativeWidget aNativeParent,
       // We have a toplevel window with transparency. Mark it as transparent
       // now as nsWindow::SetTransparencyMode() can't be called after
       // nsWindow is created (Bug 1344839).
-      if (mWindowType == eWindowType_toplevel && mHasAlphaVisual) {
+      if (mWindowType == eWindowType_toplevel &&
+          (mHasAlphaVisual || mTransparencyBitmapForTitlebar)) {
         mIsTransparent = true;
       }
 
@@ -6523,35 +6527,6 @@ bool nsWindow::HideTitlebarByDefault() {
   return hideTitlebar;
 }
 
-bool nsWindow::TopLevelWindowUseARGBVisual() {
-  static int useARGBVisual = -1;
-  if (useARGBVisual != -1) {
-    return useARGBVisual;
-  }
-
-  GdkScreen *screen = gdk_screen_get_default();
-  if (!gdk_screen_is_composited(screen)) {
-    useARGBVisual = false;
-  }
-
-  if (Preferences::HasUserValue("mozilla.widget.use-argb-visuals")) {
-    useARGBVisual =
-        Preferences::GetBool("mozilla.widget.use-argb-visuals", false);
-  } else {
-    const char *currentDesktop = getenv("XDG_CURRENT_DESKTOP");
-    useARGBVisual =
-        (currentDesktop && GetSystemCSDSupportLevel() != CSD_SUPPORT_NONE);
-
-    if (useARGBVisual) {
-      useARGBVisual =
-          (strstr(currentDesktop, "GNOME-Flashback:GNOME") != nullptr ||
-           strstr(currentDesktop, "GNOME") != nullptr);
-    }
-  }
-
-  return useARGBVisual;
-}
-
 int32_t nsWindow::RoundsWidgetCoordinatesTo() { return GdkScaleFactor(); }
 
 void nsWindow::GetCompositorWidgetInitData(
@@ -6565,7 +6540,8 @@ void nsWindow::GetCompositorWidgetInitData(
   *aInitData = mozilla::widget::GtkCompositorWidgetInitData(
       (mXWindow != X11None) ? mXWindow : (uintptr_t) nullptr,
       mXDisplay ? nsCString(XDisplayString(mXDisplay)) : nsCString(),
-      mIsTransparent && !mHasAlphaVisual, GetClientSize());
+      mIsTransparent && !mHasAlphaVisual && !mTransparencyBitmapForTitlebar,
+      GetClientSize());
 }
 
 #ifdef MOZ_WAYLAND

widget/gtk/nsWindow.h

@@ -386,7 +386,6 @@ class nsWindow final : public nsBaseWidget {
   static CSDSupportLevel GetSystemCSDSupportLevel();
 
   static bool HideTitlebarByDefault();
-  static bool TopLevelWindowUseARGBVisual();
   static bool GetTopLevelWindowActiveState(nsIFrame* aFrame);
 
  protected: