Bug 1633718 - Make sure not to call OpenThemeData if it failed before. r=mstange

As bug 1544435, we blocked wbload.dll from being loaded in content processes.
Because that modules is injected via `user32!RegisterUserApiHook` and Windows
keeps trying to load hooking modules even if the previous attempt has failed,
blocking wbload.dll caused repetitive loading attempts in the tab process,
resulting in bad rendering performance.

This patch is to mitigate that performance issue by not calling `OpenThemeData`,
which is one of the entrypoints triggering user api hook, if the previous call
has failed.  With this patch, performance is still slower than with the default
theme.  We will seek out a long-term solution to solve the issue.

Differential Revision: https://phabricator.services.mozilla.com/D73344

widget/windows/nsUXThemeData.cpp

@@ -16,8 +16,7 @@
 using namespace mozilla;
 using namespace mozilla::widget;
 
-HANDLE
+nsUXThemeData::ThemeHandle nsUXThemeData::sThemes[eUXNumClasses];
-nsUXThemeData::sThemes[eUXNumClasses];
 
 const int NUM_COMMAND_BUTTONS = 3;
 SIZE nsUXThemeData::sCommandButtonMetrics[NUM_COMMAND_BUTTONS];
@@ -30,21 +29,39 @@ bool nsUXThemeData::sFlatMenus = false;
 bool nsUXThemeData::sTitlebarInfoPopulatedAero = false;
 bool nsUXThemeData::sTitlebarInfoPopulatedThemed = false;
 
+nsUXThemeData::ThemeHandle::~ThemeHandle() { Close(); }
+
+void nsUXThemeData::ThemeHandle::OpenOnce(HWND aWindow, LPCWSTR aClassList) {
+  if (mHandle.isSome()) {
+    return;
+  }
+
+  mHandle = Some(OpenThemeData(aWindow, aClassList));
+}
+
+void nsUXThemeData::ThemeHandle::Close() {
+  if (mHandle.isNothing() || !mHandle.value()) {
+    return;
+  }
+
+  CloseThemeData(mHandle.value());
+  mHandle = Nothing();
+}
+
+nsUXThemeData::ThemeHandle::operator HANDLE() {
+  return mHandle.isSome() ? mHandle.value() : nullptr;
+}
+
 void nsUXThemeData::Teardown() { Invalidate(); }
 
 void nsUXThemeData::Initialize() {
-  ::ZeroMemory(sThemes, sizeof(sThemes));
-
   CheckForCompositor(true);
   Invalidate();
 }
 
 void nsUXThemeData::Invalidate() {
-  for (int i = 0; i < eUXNumClasses; i++) {
+  for (auto& theme : sThemes) {
-    if (sThemes[i]) {
+    theme.Close();
-      CloseThemeData(sThemes[i]);
-      sThemes[i] = nullptr;
-    }
   }
   BOOL useFlat = FALSE;
   sFlatMenus =
@@ -54,9 +71,7 @@ void nsUXThemeData::Invalidate() {
 HANDLE
 nsUXThemeData::GetTheme(nsUXThemeClass cls) {
   NS_ASSERTION(cls < eUXNumClasses, "Invalid theme class!");
-  if (!sThemes[cls]) {
+  sThemes[cls].OpenOnce(nullptr, GetClassName(cls));
-    sThemes[cls] = OpenThemeData(nullptr, GetClassName(cls));
-  }
   return sThemes[cls];
 }
 

widget/windows/nsUXThemeData.h

@@ -79,7 +79,28 @@ enum WindowsThemeColor {
 #define CMDBUTTONIDX_BUTTONBOX 3
 
 class nsUXThemeData {
-  static HANDLE sThemes[eUXNumClasses];
+  // This class makes sure we don't attempt to open a theme if the previous
+  // loading attempt has failed because OpenThemeData is a heavy task and
+  // it's less likely that the API returns a different result.
+  class ThemeHandle final {
+    Maybe<HANDLE> mHandle;
+
+   public:
+    ThemeHandle() = default;
+    ~ThemeHandle();
+
+    // Disallow copy and move
+    ThemeHandle(const ThemeHandle&) = delete;
+    ThemeHandle(ThemeHandle&&) = delete;
+    ThemeHandle& operator=(const ThemeHandle&) = delete;
+    ThemeHandle& operator=(ThemeHandle&&) = delete;
+
+    operator HANDLE();
+    void OpenOnce(HWND aWindow, LPCWSTR aClassList);
+    void Close();
+  };
+
+  static ThemeHandle sThemes[eUXNumClasses];
 
   // We initialize sCommandButtonBoxMetrics separately as a performance
   // optimization to avoid fetching dummy values for sCommandButtonMetrics