Bug 1133351: Part 1 - Make Windows IPC play nicely with COM STA marshaling; r=bsmedberg

--HG--
extra : rebase_source : ebb6b9c25565bd2721ee6537b2161511666e92ac

dom/plugins/ipc/COMMessageFilter.cpp

@@ -1,84 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "base/basictypes.h"
-
-#include "COMMessageFilter.h"
-#include "base/message_loop.h"
-#include "mozilla/plugins/PluginModuleChild.h"
-
-#include <stdio.h>
-
-namespace mozilla {
-namespace plugins {
-
-HRESULT
-COMMessageFilter::QueryInterface(REFIID riid, void** ppv)
-{
-  if (riid == IID_IUnknown || riid == IID_IMessageFilter) {
-    *ppv = static_cast<IMessageFilter*>(this);
-    AddRef();
-    return S_OK;
-  }
-  return E_NOINTERFACE;
-}
-
-DWORD COMMessageFilter::AddRef()
-{
-  ++mRefCnt;
-  return mRefCnt;
-}
-
-DWORD COMMessageFilter::Release()
-{
-  DWORD r = --mRefCnt;
-  if (0 == r)
-    delete this;
-  return r;
-}
-
-DWORD 
-COMMessageFilter::HandleInComingCall(DWORD dwCallType,
-				     HTASK htaskCaller,
-				     DWORD dwTickCount,
-				     LPINTERFACEINFO lpInterfaceInfo)
-{
-  if (mPreviousFilter)
-    return mPreviousFilter->HandleInComingCall(dwCallType, htaskCaller,
-					       dwTickCount, lpInterfaceInfo);
-  return SERVERCALL_ISHANDLED;
-}
-
-DWORD
-COMMessageFilter::RetryRejectedCall(HTASK htaskCallee,
-				    DWORD dwTickCount,
-				    DWORD dwRejectType)
-{
-  if (mPreviousFilter)
-    return mPreviousFilter->RetryRejectedCall(htaskCallee, dwTickCount,
-					      dwRejectType);
-  return -1;
-}
-
-DWORD
-COMMessageFilter::MessagePending(HTASK htaskCallee,
-				 DWORD dwTickCount,
-				 DWORD dwPendingType)
-{
-  mPlugin->FlushPendingInterruptQueue();
-  if (mPreviousFilter)
-    return mPreviousFilter->MessagePending(htaskCallee, dwTickCount,
-					   dwPendingType);
-  return PENDINGMSG_WAITNOPROCESS;
-}
-
-void
-COMMessageFilter::Initialize(PluginModuleChild* module)
-{
-  nsRefPtr<COMMessageFilter> f = new COMMessageFilter(module);
-  ::CoRegisterMessageFilter(f, getter_AddRefs(f->mPreviousFilter));
-}
-
-} // namespace plugins
-} // namespace mozilla

dom/plugins/ipc/COMMessageFilter.h

@@ -1,50 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#ifndef mozilla_plugins_COMMessageFilter_h
-#define mozilla_plugins_COMMessageFilter_h
-
-#include <objidl.h>
-#include "nsISupportsImpl.h"
-#include "nsAutoPtr.h"
-
-namespace mozilla {
-namespace plugins {
-
-class PluginModuleChild;
-
-class COMMessageFilter final : public IMessageFilter
-{
-public:
-  static void Initialize(PluginModuleChild* plugin);
-
-  COMMessageFilter(PluginModuleChild* plugin)
-    : mPlugin(plugin)
-  { }
-
-  HRESULT WINAPI QueryInterface(REFIID riid, void** ppv);
-  DWORD WINAPI AddRef();
-  DWORD WINAPI Release();
-
-  DWORD WINAPI HandleInComingCall(DWORD dwCallType,
-                                  HTASK htaskCaller,
-                                  DWORD dwTickCount,
-                                  LPINTERFACEINFO lpInterfaceInfo);
-  DWORD WINAPI RetryRejectedCall(HTASK htaskCallee,
-                                 DWORD dwTickCount,
-                                 DWORD dwRejectType);
-  DWORD WINAPI MessagePending(HTASK htaskCallee,
-                              DWORD dwTickCount,
-                              DWORD dwPendingType);
-
-private:
-  nsAutoRefCnt mRefCnt;
-  PluginModuleChild* mPlugin;
-  nsRefPtr<IMessageFilter> mPreviousFilter;
-};
-
-} // namespace plugins
-} // namespace mozilla
-
-#endif // COMMessageFilter_h

dom/plugins/ipc/PluginModuleChild.cpp

@@ -42,7 +42,6 @@
 #include "nsNPAPIPlugin.h"
 
 #ifdef XP_WIN
-#include "COMMessageFilter.h"
 #include "nsWindowsDllInterceptor.h"
 #include "mozilla/widget/AudioSession.h"
 #endif
@@ -264,10 +263,6 @@ PluginModuleChild::InitForChrome(const std::string& aPluginFilename,
                                  MessageLoop* aIOLoop,
                                  IPC::Channel* aChannel)
 {
-#ifdef XP_WIN
-    COMMessageFilter::Initialize(this);
-#endif
-
     NS_ASSERTION(aChannel, "need a channel");
 
     if (!InitGraphics())

dom/plugins/ipc/moz.build

@@ -50,7 +50,6 @@ if CONFIG['OS_ARCH'] == 'WINNT':
         'PluginSurfaceParent.h',
     ]
     UNIFIED_SOURCES += [
-        'COMMessageFilter.cpp',
         'PluginHangUIParent.cpp',
         'PluginSurfaceParent.cpp',
     ]

ipc/glue/WindowsMessageLoop.cpp

@@ -104,6 +104,7 @@ HHOOK gDeferredGetMsgHook = nullptr;
 HHOOK gDeferredCallWndProcHook = nullptr;
 
 DWORD gUIThreadId = 0;
+HWND gCOMWindow = 0;
 
 // WM_GETOBJECT id pulled from uia headers
 #define MOZOBJID_UIAROOT -25
@@ -340,13 +341,15 @@ ProcessOrDeferMessage(HWND hwnd,
    case WM_GETOBJECT: {
       if (!::GetPropW(hwnd, k3rdPartyWindowProp)) {
         DWORD objId = static_cast<DWORD>(lParam);
-        WNDPROC oldWndProc = (WNDPROC)GetProp(hwnd, kOldWndProcProp);
-        if ((objId == OBJID_CLIENT || objId == MOZOBJID_UIAROOT) && oldWndProc) {
-          return CallWindowProcW(oldWndProc, hwnd, uMsg, wParam, lParam);
+        if ((objId == OBJID_CLIENT || objId == MOZOBJID_UIAROOT)) {
+          WNDPROC oldWndProc = (WNDPROC)GetProp(hwnd, kOldWndProcProp);
+          if (oldWndProc) {
+            return CallWindowProcW(oldWndProc, hwnd, uMsg, wParam, lParam);
+          }
         }
       }
-      break;
-    }
+      return DefWindowProc(hwnd, uMsg, wParam, lParam);
+   }
 #endif // ACCESSIBILITY
 
     default: {
@@ -647,7 +650,10 @@ InitUIThread()
   // on startup.
   if (!gUIThreadId) {
     gUIThreadId = GetCurrentThreadId();
+
+    CoInitialize(nullptr);
   }
+
   MOZ_ASSERT(gUIThreadId);
   MOZ_ASSERT(gUIThreadId == GetCurrentThreadId(),
              "Called InitUIThread multiple times on different threads!");
@@ -804,6 +810,21 @@ IsTimeoutExpired(PRIntervalTime aStart, PRIntervalTime aTimeout)
     (aTimeout <= (PR_IntervalNow() - aStart));
 }
 
+HWND
+FindCOMWindow()
+{
+  MOZ_ASSERT(gUIThreadId);
+
+  HWND last = 0;
+  while ((last = FindWindowExW(HWND_MESSAGE, last, L"OleMainThreadWndClass", NULL))) {
+    if (GetWindowThreadProcessId(last, NULL) == gUIThreadId) {
+      return last;
+    }
+  }
+
+  return (HWND)0;
+}
+
 bool
 MessageChannel::WaitForSyncNotify()
 {
@@ -844,6 +865,12 @@ MessageChannel::WaitForSyncNotify()
 
   MonitorAutoUnlock unlock(*mMonitor);
 
+  MOZ_ASSERT_IF(gCOMWindow, FindCOMWindow() == gCOMWindow);
+
+  if (!gCOMWindow) {
+    gCOMWindow = FindCOMWindow();
+  }
+
   bool timedout = false;
 
   UINT_PTR timerId = 0;
@@ -912,11 +939,20 @@ MessageChannel::WaitForSyncNotify()
       bool haveSentMessagesPending =
         (HIWORD(GetQueueStatus(QS_SENDMESSAGE)) & QS_SENDMESSAGE) != 0;
 
-      // This PeekMessage call will actually process all "nonqueued" messages
-      // that are pending before returning. If we have "nonqueued" messages
-      // pending then we should have switched out all the window procedures
-      // above. In that case this PeekMessage call won't actually cause any
-      // mozilla code (or plugin code) to run.
+      // Either of the PeekMessage calls below will actually process all
+      // "nonqueued" messages that are pending before returning. If we have
+      // "nonqueued" messages pending then we should have switched out all the
+      // window procedures above. In that case this PeekMessage call won't
+      // actually cause any mozilla code (or plugin code) to run.
+
+      // We have to manually pump all COM messages *after* looking at the queue
+      // queue status but before yielding our thread below.
+      if (gCOMWindow) {
+        if (PeekMessageW(&msg, gCOMWindow, 0, 0, PM_REMOVE)) {
+          TranslateMessage(&msg);
+          ::DispatchMessageW(&msg);
+        }
+      }
 
       // If the following PeekMessage call fails to return a message for us (and
       // returns false) and we didn't run any "nonqueued" messages then we must
@@ -978,6 +1014,12 @@ MessageChannel::WaitForInterruptNotify()
 
   MonitorAutoUnlock unlock(*mMonitor);
 
+  MOZ_ASSERT_IF(gCOMWindow, FindCOMWindow() == gCOMWindow);
+
+  if (!gCOMWindow) {
+    gCOMWindow = FindCOMWindow();
+  }
+
   bool timedout = false;
 
   UINT_PTR timerId = 0;
@@ -1067,6 +1109,14 @@ MessageChannel::WaitForInterruptNotify()
     bool haveSentMessagesPending =
       (HIWORD(GetQueueStatus(QS_SENDMESSAGE)) & QS_SENDMESSAGE) != 0;
 
+    // Run all COM messages *after* looking at the queue status.
+    if (gCOMWindow) {
+        if (PeekMessageW(&msg, gCOMWindow, 0, 0, PM_REMOVE)) {
+            TranslateMessage(&msg);
+            ::DispatchMessageW(&msg);
+        }
+    }
+
     // PeekMessage markes the messages as "old" so that they don't wake up
     // MsgWaitForMultipleObjects every time.
     if (!PeekMessageW(&msg, nullptr, 0, 0, PM_NOREMOVE) &&

widget/windows/nsWindow.cpp

@@ -4144,8 +4144,8 @@ nsWindow::IsAsyncResponseEvent(UINT aMsg, LRESULT& aResult)
 void
 nsWindow::IPCWindowProcHandler(UINT& msg, WPARAM& wParam, LPARAM& lParam)
 {
-  NS_ASSERTION(!mozilla::ipc::MessageChannel::IsPumpingMessages(),
-               "Failed to prevent a nonqueued message from running!");
+  MOZ_ASSERT_IF(msg != WM_GETOBJECT,
+                !mozilla::ipc::MessageChannel::IsPumpingMessages());
 
   // Modal UI being displayed in windowless plugins.
   if (mozilla::ipc::MessageChannel::IsSpinLoopActive() &&