Bug 829909 - Fix some Plugin Hang UI synchronization problems. r=bbondy

dom/plugins/ipc/MiniShmParent.cpp

@@ -126,6 +126,8 @@ MiniShmParent::Init(MiniShmObserver* aObserver, const DWORD aTimeout,
   }
   nsresult rv = SetView(view, aSectionSize, false);
   NS_ENSURE_SUCCESS(rv, rv);
+  rv = SetGuard(childGuard, aTimeout);
+  NS_ENSURE_SUCCESS(rv, rv);
 
   MiniShmInit* initStruct = nullptr;
   rv = GetWritePtrInternal(initStruct);
@@ -174,12 +176,9 @@ MiniShmParent::GetCookie(std::wstring& cookie)
 nsresult
 MiniShmParent::Send()
 {
-  if (!mChildEvent || !mChildGuard) {
+  if (!mChildEvent) {
     return NS_ERROR_NOT_INITIALIZED;
   }
-  if (::WaitForSingleObject(mChildGuard, mTimeout) != WAIT_OBJECT_0) {
-    return NS_ERROR_FAILURE;
-  }
   if (!::SetEvent(mChildEvent)) {
     return NS_ERROR_FAILURE;
   }

dom/plugins/ipc/PluginHangUIParent.cpp

@@ -322,6 +322,12 @@ PluginHangUIParent::OnMiniShmEvent(MiniShmBase *aMiniShmObj)
   NS_ASSERTION(NS_SUCCEEDED(rv),
                "Couldn't obtain read pointer OnMiniShmEvent");
   if (NS_SUCCEEDED(rv)) {
+    // The child process has returned a response so we shouldn't worry about 
+    // its state anymore.
+    if (::UnregisterWaitEx(mRegWait, NULL)) {
+      mRegWait = NULL;
+    }
+
     RecvUserResponse(response->mResponseBits);
   }
 }

dom/plugins/ipc/hangui/MiniShmBase.h

@@ -121,17 +121,21 @@ public:
    * @return NS_OK if and only if aPtr was successfully obtained.
    *         NS_ERROR_ILLEGAL_VALUE if type T is not valid for MiniShm.
    *         NS_ERROR_NOT_INITIALIZED if there is no valid MiniShm connection.
+   *         NS_ERROR_NOT_AVAILABLE if the memory is not safe to write.
    */
   template<typename T> nsresult
   GetWritePtr(T*& aPtr)
   {
-    if (!mWriteHeader) {
+    if (!mWriteHeader || !mGuard) {
       return NS_ERROR_NOT_INITIALIZED;
     }
     if (sizeof(T) > mPayloadMaxLen ||
         T::identifier <= RESERVED_CODE_LAST) {
       return NS_ERROR_ILLEGAL_VALUE;
     }
+    if (::WaitForSingleObject(mGuard, mTimeout) != WAIT_OBJECT_0) {
+      return NS_ERROR_NOT_AVAILABLE;
+    }
     mWriteHeader->mId = T::identifier;
     mWriteHeader->mPayloadLen = sizeof(T);
     aPtr = reinterpret_cast<T*>(mWriteHeader + 1);
@@ -219,7 +223,9 @@ protected:
     : mObserver(nullptr),
       mWriteHeader(nullptr),
       mReadHeader(nullptr),
-      mPayloadMaxLen(0)
+      mPayloadMaxLen(0),
+      mGuard(NULL),
+      mTimeout(INFINITE)
   {
   }
   virtual ~MiniShmBase()
@@ -261,6 +267,17 @@ protected:
     return NS_OK;
   }
 
+  nsresult
+  SetGuard(HANDLE aGuard, DWORD aTimeout)
+  {
+    if (!aGuard || !aTimeout) {
+      return NS_ERROR_ILLEGAL_VALUE;
+    }
+    mGuard = aGuard;
+    mTimeout = aTimeout;
+    return NS_OK;
+  }
+
   inline void
   SetObserver(MiniShmObserver *aObserver) { mObserver = aObserver; }
 
@@ -304,6 +321,8 @@ private:
   MiniShmHeader*    mWriteHeader;
   MiniShmHeader*    mReadHeader;
   unsigned int      mPayloadMaxLen;
+  HANDLE            mGuard;
+  DWORD             mTimeout;
 
   DISALLOW_COPY_AND_ASSIGN(MiniShmBase);
 };

dom/plugins/ipc/hangui/MiniShmChild.cpp

@@ -28,12 +28,14 @@ MiniShmChild::~MiniShmChild()
   if (mRegWait) {
     ::UnregisterWaitEx(mRegWait, INVALID_HANDLE_VALUE);
   }
+  if (mParentGuard) {
+    // Try to avoid shutting down while the parent's event handler is running.
+    ::WaitForSingleObject(mParentGuard, mTimeout);
+    ::CloseHandle(mParentGuard);
+  }
   if (mParentEvent) {
     ::CloseHandle(mParentEvent);
   }
-  if (mParentGuard) {
-    ::CloseHandle(mParentGuard);
-  }
   if (mChildEvent) {
     ::CloseHandle(mChildEvent);
   }
@@ -95,6 +97,10 @@ MiniShmChild::Init(MiniShmObserver* aObserver, const std::wstring& aCookie,
       !initStruct->mChildEvent || !initStruct->mChildGuard) {
     return NS_ERROR_FAILURE;
   }
+  rv = SetGuard(initStruct->mParentGuard, aTimeout);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
   if (!::RegisterWaitForSingleObject(&mRegWait,
                                      initStruct->mChildEvent,
                                      &SOnEvent,
@@ -146,12 +152,9 @@ MiniShmChild::Init(MiniShmObserver* aObserver, const std::wstring& aCookie,
 nsresult
 MiniShmChild::Send()
 {
-  if (!mParentEvent || !mParentGuard) {
+  if (!mParentEvent) {
     return NS_ERROR_NOT_INITIALIZED;
   }
-  if (::WaitForSingleObject(mParentGuard, mTimeout) != WAIT_OBJECT_0) {
-    return NS_ERROR_FAILURE;
-  }
   if (!::SetEvent(mParentEvent)) {
     return NS_ERROR_FAILURE;
   }