Bug 1472018 - Limit the lock scope in WebCryptoThreadPool::Shutdown. r=bz

In bug 1364624 we switched over to SRWLock on Windows for our internal
implementation of mozilla::Mutex. This doesn't allow for re-entrancy. The
WebCryptoThreadPool shutdown code has potential for re-entrancy due to the
spinning of the main thread event loop while shutting down the worker threads.

By limiting the scope of the lock protecting mPool during shutdown we can avoid
the re-entrancy. Addtionally we track the shutdown status to avoid dispatching
events once shutdown has started.

--HG--
extra : rebase_source : 6e97a1fbdf4033ef93b3ecbafcf4b7898d9b19af

dom/crypto/WebCryptoThreadPool.cpp

@@ -66,6 +66,10 @@ WebCryptoThreadPool::DispatchInternal(nsIRunnable* aRunnable)
 {
   MutexAutoLock lock(mMutex);
 
+  if (mShutdown) {
+    return NS_ERROR_FAILURE;
+  }
+
   if (!mPool) {
     NS_ENSURE_TRUE(EnsureNSSInitializedChromeOrContent(), NS_ERROR_FAILURE);
 
@@ -85,10 +89,21 @@ void
 WebCryptoThreadPool::Shutdown()
 {
   MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
-  MutexAutoLock lock(mMutex);
 
-  if (mPool) {
-    mPool->Shutdown();
+  // Limit the scope of locking to avoid deadlocking if DispatchInternal ends
+  // up getting called during shutdown event processing.
+  nsCOMPtr<nsIThreadPool> pool;
+  {
+    MutexAutoLock lock(mMutex);
+    if (mShutdown) {
+      return;
+    }
+    pool = mPool;
+    mShutdown = true;
+  }
+
+  if (pool) {
+    pool->Shutdown();
   }
 
   nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();

dom/crypto/WebCryptoThreadPool.h

@@ -29,6 +29,7 @@ private:
   WebCryptoThreadPool()
     : mMutex("WebCryptoThreadPool::mMutex")
     , mPool(nullptr)
+    , mShutdown(false)
   { }
   virtual ~WebCryptoThreadPool()
   { }
@@ -48,6 +49,7 @@ private:
 
   mozilla::Mutex mMutex;
   nsCOMPtr<nsIThreadPool> mPool;
+  bool mShutdown;
 };
 
 } // namespace dom