Bug 1761547 - Avoid the deadlock detector for mojo Port mutexes, r=ipc-reviewers,mccr8

Due to the way that ports are locked (occasionally in large groups in
address order) these mutexes seem to occasionally overwhelm the deadlock
detector with very long chains of lock dependencies, sometimes leading
to stack overflow issues.

As the imported code from chromium is already quite careful about
avoiding deadlocks with these locks, this patch switches them to use a
custom mutex which is not registered with the deadlock detector to avoid
these issues.

Differential Revision: https://phabricator.services.mozilla.com/D142130

ipc/chromium/src/mojo/core/ports/port.h

@@ -13,7 +13,8 @@
 #include "mojo/core/ports/event.h"
 #include "mojo/core/ports/message_queue.h"
 #include "mojo/core/ports/user_data.h"
-#include "mozilla/Mutex.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/PlatformMutex.h"
 #include "mozilla/RefPtr.h"
 #include "nsISupportsImpl.h"
 
@@ -23,6 +24,43 @@ namespace ports {
 
 class PortLocker;
 
+namespace detail {
+
+// Ports cannot use mozilla::Mutex, as the acquires-before relationships handled
+// by PortLocker can overload the debug-only deadlock detector.
+class CAPABILITY PortMutex : private ::mozilla::detail::MutexImpl {
+ public:
+  void AssertCurrentThreadOwns() const ASSERT_CAPABILITY(this) {
+#ifdef DEBUG
+    MOZ_ASSERT(mOwningThread == PR_GetCurrentThread());
+#endif
+  }
+
+ private:
+  // PortMutex should only be locked/unlocked via PortLocker
+  friend class ::mojo::core::ports::PortLocker;
+
+  void Lock() CAPABILITY_ACQUIRE() {
+    ::mozilla::detail::MutexImpl::lock();
+#ifdef DEBUG
+    mOwningThread = PR_GetCurrentThread();
+#endif
+  }
+  void Unlock() CAPABILITY_RELEASE() {
+#ifdef DEBUG
+    MOZ_ASSERT(mOwningThread == PR_GetCurrentThread());
+    mOwningThread = nullptr;
+#endif
+    ::mozilla::detail::MutexImpl::unlock();
+  }
+
+#ifdef DEBUG
+  mozilla::Atomic<PRThread*, mozilla::Relaxed> mOwningThread;
+#endif
+};
+
+}  // namespace detail
+
 // A Port is essentially a node in a circular list of addresses. For the sake of
 // this documentation such a list will henceforth be referred to as a "route."
 // Routes are the fundamental medium upon which all Node event circulation takes
@@ -191,7 +229,7 @@ class Port {
   // This lock guards all fields in Port, but is locked in a unique way which is
   // unfortunately somewhat difficult to get to work with the thread-safety
   // analysis.
-  mozilla::Mutex lock_ MOZ_ANNOTATED{"Port State"};
+  detail::PortMutex lock_ MOZ_ANNOTATED;
 };
 
 }  // namespace ports