Bug 1488808 Part 16 - Avoid deadlocking in a few places after diverging from the recording, r=froydnj.

--HG--
extra : rebase_source : 2327cd9632cdc9ace793f9e9fea666d95f305201

ipc/glue/MessageChannel.cpp

@@ -995,6 +995,17 @@ MessageChannel::Echo(Message* aMsg)
 bool
 MessageChannel::Send(Message* aMsg)
 {
+    if (recordreplay::HasDivergedFromRecording() &&
+        recordreplay::child::SuppressMessageAfterDiverge(aMsg))
+    {
+        // Only certain IPDL messages are allowed to be sent in a replaying
+        // process after it has diverged from the recording, to avoid
+        // deadlocking with threads that remain idle. The browser remains
+        // paused after diverging from the recording, and other IPDL messages
+        // do not need to be sent.
+        return true;
+    }
+
     if (aMsg->size() >= kMinTelemetryMessageSize) {
         Telemetry::Accumulate(Telemetry::IPC_MESSAGE_SIZE2, aMsg->size());
     }

toolkit/recordreplay/ipc/ChildIPC.cpp

@@ -16,7 +16,10 @@
 #include "ipc/Channel.h"
 #include "mac/handler/exception_handler.h"
 #include "mozilla/dom/ContentChild.h"
+#include "mozilla/layers/CompositorBridgeChild.h"
 #include "mozilla/layers/ImageDataSerializer.h"
+#include "mozilla/layers/ImageDataSerializer.h"
+#include "mozilla/layers/LayerTransactionChild.h"
 #include "mozilla/Sprintf.h"
 #include "mozilla/VsyncDispatcher.h"
 
@@ -591,6 +594,29 @@ CompositorCanPerformMiddlemanCalls()
   return gNumPendingPaints != 0;
 }
 
+bool
+SuppressMessageAfterDiverge(IPC::Message* aMsg)
+{
+  MOZ_RELEASE_ASSERT(HasDivergedFromRecording());
+
+  // Only messages necessary for compositing can be sent after the sending
+  // thread has diverged from the recording. Sending other messages can risk
+  // deadlocking when a necessary lock is held by an idle thread (we probably
+  // need a more robust way to deal with this problem).
+
+  IPC::Message::msgid_t type = aMsg->type();
+  if (type >= layers::PLayerTransaction::PLayerTransactionStart &&
+      type <= layers::PLayerTransaction::PLayerTransactionEnd) {
+    return false;
+  }
+
+  if (type == layers::PCompositorBridge::Msg_PTextureConstructor__ID) {
+    return false;
+  }
+
+  return true;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Checkpoint Messages
 ///////////////////////////////////////////////////////////////////////////////

toolkit/recordreplay/ipc/ChildIPC.h

@@ -11,6 +11,8 @@
 #include "mozilla/gfx/2D.h"
 #include "Units.h"
 
+namespace IPC { class Message; }
+
 namespace mozilla {
 
 class VsyncObserver;
@@ -53,6 +55,10 @@ void NotifyPaintComplete();
 // Get a draw target which the compositor thread can paint to.
 already_AddRefed<gfx::DrawTarget> DrawTargetForRemoteDrawing(LayoutDeviceIntSize aSize);
 
+// Called to ignore IPDL messages sent after diverging from the recording,
+// except for those needed for compositing.
+bool SuppressMessageAfterDiverge(IPC::Message* aMsg);
+
 } // namespace child
 } // namespace recordreplay
 } // namespace mozilla

toolkit/recordreplay/ipc/DisabledIPC.cpp

@@ -75,6 +75,12 @@ DrawTargetForRemoteDrawing(LayoutDeviceIntSize aSize)
   MOZ_CRASH();
 }
 
+bool
+SuppressMessageAfterDiverge(IPC::Message* aMsg)
+{
+  MOZ_CRASH();
+}
+
 } // namespace child
 
 namespace parent {

xpcom/threads/ThreadEventTarget.cpp

@@ -134,6 +134,14 @@ ThreadEventTarget::Dispatch(already_AddRefed<nsIRunnable> aEvent, uint32_t aFlag
     return NS_ERROR_ILLEGAL_DURING_SHUTDOWN;
   }
 
+  // Don't dispatch runnables to other threads when replaying and diverged from
+  // the recording, to avoid deadlocking with other idle threads. The browser
+  // remains paused after diverging from the recording, and threads will not
+  // run their event loops.
+  if (recordreplay::HasDivergedFromRecording()) {
+    return NS_ERROR_FAILURE;
+  }
+
 #ifdef MOZ_TASK_TRACER
   nsCOMPtr<nsIRunnable> tracedRunnable = CreateTracedRunnable(event.take());
   (static_cast<TracedRunnable*>(tracedRunnable.get()))->DispatchTask();