Bug 1488808 Part 5 - Add infrastructure for performing system calls in the middleman process, r=froydnj.

--HG--
extra : rebase_source : d4e37772b0dd5103bf1ceb1602503fc8fe83d927

toolkit/recordreplay/BufferStream.h

@@ -0,0 +1,68 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_recordreplay_BufferStream_h
+#define mozilla_recordreplay_BufferStream_h
+
+#include "InfallibleVector.h"
+
+namespace mozilla {
+namespace recordreplay {
+
+// BufferStream provides similar functionality to Stream in File.h, allowing
+// reading or writing to a stream of data backed by an in memory buffer instead
+// of data stored on disk.
+class BufferStream
+{
+  InfallibleVector<char>* mOutput;
+
+  const char* mInput;
+  size_t mInputSize;
+
+public:
+  BufferStream(const char* aInput, size_t aInputSize)
+    : mOutput(nullptr), mInput(aInput), mInputSize(aInputSize)
+  {}
+
+  explicit BufferStream(InfallibleVector<char>* aOutput)
+    : mOutput(aOutput), mInput(nullptr), mInputSize(0)
+  {}
+
+  void WriteBytes(const void* aData, size_t aSize) {
+    MOZ_RELEASE_ASSERT(mOutput);
+    mOutput->append((char*) aData, aSize);
+  }
+
+  void WriteScalar(size_t aValue) {
+    WriteBytes(&aValue, sizeof(aValue));
+  }
+
+  void ReadBytes(void* aData, size_t aSize) {
+    if (aSize) {
+      MOZ_RELEASE_ASSERT(mInput);
+      MOZ_RELEASE_ASSERT(aSize <= mInputSize);
+      memcpy(aData, mInput, aSize);
+      mInput += aSize;
+      mInputSize -= aSize;
+    }
+  }
+
+  size_t ReadScalar() {
+    size_t rv;
+    ReadBytes(&rv, sizeof(rv));
+    return rv;
+  }
+
+  bool IsEmpty() {
+    MOZ_RELEASE_ASSERT(mInput);
+    return mInputSize == 0;
+  }
+};
+
+} // namespace recordreplay
+} // namespace mozilla
+
+#endif // mozilla_recordreplay_BufferStream_h

toolkit/recordreplay/MiddlemanCall.cpp

@@ -0,0 +1,426 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MiddlemanCall.h"
+
+#include <unordered_map>
+
+namespace mozilla {
+namespace recordreplay {
+
+// In a replaying or middleman process, all middleman calls that have been
+// encountered, indexed by their ID.
+static StaticInfallibleVector<MiddlemanCall*> gMiddlemanCalls;
+
+// In a replaying or middleman process, association between values produced by
+// a middleman call and the call itself.
+typedef std::unordered_map<const void*, MiddlemanCall*> MiddlemanCallMap;
+static MiddlemanCallMap* gMiddlemanCallMap;
+
+// In a middleman process, any buffers allocated for performed calls.
+static StaticInfallibleVector<void*> gAllocatedBuffers;
+
+// Lock protecting middleman call state.
+static Monitor* gMonitor;
+
+void
+InitializeMiddlemanCalls()
+{
+  MOZ_RELEASE_ASSERT(IsRecordingOrReplaying() || IsMiddleman());
+
+  gMiddlemanCallMap = new MiddlemanCallMap();
+  gMonitor = new Monitor();
+}
+
+// Apply the ReplayInput phase to aCall and any calls it depends on that have
+// not been sent to the middleman yet, filling aOutgoingCalls with the set of
+// such calls.
+static bool
+GatherDependentCalls(InfallibleVector<MiddlemanCall*>& aOutgoingCalls, MiddlemanCall* aCall)
+{
+  MOZ_RELEASE_ASSERT(!aCall->mSent);
+  aCall->mSent = true;
+
+  CallArguments arguments;
+  aCall->mArguments.CopyTo(&arguments);
+
+  InfallibleVector<MiddlemanCall*> dependentCalls;
+
+  MiddlemanCallContext cx(aCall, &arguments, MiddlemanCallPhase::ReplayInput);
+  cx.mDependentCalls = &dependentCalls;
+  gRedirections[aCall->mCallId].mMiddlemanCall(cx);
+  if (cx.mFailed) {
+    return false;
+  }
+
+  for (MiddlemanCall* dependent : dependentCalls) {
+    if (!dependent->mSent && !GatherDependentCalls(aOutgoingCalls, dependent)) {
+      return false;
+    }
+  }
+
+  aOutgoingCalls.append(aCall);
+  return true;
+}
+
+bool
+SendCallToMiddleman(size_t aCallId, CallArguments* aArguments, bool aDiverged)
+{
+  MOZ_RELEASE_ASSERT(IsReplaying());
+
+  const Redirection& redirection = gRedirections[aCallId];
+  MOZ_RELEASE_ASSERT(redirection.mMiddlemanCall);
+
+  Maybe<MonitorAutoLock> lock;
+  lock.emplace(*gMonitor);
+
+  // Allocate and fill in a new MiddlemanCall.
+  size_t id = gMiddlemanCalls.length();
+  MiddlemanCall* newCall = new MiddlemanCall();
+  gMiddlemanCalls.emplaceBack(newCall);
+  newCall->mId = id;
+  newCall->mCallId = aCallId;
+  newCall->mArguments.CopyFrom(aArguments);
+
+  // Perform the ReplayPreface phase on the new call.
+  {
+    MiddlemanCallContext cx(newCall, aArguments, MiddlemanCallPhase::ReplayPreface);
+    redirection.mMiddlemanCall(cx);
+    if (cx.mFailed) {
+      delete newCall;
+      gMiddlemanCalls.popBack();
+      return false;
+    }
+  }
+
+  // Other phases will not run if we have not diverged from the recording.
+  // Any outputs for the call have been handled by the SaveOutput hook.
+  if (!aDiverged) {
+    return true;
+  }
+
+  // Perform the ReplayInput phase on the new call and any others it depends on.
+  InfallibleVector<MiddlemanCall*> outgoingCalls;
+  if (!GatherDependentCalls(outgoingCalls, newCall)) {
+    for (MiddlemanCall* call : outgoingCalls) {
+      call->mSent = false;
+    }
+    return false;
+  }
+
+  // Encode all calls we are sending to the middleman.
+  InfallibleVector<char> inputData;
+  BufferStream inputStream(&inputData);
+  for (MiddlemanCall* call : outgoingCalls) {
+    call->EncodeInput(inputStream);
+  }
+
+  // Perform the calls synchronously in the middleman.
+  InfallibleVector<char> outputData;
+  if (!child::SendMiddlemanCallRequest(inputData.begin(), inputData.length(), &outputData)) {
+    // This thread is not allowed to perform middleman calls anymore. Release
+    // the lock and block until the process rewinds.
+    lock.reset();
+    Thread::WaitForever();
+  }
+
+  // Decode outputs for the calls just sent, and perform the ReplayOutput phase
+  // on any older dependent calls we sent.
+  BufferStream outputStream(outputData.begin(), outputData.length());
+  for (MiddlemanCall* call : outgoingCalls) {
+    call->DecodeOutput(outputStream);
+
+    if (call != newCall) {
+      CallArguments oldArguments;
+      call->mArguments.CopyTo(&oldArguments);
+      MiddlemanCallContext cx(call, &oldArguments, MiddlemanCallPhase::ReplayOutput);
+      cx.mReplayOutputIsOld = true;
+      gRedirections[call->mCallId].mMiddlemanCall(cx);
+    }
+  }
+
+  // Perform the ReplayOutput phase to fill in outputs for the current call.
+  newCall->mArguments.CopyTo(aArguments);
+  MiddlemanCallContext cx(newCall, aArguments, MiddlemanCallPhase::ReplayOutput);
+  redirection.mMiddlemanCall(cx);
+  return true;
+}
+
+void
+ProcessMiddlemanCall(const char* aInputData, size_t aInputSize,
+                     InfallibleVector<char>* aOutputData)
+{
+  MOZ_RELEASE_ASSERT(IsMiddleman());
+
+  BufferStream inputStream(aInputData, aInputSize);
+  BufferStream outputStream(aOutputData);
+
+  while (!inputStream.IsEmpty()) {
+    MiddlemanCall* call = new MiddlemanCall();
+    call->DecodeInput(inputStream);
+
+    const Redirection& redirection = gRedirections[call->mCallId];
+    MOZ_RELEASE_ASSERT(gRedirections[call->mCallId].mMiddlemanCall);
+
+    CallArguments arguments;
+    call->mArguments.CopyTo(&arguments);
+
+    {
+      MiddlemanCallContext cx(call, &arguments, MiddlemanCallPhase::MiddlemanInput);
+      redirection.mMiddlemanCall(cx);
+    }
+
+    RecordReplayInvokeCall(call->mCallId, &arguments);
+
+    {
+      MiddlemanCallContext cx(call, &arguments, MiddlemanCallPhase::MiddlemanOutput);
+      redirection.mMiddlemanCall(cx);
+    }
+
+    call->mArguments.CopyFrom(&arguments);
+    call->EncodeOutput(outputStream);
+
+    while (call->mId >= gMiddlemanCalls.length()) {
+      gMiddlemanCalls.emplaceBack(nullptr);
+    }
+    MOZ_RELEASE_ASSERT(!gMiddlemanCalls[call->mId]);
+    gMiddlemanCalls[call->mId] = call;
+  }
+}
+
+void*
+MiddlemanCallContext::AllocateBytes(size_t aSize)
+{
+  void* rv = malloc(aSize);
+
+  // In a middleman process, any buffers we allocate live until the calls are
+  // reset. In a replaying process, the buffers will either live forever
+  // (if they are allocated in the ReplayPreface phase, to match the lifetime
+  // of the MiddlemanCall itself) or will be recovered when we rewind after we
+  // are done with our divergence from the recording (any other phase).
+  if (IsMiddleman()) {
+    gAllocatedBuffers.append(rv);
+  }
+
+  return rv;
+}
+
+void
+ResetMiddlemanCalls()
+{
+  MOZ_RELEASE_ASSERT(IsMiddleman());
+
+  for (MiddlemanCall* call : gMiddlemanCalls) {
+    if (call) {
+      CallArguments arguments;
+      call->mArguments.CopyTo(&arguments);
+
+      MiddlemanCallContext cx(call, &arguments, MiddlemanCallPhase::MiddlemanRelease);
+      gRedirections[call->mCallId].mMiddlemanCall(cx);
+
+      delete call;
+    }
+  }
+
+  gMiddlemanCalls.clear();
+  for (auto buffer : gAllocatedBuffers) {
+    free(buffer);
+  }
+  gAllocatedBuffers.clear();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// System Values
+///////////////////////////////////////////////////////////////////////////////
+
+static void
+AddMiddlemanCallValue(const void* aThing, MiddlemanCall* aCall)
+{
+  gMiddlemanCallMap->erase(aThing);
+  gMiddlemanCallMap->insert(MiddlemanCallMap::value_type(aThing, aCall));
+}
+
+static MiddlemanCall*
+LookupMiddlemanCall(const void* aThing)
+{
+  MiddlemanCallMap::const_iterator iter = gMiddlemanCallMap->find(aThing);
+  if (iter != gMiddlemanCallMap->end()) {
+    return iter->second;
+  }
+  return nullptr;
+}
+
+static const void*
+GetMiddlemanCallValue(size_t aId)
+{
+  MOZ_RELEASE_ASSERT(IsMiddleman());
+  MOZ_RELEASE_ASSERT(aId < gMiddlemanCalls.length() &&
+                     gMiddlemanCalls[aId] &&
+                     gMiddlemanCalls[aId]->mMiddlemanValue.isSome());
+  return gMiddlemanCalls[aId]->mMiddlemanValue.ref();
+}
+
+bool
+Middleman_SystemInput(MiddlemanCallContext& aCx, const void** aThingPtr)
+{
+  MOZ_RELEASE_ASSERT(aCx.AccessPreface());
+
+  if (!*aThingPtr) {
+    // Null values are handled by the normal argument copying logic.
+    return true;
+  }
+
+  Maybe<size_t> callId;
+  if (aCx.mPhase == MiddlemanCallPhase::ReplayPreface) {
+    // Determine any middleman call this object came from, before the pointer
+    // has a chance to be clobbered by another call between this and the
+    // ReplayInput phase.
+    MiddlemanCall* call = LookupMiddlemanCall(*aThingPtr);
+    if (call) {
+      callId.emplace(call->mId);
+    }
+  }
+  aCx.ReadOrWritePrefaceBytes(&callId, sizeof(callId));
+
+  switch (aCx.mPhase) {
+  case MiddlemanCallPhase::ReplayPreface:
+    return true;
+  case MiddlemanCallPhase::ReplayInput:
+    if (callId.isSome()) {
+      aCx.WriteInputScalar(callId.ref());
+      aCx.mDependentCalls->append(gMiddlemanCalls[callId.ref()]);
+      return true;
+    }
+    return false;
+  case MiddlemanCallPhase::MiddlemanInput:
+    if (callId.isSome()) {
+      size_t callIndex = aCx.ReadInputScalar();
+      *aThingPtr = GetMiddlemanCallValue(callIndex);
+      return true;
+    }
+    return false;
+  default:
+    MOZ_CRASH("Bad phase");
+  }
+}
+
+// Pointer system values are preserved during the replay so that null tests
+// and equality tests work as expected. We additionally mangle the
+// pointers here by setting one of the two highest bits, depending on whether
+// the pointer came from the recording or from the middleman. This avoids
+// accidentally conflating pointers that happen to have the same value but
+// which originate from different processes.
+static const void*
+MangleSystemValue(const void* aValue, bool aFromRecording)
+{
+  return (const void*) ((size_t)aValue | (1ULL << (aFromRecording ? 63 : 62)));
+}
+
+void
+Middleman_SystemOutput(MiddlemanCallContext& aCx, const void** aOutput, bool aUpdating)
+{
+  if (!*aOutput) {
+    return;
+  }
+
+  switch (aCx.mPhase) {
+  case MiddlemanCallPhase::ReplayPreface:
+    if (!HasDivergedFromRecording()) {
+      // If we haven't diverged from the recording, use the output value saved
+      // in the recording.
+      if (!aUpdating) {
+        *aOutput = MangleSystemValue(*aOutput, true);
+      }
+      aCx.mCall->SetRecordingValue(*aOutput);
+      AddMiddlemanCallValue(*aOutput, aCx.mCall);
+    }
+    break;
+  case MiddlemanCallPhase::MiddlemanOutput:
+    aCx.mCall->SetMiddlemanValue(*aOutput);
+    AddMiddlemanCallValue(*aOutput, aCx.mCall);
+    break;
+  case MiddlemanCallPhase::ReplayOutput: {
+    if (!aUpdating) {
+      *aOutput = MangleSystemValue(*aOutput, false);
+    }
+    aCx.mCall->SetMiddlemanValue(*aOutput);
+
+    // Associate the value produced by the middleman with this call. If the
+    // call previously went through the ReplayPreface phase when we did not
+    // diverge from the recording, we will associate values from both the
+    // recording and middleman processes with this call. If a call made after
+    // diverging produced the same value as a call made before diverging, use
+    // the value saved in the recording for the first call, so that equality
+    // tests on the value work as expected.
+    MiddlemanCall* previousCall = LookupMiddlemanCall(*aOutput);
+    if (previousCall) {
+      if (previousCall->mRecordingValue.isSome()) {
+        *aOutput = previousCall->mRecordingValue.ref();
+      }
+    } else {
+      AddMiddlemanCallValue(*aOutput, aCx.mCall);
+    }
+    break;
+  }
+  default:
+    return;
+  }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// MiddlemanCall
+///////////////////////////////////////////////////////////////////////////////
+
+void
+MiddlemanCall::EncodeInput(BufferStream& aStream) const
+{
+  aStream.WriteScalar(mId);
+  aStream.WriteScalar(mCallId);
+  aStream.WriteBytes(&mArguments, sizeof(CallRegisterArguments));
+  aStream.WriteScalar(mPreface.length());
+  aStream.WriteBytes(mPreface.begin(), mPreface.length());
+  aStream.WriteScalar(mInput.length());
+  aStream.WriteBytes(mInput.begin(), mInput.length());
+}
+
+void
+MiddlemanCall::DecodeInput(BufferStream& aStream)
+{
+  mId = aStream.ReadScalar();
+  mCallId = aStream.ReadScalar();
+  aStream.ReadBytes(&mArguments, sizeof(CallRegisterArguments));
+  size_t prefaceLength = aStream.ReadScalar();
+  mPreface.appendN(0, prefaceLength);
+  aStream.ReadBytes(mPreface.begin(), prefaceLength);
+  size_t inputLength = aStream.ReadScalar();
+  mInput.appendN(0, inputLength);
+  aStream.ReadBytes(mInput.begin(), inputLength);
+}
+
+void
+MiddlemanCall::EncodeOutput(BufferStream& aStream) const
+{
+  aStream.WriteBytes(&mArguments, sizeof(CallRegisterArguments));
+  aStream.WriteScalar(mOutput.length());
+  aStream.WriteBytes(mOutput.begin(), mOutput.length());
+}
+
+void
+MiddlemanCall::DecodeOutput(BufferStream& aStream)
+{
+  // Only update the return value when decoding arguments, so that we don't
+  // clobber the call's arguments with any changes made in the middleman.
+  CallRegisterArguments newArguments;
+  aStream.ReadBytes(&newArguments, sizeof(CallRegisterArguments));
+  mArguments.CopyRvalFrom(&newArguments);
+
+  size_t outputLength = aStream.ReadScalar();
+  mOutput.appendN(0, outputLength);
+  aStream.ReadBytes(mOutput.begin(), outputLength);
+}
+
+} // namespace recordreplay
+} // namespace mozilla

toolkit/recordreplay/MiddlemanCall.h

@@ -0,0 +1,464 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_recordreplay_MiddlemanCall_h
+#define mozilla_recordreplay_MiddlemanCall_h
+
+#include "BufferStream.h"
+#include "ProcessRedirect.h"
+#include "mozilla/Maybe.h"
+
+namespace mozilla {
+namespace recordreplay {
+
+// Middleman Calls Overview
+//
+// With few exceptions, replaying processes do not interact with the underlying
+// system or call the actual versions of redirected system library functions.
+// This is problematic after diverging from the recording, as then the diverged
+// thread cannot interact with its recording either.
+//
+// Middleman calls are used in a replaying process after diverging from the
+// recording to perform calls in the middleman process instead. Inputs are
+// gathered and serialized in the replaying process, then sent to the middleman
+// process. The middleman calls the function, and its outputs are serialized
+// for reading by the replaying process.
+//
+// Calls that might need to be sent to the middleman are processed in phases,
+// per the MiddlemanCallPhase enum below. The timeline of a middleman call is
+// as follows:
+//
+// - Any redirection with a middleman call hook can potentially be sent to the
+//   middleman. In a replaying process, whenever such a call is encountered,
+//   the hook is invoked in the ReplayPreface phase to capture any input data
+//   that must be examined at the time of the call itself.
+//
+// - If the thread has not diverged from the recording, the call is remembered
+//   but no further action is necessary yet.
+//
+// - If the thread has diverged from the recording, the call needs to go
+//   through the remaining phases. The ReplayInput phase captures any
+//   additional inputs to the call, potentially including values produced by
+//   other middleman calls.
+//
+// - The transitive closure of these call dependencies is produced, and all
+//   calls found go through the ReplayInput phase. The resulting data is sent
+//   to the middleman process, which goes through the MiddlemanInput phase
+//   to decode those inputs.
+//
+// - The middleman performs each of the calls it has been given, and their
+//   outputs are encoded in the MiddlemanOutput phase. These outputs are sent
+//   to the replaying process in a response and decoded in the ReplayOutput
+//   phase, which can then resume execution.
+//
+// - The replaying process holds onto information about calls it has sent until
+//   it rewinds to a point before it diverged from the recording. This rewind
+//   will --- without any special action required --- wipe out information on
+//   all calls sent to the middleman, and retain any data gathered in the
+//   ReplayPreface phase for calls that were made prior to the rewind target.
+//
+// - Information about calls and all resources held are retained in the
+//   middleman process are retained until a replaying process asks for them to
+//   be reset, which happens any time the replaying process first diverges from
+//   the recording. The MiddlemanRelease phase is used to release any system
+//   resources held.
+
+// Ways of processing calls that can be sent to the middleman.
+enum class MiddlemanCallPhase
+{
+  // When replaying, a call is being performed that might need to be sent to
+  // the middleman later.
+  ReplayPreface,
+
+  // A call for which inputs have been gathered is now being sent to the
+  // middleman. This is separate from ReplayPreface because capturing inputs
+  // might need to dereference pointers that could be bogus values originating
+  // from the recording. Waiting to dereference these pointers until we know
+  // the call needs to be sent to the middleman avoids needing to understand
+  // the inputs to all call sites of general purpose redirections such as
+  // CFArrayCreate.
+  ReplayInput,
+
+  // In the middleman process, a call from the replaying process is being
+  // performed.
+  MiddlemanInput,
+
+  // In the middleman process, a call from the replaying process was just
+  // performed, and its outputs need to be saved.
+  MiddlemanOutput,
+
+  // Back in the replaying process, the outputs from a call have been received
+  // from the middleman.
+  ReplayOutput,
+
+  // In the middleman process, release any system resources held after this
+  // call.
+  MiddlemanRelease,
+};
+
+struct MiddlemanCall
+{
+  // Unique ID for this call.
+  size_t mId;
+
+  // ID of the redirection being invoked.
+  size_t mCallId;
+
+  // All register arguments and return values are preserved when sending the
+  // call back and forth between processes.
+  CallRegisterArguments mArguments;
+
+  // Written in ReplayPrefaceInput, read in ReplayInput and MiddlemanInput.
+  InfallibleVector<char> mPreface;
+
+  // Written in ReplayInput, read in MiddlemanInput.
+  InfallibleVector<char> mInput;
+
+  // Written in MiddlemanOutput, read in ReplayOutput.
+  InfallibleVector<char> mOutput;
+
+  // In a replaying process, whether this call has been sent to the middleman.
+  bool mSent;
+
+  // In a replaying process, any value associated with this call that was
+  // included in the recording, when the call was made before diverging from
+  // the recording.
+  Maybe<const void*> mRecordingValue;
+
+  // In a replaying or middleman process, any value associated with this call
+  // that was produced by the middleman itself.
+  Maybe<const void*> mMiddlemanValue;
+
+  MiddlemanCall()
+    : mId(0), mCallId(0), mSent(false)
+  {}
+
+  void EncodeInput(BufferStream& aStream) const;
+  void DecodeInput(BufferStream& aStream);
+
+  void EncodeOutput(BufferStream& aStream) const;
+  void DecodeOutput(BufferStream& aStream);
+
+  void SetRecordingValue(const void* aValue) {
+    MOZ_RELEASE_ASSERT(mRecordingValue.isNothing());
+    mRecordingValue.emplace(aValue);
+  }
+
+  void SetMiddlemanValue(const void* aValue) {
+    MOZ_RELEASE_ASSERT(mMiddlemanValue.isNothing());
+    mMiddlemanValue.emplace(aValue);
+  }
+};
+
+// Information needed to process one of the phases of a middleman call,
+// in either the replaying or middleman process.
+struct MiddlemanCallContext
+{
+  // Call being operated on.
+  MiddlemanCall* mCall;
+
+  // Complete arguments and return value information for the call.
+  CallArguments* mArguments;
+
+  // Current processing phase.
+  MiddlemanCallPhase mPhase;
+
+  // During the ReplayPreface or ReplayInput phases, whether capturing input
+  // data has failed. In such cases the call cannot be sent to the middleman
+  // and, if the thread has diverged from the recording, an unhandled
+  // divergence and associated rewind will occur.
+  bool mFailed;
+
+  // During the ReplayInput phase, this can be used to fill in any middleman
+  // calls whose output the current one depends on.
+  InfallibleVector<MiddlemanCall*>* mDependentCalls;
+
+  // Streams of data that can be accessed during the various phases. Streams
+  // need to be read or written from at the same points in the phases which use
+  // them, so that callbacks operating on these streams can be composed without
+  // issues.
+
+  // The preface is written during ReplayPreface, and read during both
+  // ReplayInput and MiddlemanInput.
+  Maybe<BufferStream> mPrefaceStream;
+
+  // Inputs are written during ReplayInput, and read during MiddlemanInput.
+  Maybe<BufferStream> mInputStream;
+
+  // Outputs are written during MiddlemanOutput, and read during ReplayOutput.
+  Maybe<BufferStream> mOutputStream;
+
+  // During the ReplayOutput phase, this is set if the call was made sometime
+  // in the past and pointers referred to in the arguments may no longer be
+  // valid.
+  bool mReplayOutputIsOld;
+
+  MiddlemanCallContext(MiddlemanCall* aCall, CallArguments* aArguments, MiddlemanCallPhase aPhase)
+    : mCall(aCall), mArguments(aArguments), mPhase(aPhase),
+      mFailed(false), mDependentCalls(nullptr), mReplayOutputIsOld(false)
+  {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayPreface:
+      mPrefaceStream.emplace(&mCall->mPreface);
+      break;
+    case MiddlemanCallPhase::ReplayInput:
+      mPrefaceStream.emplace(mCall->mPreface.begin(), mCall->mPreface.length());
+      mInputStream.emplace(&mCall->mInput);
+      break;
+    case MiddlemanCallPhase::MiddlemanInput:
+      mPrefaceStream.emplace(mCall->mPreface.begin(), mCall->mPreface.length());
+      mInputStream.emplace(mCall->mInput.begin(), mCall->mInput.length());
+      break;
+    case MiddlemanCallPhase::MiddlemanOutput:
+      mOutputStream.emplace(&mCall->mOutput);
+      break;
+    case MiddlemanCallPhase::ReplayOutput:
+      mOutputStream.emplace(mCall->mOutput.begin(), mCall->mOutput.length());
+      break;
+    case MiddlemanCallPhase::MiddlemanRelease:
+      break;
+    }
+  }
+
+  void MarkAsFailed() {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::ReplayPreface ||
+                       mPhase == MiddlemanCallPhase::ReplayInput);
+    mFailed = true;
+  }
+
+  void WriteInputBytes(const void* aBuffer, size_t aSize) {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::ReplayInput);
+    mInputStream.ref().WriteBytes(aBuffer, aSize);
+  }
+
+  void WriteInputScalar(size_t aValue) {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::ReplayInput);
+    mInputStream.ref().WriteScalar(aValue);
+  }
+
+  void ReadInputBytes(void* aBuffer, size_t aSize) {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::MiddlemanInput);
+    mInputStream.ref().ReadBytes(aBuffer, aSize);
+  }
+
+  size_t ReadInputScalar() {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::MiddlemanInput);
+    return mInputStream.ref().ReadScalar();
+  }
+
+  bool AccessInput() {
+    return mInputStream.isSome();
+  }
+
+  void ReadOrWriteInputBytes(void* aBuffer, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayInput:
+      WriteInputBytes(aBuffer, aSize);
+      break;
+    case MiddlemanCallPhase::MiddlemanInput:
+      ReadInputBytes(aBuffer, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  bool AccessPreface() {
+    return mPrefaceStream.isSome();
+  }
+
+  void ReadOrWritePrefaceBytes(void* aBuffer, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayPreface:
+      mPrefaceStream.ref().WriteBytes(aBuffer, aSize);
+      break;
+    case MiddlemanCallPhase::ReplayInput:
+    case MiddlemanCallPhase::MiddlemanInput:
+      mPrefaceStream.ref().ReadBytes(aBuffer, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  void ReadOrWritePrefaceBuffer(void** aBufferPtr, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayPreface:
+      mPrefaceStream.ref().WriteBytes(*aBufferPtr, aSize);
+      break;
+    case MiddlemanCallPhase::ReplayInput:
+    case MiddlemanCallPhase::MiddlemanInput:
+      *aBufferPtr = AllocateBytes(aSize);
+      mPrefaceStream.ref().ReadBytes(*aBufferPtr, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  bool AccessOutput() {
+    return mOutputStream.isSome();
+  }
+
+  void ReadOrWriteOutputBytes(void* aBuffer, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::MiddlemanOutput:
+      mOutputStream.ref().WriteBytes(aBuffer, aSize);
+      break;
+    case MiddlemanCallPhase::ReplayOutput:
+      mOutputStream.ref().ReadBytes(aBuffer, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  void ReadOrWriteOutputBuffer(void** aBuffer, size_t aSize) {
+    if (*aBuffer) {
+      if (mPhase == MiddlemanCallPhase::MiddlemanInput || mReplayOutputIsOld) {
+        *aBuffer = AllocateBytes(aSize);
+      }
+
+      if (AccessOutput()) {
+        ReadOrWriteOutputBytes(*aBuffer, aSize);
+      }
+    }
+  }
+
+  // Allocate some memory associated with the call, which will be released in
+  // the replaying process on a rewind and in the middleman process when the
+  // call state is reset.
+  void* AllocateBytes(size_t aSize);
+};
+
+// Notify the system about a call to a redirection with a middleman call hook.
+// aDiverged is set if the current thread has diverged from the recording and
+// any outputs for the call must be filled in; otherwise, they already have
+// been filled in using data from the recording. Returns false if the call was
+// unable to be processed.
+bool SendCallToMiddleman(size_t aCallId, CallArguments* aArguments, bool aDiverged);
+
+// In the middleman process, perform one or more calls encoded in aInputData
+// and encode their outputs to aOutputData.
+void ProcessMiddlemanCall(const char* aInputData, size_t aInputSize,
+                          InfallibleVector<char>* aOutputData);
+
+// In the middleman process, reset all call state.
+void ResetMiddlemanCalls();
+
+///////////////////////////////////////////////////////////////////////////////
+// Middleman Call Helpers
+///////////////////////////////////////////////////////////////////////////////
+
+// Capture the contents of an input buffer at BufferArg with element count at CountArg.
+template <size_t BufferArg, size_t CountArg, typename ElemType>
+static inline void
+Middleman_Buffer(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+    auto byteSize = aCx.mArguments->Arg<CountArg, size_t>() * sizeof(ElemType);
+    aCx.ReadOrWritePrefaceBuffer(&buffer, byteSize);
+  }
+}
+
+// Capture the contents of a fixed size input buffer.
+template <size_t BufferArg, size_t ByteSize>
+static inline void
+Middleman_BufferFixedSize(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+    if (buffer) {
+      aCx.ReadOrWritePrefaceBuffer(&buffer, ByteSize);
+    }
+  }
+}
+
+// Capture a C string argument.
+template <size_t StringArg>
+static inline void
+Middleman_CString(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto& buffer = aCx.mArguments->Arg<StringArg, char*>();
+    size_t len = (aCx.mPhase == MiddlemanCallPhase::ReplayPreface) ? strlen(buffer) + 1 : 0;
+    aCx.ReadOrWritePrefaceBytes(&len, sizeof(len));
+    aCx.ReadOrWritePrefaceBuffer((void**) &buffer, len);
+  }
+}
+
+// Capture the data written to an output buffer at BufferArg with element count at CountArg.
+template <size_t BufferArg, size_t CountArg, typename ElemType>
+static inline void
+Middleman_WriteBuffer(MiddlemanCallContext& aCx)
+{
+  auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+  auto count = aCx.mArguments->Arg<CountArg, size_t>();
+  aCx.ReadOrWriteOutputBuffer(&buffer, count * sizeof(ElemType));
+}
+
+// Capture the data written to a fixed size output buffer.
+template <size_t BufferArg, size_t ByteSize>
+static inline void
+Middleman_WriteBufferFixedSize(MiddlemanCallContext& aCx)
+{
+  auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+  aCx.ReadOrWriteOutputBuffer(&buffer, ByteSize);
+}
+
+// Capture return values that are too large for register storage.
+template <size_t ByteSize>
+static inline void
+Middleman_OversizeRval(MiddlemanCallContext& aCx)
+{
+  Middleman_WriteBufferFixedSize<0, ByteSize>(aCx);
+}
+
+// Capture a byte count of stack argument data.
+template <size_t ByteSize>
+static inline void
+Middleman_StackArgumentData(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto stack = aCx.mArguments->StackAddress<0>();
+    aCx.ReadOrWritePrefaceBytes(stack, ByteSize);
+  }
+}
+
+static inline void
+Middleman_NoOp(MiddlemanCallContext& aCx)
+{
+}
+
+template <MiddlemanCallFn Fn0,
+          MiddlemanCallFn Fn1,
+          MiddlemanCallFn Fn2 = Middleman_NoOp,
+          MiddlemanCallFn Fn3 = Middleman_NoOp,
+          MiddlemanCallFn Fn4 = Middleman_NoOp>
+static inline void
+Middleman_Compose(MiddlemanCallContext& aCx)
+{
+  Fn0(aCx);
+  Fn1(aCx);
+  Fn2(aCx);
+  Fn3(aCx);
+  Fn4(aCx);
+}
+
+// Helper for capturing inputs that are produced by other middleman calls.
+// Returns false in the ReplayInput or MiddlemanInput phases if the input
+// system value could not be found.
+bool Middleman_SystemInput(MiddlemanCallContext& aCx, const void** aThingPtr);
+
+// Helper for capturing output system values that might be consumed by other
+// middleman calls.
+void Middleman_SystemOutput(MiddlemanCallContext& aCx, const void** aOutput, bool aUpdating = false);
+
+} // namespace recordreplay
+} // namespace mozilla
+
+#endif // mozilla_recordreplay_MiddlemanCall_h

toolkit/recordreplay/ProcessRecordReplay.cpp

@@ -111,6 +111,7 @@ RecordReplayInterface_Initialize(int aArgc, char* aArgv[])
   EarlyInitializeRedirections();
 
   if (!IsRecordingOrReplaying()) {
+    InitializeMiddlemanCalls();
     return;
   }
 
@@ -144,6 +145,7 @@ RecordReplayInterface_Initialize(int aArgc, char* aArgv[])
   Thread::SpawnAllThreads();
   InitializeCountdownThread();
   SetupDirtyMemoryHandler();
+  InitializeMiddlemanCalls();
 
   // Don't create a stylo thread pool when recording or replaying.
   putenv((char*) "STYLO_THREADS=1");

toolkit/recordreplay/ProcessRedirect.cpp

@@ -7,6 +7,7 @@
 #include "ProcessRedirect.h"
 
 #include "InfallibleVector.h"
+#include "MiddlemanCall.h"
 #include "mozilla/Sprintf.h"
 
 #include <dlfcn.h>
@@ -27,6 +28,24 @@ namespace recordreplay {
 // Redirection Skeleton
 ///////////////////////////////////////////////////////////////////////////////
 
+static bool
+CallPreambleHook(PreambleFn aPreamble, size_t aCallId, CallArguments* aArguments)
+{
+  PreambleResult result = aPreamble(aArguments);
+  switch (result) {
+  case PreambleResult::Veto:
+    return true;
+  case PreambleResult::PassThrough: {
+    AutoEnsurePassThroughThreadEvents pt;
+    RecordReplayInvokeCall(aCallId, aArguments);
+    return true;
+  }
+  case PreambleResult::Redirect:
+    return false;
+  }
+  Unreachable();
+}
+
 extern "C" {
 
 __attribute__((used)) int
@@ -37,42 +56,64 @@ RecordReplayInterceptCall(int aCallId, CallArguments* aArguments)
   // Call the preamble to see if this call needs special handling, even when
   // events have been passed through.
   if (redirection.mPreamble) {
-    PreambleResult result = redirection.mPreamble(aArguments);
-    switch (result) {
-    case PreambleResult::Veto:
+    if (CallPreambleHook(redirection.mPreamble, aCallId, aArguments)) {
       return 0;
-    case PreambleResult::PassThrough: {
-      AutoEnsurePassThroughThreadEvents pt;
-      RecordReplayInvokeCall(aCallId, aArguments);
-      return 0;
-    }
-    case PreambleResult::Redirect:
-      break;
     }
   }
 
   Thread* thread = Thread::Current();
+  Maybe<RecordingEventSection> res;
+  res.emplace(thread);
 
-  // When events are passed through, invoke the call with the original stack
-  // and register state.
-  if (!thread || thread->PassThroughEvents()) {
-    // RecordReplayRedirectCall will load the function to call from the
-    // return value slot.
-    aArguments->Rval<uint8_t*>() = redirection.mOriginalFunction;
-    return 1;
+  if (!res.ref().CanAccessEvents()) {
+    // When events are passed through, invoke the call with the original stack
+    // and register state.
+    if (!thread || thread->PassThroughEvents()) {
+      // RecordReplayRedirectCall will load the function to call from the
+      // return value slot.
+      aArguments->Rval<uint8_t*>() = redirection.mOriginalFunction;
+      return 1;
+    }
+
+    MOZ_RELEASE_ASSERT(thread->HasDivergedFromRecording());
+
+    // After we have diverged from the recording, we can't access the thread's
+    // recording anymore.
+
+    // If the redirection has a middleman preamble hook, call it to see if it
+    // can handle this call. The middleman preamble hook is separate from the
+    // normal preamble hook because entering the RecordingEventSection can
+    // cause the current thread to diverge from the recording; testing for
+    // HasDivergedFromRecording() does not work reliably in the normal preamble.
+    if (redirection.mMiddlemanPreamble) {
+      if (CallPreambleHook(redirection.mMiddlemanPreamble, aCallId, aArguments)) {
+        return 0;
+      }
+    }
+
+    // If the redirection has a middleman call hook, try to perform the call in
+    // the middleman instead.
+    if (redirection.mMiddlemanCall) {
+      if (SendCallToMiddleman(aCallId, aArguments, /* aPopulateOutput = */ true)) {
+        return 0;
+      }
+    }
+
+    // Calling any redirection which performs the standard steps will cause
+    // debugger operations that have diverged from the recording to fail.
+    EnsureNotDivergedFromRecording();
+    Unreachable();
   }
 
-  // Calling any redirection which performs the standard steps will cause
-  // debugger operations that have diverged from the recording to fail.
-  EnsureNotDivergedFromRecording();
-
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
-
   if (IsRecording()) {
     // Call the original function, passing through events while we do so.
+    // Destroy the RecordingEventSection so that we don't prevent the file
+    // from being flushed in case we end up blocking.
+    res.reset();
     thread->SetPassThrough(true);
     RecordReplayInvokeCall(aCallId, aArguments);
     thread->SetPassThrough(false);
+    res.emplace(thread);
   }
 
   // Save any system error in case we want to record/replay it.
@@ -86,6 +127,13 @@ RecordReplayInterceptCall(int aCallId, CallArguments* aArguments)
     redirection.mSaveOutput(thread->Events(), aArguments, &error);
   }
 
+  // Save information about any potential middleman calls encountered if we
+  // haven't diverged from the recording, in case we diverge and later calls
+  // access data produced by this one.
+  if (IsReplaying() && redirection.mMiddlemanCall) {
+    (void) SendCallToMiddleman(aCallId, aArguments, /* aDiverged = */ false);
+  }
+
   RestoreError(error);
   return 0;
 }

toolkit/recordreplay/ProcessRedirect.h

@@ -13,7 +13,6 @@
 #include "ProcessRecordReplay.h"
 #include "ProcessRewind.h"
 #include "Thread.h"
-#include "ipc/Channel.h"
 
 #include "mozilla/Assertions.h"
 #include "mozilla/Atomics.h"
@@ -74,18 +73,11 @@ namespace recordreplay {
 // Function Redirections
 ///////////////////////////////////////////////////////////////////////////////
 
-// Capture the arguments that can be passed to a redirection, and provide
-// storage to specify the redirection's return value. We only need to capture
-// enough argument data here for calls made directly from Gecko code,
-// i.e. where events are not passed through. Calls made while events are passed
-// through are performed with the same stack and register state as when they
-// were initially invoked.
-//
-// Arguments and return value indexes refer to the register contents as passed
-// to the function originally. For functions with complex or floating point
-// arguments and return values, the right index to use might be different than
-// expected, per the requirements of the System V x64 ABI.
-struct CallArguments
+struct CallArguments;
+
+// All argument and return value data that is stored in registers and whose
+// values are preserved when calling a redirected function.
+struct CallRegisterArguments
 {
 protected:
   size_t arg0;      // 0
@@ -101,6 +93,28 @@ protected:
   size_t rval1;     // 80
   double floatrval0; // 88
   double floatrval1; // 96
+                     // Size: 104
+
+public:
+  void CopyFrom(const CallRegisterArguments* aArguments);
+  void CopyTo(CallRegisterArguments* aArguments) const;
+  void CopyRvalFrom(const CallRegisterArguments* aArguments);
+};
+
+// Capture the arguments that can be passed to a redirection, and provide
+// storage to specify the redirection's return value. We only need to capture
+// enough argument data here for calls made directly from Gecko code,
+// i.e. where events are not passed through. Calls made while events are passed
+// through are performed with the same stack and register state as when they
+// were initially invoked.
+//
+// Arguments and return value indexes refer to the register contents as passed
+// to the function originally. For functions with complex or floating point
+// arguments and return values, the right index to use might be different than
+// expected, per the requirements of the System V x64 ABI.
+struct CallArguments : public CallRegisterArguments
+{
+protected:
   size_t stack[64]; // 104
                     // Size: 616
 
@@ -121,6 +135,12 @@ public:
     }
   }
 
+  template <size_t Offset>
+  size_t* StackAddress() {
+    static_assert(Offset % sizeof(size_t) == 0, "Bad stack offset");
+    return &stack[Offset / sizeof(size_t)];
+  }
+
   template <typename T, size_t Index = 0>
   T& Rval() {
     static_assert(sizeof(T) == sizeof(size_t), "Size must match");
@@ -142,6 +162,27 @@ public:
   }
 };
 
+inline void
+CallRegisterArguments::CopyFrom(const CallRegisterArguments* aArguments)
+{
+  memcpy(this, aArguments, sizeof(CallRegisterArguments));
+}
+
+inline void
+CallRegisterArguments::CopyTo(CallRegisterArguments* aArguments) const
+{
+  memcpy(aArguments, this, sizeof(CallRegisterArguments));
+}
+
+inline void
+CallRegisterArguments::CopyRvalFrom(const CallRegisterArguments* aArguments)
+{
+  rval0 = aArguments->rval0;
+  rval1 = aArguments->rval1;
+  floatrval0 = aArguments->floatrval0;
+  floatrval1 = aArguments->floatrval1;
+}
+
 // Generic type for a system error code.
 typedef ssize_t ErrorType;
 
@@ -164,6 +205,15 @@ enum class PreambleResult {
   PassThrough
 };
 
+// Signature for a function that is called on entry to a redirection and can
+// modify its behavior.
+typedef PreambleResult (*PreambleFn)(CallArguments* aArguments);
+
+// Signature for a function that conveys data about a call to or from the
+// middleman process.
+struct MiddlemanCallContext;
+typedef void (*MiddlemanCallFn)(MiddlemanCallContext& aCx);
+
 // Information about a system library API function which is being redirected.
 struct Redirection
 {
@@ -187,7 +237,15 @@ struct Redirection
   SaveOutputFn mSaveOutput;
 
   // If specified, will be called upon entry to the redirected call.
-  PreambleResult (*mPreamble)(CallArguments* aArguments);
+  PreambleFn mPreamble;
+
+  // If specified, will be called while replaying and diverged from the
+  // recording to perform this call in the middleman process.
+  MiddlemanCallFn mMiddlemanCall;
+
+  // Additional preamble that is only called while replaying and diverged from
+  // the recording.
+  PreambleFn mMiddlemanPreamble;
 };
 
 // All platform specific redirections, indexed by the call event.

toolkit/recordreplay/ProcessRewind.cpp

@@ -193,7 +193,13 @@ DivergeFromRecording()
 
   Thread* thread = Thread::Current();
   MOZ_RELEASE_ASSERT(thread->IsMainThread());
-  thread->DivergeFromRecording();
+
+  if (!thread->HasDivergedFromRecording()) {
+    // Reset middleman call state whenever we first diverge from the recording.
+    child::SendResetMiddlemanCalls();
+
+    thread->DivergeFromRecording();
+  }
 
   gUnhandledDivergeAllowed = true;
 }

toolkit/recordreplay/moz.build

@@ -26,6 +26,7 @@ if CONFIG['OS_ARCH'] == 'Darwin' and CONFIG['NIGHTLY_BUILD']:
         'ipc/ParentIPC.cpp',
         'Lock.cpp',
         'MemorySnapshot.cpp',
+        'MiddlemanCall.cpp',
         'ProcessRecordReplay.cpp',
         'ProcessRedirectDarwin.cpp',
         'ProcessRewind.cpp',