Bug 1525086 - Part 3b - Start the RDD sandbox earlier r=Alex_Gaynor

Start the RDD process earlier by changing RDDProcessHost to pass the necessary command line arguments for enabling the sandbox.

Per lsmp output on 10.14.3, starting the RDD process sandbox removes access to WindowServer, coreservicesd, lsd and distnoted.

Add a pref (defaulting to on) to control enabling starting the RDD process earlier.

Differential Revision: https://phabricator.services.mozilla.com/D23460

--HG--
extra : moz-landing-system : lando

browser/app/profile/firefox.js

@@ -1054,6 +1054,8 @@ pref("security.sandbox.gmp.win32k-disable", false);
 // Start the Mac sandbox early during child process startup instead
 // of when messaged by the parent after the message loop is running.
 pref("security.sandbox.content.mac.earlyinit", true);
+// Remove this pref once RDD early init is stable on Release.
+pref("security.sandbox.rdd.mac.earlyinit", true);
 
 // This pref is discussed in bug 1083344, the naming is inspired from its
 // Windows counterpart, but on Mac it's an integer which means:

dom/media/ipc/PRDD.ipdl

@@ -20,8 +20,7 @@ protocol PRDD
 {
 parent:
 
-  // args TBD, sent by UI process to initiate core settings
-  async Init(FileDescriptor? sandboxBroker);
+  async Init(FileDescriptor? sandboxBroker, bool startMacSandbox);
 
   async InitProfiler(Endpoint<PProfilerChild> endpoint);
 

dom/media/ipc/RDDChild.cpp

@@ -28,7 +28,7 @@ RDDChild::RDDChild(RDDProcessHost* aHost) : mHost(aHost), mRDDReady(false) {
 
 RDDChild::~RDDChild() { MOZ_COUNT_DTOR(RDDChild); }
 
-bool RDDChild::Init() {
+bool RDDChild::Init(bool aStartMacSandbox) {
   Maybe<FileDescriptor> brokerFd;
 
 #if defined(XP_LINUX) && defined(MOZ_SANDBOX)
@@ -46,7 +46,7 @@ bool RDDChild::Init() {
   }
 #endif  // XP_LINUX && MOZ_SANDBOX
 
-  SendInit(brokerFd);
+  SendInit(brokerFd, aStartMacSandbox);
 
 #ifdef MOZ_GECKO_PROFILER
   Unused << SendInitProfiler(ProfilerParent::CreateForProcess(OtherPid()));

dom/media/ipc/RDDChild.h

@@ -32,7 +32,7 @@ class RDDChild final : public PRDDChild {
   explicit RDDChild(RDDProcessHost* aHost);
   ~RDDChild();
 
-  bool Init();
+  bool Init(bool aStartMacSandbox);
 
   bool EnsureRDDReady();
 

dom/media/ipc/RDDParent.cpp

@@ -31,6 +31,7 @@
 #  include "mozilla/Sandbox.h"
 #  include "nsMacUtilsImpl.h"
 #  include <Carbon/Carbon.h>  // for CGSSetDenyWindowServerConnections
+#  include "RDDProcessHost.h"
 #endif
 
 #include "nsDebugImpl.h"
@@ -93,11 +94,6 @@ void CGSShutdownServerConnections();
 };
 
 static void StartRDDMacSandbox() {
-  // Close all current connections to the WindowServer. This ensures that the
-  // Activity Monitor will not label the content process as "Not responding"
-  // because it's not running a native event loop. See bug 1384336.
-  CGSShutdownServerConnections();
-
   // Actual security benefits are only acheived when we additionally deny
   // future connections.
   CGError result = CGSSetDenyWindowServerConnections(true);
@@ -106,14 +102,8 @@ static void StartRDDMacSandbox() {
   Unused << result;
 #  endif
 
-  nsAutoCString appPath;
-  nsMacUtilsImpl::GetAppPath(appPath);
-
   MacSandboxInfo info;
-  info.type = MacSandboxType_Utility;
-  info.shouldLog = Preferences::GetBool("security.sandbox.logging.enabled") ||
-                   PR_GetEnv("MOZ_SANDBOX_LOGGING");
-  info.appPath.assign(appPath.get());
+  RDDProcessHost::StaticFillMacSandboxInfo(info);
 
   std::string err;
   bool rv = mozilla::StartMacSandbox(info, err);
@@ -125,11 +115,18 @@ static void StartRDDMacSandbox() {
 #endif
 
 mozilla::ipc::IPCResult RDDParent::RecvInit(
-    const Maybe<FileDescriptor>& aBrokerFd) {
+    const Maybe<FileDescriptor>& aBrokerFd, bool aStartMacSandbox) {
   Unused << SendInitComplete();
 #if defined(MOZ_SANDBOX)
 #  if defined(XP_MACOSX)
-  StartRDDMacSandbox();
+  // Close all current connections to the WindowServer. This ensures that the
+  // Activity Monitor will not label the content process as "Not responding"
+  // because it's not running a native event loop. See bug 1384336.
+  CGSShutdownServerConnections();
+
+  if (aStartMacSandbox) {
+    StartRDDMacSandbox();
+  }
 #  elif defined(XP_LINUX)
   int fd = -1;
   if (aBrokerFd.isSome()) {

dom/media/ipc/RDDParent.h

@@ -24,7 +24,7 @@ class RDDParent final : public PRDDParent {
   bool Init(base::ProcessId aParentPid, const char* aParentBuildID,
             MessageLoop* aIOLoop, IPC::Channel* aChannel);
 
-  mozilla::ipc::IPCResult RecvInit(const Maybe<ipc::FileDescriptor>& aBrokerFd);
+  mozilla::ipc::IPCResult RecvInit(const Maybe<ipc::FileDescriptor>& aBrokerFd, bool aStartMacSandbox);
   mozilla::ipc::IPCResult RecvInitProfiler(
       Endpoint<PProfilerChild>&& aEndpoint);
 

dom/media/ipc/RDDProcessHost.cpp

@@ -11,10 +11,19 @@
 
 #include "RDDChild.h"
 
+#if defined(XP_MACOSX)
+#  include "mozilla/Sandbox.h"
+#  include "nsMacUtilsImpl.h"
+#endif
+
 namespace mozilla {
 
 using namespace ipc;
 
+#if defined(XP_MACOSX) && defined(MOZ_SANDBOX)
+bool RDDProcessHost::sLaunchWithMacSandbox = false;
+#endif
+
 RDDProcessHost::RDDProcessHost(Listener* aListener)
     : GeckoChildProcessHost(GeckoProcessType_RDD),
       mListener(aListener),
@@ -24,6 +33,17 @@ RDDProcessHost::RDDProcessHost(Listener* aListener)
       mShutdownRequested(false),
       mChannelClosed(false) {
   MOZ_COUNT_CTOR(RDDProcessHost);
+
+#if defined(XP_MACOSX) && defined(MOZ_SANDBOX)
+  // sLaunchWithMacSandbox is statically initialized to false.
+  // Once we've set it to true due to the pref, avoid checking the
+  // pref on subsequent calls. As a result, changing the earlyinit
+  // pref requires restarting the browser to take effect.
+  if (!sLaunchWithMacSandbox) {
+    sLaunchWithMacSandbox =
+        Preferences::GetBool("security.sandbox.rdd.mac.earlyinit");
+  }
+#endif
 }
 
 RDDProcessHost::~RDDProcessHost() { MOZ_COUNT_DTOR(RDDProcessHost); }
@@ -128,7 +148,15 @@ void RDDProcessHost::InitAfterConnect(bool aSucceeded) {
         mRDDChild->Open(GetChannel(), base::GetProcId(GetChildProcessHandle()));
     MOZ_ASSERT(rv);
 
-    if (!mRDDChild->Init()) {
+    bool startMacSandbox = false;
+
+#if defined(XP_MACOSX) && defined(MOZ_SANDBOX)
+    // If the sandbox was started at launch time,
+    // do not start the sandbox again.
+    startMacSandbox = !sLaunchWithMacSandbox;
+#endif
+
+    if (!mRDDChild->Init(startMacSandbox)) {
       KillHard("ActorInitFailed");
     }
   }
@@ -211,4 +239,23 @@ void RDDProcessHost::DestroyProcess() {
       NS_NewRunnableFunction("DestroyProcessRunnable", [this] { Destroy(); }));
 }
 
+#if defined(XP_MACOSX) && defined(MOZ_SANDBOX)
+/* static */
+void RDDProcessHost::StaticFillMacSandboxInfo(MacSandboxInfo& aInfo) {
+  GeckoChildProcessHost::StaticFillMacSandboxInfo(aInfo);
+  if (!aInfo.shouldLog && PR_GetEnv("MOZ_SANDBOX_RDD_LOGGING")) {
+    aInfo.shouldLog = true;
+  }
+}
+
+void RDDProcessHost::FillMacSandboxInfo(MacSandboxInfo& aInfo) {
+  RDDProcessHost::StaticFillMacSandboxInfo(aInfo);
+}
+
+/* static */
+MacSandboxType RDDProcessHost::GetMacSandboxType() {
+  return GeckoChildProcessHost::GetDefaultMacSandboxType();
+}
+#endif
+
 }  // namespace mozilla

dom/media/ipc/RDDProcessHost.h

@@ -90,6 +90,16 @@ class RDDProcessHost final : public mozilla::ipc::GeckoChildProcessHost {
   // Used for tests and diagnostics
   void KillProcess();
 
+#if defined(XP_MACOSX) && defined(MOZ_SANDBOX)
+  // To allow filling a MacSandboxInfo from the child
+  // process without an instance of RDDProcessHost.
+  // Only needed for late-start sandbox enabling.
+  static void StaticFillMacSandboxInfo(MacSandboxInfo& aInfo);
+
+  // Return the sandbox type to be used with this process type.
+  static MacSandboxType GetMacSandboxType();
+#endif
+
  private:
   ~RDDProcessHost();
 
@@ -108,6 +118,16 @@ class RDDProcessHost final : public mozilla::ipc::GeckoChildProcessHost {
 
   void DestroyProcess();
 
+#if defined(XP_MACOSX) && defined(MOZ_SANDBOX)
+  static bool sLaunchWithMacSandbox;
+
+  // Sandbox the RDD process at launch for all instances
+  bool IsMacSandboxLaunchEnabled() override { return sLaunchWithMacSandbox; }
+
+  // Override so we can turn on RDD process-specific sandbox logging
+  void FillMacSandboxInfo(MacSandboxInfo& aInfo) override;
+#endif
+
   DISALLOW_COPY_AND_ASSIGN(RDDProcessHost);
 
   Listener* mListener;

ipc/glue/GeckoChildProcessHost.cpp

@@ -39,6 +39,7 @@
 #include "mozilla/ipc/EnvironmentMap.h"
 #include "mozilla/Omnijar.h"
 #include "mozilla/RecordReplay.h"
+#include "mozilla/RDDProcessHost.h"
 #include "mozilla/Scoped.h"
 #include "mozilla/Services.h"
 #include "mozilla/SharedThreadPool.h"
@@ -1437,6 +1438,9 @@ bool GeckoChildProcessHost::StartMacSandbox(int aArgc, char** aArgv,
       // to configure sandboxing so hard code the sandbox type.
       sandboxType = MacSandboxType_Content;
       break;
+    case GeckoProcessType_RDD:
+      sandboxType = RDDProcessHost::GetMacSandboxType();
+      break;
     default:
       return true;
   }