Bug 1646453: Add MSCOM channel hook to enable profiler markers for COM IPC; r=Jamie,gerald

To intercept COM IPC, we provide an `IChannelHook` interface to
`CoRegisterChannelHook`, which gives us notifications about COM IPC that we can
use to insert profiler markers. Note that `IChannelHook` is not documented on
MSDN, however it is defined in the SDK header files.

When the profiler is available, once XPCOM is up:

* If the profiler is active, we immediately register the channel hook;
* Otherwise we register an observer and hold off on registering the hook until
  the profiler is started, at which point we register the hook and remove the
  observer.

Differential Revision: https://phabricator.services.mozilla.com/D80053

ipc/mscom/ProfilerMarkers.cpp

@@ -0,0 +1,212 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "ProfilerMarkers.h"
+
+#include "GeckoProfiler.h"
+#include "MainThreadUtils.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Services.h"
+#include "nsCOMPtr.h"
+#include "nsIObserver.h"
+#include "nsIObserverService.h"
+#include "nsISupportsImpl.h"
+#include "nsXULAppAPI.h"
+
+#include <objbase.h>
+#include <objidlbase.h>
+
+// {9DBE6B28-E5E7-4FDE-AF00-9404604E74DC}
+static const GUID GUID_MozProfilerMarkerExtension = {
+    0x9dbe6b28, 0xe5e7, 0x4fde, {0xaf, 0x0, 0x94, 0x4, 0x60, 0x4e, 0x74, 0xdc}};
+
+static ULONG gMainThreadORPCDepth = 0UL;
+
+namespace {
+
+class ProfilerMarkerChannelHook final : public IChannelHook {
+  ~ProfilerMarkerChannelHook() = default;
+
+ public:
+  STDMETHODIMP QueryInterface(REFIID aIid, void** aOutInterface) override;
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ProfilerMarkerChannelHook, override)
+
+  /**
+   * IChannelHook exposes six methods: The Client* methods are called when
+   * a client is sending an IPC request, whereas the Server* methods are called
+   * when a server is receiving an IPC request.
+   *
+   * For our purposes, we only care about the client-side methods. The COM
+   * runtime invokes the methods in the following order:
+   * 1. ClientGetSize, where the hook specifies the size of its payload;
+   * 2. ClientFillBuffer, where the hook fills the channel's buffer with its
+   *    payload information. NOTE: This method is only called when ClientGetSize
+   *    specifies a non-zero payload size. For our purposes, since we are not
+   *    sending a payload, this method will never be called!
+   * 3. ClientNotify, when the response has been received from the server.
+   *
+   * Since we want to use these hooks to record the beginning and end of a COM
+   * IPC call, we use ClientGetSize for logging the start, and ClientNotify for
+   * logging the end.
+   *
+   * Finally, our implementation responds to any request matching our extension
+   * ID, however we only care about main thread COM calls.
+   *
+   * Further note that COM allows re-entrancy, however for our purposes we only
+   * care about the outermost IPC call on the main thread, so we use the
+   * gMainThreadORPCDepth variable to track that.
+   */
+
+  // IChannelHook
+  STDMETHODIMP_(void)
+  ClientGetSize(REFGUID aExtensionId, REFIID aIid,
+                ULONG* aOutDataSize) override;
+
+  // No-op (see the large comment above)
+  STDMETHODIMP_(void)
+  ClientFillBuffer(REFGUID aExtensionId, REFIID aIid, ULONG* aDataSize,
+                   void* aDataBuf) override {}
+
+  STDMETHODIMP_(void)
+  ClientNotify(REFGUID aExtensionId, REFIID aIid, ULONG aDataSize,
+               void* aDataBuffer, DWORD aDataRep, HRESULT aFault) override;
+
+  // We don't care about the server-side notifications, so leave as no-ops.
+  STDMETHODIMP_(void)
+  ServerNotify(REFGUID aExtensionId, REFIID aIid, ULONG aDataSize,
+               void* aDataBuf, DWORD aDataRep) override {}
+
+  STDMETHODIMP_(void)
+  ServerGetSize(REFGUID aExtensionId, REFIID aIid, HRESULT aFault,
+                ULONG* aOutDataSize) override {}
+
+  STDMETHODIMP_(void)
+  ServerFillBuffer(REFGUID aExtensionId, REFIID aIid, ULONG* aDataSize,
+                   void* aDataBuf, HRESULT aFault) override {}
+};
+
+HRESULT ProfilerMarkerChannelHook::QueryInterface(REFIID aIid,
+                                                  void** aOutInterface) {
+  if (aIid == IID_IChannelHook || aIid == IID_IUnknown) {
+    RefPtr<IChannelHook> ptr(this);
+    ptr.forget(aOutInterface);
+    return S_OK;
+  }
+
+  return E_NOINTERFACE;
+}
+
+void ProfilerMarkerChannelHook::ClientGetSize(REFGUID aExtensionId, REFIID aIid,
+                                              ULONG* aOutDataSize) {
+  if (aExtensionId == GUID_MozProfilerMarkerExtension) {
+    if (NS_IsMainThread()) {
+      if (!gMainThreadORPCDepth) {
+        PROFILER_TRACING_MARKER("MSCOM", "ORPC Call", IPC,
+                                TRACING_INTERVAL_START);
+      }
+
+      ++gMainThreadORPCDepth;
+    }
+
+    if (aOutDataSize) {
+      // We don't add any payload data to the channel
+      *aOutDataSize = 0UL;
+    }
+  }
+}
+
+void ProfilerMarkerChannelHook::ClientNotify(REFGUID aExtensionId, REFIID aIid,
+                                             ULONG aDataSize, void* aDataBuffer,
+                                             DWORD aDataRep, HRESULT aFault) {
+  if (aExtensionId == GUID_MozProfilerMarkerExtension && NS_IsMainThread()) {
+    MOZ_ASSERT(gMainThreadORPCDepth > 0);
+    --gMainThreadORPCDepth;
+    if (!gMainThreadORPCDepth) {
+      PROFILER_TRACING_MARKER("MSCOM", "ORPC Call", IPC, TRACING_INTERVAL_END);
+    }
+  }
+}
+
+}  // anonymous namespace
+
+static void RegisterChannelHook() {
+  RefPtr<ProfilerMarkerChannelHook> hook(new ProfilerMarkerChannelHook());
+  DebugOnly<HRESULT> hr =
+      ::CoRegisterChannelHook(GUID_MozProfilerMarkerExtension, hook);
+  MOZ_ASSERT(SUCCEEDED(hr));
+}
+
+namespace {
+
+class ProfilerStartupObserver final : public nsIObserver {
+  ~ProfilerStartupObserver() = default;
+
+ public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIOBSERVER
+};
+
+NS_IMPL_ISUPPORTS(ProfilerStartupObserver, nsIObserver)
+
+NS_IMETHODIMP ProfilerStartupObserver::Observe(nsISupports* aSubject,
+                                               const char* aTopic,
+                                               const char16_t* aData) {
+  if (strcmp(aTopic, "profiler-started")) {
+    return NS_OK;
+  }
+
+  RegisterChannelHook();
+
+  // Once we've set the channel hook, we don't care about this notification
+  // anymore; our channel hook will remain set for the lifetime of the process.
+  nsCOMPtr<nsIObserverService> obsServ(mozilla::services::GetObserverService());
+  MOZ_ASSERT(!!obsServ);
+  if (!obsServ) {
+    return NS_OK;
+  }
+
+  obsServ->RemoveObserver(this, "profiler-started");
+  return NS_OK;
+}
+
+}  // anonymous namespace
+
+namespace mozilla {
+namespace mscom {
+
+void InitProfilerMarkers() {
+  if (!XRE_IsParentProcess()) {
+    return;
+  }
+
+  MOZ_ASSERT(NS_IsMainThread());
+  if (!NS_IsMainThread()) {
+    return;
+  }
+
+  if (profiler_is_active()) {
+    // If the profiler is already running, we'll immediately register our
+    // channel hook.
+    RegisterChannelHook();
+    return;
+  }
+
+  // The profiler is not running yet. To avoid unnecessary invocations of the
+  // channel hook, we won't bother with installing it until the profiler starts.
+  // Set up an observer to watch for this.
+  nsCOMPtr<nsIObserverService> obsServ(mozilla::services::GetObserverService());
+  MOZ_ASSERT(!!obsServ);
+  if (!obsServ) {
+    return;
+  }
+
+  nsCOMPtr<nsIObserver> obs(new ProfilerStartupObserver());
+  obsServ->AddObserver(obs, "profiler-started", false);
+}
+
+}  // namespace mscom
+}  // namespace mozilla

ipc/mscom/ProfilerMarkers.h

@@ -0,0 +1,18 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_mscom_ProfilerMarkers_h
+#define mozilla_mscom_ProfilerMarkers_h
+
+namespace mozilla {
+namespace mscom {
+
+void InitProfilerMarkers();
+
+}  // namespace mscom
+}  // namespace mozilla
+
+#endif  // mozilla_mscom_ProfilerMarkers_h

ipc/mscom/moz.build

@@ -14,6 +14,7 @@ EXPORTS.mozilla.mscom += [
     'Objref.h',
     'PassthruProxy.h',
     'ProcessRuntime.h',
+    'ProfilerMarkers.h',
     'ProxyStream.h',
     'Ptr.h',
     'Utils.h',
@@ -33,6 +34,7 @@ UNIFIED_SOURCES += [
     'Objref.cpp',
     'PassthruProxy.cpp',
     'ProcessRuntime.cpp',
+    'ProfilerMarkers.cpp',
     'ProxyStream.cpp',
     'RegistrationAnnotator.cpp',
     'Utils.cpp',

toolkit/xre/nsAppRunner.cpp

@@ -105,6 +105,9 @@
 #  include "mozilla/WindowsProcessMitigations.h"
 #  include "mozilla/WinHeaderOnlyUtils.h"
 #  include "mozilla/mscom/ProcessRuntime.h"
+#  if defined(MOZ_GECKO_PROFILER)
+#    include "mozilla/mscom/ProfilerMarkers.h"
+#  endif  // defined(MOZ_GECKO_PROFILER)
 #  include "mozilla/widget/AudioSession.h"
 #  include "WinTokenUtils.h"
 
@@ -4256,7 +4259,11 @@ nsresult XREMain::XRE_mainRun() {
   dllServices->StartUntrustedModulesProcessor();
   auto dllServicesDisable =
       MakeScopeExit([&dllServices]() { dllServices->DisableFull(); });
-#endif  // defined(XP_WIN)
+
+#  if defined(MOZ_GECKO_PROFILER)
+  mozilla::mscom::InitProfilerMarkers();
+#  endif  // defined(MOZ_GECKO_PROFILER)
+#endif    // defined(XP_WIN)
 
 #ifdef NS_FUNCTION_TIMER
   // initialize some common services, so we don't pay the cost for these at odd