Bug 1160142 - For e10s plugin hangs take the minidump of the browser process before we message the chrome UI about the hang. r=billm

browser/modules/ProcessHangMonitor.jsm

@@ -313,6 +313,7 @@ let ProcessHangMonitor = {
     for (let [otherReport, otherTimer] of this._activeReports) {
       if (otherTimer === timer) {
         this.removeReport(otherReport);
+        otherReport.userCanceled();
         break;
       }
     }

dom/ipc/ProcessHangMonitor.cpp

@@ -22,6 +22,9 @@
 #include "nsITabParent.h"
 #include "nsPluginHost.h"
 #include "nsThreadUtils.h"
+#ifdef MOZ_CRASHREPORTER
+#include "nsExceptionHandler.h"
+#endif
 
 #include "base/task.h"
 #include "base/thread.h"
@@ -149,12 +152,20 @@ public:
   NS_IMETHOD EndStartingDebugger() override;
   NS_IMETHOD TerminatePlugin() override;
   NS_IMETHOD TerminateProcess() override;
+  NS_IMETHOD UserCanceled() override;
 
   NS_IMETHOD IsReportForBrowser(nsIFrameLoader* aFrameLoader, bool* aResult) override;
 
-  void Clear() { mContentParent = nullptr; mActor = nullptr; }
+  // Called on xpcom shutdown
+  void Clear() {
+    mContentParent = nullptr;
+    mActor = nullptr;
+  }
 
   void SetHangData(const HangData& aHangData) { mHangData = aHangData; }
+  void SetBrowserDumpId(nsAutoString& aId) {
+    mBrowserDumpId = aId;
+  }
 
 private:
   ~HangMonitoredProcess() {}
@@ -163,6 +174,7 @@ private:
   HangMonitorParent* mActor;
   ContentParent* mContentParent;
   HangData mHangData;
+  nsAutoString mBrowserDumpId;
 };
 
 class HangMonitorParent
@@ -185,6 +197,7 @@ public:
   void TerminateScript();
   void BeginStartingDebugger();
   void EndStartingDebugger();
+  void CleanupPluginHang(uint32_t aPluginId, bool aRemoveFiles);
 
   MessageLoop* MonitorLoop() { return mHangMonitor->MonitorLoop(); }
 
@@ -204,6 +217,9 @@ public:
   // Must be accessed with mMonitor held.
   nsRefPtr<HangMonitoredProcess> mProcess;
   bool mShutdownDone;
+  // Map from plugin ID to crash dump ID. Protected by mBrowserCrashDumpHashLock.
+  nsDataHashtable<nsUint32HashKey, nsString> mBrowserCrashDumpIds;
+  Mutex mBrowserCrashDumpHashLock;
 };
 
 } // namespace
@@ -389,10 +405,12 @@ HangMonitorChild::IsDebuggerStartupComplete()
 void
 HangMonitorChild::NotifyPluginHang(uint32_t aPluginId)
 {
+  // main thread in the child
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
 
   mSentReport = true;
 
+  // bounce to background thread
   MonitorLoop()->PostTask(
     FROM_HERE,
     NewRunnableMethod(this,
@@ -405,6 +423,7 @@ HangMonitorChild::NotifyPluginHangAsync(uint32_t aPluginId)
 {
   MOZ_RELEASE_ASSERT(MessageLoop::current() == MonitorLoop());
 
+  // bounce back to parent on background thread
   if (mIPCOpen) {
     unused << SendHangEvidence(PluginHangData(aPluginId));
   }
@@ -430,7 +449,8 @@ HangMonitorParent::HangMonitorParent(ProcessHangMonitor* aMonitor)
  : mHangMonitor(aMonitor),
    mIPCOpen(true),
    mMonitor("HangMonitorParent lock"),
-   mShutdownDone(false)
+   mShutdownDone(false),
+   mBrowserCrashDumpHashLock("mBrowserCrashDumpIds lock")
 {
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
   mReportHangs = mozilla::Preferences::GetBool("dom.ipc.reportProcessHangs", false);
@@ -501,16 +521,21 @@ HangMonitorParent::Open(Transport* aTransport, ProcessId aPid,
 class HangObserverNotifier final : public nsRunnable
 {
 public:
-  HangObserverNotifier(HangMonitoredProcess* aProcess, const HangData& aHangData)
+  HangObserverNotifier(HangMonitoredProcess* aProcess,
+                       const HangData& aHangData,
+                       const nsString& aBrowserDumpId)
     : mProcess(aProcess),
-      mHangData(aHangData)
+      mHangData(aHangData),
+      mBrowserDumpId(aBrowserDumpId)
   {}
 
   NS_IMETHOD
   Run()
   {
+    // chrome process, main thread
     MOZ_RELEASE_ASSERT(NS_IsMainThread());
     mProcess->SetHangData(mHangData);
+    mProcess->SetBrowserDumpId(mBrowserDumpId);
 
     nsCOMPtr<nsIObserverService> observerService =
       mozilla::services::GetObserverService();
@@ -521,11 +546,13 @@ public:
 private:
   nsRefPtr<HangMonitoredProcess> mProcess;
   HangData mHangData;
+  nsAutoString mBrowserDumpId;
 };
 
 bool
 HangMonitorParent::RecvHangEvidence(const HangData& aHangData)
 {
+  // chrome process, background thread
   MOZ_RELEASE_ASSERT(MessageLoop::current() == MonitorLoop());
 
   if (!mReportHangs) {
@@ -540,11 +567,33 @@ HangMonitorParent::RecvHangEvidence(const HangData& aHangData)
   }
 #endif
 
+  // Before we wake up the browser main thread we want to take a
+  // browser minidump.
+  nsAutoString crashId;
+#ifdef MOZ_CRASHREPORTER
+  if (aHangData.type() == HangData::TPluginHangData) {
+    MutexAutoLock lock(mBrowserCrashDumpHashLock);
+    const PluginHangData& phd = aHangData.get_PluginHangData();
+    if (!mBrowserCrashDumpIds.Get(phd.pluginId(), &crashId)) {
+      nsCOMPtr<nsIFile> browserDump;
+      if (CrashReporter::TakeMinidump(getter_AddRefs(browserDump), true)) {
+        if (!CrashReporter::GetIDFromMinidump(browserDump, crashId) || crashId.IsEmpty()) {
+          browserDump->Remove(false);
+          NS_WARNING("Failed to generate timely browser stack, this is bad for plugin hang analysis!");
+        } else {
+          mBrowserCrashDumpIds.Put(phd.pluginId(), crashId);
+        }
+      }
+    }
+  }
+#endif
+
   mHangMonitor->InitiateCPOWTimeout();
 
   MonitorAutoLock lock(mMonitor);
 
-  nsCOMPtr<nsIRunnable> notifier = new HangObserverNotifier(mProcess, aHangData);
+  nsCOMPtr<nsIRunnable> notifier =
+    new HangObserverNotifier(mProcess, aHangData, crashId);
   NS_DispatchToMainThread(notifier);
 
   return true;
@@ -580,6 +629,22 @@ HangMonitorParent::EndStartingDebugger()
   }
 }
 
+void
+HangMonitorParent::CleanupPluginHang(uint32_t aPluginId, bool aRemoveFiles)
+{
+  MutexAutoLock lock(mBrowserCrashDumpHashLock);
+  nsAutoString crashId;
+  if (!mBrowserCrashDumpIds.Get(aPluginId, &crashId)) {
+    return;
+  }
+  mBrowserCrashDumpIds.Remove(aPluginId);
+#ifdef MOZ_CRASHREPORTER
+  if (aRemoveFiles && !crashId.IsEmpty()) {
+    CrashReporter::DeleteMinidumpFilesForID(crashId);
+  }
+#endif
+}
+
 /* HangMonitoredProcess implementation */
 
 NS_IMPL_ISUPPORTS(HangMonitoredProcess, nsIHangReport)
@@ -738,7 +803,9 @@ HangMonitoredProcess::TerminatePlugin()
   }
 
   uint32_t id = mHangData.get_PluginHangData().pluginId();
-  plugins::TerminatePlugin(id);
+  plugins::TerminatePlugin(id, mBrowserDumpId);
+
+  mActor->CleanupPluginHang(id, false);
   return NS_OK;
 }
 
@@ -751,6 +818,11 @@ HangMonitoredProcess::TerminateProcess()
     return NS_ERROR_UNEXPECTED;
   }
 
+  if (mHangData.type() == HangData::TPluginHangData) {
+    uint32_t id = mHangData.get_PluginHangData().pluginId();
+    mActor->CleanupPluginHang(id, true);
+  }
+
   mContentParent->KillHard("HangMonitor");
   return NS_OK;
 }
@@ -775,6 +847,19 @@ HangMonitoredProcess::IsReportForBrowser(nsIFrameLoader* aFrameLoader, bool* aRe
   return NS_OK;
 }
 
+NS_IMETHODIMP
+HangMonitoredProcess::UserCanceled()
+{
+  MOZ_RELEASE_ASSERT(NS_IsMainThread());
+  if (mHangData.type() != HangData::TPluginHangData) {
+    return NS_OK;
+  }
+
+  uint32_t id = mHangData.get_PluginHangData().pluginId();
+  mActor->CleanupPluginHang(id, true);
+  return NS_OK;
+}
+
 ProcessHangMonitor* ProcessHangMonitor::sInstance;
 
 ProcessHangMonitor::ProcessHangMonitor()

dom/ipc/nsIHangReport.idl

@@ -18,7 +18,7 @@ interface nsIFrameLoader;
  * process will continue to run uninhibitedly during this time.
  */
 
-[scriptable, uuid(3b88d100-8d5b-11e4-b4a9-0800200c9a66)]
+[scriptable, uuid(90cea731-dd3e-459e-b017-f9a14697b56e)]
 interface nsIHangReport : nsISupports
 {
   const unsigned long SLOW_SCRIPT = 1;
@@ -38,6 +38,10 @@ interface nsIHangReport : nsISupports
   // Only valid for PLUGIN_HANG reports.
   readonly attribute ACString pluginName;
 
+  // Called by front end code when user ignores or cancels
+  // the notification.
+  void userCanceled();
+
   // Terminate the slow script if it is still running.
   // Only valid for SLOW_SCRIPT reports.
   void terminateScript();

dom/plugins/ipc/PluginBridge.h

@@ -25,7 +25,7 @@ FindPluginsForContent(uint32_t aPluginEpoch,
                       uint32_t* aNewPluginEpoch);
 
 void
-TerminatePlugin(uint32_t aPluginId);
+TerminatePlugin(uint32_t aPluginId, const nsString& aBrowserDumpId);
 
 } // namespace plugins
 } // namespace mozilla

dom/plugins/ipc/PluginHangUIParent.cpp

@@ -353,7 +353,8 @@ PluginHangUIParent::RecvUserResponse(const unsigned int& aResponse)
   int responseCode;
   if (aResponse & HANGUI_USER_RESPONSE_STOP) {
     // User clicked Stop
-    mModule->TerminateChildProcess(mMainThreadMessageLoop);
+    nsString dummy;
+    mModule->TerminateChildProcess(mMainThreadMessageLoop, &dummy);
     responseCode = 1;
   } else if(aResponse & HANGUI_USER_RESPONSE_CONTINUE) {
     mModule->OnHangUIContinue();

dom/plugins/ipc/PluginModuleParent.cpp

@@ -344,7 +344,7 @@ bool PluginModuleMapping::sIsLoadModuleOnStack = false;
 } // anonymous namespace
 
 void
-mozilla::plugins::TerminatePlugin(uint32_t aPluginId)
+mozilla::plugins::TerminatePlugin(uint32_t aPluginId, const nsString& aBrowserDumpId)
 {
     MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_Default);
 
@@ -353,10 +353,10 @@ mozilla::plugins::TerminatePlugin(uint32_t aPluginId)
     if (!pluginTag || !pluginTag->mPlugin) {
         return;
     }
-
+    nsAutoString dumpId(aBrowserDumpId);
     nsRefPtr<nsNPAPIPlugin> plugin = pluginTag->mPlugin;
     PluginModuleChromeParent* chromeParent = static_cast<PluginModuleChromeParent*>(plugin->GetLibrary());
-    chromeParent->TerminateChildProcess(MessageLoop::current());
+    chromeParent->TerminateChildProcess(MessageLoop::current(), &dumpId);
 }
 
 /* static */ PluginLibrary*
@@ -1148,7 +1148,8 @@ PluginModuleChromeParent::ShouldContinueFromReplyTimeout()
     // original plugin hang behaviour and kill the plugin container.
     FinishHangUI();
 #endif // XP_WIN
-    TerminateChildProcess(MessageLoop::current());
+    nsString dummy;
+    TerminateChildProcess(MessageLoop::current(), &dummy);
     GetIPCChannel()->CloseWithTimeout();
     return false;
 }
@@ -1171,7 +1172,8 @@ PluginModuleContentParent::OnExitedSyncSend()
 }
 
 void
-PluginModuleChromeParent::TerminateChildProcess(MessageLoop* aMsgLoop)
+PluginModuleChromeParent::TerminateChildProcess(MessageLoop* aMsgLoop,
+                                                nsAString* aBrowserDumpId)
 {
 #ifdef MOZ_CRASHREPORTER
 #ifdef XP_WIN
@@ -1198,8 +1200,39 @@ PluginModuleChromeParent::TerminateChildProcess(MessageLoop* aMsgLoop)
         }
     }
 #endif // XP_WIN
-    // Generate base report, includes plugin and browser process minidumps.
-    if (crashReporter->GeneratePairedMinidump(this)) {
+
+    bool reportsReady = false;
+
+    // Check to see if we already have a browser dump id - with e10s plugin
+    // hangs we take this earlier (see ProcessHangMonitor) from a background
+    // thread. We do this before we message the main thread about the hang
+    // since the posted message will trash our browser stack state.
+    bool exists;
+    nsCOMPtr<nsIFile> browserDumpFile;
+    if (aBrowserDumpId && !aBrowserDumpId->IsEmpty() &&
+        CrashReporter::GetMinidumpForID(*aBrowserDumpId, getter_AddRefs(browserDumpFile)) &&
+        browserDumpFile &&
+        NS_SUCCEEDED(browserDumpFile->Exists(&exists)) && exists)
+    {
+        // We have a single browser report, generate a new plugin process parent
+        // report and pair it up with the browser report handed in.
+        reportsReady = crashReporter->GenerateMinidumpAndPair(this, browserDumpFile,
+                                                              NS_LITERAL_CSTRING("browser"));
+        if (!reportsReady) {
+          browserDumpFile = nullptr;
+          CrashReporter::DeleteMinidumpFilesForID(*aBrowserDumpId);
+        }
+    }
+
+    // Generate crash report including plugin and browser process minidumps.
+    // The plugin process is the parent report with additional dumps including
+    // the browser process, content process when running under e10s, and
+    // various flash subprocesses if we're the flash module.
+    if (!reportsReady) {
+        reportsReady = crashReporter->GeneratePairedMinidump(this);
+    }
+
+    if (reportsReady) {
         mPluginDumpID = crashReporter->ChildDumpID();
         PLUGIN_LOG_DEBUG(
                 ("generated paired browser/plugin minidumps: %s)",

dom/plugins/ipc/PluginModuleParent.h

@@ -369,7 +369,19 @@ class PluginModuleChromeParent
 
     virtual ~PluginModuleChromeParent();
 
-    void TerminateChildProcess(MessageLoop* aMsgLoop);
+    /*
+     * Terminates the plugin process associated with this plugin module. Also
+     * generates appropriate crash reports. Takes ownership of the file
+     * associated with aBrowserDumpId on success.
+     *
+     * @param aMsgLoop the main message pump associated with the module
+     *   protocol.
+     * @param aBrowserDumpId (optional) previously taken browser dump id. If
+     *   provided TerminateChildProcess will use this browser dump file in
+     *   generating a multi-process crash report. If not provided a browser
+     *   dump will be taken at the time of this call.
+     */
+    void TerminateChildProcess(MessageLoop* aMsgLoop, nsAString* aBrowserDumpId);
 
 #ifdef XP_WIN
     /**