Bug 532751 - Notify the nsNPAPIPlugin and the related nsNPAPIPluginInstances when a plugin crashes so that reloading will create a new plugin process, r=josh

dom/plugins/PluginLibrary.h

@@ -1,4 +1,4 @@
-/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
  * vim: sw=4 ts=4 et :
  * ***** BEGIN LICENSE BLOCK *****
  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
@@ -43,6 +43,8 @@
 #include "npapi.h"
 #include "nscore.h"
 
+class nsNPAPIPlugin;
+
 namespace mozilla {
 
 class PluginLibrary
@@ -50,6 +52,12 @@ class PluginLibrary
 public:
   virtual ~PluginLibrary() { }
 
+  /**
+   * Inform this library about the nsNPAPIPlugin which owns it. This
+   * object will hold a weak pointer to the plugin.
+   */
+  virtual void SetPlugin(nsNPAPIPlugin* plugin) = 0;
+
   virtual bool HasRequiredFunctions() = 0;
 
 #if defined(XP_UNIX) && !defined(XP_MACOSX)

dom/plugins/PluginModuleParent.cpp

@@ -69,6 +69,7 @@ PluginModuleParent::PluginModuleParent(const char* aFilePath)
     : mSubprocess(new PluginProcessParent(aFilePath))
     , mShutdown(false)
     , mNPNIface(NULL)
+    , mPlugin(NULL)
 {
     NS_ASSERTION(mSubprocess, "Out of memory!");
 
@@ -97,9 +98,16 @@ PluginModuleParent::ActorDestroy(ActorDestroyReason why)
 {
     switch (why) {
     case AbnormalShutdown:
-        // TODObsmedberg: notify the plugin host to forget this plugin module
-        // and instantiate us again.
-        // FALL THROUGH
+        mShutdown = true;
+        // Defer the PluginCrashed method so that we don't re-enter
+        // and potentially modify the actor child list while enumerating it.
+        if (mPlugin) {
+            nsCOMPtr<nsIRunnable> r =
+                new nsRunnableMethod<nsNPAPIPlugin>(
+                    mPlugin, &nsNPAPIPlugin::PluginCrashed);
+            NS_DispatchToMainThread(r);
+        }
+        break;
 
     case NormalShutdown:
         mShutdown = true;

dom/plugins/PluginModuleParent.h

@@ -97,9 +97,13 @@ protected:
 
 public:
     PluginModuleParent(const char* aFilePath);
-
     virtual ~PluginModuleParent();
 
+    NS_OVERRIDE virtual void SetPlugin(nsNPAPIPlugin* plugin)
+    {
+        mPlugin = plugin;
+    }
+
     NS_OVERRIDE virtual void ActorDestroy(ActorDestroyReason why);
 
     /**
@@ -213,6 +217,7 @@ private:
     bool mShutdown;
     const NPNetscapeFuncs* mNPNIface;
     nsTHashtable<nsVoidPtrHashKey> mValidIdentifiers;
+    nsNPAPIPlugin* mPlugin;
 };
 
 } // namespace plugins

dom/plugins/PluginPRLibrary.h

@@ -83,6 +83,8 @@ public:
         // unref here??
     }
 
+    virtual void SetPlugin(nsNPAPIPlugin*) { }
+
     virtual bool HasRequiredFunctions() {
         mNP_Initialize = (NP_InitializeFunc)
             PR_FindFunctionSymbol(mLibrary, "NP_Initialize");

modules/plugin/base/src/nsNPAPIPlugin.cpp

@@ -280,6 +280,7 @@ nsNPAPIPlugin::nsNPAPIPlugin(NPPluginFuncs* callbacks,
 #endif
 
   fLibrary = aLibrary;
+  fLibrary->SetPlugin(this);
 }
 
 nsNPAPIPlugin::~nsNPAPIPlugin()
@@ -299,6 +300,15 @@ nsNPAPIPlugin::SetPluginRefNum(short aRefNum)
 }
 #endif
 
+#ifdef MOZ_IPC
+void
+nsNPAPIPlugin::PluginCrashed()
+{
+  nsRefPtr<nsPluginHost> host = dont_AddRef(nsPluginHost::GetInst());
+  host->PluginCrashed(this);
+}
+#endif
+
 namespace {
 
 #ifdef MOZ_IPC

modules/plugin/base/src/nsNPAPIPlugin.h

@@ -94,6 +94,12 @@ public:
   void SetPluginRefNum(short aRefNum);
 #endif
 
+#ifdef MOZ_IPC
+  // The IPC mechanism notifies the nsNPAPIPlugin if the plugin crashes and is
+  // no longer usable.
+  void PluginCrashed();
+#endif
+
 protected:
   // Ensures that the static CALLBACKS is properly initialized
   static void CheckClassInitialized(void);

modules/plugin/base/src/nsPluginHost.cpp

@@ -5180,6 +5180,44 @@ NS_IMETHODIMP nsPluginHost::Notify(nsITimer* timer)
   return NS_ERROR_FAILURE;
 }
 
+#ifdef MOZ_IPC
+void
+nsPluginHost::PluginCrashed(nsNPAPIPlugin* aPlugin)
+{
+  // Find the nsPluginTag corresponding to this plugin
+
+  nsPluginTag* plugin;
+  for (plugin = mPlugins; plugin; plugin = plugin->mNext) {
+    if (plugin->mEntryPoint == aPlugin)
+      break;
+  }
+  if (!plugin) {
+    NS_WARNING("nsPluginTag not found in nsPluginHost::PluginCrashed");
+    return;
+  }
+
+  // Invalidate each nsPluginInstanceTag for the crashed plugin
+
+  nsPluginInstanceTag** pinstancetag = &mPluginInstanceTagList.mFirst;
+  while (*pinstancetag) {
+    nsPluginInstanceTag* instancetag = *pinstancetag;
+    if (instancetag->mPluginTag == plugin) {
+      *pinstancetag = (*pinstancetag)->mNext;
+      delete instancetag;
+    }
+    else {
+      pinstancetag = &(*pinstancetag)->mNext;
+    }
+  }
+
+  // Only after all instances have been invalidated is it safe to null
+  // out nsPluginTag.mEntryPoint. The next time we try to create an
+  // instance of this plugin we reload it (launch a new plugin process).
+
+  plugin->mEntryPoint = nsnull;
+}
+#endif
+
 nsresult nsPluginStreamListenerPeer::ServeStreamAsFile(nsIRequest *request,
                                                        nsISupports* aContext)
 {

modules/plugin/base/src/nsPluginHost.h

@@ -161,7 +161,11 @@ public:
 
   void AddIdleTimeTarget(nsIPluginInstanceOwner* objectFrame, PRBool isVisible);
   void RemoveIdleTimeTarget(nsIPluginInstanceOwner* objectFrame);
-  
+
+#ifdef MOZ_IPC
+  void PluginCrashed(nsNPAPIPlugin* plugin);
+#endif
+
 private:
   nsresult
   TrySetUpPluginInstance(const char *aMimeType, nsIURI *aURL, nsIPluginInstanceOwner *aOwner);

modules/plugin/test/mochitest/Makefile.in

@@ -69,12 +69,12 @@ _MOCHITEST_FILES = \
 
 #		test_npruntime_npnsetexception.html \ Disabled for e10s
 
-#ifdef MOZ_IPC
-#_MOCHITEST_FILES += \
-#		test_crashing.html \
-#		crashing_subpage.html \
-#		$(NULL)
-#endif
+ifdef MOZ_IPC
+_MOCHITEST_FILES += \
+		test_crashing.html \
+		crashing_subpage.html \
+		$(NULL)
+endif
 
 ifeq ($(OS_ARCH),WINNT)
 _MOCHITEST_FILES += \