Bug 1072404: Firefox may crash when the D3D device is removed while rendering. r=mattwoodrow

gfx/layers/client/ClientLayerManager.cpp

@@ -28,6 +28,7 @@
 #include "nsXULAppAPI.h"                // for XRE_GetProcessType, etc
 #include "TiledLayerBuffer.h"
 #include "mozilla/dom/WindowBinding.h"  // for Overfill Callback
+#include "FrameLayerBuilder.h"          // for FrameLayerbuilder
 #include "gfxPrefs.h"
 #ifdef MOZ_WIDGET_ANDROID
 #include "AndroidBridge.h"
@@ -276,6 +277,10 @@ ClientLayerManager::EndTransactionInternal(DrawThebesLayerCallback aCallback,
   NS_ASSERTION(!aCallback || !mTransactionIncomplete,
                "If callback is not null, transaction must be complete");
 
+  if (gfxPlatform::GetPlatform()->DidRenderingDeviceReset()) {
+    FrameLayerBuilder::InvalidateAllLayers(this);
+  }
+
   return !mTransactionIncomplete;
 }
 

gfx/thebes/gfxContext.cpp

@@ -1005,9 +1005,13 @@ gfxContext::Paint(gfxFloat alpha)
 void
 gfxContext::PushGroup(gfxContentType content)
 {
+  DrawTarget* oldDT = mDT;
+
   PushNewDT(content);
 
-  PushClipsToDT(mDT);
+  if (oldDT != mDT) {
+    PushClipsToDT(mDT);
+  }
   mDT->SetTransform(GetDTTransform());
 }
 
@@ -1038,6 +1042,11 @@ gfxContext::PushGroupAndCopyBackground(gfxContentType content)
 
     PushNewDT(gfxContentType::COLOR);
 
+    if (oldDT == mDT) {
+      // Creating new DT failed.
+      return;
+    }
+
     Point offset = CurrentState().deviceOffset - oldDeviceOffset;
     Rect surfRect(0, 0, Float(mDT->GetSize().width), Float(mDT->GetSize().height));
     Rect sourceRect = surfRect + offset;
@@ -1588,8 +1597,11 @@ gfxContext::PushNewDT(gfxContentType content)
     newDT = mDT->CreateSimilarDrawTarget(IntSize(64, 64), format);
 
     if (!newDT) {
-      // If even this fails.. we're most likely just out of memory!
-      NS_ABORT_OOM(BytesPerPixel(format) * 64 * 64);
+      if (!gfxPlatform::GetPlatform()->DidRenderingDeviceReset()) {
+        // If even this fails.. we're most likely just out of memory!
+        NS_ABORT_OOM(BytesPerPixel(format) * 64 * 64);
+      }
+      newDT = CurrentState().drawTarget;
     }
   }
 

gfx/thebes/gfxPlatform.h

@@ -411,6 +411,8 @@ public:
     // check whether format is supported on a platform or not (if unclear, returns true)
     virtual bool IsFontFormatSupported(nsIURI *aFontURI, uint32_t aFormatFlags) { return false; }
 
+    virtual bool DidRenderingDeviceReset() { return false; }
+
     void GetPrefFonts(nsIAtom *aLanguage, nsString& array, bool aAppendUnicode = true);
 
     // in some situations, need to make decisions about ambiguous characters, may need to look at multiple pref langs

gfx/thebes/gfxWindowsPlatform.cpp

@@ -974,6 +974,12 @@ gfxWindowsPlatform::IsFontFormatSupported(nsIURI *aFontURI, uint32_t aFormatFlag
     return true;
 }
 
+bool
+gfxWindowsPlatform::DidRenderingDeviceReset()
+{
+  return GetD3D10Device() && GetD3D10Device()->GetDeviceRemovedReason() != S_OK;
+}
+
 gfxFontFamily *
 gfxWindowsPlatform::FindFontFamily(const nsAString& aName)
 {

gfx/thebes/gfxWindowsPlatform.h

@@ -219,6 +219,8 @@ public:
      */
     virtual bool IsFontFormatSupported(nsIURI *aFontURI, uint32_t aFormatFlags);
 
+    virtual bool DidRenderingDeviceReset();
+
     /* Find a FontFamily/FontEntry object that represents a font on your system given a name */
     gfxFontFamily *FindFontFamily(const nsAString& aName);
     gfxFontEntry *FindFontEntry(const nsAString& aName, const gfxFontStyle& aFontStyle);