Disable device access on textures created against stale layers. (bug 1256517 part 3, r=nical)

gfx/2d/Logging.h

@@ -133,6 +133,7 @@ enum class LogReason : int {
   IncompatibleBasicTexturedEffect,
   InvalidFont,
   AsyncTransactionTimeout,
+  PAllocTextureBackendMismatch,
   // End
   MustBeLessThanThis = 101,
 };

gfx/layers/CompositorTypes.h

@@ -63,9 +63,12 @@ enum class TextureFlags : uint32_t {
   IMMEDIATE_UPLOAD   = 1 << 10,
   // The texture is part of a component-alpha pair
   COMPONENT_ALPHA    = 1 << 11,
+  // The texture is being allocated for a compositor that no longer exists.
+  // This flag is only used in the parent process.
+  INVALID_COMPOSITOR = 1 << 12,
 
   // OR union of all valid bits
-  ALL_BITS           = (1 << 12) - 1,
+  ALL_BITS           = (1 << 13) - 1,
   // the default flags
   DEFAULT = NO_FLAGS
 };

gfx/layers/d3d11/TextureD3D11.cpp

@@ -629,6 +629,10 @@ DXGITextureHostD3D11::OpenSharedHandle()
 RefPtr<ID3D11Device>
 DXGITextureHostD3D11::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
+
   RefPtr<ID3D11Device> device;
   gfxWindowsPlatform::GetPlatform()->GetD3D11Device(&device);
   return device;
@@ -747,6 +751,10 @@ DXGIYCbCrTextureHostD3D11::OpenSharedHandle()
 RefPtr<ID3D11Device>
 DXGIYCbCrTextureHostD3D11::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
+
   RefPtr<ID3D11Device> device;
   gfxWindowsPlatform::GetPlatform()->GetD3D11Device(&device);
   return device;

gfx/layers/d3d9/TextureD3D9.cpp

@@ -911,6 +911,9 @@ TextureHostD3D9::UpdatedInternal(const nsIntRegion* aRegion)
 IDirect3DDevice9*
 TextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
@@ -976,6 +979,9 @@ DXGITextureHostD3D9::DXGITextureHostD3D9(TextureFlags aFlags,
 IDirect3DDevice9*
 DXGITextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
@@ -1070,6 +1076,9 @@ DXGIYCbCrTextureHostD3D9::DXGIYCbCrTextureHostD3D9(TextureFlags aFlags,
 IDirect3DDevice9*
 DXGIYCbCrTextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 

gfx/layers/ipc/LayerTransactionParent.cpp

@@ -969,8 +969,20 @@ LayerTransactionParent::AllocPTextureParent(const SurfaceDescriptor& aSharedData
                                             const LayersBackend& aLayersBackend,
                                             const TextureFlags& aFlags)
 {
-  MOZ_ASSERT(aLayersBackend == mLayerManager->GetCompositor()->GetBackendType());
-  return TextureHost::CreateIPDLActor(this, aSharedData, aLayersBackend, aFlags);
+  TextureFlags flags = aFlags;
+
+  if (mPendingCompositorUpdates) {
+    // The compositor was recreated, and we're receiving layers updates for a
+    // a layer manager that will soon be discarded or invalidated. We can't
+    // return null because this will mess up deserialization later and we'll
+    // kill the content process. Instead, we signal that the underlying
+    // TextureHost should not attempt to access the compositor.
+    flags |= TextureFlags::INVALID_COMPOSITOR;
+  } else if (aLayersBackend != mLayerManager->GetCompositor()->GetBackendType()) {
+    gfxDevCrash(LogReason::PAllocTextureBackendMismatch, "Texture backend is wrong");
+  }
+
+  return TextureHost::CreateIPDLActor(this, aSharedData, aLayersBackend, flags);
 }
 
 bool