Bug 1437886 - Prevent shared surfaces from being used without WebRender. r=nical

Move the initialization of SharedSurfacesParent from the compositor
thread creation to mirror the other WebRender-specific components, such
as the render thread creation. Now it will only be created if WebRender
is in use. Also prevent shared surfaces from being used by the image
frame allocator, even if image.mem.shared is set -- there is no purpose
in allowing this at present. It was causing startup crashes for users
who requested image.mem.shared and/or WebRender via gfx.webrender.all
but did not actually get WebRender at all. Surfaces would get allocated
in the shared memory, try to register themselves with the WR render
thread, and then crash since that thread was never created.

gfx/config/gfxVars.h

@@ -137,6 +137,12 @@ public:                                                         \
   static const DataType& CxxName() {                            \
     return sInstance->mVar##CxxName.Get();                      \
   }                                                             \
+  static DataType Get##CxxName##OrDefault() {                   \
+    if (!sInstance) {                                           \
+      return DefaultValue;                                      \
+    }                                                           \
+    return sInstance->mVar##CxxName.Get();                      \
+  }                                                             \
   static void Set##CxxName(const DataType& aValue) {            \
     if (sInstance->mVar##CxxName.Set(aValue)) {                 \
       sInstance->NotifyReceivers(&sInstance->mVar##CxxName);    \

gfx/ipc/GPUParent.cpp

@@ -30,6 +30,7 @@
 #include "mozilla/layers/LayerTreeOwnerTracker.h"
 #include "mozilla/layers/UiCompositorControllerParent.h"
 #include "mozilla/layers/MemoryReportingMLGPU.h"
+#include "mozilla/layers/SharedSurfacesParent.h"
 #include "mozilla/webrender/RenderThread.h"
 #include "mozilla/webrender/WebRenderAPI.h"
 #include "mozilla/HangDetails.h"
@@ -234,6 +235,7 @@ GPUParent::RecvInit(nsTArray<GfxPrefSetting>&& prefs,
     wr::WebRenderAPI::InitExternalLogHandler();
 
     wr::RenderThread::Start();
+    SharedSurfacesParent::Initialize();
   }
 
   VRManager::ManagerInit();
@@ -465,6 +467,7 @@ GPUParent::ActorDestroy(ActorDestroyReason aWhy)
   // There is a case that RenderThread exists when gfxVars::UseWebRender() is false.
   // This could happen when WebRender was fallbacked to compositor.
   if (wr::RenderThread::Get()) {
+    SharedSurfacesParent::Shutdown();
     wr::RenderThread::ShutDown();
 
     wr::WebRenderAPI::ShutdownExternalLogHandler();

gfx/layers/ipc/CompositorThread.cpp

@@ -8,7 +8,6 @@
 #include "nsThreadUtils.h"
 #include "CompositorBridgeParent.h"
 #include "mozilla/layers/ImageBridgeParent.h"
-#include "mozilla/layers/SharedSurfacesParent.h"
 #include "mozilla/media/MediaSystemResourceService.h"
 
 namespace mozilla {
@@ -70,7 +69,6 @@ CompositorThreadHolder::DestroyCompositorThread(base::Thread* aCompositorThread)
   MOZ_ASSERT(!sCompositorThreadHolder, "We shouldn't be destroying the compositor thread yet.");
 
   CompositorBridgeParent::Shutdown();
-  SharedSurfacesParent::Shutdown();
   delete aCompositorThread;
   sFinishedCompositorShutDown = true;
 }
@@ -105,7 +103,6 @@ CompositorThreadHolder::CreateCompositorThread()
     return nullptr;
   }
 
-  SharedSurfacesParent::Initialize();
   CompositorBridgeParent::Setup();
   ImageBridgeParent::Setup();
 

gfx/layers/ipc/SharedSurfacesChild.cpp

@@ -7,6 +7,7 @@
 #include "SharedSurfacesChild.h"
 #include "SharedSurfacesParent.h"
 #include "CompositorManagerChild.h"
+#include "mozilla/gfx/gfxVars.h"
 #include "mozilla/layers/IpcResourceUpdateQueue.h"
 #include "mozilla/layers/SourceSurfaceSharedData.h"
 #include "mozilla/layers/WebRenderBridgeChild.h"
@@ -192,7 +193,7 @@ SharedSurfacesChild::ShareInternal(SourceSurfaceSharedData* aSurface,
   MOZ_ASSERT(aUserData);
 
   CompositorManagerChild* manager = CompositorManagerChild::GetInstance();
-  if (NS_WARN_IF(!manager || !manager->CanSend())) {
+  if (NS_WARN_IF(!manager || !manager->CanSend() || !gfxVars::UseWebRender())) {
     // We cannot try to share the surface, most likely because the GPU process
     // crashed. Ideally, we would retry when it is ready, but the handles may be
     // a scarce resource, which can cause much more serious problems if we run
@@ -391,9 +392,13 @@ SharedSurfacesChild::Unshare(const wr::ExternalImageId& aId,
 
   if (manager->OtherPid() == base::GetCurrentProcId()) {
     // We are in the combined UI/GPU process. Call directly to it to remove its
-    // wrapper surface to free the underlying buffer.
-    MOZ_ASSERT(manager->OwnsExternalImageId(aId));
+    // wrapper surface to free the underlying buffer, but only if the external
+    // image ID is owned by the manager. It can be different if the surface was
+    // last shared with the GPU process, which crashed several times, and its
+    // job was moved into the parent process.
+    if (manager->OwnsExternalImageId(aId)) {
       SharedSurfacesParent::RemoveSameProcess(aId);
+    }
   } else if (manager->OwnsExternalImageId(aId)) {
     // Only attempt to release current mappings in the GPU process. It is
     // possible we had a surface that was previously shared, the GPU process

gfx/thebes/gfxPlatform.cpp

@@ -7,6 +7,7 @@
 #include "mozilla/layers/CompositorThread.h"
 #include "mozilla/layers/ImageBridgeChild.h"
 #include "mozilla/layers/ISurfaceAllocator.h"     // for GfxMemoryImageReporter
+#include "mozilla/layers/SharedSurfacesParent.h"
 #include "mozilla/webrender/RenderThread.h"
 #include "mozilla/webrender/WebRenderAPI.h"
 #include "mozilla/webrender/webrender_ffi.h"
@@ -1019,6 +1020,7 @@ gfxPlatform::InitLayersIPC()
   } else if (XRE_IsParentProcess()) {
     if (gfxVars::UseWebRender()) {
       wr::RenderThread::Start();
+      layers::SharedSurfacesParent::Initialize();
     }
 
     layers::CompositorThreadHolder::Start();
@@ -1055,6 +1057,7 @@ gfxPlatform::ShutdownLayersIPC()
         // There is a case that RenderThread exists when gfxVars::UseWebRender() is false.
         // This could happen when WebRender was fallbacked to compositor.
         if (wr::RenderThread::Get()) {
+          layers::SharedSurfacesParent::Shutdown();
           wr::RenderThread::ShutDown();
 
           Preferences::UnregisterCallback(WebRenderDebugPrefChangeCallback, WR_DEBUG_PREF);

image/VectorImage.cpp

@@ -876,7 +876,7 @@ VectorImage::IsImageContainerAvailableAtSize(LayerManager* aManager,
                                              uint32_t aFlags)
 {
   if (mError || !mIsFullyLoaded || aSize.IsEmpty() ||
-      mHaveAnimations || !gfxVars::UseWebRender()) {
+      mHaveAnimations || !gfxVars::GetUseWebRenderOrDefault()) {
     return false;
   }
 

image/imgFrame.cpp

@@ -101,7 +101,8 @@ AllocateBufferForImage(const IntSize& size,
   }
 #endif
 
-  if (!aIsAnimated && gfxPrefs::ImageMemShared()) {
+  if (!aIsAnimated && gfxVars::GetUseWebRenderOrDefault()
+                   && gfxPrefs::ImageMemShared()) {
     RefPtr<SourceSurfaceSharedData> newSurf = new SourceSurfaceSharedData();
     if (newSurf->Init(size, stride, format)) {
       return newSurf.forget();