Bug 1391262 - Do not use remote LayerManager when its initialization fails r=dvander

dom/ipc/TabChild.cpp

@@ -2800,6 +2800,7 @@ TabChild::InitRenderingState(const TextureFactoryIdentifier& aTextureFactoryIden
     mPuppetWidget->InitIMEState();
 
     if (!aRenderFrame) {
+      mLayersConnected = false;
       NS_WARNING("failed to construct RenderFrame");
       return;
     }
@@ -2811,6 +2812,7 @@ TabChild::InitRenderingState(const TextureFactoryIdentifier& aTextureFactoryIden
     // compositor context.
     PCompositorBridgeChild* compositorChild = CompositorBridgeChild::Get();
     if (!compositorChild) {
+      mLayersConnected = false;
       NS_WARNING("failed to get CompositorBridgeChild instance");
       return;
     }
@@ -2829,32 +2831,45 @@ TabChild::InitRenderingState(const TextureFactoryIdentifier& aTextureFactoryIden
       mLayersId = aLayersId;
     }
 
-    LayerManager* lm = mPuppetWidget->GetLayerManager();
-    if (lm->AsWebRenderLayerManager()) {
-      lm->AsWebRenderLayerManager()->Initialize(compositorChild,
-                                                wr::AsPipelineId(aLayersId),
-                                                &mTextureFactoryIdentifier);
+    MOZ_ASSERT(!mPuppetWidget->HasLayerManager());
+    bool success = false;
+    if (mLayersConnected && gfxVars::UseWebRender()) {
+      success = mPuppetWidget->CreateRemoteLayerManager([&] (LayerManager* aLayerManager) -> bool {
+        MOZ_ASSERT(aLayerManager->AsWebRenderLayerManager());
+        return aLayerManager->AsWebRenderLayerManager()->Initialize(compositorChild,
+                                                                    wr::AsPipelineId(mLayersId),
+                                                                    &mTextureFactoryIdentifier);
+      });
+    } else if (mLayersConnected) {
+      nsTArray<LayersBackend> ignored;
+      PLayerTransactionChild* shadowManager = compositorChild->SendPLayerTransactionConstructor(ignored, LayersId());
+      if (shadowManager &&
+          shadowManager->SendGetTextureFactoryIdentifier(&mTextureFactoryIdentifier) &&
+          mTextureFactoryIdentifier.mParentBackend != LayersBackend::LAYERS_NONE)
+      {
+        success = true;
+      }
+      if (!success) {
+        NS_WARNING("failed to allocate layer transaction");
+      } else {
+        success = mPuppetWidget->CreateRemoteLayerManager([&] (LayerManager* aLayerManager) -> bool {
+          ShadowLayerForwarder* lf = aLayerManager->AsShadowForwarder();
+          lf->SetShadowManager(shadowManager);
+          lf->IdentifyTextureHost(mTextureFactoryIdentifier);
+          return true;
+        });
+      }
+    }
+
+    if (success) {
+      MOZ_ASSERT(mLayersConnected);
+      // Succeeded to create "remote" layer manager
       ImageBridgeChild::IdentifyCompositorTextureHost(mTextureFactoryIdentifier);
       gfx::VRManagerChild::IdentifyTextureHost(mTextureFactoryIdentifier);
       InitAPZState();
-    }
-
-    ShadowLayerForwarder* lf =
-        mPuppetWidget->GetLayerManager(
-            nullptr, mTextureFactoryIdentifier.mParentBackend)
-                ->AsShadowForwarder();
-    if (lf) {
-      nsTArray<LayersBackend> backends;
-      backends.AppendElement(mTextureFactoryIdentifier.mParentBackend);
-      PLayerTransactionChild* shadowManager =
-          compositorChild->SendPLayerTransactionConstructor(backends, aLayersId);
-      if (shadowManager) {
-        lf->SetShadowManager(shadowManager);
-        lf->IdentifyTextureHost(mTextureFactoryIdentifier);
-        ImageBridgeChild::IdentifyCompositorTextureHost(mTextureFactoryIdentifier);
-        gfx::VRManagerChild::IdentifyTextureHost(mTextureFactoryIdentifier);
-        InitAPZState();
-      }
+    } else {
+      // Fallback to BasicManager
+      mLayersConnected = false;
     }
 
     nsCOMPtr<nsIObserverService> observerService =
@@ -3165,7 +3180,7 @@ TabChild::ReinitRendering()
   bool success = false;
   RefPtr<CompositorBridgeChild> cb = CompositorBridgeChild::Get();
   if (gfxVars::UseWebRender()) {
-    success = mPuppetWidget->RecreateLayerManager([&] (LayerManager* aLayerManager) -> bool {
+    success = mPuppetWidget->CreateRemoteLayerManager([&] (LayerManager* aLayerManager) -> bool {
       MOZ_ASSERT(aLayerManager->AsWebRenderLayerManager());
       return aLayerManager->AsWebRenderLayerManager()->Initialize(cb,
                                                                   wr::AsPipelineId(mLayersId),
@@ -3185,7 +3200,7 @@ TabChild::ReinitRendering()
       return;
     }
 
-    success = mPuppetWidget->RecreateLayerManager([&] (LayerManager* aLayerManager) -> bool {
+    success = mPuppetWidget->CreateRemoteLayerManager([&] (LayerManager* aLayerManager) -> bool {
       ShadowLayerForwarder* lf = aLayerManager->AsShadowForwarder();
       lf->SetShadowManager(shadowManager);
       lf->IdentifyTextureHost(mTextureFactoryIdentifier);

widget/PuppetWidget.cpp

@@ -596,31 +596,19 @@ PuppetWidget::GetLayerManager(PLayerTransactionChild* aShadowManager,
       return mLayerManager;
     }
 
-    if (mTabChild && !mTabChild->IsLayersConnected()) {
-      // If we know for sure that the parent side of this TabChild is not
-      // connected to the compositor, we don't want to use a "remote" layer
-      // manager like WebRender or Client. Instead we use a Basic one which
-      // can do drawing in this process.
-      mLayerManager = new BasicLayerManager(this);
-    } else if (gfxVars::UseWebRender()) {
-      MOZ_ASSERT(!aShadowManager);
-      mLayerManager = new WebRenderLayerManager(this);
-    } else {
-      mLayerManager = new ClientLayerManager(this);
-    }
-  }
-
-  // Attach a shadow forwarder if none exists.
-  ShadowLayerForwarder* lf = mLayerManager->AsShadowForwarder();
-  if (lf && !lf->HasShadowManager() && aShadowManager) {
-    lf->SetShadowManager(aShadowManager);
+    // If we know for sure that the parent side of this TabChild is not
+    // connected to the compositor, we don't want to use a "remote" layer
+    // manager like WebRender or Client. Instead we use a Basic one which
+    // can do drawing in this process.
+    MOZ_ASSERT(!mTabChild || !mTabChild->IsLayersConnected());
+    mLayerManager = new BasicLayerManager(this);
   }
 
   return mLayerManager;
 }
 
 bool
-PuppetWidget::RecreateLayerManager(const std::function<bool(LayerManager*)>& aInitializeFunc)
+PuppetWidget::CreateRemoteLayerManager(const std::function<bool(LayerManager*)>& aInitializeFunc)
 {
   RefPtr<LayerManager> lm;
   MOZ_ASSERT(mTabChild);

widget/PuppetWidget.h

@@ -182,11 +182,17 @@ public:
                   LayersBackend aBackendHint = mozilla::layers::LayersBackend::LAYERS_NONE,
                   LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT) override;
 
-  // This is used after a compositor reset. The lambda aInitializeFunc is used
-  // to perform any caller-required initialization for the newly created layer
+  // This is used for creating remote layer managers and for re-creating
+  // them after a compositor reset. The lambda aInitializeFunc is used to perform
+  // any caller-required initialization for the newly created layer
   // manager; in the event of a failure, return false and it will destroy the
   // new layer manager without changing the state of the widget.
-  bool RecreateLayerManager(const std::function<bool(LayerManager*)>& aInitializeFunc);
+  bool CreateRemoteLayerManager(const std::function<bool(LayerManager*)>& aInitializeFunc);
+
+  bool HasLayerManager()
+  {
+    return !!mLayerManager;
+  }
 
   virtual void SetInputContext(const InputContext& aContext,
                                const InputContextAction& aAction) override;