Bug 1533296 - Make WebRender error as to Fallback to Compositor rendering during disabling WebRender on android r=nical

On android, android's nsWindow creates LayerManaer only in nsWindow::Create(). When WebRender error happened, gecko just stopped rendering by disabling Webrender.
The nsWindow needs to re-create LayerManager during disabling Webrender. Further, during disabling WebRender, All GeckoSurfaceTextures should not be attached to GLContext. It is for preventing a conflict with AttachToGLContext() call in SurfaceTextureHost::EnsureAttached().

Differential Revision: https://phabricator.services.mozilla.com/D26687

--HG--
extra : moz-landing-system : lando

gfx/ipc/CompositorSession.cpp

@@ -11,16 +11,22 @@
 #include "mozilla/layers/CompositorBridgeChild.h"
 #include "mozilla/layers/CompositorBridgeParent.h"
 
+#if defined(MOZ_WIDGET_ANDROID)
+#  include "mozilla/widget/nsWindow.h"
+#endif  // defined(MOZ_WIDGET_ANDROID)
+
 namespace mozilla {
 namespace layers {
 
 using namespace gfx;
 using namespace widget;
 
-CompositorSession::CompositorSession(CompositorWidgetDelegate* aDelegate,
+CompositorSession::CompositorSession(nsBaseWidget* aWidget,
+                                     CompositorWidgetDelegate* aDelegate,
                                      CompositorBridgeChild* aChild,
                                      const LayersId& aRootLayerTreeId)
-    : mCompositorWidgetDelegate(aDelegate),
+    : mWidget(aWidget),
+      mCompositorWidgetDelegate(aDelegate),
       mCompositorBridgeChild(aChild),
       mRootLayerTreeId(aRootLayerTreeId) {}
 
@@ -30,5 +36,15 @@ CompositorBridgeChild* CompositorSession::GetCompositorBridgeChild() {
   return mCompositorBridgeChild;
 }
 
+#if defined(MOZ_WIDGET_ANDROID)
+void CompositorSession::NotifyDisablingWebRender() {
+  if (!mWidget) {
+    return;
+  }
+  nsWindow* window = static_cast<nsWindow*>(mWidget);
+  window->NotifyDisablingWebRender();
+}
+#endif  // defined(MOZ_WIDGET_ANDROID)
+
 }  // namespace layers
 }  // namespace mozilla

gfx/ipc/CompositorSession.h

@@ -14,7 +14,7 @@
 #  include "mozilla/layers/UiCompositorControllerChild.h"
 #endif  // defined(MOZ_WIDGET_ANDROID)
 
-class nsIWidget;
+class nsBaseWidget;
 
 namespace mozilla {
 namespace widget {
@@ -80,14 +80,17 @@ class CompositorSession {
   RefPtr<UiCompositorControllerChild> GetUiCompositorControllerChild() {
     return mUiCompositorControllerChild;
   }
+
+  void NotifyDisablingWebRender();
 #endif  // defined(MOZ_WIDGET_ANDROID)
  protected:
-  CompositorSession(CompositorWidgetDelegate* aDelegate,
+  CompositorSession(nsBaseWidget* aWidget, CompositorWidgetDelegate* aDelegate,
                     CompositorBridgeChild* aChild,
                     const LayersId& aRootLayerTreeId);
   virtual ~CompositorSession();
 
  protected:
+  nsBaseWidget* mWidget;
   CompositorWidgetDelegate* mCompositorWidgetDelegate;
   RefPtr<CompositorBridgeChild> mCompositorBridgeChild;
   LayersId mRootLayerTreeId;

gfx/ipc/GPUProcessManager.cpp

@@ -455,6 +455,15 @@ void GPUProcessManager::DisableWebRender(wr::WebRenderError aError) {
   }
   gfx::gfxVars::SetUseWebRender(false);
 
+#if defined(MOZ_WIDGET_ANDROID)
+  // If aError is not wr::WebRenderError::INITIALIZE, nsWindow does not
+  // re-create LayerManager. Needs to trigger re-creating LayerManager on
+  // android
+  if (aError != wr::WebRenderError::INITIALIZE) {
+    NotifyDisablingWebRender();
+  }
+#endif
+
   if (mProcess) {
     OnRemoteProcessDeviceReset(mProcess);
   } else {
@@ -641,6 +650,18 @@ void GPUProcessManager::RebuildInProcessSessions() {
   }
 }
 
+void GPUProcessManager::NotifyDisablingWebRender() {
+#if defined(MOZ_WIDGET_ANDROID)
+  for (const auto& session : mRemoteSessions) {
+    session->NotifyDisablingWebRender();
+  }
+
+  for (const auto& session : mInProcessSessions) {
+    session->NotifyDisablingWebRender();
+  }
+#endif
+}
+
 void GPUProcessManager::NotifyRemoteActorDestroyed(
     const uint64_t& aProcessToken) {
   if (!NS_IsMainThread()) {

gfx/ipc/GPUProcessManager.h

@@ -210,6 +210,8 @@ class GPUProcessManager final : public GPUProcessHost::Listener {
   void RebuildRemoteSessions();
   void RebuildInProcessSessions();
 
+  void NotifyDisablingWebRender();
+
   void FallbackToSoftware(const char* aMessage);
 
  private:

gfx/ipc/InProcessCompositorSession.cpp

@@ -13,13 +13,12 @@ namespace mozilla {
 namespace layers {
 
 InProcessCompositorSession::InProcessCompositorSession(
-    widget::CompositorWidget* aWidget, nsBaseWidget* baseWidget,
+    nsBaseWidget* aWidget, widget::CompositorWidget* aCompositorWidget,
     CompositorBridgeChild* aChild, CompositorBridgeParent* aParent)
-    : CompositorSession(aWidget->AsDelegate(), aChild,
+    : CompositorSession(aWidget, aCompositorWidget->AsDelegate(), aChild,
                         aParent->RootLayerTreeId()),
-      mWidget(baseWidget),
       mCompositorBridgeParent(aParent),
-      mCompositorWidget(aWidget) {
+      mCompositorWidget(aCompositorWidget) {
   GPUProcessManager::Get()->RegisterInProcessSession(this);
 }
 
@@ -45,7 +44,7 @@ RefPtr<InProcessCompositorSession> InProcessCompositorSession::Create(
           aLayerManager, aNamespace);
   MOZ_ASSERT(child);
 
-  return new InProcessCompositorSession(widget, aWidget, child, parent);
+  return new InProcessCompositorSession(aWidget, widget, child, parent);
 }
 
 void InProcessCompositorSession::NotifySessionLost() {

gfx/ipc/InProcessCompositorSession.h

@@ -34,13 +34,12 @@ class InProcessCompositorSession final : public CompositorSession {
   void NotifySessionLost();
 
  private:
-  InProcessCompositorSession(widget::CompositorWidget* aWidget,
-                             nsBaseWidget* baseWidget,
+  InProcessCompositorSession(nsBaseWidget* aWidget,
+                             widget::CompositorWidget* aCompositorWidget,
                              CompositorBridgeChild* aChild,
                              CompositorBridgeParent* aParent);
 
  private:
-  nsBaseWidget* mWidget;
   RefPtr<CompositorBridgeParent> mCompositorBridgeParent;
   RefPtr<CompositorWidget> mCompositorWidget;
 };

gfx/ipc/RemoteCompositorSession.cpp

@@ -24,8 +24,7 @@ RemoteCompositorSession::RemoteCompositorSession(
     nsBaseWidget* aWidget, CompositorBridgeChild* aChild,
     CompositorWidgetDelegate* aWidgetDelegate, APZCTreeManagerChild* aAPZ,
     const LayersId& aRootLayerTreeId)
-    : CompositorSession(aWidgetDelegate, aChild, aRootLayerTreeId),
-      mWidget(aWidget),
+    : CompositorSession(aWidget, aWidgetDelegate, aChild, aRootLayerTreeId),
       mAPZ(aAPZ) {
   MOZ_ASSERT(!gfxPlatform::IsHeadless());
   GPUProcessManager::Get()->RegisterRemoteProcessSession(this);

gfx/ipc/RemoteCompositorSession.h

@@ -31,7 +31,6 @@ class RemoteCompositorSession final : public CompositorSession {
   void NotifySessionLost();
 
  private:
-  nsBaseWidget* mWidget;
   RefPtr<APZCTreeManagerChild> mAPZ;
   RefPtr<GeckoContentController> mContentController;
 };

gfx/webrender_bindings/RenderAndroidSurfaceTextureHostOGL.cpp

@@ -103,6 +103,12 @@ void RenderAndroidSurfaceTextureHostOGL::DeleteTextureHandle() {
 }
 
 bool RenderAndroidSurfaceTextureHostOGL::EnsureAttachedToGLContext() {
+  // During handling WebRenderError, GeckoSurfaceTexture should not be attached
+  // to GLContext.
+  if (RenderThread::Get()->IsHandlingWebRenderError()) {
+    return false;
+  }
+
   if (mAttachedToGLContext) {
     return true;
   }
@@ -197,7 +203,9 @@ void RenderAndroidSurfaceTextureHostOGL::NofityForUse() {
     // WebRender, instead rendered to WebGL.
     // It is not a good way. But it is same to Compositor rendering.
     MOZ_ASSERT(!mSurfTex || !mSurfTex->IsSingleBuffer());
-    EnsureAttachedToGLContext();
+    if (!EnsureAttachedToGLContext()) {
+      return;
+    }
     mPrepareStatus = STATUS_PREPARE_NEEDED;
   }
 }
@@ -207,7 +215,9 @@ void RenderAndroidSurfaceTextureHostOGL::NotifyNotUsed() {
 
   if (mSurfTex && mSurfTex->IsSingleBuffer() &&
       mPrepareStatus == STATUS_PREPARED) {
-    EnsureAttachedToGLContext();
+    if (!EnsureAttachedToGLContext()) {
+      return;
+    }
     // Release SurfaceTexture's buffer to client side.
     mGL->MakeCurrent();
     mSurfTex->ReleaseTexImage();
@@ -215,7 +225,9 @@ void RenderAndroidSurfaceTextureHostOGL::NotifyNotUsed() {
     // This could happen when video frame was skipped. UpdateTexImage() neeeds
     // to be called for adjusting SurfaceTexture's buffer status.
     MOZ_ASSERT(!mSurfTex->IsSingleBuffer());
-    EnsureAttachedToGLContext();
+    if (!EnsureAttachedToGLContext()) {
+      return;
+    }
     mSurfTex->UpdateTexImage();
   }
 

gfx/webrender_bindings/RenderCompositorEGL.cpp

@@ -55,7 +55,13 @@ RenderCompositorEGL::RenderCompositorEGL(
     RefPtr<widget::CompositorWidget> aWidget)
     : RenderCompositor(std::move(aWidget)), mEGLSurface(EGL_NO_SURFACE) {}
 
-RenderCompositorEGL::~RenderCompositorEGL() { DestroyEGLSurface(); }
+RenderCompositorEGL::~RenderCompositorEGL() {
+#ifdef MOZ_WIDGET_ANDROID
+  java::GeckoSurfaceTexture::DestroyUnused((int64_t)gl());
+  java::GeckoSurfaceTexture::DetachAllFromGLContext((int64_t)gl());
+#endif
+  DestroyEGLSurface();
+}
 
 bool RenderCompositorEGL::BeginFrame() {
 #ifdef MOZ_WAYLAND

gfx/webrender_bindings/RenderThread.cpp

@@ -13,6 +13,7 @@
 #include "mozilla/gfx/GPUParent.h"
 #include "mozilla/layers/CompositorThread.h"
 #include "mozilla/layers/CompositorBridgeParent.h"
+#include "mozilla/layers/CompositorManagerParent.h"
 #include "mozilla/layers/WebRenderBridgeParent.h"
 #include "mozilla/layers/SharedSurfacesParent.h"
 #include "mozilla/StaticPtr.h"
@@ -28,6 +29,7 @@
 
 #ifdef MOZ_WIDGET_ANDROID
 #  include "GLLibraryEGL.h"
+#  include "GeneratedJNIWrappers.h"
 #endif
 
 using namespace mozilla;
@@ -46,7 +48,8 @@ RenderThread::RenderThread(base::Thread* aThread)
       mWindowInfos("RenderThread.mWindowInfos"),
       mRenderTextureMapLock("RenderThread.mRenderTextureMapLock"),
       mHasShutdown(false),
-      mHandlingDeviceReset(false) {}
+      mHandlingDeviceReset(false),
+      mHandlingWebRenderError(false) {}
 
 RenderThread::~RenderThread() {
   MOZ_ASSERT(mRenderTexturesDeferred.empty());
@@ -808,6 +811,34 @@ void RenderThread::SimulateDeviceReset() {
   }
 }
 
+static void DoNotifyWebRenderError(WebRenderError aError) {
+  layers::CompositorManagerParent::NotifyWebRenderError(aError);
+}
+
+void RenderThread::HandleWebRenderError(WebRenderError aError) {
+  if (mHandlingWebRenderError) {
+    return;
+  }
+
+  layers::CompositorThreadHolder::Loop()->PostTask(NewRunnableFunction(
+      "DoNotifyWebRenderErrorRunnable", &DoNotifyWebRenderError, aError));
+  {
+    MutexAutoLock lock(mRenderTextureMapLock);
+    mRenderTexturesDeferred.clear();
+    for (const auto& entry : mRenderTextures) {
+      entry.second->ClearCachedResources();
+    }
+  }
+  mHandlingWebRenderError = true;
+  // WebRender is going to be disabled by
+  // GPUProcessManager::NotifyWebRenderError()
+}
+
+bool RenderThread::IsHandlingWebRenderError() {
+  MOZ_ASSERT(IsInRenderThread());
+  return mHandlingWebRenderError;
+}
+
 gl::GLContext* RenderThread::SharedGL() {
   MOZ_ASSERT(IsInRenderThread());
   if (!mSharedGL) {

gfx/webrender_bindings/RenderThread.h

@@ -255,6 +255,11 @@ class RenderThread final {
   /// Can be called from any thread.
   void SimulateDeviceReset();
 
+  /// Can only be called from the render thread.
+  void HandleWebRenderError(WebRenderError aError);
+  /// Can only be called from the render thread.
+  bool IsHandlingWebRenderError();
+
   size_t RendererCount();
 
   void SetCompositionRecorderForWindow(
@@ -314,6 +319,7 @@ class RenderThread final {
   bool mHasShutdown;
 
   bool mHandlingDeviceReset;
+  bool mHandlingWebRenderError;
 };
 
 }  // namespace wr

gfx/webrender_bindings/RendererOGL.cpp

@@ -10,7 +10,6 @@
 #include "mozilla/gfx/gfxVars.h"
 #include "mozilla/gfx/Types.h"
 #include "mozilla/layers/CompositorBridgeParent.h"
-#include "mozilla/layers/CompositorManagerParent.h"
 #include "mozilla/layers/CompositorThread.h"
 #include "mozilla/layers/LayersTypes.h"
 #include "mozilla/webrender/RenderCompositor.h"
@@ -120,7 +119,8 @@ bool RendererOGL::UpdateAndRender(const Maybe<gfx::IntSize>& aReadbackSize,
 
   if (!wr_renderer_render(mRenderer, size.width, size.height, aHadSlowFrame,
                           aOutStats)) {
-    NotifyWebRenderError(WebRenderError::RENDER);
+    RenderThread::Get()->HandleWebRenderError(WebRenderError::RENDER);
+    return false;
   }
 
   if (aReadbackBuffer.isSome()) {
@@ -216,14 +216,5 @@ void RendererOGL::AccumulateMemoryReport(MemoryReport* aReport) {
   aReport->swap_chain += swapChainSize;
 }
 
-static void DoNotifyWebRenderError(WebRenderError aError) {
-  layers::CompositorManagerParent::NotifyWebRenderError(aError);
-}
-
-void RendererOGL::NotifyWebRenderError(WebRenderError aError) {
-  layers::CompositorThreadHolder::Loop()->PostTask(NewRunnableFunction(
-      "DoNotifyWebRenderErrorRunnable", &DoNotifyWebRenderError, aError));
-}
-
 }  // namespace wr
 }  // namespace mozilla

gfx/webrender_bindings/RendererOGL.h

@@ -104,8 +104,6 @@ class RendererOGL {
   gl::GLContext* gl() const;
 
  protected:
-  void NotifyWebRenderError(WebRenderError aError);
-
   RefPtr<RenderThread> mThread;
   UniquePtr<RenderCompositor> mCompositor;
   wr::Renderer* mRenderer;

widget/android/nsWindow.cpp

@@ -1428,7 +1428,8 @@ nsWindow::nsWindow()
     : mScreenId(0),  // Use 0 (primary screen) as the default value.
       mIsVisible(false),
       mParent(nullptr),
-      mIsFullScreen(false) {}
+      mIsFullScreen(false),
+      mIsDisablingWebRender(false) {}
 
 nsWindow::~nsWindow() {
   gTopLevelWindows.RemoveElement(this);
@@ -1843,6 +1844,13 @@ mozilla::layers::LayerManager* nsWindow::GetLayerManager(
   if (mLayerManager) {
     return mLayerManager;
   }
+
+  if (mIsDisablingWebRender) {
+    CreateLayerManager();
+    mIsDisablingWebRender = false;
+    return mLayerManager;
+  }
+
   return nullptr;
 }
 
@@ -1877,6 +1885,11 @@ void nsWindow::CreateLayerManager() {
   }
 }
 
+void nsWindow::NotifyDisablingWebRender() {
+  mIsDisablingWebRender = true;
+  RedrawAll();
+}
+
 void nsWindow::OnSizeChanged(const gfx::IntSize& aSize) {
   ALOG("nsWindow: %p OnSizeChanged [%d %d]", (void*)this, aSize.width,
        aSize.height);

widget/android/nsWindow.h

@@ -220,6 +220,8 @@ class nsWindow final : public nsBaseWidget {
     return nullptr;
   }
 
+  void NotifyDisablingWebRender();
+
   //
   // nsIWidget
   //
@@ -345,6 +347,7 @@ class nsWindow final : public nsBaseWidget {
   nsCOMPtr<nsIIdleServiceInternal> mIdleService;
 
   bool mIsFullScreen;
+  bool mIsDisablingWebRender;
 
   bool UseExternalCompositingSurface() const override { return true; }