Bug 1632845 - Firefox gets stuck when quit from the dock if the user cancels quitting from a beforeunload prompt r=Gijs,spohl,necko-reviewers,geckoview-reviewers,dragana,agi

Return NSTerminateCancel from applicationShouldTerminate when the user chooses to "Stay on Page".

Differential Revision: https://phabricator.services.mozilla.com/D78839

extensions/pref/autoconfig/src/nsReadConfig.cpp

@@ -102,7 +102,10 @@ NS_IMETHODIMP nsReadConfig::Observe(nsISupports* aSubject, const char* aTopic,
         if (NS_FAILED(rv)) {
           nsCOMPtr<nsIAppStartup> appStartup =
               components::AppStartup::Service();
-          if (appStartup) appStartup->Quit(nsIAppStartup::eAttemptQuit);
+          if (appStartup) {
+            bool userAllowedQuit = true;
+            appStartup->Quit(nsIAppStartup::eAttemptQuit, &userAllowedQuit);
+          }
         }
       }
     }

netwerk/ipc/SocketProcessParent.cpp

@@ -86,7 +86,8 @@ void SocketProcessParent::ActorDestroy(ActorDestroyReason aWhy) {
       nsCOMPtr<nsIAppStartup> appService =
           do_GetService("@mozilla.org/toolkit/app-startup;1");
       if (appService) {
-        appService->Quit(nsIAppStartup::eForceQuit);
+        bool userAllowedQuit = true;
+        appService->Quit(nsIAppStartup::eForceQuit, &userAllowedQuit);
       }
     }
   }

toolkit/components/startup/nsAppStartup.cpp

@@ -275,7 +275,8 @@ nsAppStartup::Run(void) {
   // Make sure that the appropriate quit notifications have been dispatched
   // regardless of whether the event loop has spun or not. Note that this call
   // is a no-op if Quit has already been called previously.
-  Quit(eForceQuit);
+  bool userAllowedQuit = true;
+  Quit(eForceQuit, &userAllowedQuit);
 
   nsresult retval = NS_OK;
   if (mozilla::AppShutdown::IsRestarting()) {
@@ -286,9 +287,14 @@ nsAppStartup::Run(void) {
 }
 
 NS_IMETHODIMP
-nsAppStartup::Quit(uint32_t aMode) {
+nsAppStartup::Quit(uint32_t aMode, bool* aUserAllowedQuit) {
   uint32_t ferocity = (aMode & 0xF);
 
+  // If the shutdown was cancelled due to a hidden window or
+  // because one of the windows was not permitted to be closed,
+  // return NS_OK with |aUserAllowedQuit| = false.
+  *aUserAllowedQuit = false;
+
   // Quit the application. We will asynchronously call the appshell's
   // Exit() method via nsAppExitEvent to allow one last pass
   // through any events in the queue. This guarantees a tidy cleanup.
@@ -349,7 +355,9 @@ nsAppStartup::Quit(uint32_t aMode) {
           windowEnumerator->GetNext(getter_AddRefs(window));
           nsCOMPtr<nsPIDOMWindowOuter> domWindow(do_QueryInterface(window));
           if (domWindow) {
-            if (!domWindow->CanClose()) return NS_OK;
+            if (!domWindow->CanClose()) {
+              return NS_OK;
+            }
           }
           windowEnumerator->HasMoreElements(&more);
         }
@@ -359,6 +367,7 @@ nsAppStartup::Quit(uint32_t aMode) {
     PROFILER_ADD_MARKER("Shutdown start", OTHER);
     mozilla::RecordShutdownStartTimeStamp();
 
+    *aUserAllowedQuit = true;
     mShuttingDown = true;
     auto shutdownMode = ((aMode & eRestart) != 0)
                             ? mozilla::AppShutdownMode::Restart
@@ -496,7 +505,10 @@ nsAppStartup::ExitLastWindowClosingSurvivalArea(void) {
   NS_ASSERTION(mConsiderQuitStopper > 0, "consider quit stopper out of bounds");
   --mConsiderQuitStopper;
 
-  if (mRunning) Quit(eConsiderQuit);
+  if (mRunning) {
+    bool userAllowedQuit = false;
+    Quit(eConsiderQuit, &userAllowedQuit);
+  }
 
   return NS_OK;
 }
@@ -940,7 +952,8 @@ nsAppStartup::TrackStartupCrashEnd() {
 NS_IMETHODIMP
 nsAppStartup::RestartInSafeMode(uint32_t aQuitMode) {
   PR_SetEnv("MOZ_SAFE_MODE_RESTART=1");
-  this->Quit(aQuitMode | nsIAppStartup::eRestart);
+  bool userAllowedQuit = false;
+  this->Quit(aQuitMode | nsIAppStartup::eRestart, &userAllowedQuit);
 
   return NS_OK;
 }

toolkit/components/startup/public/nsIAppStartup.idl

@@ -123,8 +123,12 @@ interface nsIAppStartup : nsISupports
      * @param aMode
      *        This parameter modifies how the app is shutdown, and it is
      *        constructed from the constants defined above.
+     *
+     * @return false if the shutdown was cancelled due to the presence
+     *         of a hidden window or if the user disallowed a window
+     *         to be closed.
      */
-    void quit(in uint32_t aMode);
+    bool quit(in uint32_t aMode);
 
     /**
      * True if the application is in the process of shutting down.

toolkit/xre/MacApplicationDelegate.mm

@@ -299,7 +299,13 @@ void ProcessPendingGetURLAppleEvents() {
   if (abortQuit) return NSTerminateCancel;
 
   nsCOMPtr<nsIAppStartup> appService = do_GetService("@mozilla.org/toolkit/app-startup;1");
-  if (appService) appService->Quit(nsIAppStartup::eForceQuit);
+  if (appService) {
+    bool userAllowedQuit = true;
+    appService->Quit(nsIAppStartup::eForceQuit, &userAllowedQuit);
+    if (!userAllowedQuit) {
+      return NSTerminateCancel;
+    }
+  }
 
   return NSTerminateNow;
 }

toolkit/xre/nsNativeAppSupportUnix.cpp

@@ -265,7 +265,8 @@ void nsNativeAppSupportUnix::DoInteract() {
         do_GetService("@mozilla.org/toolkit/app-startup;1");
 
     if (appService) {
-      appService->Quit(nsIAppStartup::eForceQuit);
+      bool userAllowedQuit = true;
+      appService->Quit(nsIAppStartup::eForceQuit, &userAllowedQuit);
     }
   } else {
     if (mClientState != STATE_SHUTDOWN_CANCELLED) {
@@ -353,7 +354,8 @@ void nsNativeAppSupportUnix::DieCB(SmcConn smc_conn, SmPointer client_data) {
       do_GetService("@mozilla.org/toolkit/app-startup;1");
 
   if (appService) {
-    appService->Quit(nsIAppStartup::eForceQuit);
+    bool userAllowedQuit = false;
+    appService->Quit(nsIAppStartup::eForceQuit, &userAllowedQuit);
   }
   // Quit causes the shutdown to begin but the shutdown process is asynchronous
   // so we can't DisconnectFromSM() yet

widget/android/nsAppShell.cpp

@@ -219,7 +219,8 @@ class GeckoThreadSupport final
     nsCOMPtr<nsIAppStartup> appStartup = components::AppStartup::Service();
 
     if (appStartup) {
-      appStartup->Quit(nsIAppStartup::eForceQuit);
+      bool userAllowedQuit = true;
+      appStartup->Quit(nsIAppStartup::eForceQuit, &userAllowedQuit);
     }
   }
 

widget/cocoa/nsMenuBarX.mm

@@ -918,7 +918,8 @@ static BOOL gMenuItemsExecuteCommands = YES;
     } else {
       nsCOMPtr<nsIAppStartup> appStartup = mozilla::components::AppStartup::Service();
       if (appStartup) {
-        appStartup->Quit(nsIAppStartup::eAttemptQuit);
+        bool userAllowedQuit = true;
+        appStartup->Quit(nsIAppStartup::eAttemptQuit, &userAllowedQuit);
       }
     }
     return;