Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-09 19:35:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1757186: Unnest QM shutdown timers and extend crash timer scope. r=dom-storage-reviewers,janv

Browse Source 
Depends on D139616

Differential Revision: https://phabricator.services.mozilla.com/D139795
This commit is contained in:
Jens Stutte 2022-04-07 17:01:56 +00:00
parent e9d7d40db7
commit 79f0e6cf55
2 changed files with 103 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
dom/quota/ActorsParent.cpp
View File

@ -177,7 +177,7 @@
* If shutdown takes this long, kill actors of a quota client, to avoid reaching
* the crash timeout.
*/
#define SHUTDOWN_FORCE_KILL_TIMEOUT_MS 5000
#define SHUTDOWN_KILL_ACTORS_TIMEOUT_MS 5000
/**
* Automatically crash the browser if shutdown of a quota client takes this
@ -189,7 +189,11 @@
* not hide them. On the other hand this value is less than 60 seconds which is
* used by nsTerminator to crash a hung main process.
*/
#define SHUTDOWN_FORCE_CRASH_TIMEOUT_MS 45000
#define SHUTDOWN_CRASH_BROWSER_TIMEOUT_MS 45000
static_assert(
SHUTDOWN_CRASH_BROWSER_TIMEOUT_MS > SHUTDOWN_KILL_ACTORS_TIMEOUT_MS,
"The kill actors timeout must be shorter than the crash browser one.");
// profile-before-change, when we need to shut down quota manager
#define PROFILE_BEFORE_CHANGE_QM_OBSERVER_ID "profile-before-change-qm"
@ -3700,13 +3704,6 @@ nsresult QuotaManager::Init() {
nsCOMPtr<nsIThread>, MOZ_SELECT_OVERLOAD(NS_NewNamedThread),
"QuotaManager IO"));
// Make a timer here to avoid potential failures later. We don't actually
// initialize the timer until shutdown.
nsCOMPtr shutdownTimer = NS_NewTimer();
QM_TRY(OkIf(shutdownTimer), Err(NS_ERROR_FAILURE));
mShutdownTimer.init(WrapNotNullUnchecked(std::move(shutdownTimer)));
static_assert(Client::IDB == 0 && Client::DOMCACHE == 1 && Client::SDB == 2 &&
Client::LS == 3 && Client::TYPE_MAX == 4,
"Fix the registration!");
@ -3835,6 +3832,65 @@ void QuotaManager::Shutdown() {
mShutdownStarted = true;
};
nsCOMPtr<nsITimer> crashBrowserTimer;
auto crashBrowserTimerCallback = [](nsITimer* aTimer, void* aClosure) {
auto* const quotaManager = static_cast<QuotaManager*>(aClosure);
nsCString annotation;
for (Client::Type type : quotaManager->AllClientTypes()) {
auto& quotaClient = *(*quotaManager->mClients)[type];
if (!quotaClient.IsShutdownCompleted()) {
annotation.AppendPrintf("%s: %s\nIntermediate steps:\n%s\n\n",
Client::TypeToText(type).get(),
quotaClient.GetShutdownStatus().get(),
quotaManager->mShutdownSteps[type].get());
}
}
{
MutexAutoLock lock(quotaManager->mQuotaMutex);
annotation.AppendPrintf("QM: %zu normal origin ops pending\n",
gNormalOriginOps->Length());
#ifdef MOZ_COLLECTING_RUNNABLE_TELEMETRY
for (const auto& op : *gNormalOriginOps) {
nsCString name;
op->GetName(name);
annotation.AppendPrintf("Op: %s pending\n", name.get());
}
#endif
annotation.AppendPrintf("Intermediate steps:\n%s\n",
quotaManager->mQuotaManagerShutdownSteps.get());
}
CrashReporter::AnnotateCrashReport(
CrashReporter::Annotation::QuotaManagerShutdownTimeout, annotation);
MOZ_CRASH("Quota manager shutdown timed out");
};
auto startCrashBrowserTimer = [&]() {
crashBrowserTimer = NS_NewTimer();
MOZ_ASSERT(crashBrowserTimer);
if (crashBrowserTimer) {
RecordQuotaManagerShutdownStep("startCrashBrowserTimer"_ns);
MOZ_ALWAYS_SUCCEEDS(crashBrowserTimer->InitWithNamedFuncCallback(
crashBrowserTimerCallback, this, SHUTDOWN_CRASH_BROWSER_TIMEOUT_MS,
nsITimer::TYPE_ONE_SHOT,
"quota::QuotaManager::Shutdown::crashBrowserTimer"));
}
};
auto stopCrashBrowserTimer = [&]() {
if (crashBrowserTimer) {
RecordQuotaManagerShutdownStep("stopCrashBrowserTimer"_ns);
QM_WARNONLY_TRY(QM_TO_RESULT(crashBrowserTimer->Cancel()));
}
};
auto initiateShutdownWorkThreads = [this]() {
RecordQuotaManagerShutdownStep("initiateShutdownWorkThreads"_ns);
bool needsToWait = false;
@ -3848,60 +3904,12 @@ void QuotaManager::Shutdown() {
return needsToWait;
};
auto isAllClientsShutdownComplete = [this] {
return std::all_of(AllClientTypes().cbegin(), AllClientTypes().cend(),
[&self = *this](const auto type) {
return (*self.mClients)[type]->IsShutdownCompleted();
});
};
auto forceKillTimerCallback = [](nsITimer* aTimer, void* aClosure) {
auto forceCrashTimerCallback = [](nsITimer* aTimer, void* aClosure) {
auto* const quotaManager = static_cast<QuotaManager*>(aClosure);
nsCString annotation;
{
for (Client::Type type : quotaManager->AllClientTypes()) {
auto& quotaClient = *(*quotaManager->mClients)[type];
if (!quotaClient.IsShutdownCompleted()) {
annotation.AppendPrintf("%s: %s\nIntermediate steps:\n%s\n\n",
Client::TypeToText(type).get(),
quotaClient.GetShutdownStatus().get(),
quotaManager->mShutdownSteps[type].get());
}
}
if (gNormalOriginOps) {
MutexAutoLock lock(quotaManager->mQuotaMutex);
annotation.AppendPrintf("QM: %zu normal origin ops pending\n",
gNormalOriginOps->Length());
#ifdef MOZ_COLLECTING_RUNNABLE_TELEMETRY
for (const auto& op : *gNormalOriginOps) {
nsCString name;
op->GetName(name);
annotation.AppendPrintf("Op: %s pending\n", name.get());
}
#endif
annotation.AppendPrintf(
"Intermediate steps:\n%s\n",
quotaManager->mQuotaManagerShutdownSteps.get());
}
}
CrashReporter::AnnotateCrashReport(
CrashReporter::Annotation::QuotaManagerShutdownTimeout, annotation);
MOZ_CRASH("Quota manager shutdown timed out");
};
nsCOMPtr<nsITimer> killActorsTimer;
auto killActorsTimerCallback = [](nsITimer* aTimer, void* aClosure) {
auto* const quotaManager = static_cast<QuotaManager*>(aClosure);
// If we see a QM shutdown hang we must have executed this callback for
// sure, but let's remind ourselves in the annotation, too.
quotaManager->RecordQuotaManagerShutdownStep("forceKillTimerCallback"_ns);
quotaManager->RecordQuotaManagerShutdownStep("killActorsTimerCallback"_ns);
// XXX: This abort is a workaround to unblock shutdown, which
// ought to be removed by bug 1682326. We probably need more
@ -3912,10 +3920,32 @@ void QuotaManager::Shutdown() {
for (Client::Type type : quotaManager->AllClientTypes()) {
quotaManager->GetClient(type)->ForceKillActors();
}
};
MOZ_ALWAYS_SUCCEEDS(aTimer->InitWithNamedFuncCallback(
forceCrashTimerCallback, aClosure, SHUTDOWN_FORCE_CRASH_TIMEOUT_MS,
nsITimer::TYPE_ONE_SHOT, "quota::QuotaManager::ForceCrashTimer"));
auto startKillActorsTimer = [&]() {
killActorsTimer = NS_NewTimer();
MOZ_ASSERT(killActorsTimer);
if (killActorsTimer) {
RecordQuotaManagerShutdownStep("startKillActorsTimer"_ns);
MOZ_ALWAYS_SUCCEEDS(killActorsTimer->InitWithNamedFuncCallback(
killActorsTimerCallback, this, SHUTDOWN_KILL_ACTORS_TIMEOUT_MS,
nsITimer::TYPE_ONE_SHOT,
"quota::QuotaManager::Shutdown::killActorsTimer"));
}
};
auto stopKillActorsTimer = [&]() {
if (killActorsTimer) {
RecordQuotaManagerShutdownStep("stopKillActorsTimer"_ns);
QM_WARNONLY_TRY(QM_TO_RESULT(killActorsTimer->Cancel()));
}
};
auto isAllClientsShutdownComplete = [this] {
return std::all_of(AllClientTypes().cbegin(), AllClientTypes().cend(),
[&self = *this](const auto type) {
return (*self.mClients)[type]->IsShutdownCompleted();
});
};
auto shutdownAndJoinWorkThreads = [this]() {
@ -3926,6 +3956,7 @@ void QuotaManager::Shutdown() {
};
auto shutdownAndJoinIOThread = [this]() {
RecordQuotaManagerShutdownStep("shutdownAndJoinIOThread"_ns);
// NB: It's very important that runnable is destroyed on this thread
// (i.e. after we join the IO thread) because we can't release the
// QuotaManager on the IO thread. This should probably use
@ -3944,6 +3975,7 @@ void QuotaManager::Shutdown() {
};
auto invalidatePendingDirectoryLocks = [this]() {
RecordQuotaManagerShutdownStep("invalidatePendingDirectoryLocks"_ns);
for (RefPtr<DirectoryLockImpl>& lock : mPendingDirectoryLocks) {
lock->Invalidate();
}
@ -3953,6 +3985,8 @@ void QuotaManager::Shutdown() {
flagShutdownStarted();
startCrashBrowserTimer();
// XXX: StopIdleMaintenance now just notifies all clients to abort any
// maintenance work.
// This could be done as part of QuotaClient::AbortAllOperations.
@ -3962,31 +3996,25 @@ void QuotaManager::Shutdown() {
initiateShutdownWorkThreads() | static_cast<bool>(gNormalOriginOps);
// If any clients cannot shutdown immediately, spin the event loop while we
// wait on all the threads to close. Our timer may fire during that loop.
// wait on all the threads to close.
if (needsToWait) {
MOZ_ALWAYS_SUCCEEDS(
(*mShutdownTimer)
->InitWithNamedFuncCallback(forceKillTimerCallback, this,
SHUTDOWN_FORCE_KILL_TIMEOUT_MS,
nsITimer::TYPE_ONE_SHOT,
"quota::QuotaManager::ForceKillTimer"));
startKillActorsTimer();
MOZ_ALWAYS_TRUE(SpinEventLoopUntil(
"QuotaManager::Shutdown"_ns, [isAllClientsShutdownComplete]() {
return !gNormalOriginOps && isAllClientsShutdownComplete();
}));
stopKillActorsTimer();
}
shutdownAndJoinWorkThreads();
// Cancel the timer regardless of whether it actually fired.
// XXX bug 1757186: Unnest these timers to extend crash timer duration
// until the end of shutdown.
QM_WARNONLY_TRY(QM_TO_RESULT((*mShutdownTimer)->Cancel()));
shutdownAndJoinIOThread();
invalidatePendingDirectoryLocks();
stopCrashBrowserTimer();
}
void QuotaManager::InitQuotaForOrigin(

3
dom/quota/QuotaManager.h
View File

@ -590,9 +590,6 @@ class QuotaManager final : public BackgroundThreadObject {
nsCOMPtr<mozIStorageConnection> mStorageConnection;
// A timer that gets activated at shutdown to ensure we close all storages.
LazyInitializedOnceNotNull<const nsCOMPtr<nsITimer>> mShutdownTimer;
EnumeratedArray<Client::Type, Client::TYPE_MAX, nsCString> mShutdownSteps;
LazyInitializedOnce<const TimeStamp> mShutdownStartedAt;
Atomic<bool> mShutdownStarted;