2013-11-22 19:17:29 +00:00
|
|
|
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
2014-06-30 15:39:45 +00:00
|
|
|
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
|
2013-11-22 19:17:29 +00:00
|
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
|
2013-12-09 02:52:54 +00:00
|
|
|
#include "mozilla/ArrayUtils.h"
|
2013-11-22 19:17:29 +00:00
|
|
|
#include "mozilla/BackgroundHangMonitor.h"
|
|
|
|
#include "mozilla/LinkedList.h"
|
|
|
|
#include "mozilla/Monitor.h"
|
2013-11-22 19:17:31 +00:00
|
|
|
#include "mozilla/Move.h"
|
2015-03-09 08:58:00 +00:00
|
|
|
#include "mozilla/Preferences.h"
|
2013-11-22 19:17:29 +00:00
|
|
|
#include "mozilla/StaticPtr.h"
|
2013-11-22 19:17:31 +00:00
|
|
|
#include "mozilla/Telemetry.h"
|
2013-11-22 19:17:31 +00:00
|
|
|
#include "mozilla/ThreadHangStats.h"
|
2013-11-22 19:17:29 +00:00
|
|
|
#include "mozilla/ThreadLocal.h"
|
|
|
|
|
|
|
|
#include "prinrval.h"
|
|
|
|
#include "prthread.h"
|
2013-11-22 19:17:31 +00:00
|
|
|
#include "ThreadStackHelper.h"
|
2015-03-09 08:58:00 +00:00
|
|
|
#include "nsIObserverService.h"
|
|
|
|
#include "nsIObserver.h"
|
|
|
|
#include "mozilla/Services.h"
|
|
|
|
#include "nsXULAppAPI.h"
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
#include <algorithm>
|
|
|
|
|
2015-03-09 08:58:00 +00:00
|
|
|
// Activate BHR only for one every BHR_BETA_MOD users.
|
2016-01-21 15:37:33 +00:00
|
|
|
// This is now 100% of Beta population for the Beta 45/46 e10s A/B trials
|
|
|
|
// It can be scaled back again in the future
|
|
|
|
#define BHR_BETA_MOD 1;
|
2015-03-09 08:58:00 +00:00
|
|
|
|
2015-10-16 05:04:00 +00:00
|
|
|
// Maximum depth of the call stack in the reported thread hangs. This value represents
|
2015-10-29 07:05:00 +00:00
|
|
|
// the 99.9th percentile of the thread hangs stack depths reported by Telemetry.
|
|
|
|
static const size_t kMaxThreadHangStackDepth = 30;
|
2015-10-16 05:04:00 +00:00
|
|
|
|
|
|
|
// An utility comparator function used by std::unique to collapse "(* script)" entries in
|
|
|
|
// a vector representing a call stack.
|
|
|
|
bool StackScriptEntriesCollapser(const char* aStackEntry, const char *aAnotherStackEntry)
|
|
|
|
{
|
|
|
|
return !strcmp(aStackEntry, aAnotherStackEntry) &&
|
|
|
|
(!strcmp(aStackEntry, "(chrome script)") || !strcmp(aStackEntry, "(content script)"));
|
|
|
|
}
|
|
|
|
|
2013-11-22 19:17:29 +00:00
|
|
|
namespace mozilla {
|
|
|
|
|
|
|
|
/**
|
|
|
|
* BackgroundHangManager is the global object that
|
|
|
|
* manages all instances of BackgroundHangThread.
|
|
|
|
*/
|
2015-03-09 08:58:00 +00:00
|
|
|
class BackgroundHangManager : public nsIObserver
|
2013-11-22 19:17:29 +00:00
|
|
|
{
|
|
|
|
private:
|
|
|
|
// Background hang monitor thread function
|
|
|
|
static void MonitorThread(void* aData)
|
|
|
|
{
|
|
|
|
PR_SetCurrentThreadName("BgHangManager");
|
2013-10-15 09:43:08 +00:00
|
|
|
|
2013-11-22 19:17:30 +00:00
|
|
|
/* We do not hold a reference to BackgroundHangManager here
|
|
|
|
because the monitor thread only exists as long as the
|
|
|
|
BackgroundHangManager instance exists. We stop the monitor
|
|
|
|
thread in the BackgroundHangManager destructor, and we can
|
|
|
|
only get to the destructor if we don't hold a reference here. */
|
|
|
|
static_cast<BackgroundHangManager*>(aData)->RunMonitorThread();
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Hang monitor thread
|
|
|
|
PRThread* mHangMonitorThread;
|
|
|
|
// Stop hang monitoring
|
|
|
|
bool mShutdown;
|
|
|
|
|
|
|
|
BackgroundHangManager(const BackgroundHangManager&);
|
|
|
|
BackgroundHangManager& operator=(const BackgroundHangManager&);
|
|
|
|
void RunMonitorThread();
|
|
|
|
|
|
|
|
public:
|
2015-03-09 08:58:00 +00:00
|
|
|
NS_DECL_THREADSAFE_ISUPPORTS
|
|
|
|
NS_DECL_NSIOBSERVER
|
2013-11-22 19:17:29 +00:00
|
|
|
static StaticRefPtr<BackgroundHangManager> sInstance;
|
2014-07-30 07:24:00 +00:00
|
|
|
static bool sProhibited;
|
2015-03-09 08:58:00 +00:00
|
|
|
static bool sDisabled;
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
// Lock for access to members of this class
|
|
|
|
Monitor mLock;
|
|
|
|
// Current time as seen by hang monitors
|
|
|
|
PRIntervalTime mIntervalNow;
|
|
|
|
// List of BackgroundHangThread instances associated with each thread
|
|
|
|
LinkedList<BackgroundHangThread> mHangThreads;
|
|
|
|
|
|
|
|
void Shutdown()
|
|
|
|
{
|
|
|
|
MonitorAutoLock autoLock(mLock);
|
|
|
|
mShutdown = true;
|
|
|
|
autoLock.Notify();
|
|
|
|
}
|
|
|
|
|
|
|
|
void Wakeup()
|
|
|
|
{
|
2013-11-22 19:17:30 +00:00
|
|
|
// PR_CreateThread could have failed earlier
|
|
|
|
if (mHangMonitorThread) {
|
|
|
|
// Use PR_Interrupt to avoid potentially taking a lock
|
|
|
|
PR_Interrupt(mHangMonitorThread);
|
|
|
|
}
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BackgroundHangManager();
|
2014-06-19 02:29:00 +00:00
|
|
|
private:
|
2015-03-09 08:58:00 +00:00
|
|
|
virtual ~BackgroundHangManager();
|
2013-11-22 19:17:29 +00:00
|
|
|
};
|
|
|
|
|
2015-03-09 08:58:00 +00:00
|
|
|
NS_IMPL_ISUPPORTS(BackgroundHangManager, nsIObserver)
|
|
|
|
|
|
|
|
NS_IMETHODIMP
|
|
|
|
BackgroundHangManager::Observe(nsISupports* aSubject, const char* aTopic, const char16_t* aData) {
|
|
|
|
NS_ENSURE_TRUE(!strcmp(aTopic, "profile-after-change"), NS_ERROR_UNEXPECTED);
|
|
|
|
BackgroundHangMonitor::DisableOnBeta();
|
|
|
|
|
|
|
|
nsCOMPtr<nsIObserverService> observerService = mozilla::services::GetObserverService();
|
|
|
|
MOZ_ASSERT(observerService);
|
|
|
|
observerService->RemoveObserver(this, "profile-after-change");
|
|
|
|
|
|
|
|
return NS_OK;
|
|
|
|
}
|
|
|
|
|
2013-11-22 19:17:29 +00:00
|
|
|
/**
|
|
|
|
* BackgroundHangThread is a per-thread object that is used
|
|
|
|
* by all instances of BackgroundHangMonitor to monitor hangs.
|
|
|
|
*/
|
2014-04-23 13:45:56 +00:00
|
|
|
class BackgroundHangThread : public LinkedListElement<BackgroundHangThread>
|
2013-11-22 19:17:29 +00:00
|
|
|
{
|
|
|
|
private:
|
2015-11-23 19:11:22 +00:00
|
|
|
static MOZ_THREAD_LOCAL(BackgroundHangThread*) sTlsKey;
|
|
|
|
static bool sTlsKeyInitialized;
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
BackgroundHangThread(const BackgroundHangThread&);
|
|
|
|
BackgroundHangThread& operator=(const BackgroundHangThread&);
|
2014-04-23 13:45:56 +00:00
|
|
|
~BackgroundHangThread();
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
/* Keep a reference to the manager, so we can keep going even
|
|
|
|
after BackgroundHangManager::Shutdown is called. */
|
2015-10-18 05:24:48 +00:00
|
|
|
const RefPtr<BackgroundHangManager> mManager;
|
2013-11-22 19:17:29 +00:00
|
|
|
// Unique thread ID for identification
|
|
|
|
const PRThread* mThreadID;
|
|
|
|
|
|
|
|
public:
|
2014-04-23 13:45:56 +00:00
|
|
|
NS_INLINE_DECL_REFCOUNTING(BackgroundHangThread)
|
2013-11-22 19:17:29 +00:00
|
|
|
static BackgroundHangThread* FindThread();
|
|
|
|
|
|
|
|
static void Startup()
|
|
|
|
{
|
2014-05-27 07:15:35 +00:00
|
|
|
/* We can tolerate init() failing. */
|
|
|
|
(void)!sTlsKey.init();
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Hang timeout in ticks
|
|
|
|
const PRIntervalTime mTimeout;
|
|
|
|
// PermaHang timeout in ticks
|
|
|
|
const PRIntervalTime mMaxTimeout;
|
|
|
|
// Time at last activity
|
|
|
|
PRIntervalTime mInterval;
|
2013-11-22 19:17:29 +00:00
|
|
|
// Time when a hang started
|
|
|
|
PRIntervalTime mHangStart;
|
|
|
|
// Is the thread in a hang
|
|
|
|
bool mHanging;
|
2013-11-22 19:17:29 +00:00
|
|
|
// Is the thread in a waiting state
|
|
|
|
bool mWaiting;
|
2013-11-22 19:17:31 +00:00
|
|
|
// Platform-specific helper to get hang stacks
|
|
|
|
ThreadStackHelper mStackHelper;
|
|
|
|
// Stack of current hang
|
2014-06-20 18:28:10 +00:00
|
|
|
Telemetry::HangStack mHangStack;
|
2013-11-22 19:17:31 +00:00
|
|
|
// Statistics for telemetry
|
|
|
|
Telemetry::ThreadHangStats mStats;
|
2015-02-19 06:22:01 +00:00
|
|
|
// Annotations for the current hang
|
|
|
|
UniquePtr<HangMonitor::HangAnnotations> mAnnotations;
|
|
|
|
// Annotators registered for this thread
|
|
|
|
HangMonitor::Observer::Annotators mAnnotators;
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
BackgroundHangThread(const char* aName,
|
|
|
|
uint32_t aTimeoutMs,
|
|
|
|
uint32_t aMaxTimeoutMs);
|
|
|
|
|
2013-11-22 19:17:29 +00:00
|
|
|
// Report a hang; aManager->mLock IS locked
|
2014-07-29 14:33:11 +00:00
|
|
|
Telemetry::HangHistogram& ReportHang(PRIntervalTime aHangTime);
|
2013-11-22 19:17:29 +00:00
|
|
|
// Report a permanent hang; aManager->mLock IS locked
|
2014-01-14 16:33:31 +00:00
|
|
|
void ReportPermaHang();
|
2013-11-22 19:17:29 +00:00
|
|
|
// Called by BackgroundHangMonitor::NotifyActivity
|
|
|
|
void NotifyActivity();
|
|
|
|
// Called by BackgroundHangMonitor::NotifyWait
|
|
|
|
void NotifyWait()
|
|
|
|
{
|
|
|
|
NotifyActivity();
|
|
|
|
mWaiting = true;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
StaticRefPtr<BackgroundHangManager> BackgroundHangManager::sInstance;
|
2014-07-30 07:24:00 +00:00
|
|
|
bool BackgroundHangManager::sProhibited = false;
|
2015-03-09 08:58:00 +00:00
|
|
|
bool BackgroundHangManager::sDisabled = false;
|
2013-11-22 19:17:29 +00:00
|
|
|
|
2015-11-23 19:11:22 +00:00
|
|
|
MOZ_THREAD_LOCAL(BackgroundHangThread*) BackgroundHangThread::sTlsKey;
|
|
|
|
bool BackgroundHangThread::sTlsKeyInitialized;
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
BackgroundHangManager::BackgroundHangManager()
|
|
|
|
: mShutdown(false)
|
|
|
|
, mLock("BackgroundHangManager")
|
|
|
|
, mIntervalNow(0)
|
|
|
|
{
|
|
|
|
// Lock so we don't race against the new monitor thread
|
|
|
|
MonitorAutoLock autoLock(mLock);
|
|
|
|
mHangMonitorThread = PR_CreateThread(
|
|
|
|
PR_USER_THREAD, MonitorThread, this,
|
2013-11-22 19:17:30 +00:00
|
|
|
PR_PRIORITY_LOW, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD, 0);
|
|
|
|
|
2014-05-27 07:15:35 +00:00
|
|
|
MOZ_ASSERT(mHangMonitorThread, "Failed to create monitor thread");
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BackgroundHangManager::~BackgroundHangManager()
|
|
|
|
{
|
2014-05-27 07:15:35 +00:00
|
|
|
MOZ_ASSERT(mShutdown, "Destruction without Shutdown call");
|
|
|
|
MOZ_ASSERT(mHangThreads.isEmpty(), "Destruction with outstanding monitors");
|
|
|
|
MOZ_ASSERT(mHangMonitorThread, "No monitor thread");
|
2013-11-22 19:17:30 +00:00
|
|
|
|
|
|
|
// PR_CreateThread could have failed above due to resource limitation
|
|
|
|
if (mHangMonitorThread) {
|
|
|
|
// The monitor thread can only live as long as the instance lives
|
|
|
|
PR_JoinThread(mHangMonitorThread);
|
|
|
|
}
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
BackgroundHangManager::RunMonitorThread()
|
|
|
|
{
|
|
|
|
// Keep us locked except when waiting
|
|
|
|
MonitorAutoLock autoLock(mLock);
|
|
|
|
|
|
|
|
/* mIntervalNow is updated at various intervals determined by waitTime.
|
|
|
|
However, if an update latency is too long (due to CPU scheduling, system
|
|
|
|
sleep, etc.), we don't update mIntervalNow at all. This is done so that
|
|
|
|
long latencies in our timing are not detected as hangs. systemTime is
|
|
|
|
used to track PR_IntervalNow() and determine our latency. */
|
|
|
|
|
|
|
|
PRIntervalTime systemTime = PR_IntervalNow();
|
|
|
|
// Default values for the first iteration of thread loop
|
|
|
|
PRIntervalTime waitTime = PR_INTERVAL_NO_WAIT;
|
2013-11-22 19:17:29 +00:00
|
|
|
PRIntervalTime recheckTimeout = PR_INTERVAL_NO_WAIT;
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
while (!mShutdown) {
|
|
|
|
|
|
|
|
PR_ClearInterrupt();
|
|
|
|
nsresult rv = autoLock.Wait(waitTime);
|
|
|
|
|
|
|
|
PRIntervalTime newTime = PR_IntervalNow();
|
|
|
|
PRIntervalTime systemInterval = newTime - systemTime;
|
|
|
|
systemTime = newTime;
|
|
|
|
|
|
|
|
/* waitTime is a quarter of the shortest timeout value; If our timing
|
|
|
|
latency is low enough (less than half the shortest timeout value),
|
|
|
|
we can update mIntervalNow. */
|
|
|
|
if (MOZ_LIKELY(waitTime != PR_INTERVAL_NO_TIMEOUT &&
|
|
|
|
systemInterval < 2 * waitTime)) {
|
|
|
|
mIntervalNow += systemInterval;
|
|
|
|
}
|
|
|
|
|
2013-11-22 19:17:29 +00:00
|
|
|
/* If it's before the next recheck timeout, and our wait did not
|
2013-11-22 19:17:29 +00:00
|
|
|
get interrupted (either through Notify or PR_Interrupt), we can
|
|
|
|
keep the current waitTime and skip iterating through hang monitors. */
|
2013-11-22 19:17:29 +00:00
|
|
|
if (MOZ_LIKELY(systemInterval < recheckTimeout &&
|
2013-11-22 19:17:29 +00:00
|
|
|
systemInterval >= waitTime &&
|
|
|
|
rv == NS_OK)) {
|
2013-11-22 19:17:29 +00:00
|
|
|
recheckTimeout -= systemInterval;
|
2013-11-22 19:17:29 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We are in one of the following scenarios,
|
2013-11-22 19:17:29 +00:00
|
|
|
- Hang or permahang recheck timeout
|
2013-11-22 19:17:29 +00:00
|
|
|
- Thread added/removed
|
2013-11-22 19:17:29 +00:00
|
|
|
- Thread wait or hang ended
|
2013-11-22 19:17:29 +00:00
|
|
|
In all cases, we want to go through our list of hang
|
2013-11-22 19:17:29 +00:00
|
|
|
monitors and update waitTime and recheckTimeout. */
|
2013-11-22 19:17:29 +00:00
|
|
|
waitTime = PR_INTERVAL_NO_TIMEOUT;
|
2013-11-22 19:17:29 +00:00
|
|
|
recheckTimeout = PR_INTERVAL_NO_TIMEOUT;
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
// Locally hold mIntervalNow
|
|
|
|
PRIntervalTime intervalNow = mIntervalNow;
|
|
|
|
|
|
|
|
// iterate through hang monitors
|
|
|
|
for (BackgroundHangThread* currentThread = mHangThreads.getFirst();
|
|
|
|
currentThread; currentThread = currentThread->getNext()) {
|
|
|
|
|
|
|
|
if (currentThread->mWaiting) {
|
|
|
|
// Thread is waiting, not hanging
|
|
|
|
continue;
|
|
|
|
}
|
2013-11-22 19:17:29 +00:00
|
|
|
PRIntervalTime interval = currentThread->mInterval;
|
|
|
|
PRIntervalTime hangTime = intervalNow - interval;
|
2013-11-22 19:17:29 +00:00
|
|
|
if (MOZ_UNLIKELY(hangTime >= currentThread->mMaxTimeout)) {
|
2013-11-22 19:17:29 +00:00
|
|
|
// A permahang started
|
2013-11-22 19:17:29 +00:00
|
|
|
// Skip subsequent iterations and tolerate a race on mWaiting here
|
|
|
|
currentThread->mWaiting = true;
|
2014-01-14 16:33:31 +00:00
|
|
|
currentThread->mHanging = false;
|
2013-11-22 19:17:29 +00:00
|
|
|
currentThread->ReportPermaHang();
|
|
|
|
continue;
|
|
|
|
}
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
if (MOZ_LIKELY(!currentThread->mHanging)) {
|
|
|
|
if (MOZ_UNLIKELY(hangTime >= currentThread->mTimeout)) {
|
|
|
|
// A hang started
|
2013-11-22 19:17:31 +00:00
|
|
|
currentThread->mStackHelper.GetStack(currentThread->mHangStack);
|
2013-11-22 19:17:29 +00:00
|
|
|
currentThread->mHangStart = interval;
|
|
|
|
currentThread->mHanging = true;
|
2015-02-19 06:22:01 +00:00
|
|
|
currentThread->mAnnotations =
|
|
|
|
currentThread->mAnnotators.GatherAnnotations();
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (MOZ_LIKELY(interval != currentThread->mHangStart)) {
|
|
|
|
// A hang ended
|
|
|
|
currentThread->ReportHang(intervalNow - currentThread->mHangStart);
|
|
|
|
currentThread->mHanging = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If we are hanging, the next time we check for hang status is when
|
|
|
|
the hang turns into a permahang. If we're not hanging, the next
|
|
|
|
recheck timeout is when we may be entering a hang. */
|
|
|
|
PRIntervalTime nextRecheck;
|
|
|
|
if (currentThread->mHanging) {
|
|
|
|
nextRecheck = currentThread->mMaxTimeout;
|
|
|
|
} else {
|
|
|
|
nextRecheck = currentThread->mTimeout;
|
|
|
|
}
|
|
|
|
recheckTimeout = std::min(recheckTimeout, nextRecheck - hangTime);
|
|
|
|
|
2015-12-22 23:27:39 +00:00
|
|
|
if (currentThread->mTimeout != PR_INTERVAL_NO_TIMEOUT) {
|
|
|
|
/* We wait for a quarter of the shortest timeout
|
|
|
|
value to give mIntervalNow enough granularity. */
|
|
|
|
waitTime = std::min(waitTime, currentThread->mTimeout / 4);
|
|
|
|
}
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We are shutting down now.
|
|
|
|
Wait for all outstanding monitors to unregister. */
|
|
|
|
while (!mHangThreads.isEmpty()) {
|
|
|
|
autoLock.Wait(PR_INTERVAL_NO_TIMEOUT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
BackgroundHangThread::BackgroundHangThread(const char* aName,
|
|
|
|
uint32_t aTimeoutMs,
|
|
|
|
uint32_t aMaxTimeoutMs)
|
|
|
|
: mManager(BackgroundHangManager::sInstance)
|
|
|
|
, mThreadID(PR_GetCurrentThread())
|
2013-12-05 02:24:28 +00:00
|
|
|
, mTimeout(aTimeoutMs == BackgroundHangMonitor::kNoTimeout
|
|
|
|
? PR_INTERVAL_NO_TIMEOUT
|
|
|
|
: PR_MillisecondsToInterval(aTimeoutMs))
|
|
|
|
, mMaxTimeout(aMaxTimeoutMs == BackgroundHangMonitor::kNoTimeout
|
|
|
|
? PR_INTERVAL_NO_TIMEOUT
|
|
|
|
: PR_MillisecondsToInterval(aMaxTimeoutMs))
|
2013-11-22 19:17:29 +00:00
|
|
|
, mInterval(mManager->mIntervalNow)
|
2013-11-22 19:17:29 +00:00
|
|
|
, mHangStart(mInterval)
|
|
|
|
, mHanging(false)
|
2013-11-22 19:17:29 +00:00
|
|
|
, mWaiting(true)
|
2013-11-22 19:17:31 +00:00
|
|
|
, mStats(aName)
|
2013-11-22 19:17:29 +00:00
|
|
|
{
|
2015-11-23 19:11:22 +00:00
|
|
|
if (sTlsKeyInitialized) {
|
2013-11-22 19:17:29 +00:00
|
|
|
sTlsKey.set(this);
|
|
|
|
}
|
|
|
|
// Lock here because LinkedList is not thread-safe
|
|
|
|
MonitorAutoLock autoLock(mManager->mLock);
|
|
|
|
// Add to thread list
|
|
|
|
mManager->mHangThreads.insertBack(this);
|
|
|
|
// Wake up monitor thread to process new thread
|
|
|
|
autoLock.Notify();
|
|
|
|
}
|
|
|
|
|
|
|
|
BackgroundHangThread::~BackgroundHangThread()
|
|
|
|
{
|
|
|
|
// Lock here because LinkedList is not thread-safe
|
|
|
|
MonitorAutoLock autoLock(mManager->mLock);
|
|
|
|
// Remove from thread list
|
|
|
|
remove();
|
|
|
|
// Wake up monitor thread to process removed thread
|
|
|
|
autoLock.Notify();
|
|
|
|
|
|
|
|
// We no longer have a thread
|
2015-11-23 19:11:22 +00:00
|
|
|
if (sTlsKeyInitialized) {
|
2013-11-22 19:17:29 +00:00
|
|
|
sTlsKey.set(nullptr);
|
|
|
|
}
|
2013-11-22 19:17:31 +00:00
|
|
|
|
|
|
|
// Move our copy of ThreadHangStats to Telemetry storage
|
|
|
|
Telemetry::RecordThreadHangStats(mStats);
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
2014-07-29 14:33:11 +00:00
|
|
|
Telemetry::HangHistogram&
|
2013-11-22 19:17:31 +00:00
|
|
|
BackgroundHangThread::ReportHang(PRIntervalTime aHangTime)
|
2013-11-22 19:17:29 +00:00
|
|
|
{
|
2013-11-22 19:17:29 +00:00
|
|
|
// Recovered from a hang; called on the monitor thread
|
|
|
|
// mManager->mLock IS locked
|
2013-11-22 19:17:29 +00:00
|
|
|
|
2015-01-10 17:41:48 +00:00
|
|
|
// Remove unwanted "js::RunScript" frame from the stack
|
|
|
|
for (const char** f = &mHangStack.back(); f >= mHangStack.begin(); f--) {
|
|
|
|
if (!mHangStack.IsInBuffer(*f) && !strcmp(*f, "js::RunScript")) {
|
|
|
|
mHangStack.erase(f);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-16 05:04:00 +00:00
|
|
|
// Collapse duplicated "(chrome script)" and "(content script)" entries in the stack.
|
|
|
|
auto it = std::unique(mHangStack.begin(), mHangStack.end(), StackScriptEntriesCollapser);
|
|
|
|
mHangStack.erase(it, mHangStack.end());
|
|
|
|
|
|
|
|
// Limit the depth of the reported stack if greater than our limit. Only keep its
|
|
|
|
// last entries, since the most recent frames are at the end of the vector.
|
|
|
|
if (mHangStack.length() > kMaxThreadHangStackDepth) {
|
|
|
|
const int elementsToRemove = mHangStack.length() - kMaxThreadHangStackDepth;
|
|
|
|
// Replace the oldest frame with a known label so that we can tell this stack
|
|
|
|
// was limited.
|
|
|
|
mHangStack[0] = "(reduced stack)";
|
|
|
|
mHangStack.erase(mHangStack.begin() + 1, mHangStack.begin() + elementsToRemove);
|
|
|
|
}
|
|
|
|
|
2013-11-22 19:17:31 +00:00
|
|
|
Telemetry::HangHistogram newHistogram(Move(mHangStack));
|
|
|
|
for (Telemetry::HangHistogram* oldHistogram = mStats.mHangs.begin();
|
|
|
|
oldHistogram != mStats.mHangs.end(); oldHistogram++) {
|
|
|
|
if (newHistogram == *oldHistogram) {
|
|
|
|
// New histogram matches old one
|
2015-02-19 06:22:01 +00:00
|
|
|
oldHistogram->Add(aHangTime, Move(mAnnotations));
|
2014-07-29 14:33:11 +00:00
|
|
|
return *oldHistogram;
|
2013-11-22 19:17:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// Add new histogram
|
2015-02-19 06:22:01 +00:00
|
|
|
newHistogram.Add(aHangTime, Move(mAnnotations));
|
2016-01-14 14:19:37 +00:00
|
|
|
if (!mStats.mHangs.append(Move(newHistogram))) {
|
|
|
|
MOZ_CRASH();
|
|
|
|
}
|
2014-07-29 14:33:11 +00:00
|
|
|
return mStats.mHangs.back();
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2014-01-14 16:33:31 +00:00
|
|
|
BackgroundHangThread::ReportPermaHang()
|
2013-11-22 19:17:29 +00:00
|
|
|
{
|
|
|
|
// Permanently hanged; called on the monitor thread
|
|
|
|
// mManager->mLock IS locked
|
|
|
|
|
2014-07-29 14:33:11 +00:00
|
|
|
Telemetry::HangHistogram& hang = ReportHang(mMaxTimeout);
|
|
|
|
Telemetry::HangStack& stack = hang.GetNativeStack();
|
|
|
|
if (stack.empty()) {
|
|
|
|
mStackHelper.GetNativeStack(stack);
|
|
|
|
}
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
MOZ_ALWAYS_INLINE void
|
|
|
|
BackgroundHangThread::NotifyActivity()
|
|
|
|
{
|
|
|
|
PRIntervalTime intervalNow = mManager->mIntervalNow;
|
|
|
|
if (mWaiting) {
|
|
|
|
mInterval = intervalNow;
|
|
|
|
mWaiting = false;
|
|
|
|
/* We have to wake up the manager thread because when all threads
|
|
|
|
are waiting, the manager thread waits indefinitely as well. */
|
|
|
|
mManager->Wakeup();
|
|
|
|
} else {
|
|
|
|
PRIntervalTime duration = intervalNow - mInterval;
|
2013-11-22 19:17:31 +00:00
|
|
|
mStats.mActivity.Add(duration);
|
2013-11-22 19:17:29 +00:00
|
|
|
if (MOZ_UNLIKELY(duration >= mTimeout)) {
|
2013-11-22 19:17:29 +00:00
|
|
|
/* Wake up the manager thread to tell it that a hang ended */
|
|
|
|
mManager->Wakeup();
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
mInterval = intervalNow;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
BackgroundHangThread*
|
|
|
|
BackgroundHangThread::FindThread()
|
|
|
|
{
|
2014-02-12 15:24:21 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2014-07-30 07:24:00 +00:00
|
|
|
if (BackgroundHangManager::sInstance == nullptr) {
|
2015-03-09 08:58:00 +00:00
|
|
|
MOZ_ASSERT(BackgroundHangManager::sProhibited || BackgroundHangManager::sDisabled,
|
2014-07-30 07:24:00 +00:00
|
|
|
"BackgroundHandleManager is not initialized");
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
2015-11-23 19:11:22 +00:00
|
|
|
if (sTlsKeyInitialized) {
|
2013-11-22 19:17:29 +00:00
|
|
|
// Use TLS if available
|
2014-07-30 07:24:00 +00:00
|
|
|
MOZ_ASSERT(!BackgroundHangManager::sProhibited,
|
|
|
|
"BackgroundHandleManager is not initialized");
|
2013-11-22 19:17:29 +00:00
|
|
|
return sTlsKey.get();
|
|
|
|
}
|
|
|
|
// If TLS is unavailable, we can search through the thread list
|
2015-10-18 05:24:48 +00:00
|
|
|
RefPtr<BackgroundHangManager> manager(BackgroundHangManager::sInstance);
|
2013-11-22 19:17:29 +00:00
|
|
|
MOZ_ASSERT(manager, "Creating BackgroundHangMonitor after shutdown");
|
|
|
|
|
|
|
|
PRThread* threadID = PR_GetCurrentThread();
|
|
|
|
// Lock thread list for traversal
|
|
|
|
MonitorAutoLock autoLock(manager->mLock);
|
|
|
|
for (BackgroundHangThread* thread = manager->mHangThreads.getFirst();
|
|
|
|
thread; thread = thread->getNext()) {
|
|
|
|
if (thread->mThreadID == threadID) {
|
|
|
|
return thread;
|
|
|
|
}
|
|
|
|
}
|
2014-02-12 15:24:21 +00:00
|
|
|
#endif
|
2013-11-22 19:17:29 +00:00
|
|
|
// Current thread is not initialized
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
2015-03-09 08:58:00 +00:00
|
|
|
bool
|
|
|
|
BackgroundHangMonitor::ShouldDisableOnBeta(const nsCString &clientID) {
|
|
|
|
MOZ_ASSERT(clientID.Length() == 36, "clientID is invalid");
|
|
|
|
const char *suffix = clientID.get() + clientID.Length() - 4;
|
|
|
|
return strtol(suffix, NULL, 16) % BHR_BETA_MOD;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
BackgroundHangMonitor::IsDisabled() {
|
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
|
|
|
return BackgroundHangManager::sDisabled;
|
|
|
|
#else
|
|
|
|
return true;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
BackgroundHangMonitor::DisableOnBeta() {
|
|
|
|
nsAdoptingCString clientID = Preferences::GetCString("toolkit.telemetry.cachedClientID");
|
|
|
|
bool telemetryEnabled = Preferences::GetBool("toolkit.telemetry.enabled");
|
|
|
|
|
|
|
|
if (!telemetryEnabled || !clientID || BackgroundHangMonitor::ShouldDisableOnBeta(clientID)) {
|
|
|
|
if (XRE_IsParentProcess()) {
|
|
|
|
BackgroundHangMonitor::Shutdown();
|
|
|
|
} else {
|
|
|
|
BackgroundHangManager::sDisabled = true;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2013-11-22 19:17:29 +00:00
|
|
|
|
|
|
|
void
|
|
|
|
BackgroundHangMonitor::Startup()
|
|
|
|
{
|
2014-02-12 15:24:21 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2014-07-30 07:24:00 +00:00
|
|
|
MOZ_ASSERT(!BackgroundHangManager::sProhibited, "Prohibited");
|
2013-11-22 19:17:29 +00:00
|
|
|
MOZ_ASSERT(!BackgroundHangManager::sInstance, "Already initialized");
|
2015-03-09 08:58:00 +00:00
|
|
|
|
|
|
|
if (!strcmp(NS_STRINGIFY(MOZ_UPDATE_CHANNEL), "beta")) {
|
|
|
|
if (XRE_IsParentProcess()) { // cached ClientID hasn't been read yet
|
|
|
|
ThreadStackHelper::Startup();
|
|
|
|
BackgroundHangThread::Startup();
|
|
|
|
BackgroundHangManager::sInstance = new BackgroundHangManager();
|
|
|
|
|
|
|
|
nsCOMPtr<nsIObserverService> observerService = mozilla::services::GetObserverService();
|
|
|
|
MOZ_ASSERT(observerService);
|
|
|
|
|
|
|
|
observerService->AddObserver(BackgroundHangManager::sInstance, "profile-after-change", false);
|
|
|
|
return;
|
|
|
|
} else if(DisableOnBeta()){
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-11-22 19:17:31 +00:00
|
|
|
ThreadStackHelper::Startup();
|
2013-11-22 19:17:29 +00:00
|
|
|
BackgroundHangThread::Startup();
|
|
|
|
BackgroundHangManager::sInstance = new BackgroundHangManager();
|
2014-02-12 15:24:21 +00:00
|
|
|
#endif
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
BackgroundHangMonitor::Shutdown()
|
|
|
|
{
|
2014-02-12 15:24:21 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2015-03-09 08:58:00 +00:00
|
|
|
if (BackgroundHangManager::sDisabled) {
|
|
|
|
MOZ_ASSERT(!BackgroundHangManager::sInstance, "Initialized");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2014-07-30 07:24:00 +00:00
|
|
|
MOZ_ASSERT(!BackgroundHangManager::sProhibited, "Prohibited");
|
2013-11-22 19:17:29 +00:00
|
|
|
MOZ_ASSERT(BackgroundHangManager::sInstance, "Not initialized");
|
|
|
|
/* Scope our lock inside Shutdown() because the sInstance object can
|
|
|
|
be destroyed as soon as we set sInstance to nullptr below, and
|
|
|
|
we don't want to hold the lock when it's being destroyed. */
|
|
|
|
BackgroundHangManager::sInstance->Shutdown();
|
|
|
|
BackgroundHangManager::sInstance = nullptr;
|
2013-11-22 19:17:31 +00:00
|
|
|
ThreadStackHelper::Shutdown();
|
2015-03-09 08:58:00 +00:00
|
|
|
BackgroundHangManager::sDisabled = true;
|
2014-02-12 15:24:21 +00:00
|
|
|
#endif
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BackgroundHangMonitor::BackgroundHangMonitor(const char* aName,
|
|
|
|
uint32_t aTimeoutMs,
|
|
|
|
uint32_t aMaxTimeoutMs)
|
|
|
|
: mThread(BackgroundHangThread::FindThread())
|
|
|
|
{
|
2014-02-12 15:24:21 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2015-03-09 08:58:00 +00:00
|
|
|
if (!BackgroundHangManager::sDisabled && !BackgroundHangManager::sProhibited && !mThread) {
|
2014-07-30 07:24:00 +00:00
|
|
|
// If sProhibit is true, mThread would be null, and no monitoring.
|
2013-11-22 19:17:29 +00:00
|
|
|
mThread = new BackgroundHangThread(aName, aTimeoutMs, aMaxTimeoutMs);
|
|
|
|
}
|
2014-02-12 15:24:21 +00:00
|
|
|
#endif
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BackgroundHangMonitor::BackgroundHangMonitor()
|
|
|
|
: mThread(BackgroundHangThread::FindThread())
|
|
|
|
{
|
2014-02-12 15:24:21 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2015-03-09 08:58:00 +00:00
|
|
|
if (BackgroundHangManager::sDisabled) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2014-07-30 07:24:00 +00:00
|
|
|
MOZ_ASSERT(!BackgroundHangManager::sProhibited || mThread,
|
|
|
|
"This thread is not initialized for hang monitoring");
|
2014-02-12 15:24:21 +00:00
|
|
|
#endif
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BackgroundHangMonitor::~BackgroundHangMonitor()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
BackgroundHangMonitor::NotifyActivity()
|
|
|
|
{
|
2014-02-12 15:24:21 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2014-07-30 07:24:00 +00:00
|
|
|
if (mThread == nullptr) {
|
2015-03-09 08:58:00 +00:00
|
|
|
MOZ_ASSERT(BackgroundHangManager::sProhibited ||
|
|
|
|
BackgroundHangManager::sDisabled,
|
2014-07-30 07:24:00 +00:00
|
|
|
"This thread is not initialized for hang monitoring");
|
|
|
|
return;
|
|
|
|
}
|
2015-01-13 04:41:15 +00:00
|
|
|
|
2015-03-23 03:39:00 +00:00
|
|
|
if (Telemetry::CanRecordExtended()) {
|
2015-01-13 04:41:15 +00:00
|
|
|
mThread->NotifyActivity();
|
|
|
|
}
|
2014-02-12 15:24:21 +00:00
|
|
|
#endif
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
BackgroundHangMonitor::NotifyWait()
|
|
|
|
{
|
2014-02-12 15:24:21 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2014-07-30 07:24:00 +00:00
|
|
|
if (mThread == nullptr) {
|
2015-03-09 08:58:00 +00:00
|
|
|
MOZ_ASSERT(BackgroundHangManager::sProhibited ||
|
|
|
|
BackgroundHangManager::sDisabled,
|
2014-07-30 07:24:00 +00:00
|
|
|
"This thread is not initialized for hang monitoring");
|
|
|
|
return;
|
|
|
|
}
|
2015-01-13 04:41:15 +00:00
|
|
|
|
2015-03-23 03:39:00 +00:00
|
|
|
if (Telemetry::CanRecordExtended()) {
|
2015-01-13 04:41:15 +00:00
|
|
|
mThread->NotifyWait();
|
|
|
|
}
|
2014-02-12 15:24:21 +00:00
|
|
|
#endif
|
2013-11-22 19:17:29 +00:00
|
|
|
}
|
|
|
|
|
2014-07-30 07:24:00 +00:00
|
|
|
void
|
|
|
|
BackgroundHangMonitor::Prohibit()
|
|
|
|
{
|
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
|
|
|
MOZ_ASSERT(BackgroundHangManager::sInstance == nullptr,
|
|
|
|
"The background hang monitor is already initialized");
|
|
|
|
BackgroundHangManager::sProhibited = true;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
BackgroundHangMonitor::Allow()
|
|
|
|
{
|
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
|
|
|
MOZ_ASSERT(BackgroundHangManager::sInstance == nullptr,
|
|
|
|
"The background hang monitor is already initialized");
|
|
|
|
BackgroundHangManager::sProhibited = false;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2015-02-19 06:22:01 +00:00
|
|
|
bool
|
|
|
|
BackgroundHangMonitor::RegisterAnnotator(HangMonitor::Annotator& aAnnotator)
|
|
|
|
{
|
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
|
|
|
BackgroundHangThread* thisThread = BackgroundHangThread::FindThread();
|
|
|
|
if (!thisThread) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return thisThread->mAnnotators.Register(aAnnotator);
|
|
|
|
#else
|
|
|
|
return false;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
BackgroundHangMonitor::UnregisterAnnotator(HangMonitor::Annotator& aAnnotator)
|
|
|
|
{
|
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
|
|
|
BackgroundHangThread* thisThread = BackgroundHangThread::FindThread();
|
|
|
|
if (!thisThread) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return thisThread->mAnnotators.Unregister(aAnnotator);
|
|
|
|
#else
|
|
|
|
return false;
|
|
|
|
#endif
|
|
|
|
}
|
2013-11-22 19:17:31 +00:00
|
|
|
|
|
|
|
/* Because we are iterating through the BackgroundHangThread linked list,
|
|
|
|
we need to take a lock. Using MonitorAutoLock as a base class makes
|
|
|
|
sure all of that is taken care of for us. */
|
|
|
|
BackgroundHangMonitor::ThreadHangStatsIterator::ThreadHangStatsIterator()
|
|
|
|
: MonitorAutoLock(BackgroundHangManager::sInstance->mLock)
|
2014-07-30 07:24:00 +00:00
|
|
|
, mThread(BackgroundHangManager::sInstance ?
|
|
|
|
BackgroundHangManager::sInstance->mHangThreads.getFirst() :
|
|
|
|
nullptr)
|
2013-11-22 19:17:31 +00:00
|
|
|
{
|
2014-07-30 07:24:00 +00:00
|
|
|
#ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
|
2015-03-09 08:58:00 +00:00
|
|
|
MOZ_ASSERT(BackgroundHangManager::sInstance ||
|
|
|
|
BackgroundHangManager::sProhibited ||
|
|
|
|
BackgroundHangManager::sDisabled,
|
2014-07-30 07:24:00 +00:00
|
|
|
"Inconsistent state");
|
|
|
|
#endif
|
2013-11-22 19:17:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Telemetry::ThreadHangStats*
|
|
|
|
BackgroundHangMonitor::ThreadHangStatsIterator::GetNext()
|
|
|
|
{
|
|
|
|
if (!mThread) {
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
Telemetry::ThreadHangStats* stats = &mThread->mStats;
|
|
|
|
mThread = mThread->getNext();
|
|
|
|
return stats;
|
|
|
|
}
|
|
|
|
|
2013-11-22 19:17:29 +00:00
|
|
|
} // namespace mozilla
|