Bug 695003 - Race condition when deleting cache directory

This commit is contained in:
Michal Novotny 2011-12-02 18:10:57 +01:00
parent d58e948c0c
commit ff08b0851d
5 changed files with 467 additions and 107 deletions

View File

@ -697,7 +697,7 @@ nsCacheProfilePrefObserver::ReadPrefs(nsIPrefBranch* branch)
if (NS_SUCCEEDED(rv)) {
bool exists;
if (NS_SUCCEEDED(profDir->Exists(&exists)) && exists)
DeleteDir(profDir, false, false);
nsDeleteDir::DeleteDir(profDir, false);
}
}
}
@ -1038,6 +1038,11 @@ nsCacheService::Init()
NS_WARNING("Can't create cache IO thread");
}
rv = nsDeleteDir::Init();
if (NS_FAILED(rv)) {
NS_WARNING("Can't initialize nsDeleteDir");
}
// initialize hashtable for active cache entries
rv = mActiveEntries.Init();
if (NS_FAILED(rv)) return rv;
@ -1061,6 +1066,7 @@ void
nsCacheService::Shutdown()
{
nsCOMPtr<nsIThread> cacheIOThread;
Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_SHUTDOWN> totalTimer;
{
nsCacheServiceAutoLock lock;
@ -1106,6 +1112,29 @@ nsCacheService::Shutdown()
if (cacheIOThread)
cacheIOThread->Shutdown();
bool finishDeleting = false;
nsresult rv;
nsCOMPtr<nsIPrefBranch2> branch = do_GetService(NS_PREFSERVICE_CONTRACTID);
if (!branch) {
NS_WARNING("Failed to get pref service!");
} else {
bool isSet;
rv = branch->GetBoolPref("privacy.sanitize.sanitizeOnShutdown", &isSet);
if (NS_SUCCEEDED(rv) && isSet) {
rv = branch->GetBoolPref("privacy.clearOnShutdown.cache", &isSet);
if (NS_SUCCEEDED(rv) && isSet) {
finishDeleting = true;
}
}
}
if (finishDeleting) {
Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_SHUTDOWN_CLEAR_PRIVATE> timer;
nsDeleteDir::Shutdown(finishDeleting);
} else {
Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_DELETEDIR_SHUTDOWN> timer;
nsDeleteDir::Shutdown(finishDeleting);
}
}

View File

@ -22,6 +22,7 @@
* Contributor(s):
* Darin Fisher <darin@meer.net>
* Jason Duell <jduell.mcbugs@gmail.com>
* Michal Novotny <michal.novotny@gmail.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
@ -40,33 +41,195 @@
#include "nsDeleteDir.h"
#include "nsIFile.h"
#include "nsString.h"
#include "prthread.h"
#include "mozilla/Telemetry.h"
#include "nsITimer.h"
#include "nsISimpleEnumerator.h"
#include "nsAutoPtr.h"
#include "nsThreadUtils.h"
#include "nsISupportsPriority.h"
using namespace mozilla;
static void DeleteDirThreadFunc(void *arg)
class nsBlockOnBackgroundThreadEvent : public nsRunnable {
public:
nsBlockOnBackgroundThreadEvent() {}
NS_IMETHOD Run()
{
MutexAutoLock lock(nsDeleteDir::gInstance->mLock);
nsDeleteDir::gInstance->mCondVar.Notify();
return NS_OK;
}
};
class nsDestroyThreadEvent : public nsRunnable {
public:
nsDestroyThreadEvent(nsIThread *thread)
: mThread(thread)
{}
NS_IMETHOD Run()
{
mThread->Shutdown();
return NS_OK;
}
private:
nsCOMPtr<nsIThread> mThread;
};
nsDeleteDir * nsDeleteDir::gInstance = nsnull;
nsDeleteDir::nsDeleteDir()
: mLock("nsDeleteDir.mLock"),
mCondVar(mLock, "nsDeleteDir.mCondVar"),
mShutdownPending(false),
mStopDeleting(false)
{
NS_ASSERTION(gInstance==nsnull, "multiple nsCacheService instances!");
}
nsDeleteDir::~nsDeleteDir()
{
gInstance = nsnull;
}
nsresult
nsDeleteDir::Init()
{
if (gInstance)
return NS_ERROR_ALREADY_INITIALIZED;
gInstance = new nsDeleteDir();
return NS_OK;
}
nsresult
nsDeleteDir::Shutdown(bool finishDeleting)
{
if (!gInstance)
return NS_ERROR_NOT_INITIALIZED;
nsCOMArray<nsIFile> dirsToRemove;
nsCOMPtr<nsIThread> thread;
{
MutexAutoLock lock(gInstance->mLock);
NS_ASSERTION(!gInstance->mShutdownPending,
"Unexpected state in nsDeleteDir::Shutdown()");
gInstance->mShutdownPending = true;
if (!finishDeleting)
gInstance->mStopDeleting = true;
// remove all pending timers
for (PRInt32 i = gInstance->mTimers.Count(); i > 0; i--) {
nsCOMPtr<nsITimer> timer = gInstance->mTimers[i-1];
gInstance->mTimers.RemoveObjectAt(i-1);
timer->Cancel();
nsCOMArray<nsIFile> *arg;
timer->GetClosure((reinterpret_cast<void**>(&arg)));
if (finishDeleting)
dirsToRemove.AppendObjects(*arg);
// delete argument passed to the timer
delete arg;
}
thread.swap(gInstance->mThread);
if (thread) {
// dispatch event and wait for it to run and notify us, so we know thread
// has completed all work and can be shutdown
nsCOMPtr<nsIRunnable> event = new nsBlockOnBackgroundThreadEvent();
nsresult rv = thread->Dispatch(event, NS_DISPATCH_NORMAL);
if (NS_FAILED(rv)) {
NS_WARNING("Failed dispatching block-event");
return NS_ERROR_UNEXPECTED;
}
rv = gInstance->mCondVar.Wait();
thread->Shutdown();
}
}
delete gInstance;
for (PRInt32 i = 0; i < dirsToRemove.Count(); i++)
dirsToRemove[i]->Remove(true);
return NS_OK;
}
nsresult
nsDeleteDir::InitThread()
{
if (mThread)
return NS_OK;
nsresult rv = NS_NewThread(getter_AddRefs(mThread));
if (NS_FAILED(rv)) {
NS_WARNING("Can't create background thread");
return rv;
}
nsCOMPtr<nsISupportsPriority> p = do_QueryInterface(mThread);
if (p) {
p->SetPriority(nsISupportsPriority::PRIORITY_LOWEST);
}
return NS_OK;
}
void
nsDeleteDir::DestroyThread()
{
if (!mThread)
return;
if (mTimers.Count())
// more work to do, so don't delete thread.
return;
NS_DispatchToMainThread(new nsDestroyThreadEvent(mThread));
mThread = nsnull;
}
void
nsDeleteDir::TimerCallback(nsITimer *aTimer, void *arg)
{
Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_DELETEDIR> timer;
nsIFile *dir = static_cast<nsIFile *>(arg);
dir->Remove(true);
NS_RELEASE(dir);
{
MutexAutoLock lock(gInstance->mLock);
PRInt32 idx = gInstance->mTimers.IndexOf(aTimer);
if (idx == -1) {
// Timer was canceled and removed during shutdown.
return;
}
gInstance->mTimers.RemoveObjectAt(idx);
}
nsAutoPtr<nsCOMArray<nsIFile> > dirList;
dirList = static_cast<nsCOMArray<nsIFile> *>(arg);
bool shuttingDown = false;
for (PRInt32 i = 0; i < dirList->Count() && !shuttingDown; i++) {
gInstance->RemoveDir((*dirList)[i], &shuttingDown);
}
{
MutexAutoLock lock(gInstance->mLock);
gInstance->DestroyThread();
}
}
static void CreateDeleterThread(nsITimer *aTimer, void *arg)
{
nsIFile *dir = static_cast<nsIFile *>(arg);
// create the worker thread
PR_CreateThread(PR_USER_THREAD, DeleteDirThreadFunc, dir, PR_PRIORITY_LOW,
PR_GLOBAL_THREAD, PR_UNJOINABLE_THREAD, 0);
}
nsresult DeleteDir(nsIFile *dirIn, bool moveToTrash, bool sync,
PRUint32 delay)
nsresult
nsDeleteDir::DeleteDir(nsIFile *dirIn, bool moveToTrash, PRUint32 delay)
{
Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_TRASHRENAME> timer;
if (!gInstance)
return NS_ERROR_NOT_INITIALIZED;
nsresult rv;
nsCOMPtr<nsIFile> trash, dir;
@ -80,55 +243,56 @@ nsresult DeleteDir(nsIFile *dirIn, bool moveToTrash, bool sync,
rv = GetTrashDir(dir, &trash);
if (NS_FAILED(rv))
return rv;
nsCAutoString leaf;
rv = trash->GetNativeLeafName(leaf);
nsCAutoString origLeaf;
rv = trash->GetNativeLeafName(origLeaf);
if (NS_FAILED(rv))
return rv;
// Important: must rename directory w/o changing parent directory: else on
// NTFS we'll wait (with cache lock) while nsIFile's ACL reset walks file
// tree: was hanging GUI for *minutes* on large cache dirs.
rv = dir->MoveToNative(nsnull, leaf);
if (NS_FAILED(rv)) {
nsresult rvMove = rv;
// TrashDir may already exist (if we crashed while deleting it, etc.)
// In that case current Cache dir should be small--just move it to
// subdirectory of TrashDir and eat the NTFS ACL overhead.
leaf.AppendInt(rand()); // support this happening multiple times
rv = dir->MoveToNative(trash, leaf);
// Append random number to the trash directory and check if it exists.
nsCAutoString leaf;
for (PRInt32 i = 0; i < 10; i++) {
leaf = origLeaf;
leaf.AppendInt(rand());
rv = trash->SetNativeLeafName(leaf);
if (NS_FAILED(rv))
return rvMove;
// Be paranoid and delete immediately if we're seeing old trash when
// we're creating a new one
delay = 0;
return rv;
bool exists;
if (NS_SUCCEEDED(trash->Exists(&exists)) && !exists) {
break;
}
leaf.Truncate();
}
// Fail if we didn't find unused trash directory within the limit
if (!leaf.Length())
return NS_ERROR_FAILURE;
rv = dir->MoveToNative(nsnull, leaf);
if (NS_FAILED(rv))
return rv;
} else {
// we want to pass a clone of the original off to the worker thread.
trash.swap(dir);
}
// Steal ownership of trash directory; let the thread release it.
nsIFile *trashRef = nsnull;
trash.swap(trashRef);
nsAutoPtr<nsCOMArray<nsIFile> > arg(new nsCOMArray<nsIFile>);
arg->AppendObject(trash);
if (sync) {
DeleteDirThreadFunc(trashRef);
} else {
if (delay) {
nsCOMPtr<nsITimer> timer = do_CreateInstance("@mozilla.org/timer;1", &rv);
if (NS_FAILED(rv))
return NS_ERROR_UNEXPECTED;
timer->InitWithFuncCallback(CreateDeleterThread, trashRef, delay,
nsITimer::TYPE_ONE_SHOT);
} else {
CreateDeleterThread(nsnull, trashRef);
}
}
rv = gInstance->PostTimer(arg, delay);
if (NS_FAILED(rv))
return rv;
arg.forget();
return NS_OK;
}
nsresult GetTrashDir(nsIFile *target, nsCOMPtr<nsIFile> *result)
nsresult
nsDeleteDir::GetTrashDir(nsIFile *target, nsCOMPtr<nsIFile> *result)
{
nsresult rv = target->Clone(getter_AddRefs(*result));
if (NS_FAILED(rv))
@ -142,3 +306,157 @@ nsresult GetTrashDir(nsIFile *target, nsCOMPtr<nsIFile> *result)
return (*result)->SetNativeLeafName(leaf);
}
nsresult
nsDeleteDir::RemoveOldTrashes(nsIFile *cacheDir)
{
if (!gInstance)
return NS_ERROR_NOT_INITIALIZED;
nsresult rv;
static bool firstRun = true;
if (!firstRun)
return NS_OK;
firstRun = false;
nsCOMPtr<nsIFile> trash;
rv = GetTrashDir(cacheDir, &trash);
if (NS_FAILED(rv))
return rv;
nsAutoString trashName;
rv = trash->GetLeafName(trashName);
if (NS_FAILED(rv))
return rv;
nsCOMPtr<nsIFile> parent;
rv = cacheDir->GetParent(getter_AddRefs(parent));
if (NS_FAILED(rv))
return rv;
nsCOMPtr<nsISimpleEnumerator> iter;
rv = parent->GetDirectoryEntries(getter_AddRefs(iter));
if (NS_FAILED(rv))
return rv;
bool more;
nsCOMPtr<nsISupports> elem;
nsAutoPtr<nsCOMArray<nsIFile> > dirList;
while (NS_SUCCEEDED(iter->HasMoreElements(&more)) && more) {
rv = iter->GetNext(getter_AddRefs(elem));
if (NS_FAILED(rv))
continue;
nsCOMPtr<nsIFile> file = do_QueryInterface(elem);
if (!file)
continue;
nsAutoString leafName;
rv = file->GetLeafName(leafName);
if (NS_FAILED(rv))
continue;
// match all names that begin with the trash name (i.e. "Cache.Trash")
if (Substring(leafName, 0, trashName.Length()).Equals(trashName)) {
if (!dirList)
dirList = new nsCOMArray<nsIFile>;
dirList->AppendObject(file);
}
}
if (dirList) {
rv = gInstance->PostTimer(dirList, 90000);
if (NS_FAILED(rv))
return rv;
dirList.forget();
}
return NS_OK;
}
nsresult
nsDeleteDir::PostTimer(void *arg, PRUint32 delay)
{
nsresult rv;
nsCOMPtr<nsITimer> timer = do_CreateInstance("@mozilla.org/timer;1", &rv);
if (NS_FAILED(rv))
return NS_ERROR_UNEXPECTED;
MutexAutoLock lock(mLock);
rv = InitThread();
if (NS_FAILED(rv))
return rv;
rv = timer->SetTarget(mThread);
if (NS_FAILED(rv))
return rv;
rv = timer->InitWithFuncCallback(TimerCallback, arg, delay,
nsITimer::TYPE_ONE_SHOT);
if (NS_FAILED(rv))
return rv;
mTimers.AppendObject(timer);
return NS_OK;
}
nsresult
nsDeleteDir::RemoveDir(nsIFile *file, bool *stopDeleting)
{
nsresult rv;
bool isLink;
rv = file->IsSymlink(&isLink);
if (NS_FAILED(rv) || isLink)
return NS_ERROR_UNEXPECTED;
bool isDir;
rv = file->IsDirectory(&isDir);
if (NS_FAILED(rv))
return rv;
if (isDir) {
nsCOMPtr<nsISimpleEnumerator> iter;
rv = file->GetDirectoryEntries(getter_AddRefs(iter));
if (NS_FAILED(rv))
return rv;
bool more;
nsCOMPtr<nsISupports> elem;
while (NS_SUCCEEDED(iter->HasMoreElements(&more)) && more) {
rv = iter->GetNext(getter_AddRefs(elem));
if (NS_FAILED(rv)) {
NS_WARNING("Unexpected failure in nsDeleteDir::RemoveDir");
continue;
}
nsCOMPtr<nsIFile> file2 = do_QueryInterface(elem);
if (!file2) {
NS_WARNING("Unexpected failure in nsDeleteDir::RemoveDir");
continue;
}
RemoveDir(file2, stopDeleting);
// No check for errors to remove as much as possible
if (*stopDeleting)
return NS_OK;
}
}
file->Remove(false);
// No check for errors to remove as much as possible
MutexAutoLock lock(mLock);
if (mStopDeleting)
*stopDeleting = true;
return NS_OK;
}

View File

@ -21,6 +21,7 @@
*
* Contributor(s):
* Darin Fisher <darin@meer.net>
* Michal Novotny <michal.novotny@gmail.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
@ -40,39 +41,72 @@
#define nsDeleteDir_h__
#include "nsCOMPtr.h"
#include "nsCOMArray.h"
#include "mozilla/Mutex.h"
#include "mozilla/CondVar.h"
class nsIFile;
class nsIThread;
class nsITimer;
/**
* This routine attempts to delete a directory that may contain some files that
* are still in use. This later point is only an issue on Windows and a few
* other systems.
*
* If the moveToTrash parameter is true, then the process for deleting the
* directory creates a sibling directory of the same name with the ".Trash"
* suffix. It then attempts to move the given directory into the corresponding
* trash folder (moving individual files if necessary). Next, it proceeds to
* delete each file in the trash folder on a low-priority background thread.
*
* If the moveToTrash parameter is false, then the given directory is deleted
* directly.
*
* If the sync flag is true, then the delete operation runs to completion
* before this function returns. Otherwise, deletion occurs asynchronously.
*
* If 'delay' is non-zero, the directory will not be deleted until the
* specified number of milliseconds have passed. (The directory is still
* renamed immediately if 'moveToTrash' is passed, so upon return it is safe
* to create a directory with the same name). This parameter is ignored if
* 'sync' is true.
*/
NS_HIDDEN_(nsresult) DeleteDir(nsIFile *dir, bool moveToTrash, bool sync,
PRUint32 delay = 0);
/**
* This routine returns the trash directory corresponding to the given
* directory.
*/
NS_HIDDEN_(nsresult) GetTrashDir(nsIFile *dir, nsCOMPtr<nsIFile> *result);
class nsDeleteDir {
public:
nsDeleteDir();
~nsDeleteDir();
static nsresult Init();
static nsresult Shutdown(bool finishDeleting);
/**
* This routine attempts to delete a directory that may contain some files
* that are still in use. This latter point is only an issue on Windows and
* a few other systems.
*
* If the moveToTrash parameter is true we first rename the given directory
* "foo.Trash123" (where "foo" is the original directory name, and "123" is
* a random number, in order to support multiple concurrent deletes). The
* directory is then deleted, file-by-file, on a background thread.
*
* If the moveToTrash parameter is false, then the given directory is deleted
* directly.
*
* If 'delay' is non-zero, the directory will not be deleted until the
* specified number of milliseconds have passed. (The directory is still
* renamed immediately if 'moveToTrash' is passed, so upon return it is safe
* to create a directory with the same name).
*/
static nsresult DeleteDir(nsIFile *dir, bool moveToTrash, PRUint32 delay = 0);
/**
* Returns the trash directory corresponding to the given directory.
*/
static nsresult GetTrashDir(nsIFile *dir, nsCOMPtr<nsIFile> *result);
/**
* Remove all trashes left from previous run. This function does nothing when
* called second and more times.
*/
static nsresult RemoveOldTrashes(nsIFile *cacheDir);
static void TimerCallback(nsITimer *aTimer, void *arg);
private:
friend class nsBlockOnBackgroundThreadEvent;
friend class nsDestroyThreadEvent;
nsresult InitThread();
void DestroyThread();
nsresult PostTimer(void *arg, PRUint32 delay);
nsresult RemoveDir(nsIFile *file, bool *stopDeleting);
static nsDeleteDir * gInstance;
mozilla::Mutex mLock;
mozilla::CondVar mCondVar;
nsCOMArray<nsITimer> mTimers;
nsCOMPtr<nsIThread> mThread;
bool mShutdownPending;
bool mStopDeleting;
};
#endif // nsDeleteDir_h__

View File

@ -419,7 +419,9 @@ nsDiskCacheDevice::Init()
rv = mBindery.Init();
if (NS_FAILED(rv))
return rv;
nsDeleteDir::RemoveOldTrashes(mCacheDirectory);
// Open Disk Cache
rv = OpenDiskCache();
if (NS_FAILED(rv)) {
@ -444,17 +446,6 @@ nsDiskCacheDevice::Shutdown()
if (NS_FAILED(rv))
return rv;
if (mCacheDirectory) {
// delete any trash files left-over before shutting down.
nsCOMPtr<nsIFile> trashDir;
GetTrashDir(mCacheDirectory, &trashDir);
if (trashDir) {
bool exists;
if (NS_SUCCEEDED(trashDir->Exists(&exists)) && exists)
DeleteDir(trashDir, false, true);
}
}
return NS_OK;
}
@ -1004,18 +995,16 @@ nsDiskCacheDevice::OpenDiskCache()
if (NS_FAILED(rv))
return rv;
bool trashing = false;
if (exists) {
// Try opening cache map file.
rv = mCacheMap.Open(mCacheDirectory);
// move "corrupt" caches to trash
if (rv == NS_ERROR_FILE_CORRUPTED) {
// delay delete by 1 minute to avoid IO thrash at startup
rv = DeleteDir(mCacheDirectory, true, false, 60000);
rv = nsDeleteDir::DeleteDir(mCacheDirectory, true, 60000);
if (NS_FAILED(rv))
return rv;
exists = false;
trashing = true;
}
else if (NS_FAILED(rv))
return rv;
@ -1035,19 +1024,6 @@ nsDiskCacheDevice::OpenDiskCache()
return rv;
}
if (!trashing) {
// delete any trash files leftover from a previous run
nsCOMPtr<nsIFile> trashDir;
GetTrashDir(mCacheDirectory, &trashDir);
if (trashDir) {
bool exists;
if (NS_SUCCEEDED(trashDir->Exists(&exists)) && exists) {
// be paranoid and delete immediately if leftover
DeleteDir(trashDir, false, false);
}
}
}
return NS_OK;
}
@ -1064,7 +1040,7 @@ nsDiskCacheDevice::ClearDiskCache()
// If the disk cache directory is already gone, then it's not an error if
// we fail to delete it ;-)
rv = DeleteDir(mCacheDirectory, true, false);
rv = nsDeleteDir::DeleteDir(mCacheDirectory, true);
if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST)
return rv;

View File

@ -235,6 +235,9 @@ HISTOGRAM(STARTUP_MEASUREMENT_ERRORS, 1, mozilla::StartupTimeline::MAX_EVENT_ID,
HISTOGRAM(NETWORK_DISK_CACHE_OPEN, 1, 10000, 10, EXPONENTIAL, "Time spent opening disk cache (ms)")
HISTOGRAM(NETWORK_DISK_CACHE_TRASHRENAME, 1, 10000, 10, EXPONENTIAL, "Time spent renaming bad Cache to Cache.Trash (ms)")
HISTOGRAM(NETWORK_DISK_CACHE_DELETEDIR, 1, 10000, 10, EXPONENTIAL, "Time spent deleting disk cache (ms)")
HISTOGRAM(NETWORK_DISK_CACHE_DELETEDIR_SHUTDOWN, 1, 10000, 10, EXPONENTIAL, "Time spent during showdown stopping thread deleting old disk cache (ms)")
HISTOGRAM(NETWORK_DISK_CACHE_SHUTDOWN, 1, 10000, 10, EXPONENTIAL, "Total Time spent (ms) during disk cache showdown")
HISTOGRAM(NETWORK_DISK_CACHE_SHUTDOWN_CLEAR_PRIVATE, 1, 10000, 10, EXPONENTIAL, "Time spent (ms) during showdown deleting disk cache for 'clear private data' option")
/**
* Url-Classifier telemetry