From 0d3e56e976f3b7e2945ce3f2e5e3b1c526cd6e05 Mon Sep 17 00:00:00 2001 From: Nick Hurley Date: Mon, 5 Nov 2012 10:22:33 -0800 Subject: [PATCH] Bug 699951 - add a memory reporter for heap usage by the disk cache. r=michal --- netwerk/cache/nsCacheService.cpp | 29 ++++++++++- netwerk/cache/nsCacheService.h | 2 + netwerk/cache/nsDiskCacheBinding.cpp | 52 ++++++++++++++++++++ netwerk/cache/nsDiskCacheBinding.h | 4 +- netwerk/cache/nsDiskCacheBlockFile.cpp | 6 +++ netwerk/cache/nsDiskCacheBlockFile.h | 4 +- netwerk/cache/nsDiskCacheDevice.cpp | 12 +++++ netwerk/cache/nsDiskCacheDevice.h | 2 + netwerk/cache/nsDiskCacheMap.cpp | 26 +++++++--- netwerk/cache/nsDiskCacheMap.h | 2 + netwerk/cache/nsDiskCacheStreams.cpp | 12 +++++ netwerk/cache/nsDiskCacheStreams.h | 2 + toolkit/components/telemetry/Histograms.json | 6 +++ 13 files changed, 149 insertions(+), 10 deletions(-) diff --git a/netwerk/cache/nsCacheService.cpp b/netwerk/cache/nsCacheService.cpp index f41c1f447c1c..204a2a5322b7 100644 --- a/netwerk/cache/nsCacheService.cpp +++ b/netwerk/cache/nsCacheService.cpp @@ -1060,12 +1060,23 @@ nsCacheService * nsCacheService::gService = nullptr; static nsCOMPtr MemoryCacheReporter = nullptr; NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkMemoryCache, - "explicit/network-memory-cache", + "explicit/network/memory-cache", KIND_HEAP, UNITS_BYTES, nsCacheService::MemoryDeviceSize, "Memory used by the network memory cache.") +NS_MEMORY_REPORTER_MALLOC_SIZEOF_FUN(NetworkDiskCacheSizeOfFun, "network-disk-cache") + +static nsCOMPtr DiskCacheReporter = nullptr; + +NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkDiskCache, + "explicit/network/disk-cache", + KIND_HEAP, + UNITS_BYTES, + nsCacheService::DiskDeviceHeapSize, + "Memory used by the network disk cache.") + NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheService, nsICacheService) nsCacheService::nsCacheService() @@ -1206,11 +1217,14 @@ nsCacheService::Shutdown() mObserver->Remove(); NS_RELEASE(mObserver); - // unregister memory reporter, before deleting the memory device, just + // unregister memory reporters, before deleting the devices, just // to be safe NS_UnregisterMemoryReporter(MemoryCacheReporter); MemoryCacheReporter = nullptr; + NS_UnregisterMemoryReporter(DiskCacheReporter); + DiskCacheReporter = nullptr; + // deallocate memory and disk caches delete mMemoryDevice; mMemoryDevice = nullptr; @@ -1564,6 +1578,9 @@ nsCacheService::CreateDiskDevice() // Ignore state of the timer and return success since the purpose of the // method (create the disk-device) has been fulfilled + DiskCacheReporter = new NS_MEMORY_REPORTER_NAME(NetworkDiskCache); + NS_RegisterMemoryReporter(DiskCacheReporter); + return NS_OK; } @@ -2236,6 +2253,14 @@ nsCacheService::MemoryDeviceSize() return memoryDevice ? memoryDevice->TotalSize() : 0; } +int64_t +nsCacheService::DiskDeviceHeapSize() +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_DISKDEVICEHEAPSIZE)); + nsDiskCacheDevice *diskDevice = GlobalInstance()->mDiskDevice; + return (int64_t)(diskDevice ? diskDevice->SizeOfIncludingThis(NetworkDiskCacheSizeOfFun) : 0); +} + nsresult nsCacheService::DoomEntry(nsCacheEntry * entry) { diff --git a/netwerk/cache/nsCacheService.h b/netwerk/cache/nsCacheService.h index 7a867fb0339a..d425d9495b4e 100644 --- a/netwerk/cache/nsCacheService.h +++ b/netwerk/cache/nsCacheService.h @@ -129,6 +129,8 @@ public: nsCacheService * GlobalInstance() { return gService; } static int64_t MemoryDeviceSize(); + + static int64_t DiskDeviceHeapSize(); static nsresult DoomEntry(nsCacheEntry * entry); diff --git a/netwerk/cache/nsDiskCacheBinding.cpp b/netwerk/cache/nsDiskCacheBinding.cpp index 059db4f37055..b4f6390db537 100644 --- a/netwerk/cache/nsDiskCacheBinding.cpp +++ b/netwerk/cache/nsDiskCacheBinding.cpp @@ -368,3 +368,55 @@ nsDiskCacheBindery::ActiveBindings() return activeBinding; } + +struct AccumulatorArg { + size_t mUsage; + nsMallocSizeOfFun mMallocSizeOf; +}; + +PLDHashOperator +AccumulateHeapUsage(PLDHashTable *table, PLDHashEntryHdr *hdr, uint32_t number, + void *arg) +{ + nsDiskCacheBinding *binding = ((HashTableEntry *)hdr)->mBinding; + NS_ASSERTION(binding, "### disk cache binding = nsnull!"); + + AccumulatorArg *acc = (AccumulatorArg *)arg; + + nsDiskCacheBinding *head = binding; + do { + acc->mUsage += acc->mMallocSizeOf(binding); + + if (binding->mStreamIO) { + acc->mUsage += binding->mStreamIO->SizeOfIncludingThis(acc->mMallocSizeOf); + } + + /* No good way to get at mDeactivateEvent internals for proper size, so + we use this as an estimate. */ + if (binding->mDeactivateEvent) { + acc->mUsage += acc->mMallocSizeOf(binding->mDeactivateEvent); + } + + binding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding); + } while (binding != head); + + return PL_DHASH_NEXT; +} + +/** + * SizeOfExcludingThis: return the amount of heap memory (bytes) being used by the bindery + */ +size_t +nsDiskCacheBindery::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf) +{ + NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized"); + if (!initialized) return 0; + + AccumulatorArg arg; + arg.mUsage = 0; + arg.mMallocSizeOf = aMallocSizeOf; + + PL_DHashTableEnumerate(&table, AccumulateHeapUsage, &arg); + + return arg.mUsage; +} diff --git a/netwerk/cache/nsDiskCacheBinding.h b/netwerk/cache/nsDiskCacheBinding.h index 21b6e0e6521f..7e607c38474c 100644 --- a/netwerk/cache/nsDiskCacheBinding.h +++ b/netwerk/cache/nsDiskCacheBinding.h @@ -104,7 +104,9 @@ public: nsDiskCacheBinding * FindActiveBinding(uint32_t hashNumber); void RemoveBinding(nsDiskCacheBinding * binding); bool ActiveBindings(); - + + size_t SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf); + private: nsresult AddBinding(nsDiskCacheBinding * binding); diff --git a/netwerk/cache/nsDiskCacheBlockFile.cpp b/netwerk/cache/nsDiskCacheBlockFile.cpp index 4277f395303f..5f06aa58d391 100644 --- a/netwerk/cache/nsDiskCacheBlockFile.cpp +++ b/netwerk/cache/nsDiskCacheBlockFile.cpp @@ -393,3 +393,9 @@ nsDiskCacheBlockFile::Write(int32_t offset, const void *buf, int32_t amount) return false; return PR_Write(mFD, buf, amount) == amount; } + +size_t +nsDiskCacheBlockFile::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf) +{ + return aMallocSizeOf(mBitMap) + aMallocSizeOf(mFD); +} diff --git a/netwerk/cache/nsDiskCacheBlockFile.h b/netwerk/cache/nsDiskCacheBlockFile.h index 690df84a9e07..5e5caa2e6716 100644 --- a/netwerk/cache/nsDiskCacheBlockFile.h +++ b/netwerk/cache/nsDiskCacheBlockFile.h @@ -44,7 +44,9 @@ public: int32_t * startBlock); nsresult ReadBlocks( void * buffer, int32_t startBlock, int32_t numBlocks, int32_t * bytesRead); - + + size_t SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf); + private: nsresult FlushBitMap(); int32_t AllocateBlocks( int32_t numBlocks); diff --git a/netwerk/cache/nsDiskCacheDevice.cpp b/netwerk/cache/nsDiskCacheDevice.cpp index 632a9f0d60ce..3b6a3621a44c 100644 --- a/netwerk/cache/nsDiskCacheDevice.cpp +++ b/netwerk/cache/nsDiskCacheDevice.cpp @@ -1160,3 +1160,15 @@ nsDiskCacheDevice::SetMaxEntrySize(int32_t maxSizeInKilobytes) else mMaxEntrySize = -1; } + +size_t +nsDiskCacheDevice::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf) +{ + size_t usage = aMallocSizeOf(this); + + usage += mCacheMap.SizeOfExcludingThis(aMallocSizeOf); + usage += mBindery.SizeOfExcludingThis(aMallocSizeOf); + + return usage; +} + diff --git a/netwerk/cache/nsDiskCacheDevice.h b/netwerk/cache/nsDiskCacheDevice.h index 0f38fea49932..c402b1c6293a 100644 --- a/netwerk/cache/nsDiskCacheDevice.h +++ b/netwerk/cache/nsDiskCacheDevice.h @@ -54,6 +54,8 @@ public: bool EntryIsTooBig(int64_t entrySize); + size_t SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf); + /** * Preference accessors */ diff --git a/netwerk/cache/nsDiskCacheMap.cpp b/netwerk/cache/nsDiskCacheMap.cpp index 96082cbd9d61..1194d565e64e 100644 --- a/netwerk/cache/nsDiskCacheMap.cpp +++ b/netwerk/cache/nsDiskCacheMap.cpp @@ -181,12 +181,8 @@ nsDiskCacheMap::Open(nsIFile * cacheDirectory, goto error_exit; } - { - // extra scope so the compiler doesn't barf on the above gotos jumping - // past this declaration down here - uint32_t overhead = moz_malloc_size_of(mRecordArray); - Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, overhead); - } + Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, + (uint32_t)SizeOfExcludingThis(moz_malloc_size_of)); *corruptInfo = nsDiskCache::kNotCorrupt; return NS_OK; @@ -1217,6 +1213,24 @@ nsDiskCacheMap::NotifyCapacityChange(uint32_t capacity) } } +size_t +nsDiskCacheMap::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf) +{ + size_t usage = aMallocSizeOf(mRecordArray); + + usage += aMallocSizeOf(mBuffer); + usage += aMallocSizeOf(mMapFD); + usage += aMallocSizeOf(mCleanFD); + usage += aMallocSizeOf(mCacheDirectory); + usage += aMallocSizeOf(mCleanCacheTimer); + + for (int i = 0; i < kNumBlockFiles; i++) { + usage += mBlockFile[i].SizeOfExcludingThis(aMallocSizeOf); + } + + return usage; +} + nsresult nsDiskCacheMap::InitCacheClean(nsIFile * cacheDirectory, nsDiskCache::CorruptCacheInfo * corruptInfo, diff --git a/netwerk/cache/nsDiskCacheMap.h b/netwerk/cache/nsDiskCacheMap.h index 0bc2a544c0db..71ae9feb5163 100644 --- a/netwerk/cache/nsDiskCacheMap.h +++ b/netwerk/cache/nsDiskCacheMap.h @@ -485,6 +485,8 @@ public: int32_t EntryCount() { return mHeader.mEntryCount; } + size_t SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf); + private: diff --git a/netwerk/cache/nsDiskCacheStreams.cpp b/netwerk/cache/nsDiskCacheStreams.cpp index 00b2aeba6c25..8016ff676ea7 100644 --- a/netwerk/cache/nsDiskCacheStreams.cpp +++ b/netwerk/cache/nsDiskCacheStreams.cpp @@ -734,3 +734,15 @@ nsDiskCacheStreamIO::DeleteBuffer() mBufSize = 0; } } + +size_t +nsDiskCacheStreamIO::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf) +{ + size_t usage = aMallocSizeOf(this); + + usage += aMallocSizeOf(mLocalFile); + usage += aMallocSizeOf(mFD); + usage += aMallocSizeOf(mBuffer); + + return usage; +} diff --git a/netwerk/cache/nsDiskCacheStreams.h b/netwerk/cache/nsDiskCacheStreams.h index bf43710425c0..995aa963dec9 100644 --- a/netwerk/cache/nsDiskCacheStreams.h +++ b/netwerk/cache/nsDiskCacheStreams.h @@ -46,6 +46,8 @@ public: NS_ASSERTION(mInStreamCount >= 0, "mInStreamCount has gone negative"); } + size_t SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf); + // GCC 2.95.2 requires this to be defined, although we never call it. // and OS/2 requires that it not be private nsDiskCacheStreamIO() { NS_NOTREACHED("oops"); } diff --git a/toolkit/components/telemetry/Histograms.json b/toolkit/components/telemetry/Histograms.json index 89dd5949133b..87f2bd3b6550 100644 --- a/toolkit/components/telemetry/Histograms.json +++ b/toolkit/components/telemetry/Histograms.json @@ -1097,6 +1097,12 @@ "n_buckets": 50, "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_EVICTENTRIESFORCLIENT" }, + "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHESERVICE_DISKDEVICEHEAPSIZE": { + "kind": "exponential", + "high": "10 * 1000", + "n_buckets": 50, + "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_DISKDEVICEHEAPSIZE" + }, "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHEENTRYDESCRIPTOR_DOOM": { "kind": "exponential", "high": "10 * 1000",