Bug 699951 - add a memory reporter for heap usage by the disk cache. r=michal

netwerk/cache/nsCacheService.cpp

@@ -1060,12 +1060,23 @@ nsCacheService *   nsCacheService::gService = nullptr;
 static nsCOMPtr<nsIMemoryReporter> MemoryCacheReporter = nullptr;
 
 NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkMemoryCache,
-    "explicit/network-memory-cache",
+    "explicit/network/memory-cache",
     KIND_HEAP,
     UNITS_BYTES,
     nsCacheService::MemoryDeviceSize,
     "Memory used by the network memory cache.")
 
+NS_MEMORY_REPORTER_MALLOC_SIZEOF_FUN(NetworkDiskCacheSizeOfFun, "network-disk-cache")
+
+static nsCOMPtr<nsIMemoryReporter> DiskCacheReporter = nullptr;
+
+NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkDiskCache,
+    "explicit/network/disk-cache",
+    KIND_HEAP,
+    UNITS_BYTES,
+    nsCacheService::DiskDeviceHeapSize,
+    "Memory used by the network disk cache.")
+
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheService, nsICacheService)
 
 nsCacheService::nsCacheService()
@@ -1206,11 +1217,14 @@ nsCacheService::Shutdown()
         mObserver->Remove();
         NS_RELEASE(mObserver);
         
-        // unregister memory reporter, before deleting the memory device, just
+        // unregister memory reporters, before deleting the devices, just
         // to be safe
         NS_UnregisterMemoryReporter(MemoryCacheReporter);
         MemoryCacheReporter = nullptr;
 
+        NS_UnregisterMemoryReporter(DiskCacheReporter);
+        DiskCacheReporter = nullptr;
+
         // deallocate memory and disk caches
         delete mMemoryDevice;
         mMemoryDevice = nullptr;
@@ -1564,6 +1578,9 @@ nsCacheService::CreateDiskDevice()
     // Ignore state of the timer and return success since the purpose of the
     // method (create the disk-device) has been fulfilled
 
+    DiskCacheReporter = new NS_MEMORY_REPORTER_NAME(NetworkDiskCache);
+    NS_RegisterMemoryReporter(DiskCacheReporter);
+
     return NS_OK;
 }
 
@@ -2236,6 +2253,14 @@ nsCacheService::MemoryDeviceSize()
     return memoryDevice ? memoryDevice->TotalSize() : 0;
 }
 
+int64_t
+nsCacheService::DiskDeviceHeapSize()
+{
+    nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_DISKDEVICEHEAPSIZE));
+    nsDiskCacheDevice *diskDevice = GlobalInstance()->mDiskDevice;
+    return (int64_t)(diskDevice ? diskDevice->SizeOfIncludingThis(NetworkDiskCacheSizeOfFun) : 0);
+}
+
 nsresult
 nsCacheService::DoomEntry(nsCacheEntry * entry)
 {

netwerk/cache/nsCacheService.h

@@ -129,6 +129,8 @@ public:
     nsCacheService * GlobalInstance()   { return gService; }
 
     static int64_t   MemoryDeviceSize();
+
+    static int64_t   DiskDeviceHeapSize();
     
     static nsresult  DoomEntry(nsCacheEntry * entry);
 

netwerk/cache/nsDiskCacheBinding.cpp

@@ -368,3 +368,55 @@ nsDiskCacheBindery::ActiveBindings()
 
     return activeBinding;
 }
+
+struct AccumulatorArg {
+    size_t mUsage;
+    nsMallocSizeOfFun mMallocSizeOf;
+};
+
+PLDHashOperator
+AccumulateHeapUsage(PLDHashTable *table, PLDHashEntryHdr *hdr, uint32_t number,
+                    void *arg)
+{
+    nsDiskCacheBinding *binding = ((HashTableEntry *)hdr)->mBinding;
+    NS_ASSERTION(binding, "### disk cache binding = nsnull!");
+
+    AccumulatorArg *acc = (AccumulatorArg *)arg;
+
+    nsDiskCacheBinding *head = binding;
+    do {
+        acc->mUsage += acc->mMallocSizeOf(binding);
+
+        if (binding->mStreamIO) {
+            acc->mUsage += binding->mStreamIO->SizeOfIncludingThis(acc->mMallocSizeOf);
+        }
+
+        /* No good way to get at mDeactivateEvent internals for proper size, so
+           we use this as an estimate. */
+        if (binding->mDeactivateEvent) {
+            acc->mUsage += acc->mMallocSizeOf(binding->mDeactivateEvent);
+        }
+
+        binding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding);
+    } while (binding != head);
+
+    return PL_DHASH_NEXT;
+}
+
+/**
+ * SizeOfExcludingThis: return the amount of heap memory (bytes) being used by the bindery
+ */
+size_t
+nsDiskCacheBindery::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized");
+    if (!initialized) return 0;
+
+    AccumulatorArg arg;
+    arg.mUsage = 0;
+    arg.mMallocSizeOf = aMallocSizeOf;
+
+    PL_DHashTableEnumerate(&table, AccumulateHeapUsage, &arg);
+
+    return arg.mUsage;
+}

netwerk/cache/nsDiskCacheBinding.h

@@ -104,7 +104,9 @@ public:
     nsDiskCacheBinding *    FindActiveBinding(uint32_t  hashNumber);
     void                    RemoveBinding(nsDiskCacheBinding * binding);
     bool                    ActiveBindings();
-    
+
+    size_t                 SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 private:
     nsresult                AddBinding(nsDiskCacheBinding * binding);
 

netwerk/cache/nsDiskCacheBlockFile.cpp

@@ -393,3 +393,9 @@ nsDiskCacheBlockFile::Write(int32_t offset, const void *buf, int32_t amount)
         return false;
     return PR_Write(mFD, buf, amount) == amount;
 }
+
+size_t
+nsDiskCacheBlockFile::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    return aMallocSizeOf(mBitMap) + aMallocSizeOf(mFD);
+}

netwerk/cache/nsDiskCacheBlockFile.h

@@ -44,7 +44,9 @@ public:
                            int32_t * startBlock);
     nsresult  ReadBlocks( void * buffer, int32_t  startBlock, int32_t  numBlocks, 
                           int32_t * bytesRead);
-    
+
+    size_t   SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 private:
     nsresult  FlushBitMap();
     int32_t   AllocateBlocks( int32_t  numBlocks);

netwerk/cache/nsDiskCacheDevice.cpp

@@ -1160,3 +1160,15 @@ nsDiskCacheDevice::SetMaxEntrySize(int32_t maxSizeInKilobytes)
     else
         mMaxEntrySize = -1;
 }
+
+size_t
+nsDiskCacheDevice::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    size_t usage = aMallocSizeOf(this);
+
+    usage += mCacheMap.SizeOfExcludingThis(aMallocSizeOf);
+    usage += mBindery.SizeOfExcludingThis(aMallocSizeOf);
+
+    return usage;
+}
+

netwerk/cache/nsDiskCacheDevice.h

@@ -54,6 +54,8 @@ public:
 
     bool                    EntryIsTooBig(int64_t entrySize);
 
+    size_t                 SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
     /**
      * Preference accessors
      */

netwerk/cache/nsDiskCacheMap.cpp

@@ -181,12 +181,8 @@ nsDiskCacheMap::Open(nsIFile *  cacheDirectory,
         goto error_exit;
     }
     
-    {
-        // extra scope so the compiler doesn't barf on the above gotos jumping
-        // past this declaration down here
-        uint32_t overhead = moz_malloc_size_of(mRecordArray);
-        Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, overhead);
-    }
+    Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD,
+                          (uint32_t)SizeOfExcludingThis(moz_malloc_size_of));
 
     *corruptInfo = nsDiskCache::kNotCorrupt;
     return NS_OK;
@@ -1217,6 +1213,24 @@ nsDiskCacheMap::NotifyCapacityChange(uint32_t capacity)
   }
 }
 
+size_t
+nsDiskCacheMap::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+  size_t usage = aMallocSizeOf(mRecordArray);
+
+  usage += aMallocSizeOf(mBuffer);
+  usage += aMallocSizeOf(mMapFD);
+  usage += aMallocSizeOf(mCleanFD);
+  usage += aMallocSizeOf(mCacheDirectory);
+  usage += aMallocSizeOf(mCleanCacheTimer);
+
+  for (int i = 0; i < kNumBlockFiles; i++) {
+    usage += mBlockFile[i].SizeOfExcludingThis(aMallocSizeOf);
+  }
+
+  return usage;
+}
+
 nsresult
 nsDiskCacheMap::InitCacheClean(nsIFile *  cacheDirectory,
                                nsDiskCache::CorruptCacheInfo *  corruptInfo,

netwerk/cache/nsDiskCacheMap.h

@@ -485,6 +485,8 @@ public:
     
     int32_t  EntryCount()  { return mHeader.mEntryCount; }
 
+    size_t  SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 
 private:
 

netwerk/cache/nsDiskCacheStreams.cpp

@@ -734,3 +734,15 @@ nsDiskCacheStreamIO::DeleteBuffer()
         mBufSize = 0;
     }
 }
+
+size_t
+nsDiskCacheStreamIO::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    size_t usage = aMallocSizeOf(this);
+
+    usage += aMallocSizeOf(mLocalFile);
+    usage += aMallocSizeOf(mFD);
+    usage += aMallocSizeOf(mBuffer);
+
+    return usage;
+}

netwerk/cache/nsDiskCacheStreams.h

@@ -46,6 +46,8 @@ public:
                     NS_ASSERTION(mInStreamCount >= 0, "mInStreamCount has gone negative");
                 }
 
+    size_t     SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
     // GCC 2.95.2 requires this to be defined, although we never call it.
     // and OS/2 requires that it not be private
     nsDiskCacheStreamIO() { NS_NOTREACHED("oops"); }

toolkit/components/telemetry/Histograms.json

@@ -1097,6 +1097,12 @@
     "n_buckets": 50,
     "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_EVICTENTRIESFORCLIENT"
   },
+  "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHESERVICE_DISKDEVICEHEAPSIZE": {
+    "kind": "exponential",
+    "high": "10 * 1000",
+    "n_buckets": 50,
+    "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_DISKDEVICEHEAPSIZE"
+  },
   "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHEENTRYDESCRIPTOR_DOOM": {
     "kind": "exponential",
     "high": "10 * 1000",