Bug 1151812 - Add telemetry to find optimal cache entry hash size, r=honzab

netwerk/cache2/CacheIndex.cpp

@@ -20,6 +20,7 @@
 #include "nsITimer.h"
 #include "mozilla/AutoRestore.h"
 #include <algorithm>
+#include "mozilla/Telemetry.h"
 
 
 #define kMinUnwrittenChanges   300
@@ -3130,6 +3131,11 @@ CacheIndex::ChangeState(EState aNewState)
     return;
   }
 
+  if ((mState == READING || mState == BUILDING || mState == UPDATING) &&
+      aNewState == READY) {
+    ReportHashStats();
+  }
+
   // Try to evict entries over limit everytime we're leaving state READING,
   // BUILDING or UPDATING, but not during shutdown or when removing all
   // entries.
@@ -3627,5 +3633,73 @@ CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf)
   return mallocSizeOf(gInstance) + SizeOfExcludingThis(mallocSizeOf);
 }
 
+namespace { // anon
+
+class HashComparator
+{
+public:
+  bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
+    return memcmp(&a->mHash, &b->mHash, sizeof(SHA1Sum::Hash)) == 0;
+  }
+  bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
+    return memcmp(&a->mHash, &b->mHash, sizeof(SHA1Sum::Hash)) < 0;
+  }
+};
+
+void
+ReportHashSizeMatch(const SHA1Sum::Hash *aHash1, const SHA1Sum::Hash *aHash2)
+{
+  const uint32_t *h1 = reinterpret_cast<const uint32_t *>(aHash1);
+  const uint32_t *h2 = reinterpret_cast<const uint32_t *>(aHash2);
+
+  for (uint32_t i = 0; i < 5; ++i) {
+    if (h1[i] != h2[i]) {
+      uint32_t bitsDiff = h1[i] ^ h2[i];
+      bitsDiff = NetworkEndian::readUint32(&bitsDiff);
+
+      // count leading zeros in bitsDiff
+      static const uint8_t debruijn32[32] =
+        { 0, 31, 9, 30, 3, 8, 13, 29, 2, 5, 7, 21, 12, 24, 28, 19,
+          1, 10, 4, 14, 6, 22, 25, 20, 11, 15, 23, 26, 16, 27, 17, 18};
+
+      bitsDiff |= bitsDiff>>1;
+      bitsDiff |= bitsDiff>>2;
+      bitsDiff |= bitsDiff>>4;
+      bitsDiff |= bitsDiff>>8;
+      bitsDiff |= bitsDiff>>16;
+      bitsDiff++;
+
+      uint8_t hashSizeMatch = debruijn32[bitsDiff*0x076be629>>27] + (i<<5);
+      Telemetry::Accumulate(Telemetry::NETWORK_CACHE_HASH_STATS, hashSizeMatch);
+
+      return;
+    }
+  }
+
+  MOZ_ASSERT(false, "Found a collision in the index!");
+}
+
+} // anon
+
+void
+CacheIndex::ReportHashStats()
+{
+  // We're gathering the hash stats only once, exclude too small caches.
+  if (CacheObserver::HashStatsReported() || mFrecencyArray.Length() < 15000) {
+    return;
+  }
+
+  nsTArray<CacheIndexRecord *> records;
+  records.AppendElements(mFrecencyArray);
+
+  records.Sort(HashComparator());
+
+  for (uint32_t i = 1; i < records.Length(); i++) {
+    ReportHashSizeMatch(&records[i-1]->mHash, &records[i]->mHash);
+  }
+
+  CacheObserver::SetHashStatsReported();
+}
+
 } // net
 } // mozilla

netwerk/cache2/CacheIndex.h

@@ -926,6 +926,8 @@ private:
   // Memory reporting (private part)
   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
 
+  void ReportHashStats();
+
   static CacheIndex *gInstance;
 
   nsCOMPtr<nsIFile> mCacheDirectory;

netwerk/cache2/CacheObserver.cpp

@@ -89,6 +89,9 @@ bool CacheObserver::sClearCacheOnShutdown = kDefaultClearCacheOnShutdown;
 static bool kDefaultCacheFSReported = false;
 bool CacheObserver::sCacheFSReported = kDefaultCacheFSReported;
 
+static bool kDefaultHashStatsReported = false;
+bool CacheObserver::sHashStatsReported = kDefaultHashStatsReported;
+
 NS_IMPL_ISUPPORTS(CacheObserver,
                   nsIObserver,
                   nsISupportsWeakReference)
@@ -346,6 +349,32 @@ CacheObserver::StoreCacheFSReported()
                                sCacheFSReported);
 }
 
+// static
+void
+CacheObserver::SetHashStatsReported()
+{
+  sHashStatsReported = true;
+
+  if (!sSelf) {
+    return;
+  }
+
+  if (NS_IsMainThread()) {
+    sSelf->StoreHashStatsReported();
+  } else {
+    nsCOMPtr<nsIRunnable> event =
+      NS_NewRunnableMethod(sSelf, &CacheObserver::StoreHashStatsReported);
+    NS_DispatchToMainThread(event);
+  }
+}
+
+void
+CacheObserver::StoreHashStatsReported()
+{
+  mozilla::Preferences::SetInt("browser.cache.disk.hashstats_reported",
+                               sHashStatsReported);
+}
+
 // static
 void CacheObserver::ParentDirOverride(nsIFile** aDir)
 {

netwerk/cache2/CacheObserver.h

@@ -64,6 +64,9 @@ class CacheObserver : public nsIObserver
   static bool const CacheFSReported()
     { return sCacheFSReported; }
   static void SetCacheFSReported();
+  static bool const HashStatsReported()
+    { return sHashStatsReported; }
+  static void SetHashStatsReported();
   static void ParentDirOverride(nsIFile ** aDir);
 
   static bool const EntryIsTooBig(int64_t aSize, bool aUsingDisk);
@@ -73,6 +76,7 @@ private:
 
   void StoreDiskCacheCapacity();
   void StoreCacheFSReported();
+  void StoreHashStatsReported();
   void AttachToPreferences();
 
   static uint32_t sUseNewCache;
@@ -96,6 +100,7 @@ private:
   static bool sSanitizeOnShutdown;
   static bool sClearCacheOnShutdown;
   static bool sCacheFSReported;
+  static bool sHashStatsReported;
 
   // Non static properties, accessible via sSelf
   nsCOMPtr<nsIFile> mCacheParentDirectoryOverride;

toolkit/components/telemetry/Histograms.json

@@ -7056,6 +7056,12 @@
     "n_buckets": 256,
     "description": "Actual size of the metadata parsed from the disk."
   },
+  "NETWORK_CACHE_HASH_STATS": {
+    "expires_in_version": "46",
+    "kind": "enumerated",
+    "n_values": "160",
+    "description": "The longest hash match between a newly added entry and all the existing entries."
+  },
   "DATABASE_LOCKED_EXCEPTION": {
     "expires_in_version": "42",
     "kind": "enumerated",