Bug 1175562 - Persist last update time for SafeBrowsing. r=francois

toolkit/components/build/nsToolkitCompsCID.h

@@ -140,9 +140,9 @@
  #define NS_URLCLASSIFIERPREFIXSET_CID \
 { 0x3d8579f0, 0x75fa, 0x4e00, { 0xba, 0x41, 0x38, 0x66, 0x1d, 0x5b, 0x5d, 0x17} }
 
-// {8a389f21-f821-4e29-9c6b-3de6f33cd7cf}
+// {7a258022-6765-11e5-b379-b37b1f2354be}
 #define NS_URLCLASSIFIERDBSERVICE_CID \
-{ 0x8a389f21, 0xf821, 0x4e29, { 0x9c, 0x6b, 0x3d, 0xe6, 0xf3, 0x3c, 0xd7, 0xcf} }
+{ 0x7a258022, 0x6765, 0x11e5, { 0xb3, 0x79, 0xb3, 0x7b, 0x1f, 0x23, 0x54, 0xbe} }
 
 // e1797597-f4d6-4dd3-a1e1-745ad352cd80
 #define NS_URLCLASSIFIERSTREAMUPDATER_CID \

toolkit/components/url-classifier/Classifier.cpp

@@ -368,6 +368,15 @@ Classifier::MarkSpoiled(nsTArray<nsCString>& aTables)
   return NS_OK;
 }
 
+void
+Classifier::SetLastUpdateTime(const nsACString &aTable,
+                              uint64_t updateTime)
+{
+  LOG(("Marking table %s as last updated on %u",
+       PromiseFlatCString(aTable).get(), updateTime));
+  mTableFreshness.Put(aTable, updateTime / PR_MSEC_PER_SEC);
+}
+
 void
 Classifier::DropStores()
 {

toolkit/components/url-classifier/Classifier.h

@@ -60,6 +60,7 @@ public:
    * unnecessarily
    */
   nsresult MarkSpoiled(nsTArray<nsCString>& aTables);
+  void SetLastUpdateTime(const nsACString& aTableName, uint64_t updateTime);
   nsresult CacheCompletions(const CacheResultArray& aResults);
   uint32_t GetHashKey(void) { return mHashKey; }
   /*

toolkit/components/url-classifier/SafeBrowsing.jsm

@@ -80,7 +80,7 @@ this.SafeBrowsing = {
     let providerName = this.listToProvider[listname];
     let provider = this.providers[providerName];
 
-    listManager.registerTable(listname, provider.updateURL, provider.gethashURL);
+    listManager.registerTable(listname, providerName, provider.updateURL, provider.gethashURL);
   },
 
   registerTables: function() {

toolkit/components/url-classifier/content/listmanager.js

@@ -93,6 +93,7 @@ PROT_ListManager.prototype.shutdown_ = function() {
  * @returns true if the table could be created; false otherwise
  */
 PROT_ListManager.prototype.registerTable = function(tableName,
+                                                    providerName,
                                                     updateUrl,
                                                     gethashUrl) {
   log("registering " + tableName + " with " + updateUrl);
@@ -103,6 +104,7 @@ PROT_ListManager.prototype.registerTable = function(tableName,
   this.tablesData[tableName] = {};
   this.tablesData[tableName].updateUrl = updateUrl;
   this.tablesData[tableName].gethashUrl = gethashUrl;
+  this.tablesData[tableName].provider = providerName;
 
   // Keep track of all of our update URLs.
   if (!this.needsUpdate_[updateUrl]) {
@@ -196,6 +198,8 @@ PROT_ListManager.prototype.kickoffUpdate_ = function (onDiskTableData)
 {
   this.startingUpdate_ = false;
   var initialUpdateDelay = 3000;
+  // Add a fuzz of 0-5 minutes.
+  initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
 
   // If the user has never downloaded tables, do the check now.
   log("needsUpdate: " + JSON.stringify(this.needsUpdate_, undefined, 2));
@@ -206,10 +210,46 @@ PROT_ListManager.prototype.kickoffUpdate_ = function (onDiskTableData)
     // Don't set the updateChecker unless at least one table has updates
     // enabled.
     if (this.updatesNeeded_(updateUrl) && !this.updateCheckers_[updateUrl]) {
-      log("Initializing update checker for " + updateUrl);
+      let provider = null;
+      Object.keys(this.tablesData).forEach(function(table) {
+        if (this.tablesData[table].updateUrl === updateUrl) {
+          let newProvider = this.tablesData[table].provider;
+          if (provider) {
+            if (newProvider !== provider) {
+              log("Multiple tables for the same updateURL have a different provider?!");
+            }
+          } else {
+            provider = newProvider;
+          }
+        }
+      }, this);
+      log("Initializing update checker for " + updateUrl
+          + " provided by " + provider);
+
+      // Use the initialUpdateDelay + fuzz unless we had previous updates
+      // and the server told us when to try again.
+      let updateDelay = initialUpdateDelay;
+      let targetPref = "browser.safebrowsing.provider." + provider + ".nextupdatetime";
+      let nextUpdate = this.prefs_.getPref(targetPref);
+      if (nextUpdate) {
+        updateDelay = Math.max(0, nextUpdate - Date.now());
+        log("Next update at " + nextUpdate
+            + " which is " + updateDelay + "ms from now");
+      }
+
+      // Set the last update time to verify if data is still valid.
+      let freshnessPref = "browser.safebrowsing.provider." + provider + ".lastupdatetime";
+      let freshness = this.prefs_.getPref(freshnessPref);
+      if (freshness) {
+        Object.keys(this.tablesData).forEach(function(table) {
+        if (this.tablesData[table].provider === provider) {
+          this.dbService_.setLastUpdateTime(table, freshness);
+        }}, this);
+      }
+
       this.updateCheckers_[updateUrl] =
         new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl),
-                    initialUpdateDelay, false /* repeating */);
+                    updateDelay, false /* repeating */);
     } else {
       log("No updates needed or already initialized for " + updateUrl);
     }
@@ -407,6 +447,34 @@ PROT_ListManager.prototype.updateSuccess_ = function(tableList, updateUrl,
 
   // Let the backoff object know that we completed successfully.
   this.requestBackoffs_[updateUrl].noteServerResponse(200);
+
+  // Set last update time for provider
+  // Get the provider for these tables, check for consistency
+  let tables = tableList.split(",");
+  let provider = null;
+  for (let table of tables) {
+    let newProvider = this.tablesData[table].provider;
+    if (provider) {
+      if (newProvider !== provider) {
+        log("Multiple tables for the same updateURL have a different provider?!");
+      }
+    } else {
+      provider = newProvider;
+    }
+  }
+
+  // Store the last update time (needed to know if the table is "fresh")
+  // and the next update time (to know when to update next).
+  let lastUpdatePref = "browser.safebrowsing.provider." + provider + ".lastupdatetime";
+  let now = Date.now();
+  log("Setting last update of " + provider + " to " + now);
+  this.prefs_.setPref(lastUpdatePref, now.toString());
+
+  let nextUpdatePref = "browser.safebrowsing.provider." + provider + ".nextupdatetime";
+  let targetTime = now + delay;
+  log("Setting next update of " + provider + " to " + targetTime
+      + " (" + delay + "ms from now)");
+  this.prefs_.setPref(nextUpdatePref, targetTime.toString());
 }
 
 /**

toolkit/components/url-classifier/nsIUrlClassifierDBService.idl

@@ -66,7 +66,7 @@ interface nsIUrlClassifierUpdateObserver : nsISupports {
  * It provides async methods for querying and updating the database.  As the
  * methods complete, they call the callback function.
  */
-[scriptable, uuid(3f9e61e5-01bd-45d0-8dd2-f1abcd20dbb7)]
+[scriptable, uuid(7a258022-6765-11e5-b379-b37b1f2354be)]
 interface nsIUrlClassifierDBService : nsISupports
 {
   /**
@@ -100,6 +100,13 @@ interface nsIUrlClassifierDBService : nsISupports
   void setHashCompleter(in ACString tableName,
                         in nsIUrlClassifierHashCompleter completer);
 
+  /**
+   * Set the last update time for the given table. We use this to
+   * remember freshness past restarts. Time is in milliseconds since epoch.
+   */
+  void setLastUpdateTime(in ACString tableName,
+                         in unsigned long long lastUpdateTime);
+
   ////////////////////////////////////////////////////////////////////////////
   // Incremental update methods.
   //

toolkit/components/url-classifier/nsIUrlListManager.idl

@@ -18,7 +18,7 @@ interface nsIUrlListManagerCallback : nsISupports {
 };
 
 
-[scriptable, uuid(5d5ed98f-72cd-46b6-a9fe-76418adfdfeb)]
+[scriptable, uuid(d60a08ee-5c83-4eb6-bdfb-79fd0716501e)]
 interface nsIUrlListManager : nsISupports
 {
     /**
@@ -32,10 +32,12 @@ interface nsIUrlListManager : nsISupports
      * @param tableName A string of the format
      *        provider_name-semantic_type-table_type.  For example,
      *        goog-white-enchash or goog-black-url.
+     * @param providerName The name of the entity providing the list.
      * @param updateUrl The URL from which to fetch updates.
      * @param gethashUrl The URL from which to fetch hash completions.
      */
     boolean registerTable(in ACString tableName,
+                          in ACString providerName,
                           in ACString updateUrl,
                           in ACString gethashUrl);
 

toolkit/components/url-classifier/nsUrlClassifierDBService.cpp

@@ -735,6 +735,18 @@ nsUrlClassifierDBServiceWorker::OpenDb()
   return NS_OK;
 }
 
+nsresult
+nsUrlClassifierDBServiceWorker::SetLastUpdateTime(const nsACString &table,
+                                                  uint64_t updateTime)
+{
+  MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread");
+  MOZ_ASSERT(mClassifier, "Classifier connection must be opened");
+
+  mClassifier->SetLastUpdateTime(table, updateTime);
+
+  return NS_OK;
+}
+
 // -------------------------------------------------------------------------
 // nsUrlClassifierLookupCallback
 //
@@ -1408,6 +1420,15 @@ nsUrlClassifierDBService::SetHashCompleter(const nsACString &tableName,
   return NS_OK;
 }
 
+NS_IMETHODIMP
+nsUrlClassifierDBService::SetLastUpdateTime(const nsACString &tableName,
+                                            uint64_t lastUpdateTime)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->SetLastUpdateTime(tableName, lastUpdateTime);
+}
+
 NS_IMETHODIMP
 nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
                                       const nsACString &updateTables)

toolkit/components/url-classifier/nsUrlClassifierProxies.cpp

@@ -216,6 +216,21 @@ UrlClassifierDBServiceWorkerProxy::CacheMissesRunnable::Run()
   return NS_OK;
 }
 
+NS_IMETHODIMP
+UrlClassifierDBServiceWorkerProxy::SetLastUpdateTime(const nsACString& table,
+                                                     uint64_t lastUpdateTime)
+{
+  nsCOMPtr<nsIRunnable> r =
+    new SetLastUpdateTimeRunnable(mTarget, table, lastUpdateTime);
+  return DispatchToWorkerThread(r);
+}
+
+NS_IMETHODIMP
+UrlClassifierDBServiceWorkerProxy::SetLastUpdateTimeRunnable::Run()
+{
+  mTarget->SetLastUpdateTime(mTable, mUpdateTime);
+  return NS_OK;
+}
 
 NS_IMPL_ISUPPORTS(UrlClassifierLookupCallbackProxy,
                   nsIUrlClassifierLookupCallback)

toolkit/components/url-classifier/nsUrlClassifierProxies.h

@@ -173,6 +173,24 @@ public:
     mozilla::safebrowsing::LookupResultArray* mResults;
   };
 
+  class SetLastUpdateTimeRunnable : public nsRunnable
+  {
+  public:
+    SetLastUpdateTimeRunnable(nsUrlClassifierDBServiceWorker* aTarget,
+                              const nsACString& table,
+                              uint64_t updateTime)
+      : mTarget(aTarget),
+        mTable(table),
+        mUpdateTime(updateTime)
+    { }
+
+    NS_DECL_NSIRUNNABLE
+  private:
+    nsRefPtr<nsUrlClassifierDBServiceWorker> mTarget;
+    nsCString mTable;
+    uint64_t mUpdateTime;
+  };
+
 public:
   nsresult DoLocalLookup(const nsACString& spec,
                          const nsACString& tables,