Bug 243136 - saved form data should expire after a time period defined by user. r=sdwilsh, r=gavin

toolkit/components/satchel/src/nsStorageFormHistory.cpp

@@ -66,7 +66,7 @@
 #include "nsIPrivateBrowsingService.h"
 #include "nsNetCID.h"
 
-#define DB_SCHEMA_VERSION   1
+#define DB_SCHEMA_VERSION   2
 #define DB_FILENAME         NS_LITERAL_STRING("formhistory.sqlite")
 #define DB_CORRUPT_FILENAME NS_LITERAL_STRING("formhistory.sqlite.corrupt")
 
@@ -205,8 +205,11 @@ nsFormHistory::Init()
 #endif
 
   nsCOMPtr<nsIObserverService> service = do_GetService("@mozilla.org/observer-service;1");
-  if (service)
+  if (service) {
     service->AddObserver(this, NS_EARLYFORMSUBMIT_SUBJECT, PR_TRUE);
+    service->AddObserver(this, "idle-daily", PR_TRUE);
+    service->AddObserver(this, "formhistory-expire-now", PR_TRUE);
+  }
 
   return NS_OK;
 }
@@ -447,6 +450,9 @@ nsFormHistory::Observe(nsISupports *aSubject, const char *aTopic, const PRUnicha
 {
   if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
     mPrefBranch->GetBoolPref(PREF_FORMFILL_ENABLE, &gFormHistoryEnabled);
+  } else if (!strcmp(aTopic, "idle-daily") ||
+             !strcmp(aTopic, "formhistory-expire-now")) {
+      ExpireOldEntries();
   }
 
   return NS_OK;
@@ -518,6 +524,68 @@ nsFormHistory::Notify(nsIDOMHTMLFormElement* formElt, nsIDOMWindowInternal* aWin
   return transaction.Commit();
 }
 
+nsresult
+nsFormHistory::ExpireOldEntries()
+{
+  // Determine how many days of history we're supposed to keep.
+  nsresult rv;
+  nsCOMPtr<nsIPrefBranch> prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  PRInt32 expireDays;
+  rv = prefBranch->GetIntPref("browser.formfill.expire_days", &expireDays);
+  if (NS_FAILED(rv))
+    rv = prefBranch->GetIntPref("browser.history_expire_days", &expireDays);
+  NS_ENSURE_SUCCESS(rv, rv);
+  PRInt64 expireTime = PR_Now() - expireDays * 24 * PR_HOURS;
+
+
+  PRInt32 beginningCount = CountAllEntries();
+
+  // Purge the form history...
+  nsCOMPtr<mozIStorageStatement> stmt;
+  rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
+          "DELETE FROM moz_formhistory WHERE lastUsed<=?1"),
+          getter_AddRefs(stmt));
+  NS_ENSURE_SUCCESS(rv,rv);
+  rv = stmt->BindInt64Parameter(0, expireTime);
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = stmt->Execute();
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  PRInt32 endingCount = CountAllEntries();
+
+  // If we expired a large batch of entries, shrink the DB to reclaim wasted
+  // space. This is expected to happen when entries predating timestamps
+  // (added in the v.1 schema) expire in mass, 180 days after the DB was
+  // upgraded -- entries not used since then expire all at once.
+  if (beginningCount - endingCount > 500) {
+    rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM"));
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+
+  return NS_OK;
+}
+
+PRInt32
+nsFormHistory::CountAllEntries()
+{
+  nsCOMPtr<mozIStorageStatement> stmt;
+  nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
+                  "SELECT COUNT(*) FROM moz_formhistory"),
+                  getter_AddRefs(stmt));
+
+  PRBool hasResult;
+  rv = stmt->ExecuteStep(&hasResult);
+  NS_ENSURE_SUCCESS(rv, 0);
+
+  PRInt32 count = 0;
+  if (hasResult)
+    count = stmt->AsInt32(0);
+
+  return count;
+}
+
 nsresult
 nsFormHistory::CreateTable()
 {
@@ -531,6 +599,8 @@ nsFormHistory::CreateTable()
   NS_ENSURE_SUCCESS(rv, rv);
   rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("CREATE INDEX moz_formhistory_index ON moz_formhistory (fieldname)"));
   NS_ENSURE_SUCCESS(rv, rv);
+  rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("CREATE INDEX moz_formhistory_lastused_index ON moz_formhistory (lastUsed)"));
+  NS_ENSURE_SUCCESS(rv, rv);
 
   rv = mDBConn->SetSchemaVersion(DB_SCHEMA_VERSION);
   NS_ENSURE_SUCCESS(rv, rv);
@@ -637,7 +707,10 @@ nsFormHistory::dbMigrate()
       rv = MigrateToVersion1();
       NS_ENSURE_SUCCESS(rv, rv);
       // (fallthrough to the next upgrade)
-
+    case 1:
+      rv = MigrateToVersion2();
+      NS_ENSURE_SUCCESS(rv, rv);
+      // (fallthrough to the next upgrade)
     case DB_SCHEMA_VERSION:
       // (current version, nothing more to do)
       break;
@@ -716,6 +789,30 @@ nsFormHistory::MigrateToVersion1()
 }
 
 
+/*
+ * MigrateToVersion2
+ *
+ * Updates the DB schema to v2 (bug 243136).
+ * Adds lastUsed index, removes moz_dummy_table
+ */
+nsresult
+nsFormHistory::MigrateToVersion2()
+{
+  nsresult rv;
+
+  rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DROP TABLE IF EXISTS moz_dummy_table"));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS moz_formhistory_lastused_index ON moz_formhistory (lastUsed)"));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = mDBConn->SetSchemaVersion(2);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return NS_OK;
+}
+
+
 nsresult
 nsFormHistory::GetDatabaseFile(nsIFile** aFile)
 {

toolkit/components/satchel/src/nsStorageFormHistory.h

@@ -120,6 +120,7 @@ public:
   nsresult dbMigrate();
   nsresult dbCleanup();
   nsresult MigrateToVersion1();
+  nsresult MigrateToVersion2();
   PRBool   dbAreExpectedColumnsPresent();
 
   nsresult CreateTable();
@@ -130,6 +131,8 @@ public:
   static PRBool gFormHistoryEnabled;
   static PRBool gPrefsInitialized;
 
+  nsresult ExpireOldEntries();
+  PRInt32 CountAllEntries();
   PRInt64 GetExistingEntryID(const nsAString &aName, const nsAString &aValue);
 
   nsCOMPtr<nsIPrefBranch> mPrefBranch;

toolkit/components/satchel/test/unit/head_satchel.js

@@ -38,7 +38,7 @@
 const Ci = Components.interfaces;
 const Cc = Components.classes;
 
-const CURRENT_SCHEMA = 1;
+const CURRENT_SCHEMA = 2;
 const PR_HOURS = 60 * 60 * 1000000;
 
 

toolkit/components/satchel/test/unit/test_db_update_v2.js

@@ -0,0 +1,88 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Satchel Test Code.
+ *
+ * The Initial Developer of the Original Code is
+ * Mozilla Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2009
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Justin Dolske <dolske@mozilla.com> (Original Author)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+var testnum = 0;
+var fh;
+
+function run_test()
+{
+  try {
+
+  // ===== test init =====
+  var testfile = do_get_file("formhistory_v1.sqlite");
+  var profileDir = dirSvc.get("ProfD", Ci.nsIFile);
+
+  // Cleanup from any previous tests or failures.
+  var destFile = profileDir.clone();
+  destFile.append("formhistory.sqlite");
+  if (destFile.exists())
+    destFile.remove(false);
+
+  testfile.copyTo(profileDir, "formhistory.sqlite");
+  do_check_eq(1, getDBVersion(testfile));
+
+  fh = Cc["@mozilla.org/satchel/form-history;1"].
+       getService(Ci.nsIFormHistory2);
+
+
+  // ===== 1 =====
+  testnum++;
+
+  // Check that the index was added (which is all the v2 upgrade does)
+  do_check_true(fh.DBConnection.indexExists("moz_formhistory_lastused_index"));
+  // check for upgraded schema.
+  do_check_eq(CURRENT_SCHEMA, fh.DBConnection.schemaVersion);
+  // Check that old table was removed
+  do_check_false(fh.DBConnection.tableExists("moz_dummy_table"));
+
+
+  // ===== 2 =====
+  testnum++;
+
+  // Just sanity check for expected contents and that DB is working.
+  do_check_true(fh.entryExists("name-A", "value-A"));
+  do_check_false(fh.entryExists("name-B", "value-B"));
+  fh.addEntry("name-B", "value-B");
+  do_check_true(fh.entryExists("name-B", "value-B"));
+  fh.removeEntry("name-B", "value-B");
+  do_check_false(fh.entryExists("name-B", "value-B"));
+
+
+  } catch (e) {
+    throw "FAILED in test #" + testnum + " -- " + e;
+  }
+}

toolkit/components/satchel/test/unit/test_db_update_v2b.js

@@ -0,0 +1,88 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Satchel Test Code.
+ *
+ * The Initial Developer of the Original Code is
+ * Mozilla Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2009
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Justin Dolske <dolske@mozilla.com> (Original Author)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+var testnum = 0;
+var fh;
+
+function run_test()
+{
+  try {
+
+  // ===== test init =====
+  var testfile = do_get_file("formhistory_v1v2.sqlite");
+  var profileDir = dirSvc.get("ProfD", Ci.nsIFile);
+
+  // Cleanup from any previous tests or failures.
+  var destFile = profileDir.clone();
+  destFile.append("formhistory.sqlite");
+  if (destFile.exists())
+    destFile.remove(false);
+
+  testfile.copyTo(profileDir, "formhistory.sqlite");
+  do_check_eq(1, getDBVersion(testfile));
+
+  fh = Cc["@mozilla.org/satchel/form-history;1"].
+       getService(Ci.nsIFormHistory2);
+
+
+  // ===== 1 =====
+  testnum++;
+
+  // Check that the index was added / still exists (which is all the v2 upgrade does)
+  do_check_true(fh.DBConnection.indexExists("moz_formhistory_lastused_index"));
+  // check for upgraded schema.
+  do_check_eq(CURRENT_SCHEMA, fh.DBConnection.schemaVersion);
+  // Check that old table was removed
+  do_check_false(fh.DBConnection.tableExists("moz_dummy_table"));
+
+
+  // ===== 2 =====
+  testnum++;
+
+  // Just sanity check for expected contents and that DB is working.
+  do_check_true(fh.entryExists("name-A", "value-A"));
+  do_check_false(fh.entryExists("name-B", "value-B"));
+  fh.addEntry("name-B", "value-B");
+  do_check_true(fh.entryExists("name-B", "value-B"));
+  fh.removeEntry("name-B", "value-B");
+  do_check_false(fh.entryExists("name-B", "value-B"));
+
+
+  } catch (e) {
+    throw "FAILED in test #" + testnum + " -- " + e;
+  }
+}

toolkit/components/satchel/test/unit/test_expire.js

@@ -0,0 +1,196 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Satchel Test Code.
+ *
+ * The Initial Developer of the Original Code is
+ * Mozilla Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2009
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Justin Dolske <dolske@mozilla.com> (Original Author)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+var testnum = 0;
+var fh, prefs;
+
+function countAllEntries() {
+    let stmt = fh.DBConnection.createStatement("SELECT COUNT(*) as numEntries FROM moz_formhistory");
+    do_check_true(stmt.step());
+    let numEntries = stmt.row.numEntries;
+    stmt.finalize();
+    return numEntries;
+}
+
+function triggerExpiration() {
+    // We can't easily fake a "daily idle" event, so for testing purposes form
+    // history listens for another notification to trigger an immediate
+    // expiration.
+    var os = Cc["@mozilla.org/observer-service;1"].
+             getService(Ci.nsIObserverService);
+    os.notifyObservers(null, "formhistory-expire-now", null);
+}
+
+function run_test()
+{
+  try {
+
+  // ===== test init =====
+  var testfile = do_get_file("formhistory_expire.sqlite");
+  var profileDir = dirSvc.get("ProfD", Ci.nsIFile);
+
+  // Cleanup from any previous tests or failures.
+  var dbFile = profileDir.clone();
+  dbFile.append("formhistory.sqlite");
+  if (dbFile.exists())
+    dbFile.remove(false);
+
+  testfile.copyTo(profileDir, "formhistory.sqlite");
+  do_check_true(dbFile.exists());
+
+  fh = Cc["@mozilla.org/satchel/form-history;1"].
+       getService(Ci.nsIFormHistory2);
+
+  prefs = Cc["@mozilla.org/preferences-service;1"].
+          getService(Ci.nsIPrefBranch);
+
+  // We're going to clear this at the end, so it better have the default value now.
+  do_check_false(prefs.prefHasUserValue("browser.history_expire_days"));
+  do_check_false(prefs.prefHasUserValue("browser.formfill.expire_days"));
+
+
+  // ===== 1 =====
+  testnum++;
+
+  // Sanity check initial state
+  do_check_eq(CURRENT_SCHEMA, fh.DBConnection.schemaVersion);
+  do_check_eq(508, countAllEntries());
+  do_check_true(fh.entryExists("name-A", "value-A")); // lastUsed == distant past
+  do_check_true(fh.entryExists("name-B", "value-B")); // lastUsed == distant future
+
+  // Add a new entry
+  do_check_false(fh.entryExists("name-C", "value-C"));
+  fh.addEntry("name-C", "value-C");
+  do_check_true(fh.entryExists("name-C", "value-C"));
+
+  // Update some existing entries to have ages relative to when the test runs.
+  var now = 1000 * Date.now();
+  var age181 = now - 181 * 24 * PR_HOURS;
+  var age179 = now - 179 * 24 * PR_HOURS;
+  var age31  = now -  31 * 24 * PR_HOURS;
+  var age29  = now -  29 * 24 * PR_HOURS;
+  var age11  = now -  11 * 24 * PR_HOURS;
+  var age9   = now -   9 * 24 * PR_HOURS;
+
+  fh.DBConnection.executeSimpleSQL("UPDATE moz_formhistory SET lastUsed=" + age181 + " WHERE lastUsed=181");
+  fh.DBConnection.executeSimpleSQL("UPDATE moz_formhistory SET lastUsed=" + age179 + " WHERE lastUsed=179");
+  fh.DBConnection.executeSimpleSQL("UPDATE moz_formhistory SET lastUsed=" + age31  + " WHERE lastUsed=31");
+  fh.DBConnection.executeSimpleSQL("UPDATE moz_formhistory SET lastUsed=" + age29  + " WHERE lastUsed=29");
+  fh.DBConnection.executeSimpleSQL("UPDATE moz_formhistory SET lastUsed=" + age11  + " WHERE lastUsed=9999");
+  fh.DBConnection.executeSimpleSQL("UPDATE moz_formhistory SET lastUsed=" + age9   + " WHERE lastUsed=9");
+
+
+  // ===== 2 =====
+  testnum++;
+
+  // Expire history with default pref (180 days)
+  do_check_true(fh.entryExists("name-A", "value-A"));
+  do_check_true(fh.entryExists("181DaysOld", "foo"));
+  do_check_true(fh.entryExists("179DaysOld", "foo"));
+  do_check_eq(509, countAllEntries());
+
+  // 2 entries are expected to expire.
+  triggerExpiration();
+
+  do_check_false(fh.entryExists("name-A", "value-A"));
+  do_check_false(fh.entryExists("181DaysOld", "foo"));
+  do_check_true(fh.entryExists("179DaysOld", "foo"));
+  do_check_eq(507, countAllEntries());
+
+
+  // ===== 3 =====
+  testnum++;
+
+  // And again. No change expected.
+  triggerExpiration();
+  do_check_eq(507, countAllEntries());
+
+
+  // ===== 4 =====
+  testnum++;
+
+  // Set pref to 30 days and expire.
+  prefs.setIntPref("browser.history_expire_days", 30);
+  do_check_true(fh.entryExists("179DaysOld", "foo"));
+  do_check_true(fh.entryExists("bar", "31days"));
+  do_check_true(fh.entryExists("bar", "29days"));
+  do_check_eq(507, countAllEntries());
+
+  triggerExpiration();
+
+  do_check_false(fh.entryExists("179DaysOld", "foo"));
+  do_check_false(fh.entryExists("bar", "31days"));
+  do_check_true(fh.entryExists("bar", "29days"));
+  do_check_eq(505, countAllEntries());
+
+
+  // ===== 5 =====
+  testnum++;
+
+  // Set override pref to 10 days and expire. This expires a large batch of
+  // entries, and should trigger a VACCUM to reduce file size.
+  prefs.setIntPref("browser.formfill.expire_days", 10);
+  prefs.setIntPref("browser.history_expire_days", 1);
+
+  do_check_true(fh.entryExists("bar", "29days"));
+  do_check_true(fh.entryExists("9DaysOld", "foo"));
+  do_check_eq(505, countAllEntries());
+
+  do_check_true(dbFile.fileSize > 70000);
+
+  triggerExpiration();
+
+  do_check_false(fh.entryExists("bar", "29days"));
+  do_check_true(fh.entryExists("9DaysOld", "foo"));
+  do_check_true(fh.entryExists("name-B", "value-B"));
+  do_check_true(fh.entryExists("name-C", "value-C"));
+  do_check_eq(3, countAllEntries());
+
+  // Check that the file size was reduced.
+  // Need to clone the nsIFile because the size is being cached on Windows.
+  dbFile = dbFile.clone();
+  do_check_true(dbFile.fileSize < 6000);
+
+
+  } catch (e) {
+      throw "FAILED in test #" + testnum + " -- " + e;
+  } finally {
+      // Make sure we always reset prefs.
+      prefs.clearUserPref("browser.history_expire_days");
+      prefs.clearUserPref("browser.formfill.expire_days");
+  }
+}