Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-02 15:15:23 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 690354 - Idle expiration never runs for clean databases.

Browse Source 
r=dietrich
This commit is contained in:
Marco Bonardo 2011-10-10 12:15:12 +02:00
parent 6834886ad9
commit 82773d3ab9
4 changed files with 204 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
toolkit/components/places/nsPlacesExpiration.js
View File

@ -87,6 +87,7 @@ const TOPIC_DEBUG_START_EXPIRATION = "places-debug-start-expiration";
const TOPIC_EXPIRATION_FINISHED = "places-expiration-finished";
const TOPIC_IDLE_BEGIN = "idle";
const TOPIC_IDLE_END = "back";
const TOPIC_IDLE_DAILY = "idle-daily";
// Branch for all expiration preferences.
const PREF_BRANCH = "places.history.expiration.";
@ -150,6 +151,10 @@ const IDLE_TIMEOUT_SECONDS = 5 * 60;
// clearHistory to decide to skip expiration at shutdown.
const SHUTDOWN_WITH_RECENT_CLEARHISTORY_TIMEOUT_SECONDS = 10;
// If the pages delta from the last ANALYZE is over this threashold, the tables
// should be analyzed again.
const ANALYZE_PAGES_THRESHOLD = 100;
const USECS_PER_DAY = 86400000000;
const ANNOS_EXPIRE_POLICIES = [
{ bind: "expire_days",
@ -184,22 +189,22 @@ const STATUS = {
// Represents actions on which a query will run.
const ACTION = {
TIMED: 1 << 0, // happens every this._interval
CLEAR_HISTORY: 1 << 1, // happens when history is cleared
SHUTDOWN: 1 << 2, // happens at shutdown when the db has a DIRTY state
CLEAN_SHUTDOWN: 1 << 3, // happens at shutdown when the db has a CLEAN or
// UNKNOWN state
IDLE: 1 << 4, // happens once on idle
DEBUG: 1 << 5, // happens whenever TOPIC_DEBUG_START_EXPIRATION is dispatched
TIMED_OVERLIMIT: 1 << 6, // just like TIMED, but also when we have too much
// history
TIMED: 1 << 0, // happens every this._interval
TIMED_OVERLIMIT: 1 << 1, // like TIMED but only when history is over limits
TIMED_ANALYZE: 1 << 2, // happens when ANALYZE statistics should be updated
CLEAR_HISTORY: 1 << 3, // happens when history is cleared
SHUTDOWN_DIRTY: 1 << 4, // happens at shutdown for DIRTY state
SHUTDOWN_CLEAN: 1 << 5, // happens at shutdown for CLEAN or UNKNOWN states
IDLE_DIRTY: 1 << 6, // happens on idle for DIRTY state
IDLE_DAILY: 1 << 7, // happens once a day on idle
DEBUG: 1 << 8, // happens on TOPIC_DEBUG_START_EXPIRATION
};
// The queries we use to expire.
const EXPIRATION_QUERIES = {
// Finds visits to be expired. Will return nothing if we are not over the
// unique URIs limit.
// Finds visits to be expired when history is over the unique pages limit,
// otherwise will return nothing.
// This explicitly excludes any visits added in the last 7 days, to protect
// users with thousands of bookmarks from constantly losing history.
QUERY_FIND_VISITS_TO_EXPIRE: {
@ -212,7 +217,7 @@ const EXPIRATION_QUERIES = {
+ "AND visit_date < strftime('%s','now','localtime','start of day','-7 days','utc') * 1000000 "
+ "ORDER BY v.visit_date ASC "
+ "LIMIT :limit_visits",
actions: ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE |
actions: ACTION.TIMED_OVERLIMIT | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
@ -221,7 +226,7 @@ const EXPIRATION_QUERIES = {
sql: "DELETE FROM moz_historyvisits WHERE id IN ( "
+ "SELECT v_id FROM expiration_notify WHERE v_id NOTNULL "
+ ")",
actions: ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE |
actions: ACTION.TIMED_OVERLIMIT | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
@ -243,8 +248,8 @@ const EXPIRATION_QUERIES = {
+ "AND b.id IS NULL "
+ "AND frecency <> -1 "
+ "LIMIT :limit_uris",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN |
ACTION.IDLE | ACTION.DEBUG
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
// Expire found URIs from the database.
@ -252,8 +257,8 @@ const EXPIRATION_QUERIES = {
sql: "DELETE FROM moz_places WHERE id IN ( "
+ "SELECT p_id FROM expiration_notify WHERE p_id NOTNULL "
+ ")",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN |
ACTION.IDLE | ACTION.DEBUG
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
// Expire orphan URIs from the database.
@ -279,7 +284,8 @@ const EXPIRATION_QUERIES = {
+ "LIMIT :limit_favicons "
+ ")",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
// Expire orphan page annotations from the database.
@ -293,7 +299,8 @@ const EXPIRATION_QUERIES = {
+ "LIMIT :limit_annos "
+ ")",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
// Expire page annotations based on expiration policy.
@ -306,7 +313,8 @@ const EXPIRATION_QUERIES = {
+ "OR (expiration = :expire_months "
+ "AND :expire_months_time > MAX(lastModified, dateAdded))",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
// Expire items annotations based on expiration policy.
@ -319,7 +327,8 @@ const EXPIRATION_QUERIES = {
+ "OR (expiration = :expire_months "
+ "AND :expire_months_time > MAX(lastModified, dateAdded))",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
// Expire page annotations based on expiration policy.
@ -329,7 +338,8 @@ const EXPIRATION_QUERIES = {
+ "AND NOT EXISTS (SELECT id FROM moz_historyvisits "
+ "WHERE place_id = moz_annos.place_id LIMIT 1)",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
// Expire item annos without a corresponding item id.
@ -340,7 +350,7 @@ const EXPIRATION_QUERIES = {
+ "WHERE b.id IS NULL "
+ "LIMIT :limit_annos "
+ ")",
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
actions: ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
// Expire all annotation names without a corresponding annotation.
@ -353,7 +363,8 @@ const EXPIRATION_QUERIES = {
+ "AND t.anno_attribute_id IS NULL "
+ "LIMIT :limit_annos"
+ ")",
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY |
ACTION.IDLE_DAILY | ACTION.DEBUG
},
// Expire orphan inputhistory.
@ -364,21 +375,23 @@ const EXPIRATION_QUERIES = {
+ "WHERE h.id IS NULL "
+ "LIMIT :limit_inputhistory "
+ ")",
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN | ACTION.IDLE | ACTION.DEBUG
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
ACTION.DEBUG
},
// Expire all session annotations. Should only be called at shutdown.
QUERY_EXPIRE_ANNOS_SESSION: {
sql: "DELETE FROM moz_annos WHERE expiration = :expire_session",
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN | ACTION.CLEAN_SHUTDOWN |
ACTION.DEBUG
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN_DIRTY |
ACTION.SHUTDOWN_CLEAN | ACTION.DEBUG
},
// Expire all session item annotations. Should only be called at shutdown.
QUERY_EXPIRE_ITEMS_ANNOS_SESSION: {
sql: "DELETE FROM moz_items_annos WHERE expiration = :expire_session",
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN | ACTION.CLEAN_SHUTDOWN |
ACTION.DEBUG
actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN_DIRTY |
ACTION.SHUTDOWN_CLEAN | ACTION.DEBUG
},
// Select entries for notifications.
@ -390,15 +403,15 @@ const EXPIRATION_QUERIES = {
+ "expected_results "
+ "FROM expiration_notify "
+ "GROUP BY url",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN |
ACTION.IDLE | ACTION.DEBUG
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
// Empty the notifications table.
QUERY_DELETE_NOTIFICATIONS: {
sql: "DELETE FROM expiration_notify",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN |
ACTION.IDLE | ACTION.DEBUG
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
// The following queries are used to adjust the sqlite_stat1 table to help the
@ -409,22 +422,22 @@ const EXPIRATION_QUERIES = {
QUERY_ANALYZE_MOZ_PLACES: {
sql: "ANALYZE moz_places",
actions: ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | ACTION.IDLE |
ACTION.DEBUG
actions: ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE |
ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
QUERY_ANALYZE_MOZ_BOOKMARKS: {
sql: "ANALYZE moz_bookmarks",
actions: ACTION.IDLE | ACTION.DEBUG
actions: ACTION.TIMED_ANALYZE | ACTION.IDLE_DAILY | ACTION.DEBUG
},
QUERY_ANALYZE_MOZ_HISTORYVISITS: {
sql: "ANALYZE moz_historyvisits",
actions: ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | ACTION.IDLE |
ACTION.DEBUG
actions: ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE |
ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
QUERY_ANALYZE_MOZ_INPUTHISTORY: {
sql: "ANALYZE moz_inputhistory",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
ACTION.IDLE | ACTION.DEBUG
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE |
ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
},
};
@ -480,6 +493,7 @@ function nsPlacesExpiration()
// Register topic observers.
Services.obs.addObserver(this, TOPIC_SHUTDOWN, false);
Services.obs.addObserver(this, TOPIC_DEBUG_START_EXPIRATION, false);
Services.obs.addObserver(this, TOPIC_IDLE_DAILY, false);
// Create our expiration timer.
this._newTimer();
@ -496,6 +510,7 @@ nsPlacesExpiration.prototype = {
this._shuttingDown = true;
Services.obs.removeObserver(this, TOPIC_SHUTDOWN);
Services.obs.removeObserver(this, TOPIC_DEBUG_START_EXPIRATION);
Services.obs.removeObserver(this, TOPIC_IDLE_DAILY);
this._prefBranch.removeObserver("", this);
@ -512,8 +527,8 @@ nsPlacesExpiration.prototype = {
Date.now() - this._lastClearHistoryTime <
SHUTDOWN_WITH_RECENT_CLEARHISTORY_TIMEOUT_SECONDS * 1000;
let action = hasRecentClearHistory ||
this.status != STATUS.DIRTY ? ACTION.CLEAN_SHUTDOWN
: ACTION.SHUTDOWN;
this.status != STATUS.DIRTY ? ACTION.SHUTDOWN_CLEAN
: ACTION.SHUTDOWN_DIRTY;
this._expireWithActionAndLimit(action, LIMIT.LARGE);
this._finalizeInternalStatements();
}
@ -558,13 +573,16 @@ nsPlacesExpiration.prototype = {
this._timer = null;
}
if (this.expireOnIdle)
this._expireWithActionAndLimit(ACTION.IDLE, LIMIT.LARGE);
this._expireWithActionAndLimit(ACTION.IDLE_DIRTY, LIMIT.LARGE);
}
else if (aTopic == TOPIC_IDLE_END) {
// Restart the expiration timer.
if (!this._timer)
this._newTimer();
}
else if (aTopic == TOPIC_IDLE_DAILY) {
this._expireWithActionAndLimit(ACTION.IDLE_DAILY, LIMIT.LARGE);
}
},
//////////////////////////////////////////////////////////////////////////////
@ -612,28 +630,18 @@ nsPlacesExpiration.prototype = {
notify: function PEX_timerCallback()
{
// Check if we are over history capacity, if so visits must be expired.
if (!this._cachedStatements["LIMIT_COUNT"]) {
this._cachedStatements["LIMIT_COUNT"] = this._db.createAsyncStatement(
"SELECT COUNT(*) FROM moz_places"
);
}
let self = this;
this._cachedStatements["LIMIT_COUNT"].executeAsync({
handleResult: function(aResults) {
let row = aResults.getNextRow();
self._overLimit = row.getResultByIndex(0) > self._urisLimit;
},
handleCompletion: function (aReason) {
if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED)
return;
let action = self._overLimit ? ACTION.TIMED_OVERLIMIT : ACTION.TIMED;
self._expireWithActionAndLimit(action, LIMIT.SMALL);
},
handleError: function(aError) {
Cu.reportError("Async statement execution returned with '" +
aError.result + "', '" + aError.message + "'");
this._getPagesStats((function onPagesCount(aPagesCount, aStatsCount) {
this._overLimit = aPagesCount > this._urisLimit;
let action = this._overLimit ? ACTION.TIMED_OVERLIMIT : ACTION.TIMED;
// If the number of pages changed significantly from the last ANALYZE
// update SQLite statistics.
if (Math.abs(aPagesCount - aStatsCount) >= ANALYZE_PAGES_THRESHOLD) {
action = action | ACTION.TIMED_ANALYZE;
}
});
this._expireWithActionAndLimit(action, LIMIT.SMALL);
}).bind(this));
},
//////////////////////////////////////////////////////////////////////////////
@ -789,6 +797,41 @@ nsPlacesExpiration.prototype = {
this._interval = PREF_INTERVAL_SECONDS_NOTSET;
},
/**
* Evaluates the real number of pages in the database and the value currently
* used by the SQLite query planner.
*
* @param aCallback
* invoked on success, function (aPagesCount, aStatsCount).
*/
_getPagesStats: function PEX__getPagesStats(aCallback) {
if (!this._cachedStatements["LIMIT_COUNT"]) {
this._cachedStatements["LIMIT_COUNT"] = this._db.createAsyncStatement(
"SELECT (SELECT COUNT(*) FROM moz_places), "
+ "(SELECT SUBSTR(stat,1,LENGTH(stat)-2) FROM sqlite_stat1 "
+ "WHERE idx = 'moz_places_url_uniqueindex')"
);
}
this._cachedStatements["LIMIT_COUNT"].executeAsync({
_pagesCount: 0,
_statsCount: 0,
handleResult: function(aResults) {
let row = aResults.getNextRow();
this._pagesCount = row.getResultByIndex(0);
this._statsCount = row.getResultByIndex(1);
},
handleCompletion: function (aReason) {
if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
aCallback(this._pagesCount, this._statsCount);
}
},
handleError: function(aError) {
Cu.reportError("Async statement execution returned with '" +
aError.result + "', '" + aError.message + "'");
}
});
},
/**
* Execute async statements to expire with the specified queries.
*
@ -806,7 +849,7 @@ nsPlacesExpiration.prototype = {
return;
// Don't try to further expire after shutdown.
if (this._shuttingDown &&
aAction != ACTION.SHUTDOWN && aAction != ACTION.CLEAN_SHUTDOWN) {
aAction != ACTION.SHUTDOWN_DIRTY && aAction != ACTION.SHUTDOWN_CLEAN) {
return;
}

21
toolkit/components/places/tests/expiration/test_idle_daily.js Normal file
View File

@ -0,0 +1,21 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
// Test that expiration runs on idle-daily.
function run_test() {
do_test_pending();
// Set interval to a large value so we don't expire on it.
setInterval(3600); // 1h
Services.obs.addObserver(function observeExpiration(aSubject, aTopic, aData) {
Services.obs.removeObserver(observeExpiration,
PlacesUtils.TOPIC_EXPIRATION_FINISHED);
do_test_finished();
}, PlacesUtils.TOPIC_EXPIRATION_FINISHED, false);
let expire = Cc["@mozilla.org/places/expiration;1"].
getService(Ci.nsIObserver);
expire.observe(null, "idle-daily", null);
}

72
toolkit/components/places/tests/expiration/test_outdated_analyze.js Normal file
View File

@ -0,0 +1,72 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
// Test that expiration executes ANALYZE when statistics are outdated.
const TEST_URL = "http://www.mozilla.org/";
XPCOMUtils.defineLazyServiceGetter(this, "gHistory",
"@mozilla.org/browser/history;1",
"mozIAsyncHistory");
/**
* Object that represents a mozIVisitInfo object.
*
* @param [optional] aTransitionType
* The transition type of the visit. Defaults to TRANSITION_LINK if not
* provided.
* @param [optional] aVisitTime
* The time of the visit. Defaults to now if not provided.
*/
function VisitInfo(aTransitionType, aVisitTime) {
this.transitionType =
aTransitionType === undefined ? TRANSITION_LINK : aTransitionType;
this.visitDate = aVisitTime || Date.now() * 1000;
}
function run_test() {
do_test_pending();
// Init expiration before "importing".
force_expiration_start();
// Add a bunch of pages (at laast IMPORT_PAGES_THRESHOLD pages).
let places = [];
for (let i = 0; i < 100; i++) {
places.push({
uri: NetUtil.newURI(TEST_URL + i),
title: "Title" + i,
visits: [new VisitInfo]
});
};
gHistory.updatePlaces(places);
// Set interval to a small value to expire on it.
setInterval(1); // 1s
Services.obs.addObserver(function observeExpiration(aSubject, aTopic, aData) {
Services.obs.removeObserver(observeExpiration,
PlacesUtils.TOPIC_EXPIRATION_FINISHED);
// Check that statistica are up-to-date.
let stmt = DBConn().createAsyncStatement(
"SELECT (SELECT COUNT(*) FROM moz_places) - "
+ "(SELECT SUBSTR(stat,1,LENGTH(stat)-2) FROM sqlite_stat1 "
+ "WHERE idx = 'moz_places_url_uniqueindex')"
);
stmt.executeAsync({
handleResult: function(aResultSet) {
let row = aResultSet.getNextRow();
this._difference = row.getResultByIndex(0);
},
handleError: function(aError) {
do_throw("Unexpected error (" + aError.result + "): " + aError.message);
},
handleCompletion: function(aReason) {
do_check_true(this._difference === 0);
do_test_finished();
}
});
stmt.finalize();
}, PlacesUtils.TOPIC_EXPIRATION_FINISHED, false);
}

4
toolkit/components/places/tests/expiration/xpcshell.ini
View File

@ -9,10 +9,12 @@ skip-if = os == "android"
[test_annos_expire_never.js]
[test_annos_expire_policy.js]
[test_annos_expire_session.js]
[test_debug_expiration.js]
[test_idle_daily.js]
[test_notifications.js]
[test_notifications_onDeleteURI.js]
[test_notifications_onDeleteVisits.js]
[test_outdated_analyze.js]
[test_pref_interval.js]
[test_pref_maxpages.js]
[test_removeAllPages.js]
[test_debug_expiration.js]