From 9fbe8f25d18234d293844a2ae0938e484cd052ca Mon Sep 17 00:00:00 2001 From: "Marco Bonardo ext:(%2C%20Shawn%20Wilsher%20%3Cme%40shawnwilsher.com%3E)" Date: Mon, 13 Sep 2010 09:29:42 -0700 Subject: [PATCH] Bug 573492 - Use WAL journaling for places.sqlite. r=sdwilsh r=mak a=blocking2.0 --- storage/public/Makefile.in | 3 + storage/public/storage.h | 2 + toolkit/components/places/src/Makefile.in | 3 + .../components/places/src/nsNavHistory.cpp | 159 +++++++++++------- toolkit/components/places/src/nsNavHistory.h | 19 +++ 5 files changed, 121 insertions(+), 65 deletions(-) diff --git a/storage/public/Makefile.in b/storage/public/Makefile.in index 7d6e01259ab6..77820230d40a 100644 --- a/storage/public/Makefile.in +++ b/storage/public/Makefile.in @@ -48,6 +48,8 @@ MODULE = storage XPIDL_MODULE = storage GRE_MODULE = 1 +# NOTE When adding something to this list, you probably need to add it to the +# storage.h file too. XPIDLSRCS = \ mozIStorageService.idl \ mozIStorageConnection.idl \ @@ -69,6 +71,7 @@ XPIDLSRCS = \ mozIStorageAsyncStatement.idl \ mozIStorageServiceQuotaManagement.idl \ $(NULL) +# SEE ABOVE NOTE! EXPORTS_NAMESPACES = mozilla diff --git a/storage/public/storage.h b/storage/public/storage.h index 2bfa3f520fc9..df6fffffb37d 100644 --- a/storage/public/storage.h +++ b/storage/public/storage.h @@ -58,6 +58,8 @@ #include "mozIStorageBindingParamsArray.h" #include "mozIStorageBindingParams.h" #include "mozIStorageServiceQuotaManagement.h" +#include "mozIStorageCompletionCallback.h" +#include "mozIStorageAsyncStatement.h" //////////////////////////////////////////////////////////////////////////////// //// Native Language Helpers diff --git a/toolkit/components/places/src/Makefile.in b/toolkit/components/places/src/Makefile.in index 7cc1047eb920..ded0ff87015f 100644 --- a/toolkit/components/places/src/Makefile.in +++ b/toolkit/components/places/src/Makefile.in @@ -82,6 +82,9 @@ EXTRA_DSO_LDOPTS += \ LOCAL_INCLUDES += -I$(srcdir)/../../build +# This is the default value. Must be in sync with the one defined in SQLite. +DEFINES += -DSQLITE_DEFAULT_PAGE_SIZE=32768 + EXTRA_COMPONENTS = \ toolkitplaces.manifest \ nsLivemarkService.js \ diff --git a/toolkit/components/places/src/nsNavHistory.cpp b/toolkit/components/places/src/nsNavHistory.cpp index 4015751525f7..d3820c0a8a3d 100644 --- a/toolkit/components/places/src/nsNavHistory.cpp +++ b/toolkit/components/places/src/nsNavHistory.cpp @@ -65,7 +65,6 @@ #include "nsThreadUtils.h" #include "nsAppDirectoryServiceDefs.h" #include "nsMathUtils.h" -#include "mozIStorageCompletionCallback.h" #include "nsNavBookmarks.h" #include "nsAnnotationService.h" @@ -142,10 +141,6 @@ using namespace mozilla::places; // Filename used to backup corrupt databases. #define DATABASE_CORRUPT_FILENAME NS_LITERAL_STRING("places.sqlite.corrupt") -// We use the TRUNCATE journal mode to reduce the number of fsyncs. Without -// this setting we had a Ts hit on Linux. See bug 460315 for details. -#define DATABASE_JOURNAL_MODE "TRUNCATE" - // Fraction of free pages in the database to force a vacuum between // DATABASE_MAX_TIME_BEFORE_VACUUM and DATABASE_MIN_TIME_BEFORE_VACUUM. #define DATABASE_VACUUM_FREEPAGES_THRESHOLD 0.1 @@ -425,6 +420,8 @@ PLACES_FACTORY_SINGLETON_IMPLEMENTATION(nsNavHistory, gHistoryService) nsNavHistory::nsNavHistory() : mBatchLevel(0) , mBatchDBTransaction(nsnull) +, mDBPageSize(0) +, mCurrentJournalMode(JOURNAL_DELETE) , mCachedNow(0) , mExpireNowTimer(nsnull) , mLastSessionID(0) @@ -669,6 +666,57 @@ nsNavHistory::InitDBFile(PRBool aForceInit) } +nsresult +nsNavHistory::SetJournalMode(enum JournalMode aJournalMode) +{ + nsCAutoString journalMode; + switch (aJournalMode) { + default: + NS_NOTREACHED("Trying to set an unknown journal mode."); + // Fall through to the default mode of DELETE. + case JOURNAL_DELETE: + journalMode.AssignLiteral("delete"); + break; + case JOURNAL_TRUNCATE: + journalMode.AssignLiteral("truncate"); + break; + case JOURNAL_MEMORY: + journalMode.AssignLiteral("memory"); + break; + case JOURNAL_WAL: + journalMode.AssignLiteral("wal"); + break; + } + + nsCOMPtr statement; + nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA journal_mode = ") + journalMode, + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + mozStorageStatementScoper scoper(statement); + PRBool hasResult; + rv = statement->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(hasResult, NS_ERROR_FAILURE); + + nsCAutoString currentJournalMode; + rv = statement->GetUTF8String(0, currentJournalMode); + NS_ENSURE_SUCCESS(rv, rv); + bool succeeded = currentJournalMode.Equals(journalMode); + if (succeeded) { + mCurrentJournalMode = aJournalMode; + } + else { + NS_WARNING(nsPrintfCString(128, "Setting journal mode failed: %s", + PromiseFlatCString(journalMode).get()).get()); + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + + nsresult nsNavHistory::InitDB() { @@ -676,19 +724,22 @@ nsNavHistory::InitDB() PRInt32 currentSchemaVersion = 0; nsresult rv = mDBConn->GetSchemaVersion(¤tSchemaVersion); NS_ENSURE_SUCCESS(rv, rv); + { + // Get the page size. This may be different than the default if the + // database file already existed with a different page size. + nsCOMPtr statement; + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); - // Get the page size. This may be different than the default if the - // database file already existed with a different page size. - nsCOMPtr statement; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"), - getter_AddRefs(statement)); - NS_ENSURE_SUCCESS(rv, rv); - - PRBool hasResult; - mozStorageStatementScoper scoper(statement); - rv = statement->ExecuteStep(&hasResult); - NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE); - PRInt32 pageSize = statement->AsInt32(0); + PRBool hasResult; + mozStorageStatementScoper scoper(statement); + rv = statement->ExecuteStep(&hasResult); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE); + rv = statement->GetInt32(0, &mDBPageSize); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(mDBPageSize > 0, NS_ERROR_UNEXPECTED); + } // Ensure that temp tables are held in memory, not on disk. We use temp // tables mainly for fsync and I/O reduction. @@ -720,10 +771,10 @@ nsNavHistory::InitDB() PRInt64 cacheSize = physMem * cachePercentage / 100; // Compute number of cached pages, this will be our cache size. - PRInt64 cachePages = cacheSize / pageSize; - nsCAutoString pageSizePragma("PRAGMA cache_size = "); - pageSizePragma.AppendInt(cachePages); - rv = mDBConn->ExecuteSimpleSQL(pageSizePragma); + PRInt64 cachePages = cacheSize / mDBPageSize; + nsCAutoString cacheSizePragma("PRAGMA cache_size = "); + cacheSizePragma.AppendInt(cachePages); + rv = mDBConn->ExecuteSimpleSQL(cacheSizePragma); NS_ENSURE_SUCCESS(rv, rv); // Lock the database file. This is done partly to avoid third party @@ -732,9 +783,14 @@ nsNavHistory::InitDB() "PRAGMA locking_mode = EXCLUSIVE")); NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE)); - NS_ENSURE_SUCCESS(rv, rv); + // Be sure to set journal mode after page_size. WAL would prevent the change + // otherwise. + if (NS_FAILED(SetJournalMode(JOURNAL_WAL))) { + // Ignore errors, if we fail here the database could be considered corrupt + // and we won't be able to go on, even if it's just matter of a bogus file + // system. The default mode (DELETE) will be fine in such a case. + (void)SetJournalMode(JOURNAL_TRUNCATE); + } // We are going to initialize tables, so everything from now on should be in // a transaction for performances. @@ -1828,25 +1884,12 @@ nsNavHistory::MigrateV9Up(mozIStorageConnection *aDBConn) // This query can be really slow due to disk access, since it will basically // dupe the table contents in the journal file, and then write them down // in the database. - // We will temporary use a memory journal file, this has the advantage of - // reducing write times by a half, but will temporary consume more memory - // and increase risks of corruption if we should crash in the middle of this - // update. - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = MEMORY")); - NS_ENSURE_SUCCESS(rv, rv); - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "UPDATE moz_places SET last_visit_date = " "(SELECT MAX(visit_date) " "FROM moz_historyvisits " "WHERE place_id = moz_places.id)")); NS_ENSURE_SUCCESS(rv, rv); - - // Restore the default journal mode. - rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE)); - NS_ENSURE_SUCCESS(rv, rv); } return transaction.Commit(); @@ -5928,39 +5971,25 @@ nsNavHistory::VacuumDatabase() nsnull); } - // Actually vacuuming a database is a slow operation, since it could take - // seconds. Part of the time is spent in updating the journal file on disk - // and this is particularly bad on devices with slow I/O. Temporary - // moving the journal to memory could increase a bit the possibility of - // corruption if we crash during this time, but makes the process really - // faster. - nsCOMPtr journalToMemory; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = MEMORY"), - getter_AddRefs(journalToMemory)); - NS_ENSURE_SUCCESS(rv, rv); + // If journal mode is WAL, a VACUUM cannot upgrade page_size value. + // If current page_size is not the expected one, journal mode must be + // changed to a rollback one. Once done we won't be able to go back to WAL + // mode though, since non-reset statements exist. Just keep using + // compatible mode till next restart. + // See http://www.sqlite.org/wal.html + if (mCurrentJournalMode == JOURNAL_WAL && + mDBPageSize != SQLITE_DEFAULT_PAGE_SIZE) { + (void)SetJournalMode(JOURNAL_TRUNCATE); + } - nsCOMPtr vacuum; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("VACUUM"), - getter_AddRefs(vacuum)); + nsCOMPtr vacuum; + rv = mDBConn->CreateAsyncStatement(NS_LITERAL_CSTRING("VACUUM"), + getter_AddRefs(vacuum)); NS_ENSURE_SUCCESS(rv, rv); - - nsCOMPtr journalToDefault; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE), - getter_AddRefs(journalToDefault)); - NS_ENSURE_SUCCESS(rv, rv); - - mozIStorageBaseStatement *stmts[] = { - journalToMemory, - vacuum, - journalToDefault - }; nsCOMPtr ps; nsRefPtr vacuumDBListener = new VacuumDBListener(mPrefBranch); - rv = mDBConn->ExecuteAsync(stmts, NS_ARRAY_LENGTH(stmts), - vacuumDBListener, getter_AddRefs(ps)); + rv = vacuum->ExecuteAsync(vacuumDBListener, getter_AddRefs(ps)); NS_ENSURE_SUCCESS(rv, rv); } diff --git a/toolkit/components/places/src/nsNavHistory.h b/toolkit/components/places/src/nsNavHistory.h index 95f127593849..067cdd734f96 100644 --- a/toolkit/components/places/src/nsNavHistory.h +++ b/toolkit/components/places/src/nsNavHistory.h @@ -132,6 +132,18 @@ namespace places { , DB_SET_PLACE_TITLE = 9 }; + enum JournalMode { + // Default SQLite journal mode. + JOURNAL_DELETE = 0 + // Can reduce fsyncs on Linux when journal is deleted (See bug 460315). + // We fallback to this mode when WAL is unavailable. + , JOURNAL_TRUNCATE + // Unsafe in case of crashes on database swap or low memory. + , JOURNAL_MEMORY + // Can reduce number of fsyncs. We try to use this mode by default. + , JOURNAL_WAL + }; + } // namespace places } // namespace mozilla @@ -479,6 +491,7 @@ protected: nsCOMPtr mDBService; nsCOMPtr mDBConn; nsCOMPtr mDBFile; + PRInt32 mDBPageSize; nsCOMPtr mDBGetURLPageInfo; // kGetInfoIndex_* results nsCOMPtr mDBGetIdPageInfo; // kGetInfoIndex_* results @@ -546,6 +559,12 @@ protected: */ nsresult InitDBFile(PRBool aForceInit); + /** + * Set journal mode on the database. + */ + nsresult SetJournalMode(enum mozilla::places::JournalMode aJournalMode); + enum mozilla::places::JournalMode mCurrentJournalMode; + /** * Initializes the database. This performs any necessary migrations for the * database. All migration is done inside a transaction that is rolled back