diff --git a/toolkit/components/url-classifier/Classifier.cpp b/toolkit/components/url-classifier/Classifier.cpp index dcab72c0c3e4..46b5f922e3ab 100644 --- a/toolkit/components/url-classifier/Classifier.cpp +++ b/toolkit/components/url-classifier/Classifier.cpp @@ -34,6 +34,7 @@ extern mozilla::LazyLogModule gUrlClassifierDbServiceLog; #define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing") #define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete") #define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup") +#define UPDATING_DIR_SUFFIX NS_LITERAL_CSTRING("-updating") #define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata") @@ -180,6 +181,13 @@ Classifier::SetupPathNames() rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX); NS_ENSURE_SUCCESS(rv, rv); + // Directory where to be working on the update. + rv = mCacheDirectory->Clone(getter_AddRefs(mUpdatingDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mUpdatingDirectory->AppendNative(STORE_DIRECTORY + UPDATING_DIR_SUFFIX); + NS_ENSURE_SUCCESS(rv, rv); + // Directory where to move the backup so we can atomically // delete (really move) it. rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory)); @@ -225,9 +233,19 @@ Classifier::Open(nsIFile& aCacheDirectory) NS_ENSURE_SUCCESS(rv, rv); // Clean up any to-delete directories that haven't been deleted yet. + // This is still required for backward compatibility. rv = CleanToDelete(); NS_ENSURE_SUCCESS(rv, rv); + // If we met a crash during the previous update, "safebrowsing-updating" + // directory will exist and let's remove it. + rv = mUpdatingDirectory->Remove(true); + if (NS_SUCCEEDED(rv)) { + // If the "safebrowsing-updating" exists, it implies a crash occurred + // in the previous update. + LOG(("We may have hit a crash in the previous update.")); + } + // Check whether we have an incomplete update and recover from the // backup if so. rv = RecoverBackups(); @@ -260,6 +278,7 @@ Classifier::Reset() mRootStoreDirectory->Remove(true); mBackupDirectory->Remove(true); + mUpdatingDirectory->Remove(true); mToDeleteDirectory->Remove(true); CreateStoreDirectory(); @@ -347,11 +366,6 @@ Classifier::AbortUpdateAndReset(const nsCString& aTable) // ResetTables will clear both in-memory & on-disk data. ResetTables(Clear_All, nsTArray { aTable }); - - // Remove the backup and delete directory since we are aborting - // from an update. - Unused << RemoveBackupTables(); - Unused << CleanToDelete(); } void @@ -527,9 +541,144 @@ Classifier::Check(const nsACString& aSpec, return NS_OK; } +static nsresult +SwapDirectoryContent(nsIFile* aDir1, + nsIFile* aDir2, + nsIFile* aParentDir, + nsIFile* aTempDir) +{ + // Pre-condition: |aDir1| and |aDir2| are directory and their parent + // are both |aParentDir|. + // + // Post-condition: The locations where aDir1 and aDir2 point to will not + // change but their contents will be exchanged. If we failed + // to swap their content, everything will be rolled back. + + nsAutoCString tempDirName; + aTempDir->GetNativeLeafName(tempDirName); + + nsresult rv; + + nsAutoCString dirName1, dirName2; + aDir1->GetNativeLeafName(dirName1); + aDir2->GetNativeLeafName(dirName2); + + LOG(("Swapping directories %s and %s...", dirName1.get(), + dirName2.get())); + + // 1. Rename "dirName1" to "temp" + rv = aDir1->RenameToNative(nullptr, tempDirName); + if (NS_FAILED(rv)) { + LOG(("Unable to rename %s to %s", dirName1.get(), + tempDirName.get())); + return rv; // Nothing to roll back. + } + + // 1.1. Create a handle for temp directory. This is required since + // |nsIFile.rename| will not change the location where the + // object points to. + nsCOMPtr tempDirectory; + rv = aParentDir->Clone(getter_AddRefs(tempDirectory)); + rv = tempDirectory->AppendNative(tempDirName); + + // 2. Rename "dirName2" to "dirName1". + rv = aDir2->RenameToNative(nullptr, dirName1); + if (NS_FAILED(rv)) { + LOG(("Failed to rename %s to %s. Rename temp directory back to %s", + dirName2.get(), dirName1.get(), dirName1.get())); + nsresult rbrv = tempDirectory->RenameToNative(nullptr, dirName1); + NS_ENSURE_SUCCESS(rbrv, rbrv); + return rv; + } + + // 3. Rename "temp" to "dirName2". + rv = tempDirectory->RenameToNative(nullptr, dirName2); + if (NS_FAILED(rv)) { + LOG(("Failed to rename temp directory to %s. ", dirName2.get())); + // We've done (1) renaming "dir1 to temp" and + // (2) renaming "dir2 to dir1" + // so the rollback is + // (1) renaming "dir1 to dir2" and + // (2) renaming "temp to dir1" + nsresult rbrv; // rollback result + rbrv = aDir1->RenameToNative(nullptr, dirName2); + NS_ENSURE_SUCCESS(rbrv, rbrv); + rbrv = tempDirectory->RenameToNative(nullptr, dirName1); + NS_ENSURE_SUCCESS(rbrv, rbrv); + return rv; + } + + return rv; +} + +void +Classifier::RemoveUpdateIntermediaries() +{ + // Remove old LookupCaches. + for (auto c: mNewLookupCaches) { + delete c; + } + mNewLookupCaches.Clear(); + + // Remove the "old" directory. (despite its looking-new name) + if (NS_FAILED(mUpdatingDirectory->Remove(true))) { + // If the directory is locked from removal for some reason, + // we will fail here and it doesn't matter until the next + // update. (the next udpate will fail due to the removable + // "safebrowsing-udpating" directory.) + LOG(("Failed to remove updating directory.")); + } +} + +nsresult +Classifier::SwapInNewTablesAndCleanup() +{ + nsresult rv; + + // Step 1. Swap in on-disk tables. The idea of using "safebrowsing-backup" + // as the intermediary directory is we can get databases recovered if + // crash occurred in any step of the swap. (We will recover from + // "safebrowsing-backup" in OpenDb().) + rv = SwapDirectoryContent(mUpdatingDirectory, // contains new tables + mRootStoreDirectory, // contains old tables + mCacheDirectory, // common parent dir + mBackupDirectory); // intermediary dir for swap + if (NS_FAILED(rv)) { + LOG(("Failed to swap in on-disk tables.")); + RemoveUpdateIntermediaries(); + return rv; + } + + // Step 2. Swap in in-memory tables and update root directroy handles. + mLookupCaches.SwapElements(mNewLookupCaches); + for (auto c: mLookupCaches) { + c->UpdateRootDirHandle(mRootStoreDirectory); + } + + // Step 3. Re-generate active tables based on on-disk tables. + rv = RegenActiveTables(); + if (NS_FAILED(rv)) { + LOG(("Failed to re-generate active tables!")); + } + + // Step 4. Clean up intermediaries for update. + RemoveUpdateIntermediaries(); + + // Step 5. Invalidate cached tableRequest request. + mIsTableRequestResultOutdated = true; + + LOG(("Done swap in updated tables.")); + + return rv; +} + nsresult Classifier::ApplyUpdates(nsTArray* aUpdates) { + // Will run on the update thread after Bug 1339760. + // No lock is required except for CopyInUseDirForUpdate() since + // the table lookup may lead to database removal. + if (!aUpdates || aUpdates->Length() == 0) { return NS_OK; } @@ -554,10 +703,18 @@ Classifier::ApplyUpdates(nsTArray* aUpdates) { ScopedUpdatesClearer scopedUpdatesClearer(aUpdates); - LOG(("Backup before update.")); - - rv = BackupTables(); - NS_ENSURE_SUCCESS(rv, rv); + { + // TODO: Bug 1339050. This section should be mutual exclusive to + // LookupCache::Open() and HashStore::Open() since those operations + // might fail and cause the database to be removed. It'll be fine + // until we move "apply update" code off the worker thread. + rv = CopyInUseDirForUpdate(); // i.e. mUpdatingDirectory will be setup. + if (NS_FAILED(rv)) { + LOG(("Failed to copy in-use directory for update.")); + return rv; + } + CopyInUseLookupCacheForUpdate(); // i.e. mNewLookupCaches will be setup. + } LOG(("Applying %" PRIuSIZE " table updates.", aUpdates->Length())); @@ -567,23 +724,22 @@ Classifier::ApplyUpdates(nsTArray* aUpdates) // Run all updates for one table nsCString updateTable(aUpdates->ElementAt(i)->TableName()); + // Will update the mirrored in-memory and on-disk databases. if (TableUpdate::Cast((*aUpdates)[i])) { rv = UpdateHashStore(aUpdates, updateTable); } else { rv = UpdateTableV4(aUpdates, updateTable); } - // We mark the table associated info outdated no matter the - // update is successful to avoid any possibile non-atomic update. - mIsTableRequestResultOutdated = true; - if (NS_FAILED(rv)) { if (rv != NS_ERROR_OUT_OF_MEMORY) { #ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES DumpFailedUpdate(); #endif + // TODO: Bug 1339050 - Should be run on worker thread. AbortUpdateAndReset(updateTable); } + RemoveUpdateIntermediaries(); return rv; } } @@ -591,21 +747,10 @@ Classifier::ApplyUpdates(nsTArray* aUpdates) } // End of scopedUpdatesClearer scope. - rv = RegenActiveTables(); - NS_ENSURE_SUCCESS(rv, rv); - - LOG(("Cleaning up backups.")); - - // Move the backup directory away (signaling the transaction finished - // successfully). This is atomic. - rv = RemoveBackupTables(); - NS_ENSURE_SUCCESS(rv, rv); - - // Do the actual deletion of the backup files. - rv = CleanToDelete(); - NS_ENSURE_SUCCESS(rv, rv); - - LOG(("Done applying updates.")); + rv = SwapInNewTablesAndCleanup(); + if (NS_FAILED(rv)) { + LOG(("Failed to swap in new tables.")); + } if (LOG_ENABLED()) { PRIntervalTime clockEnd = PR_IntervalNow(); @@ -613,7 +758,7 @@ Classifier::ApplyUpdates(nsTArray* aUpdates) PR_IntervalToMilliseconds(clockEnd - clockStart))); } - return NS_OK; + return rv; } nsresult @@ -815,11 +960,11 @@ Classifier::DumpFailedUpdate() nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName); NS_ENSURE_SUCCESS(rv, rv); - // Move backup to a clean "failed update" directory. - nsCOMPtr backupDirectory; - if (NS_FAILED(mBackupDirectory->Clone(getter_AddRefs(backupDirectory))) || - NS_FAILED(backupDirectory->MoveToNative(nullptr, failedUpdatekDirName))) { - LOG(("Failed to move backup to the \"failed update\" directory %s", + // Copy the in-use directory to a clean "failed update" directory. + nsCOMPtr inUseDirectory; + if (NS_FAILED(mRootStoreDirectory->Clone(getter_AddRefs(inUseDirectory))) || + NS_FAILED(inUseDirectory->CopyToNative(nullptr, failedUpdatekDirName))) { + LOG(("Failed to move in-use to the \"failed update\" directory %s", failedUpdatekDirName.get())); return NS_ERROR_FAILURE; } @@ -830,26 +975,20 @@ Classifier::DumpFailedUpdate() #endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES nsresult -Classifier::BackupTables() +Classifier::CopyInUseDirForUpdate() { - // We have to work in reverse here: first move the normal directory - // away to be the backup directory, then copy the files over - // to the normal directory. This ensures that if we crash the backup - // dir always has a valid, complete copy, instead of a partial one, - // because that's the one we will copy over the normal store dir. + LOG(("Copy in-use directory content for update.")); - nsCString backupDirName; - nsresult rv = mBackupDirectory->GetNativeLeafName(backupDirName); + // We copy everything from in-use directory to a temporary directory + // for updating. + + nsCString updatingDirName; + nsresult rv = mUpdatingDirectory->GetNativeLeafName(updatingDirName); NS_ENSURE_SUCCESS(rv, rv); - nsCString storeDirName; - rv = mRootStoreDirectory->GetNativeLeafName(storeDirName); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mRootStoreDirectory->MoveToNative(nullptr, backupDirName); - NS_ENSURE_SUCCESS(rv, rv); - - rv = mRootStoreDirectory->CopyToNative(nullptr, storeDirName); + // Remove the destination directory first (just in case) the do the copy. + mUpdatingDirectory->Remove(true); + rv = mRootStoreDirectory->CopyToNative(nullptr, updatingDirName); NS_ENSURE_SUCCESS(rv, rv); // We moved some things to new places, so move the handles around, too. @@ -859,21 +998,19 @@ Classifier::BackupTables() return NS_OK; } -nsresult -Classifier::RemoveBackupTables() +void +Classifier::CopyInUseLookupCacheForUpdate() { - nsCString toDeleteName; - nsresult rv = mToDeleteDirectory->GetNativeLeafName(toDeleteName); - NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(mNewLookupCaches.IsEmpty(), "Update intermediaries is forgotten to " + "be removed in the previous update."); - rv = mBackupDirectory->MoveToNative(nullptr, toDeleteName); - NS_ENSURE_SUCCESS(rv, rv); - - // mBackupDirectory now points to toDelete, fix that up. - rv = SetupPathNames(); - NS_ENSURE_SUCCESS(rv, rv); - - return NS_OK; + // TODO: Bug 1341183 - Implement deep copy function or copy ctor. + // For now we just re-open every table we have opened. + // Another option is to NOT pre-open these LookupCaches and + // do a "merge" instead of "swap" after update. + for (auto c: mLookupCaches) { + Unused << GetLookupCacheForUpdate(c->TableName()); + } } nsresult @@ -962,7 +1099,7 @@ Classifier::UpdateHashStore(nsTArray* aUpdates, LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get())); - HashStore store(aTable, GetProvider(aTable), mRootStoreDirectory); + HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory); if (!CheckValidUpdate(aUpdates, store.TableName())) { return NS_OK; @@ -975,7 +1112,7 @@ Classifier::UpdateHashStore(nsTArray* aUpdates, // Read the part of the store that is (only) in the cache LookupCacheV2* lookupCache = - LookupCache::Cast(GetLookupCache(store.TableName())); + LookupCache::Cast(GetLookupCacheForUpdate(store.TableName())); if (!lookupCache) { return NS_ERROR_FAILURE; } @@ -1069,7 +1206,7 @@ Classifier::UpdateTableV4(nsTArray* aUpdates, } LookupCacheV4* lookupCache = - LookupCache::Cast(GetLookupCache(aTable)); + LookupCache::Cast(GetLookupCacheForUpdate(aTable)); if (!lookupCache) { return NS_ERROR_FAILURE; } @@ -1175,11 +1312,22 @@ Classifier::UpdateCache(TableUpdate* aUpdate) } LookupCache * -Classifier::GetLookupCache(const nsACString& aTable) +Classifier::GetLookupCache(const nsACString& aTable, bool aForUpdate) { - for (uint32_t i = 0; i < mLookupCaches.Length(); i++) { - if (mLookupCaches[i]->TableName().Equals(aTable)) { - return mLookupCaches[i]; + if (aForUpdate) { + return GetLookupCacheFrom(aTable, mNewLookupCaches, mUpdatingDirectory); + } + return GetLookupCacheFrom(aTable, mLookupCaches, mRootStoreDirectory); +} + +LookupCache * +Classifier::GetLookupCacheFrom(const nsACString& aTable, + nsTArray& aLookupCaches, + nsIFile* aRootStoreDirectory) +{ + for (uint32_t i = 0; i < aLookupCaches.Length(); i++) { + if (aLookupCaches[i]->TableName().Equals(aTable)) { + return aLookupCaches[i]; } } @@ -1189,9 +1337,9 @@ Classifier::GetLookupCache(const nsACString& aTable) UniquePtr cache; nsCString provider = GetProvider(aTable); if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) { - cache = MakeUnique(aTable, provider, mRootStoreDirectory); + cache = MakeUnique(aTable, provider, aRootStoreDirectory); } else { - cache = MakeUnique(aTable, provider, mRootStoreDirectory); + cache = MakeUnique(aTable, provider, aRootStoreDirectory); } nsresult rv = cache->Init(); @@ -1205,7 +1353,7 @@ Classifier::GetLookupCache(const nsACString& aTable) } return nullptr; } - mLookupCaches.AppendElement(cache.get()); + aLookupCaches.AppendElement(cache.get()); return cache.release(); } diff --git a/toolkit/components/url-classifier/Classifier.h b/toolkit/components/url-classifier/Classifier.h index 3bcda1923b21..c6fae989930f 100644 --- a/toolkit/components/url-classifier/Classifier.h +++ b/toolkit/components/url-classifier/Classifier.h @@ -109,6 +109,10 @@ public: const nsACString& aProvider, nsIFile** aPrivateStoreDirectory); + // Swap in in-memory and on-disk database and remove all + // update intermediaries. + nsresult SwapInNewTablesAndCleanup(); + private: void DropStores(); void DeleteTables(nsIFile* aDirectory, const nsTArray& aTables); @@ -118,10 +122,16 @@ private: nsresult SetupPathNames(); nsresult RecoverBackups(); nsresult CleanToDelete(); - nsresult BackupTables(); - nsresult RemoveBackupTables(); + nsresult CopyInUseDirForUpdate(); + void CopyInUseLookupCacheForUpdate(); nsresult RegenActiveTables(); + + + // Remove any intermediary for update, including in-memory + // and on-disk data. + void RemoveUpdateIntermediaries(); + #ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES already_AddRefed GetFailedUpdateDirectroy(); nsresult DumpFailedUpdate(); @@ -137,7 +147,17 @@ private: nsresult UpdateCache(TableUpdate* aUpdates); - LookupCache *GetLookupCache(const nsACString& aTable); + LookupCache *GetLookupCache(const nsACString& aTable, + bool aForUpdate = false); + + LookupCache *GetLookupCacheForUpdate(const nsACString& aTable) { + return GetLookupCache(aTable, true); + } + + LookupCache *GetLookupCacheFrom(const nsACString& aTable, + nsTArray& aLookupCaches, + nsIFile* aRootStoreDirectory); + bool CheckValidUpdate(nsTArray* aUpdates, const nsACString& aTable); @@ -152,9 +172,10 @@ private: nsCOMPtr mRootStoreDirectory; // Used for atomically updating the other dirs. nsCOMPtr mBackupDirectory; + nsCOMPtr mUpdatingDirectory; // For update only. nsCOMPtr mToDeleteDirectory; nsCOMPtr mCryptoHash; - nsTArray mLookupCaches; + nsTArray mLookupCaches; // For query only. nsTArray mActiveTablesCache; uint32_t mHashKey; // Stores the last time a given table was updated (seconds). @@ -167,6 +188,9 @@ private: // Whether mTableRequestResult is outdated and needs to // be reloaded from disk. bool mIsTableRequestResultOutdated; + + // The copy of mLookupCaches for update only. + nsTArray mNewLookupCaches; }; } // namespace safebrowsing