Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-26 14:22:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1338970 - Introduce 2-step DB update scheme. (background update then swap in) r=francois,gcp

Browse Source 
MozReview-Commit-ID: 4C9N0tuSso0

--HG--
extra : rebase_source : 80561fb0beec7fc9f6e852d81984b239ae62700e
This commit is contained in:
Henry Chang 2017-02-21 14:18:29 +08:00
parent aaa6c009ca
commit 2b62e1dabf
2 changed files with 249 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

294
toolkit/components/url-classifier/Classifier.cpp
View File

@ -34,6 +34,7 @@ extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
#define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing")
#define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete")
#define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup")
#define UPDATING_DIR_SUFFIX NS_LITERAL_CSTRING("-updating")
#define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata")
@ -180,6 +181,13 @@ Classifier::SetupPathNames()
rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX);
NS_ENSURE_SUCCESS(rv, rv);
// Directory where to be working on the update.
rv = mCacheDirectory->Clone(getter_AddRefs(mUpdatingDirectory));
NS_ENSURE_SUCCESS(rv, rv);
rv = mUpdatingDirectory->AppendNative(STORE_DIRECTORY + UPDATING_DIR_SUFFIX);
NS_ENSURE_SUCCESS(rv, rv);
// Directory where to move the backup so we can atomically
// delete (really move) it.
rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory));
@ -225,9 +233,19 @@ Classifier::Open(nsIFile& aCacheDirectory)
NS_ENSURE_SUCCESS(rv, rv);
// Clean up any to-delete directories that haven't been deleted yet.
// This is still required for backward compatibility.
rv = CleanToDelete();
NS_ENSURE_SUCCESS(rv, rv);
// If we met a crash during the previous update, "safebrowsing-updating"
// directory will exist and let's remove it.
rv = mUpdatingDirectory->Remove(true);
if (NS_SUCCEEDED(rv)) {
// If the "safebrowsing-updating" exists, it implies a crash occurred
// in the previous update.
LOG(("We may have hit a crash in the previous update."));
}
// Check whether we have an incomplete update and recover from the
// backup if so.
rv = RecoverBackups();
@ -260,6 +278,7 @@ Classifier::Reset()
mRootStoreDirectory->Remove(true);
mBackupDirectory->Remove(true);
mUpdatingDirectory->Remove(true);
mToDeleteDirectory->Remove(true);
CreateStoreDirectory();
@ -347,11 +366,6 @@ Classifier::AbortUpdateAndReset(const nsCString& aTable)
// ResetTables will clear both in-memory & on-disk data.
ResetTables(Clear_All, nsTArray<nsCString> { aTable });
// Remove the backup and delete directory since we are aborting
// from an update.
Unused << RemoveBackupTables();
Unused << CleanToDelete();
}
void
@ -527,9 +541,144 @@ Classifier::Check(const nsACString& aSpec,
return NS_OK;
}
static nsresult
SwapDirectoryContent(nsIFile* aDir1,
nsIFile* aDir2,
nsIFile* aParentDir,
nsIFile* aTempDir)
{
// Pre-condition: |aDir1| and |aDir2| are directory and their parent
// are both |aParentDir|.
//
// Post-condition: The locations where aDir1 and aDir2 point to will not
// change but their contents will be exchanged. If we failed
// to swap their content, everything will be rolled back.
nsAutoCString tempDirName;
aTempDir->GetNativeLeafName(tempDirName);
nsresult rv;
nsAutoCString dirName1, dirName2;
aDir1->GetNativeLeafName(dirName1);
aDir2->GetNativeLeafName(dirName2);
LOG(("Swapping directories %s and %s...", dirName1.get(),
dirName2.get()));
// 1. Rename "dirName1" to "temp"
rv = aDir1->RenameToNative(nullptr, tempDirName);
if (NS_FAILED(rv)) {
LOG(("Unable to rename %s to %s", dirName1.get(),
tempDirName.get()));
return rv; // Nothing to roll back.
}
// 1.1. Create a handle for temp directory. This is required since
// |nsIFile.rename| will not change the location where the
// object points to.
nsCOMPtr<nsIFile> tempDirectory;
rv = aParentDir->Clone(getter_AddRefs(tempDirectory));
rv = tempDirectory->AppendNative(tempDirName);
// 2. Rename "dirName2" to "dirName1".
rv = aDir2->RenameToNative(nullptr, dirName1);
if (NS_FAILED(rv)) {
LOG(("Failed to rename %s to %s. Rename temp directory back to %s",
dirName2.get(), dirName1.get(), dirName1.get()));
nsresult rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
NS_ENSURE_SUCCESS(rbrv, rbrv);
return rv;
}
// 3. Rename "temp" to "dirName2".
rv = tempDirectory->RenameToNative(nullptr, dirName2);
if (NS_FAILED(rv)) {
LOG(("Failed to rename temp directory to %s. ", dirName2.get()));
// We've done (1) renaming "dir1 to temp" and
// (2) renaming "dir2 to dir1"
// so the rollback is
// (1) renaming "dir1 to dir2" and
// (2) renaming "temp to dir1"
nsresult rbrv; // rollback result
rbrv = aDir1->RenameToNative(nullptr, dirName2);
NS_ENSURE_SUCCESS(rbrv, rbrv);
rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
NS_ENSURE_SUCCESS(rbrv, rbrv);
return rv;
}
return rv;
}
void
Classifier::RemoveUpdateIntermediaries()
{
// Remove old LookupCaches.
for (auto c: mNewLookupCaches) {
delete c;
}
mNewLookupCaches.Clear();
// Remove the "old" directory. (despite its looking-new name)
if (NS_FAILED(mUpdatingDirectory->Remove(true))) {
// If the directory is locked from removal for some reason,
// we will fail here and it doesn't matter until the next
// update. (the next udpate will fail due to the removable
// "safebrowsing-udpating" directory.)
LOG(("Failed to remove updating directory."));
}
}
nsresult
Classifier::SwapInNewTablesAndCleanup()
{
nsresult rv;
// Step 1. Swap in on-disk tables. The idea of using "safebrowsing-backup"
// as the intermediary directory is we can get databases recovered if
// crash occurred in any step of the swap. (We will recover from
// "safebrowsing-backup" in OpenDb().)
rv = SwapDirectoryContent(mUpdatingDirectory, // contains new tables
mRootStoreDirectory, // contains old tables
mCacheDirectory, // common parent dir
mBackupDirectory); // intermediary dir for swap
if (NS_FAILED(rv)) {
LOG(("Failed to swap in on-disk tables."));
RemoveUpdateIntermediaries();
return rv;
}
// Step 2. Swap in in-memory tables and update root directroy handles.
mLookupCaches.SwapElements(mNewLookupCaches);
for (auto c: mLookupCaches) {
c->UpdateRootDirHandle(mRootStoreDirectory);
}
// Step 3. Re-generate active tables based on on-disk tables.
rv = RegenActiveTables();
if (NS_FAILED(rv)) {
LOG(("Failed to re-generate active tables!"));
}
// Step 4. Clean up intermediaries for update.
RemoveUpdateIntermediaries();
// Step 5. Invalidate cached tableRequest request.
mIsTableRequestResultOutdated = true;
LOG(("Done swap in updated tables."));
return rv;
}
nsresult
Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
{
// Will run on the update thread after Bug 1339760.
// No lock is required except for CopyInUseDirForUpdate() since
// the table lookup may lead to database removal.
if (!aUpdates || aUpdates->Length() == 0) {
return NS_OK;
}
@ -554,10 +703,18 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
{
ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
LOG(("Backup before update."));
rv = BackupTables();
NS_ENSURE_SUCCESS(rv, rv);
{
// TODO: Bug 1339050. This section should be mutual exclusive to
// LookupCache::Open() and HashStore::Open() since those operations
// might fail and cause the database to be removed. It'll be fine
// until we move "apply update" code off the worker thread.
rv = CopyInUseDirForUpdate(); // i.e. mUpdatingDirectory will be setup.
if (NS_FAILED(rv)) {
LOG(("Failed to copy in-use directory for update."));
return rv;
}
CopyInUseLookupCacheForUpdate(); // i.e. mNewLookupCaches will be setup.
}
LOG(("Applying %" PRIuSIZE " table updates.", aUpdates->Length()));
@ -567,23 +724,22 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
// Run all updates for one table
nsCString updateTable(aUpdates->ElementAt(i)->TableName());
// Will update the mirrored in-memory and on-disk databases.
if (TableUpdate::Cast<TableUpdateV2>((*aUpdates)[i])) {
rv = UpdateHashStore(aUpdates, updateTable);
} else {
rv = UpdateTableV4(aUpdates, updateTable);
}
// We mark the table associated info outdated no matter the
// update is successful to avoid any possibile non-atomic update.
mIsTableRequestResultOutdated = true;
if (NS_FAILED(rv)) {
if (rv != NS_ERROR_OUT_OF_MEMORY) {
#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
DumpFailedUpdate();
#endif
// TODO: Bug 1339050 - Should be run on worker thread.
AbortUpdateAndReset(updateTable);
}
RemoveUpdateIntermediaries();
return rv;
}
}
@ -591,21 +747,10 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
} // End of scopedUpdatesClearer scope.
rv = RegenActiveTables();
NS_ENSURE_SUCCESS(rv, rv);
LOG(("Cleaning up backups."));
// Move the backup directory away (signaling the transaction finished
// successfully). This is atomic.
rv = RemoveBackupTables();
NS_ENSURE_SUCCESS(rv, rv);
// Do the actual deletion of the backup files.
rv = CleanToDelete();
NS_ENSURE_SUCCESS(rv, rv);
LOG(("Done applying updates."));
rv = SwapInNewTablesAndCleanup();
if (NS_FAILED(rv)) {
LOG(("Failed to swap in new tables."));
}
if (LOG_ENABLED()) {
PRIntervalTime clockEnd = PR_IntervalNow();
@ -613,7 +758,7 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
PR_IntervalToMilliseconds(clockEnd - clockStart)));
}
return NS_OK;
return rv;
}
nsresult
@ -815,11 +960,11 @@ Classifier::DumpFailedUpdate()
nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName);
NS_ENSURE_SUCCESS(rv, rv);
// Move backup to a clean "failed update" directory.
nsCOMPtr<nsIFile> backupDirectory;
if (NS_FAILED(mBackupDirectory->Clone(getter_AddRefs(backupDirectory))) ||
NS_FAILED(backupDirectory->MoveToNative(nullptr, failedUpdatekDirName))) {
LOG(("Failed to move backup to the \"failed update\" directory %s",
// Copy the in-use directory to a clean "failed update" directory.
nsCOMPtr<nsIFile> inUseDirectory;
if (NS_FAILED(mRootStoreDirectory->Clone(getter_AddRefs(inUseDirectory))) ||
NS_FAILED(inUseDirectory->CopyToNative(nullptr, failedUpdatekDirName))) {
LOG(("Failed to move in-use to the \"failed update\" directory %s",
failedUpdatekDirName.get()));
return NS_ERROR_FAILURE;
}
@ -830,26 +975,20 @@ Classifier::DumpFailedUpdate()
#endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
nsresult
Classifier::BackupTables()
Classifier::CopyInUseDirForUpdate()
{
// We have to work in reverse here: first move the normal directory
// away to be the backup directory, then copy the files over
// to the normal directory. This ensures that if we crash the backup
// dir always has a valid, complete copy, instead of a partial one,
// because that's the one we will copy over the normal store dir.
LOG(("Copy in-use directory content for update."));
nsCString backupDirName;
nsresult rv = mBackupDirectory->GetNativeLeafName(backupDirName);
// We copy everything from in-use directory to a temporary directory
// for updating.
nsCString updatingDirName;
nsresult rv = mUpdatingDirectory->GetNativeLeafName(updatingDirName);
NS_ENSURE_SUCCESS(rv, rv);
nsCString storeDirName;
rv = mRootStoreDirectory->GetNativeLeafName(storeDirName);
NS_ENSURE_SUCCESS(rv, rv);
rv = mRootStoreDirectory->MoveToNative(nullptr, backupDirName);
NS_ENSURE_SUCCESS(rv, rv);
rv = mRootStoreDirectory->CopyToNative(nullptr, storeDirName);
// Remove the destination directory first (just in case) the do the copy.
mUpdatingDirectory->Remove(true);
rv = mRootStoreDirectory->CopyToNative(nullptr, updatingDirName);
NS_ENSURE_SUCCESS(rv, rv);
// We moved some things to new places, so move the handles around, too.
@ -859,21 +998,19 @@ Classifier::BackupTables()
return NS_OK;
}
nsresult
Classifier::RemoveBackupTables()
void
Classifier::CopyInUseLookupCacheForUpdate()
{
nsCString toDeleteName;
nsresult rv = mToDeleteDirectory->GetNativeLeafName(toDeleteName);
NS_ENSURE_SUCCESS(rv, rv);
MOZ_ASSERT(mNewLookupCaches.IsEmpty(), "Update intermediaries is forgotten to "
"be removed in the previous update.");
rv = mBackupDirectory->MoveToNative(nullptr, toDeleteName);
NS_ENSURE_SUCCESS(rv, rv);
// mBackupDirectory now points to toDelete, fix that up.
rv = SetupPathNames();
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
// TODO: Bug 1341183 - Implement deep copy function or copy ctor.
// For now we just re-open every table we have opened.
// Another option is to NOT pre-open these LookupCaches and
// do a "merge" instead of "swap" after update.
for (auto c: mLookupCaches) {
Unused << GetLookupCacheForUpdate(c->TableName());
}
}
nsresult
@ -962,7 +1099,7 @@ Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates,
LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get()));
HashStore store(aTable, GetProvider(aTable), mRootStoreDirectory);
HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory);
if (!CheckValidUpdate(aUpdates, store.TableName())) {
return NS_OK;
@ -975,7 +1112,7 @@ Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates,
// Read the part of the store that is (only) in the cache
LookupCacheV2* lookupCache =
LookupCache::Cast<LookupCacheV2>(GetLookupCache(store.TableName()));
LookupCache::Cast<LookupCacheV2>(GetLookupCacheForUpdate(store.TableName()));
if (!lookupCache) {
return NS_ERROR_FAILURE;
}
@ -1069,7 +1206,7 @@ Classifier::UpdateTableV4(nsTArray<TableUpdate*>* aUpdates,
}
LookupCacheV4* lookupCache =
LookupCache::Cast<LookupCacheV4>(GetLookupCache(aTable));
LookupCache::Cast<LookupCacheV4>(GetLookupCacheForUpdate(aTable));
if (!lookupCache) {
return NS_ERROR_FAILURE;
}
@ -1175,11 +1312,22 @@ Classifier::UpdateCache(TableUpdate* aUpdate)
}
LookupCache *
Classifier::GetLookupCache(const nsACString& aTable)
Classifier::GetLookupCache(const nsACString& aTable, bool aForUpdate)
{
for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
if (mLookupCaches[i]->TableName().Equals(aTable)) {
return mLookupCaches[i];
if (aForUpdate) {
return GetLookupCacheFrom(aTable, mNewLookupCaches, mUpdatingDirectory);
}
return GetLookupCacheFrom(aTable, mLookupCaches, mRootStoreDirectory);
}
LookupCache *
Classifier::GetLookupCacheFrom(const nsACString& aTable,
nsTArray<LookupCache*>& aLookupCaches,
nsIFile* aRootStoreDirectory)
{
for (uint32_t i = 0; i < aLookupCaches.Length(); i++) {
if (aLookupCaches[i]->TableName().Equals(aTable)) {
return aLookupCaches[i];
}
}
@ -1189,9 +1337,9 @@ Classifier::GetLookupCache(const nsACString& aTable)
UniquePtr<LookupCache> cache;
nsCString provider = GetProvider(aTable);
if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) {
cache = MakeUnique<LookupCacheV4>(aTable, provider, mRootStoreDirectory);
cache = MakeUnique<LookupCacheV4>(aTable, provider, aRootStoreDirectory);
} else {
cache = MakeUnique<LookupCacheV2>(aTable, provider, mRootStoreDirectory);
cache = MakeUnique<LookupCacheV2>(aTable, provider, aRootStoreDirectory);
}
nsresult rv = cache->Init();
@ -1205,7 +1353,7 @@ Classifier::GetLookupCache(const nsACString& aTable)
}
return nullptr;
}
mLookupCaches.AppendElement(cache.get());
aLookupCaches.AppendElement(cache.get());
return cache.release();
}

32
toolkit/components/url-classifier/Classifier.h
View File

@ -109,6 +109,10 @@ public:
const nsACString& aProvider,
nsIFile** aPrivateStoreDirectory);
// Swap in in-memory and on-disk database and remove all
// update intermediaries.
nsresult SwapInNewTablesAndCleanup();
private:
void DropStores();
void DeleteTables(nsIFile* aDirectory, const nsTArray<nsCString>& aTables);
@ -118,10 +122,16 @@ private:
nsresult SetupPathNames();
nsresult RecoverBackups();
nsresult CleanToDelete();
nsresult BackupTables();
nsresult RemoveBackupTables();
nsresult CopyInUseDirForUpdate();
void CopyInUseLookupCacheForUpdate();
nsresult RegenActiveTables();
// Remove any intermediary for update, including in-memory
// and on-disk data.
void RemoveUpdateIntermediaries();
#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
already_AddRefed<nsIFile> GetFailedUpdateDirectroy();
nsresult DumpFailedUpdate();
@ -137,7 +147,17 @@ private:
nsresult UpdateCache(TableUpdate* aUpdates);
LookupCache *GetLookupCache(const nsACString& aTable);
LookupCache *GetLookupCache(const nsACString& aTable,
bool aForUpdate = false);
LookupCache *GetLookupCacheForUpdate(const nsACString& aTable) {
return GetLookupCache(aTable, true);
}
LookupCache *GetLookupCacheFrom(const nsACString& aTable,
nsTArray<LookupCache*>& aLookupCaches,
nsIFile* aRootStoreDirectory);
bool CheckValidUpdate(nsTArray<TableUpdate*>* aUpdates,
const nsACString& aTable);
@ -152,9 +172,10 @@ private:
nsCOMPtr<nsIFile> mRootStoreDirectory;
// Used for atomically updating the other dirs.
nsCOMPtr<nsIFile> mBackupDirectory;
nsCOMPtr<nsIFile> mUpdatingDirectory; // For update only.
nsCOMPtr<nsIFile> mToDeleteDirectory;
nsCOMPtr<nsICryptoHash> mCryptoHash;
nsTArray<LookupCache*> mLookupCaches;
nsTArray<LookupCache*> mLookupCaches; // For query only.
nsTArray<nsCString> mActiveTablesCache;
uint32_t mHashKey;
// Stores the last time a given table was updated (seconds).
@ -167,6 +188,9 @@ private:
// Whether mTableRequestResult is outdated and needs to
// be reloaded from disk.
bool mIsTableRequestResultOutdated;
// The copy of mLookupCaches for update only.
nsTArray<LookupCache*> mNewLookupCaches;
};
} // namespace safebrowsing