Bug 1315097 - Build the provider dictionary on the main thread to be used everywhere. r=francois,gcp

MozReview-Commit-ID: Ft1deSNKuVB

--HG--
extra : rebase_source : 0cf746b1d2def35606f012c994ddbda0857ccf2d
This commit is contained in:
Henry Chang 2016-11-04 17:54:05 +08:00
parent 9d38145570
commit 1a53231595
15 changed files with 225 additions and 113 deletions

View File

@ -21,6 +21,7 @@
#include "mozilla/Base64.h"
#include "mozilla/Unused.h"
#include "mozilla/TypedEnumBits.h"
#include "nsIUrlClassifierUtils.h"
// MOZ_LOG=UrlClassifierDbService:5
extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
@ -85,96 +86,10 @@ Classifier::SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
}
}
static nsresult
DeriveProviderFromPref(const nsACString& aTableName, nsCString& aProviderName)
{
// Check all preferences "browser.safebrowsing.provider.[provider].list"
// to see which one contains |aTableName|.
nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE);
nsCOMPtr<nsIPrefBranch> prefBranch;
nsresult rv = prefs->GetBranch("browser.safebrowsing.provider.",
getter_AddRefs(prefBranch));
NS_ENSURE_SUCCESS(rv, rv);
// We've got a pref branch for "browser.safebrowsing.provider.".
// Enumerate all children prefs and parse providers.
uint32_t childCount;
char** childArray;
rv = prefBranch->GetChildList("", &childCount, &childArray);
NS_ENSURE_SUCCESS(rv, rv);
// Collect providers from childArray.
nsTHashtable<nsCStringHashKey> providers;
for (uint32_t i = 0; i < childCount; i++) {
nsCString child(childArray[i]);
auto dotPos = child.FindChar('.');
if (dotPos < 0) {
continue;
}
nsDependentCSubstring provider = Substring(child, 0, dotPos);
providers.PutEntry(provider);
}
NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(childCount, childArray);
// Now we have all providers. Check which one owns |aTableName|.
// e.g. The owning lists of provider "google" is defined in
// "browser.safebrowsing.provider.google.lists".
for (auto itr = providers.Iter(); !itr.Done(); itr.Next()) {
auto entry = itr.Get();
nsCString provider(entry->GetKey());
nsPrintfCString owninListsPref("%s.lists", provider.get());
nsXPIDLCString owningLists;
nsresult rv = prefBranch->GetCharPref(owninListsPref.get(),
getter_Copies(owningLists));
if (NS_FAILED(rv)) {
continue;
}
// We've got the owning lists (represented as string) of |provider|.
// Parse the string and see if |aTableName| is included.
nsTArray<nsCString> tables;
Classifier::SplitTables(owningLists, tables);
if (tables.Contains(aTableName)) {
aProviderName = provider;
return NS_OK;
}
}
return NS_ERROR_FAILURE;
}
// Lookup the provider name by table name on non-main thread.
// On main thread, just call DeriveProviderFromPref() instead
// but DeriveProviderFromPref is supposed to used internally.
static nsCString
GetProviderByTableName(const nsACString& aTableName)
{
MOZ_ASSERT(!NS_IsMainThread(), "GetProviderByTableName MUST be called "
"on non-main thread.");
nsCString providerName;
nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([&aTableName,
&providerName] () -> void {
nsresult rv = DeriveProviderFromPref(aTableName, providerName);
if (NS_FAILED(rv)) {
LOG(("No provider found for %s", nsCString(aTableName).get()));
}
});
nsCOMPtr<nsIThread> mainThread = do_GetMainThread();
SyncRunnable::DispatchToThread(mainThread, r);
return providerName;
}
nsresult
Classifier::GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory,
const nsACString& aTableName,
const nsACString& aProvider,
nsIFile** aPrivateStoreDirectory)
{
NS_ENSURE_ARG_POINTER(aPrivateStoreDirectory);
@ -186,8 +101,7 @@ Classifier::GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory,
return NS_OK;
}
nsCString providerName = GetProviderByTableName(aTableName);
if (providerName.IsEmpty()) {
if (aProvider.IsEmpty()) {
// When failing to get provider, just store in the root folder.
nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory);
return NS_OK;
@ -200,7 +114,7 @@ Classifier::GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory,
NS_ENSURE_SUCCESS(rv, rv);
// Append the provider name to the root store directory.
rv = providerDirectory->AppendNative(providerName);
rv = providerDirectory->AppendNative(aProvider);
NS_ENSURE_SUCCESS(rv, rv);
// Ensure existence of the provider directory.
@ -438,7 +352,7 @@ Classifier::TableRequest(nsACString& aResult)
nsTArray<nsCString> tables;
ActiveTables(tables);
for (uint32_t i = 0; i < tables.Length(); i++) {
HashStore store(tables[i], mRootStoreDirectory);
HashStore store(tables[i], GetProvider(tables[i]), mRootStoreDirectory);
nsresult rv = store.Open();
if (NS_FAILED(rv))
@ -713,7 +627,7 @@ Classifier::RegenActiveTables()
for (uint32_t i = 0; i < foundTables.Length(); i++) {
nsCString table(foundTables[i]);
HashStore store(table, mRootStoreDirectory);
HashStore store(table, GetProvider(table), mRootStoreDirectory);
nsresult rv = store.Open();
if (NS_FAILED(rv))
@ -980,6 +894,18 @@ Classifier::CheckValidUpdate(nsTArray<TableUpdate*>* aUpdates,
return true;
}
nsCString
Classifier::GetProvider(const nsACString& aTableName)
{
nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
nsCString provider;
nsresult rv = urlUtil->GetProvider(aTableName, provider);
return NS_SUCCEEDED(rv) ? provider : EmptyCString();
}
/*
* This will consume+delete updates from the passed nsTArray.
*/
@ -989,7 +915,7 @@ Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates,
{
LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get()));
HashStore store(aTable, mRootStoreDirectory);
HashStore store(aTable, GetProvider(aTable), mRootStoreDirectory);
if (!CheckValidUpdate(aUpdates, store.TableName())) {
return NS_OK;
@ -1208,10 +1134,11 @@ Classifier::GetLookupCache(const nsACString& aTable)
// thread method to convert table name to protocol version. Currently
// we can only know this by checking if the table name ends with '-proto'.
UniquePtr<LookupCache> cache;
nsCString provider = GetProvider(aTable);
if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) {
cache = MakeUnique<LookupCacheV4>(aTable, mRootStoreDirectory);
cache = MakeUnique<LookupCacheV4>(aTable, provider, mRootStoreDirectory);
} else {
cache = MakeUnique<LookupCacheV2>(aTable, mRootStoreDirectory);
cache = MakeUnique<LookupCacheV2>(aTable, provider, mRootStoreDirectory);
}
nsresult rv = cache->Init();

View File

@ -23,6 +23,9 @@ namespace safebrowsing {
* Maintains the stores and LookupCaches for the url classifier.
*/
class Classifier {
public:
typedef nsClassHashtable<nsCStringHashKey, nsCString> ProviderDictType;
public:
Classifier();
~Classifier();
@ -103,6 +106,7 @@ public:
// the root directory.
static nsresult GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory,
const nsACString& aTableName,
const nsACString& aProvider,
nsIFile** aPrivateStoreDirectory);
private:
@ -140,6 +144,8 @@ private:
nsresult LoadMetadata(nsIFile* aDirectory, nsACString& aResult);
nsCString GetProvider(const nsACString& aTableName);
// Root dir of the Local profile.
nsCOMPtr<nsIFile> mCacheDirectory;
// Main directory where to store the databases.

View File

@ -206,13 +206,16 @@ TableUpdateV4::NewChecksum(const std::string& aChecksum)
mChecksum.Assign(aChecksum.data(), aChecksum.size());
}
HashStore::HashStore(const nsACString& aTableName, nsIFile* aRootStoreDir)
HashStore::HashStore(const nsACString& aTableName,
const nsACString& aProvider,
nsIFile* aRootStoreDir)
: mTableName(aTableName)
, mInUpdate(false)
, mFileSize(0)
{
nsresult rv = Classifier::GetPrivateStoreDirectory(aRootStoreDir,
aTableName,
aProvider,
getter_AddRefs(mStoreDirectory));
if (NS_FAILED(rv)) {
LOG(("Failed to get private store directory for %s", mTableName.get()));

View File

@ -190,7 +190,9 @@ private:
// There is one hash store per table.
class HashStore {
public:
HashStore(const nsACString& aTableName, nsIFile* aRootStoreFile);
HashStore(const nsACString& aTableName,
const nsACString& aProvider,
nsIFile* aRootStoreFile);
~HashStore();
const nsCString& TableName() const { return mTableName; }

View File

@ -42,9 +42,12 @@ namespace safebrowsing {
const int LookupCacheV2::VER = 2;
LookupCache::LookupCache(const nsACString& aTableName, nsIFile* aRootStoreDir)
LookupCache::LookupCache(const nsACString& aTableName,
const nsACString& aProvider,
nsIFile* aRootStoreDir)
: mPrimed(false)
, mTableName(aTableName)
, mProvider(aProvider)
, mRootStoreDirectory(aRootStoreDir)
{
UpdateRootDirHandle(mRootStoreDirectory);
@ -72,6 +75,7 @@ LookupCache::UpdateRootDirHandle(nsIFile* aNewRootStoreDirectory)
rv = Classifier::GetPrivateStoreDirectory(mRootStoreDirectory,
mTableName,
mProvider,
getter_AddRefs(mStoreDirectory));
if (NS_FAILED(rv)) {
@ -478,7 +482,7 @@ LookupCacheV2::GetPrefixes(FallibleTArray<uint32_t>& aAddPrefixes)
nsresult
LookupCacheV2::ReadCompletions()
{
HashStore store(mTableName, mRootStoreDirectory);
HashStore store(mTableName, mProvider, mRootStoreDirectory);
nsresult rv = store.Open();
NS_ENSURE_SUCCESS(rv, rv);

View File

@ -96,7 +96,9 @@ public:
static nsresult GetHostKeys(const nsACString& aSpec,
nsTArray<nsCString>* aHostKeys);
LookupCache(const nsACString& aTableName, nsIFile* aStoreFile);
LookupCache(const nsACString& aTableName,
const nsACString& aProvider,
nsIFile* aStoreFile);
virtual ~LookupCache() {}
const nsCString &TableName() const { return mTableName; }
@ -146,6 +148,7 @@ private:
protected:
bool mPrimed;
nsCString mTableName;
nsCString mProvider;
nsCOMPtr<nsIFile> mRootStoreDirectory;
nsCOMPtr<nsIFile> mStoreDirectory;
@ -159,8 +162,10 @@ protected:
class LookupCacheV2 final : public LookupCache
{
public:
explicit LookupCacheV2(const nsACString& aTableName, nsIFile* aStoreFile)
: LookupCache(aTableName, aStoreFile) {}
explicit LookupCacheV2(const nsACString& aTableName,
const nsACString& aProvider,
nsIFile* aStoreFile)
: LookupCache(aTableName, aProvider, aStoreFile) {}
~LookupCacheV2() {}
virtual nsresult Init() override;

View File

@ -17,8 +17,10 @@ class TableUpdateV4;
class LookupCacheV4 final : public LookupCache
{
public:
explicit LookupCacheV4(const nsACString& aTableName, nsIFile* aStoreFile)
: LookupCache(aTableName, aStoreFile) {}
explicit LookupCacheV4(const nsACString& aTableName,
const nsACString& aProvider,
nsIFile* aStoreFile)
: LookupCache(aTableName, aProvider, aStoreFile) {}
~LookupCacheV4() {}
virtual nsresult Init() override;

View File

@ -22,6 +22,15 @@ interface nsIUrlClassifierUtils : nsISupports
*/
ACString getKeyForURI(in nsIURI uri);
/**
* Get the provider by table name.
*
* @param tableName The table name that we want to lookup
*
* @returns the provider name that the given table belongs.
*/
ACString getProvider(in ACString tableName);
/**
* Get the protocol version for the given provider.
*

View File

@ -1268,10 +1268,20 @@ nsUrlClassifierDBService::Init()
CONFIRM_AGE_DEFAULT_SEC);
ReadTablesFromPrefs();
// Force PSM loading on main thread
nsresult rv;
nsCOMPtr<nsICryptoHash> dummy = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
NS_ENSURE_SUCCESS(rv, rv);
{
// Force PSM loading on main thread
nsCOMPtr<nsICryptoHash> dummy = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
NS_ENSURE_SUCCESS(rv, rv);
}
{
// Force nsIUrlClassifierUtils loading on main thread.
nsCOMPtr<nsIUrlClassifierUtils> dummy =
do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID, &rv);
NS_ENSURE_SUCCESS(rv, rv);
}
// Directory providers must also be accessed on the main thread.
nsCOMPtr<nsIFile> cacheDir;

View File

@ -12,6 +12,7 @@
#include "nsPrintfCString.h"
#include "safebrowsing.pb.h"
#include "mozilla/Sprintf.h"
#include "mozilla/Mutex.h"
#define DEFAULT_PROTOCOL_VERSION "2.2"
@ -145,7 +146,9 @@ CreateClientInfo()
} // end of namespace safebrowsing.
} // end of namespace mozilla.
nsUrlClassifierUtils::nsUrlClassifierUtils() : mEscapeCharmap(nullptr)
nsUrlClassifierUtils::nsUrlClassifierUtils()
: mEscapeCharmap(nullptr)
, mProviderDictLock("nsUrlClassifierUtils.mProviderDictLock")
{
}
@ -157,10 +160,31 @@ nsUrlClassifierUtils::Init()
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff);
if (!mEscapeCharmap)
return NS_ERROR_OUT_OF_MEMORY;
// nsIUrlClassifierUtils is a thread-safe service so it's
// allowed to use on non-main threads. However, building
// the provider dictionary must be on the main thread.
// We forcefully load nsUrlClassifierUtils in
// nsUrlClassifierDBService::Init() to ensure we must
// now be on the main thread.
nsresult rv = ReadProvidersFromPrefs(mProviderDict);
NS_ENSURE_SUCCESS(rv, rv);
// Add an observer for shutdown
nsCOMPtr<nsIObserverService> observerService =
mozilla::services::GetObserverService();
if (!observerService)
return NS_ERROR_FAILURE;
observerService->AddObserver(this, "xpcom-shutdown-threads", false);
Preferences::AddStrongObserver(this, "browser.safebrowsing");
return NS_OK;
}
NS_IMPL_ISUPPORTS(nsUrlClassifierUtils, nsIUrlClassifierUtils)
NS_IMPL_ISUPPORTS(nsUrlClassifierUtils,
nsIUrlClassifierUtils,
nsIObserver)
/////////////////////////////////////////////////////////////////////////////
// nsIUrlClassifierUtils
@ -249,6 +273,20 @@ nsUrlClassifierUtils::ConvertListNameToThreatType(const nsACString& aListName,
return NS_ERROR_FAILURE;
}
NS_IMETHODIMP
nsUrlClassifierUtils::GetProvider(const nsACString& aTableName,
nsACString& aProvider)
{
MutexAutoLock lock(mProviderDictLock);
nsCString* provider = nullptr;
if (mProviderDict.Get(aTableName, &provider)) {
aProvider = provider ? *provider : EmptyCString();
} else {
aProvider = EmptyCString();
}
return NS_OK;
}
NS_IMETHODIMP
nsUrlClassifierUtils::GetProtocolVersion(const nsACString& aProvider,
nsACString& aVersion)
@ -306,9 +344,91 @@ nsUrlClassifierUtils::MakeUpdateRequestV4(const char** aListNames,
return NS_OK;
}
//////////////////////////////////////////////////////////
// nsIObserver
NS_IMETHODIMP
nsUrlClassifierUtils::Observe(nsISupports *aSubject, const char *aTopic,
const char16_t *aData)
{
if (0 == strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
MutexAutoLock lock(mProviderDictLock);
return ReadProvidersFromPrefs(mProviderDict);
}
if (0 == strcmp(aTopic, "xpcom-shutdown-threads")) {
nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE);
return prefs->RemoveObserver("browser.safebrowsing", this);
}
return NS_ERROR_UNEXPECTED;
}
/////////////////////////////////////////////////////////////////////////////
// non-interface methods
nsresult
nsUrlClassifierUtils::ReadProvidersFromPrefs(ProviderDictType& aDict)
{
MOZ_ASSERT(NS_IsMainThread(), "ReadProvidersFromPrefs must be on main thread");
nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE);
nsCOMPtr<nsIPrefBranch> prefBranch;
nsresult rv = prefs->GetBranch("browser.safebrowsing.provider.",
getter_AddRefs(prefBranch));
NS_ENSURE_SUCCESS(rv, rv);
// We've got a pref branch for "browser.safebrowsing.provider.".
// Enumerate all children prefs and parse providers.
uint32_t childCount;
char** childArray;
rv = prefBranch->GetChildList("", &childCount, &childArray);
NS_ENSURE_SUCCESS(rv, rv);
// Collect providers from childArray.
nsTHashtable<nsCStringHashKey> providers;
for (uint32_t i = 0; i < childCount; i++) {
nsCString child(childArray[i]);
auto dotPos = child.FindChar('.');
if (dotPos < 0) {
continue;
}
nsDependentCSubstring provider = Substring(child, 0, dotPos);
providers.PutEntry(provider);
}
NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(childCount, childArray);
// Now we have all providers. Check which one owns |aTableName|.
// e.g. The owning lists of provider "google" is defined in
// "browser.safebrowsing.provider.google.lists".
for (auto itr = providers.Iter(); !itr.Done(); itr.Next()) {
auto entry = itr.Get();
nsCString provider(entry->GetKey());
nsPrintfCString owninListsPref("%s.lists", provider.get());
nsXPIDLCString owningLists;
nsresult rv = prefBranch->GetCharPref(owninListsPref.get(),
getter_Copies(owningLists));
if (NS_FAILED(rv)) {
continue;
}
// We've got the owning lists (represented as string) of |provider|.
// Build the dictionary for the owning list and the current provider.
nsTArray<nsCString> tables;
Classifier::SplitTables(owningLists, tables);
for (auto tableName : tables) {
aDict.Put(tableName, new nsCString(provider));
}
}
return NS_OK;
}
nsresult
nsUrlClassifierUtils::CanonicalizeHostname(const nsACString & hostname,
nsACString & _retval)

View File

@ -7,9 +7,15 @@
#include "nsAutoPtr.h"
#include "nsIUrlClassifierUtils.h"
#include "nsClassHashtable.h"
#include "nsIObserver.h"
class nsUrlClassifierUtils final : public nsIUrlClassifierUtils
class nsUrlClassifierUtils final : public nsIUrlClassifierUtils,
public nsIObserver
{
public:
typedef nsClassHashtable<nsCStringHashKey, nsCString> ProviderDictType;
private:
/**
* A fast, bit-vector map for ascii characters.
@ -46,6 +52,7 @@ public:
NS_DECL_THREADSAFE_ISUPPORTS
NS_DECL_NSIURLCLASSIFIERUTILS
NS_DECL_NSIOBSERVER
nsresult Init();
@ -80,7 +87,13 @@ private:
void CleanupHostname(const nsACString & host, nsACString & _retval);
nsresult ReadProvidersFromPrefs(ProviderDictType& aDict);
nsAutoPtr<Charmap> mEscapeCharmap;
// The provider lookup table and its mutex.
ProviderDictType mProviderDict;
mozilla::Mutex mProviderDictLock;
};
#endif // nsUrlClassifierUtils_h_

View File

@ -5,6 +5,7 @@
#include "nsTArray.h"
#include "nsIThread.h"
#include "nsThreadUtils.h"
#include "nsUrlClassifierUtils.h"
using namespace mozilla;
using namespace mozilla::safebrowsing;
@ -41,6 +42,16 @@ void ApplyUpdate(nsTArray<TableUpdate*>& updates)
UniquePtr<Classifier> classifier(new Classifier());
classifier->Open(*file);
{
// Force nsIUrlClassifierUtils loading on main thread
// because nsIUrlClassifierDBService will not run in advance
// in gtest.
nsresult rv;
nsCOMPtr<nsIUrlClassifierUtils> dummy =
do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID, &rv);
ASSERT_TRUE(NS_SUCCEEDED(rv));
}
RunTestInNewThread([&] () -> void {
classifier->ApplyUpdates(&updates);
});

View File

@ -32,7 +32,7 @@ SetupLookupCacheV4(const _PrefixArray& prefixArray)
file->AppendNative(GTEST_SAFEBROWSING_DIR);
UniquePtr<LookupCacheV4> cache = MakeUnique<LookupCacheV4>(GTEST_TABLE, file);
UniquePtr<LookupCacheV4> cache = MakeUnique<LookupCacheV4>(GTEST_TABLE, EmptyCString(), file);
nsresult rv = cache->Init();
EXPECT_EQ(rv, NS_OK);

View File

@ -32,7 +32,7 @@ void VerifyPrivateStorePath(const char* aTableName,
nsresult rv = aRootDir->GetPath(rootStorePath);
EXPECT_EQ(rv, NS_OK);
T target(nsCString(aTableName), aRootDir);
T target(nsCString(aTableName), nsCString(aProvider), aRootDir);
nsIFile* privateStoreDirectory =
PerProviderDirectoryTestUtils::InspectStoreDirectory(target);

View File

@ -252,7 +252,7 @@ testOpenLookupCache()
file->AppendNative(GTEST_SAFEBROWSING_DIR);
RunTestInNewThread([&] () -> void {
LookupCacheV4 cache(nsCString(GTEST_TABLE), file);
LookupCacheV4 cache(nsCString(GTEST_TABLE), EmptyCString(), file);
nsresult rv = cache.Init();
ASSERT_EQ(rv, NS_OK);