Bug 1542744 - P2. Improve performance of MakePrefixSet by using different algorithm according to the number of prefixes. r=gcp

The goal of this patch is to reduce the number of memory reallocation during
|MakePrefixSet|[1].

Here is the number of nsTArray memory reallocation occur during |MakePrefixSet|
(test in my local platform):
googpub-phish-proto: 58k times
goog-malware-proto: 9k times
goog-unwanted-proto: 25k times
goog-badbinurl-proto: 6k times

This patch improves the performance by:
1. For tables whose prefixes are less than 128*1024(malware, unwanted,
badinurl).

Store prefixes directly without dividing allocation into smaller chunks.
Because the maximum size to store all the prefixes in a single array for
these tables will be less than 512k, we can avoid Bug 1046038.

This simplifies the internal prefixset data structure generation and total
memory usage is also saved:
goog-malware-proto  : 437K -> 163k
goog-unwanted-proto : 658k -> 446k
goog-badbinurl-proto: 320k -> 110k

The single largest allocated continuous memory size is:
goog-malware-proto  : 86k -> 163k
goog-unwanted-proto : 86k -> 446k
goog-badbinurl-proto: 77k -> 110k

A further improvement can be done for this part is for tables with fewer
prefixes, we can use an one-dimension delta array to reduce the size of a
single continuous memory allocation.

2. For tables with more prefixes:

According to experiment, when prefixes are more than 400k
the delta arrays have very high chance that are full, in the case of
phishing table, we can estimate the capacity accurately before
applying delta algorithm.

The shortcoming of this part is when prefixes are between 130k~400k,
the capacity estimation is not accurate.

[1] https://searchfox.org/mozilla-central/rev/b2015fdd464f598d645342614593d4ebda922d95/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp#99

Differential Revision: https://phabricator.services.mozilla.com/D30046

--HG--
extra : moz-landing-system : lando
This commit is contained in:
dlee 2019-05-14 21:05:41 +00:00
parent 59923e57ab
commit 8370cfe1bf
3 changed files with 190 additions and 93 deletions

View File

@ -2581,6 +2581,13 @@ VARCACHE_PREF(
bool, true
)
// Maximum size for an array to store the safebrowsing prefixset.
VARCACHE_PREF(
"browser.safebrowsing.prefixset.max_array_size",
browser_safebrowsing_prefixset_max_array_size,
uint32_t, 512*1024
)
// When this pref is enabled document loads with a mismatched
// Cross-Origin header will fail to load
VARCACHE_PREF("browser.tabs.remote.useCrossOriginPolicy",

View File

@ -17,6 +17,7 @@
#include "nsISeekableStream.h"
#include "nsIBufferedStreams.h"
#include "mozilla/MemoryReporting.h"
#include "mozilla/StaticPrefs.h"
#include "mozilla/Telemetry.h"
#include "mozilla/Logging.h"
#include "mozilla/Unused.h"
@ -77,7 +78,10 @@ nsUrlClassifierPrefixSet::SetPrefixes(const uint32_t* aArray,
}
}
MOZ_ASSERT(mIndexPrefixes.Length() == mIndexDeltas.Length());
MOZ_ASSERT_IF(mIndexDeltas.IsEmpty(),
mIndexPrefixes.Length() == mTotalPrefixes);
MOZ_ASSERT_IF(!mIndexDeltas.IsEmpty(),
mIndexPrefixes.Length() == mIndexDeltas.Length());
return rv;
}
@ -94,47 +98,88 @@ nsresult nsUrlClassifierPrefixSet::MakePrefixSet(const uint32_t* aPrefixes,
}
#endif
mIndexPrefixes.AppendElement(aPrefixes[0]);
mIndexDeltas.AppendElement();
uint32_t numOfDeltas = 0;
uint32_t totalDeltas = 0;
uint32_t previousItem = aPrefixes[0];
for (uint32_t i = 1; i < aLength; i++) {
if ((numOfDeltas >= DELTAS_LIMIT) ||
(aPrefixes[i] - previousItem >= MAX_INDEX_DIFF)) {
// Compact the previous element.
// Note there is always at least one element when we get here,
// because we created the first element before the loop.
if (!mIndexDeltas.AppendElement(fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
if (!mIndexPrefixes.AppendElement(aPrefixes[i], fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
numOfDeltas = 0;
} else {
uint16_t delta = aPrefixes[i] - previousItem;
if (!mIndexDeltas.LastElement().AppendElement(delta, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
numOfDeltas++;
totalDeltas++;
// Request one memory space to store all the prefixes may lead to
// memory allocation failure on certain platforms(See Bug 1046038).
// So if required size to store all the prefixes exceeds defined
// threshold(512k), we divide prefixes into delta chunks instead. Note that
// the potential overhead of this approach is that it may reuqire more memory
// compared to store all prefixes in one array because of jemalloc's
// implementation.
if (aLength * sizeof(uint32_t) <
StaticPrefs::browser_safebrowsing_prefixset_max_array_size()) {
// Not over the threshold, store all prefixes into mIndexPrefixes.
// mIndexDeltas is empty in this case.
mIndexPrefixes.SetCapacity(aLength);
for (uint32_t i = 0; i < aLength; i++) {
mIndexPrefixes.AppendElement(aPrefixes[i]);
}
previousItem = aPrefixes[i];
} else {
// Apply delta algorithm to split prefixes into smaller delta chunk.
// We estimate the capacity of mIndexPrefixes & mIndexDeltas by assuming
// each element in mIndexDeltas stores DELTAS_LIMITS deltas, so the
// number of indexed prefixes is round up of
// TotalPrefixes / (DELTA_LIMIT + 1)
// The estimation only works when the number of prefixes are over a
// certain limit, which means, arrays in mIndexDeltas are always full.
uint32_t estimateCapacity =
(aLength + (DELTAS_LIMIT + 1) - 1) / (DELTAS_LIMIT + 1);
mIndexPrefixes.SetCapacity(estimateCapacity);
mIndexDeltas.SetCapacity(estimateCapacity);
mIndexPrefixes.AppendElement(aPrefixes[0]);
mIndexDeltas.AppendElement();
mIndexDeltas.LastElement().SetCapacity(DELTAS_LIMIT);
uint32_t numOfDeltas = 0;
uint32_t previousItem = aPrefixes[0];
for (uint32_t i = 1; i < aLength; i++) {
if ((numOfDeltas >= DELTAS_LIMIT) ||
(aPrefixes[i] - previousItem >= MAX_INDEX_DIFF)) {
// Compact the previous element.
// Note there is always at least one element when we get here,
// because we created the first element before the loop.
mIndexDeltas.LastElement().Compact();
if (!mIndexDeltas.AppendElement(fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
mIndexDeltas.LastElement().SetCapacity(DELTAS_LIMIT);
if (!mIndexPrefixes.AppendElement(aPrefixes[i], fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
numOfDeltas = 0;
} else {
uint16_t delta = aPrefixes[i] - previousItem;
if (!mIndexDeltas.LastElement().AppendElement(delta, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
numOfDeltas++;
totalDeltas++;
}
previousItem = aPrefixes[i];
}
mIndexDeltas.LastElement().Compact();
mIndexDeltas.Compact();
mIndexPrefixes.Compact();
MOZ_ASSERT(mIndexPrefixes.Length() == mIndexDeltas.Length());
}
if (totalDeltas == 0) {
// We have to clear mIndexDeltas here because it is still possible
// that the delta generation algorithm produces no deltas at all. When that
// happens, mIndexDeltas is not empty, which conflicts with the assumption
// that when there is no delta, mIndexDeltas is empty.
mIndexDeltas.Clear();
}
mTotalPrefixes = aLength;
mIndexDeltas.LastElement().Compact();
mIndexDeltas.Compact();
mIndexPrefixes.Compact();
MOZ_ASSERT(mIndexPrefixes.Length() == mIndexDeltas.Length());
LOG(("Total number of indices: %d", aLength));
LOG(("Total number of deltas: %d", totalDeltas));
LOG(("Total number of delta chunks: %zu", mIndexDeltas.Length()));
@ -161,6 +206,10 @@ nsresult nsUrlClassifierPrefixSet::GetPrefixesNative(
}
outArray[prefixCnt++] = prefix;
if (mIndexDeltas.IsEmpty()) {
continue;
}
for (uint32_t j = 0; j < mIndexDeltas[i].Length(); j++) {
prefix += mIndexDeltas[i][j];
if (prefixCnt >= mTotalPrefixes) {
@ -253,12 +302,15 @@ nsUrlClassifierPrefixSet::Contains(uint32_t aPrefix, bool* aFound) {
// Now search through the deltas for the target.
uint32_t diff = target - mIndexPrefixes[i];
uint32_t deltaSize = mIndexDeltas[i].Length();
uint32_t deltaIndex = 0;
while (diff > 0 && deltaIndex < deltaSize) {
diff -= mIndexDeltas[i][deltaIndex];
deltaIndex++;
if (!mIndexDeltas.IsEmpty()) {
uint32_t deltaSize = mIndexDeltas[i].Length();
uint32_t deltaIndex = 0;
while (diff > 0 && deltaIndex < deltaSize) {
diff -= mIndexDeltas[i][deltaIndex];
deltaIndex++;
}
}
if (diff == 0) {
@ -335,13 +387,14 @@ nsresult nsUrlClassifierPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE);
if (magic == PREFIXSET_VERSION_MAGIC) {
// Read the number of indexed prefixes
uint32_t indexSize;
uint32_t deltaSize;
rv = in->Read(reinterpret_cast<char*>(&indexSize), sizeof(uint32_t), &read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE);
// Read the number of delta prefixes
uint32_t deltaSize;
rv = in->Read(reinterpret_cast<char*>(&deltaSize), sizeof(uint32_t), &read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE);
@ -351,58 +404,71 @@ nsresult nsUrlClassifierPrefixSet::LoadPrefixes(nsCOMPtr<nsIInputStream>& in) {
return NS_OK;
}
if (deltaSize > (indexSize * DELTAS_LIMIT)) {
return NS_ERROR_FILE_CORRUPTED;
}
nsTArray<uint32_t> indexStarts;
if (!indexStarts.SetLength(indexSize, fallible) ||
!mIndexPrefixes.SetLength(indexSize, fallible) ||
!mIndexDeltas.SetLength(indexSize, fallible)) {
if (!mIndexPrefixes.SetLength(indexSize, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
mTotalPrefixes = indexSize;
if (deltaSize > (indexSize * DELTAS_LIMIT)) {
return NS_ERROR_FILE_CORRUPTED;
}
// Read index prefixes
uint32_t toRead = indexSize * sizeof(uint32_t);
rv = in->Read(reinterpret_cast<char*>(mIndexPrefixes.Elements()), toRead,
&read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE);
rv = in->Read(reinterpret_cast<char*>(indexStarts.Elements()), toRead,
&read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE);
if (deltaSize) {
nsTArray<uint32_t> indexStarts;
if (indexSize != 0 && indexStarts[0] != 0) {
return NS_ERROR_FILE_CORRUPTED;
}
for (uint32_t i = 0; i < indexSize; i++) {
uint32_t numInDelta = i == indexSize - 1
? deltaSize - indexStarts[i]
: indexStarts[i + 1] - indexStarts[i];
if (numInDelta > DELTAS_LIMIT) {
if (!indexStarts.SetLength(indexSize, fallible) ||
!mIndexDeltas.SetLength(indexSize, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
// Read index start array to construct mIndexDeltas
rv = in->Read(reinterpret_cast<char*>(indexStarts.Elements()), toRead,
&read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE);
if (indexStarts[0] != 0) {
return NS_ERROR_FILE_CORRUPTED;
}
if (numInDelta > 0) {
if (!mIndexDeltas[i].SetLength(numInDelta, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
for (uint32_t i = 0; i < indexSize; i++) {
uint32_t numInDelta = i == indexSize - 1
? deltaSize - indexStarts[i]
: indexStarts[i + 1] - indexStarts[i];
if (numInDelta > DELTAS_LIMIT) {
return NS_ERROR_FILE_CORRUPTED;
}
if (numInDelta > 0) {
if (!mIndexDeltas[i].SetLength(numInDelta, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
mTotalPrefixes += numInDelta;
toRead = numInDelta * sizeof(uint16_t);
rv = in->Read(reinterpret_cast<char*>(mIndexDeltas[i].Elements()),
toRead, &read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE);
}
mTotalPrefixes += numInDelta;
toRead = numInDelta * sizeof(uint16_t);
rv = in->Read(reinterpret_cast<char*>(mIndexDeltas[i].Elements()),
toRead, &read);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE);
}
} else {
mIndexDeltas.Clear();
}
} else {
LOG(("[%s] Version magic mismatch, not loading", mName.get()));
return NS_ERROR_FILE_CORRUPTED;
}
MOZ_ASSERT(mIndexPrefixes.Length() == mIndexDeltas.Length());
MOZ_ASSERT_IF(mIndexDeltas.IsEmpty(),
mIndexPrefixes.Length() == mTotalPrefixes);
MOZ_ASSERT_IF(!mIndexDeltas.IsEmpty(),
mIndexPrefixes.Length() == mIndexDeltas.Length());
LOG(("[%s] Loading PrefixSet successful (%u total prefixes)", mName.get(),
mTotalPrefixes));
@ -413,8 +479,14 @@ uint32_t nsUrlClassifierPrefixSet::CalculatePreallocateSize() const {
uint32_t fileSize = 4 * sizeof(uint32_t);
MOZ_RELEASE_ASSERT(mTotalPrefixes >= mIndexPrefixes.Length());
uint32_t deltas = mTotalPrefixes - mIndexPrefixes.Length();
fileSize += 2 * mIndexPrefixes.Length() * sizeof(uint32_t);
fileSize += deltas * sizeof(uint16_t);
fileSize += mIndexPrefixes.Length() * sizeof(uint32_t);
if (deltas) {
MOZ_ASSERT(mIndexPrefixes.Length() == mIndexDeltas.Length());
fileSize += mIndexPrefixes.Length() * sizeof(uint32_t);
fileSize += mIndexDeltas.Length() * sizeof(uint32_t);
fileSize += deltas * sizeof(uint16_t);
}
return fileSize;
}
@ -422,6 +494,11 @@ nsresult nsUrlClassifierPrefixSet::WritePrefixes(
nsCOMPtr<nsIOutputStream>& out) const {
MutexAutoLock lock(mLock);
MOZ_ASSERT_IF(mIndexDeltas.IsEmpty(),
mIndexPrefixes.Length() == mTotalPrefixes);
MOZ_ASSERT_IF(!mIndexDeltas.IsEmpty(),
mIndexPrefixes.Length() == mIndexDeltas.Length());
mCanary.Check();
uint32_t written;
@ -433,10 +510,18 @@ nsresult nsUrlClassifierPrefixSet::WritePrefixes(
NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE);
const uint32_t indexSize = mIndexPrefixes.Length();
if (NS_WARN_IF(mIndexDeltas.Length() != indexSize)) {
LOG(("[%s] mIndexPrefixes doesn't have the same length as mIndexDeltas",
mName.get()));
return NS_ERROR_FAILURE;
if (mIndexDeltas.IsEmpty()) {
if (NS_WARN_IF(mTotalPrefixes != indexSize)) {
LOG(("[%s] mIndexPrefixes doesn't have the same length as mTotalPrefixes",
mName.get()));
return NS_ERROR_FAILURE;
}
} else {
if (NS_WARN_IF(mIndexDeltas.Length() != indexSize)) {
LOG(("[%s] mIndexPrefixes doesn't have the same length as mIndexDeltas",
mName.get()));
return NS_ERROR_FAILURE;
}
}
uint32_t totalDeltas = 0;
@ -446,18 +531,20 @@ nsresult nsUrlClassifierPrefixSet::WritePrefixes(
// If we ever update the format, we can gain space by storing the delta
// subarray sizes, which fit in bytes.
nsTArray<uint32_t> indexStarts;
if (!indexStarts.SetCapacity(indexSize + 1, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
indexStarts.AppendElement(0);
if (!mIndexDeltas.IsEmpty()) {
if (!indexStarts.SetCapacity(indexSize + 1, fallible)) {
return NS_ERROR_OUT_OF_MEMORY;
}
indexStarts.AppendElement(0);
for (uint32_t i = 0; i < indexSize; i++) {
uint32_t deltaLength = mIndexDeltas[i].Length();
totalDeltas += deltaLength;
indexStarts.AppendElement(totalDeltas);
for (uint32_t i = 0; i < indexSize; i++) {
uint32_t deltaLength = mIndexDeltas[i].Length();
totalDeltas += deltaLength;
indexStarts.AppendElement(totalDeltas);
}
indexStarts.RemoveElementAt(indexSize); // we don't use the last element
MOZ_ASSERT(indexStarts.Length() == indexSize);
}
indexStarts.RemoveElementAt(indexSize); // we don't use the last element
MOZ_ASSERT(indexStarts.Length() == indexSize);
rv =
out->Write(reinterpret_cast<const char*>(&indexSize), writelen, &written);
@ -475,12 +562,13 @@ nsresult nsUrlClassifierPrefixSet::WritePrefixes(
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE);
rv = out->Write(reinterpret_cast<const char*>(indexStarts.Elements()),
writelen, &written);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE);
if (!mIndexDeltas.IsEmpty()) {
MOZ_ASSERT(!indexStarts.IsEmpty() && totalDeltas > 0);
rv = out->Write(reinterpret_cast<const char*>(indexStarts.Elements()),
writelen, &written);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE);
if (totalDeltas > 0) {
for (uint32_t i = 0; i < indexSize; i++) {
writelen = mIndexDeltas[i].Length() * sizeof(uint16_t);
rv = out->Write(reinterpret_cast<const char*>(mIndexDeltas[i].Elements()),

View File

@ -76,6 +76,8 @@ class nsUrlClassifierPrefixSet final : public nsIUrlClassifierPrefixSet,
// Index to the place that matches the closest lower
// prefix from mIndexPrefix. Then every "delta" corresponds
// to a prefix in the PrefixSet.
// This array could be empty when we decide to store all the prefixes
// in mIndexPrefixes.
nsTArray<nsTArray<uint16_t> > mIndexDeltas;
// how many prefixes we have.