facebookresearch/faiss
1
0
Fork 0
mirror of https://github.com/facebookresearch/faiss.git synced 2026-10-12 02:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Validate inverted list entry sizes against deserialization byte limit (#5117)

Browse Source 
Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/5117

Add deserialization byte limit checks before vector::resize calls in read_InvertedLists_up() for both ArrayInvertedListsPanorama ("ilpn") and ArrayInvertedLists ("ilar") paths. Previously, per-list sizes read from serialized data were used directly in .resize() calls without checking against get_deserialization_vector_byte_limit(). The READVECTOR macro enforces this limit, but explicit .resize() calls bypassed it.

Also add mul_no_overflow protection for the ilpn codes allocation (num_elems * code_size) which previously had no overflow check.

Reviewed By: mnorris11

Differential Revision: D101260923

fbshipit-source-id: 24287740642cc9647115676c71508faf8bf8f48e
This commit is contained in:
Justin Gibbs
2026-04-19 08:54:55 -07:00
committed by meta-codesync[bot]
parent e6be16241a
commit 817ecf9cd4
2 changed files with 71 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
faiss/impl/index_read.cpp
+44 -4
View File
@@ -514,14 +514,40 @@ std::unique_ptr<InvertedLists> read_InvertedLists_up(
nlist, code_size, n_levels);
std::vector<size_t> sizes(nlist);
read_ArrayInvertedLists_sizes(f, sizes);
size_t byte_limit = get_deserialization_vector_byte_limit();
for (size_t i = 0; i < nlist; i++) {
FAISS_THROW_IF_NOT_FMT(
sizes[i] <= byte_limit / sizeof(idx_t),
"inverted list %zu ids size %zu exceeds "
"deserialization byte limit",
i,
sizes[i]);
ailp->ids[i].resize(sizes[i]);
size_t num_elems =
((sizes[i] + ArrayInvertedListsPanorama::kBatchSize - 1) /
ArrayInvertedListsPanorama::kBatchSize) *
ArrayInvertedListsPanorama::kBatchSize;
ailp->codes[i].resize(num_elems * code_size);
ailp->cum_sums[i].resize(num_elems * (n_levels + 1));
size_t codes_bytes = mul_no_overflow(
num_elems, code_size, "inverted list codes");
FAISS_THROW_IF_NOT_FMT(
codes_bytes <= byte_limit,
"inverted list %zu codes size %zu exceeds "
"deserialization byte limit",
i,
codes_bytes);
ailp->codes[i].resize(codes_bytes);
size_t cum_sums_count = mul_no_overflow(
num_elems,
add_no_overflow(
n_levels, 1, "inverted list cum_sums n_levels"),
"inverted list cum_sums");
FAISS_THROW_IF_NOT_FMT(
cum_sums_count <= byte_limit / sizeof(ailp->cum_sums[0][0]),
"inverted list %zu cum_sums size %zu exceeds "
"deserialization byte limit",
i,
cum_sums_count);
ailp->cum_sums[i].resize(cum_sums_count);
}
for (size_t i = 0; i < nlist; i++) {
size_t n = sizes[i];
@@ -543,10 +569,24 @@ std::unique_ptr<InvertedLists> read_InvertedLists_up(
ails->codes.resize(ails->nlist);
std::vector<size_t> sizes(ails->nlist);
read_ArrayInvertedLists_sizes(f, sizes);
size_t ilar_byte_limit = get_deserialization_vector_byte_limit();
for (size_t i = 0; i < ails->nlist; i++) {
FAISS_THROW_IF_NOT_FMT(
sizes[i] <= ilar_byte_limit / sizeof(idx_t),
"inverted list %zu ids size %zu exceeds "
"deserialization byte limit",
i,
sizes[i]);
ails->ids[i].resize(sizes[i]);
ails->codes[i].resize(mul_no_overflow(
sizes[i], ails->code_size, "inverted list codes"));
size_t codes_bytes = mul_no_overflow(
sizes[i], ails->code_size, "inverted list codes");
FAISS_THROW_IF_NOT_FMT(
codes_bytes <= ilar_byte_limit,
"inverted list %zu codes size %zu exceeds "
"deserialization byte limit",
i,
codes_bytes);
ails->codes[i].resize(codes_bytes);
}
for (size_t i = 0; i < ails->nlist; i++) {
size_t n = ails->ids[i].size();
tests/test_read_index_deserialize.cpp
+27
View File
@@ -2034,6 +2034,33 @@ static void push_ivf_header(std::vector<uint8_t>& buf, int d) {
push_empty_direct_map(buf);
}
// -----------------------------------------------------------------------
// Test: Inverted list with oversized entry exceeding deserialization byte
// limit is rejected. Protects against corrupt data causing OOM via
// unchecked vector::resize in read_InvertedLists_up.
// -----------------------------------------------------------------------
TEST(ReadIndexDeserialize, InvertedListOversizedEntry) {
const size_t old_limit = get_deserialization_vector_byte_limit();
set_deserialization_vector_byte_limit(1024); // 1 KB limit
std::vector<uint8_t> buf;
push_fourcc(buf, "IwFl");
push_ivf_header(buf, /*d=*/4);
// "ilar" inverted lists with 1 list, code_size=16
push_fourcc(buf, "ilar");
push_val<size_t>(buf, 1); // nlist = 1
push_val<size_t>(buf, 16); // code_size = 16
// "full" list type with 1 size entry
push_fourcc(buf, "full");
// sizes vector: 1 entry with absurdly large value
push_val<size_t>(buf, 1); // vector length
push_val<size_t>(buf, (size_t)1 << 40); // sizes[0] = 1 TB
expect_read_throws_with(buf, "deserialization byte limit");
set_deserialization_vector_byte_limit(old_limit);
}
// -- Ixrq (IndexRaBitQ, single-bit) --
// qb=0 is valid for Ixrq: disables query quantization, uses raw fp32 values.
// See IndexRaBitQ.h comment on qb and RaBitQDistanceComputerNotQ.