From 817ecf9cd444dbd7461d3a09a0035b16448d2c6f Mon Sep 17 00:00:00 2001 From: Justin Gibbs Date: Sun, 19 Apr 2026 08:54:55 -0700 Subject: [PATCH] Validate inverted list entry sizes against deserialization byte limit (#5117) Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/5117 Add deserialization byte limit checks before vector::resize calls in read_InvertedLists_up() for both ArrayInvertedListsPanorama ("ilpn") and ArrayInvertedLists ("ilar") paths. Previously, per-list sizes read from serialized data were used directly in .resize() calls without checking against get_deserialization_vector_byte_limit(). The READVECTOR macro enforces this limit, but explicit .resize() calls bypassed it. Also add mul_no_overflow protection for the ilpn codes allocation (num_elems * code_size) which previously had no overflow check. Reviewed By: mnorris11 Differential Revision: D101260923 fbshipit-source-id: 24287740642cc9647115676c71508faf8bf8f48e --- faiss/impl/index_read.cpp | 48 ++++++++++++++++++++++++--- tests/test_read_index_deserialize.cpp | 27 +++++++++++++++ 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/faiss/impl/index_read.cpp b/faiss/impl/index_read.cpp index 9d13e40ec..3dcac31c0 100644 --- a/faiss/impl/index_read.cpp +++ b/faiss/impl/index_read.cpp @@ -514,14 +514,40 @@ std::unique_ptr read_InvertedLists_up( nlist, code_size, n_levels); std::vector sizes(nlist); read_ArrayInvertedLists_sizes(f, sizes); + size_t byte_limit = get_deserialization_vector_byte_limit(); for (size_t i = 0; i < nlist; i++) { + FAISS_THROW_IF_NOT_FMT( + sizes[i] <= byte_limit / sizeof(idx_t), + "inverted list %zu ids size %zu exceeds " + "deserialization byte limit", + i, + sizes[i]); ailp->ids[i].resize(sizes[i]); size_t num_elems = ((sizes[i] + ArrayInvertedListsPanorama::kBatchSize - 1) / ArrayInvertedListsPanorama::kBatchSize) * ArrayInvertedListsPanorama::kBatchSize; - ailp->codes[i].resize(num_elems * code_size); - ailp->cum_sums[i].resize(num_elems * (n_levels + 1)); + size_t codes_bytes = mul_no_overflow( + num_elems, code_size, "inverted list codes"); + FAISS_THROW_IF_NOT_FMT( + codes_bytes <= byte_limit, + "inverted list %zu codes size %zu exceeds " + "deserialization byte limit", + i, + codes_bytes); + ailp->codes[i].resize(codes_bytes); + size_t cum_sums_count = mul_no_overflow( + num_elems, + add_no_overflow( + n_levels, 1, "inverted list cum_sums n_levels"), + "inverted list cum_sums"); + FAISS_THROW_IF_NOT_FMT( + cum_sums_count <= byte_limit / sizeof(ailp->cum_sums[0][0]), + "inverted list %zu cum_sums size %zu exceeds " + "deserialization byte limit", + i, + cum_sums_count); + ailp->cum_sums[i].resize(cum_sums_count); } for (size_t i = 0; i < nlist; i++) { size_t n = sizes[i]; @@ -543,10 +569,24 @@ std::unique_ptr read_InvertedLists_up( ails->codes.resize(ails->nlist); std::vector sizes(ails->nlist); read_ArrayInvertedLists_sizes(f, sizes); + size_t ilar_byte_limit = get_deserialization_vector_byte_limit(); for (size_t i = 0; i < ails->nlist; i++) { + FAISS_THROW_IF_NOT_FMT( + sizes[i] <= ilar_byte_limit / sizeof(idx_t), + "inverted list %zu ids size %zu exceeds " + "deserialization byte limit", + i, + sizes[i]); ails->ids[i].resize(sizes[i]); - ails->codes[i].resize(mul_no_overflow( - sizes[i], ails->code_size, "inverted list codes")); + size_t codes_bytes = mul_no_overflow( + sizes[i], ails->code_size, "inverted list codes"); + FAISS_THROW_IF_NOT_FMT( + codes_bytes <= ilar_byte_limit, + "inverted list %zu codes size %zu exceeds " + "deserialization byte limit", + i, + codes_bytes); + ails->codes[i].resize(codes_bytes); } for (size_t i = 0; i < ails->nlist; i++) { size_t n = ails->ids[i].size(); diff --git a/tests/test_read_index_deserialize.cpp b/tests/test_read_index_deserialize.cpp index 431bfa4a5..659420a90 100644 --- a/tests/test_read_index_deserialize.cpp +++ b/tests/test_read_index_deserialize.cpp @@ -2034,6 +2034,33 @@ static void push_ivf_header(std::vector& buf, int d) { push_empty_direct_map(buf); } +// ----------------------------------------------------------------------- +// Test: Inverted list with oversized entry exceeding deserialization byte +// limit is rejected. Protects against corrupt data causing OOM via +// unchecked vector::resize in read_InvertedLists_up. +// ----------------------------------------------------------------------- +TEST(ReadIndexDeserialize, InvertedListOversizedEntry) { + const size_t old_limit = get_deserialization_vector_byte_limit(); + set_deserialization_vector_byte_limit(1024); // 1 KB limit + + std::vector buf; + push_fourcc(buf, "IwFl"); + push_ivf_header(buf, /*d=*/4); + // "ilar" inverted lists with 1 list, code_size=16 + push_fourcc(buf, "ilar"); + push_val(buf, 1); // nlist = 1 + push_val(buf, 16); // code_size = 16 + // "full" list type with 1 size entry + push_fourcc(buf, "full"); + // sizes vector: 1 entry with absurdly large value + push_val(buf, 1); // vector length + push_val(buf, (size_t)1 << 40); // sizes[0] = 1 TB + + expect_read_throws_with(buf, "deserialization byte limit"); + + set_deserialization_vector_byte_limit(old_limit); +} + // -- Ixrq (IndexRaBitQ, single-bit) -- // qb=0 is valid for Ixrq: disables query quantization, uses raw fp32 values. // See IndexRaBitQ.h comment on qb and RaBitQDistanceComputerNotQ.