mirror of
https://github.com/facebookresearch/faiss.git
synced 2026-10-12 02:13:36 +00:00
Subhadeep Karan
7721a01e62
Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/4687 # Bug: vmin > vmax When o > n/2 ### How the Code Works - You compute `o = int(rs_arg * n)`, then clamp it so that `o <= n/2`. - You use `std::nth_element` to find: - `vmin = x_copy[o]` (the o-th smallest element) - `vmax = x_copy[n-1-o]` (the (n-1-o)-th smallest element) ### What Happens When o > n/2? - The code clamps `o` to `n/2` if `o > n/2`. - For even `n`, `o = n/2` means: - `vmin = x_copy[n/2]` - `vmax = x_copy[n-1-n/2] = x_copy[n/2 - 1]` - For odd `n`, integer division means `o = floor(n/2)`. ### Can vmin > vmax? - Yes, if `o > n/2` (or is clamped to `n/2`), then `o > n-1-o`. - This means the index for `vmin` is greater than the index for `vmax`. - In a sorted array, this would mean `vmin >= vmax`. - In practice, since `nth_element` only guarantees the element at the index is in the correct position, but does not fully sort the array, you can end up with `vmin > vmax`. ## Example Suppose `n = 10`, so `o = 5` after clamping. - `vmin = x_copy[5]` (6th smallest) - `vmax = x_copy[4]` (5th smallest) If the array is not sorted, and you only use `nth_element`, you could have `vmin > vmax`. Reviewed By: mnorris11 Differential Revision: D86571612 fbshipit-source-id: 1fb8225fd021a2984b525a2c508062e2284213b7
98 lines
2.4 KiB
C++
98 lines
2.4 KiB
C++
/*
|
|
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
*
|
|
* This source code is licensed under the MIT license found in the
|
|
* LICENSE file in the root directory of this source tree.
|
|
*/
|
|
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <vector>
|
|
|
|
#include <faiss/impl/ScalarQuantizer.h>
|
|
|
|
TEST(ScalarQuantizer, RSQuantilesClamping) {
|
|
int d = 8;
|
|
int n = 100;
|
|
|
|
std::vector<float> x(d * n);
|
|
for (size_t i = 0; i < x.size(); i++) {
|
|
x[i] = static_cast<float>(i % 100);
|
|
}
|
|
|
|
faiss::ScalarQuantizer sq(d, faiss::ScalarQuantizer::QT_8bit);
|
|
sq.rangestat = faiss::ScalarQuantizer::RS_quantiles;
|
|
|
|
sq.rangestat_arg = 0.05f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
|
|
sq.rangestat_arg = 0.0f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
|
|
sq.rangestat_arg = -0.1f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
|
|
sq.rangestat_arg = 0.8f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
|
|
sq.rangestat_arg = 0.5f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
}
|
|
|
|
TEST(ScalarQuantizer, RSQuantilesOddSize) {
|
|
int d = 4;
|
|
int n = 5;
|
|
|
|
std::vector<float> x(d * n);
|
|
for (size_t i = 0; i < x.size(); i++) {
|
|
x[i] = static_cast<float>(i);
|
|
}
|
|
|
|
faiss::ScalarQuantizer sq(d, faiss::ScalarQuantizer::QT_8bit);
|
|
sq.rangestat = faiss::ScalarQuantizer::RS_quantiles;
|
|
|
|
sq.rangestat_arg = 0.4f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
|
|
sq.rangestat_arg = 0.5f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
|
|
sq.rangestat_arg = 0.6f;
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
}
|
|
|
|
TEST(ScalarQuantizer, RSQuantilesValidRange) {
|
|
int d = 8;
|
|
int n = 100;
|
|
|
|
std::vector<float> x(d * n);
|
|
for (size_t i = 0; i < x.size(); i++) {
|
|
x[i] = static_cast<float>(i);
|
|
}
|
|
|
|
faiss::ScalarQuantizer sq(d, faiss::ScalarQuantizer::QT_8bit);
|
|
sq.rangestat = faiss::ScalarQuantizer::RS_quantiles;
|
|
sq.rangestat_arg = 0.1f;
|
|
|
|
sq.train(n, x.data());
|
|
|
|
std::vector<uint8_t> codes(sq.code_size * n);
|
|
ASSERT_NO_THROW(sq.compute_codes(x.data(), codes.data(), n));
|
|
|
|
std::vector<float> decoded(d * n);
|
|
ASSERT_NO_THROW(sq.decode(codes.data(), decoded.data(), n));
|
|
}
|
|
|
|
TEST(ScalarQuantizer, RSQuantilesSmallDataset) {
|
|
int d = 2;
|
|
int n = 2;
|
|
|
|
std::vector<float> x = {1.0f, 2.0f, 3.0f, 4.0f};
|
|
|
|
faiss::ScalarQuantizer sq(d, faiss::ScalarQuantizer::QT_8bit);
|
|
sq.rangestat = faiss::ScalarQuantizer::RS_quantiles;
|
|
sq.rangestat_arg = 0.1f;
|
|
|
|
ASSERT_NO_THROW(sq.train(n, x.data()));
|
|
}
|