Remove dead TurboQuant projection state (#5618)

Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/5618

Remove the unused projection buffers and eager `init_projection` work from `ScalarQuantizer::TurboQuantRefine`. Quantizers construct the required projection from `trained` when used, so rebuilding it during training and deserialization only added an unbounded quadratic allocation and cubic QR operation.

Keep `seed` and `qjl_type` as the serialized configuration, enforce that the helper remains a small config-only struct, and cover both projection modes in deserialization tests.

___

Differential Revision: D119243693

fbshipit-source-id: 53f982b00dd849cbe88983ffd27809c2fa738a4c

faiss/impl/ScalarQuantizer.cpp

@@ -635,7 +635,6 @@ void ScalarQuantizer::train(size_t n, const float* x) {
             trained.push_back(seed_f[0]);
             trained.push_back(seed_f[1]);
             trained.push_back(static_cast<float>(turboq_refine.qjl_type));
-            turboq_refine.init_projection(d);
             break;
         }
         default:
@@ -643,24 +642,6 @@ void ScalarQuantizer::train(size_t n, const float* x) {
     }
 }
 
-void ScalarQuantizer::TurboQuantRefine::init_projection(size_t d) {
-    if (use_fwht()) {
-        padded_d = 1;
-        while (padded_d < d) {
-            padded_d <<= 1;
-        }
-        fwht_signs.resize(padded_d);
-        RandomGenerator rng(seed);
-        for (size_t i = 0; i < padded_d; i++) {
-            fwht_signs[i] = (rng.rand_int(2) == 0) ? 1.0f : -1.0f;
-        }
-    } else {
-        rr_matrix.resize(d * d);
-        float_randn(rr_matrix.data(), d * d, seed);
-        matrix_qr(static_cast<int>(d), static_cast<int>(d), rr_matrix.data());
-    }
-}
-
 ScalarQuantizer::SQuantizer* ScalarQuantizer::select_quantizer() const {
     // A SIMD level's factory returns nullptr when the dimension is
     // incompatible (e.g. AVX-512 needs d % 16 == 0); the dispatcher then falls

faiss/impl/ScalarQuantizer.h

@@ -177,18 +177,9 @@ struct ScalarQuantizer : Quantizer {
             return s;
         }
 
+        /// The selected projection is built from `trained` by the quantizer.
         uint8_t qjl_type = 0;
         uint64_t seed = 42;
-        size_t padded_d = 0;
-        std::vector<float> fwht_signs;
-        std::vector<float> rr_matrix;
-        size_t nb_bits_lo = 0;
-        size_t n_hi_dims = 0;
-
-        void init_projection(size_t d);
-        bool use_fwht() const {
-            return qjl_type == 0;
-        }
 
         struct DistanceComputer : SQDistanceComputer {
             virtual void configure(uint8_t qb, bool int_qjl) = 0;
@@ -199,6 +190,11 @@ struct ScalarQuantizer : Quantizer {
         };
     };
 
+    static_assert(
+            sizeof(TurboQuantRefine) <= 16,
+            "keep this a small config struct -- do not add projection buffers "
+            "here (T287092602)");
+
     TurboQuantRefine turboq_refine;
 
     SQDistanceComputer* get_distance_computer(

faiss/impl/index_read.cpp

@@ -1164,8 +1164,7 @@ void read_ScalarQuantizer(
         }
     }
 
-    // TurboQ full types: extract seed and qjl_type from trained,
-    // regenerate projection matrix.
+    // TurboQ full types: extract seed and qjl_type from trained.
     if (ScalarQuantizer::TurboQuantRefine::is_turboq_full(ivsc->qtype) &&
         ivsc->trained.size() >= 3) {
         size_t n = ivsc->trained.size();
@@ -1174,7 +1173,6 @@ void read_ScalarQuantizer(
         ivsc->turboq_refine.seed =
                 ScalarQuantizer::TurboQuantRefine::unpack_seed(
                         ivsc->trained[n - 3], ivsc->trained[n - 2]);
-        ivsc->turboq_refine.init_projection(ivsc->d);
     }
 }
 

tests/test_read_index_deserialize.cpp

@@ -4924,3 +4924,80 @@ TEST(IndexFlatPanoramaSafety, ReconstructRejectsOutOfRangeKey) {
     EXPECT_NO_THROW(index.reconstruct_n(0, 2, recons.data()));
     EXPECT_THROW(index.reconstruct_n(1, 2, recons.data()), FaissException);
 }
+
+namespace {
+
+constexpr size_t kTurboQFullDimension = 16;
+constexpr uint64_t kTurboQFullSeed = 0x0123456789abcdefULL;
+constexpr auto kTurboQFullQtype = ScalarQuantizer::QT_3bit_tq;
+
+// Total TurboQ bits/component for a QT_*_tq qtype (matches the mapping in
+// ScalarQuantizer::set_derived_sizes). Full TurboQ splits these into
+// (bits - 1) MSE bits + 1 QJL bit, so the reader-side trained-vector
+// centroid count is k = 1 << (bits - 1).
+constexpr size_t turboq_full_nb_bits(ScalarQuantizer::QuantizerType qt) {
+    return qt == ScalarQuantizer::QT_2bit_tq    ? 2
+            : qt == ScalarQuantizer::QT_3bit_tq ? 3
+            : qt == ScalarQuantizer::QT_4bit_tq ? 4
+            : qt == ScalarQuantizer::QT_5bit_tq ? 5
+                                                : 0;
+}
+
+// Derived rather than hardcoded so that changing kTurboQFullQtype (e.g. to
+// QT_4bit_tq) automatically resizes the payload; hardcoding kept the two
+// constants coupled and a mismatch would fail the read for an unrelated
+// reason (trained.size() != k + (k-1) + 3).
+//
+// `turboq_full_nb_bits` returns 0 for qtypes it does not recognise; guard
+// against that sentinel here so a future change to `kTurboQFullQtype` that
+// forgets to extend the mapping fails loudly at compile time instead of
+// tripping UB via `size_t{1} << (0 - 1)`.
+static_assert(
+        turboq_full_nb_bits(kTurboQFullQtype) != 0,
+        "kTurboQFullQtype must be a QT_*_tq qtype recognised by "
+        "turboq_full_nb_bits; extend the mapping when adding new tq qtypes.");
+constexpr size_t kTurboQFullCentroids = size_t{1}
+        << (turboq_full_nb_bits(kTurboQFullQtype) - 1);
+
+std::vector<uint8_t> make_turboq_full_sq_payload(
+        uint8_t qjl_type,
+        size_t d = kTurboQFullDimension) {
+    float seed[2];
+    ScalarQuantizer::TurboQuantRefine::pack_seed(kTurboQFullSeed, seed);
+    std::vector<float> trained(
+            kTurboQFullCentroids + (kTurboQFullCentroids - 1) + 3, 0.0f);
+    trained[trained.size() - 3] = seed[0];
+    trained[trained.size() - 2] = seed[1];
+    trained[trained.size() - 1] = static_cast<float>(qjl_type);
+
+    std::vector<uint8_t> buf;
+    push_fourcc(buf, "IxSQ");
+    push_index_header(buf, static_cast<int>(d), /*ntotal=*/0);
+    push_val<int>(buf, kTurboQFullQtype);
+    push_val<int>(buf, 0);      // rangestat
+    push_val<float>(buf, 0.0f); // rangestat_arg
+    push_val<size_t>(buf, d);
+    push_val<size_t>(buf, 0); // code_size is recomputed on read
+    push_vector<float>(buf, trained);
+    push_vector<uint8_t>(buf, {}); // codes
+    return buf;
+}
+
+} // namespace
+
+TEST(ReadIndexDeserialize, SQTurboQuantFullRecoversConfigFromTrained) {
+    for (const uint8_t qjl_type : {uint8_t{0}, uint8_t{2}}) {
+        SCOPED_TRACE(testing::Message() << "qjl_type = " << int(qjl_type));
+
+        VectorIOReader reader;
+        reader.data = make_turboq_full_sq_payload(qjl_type);
+        std::unique_ptr<Index> index;
+        ASSERT_NO_THROW(index = read_index_up(&reader));
+
+        const auto* sq_index = dynamic_cast<IndexScalarQuantizer*>(index.get());
+        ASSERT_NE(sq_index, nullptr);
+        EXPECT_EQ(sq_index->sq.d, kTurboQFullDimension);
+        EXPECT_EQ(sq_index->sq.turboq_refine.qjl_type, qjl_type);
+        EXPECT_EQ(sq_index->sq.turboq_refine.seed, kTurboQFullSeed);
+    }
+}