Support SVSVamana as IVF coarse quantizer via index_factory (#5175)

Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/5175

Enable IndexSVSVamana to be used as a coarse quantizer for Faiss IVF indexes, following the same pattern as IndexHNSWFlat and IndexNSGFlat.

Changes:

1. index_factory.cpp: Add `IVF<nlist>_SVSVamana<degree>` regex pattern in `parse_coarse_quantizer()` and explicit `get_trains_alone()` return for IndexSVSVamana (training mode 2: k-means on flat index, then centroids added to the graph). Both guarded by `#ifdef FAISS_ENABLE_SVS`.

2. IndexSVSVamana.h/.cpp: Add `reconstruct()` support with a `stored_vectors` buffer. The SVS runtime API does not expose vector retrieval, so we keep a copy of added vectors. This is needed for IVF residual computation (`by_residual = true`) and `check_compatible_for_merge()`. When used as a coarse quantizer, the buffer holds only `nlist` centroids (trivial memory cost). Also clears `stored_vectors` on `reset()`.

Example factory string: `"IVF65536_SVSVamana32,SQ8"`

Reviewed By: junjieqi

Differential Revision: D103568560

fbshipit-source-id: aa67f076ff061bb5afde80d14cf5a8d847f99eae

faiss/impl/index_read.cpp

@@ -2522,13 +2522,14 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
     }
 #ifdef FAISS_ENABLE_SVS
     else if (
-            h == fourcc("ILVQ") || h == fourcc("ISVL") || h == fourcc("ISVD")) {
+            h == fourcc("ILVQ") || h == fourcc("ISVL") || h == fourcc("ISVD") ||
+            h == fourcc("ISV2")) {
         std::unique_ptr<IndexSVSVamana> svs;
         if (h == fourcc("ILVQ")) {
             svs = std::make_unique<IndexSVSVamanaLVQ>();
         } else if (h == fourcc("ISVL")) {
             svs = std::make_unique<IndexSVSVamanaLeanVec>();
-        } else if (h == fourcc("ISVD")) {
+        } else if (h == fourcc("ISVD") || h == fourcc("ISV2")) {
             svs = std::make_unique<IndexSVSVamana>();
         }
 
@@ -2580,6 +2581,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
                 leanvec->deserialize_training_data(is);
             }
         }
+        if (h == fourcc("ISV2")) {
+            READVECTOR(svs->stored_vectors);
+        } else {
+            svs->stored_vectors_valid = false;
+        }
         idx = std::move(svs);
     } else if (h == fourcc("ISVF")) {
         auto svs = std::make_unique<IndexSVSFlat>();

faiss/impl/index_write.cpp

@@ -1027,6 +1027,8 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
             h = fourcc("ILVQ"); // LVQ
         } else if (lean != nullptr) {
             h = fourcc("ISVL"); // LeanVec
+        } else if (svs->stored_vectors_valid && !svs->stored_vectors.empty()) {
+            h = fourcc("ISV2"); // uncompressed + stored_vectors
         } else {
             h = fourcc("ISVD"); // uncompressed
         }
@@ -1069,6 +1071,10 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
                 os.flush();
             }
         }
+
+        if (h == fourcc("ISV2")) {
+            WRITEVECTOR(svs->stored_vectors);
+        }
     } else if (
             const IndexSVSFlat* svs = dynamic_cast<const IndexSVSFlat*>(idx)) {
         uint32_t h = fourcc("ISVF");

faiss/index_factory.cpp

@@ -130,6 +130,11 @@ char get_trains_alone(const Index* coarse_quantizer) {
     if (dynamic_cast<const IndexHNSWFlat*>(coarse_quantizer)) {
         return 2;
     }
+#ifdef FAISS_ENABLE_SVS
+    if (dynamic_cast<const IndexSVSVamana*>(coarse_quantizer)) {
+        return 2;
+    }
+#endif
     return 2; // for complicated indexes, we assume they can't be used as a
               // kmeans index
 }
@@ -299,6 +304,13 @@ Index* parse_coarse_quantizer(
         int R = std::stoi(sm[2]);
         return new IndexNSGFlat(d, R, mt);
     }
+#ifdef FAISS_ENABLE_SVS
+    if (match("IVF([0-9]+[kM]?)_SVSVamana([0-9]*)")) {
+        nlist = parse_nlist(sm[1].str());
+        int degree = sm[2].length() > 0 ? std::stoi(sm[2]) : 32;
+        return new IndexSVSVamana(d, degree, mt);
+    }
+#endif
     if (match("IVF([0-9]+[kM]?)\\(Index([0-9])\\)")) {
         nlist = parse_nlist(sm[1].str());
         int no = std::stoi(sm[2].str());

faiss/svs/IndexSVSVamana.cpp

@@ -31,6 +31,7 @@
 #include <svs/runtime/vamana_index.h>
 
 #include <cstddef>
+#include <cstring>
 #include <numeric>
 #include <span>
 #include <type_traits>
@@ -114,13 +115,32 @@ void IndexSVSVamana::add(idx_t n, const float* x) {
     if (!status.ok()) {
         FAISS_THROW_MSG(status.message());
     }
+
+    if (stored_vectors_valid) {
+        size_t prev = static_cast<size_t>(ntotal) * d;
+        stored_vectors.resize(prev + static_cast<size_t>(n) * d);
+        std::memcpy(stored_vectors.data() + prev, x, sizeof(float) * n * d);
+    }
     ntotal += n;
 }
 
+void IndexSVSVamana::reconstruct(idx_t key, float* recons) const {
+    FAISS_THROW_IF_NOT_MSG(
+            key >= 0 && key < ntotal,
+            "IndexSVSVamana::reconstruct: key out of range");
+    FAISS_THROW_IF_NOT_MSG(
+            stored_vectors_valid && !stored_vectors.empty(),
+            "IndexSVSVamana::reconstruct: stored_vectors unavailable "
+            "(invalidated by remove_ids or not restored after deserialization)");
+    std::memcpy(recons, stored_vectors.data() + key * d, sizeof(float) * d);
+}
+
 void IndexSVSVamana::reset() {
     if (impl) {
         impl->reset();
     }
+    stored_vectors.clear();
+    stored_vectors_valid = true;
     is_trained = false;
     ntotal = 0;
 }
@@ -189,6 +209,8 @@ size_t IndexSVSVamana::remove_ids(const IDSelector& sel) {
     size_t removed = 0;
     auto Status = impl->remove_selected(&removed, id_filter);
     ntotal -= removed;
+    stored_vectors.clear();
+    stored_vectors_valid = false;
     return removed;
 }
 

faiss/svs/IndexSVSVamana.h

@@ -31,6 +31,7 @@
 
 #include <iostream>
 #include <type_traits>
+#include <vector>
 
 namespace faiss {
 
@@ -111,6 +112,8 @@ struct IndexSVSVamana : Index {
 
     void add(idx_t n, const float* x) override;
 
+    void reconstruct(idx_t key, float* recons) const override;
+
     void search(
             idx_t n,
             const float* x,
@@ -137,6 +140,12 @@ struct IndexSVSVamana : Index {
     /* The actual SVS implementation */
     svs_runtime::DynamicVamanaIndex* impl{nullptr};
 
+    // The SVS runtime API does not expose vector retrieval, so we keep a copy
+    // of added vectors to support reconstruct(). When used as a coarse
+    // quantizer this holds only nlist centroids.
+    std::vector<float> stored_vectors;
+    bool stored_vectors_valid{true};
+
    protected:
     /* Initializes the implementation*/
     virtual void create_impl();