diff --git a/contrib/factory_tools.py b/contrib/factory_tools.py index 1470d3433..f70a85e00 100644 --- a/contrib/factory_tools.py +++ b/contrib/factory_tools.py @@ -25,7 +25,7 @@ def get_code_size(d, indexkey): if mo: return get_code_size(d, mo.group(1)) - mo = re.match("IMI\\d+x2,(.*)$", indexkey) + mo = re.match("IMI\\d+x\\d+,(.*)$", indexkey) if mo: return get_code_size(d, mo.group(1)) @@ -50,6 +50,11 @@ def get_code_size(d, indexkey): M = int(mo.group(1)) return d * 4 + M * 2 * 4 # roughly + mo = re.match("HNSW(\\d+),(.*)$", indexkey) + if mo: + M = int(mo.group(1)) + return get_code_size(d, mo.group(2)) + M * 2 * 4 # roughly + if indexkey == "SQ8": return d elif indexkey == "SQ4": @@ -127,6 +132,9 @@ def reverse_index_factory(index): return prefix + ",Flat" if isinstance(index, faiss.IndexIVFScalarQuantizer): return prefix + "," + sq_names[index.sq.qtype] + # IndexIVFPQR subclasses IndexIVFPQ, so it must be checked first. + if isinstance(index, faiss.IndexIVFPQR): + return prefix + f",PQ{index.pq.M}+{index.refine_pq.M}" if isinstance(index, faiss.IndexIVFPQ): return prefix + f",PQ{index.pq.M}x{index.pq.nbits}" if isinstance(index, faiss.IndexIVFPQFastScan): @@ -152,7 +160,12 @@ def reverse_index_factory(index): return f"{prefix},{reverse_index_factory(index.index)}" elif isinstance(index, faiss.IndexHNSW): - return f"HNSW{get_hnsw_M(index)}" + prefix = f"HNSW{get_hnsw_M(index)}" + storage = faiss.downcast_index(index.storage) + # flat storage is implicit in the bare HNSW factory string + if storage is None or isinstance(storage, faiss.IndexFlat): + return prefix + return f"{prefix},{reverse_index_factory(storage)}" elif isinstance(index, faiss.IndexRefine): return ( diff --git a/tests/test_contrib.py b/tests/test_contrib.py index 6d363684c..7d5faeda1 100644 --- a/tests/test_contrib.py +++ b/tests/test_contrib.py @@ -1021,6 +1021,56 @@ class TestFactoryTools(unittest.TestCase): code_size = factory_tools.get_code_size(d, factory_str) self.assertEqual(code_size, d * 4 + 16 * 2 * 4) + def test_hnsw_storage_reverse_index_factory(self): + d = 128 + # flat storage stays implicit, everything else must be spelled out + cases = { + "HNSW32,Flat": "HNSW32", + "HNSW32,SQ8": "HNSW32,SQ8", + "HNSW32,SQfp16": "HNSW32,SQfp16", + "HNSW32,PQ16x8": "HNSW32,PQ16x8", + } + for key, expected in cases.items(): + index = faiss.index_factory(d, key) + factory_str = factory_tools.reverse_index_factory(index) + self.assertEqual(factory_str, expected) + # the string must rebuild the same index class + rebuilt = faiss.index_factory(d, factory_str) + self.assertEqual(type(rebuilt), type(index)) + + def test_get_code_size_hnsw_non_flat_storage(self): + d = 128 + graph = 32 * 2 * 4 + self.assertEqual( + factory_tools.get_code_size(d, "HNSW32,SQ8"), d + graph + ) + self.assertEqual( + factory_tools.get_code_size(d, "HNSW32,SQfp16"), d * 2 + graph + ) + self.assertEqual( + factory_tools.get_code_size(d, "HNSW32,PQ16x8"), 16 + graph + ) + + def test_get_code_size_imi(self): + # the coarse quantizer adds no per-vector bytes + d = 64 + self.assertEqual(factory_tools.get_code_size(d, "IMI2x10,PQ16"), 16) + self.assertEqual(factory_tools.get_code_size(d, "IMI2x5,Flat"), d * 4) + # round-trip: get_code_size must parse what reverse emits + index = faiss.index_factory(d, "IMI2x5,PQ8") + factory_str = factory_tools.reverse_index_factory(index) + self.assertEqual(factory_str, "IMI2x5,PQ8x8") + self.assertEqual(factory_tools.get_code_size(d, factory_str), 8) + + def test_ivfpqr_reverse_index_factory(self): + d = 128 + index = faiss.index_factory(d, "IVF64,PQ8+16") + factory_str = factory_tools.reverse_index_factory(index) + self.assertEqual(factory_str, "IVF64,PQ8+16") + rebuilt = faiss.index_factory(d, factory_str) + self.assertEqual(type(rebuilt), type(index)) + self.assertEqual(factory_tools.get_code_size(d, factory_str), 8 + 16) + def test_rabitq_reverse_index_factory(self): d = 64 quantizer = faiss.IndexFlatL2(d)