Re-factor factory string parsing (#2134)

Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/2134

The old parsing was very complex and grew out of hand.
this diff just uses regex parsing.

Reviewed By: wickedfoo

Differential Revision: D32759110

fbshipit-source-id: 243029bba8a7fe70c71323f5edc7e2ce4e669757

.clang-format

@@ -15,8 +15,8 @@ AllowShortLoopsOnASingleLine: false
 AlwaysBreakAfterReturnType: None
 AlwaysBreakBeforeMultilineStrings: true
 AlwaysBreakTemplateDeclarations: true
-BinPackArguments: false
-BinPackParameters: false
+BinPackArguments: false  # at some point, set this to true
+BinPackParameters: false # at some point, set this to true
 BraceWrapping:
   AfterClass:      false
   AfterControlStatement: false

faiss/index_factory.h

@@ -19,6 +19,9 @@ Index* index_factory(
         const char* description,
         MetricType metric = METRIC_L2);
 
+/// set to > 0 to get more logs from index_factory
+FAISS_API extern int index_factory_verbose;
+
 IndexBinary* index_binary_factory(int d, const char* description);
 
 } // namespace faiss

tests/test_factory.py

@@ -61,14 +61,16 @@ class TestFactory(unittest.TestCase):
         index = faiss.index_factory(12, "HNSW32_PQ4")
         assert index.storage.sa_code_size() == 4
 
+
     def test_factory_HNSW_newstyle(self):
         index = faiss.index_factory(12, "HNSW32,Flat")
         assert index.storage.sa_code_size() == 12 * 4
         index = faiss.index_factory(12, "HNSW32,SQ8", faiss.METRIC_INNER_PRODUCT)
         assert index.storage.sa_code_size() == 12
         assert index.metric_type == faiss.METRIC_INNER_PRODUCT
-        index = faiss.index_factory(12, "HNSW32,PQ4")
+        index = faiss.index_factory(12, "HNSW,PQ4")
         assert index.storage.sa_code_size() == 4
+        self.assertEqual(index.hnsw.nb_neighbors(1), 32)
         index = faiss.index_factory(12, "HNSW32,PQ4np")
         indexpq = faiss.downcast_index(index.storage)
         assert not indexpq.do_polysemous_training
@@ -110,6 +112,8 @@ class TestFactory(unittest.TestCase):
         self.assertEqual(index.bbs, 64)
         self.assertEqual(index.nlist, 50)
         self.assertTrue(index.cp.spherical)
+        index = faiss.index_factory(56, "IVF50,PQ28x4fsr_64")
+        self.assertTrue(index.by_residual)
         index = faiss.index_factory(56, "PQ28x4fs,RFlat")
         self.assertEqual(index.k_factor, 1.0)
 
@@ -187,3 +191,70 @@ class TestVTDowncast(unittest.TestCase):
 
         itqt = faiss.downcast_VectorTransform(codec.chain.at(0))
         itqt.pca_then_itq
+
+
+# tests after re-factoring
+class TestFactoryV2(unittest.TestCase):
+
+    def test_refine(self):
+        index = faiss.index_factory(123, "Flat,RFlat")
+        index.k_factor
+
+    def test_refine_2(self):
+        index = faiss.index_factory(123, "LSHrt,Refine(Flat)")
+        index1 = faiss.downcast_index(index.base_index)
+        self.assertTrue(index1.rotate_data)
+        self.assertTrue(index1.train_thresholds)
+
+    def test_pre_transform(self):
+        index = faiss.index_factory(123, "PCAR100,L2Norm,PCAW50,LSHr")
+        self.assertTrue(index.chain.size() == 3)
+
+    def test_ivf(self):
+        index = faiss.index_factory(123, "IVF456,Flat")
+        self.assertEqual(index.__class__, faiss.IndexIVFFlat)
+
+    def test_idmap(self):
+        index = faiss.index_factory(123, "Flat,IDMap")
+        self.assertEqual(index.__class__, faiss.IndexIDMap)
+
+    def test_ivf_hnsw(self):
+        index = faiss.index_factory(123, "IVF100_HNSW,Flat")
+        quantizer = faiss.downcast_index(index.quantizer)
+        self.assertEqual(quantizer.hnsw.nb_neighbors(1), 32)
+
+    def test_ivf_parent(self):
+        index = faiss.index_factory(123, "IVF100(LSHr),Flat")
+        quantizer = faiss.downcast_index(index.quantizer)
+        self.assertEqual(quantizer.__class__, faiss.IndexLSH)
+
+
+class TestAdditive(unittest.TestCase):
+
+    def test_rcq(self):
+        index = faiss.index_factory(12, "IVF256(RCQ2x4),RQ3x4")
+        self.assertEqual(
+            faiss.downcast_index(index.quantizer).__class__,
+            faiss.ResidualCoarseQuantizer
+        )
+
+    def test_rq3(self):
+        index = faiss.index_factory(5, "RQ2x16_3x8_6x4")
+
+        np.testing.assert_array_equal(
+            faiss.vector_to_array(index.rq.nbits),
+            np.array([16, 16, 8, 8, 8, 4, 4, 4, 4, 4, 4])
+        )
+
+    def test_norm(self):
+        index = faiss.index_factory(5, "RQ8x8_Nqint8")
+        self.assertEqual(
+            index.rq.search_type,
+            faiss.AdditiveQuantizer.ST_norm_qint8)
+
+
+class TestSpectralHash(unittest.TestCase):
+
+    def test_sh(self):
+        index = faiss.index_factory(123, "IVF256,ITQ64,SH1.2")
+        self.assertEqual(index.__class__, faiss.IndexIVFSpectralHash)