Add ProductAdditiveQuantizer (#2286)

Summary:
This diff added ProductAdditiveQuantizer.

A Simple Algo description:

1. Divide the vector space into several orthogonal sub-spaces, just like PQ does.
2. Quantize each sub-space by an independent additive quantizer.

Usage:

Construct a ProductAdditiveQuantizer object:
- `d`: dimensionality of the input vectors
- `nsplits`: number of sub-spaces divided into
- `Msub`: `M` of each additive quantizer
- `nbits`: `nbits` of each additive quantizer

```python
d = 128
nsplits = 2
Msub = 4
nbits = 8
plsq = faiss.ProductLocalSearchQuantizer(d, nsplits, Msub, nbits)
prq = faiss.ProductResidualQuantizer(d, nsplits, Msub, nbits)
```

Pull Request resolved: https://github.com/facebookresearch/faiss/pull/2286

Test Plan:
```

buck test //faiss/tests/:test_local_search_quantizer -- TestProductLocalSearchQuantizer

buck test //faiss/tests/:test_residual_quantizer -- TestProductResidualQuantizer

```

Reviewed By: alexanderguzhva

Differential Revision: D35907702

Pulled By: mdouze

fbshipit-source-id: 7428a196e6bd323569caa585c57281dd70e547b1
This commit is contained in:
Check Deng
2022-05-05 15:14:07 -07:00
committed by Facebook GitHub Bot
parent 88f550deee
commit 9b1982262a
10 changed files with 680 additions and 13 deletions
+46
View File
@@ -1130,3 +1130,49 @@ class TestCrossCodebookComputations(unittest.TestCase):
rq.use_beam_LUT = 1
codes_new = rq.compute_codes(xb)
np.testing.assert_array_equal(codes_ref_residuals, codes_new)
class TestProductResidualQuantizer(unittest.TestCase):
def test_codec(self):
"""check that the error is in the same ballpark as PQ."""
ds = datasets.SyntheticDataset(64, 3000, 3000, 0)
xt = ds.get_train()
xb = ds.get_database()
nsplits = 2
Msub = 2
nbits = 4
prq = faiss.ProductResidualQuantizer(ds.d, nsplits, Msub, nbits)
prq.train(xt)
err_prq = eval_codec(prq, xb)
pq = faiss.ProductQuantizer(ds.d, nsplits * Msub, nbits)
pq.train(xt)
err_pq = eval_codec(pq, xb)
print(err_prq, err_pq)
self.assertLess(err_prq, err_pq)
def test_with_rq(self):
"""compare with RQ when nsplits = 1"""
ds = datasets.SyntheticDataset(32, 3000, 3000, 0)
xt = ds.get_train()
xb = ds.get_database()
M = 4
nbits = 4
prq = faiss.ProductResidualQuantizer(ds.d, 1, M, nbits)
prq.train(xt)
err_prq = eval_codec(prq, xb)
rq = faiss.ResidualQuantizer(ds.d, M, nbits)
rq.train(xt)
err_rq = eval_codec(rq, xb)
print(err_prq, err_rq)
self.assertEqual(err_prq, err_rq)