mirror of
https://github.com/facebookresearch/faiss.git
synced 2026-10-12 02:13:36 +00:00
Tia Lim
0f0f728166
Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/5349 # this is a no-op change Ran `black -l 80` over all tracked Python files under `fbcode/faiss/`, using line length 80 to match faiss's documented coding style (`CONTRIBUTING.md`: "80 character line length (both for C++ and Python)"). 172 files were reformatted; 28 were already compliant. This is a pure-formatting change. `black` runs its AST-equivalence safety check on every file, so no identifiers, string/numeric values, operators, or control flow were altered — only whitespace, indentation, line wrapping, blank lines, trailing commas, and string-quote normalization (single to double quotes). `fbcode/faiss/` is intentionally excluded from Meta's standard Python autoformatter (`arc f` / `arc lint -a` report no formatter for these paths) because it mirrors to public GitHub, so the Python had drifted out of any consistent style. The C++ is already clang-format-clean (0 of 807 files change), so this effort is Python-only. This diff brings the Python to canonical `black` formatting. Excluded `demos/index_pq_flat_separate_codes_from_codebook.py`: it contains a Jupyter `!rm` shell-magic line and is not valid Python, so `black` cannot parse it. This is the base of a 3-diff stack: (1) this black reformat; (2) `.flake8` alignment (max-line-length 80 + black-compatible ignores E203/W503); (3) genuine lint fixes (F401/F403/F841/B011). Reviewed By: mnorris11 Differential Revision: D109736099 fbshipit-source-id: 9a054635766ed11aa3760a788395babd3d198abf
230 lines
6.1 KiB
Python
230 lines
6.1 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
"""Binary indexes (de)serialization"""
|
|
|
|
from __future__ import (
|
|
absolute_import,
|
|
division,
|
|
print_function,
|
|
unicode_literals,
|
|
)
|
|
|
|
import numpy as np
|
|
import unittest
|
|
import faiss
|
|
|
|
|
|
def make_binary_dataset(d, nb, nt, nq):
|
|
assert d % 8 == 0
|
|
x = np.random.randint(256, size=(nb + nq + nt, int(d / 8))).astype("uint8")
|
|
return x[:nt], x[nt:-nq], x[-nq:]
|
|
|
|
|
|
class TestBinaryFlat(unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
d = 32
|
|
nt = 0
|
|
nb = 1500
|
|
nq = 500
|
|
|
|
(_, self.xb, self.xq) = make_binary_dataset(d, nb, nt, nq)
|
|
|
|
def test_flat(self):
|
|
d = self.xq.shape[1] * 8
|
|
|
|
index = faiss.IndexBinaryFlat(d)
|
|
index.add(self.xb)
|
|
D, I = index.search(self.xq, 3)
|
|
index2 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index)
|
|
)
|
|
|
|
D2, I2 = index2.search(self.xq, 3)
|
|
|
|
assert (I2 == I).all()
|
|
assert (D2 == D).all()
|
|
|
|
# Verify deserialized index is serializable again
|
|
index3 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index2)
|
|
)
|
|
D3, I3 = index3.search(self.xq, 3)
|
|
assert (I3 == I).all()
|
|
assert (D3 == D).all()
|
|
|
|
|
|
class TestBinaryIVF(unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
d = 32
|
|
nt = 200
|
|
nb = 1500
|
|
nq = 500
|
|
|
|
(self.xt, self.xb, self.xq) = make_binary_dataset(d, nb, nt, nq)
|
|
|
|
def test_ivf_flat(self):
|
|
d = self.xq.shape[1] * 8
|
|
|
|
quantizer = faiss.IndexBinaryFlat(d)
|
|
index = faiss.IndexBinaryIVF(quantizer, d, 8)
|
|
index.cp.min_points_per_centroid = 5 # quiet warning
|
|
index.nprobe = 4
|
|
index.train(self.xt)
|
|
index.add(self.xb)
|
|
D, I = index.search(self.xq, 3)
|
|
|
|
index2 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index)
|
|
)
|
|
|
|
D2, I2 = index2.search(self.xq, 3)
|
|
|
|
assert (I2 == I).all()
|
|
assert (D2 == D).all()
|
|
|
|
# Verify deserialized index is serializable again
|
|
index3 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index2)
|
|
)
|
|
D3, I3 = index3.search(self.xq, 3)
|
|
assert (I3 == I).all()
|
|
assert (D3 == D).all()
|
|
|
|
|
|
class TestObjectOwnership(unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
d = 32
|
|
nt = 200
|
|
nb = 1500
|
|
nq = 500
|
|
|
|
(self.xt, self.xb, self.xq) = make_binary_dataset(d, nb, nt, nq)
|
|
|
|
def test_read_index_ownership(self):
|
|
d = self.xq.shape[1] * 8
|
|
|
|
index = faiss.IndexBinaryFlat(d)
|
|
index.add(self.xb)
|
|
|
|
# this is the output of read_index_binary (==> checks ownership)
|
|
index2 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index)
|
|
)
|
|
|
|
assert index2.thisown
|
|
|
|
# Verify deserialized index is serializable again
|
|
index3 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index2)
|
|
)
|
|
assert index3.thisown
|
|
|
|
|
|
class TestBinaryFromFloat(unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
d = 32
|
|
nt = 200
|
|
nb = 1500
|
|
nq = 500
|
|
|
|
(self.xt, self.xb, self.xq) = make_binary_dataset(d, nb, nt, nq)
|
|
|
|
def test_binary_from_float(self):
|
|
d = self.xq.shape[1] * 8
|
|
|
|
float_index = faiss.IndexHNSWFlat(d, 16)
|
|
index = faiss.IndexBinaryFromFloat(float_index)
|
|
index.add(self.xb)
|
|
D, I = index.search(self.xq, 3)
|
|
|
|
index2 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index)
|
|
)
|
|
D2, I2 = index2.search(self.xq, 3)
|
|
|
|
assert (I2 == I).all()
|
|
assert (D2 == D).all()
|
|
|
|
# Verify deserialized index is serializable again
|
|
index3 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index2)
|
|
)
|
|
D3, I3 = index3.search(self.xq, 3)
|
|
assert (I3 == I).all()
|
|
assert (D3 == D).all()
|
|
|
|
|
|
class TestBinaryHNSW(unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
d = 32
|
|
nt = 200
|
|
nb = 1500
|
|
nq = 500
|
|
|
|
(self.xt, self.xb, self.xq) = make_binary_dataset(d, nb, nt, nq)
|
|
|
|
def test_hnsw(self):
|
|
d = self.xq.shape[1] * 8
|
|
|
|
index = faiss.IndexBinaryHNSW(d)
|
|
index.add(self.xb)
|
|
D, I = index.search(self.xq, 3)
|
|
|
|
index2 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index)
|
|
)
|
|
|
|
D2, I2 = index2.search(self.xq, 3)
|
|
|
|
assert (I2 == I).all()
|
|
assert (D2 == D).all()
|
|
|
|
# Verify deserialized index is serializable again
|
|
index3 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index2)
|
|
)
|
|
D3, I3 = index3.search(self.xq, 3)
|
|
assert (I3 == I).all()
|
|
assert (D3 == D).all()
|
|
|
|
def test_ivf_hnsw(self):
|
|
d = self.xq.shape[1] * 8
|
|
|
|
quantizer = faiss.IndexBinaryHNSW(d)
|
|
index = faiss.IndexBinaryIVF(quantizer, d, 8)
|
|
index.cp.min_points_per_centroid = 5 # quiet warning
|
|
index.nprobe = 4
|
|
index.train(self.xt)
|
|
index.add(self.xb)
|
|
D, I = index.search(self.xq, 3)
|
|
|
|
index2 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index)
|
|
)
|
|
|
|
D2, I2 = index2.search(self.xq, 3)
|
|
|
|
assert (I2 == I).all()
|
|
assert (D2 == D).all()
|
|
|
|
# Verify deserialized index is serializable again
|
|
index3 = faiss.deserialize_index_binary(
|
|
faiss.serialize_index_binary(index2)
|
|
)
|
|
D3, I3 = index3.search(self.xq, 3)
|
|
assert (I3 == I).all()
|
|
assert (D3 == D).all()
|