diff --git a/benchs/bench_all_ivf/bench_all_ivf.py b/benchs/bench_all_ivf/bench_all_ivf.py index da6402b76..b5472c90d 100644 --- a/benchs/bench_all_ivf/bench_all_ivf.py +++ b/benchs/bench_all_ivf/bench_all_ivf.py @@ -438,7 +438,10 @@ def main(): default=["check_files"], choices=["train", "add", "search", "check_files"], nargs="+", - help="what to do (check_files means decide depending on which index files exist)", + help=( + "what to do (check_files means decide depending on which " + "index files exist)" + ), ) group = parser.add_argument_group("dataset options") @@ -567,7 +570,10 @@ def main(): "--autotune_max", default=[], nargs="*", - help='set max value for autotune variables format "var:val" (exclusive)', + help=( + 'set max value for autotune variables format "var:val" ' + "(exclusive)" + ), ) aa( "--autotune_range", diff --git a/benchs/bench_all_ivf/make_groundtruth.py b/benchs/bench_all_ivf/make_groundtruth.py index 8f325817e..2f519087d 100644 --- a/benchs/bench_all_ivf/make_groundtruth.py +++ b/benchs/bench_all_ivf/make_groundtruth.py @@ -5,7 +5,7 @@ import logging -# https://stackoverflow.com/questions/7016056/python-logging-not-outputting-anything +# https://stackoverflow.com/questions/7016056/python-logging-not-outputting-anything # noqa: E501 logging.basicConfig() logger = logging.getLogger("faiss.contrib.exhaustive_search") logger.setLevel(logging.INFO) diff --git a/benchs/bench_fw/benchmark.py b/benchs/bench_fw/benchmark.py index 5712e30c6..4ba3fdf0e 100644 --- a/benchs/bench_fw/benchmark.py +++ b/benchs/bench_fw/benchmark.py @@ -93,7 +93,8 @@ def optimizer(op, search, cost_metric, perf_metric): (max_perf, min_cost) = op.predict_bounds(key) if not op.is_pareto_optimal(max_perf, min_cost): logger.info( - f"{cno=:4d} {str(parameters):50}: SKIP, {max_perf=:.3f} {min_cost=:.3f}", + f"{cno=:4d} {str(parameters):50}: SKIP, " + f"{max_perf=:.3f} {min_cost=:.3f}", ) continue @@ -150,7 +151,8 @@ def get_range_search_metric_function(range_metric, D, R): else: real_radius = mean([radius_from, radius_to]) logger.info( - f"range_search_metric_function {radius_from=} {radius_to=} {real_radius=} {score=}" + f"range_search_metric_function {radius_from=} " + f"{radius_to=} {real_radius=} {score=}" ) aradius.append(real_radius) ascore.append(score) @@ -518,7 +520,8 @@ class SearchOperator(IndexOperator): if flat_desc is None: flat_desc = self.get_flat_desc() self.build_index_wrapper(flat_desc) - # TODO(kuarora): Consider moving gt results(gt_knn_D, gt_knn_I) to the index as there can be multiple ground truths. + # TODO(kuarora): Consider moving gt results(gt_knn_D, gt_knn_I) to + # the index as there can be multiple ground truths. ( self.gt_knn_D, self.gt_knn_I, @@ -734,7 +737,8 @@ class SearchOperator(IndexOperator): ref_index_desc = self.get_desc(knn_desc.range_ref_index_desc) if ref_index_desc is None: raise ValueError( - f"{knn_desc.get_name()}: Unknown range index {knn_desc.range_ref_index_desc}" + f"{knn_desc.get_name()}: Unknown range index " + f"{knn_desc.range_ref_index_desc}" ) if ref_index_desc.range_metrics is None: raise ValueError( @@ -779,7 +783,9 @@ class SearchOperator(IndexOperator): metric_key=metric_key, radius=knn_desc.radius, gt_radius=gt_radius, - range_search_metric_function=range_search_metric_function, + range_search_metric_function=( + range_search_metric_function + ), gt_rsm=gt_rsm, query_dataset=knn_desc.query_dataset, ) @@ -1112,7 +1118,8 @@ class Benchmark: reconstruct, range, ) -> ExecutionOperator: - # all operators are created, as ground truth are always created in benchmarking + # all operators are created, as ground truth are always created in + # benchmarking train_op = TrainOperator( num_threads=self.num_threads, distance_metric=self.distance_metric ) diff --git a/benchs/bench_fw/descriptors.py b/benchs/bench_fw/descriptors.py index 9dbedd70c..8138820c6 100644 --- a/benchs/bench_fw/descriptors.py +++ b/benchs/bench_fw/descriptors.py @@ -347,7 +347,8 @@ class IndexDescriptor(IndexBaseDescriptor): ) return self.flat_desc_name - # alias is used to refer when index is uploaded to blobstore and referred again + # alias is used to refer when index is uploaded to blobstore and + # referred again def alias(self, benchmark_io: BenchmarkIO): if hasattr(benchmark_io, "bucket"): return IndexDescriptor( diff --git a/benchs/bench_fw/index.py b/benchs/bench_fw/index.py index 19740f829..aabd41984 100644 --- a/benchs/bench_fw/index.py +++ b/benchs/bench_fw/index.py @@ -19,7 +19,7 @@ from faiss.contrib.evaluation import ( # @manual=//faiss/contrib:faiss_contrib knn_intersection_measure, OperatingPointsWithRanges, ) -from faiss.contrib.factory_tools import ( # @manual=//faiss/contrib:faiss_contrib +from faiss.contrib.factory_tools import ( # @manual=//faiss/contrib:faiss_contrib # noqa: E501 reverse_index_factory, ) from faiss.contrib.ivf_tools import ( # @manual=//faiss/contrib:faiss_contrib @@ -206,7 +206,9 @@ class IndexBase: def transform(self, vectors): transformed_vectors = DatasetDescriptor( - tablename=f"{vectors.get_filename()}{self.get_codec_name()}transform.npy" + tablename=( + f"{vectors.get_filename()}{self.get_codec_name()}transform.npy" + ) ) if not self.io.file_exist(transformed_vectors.tablename): codec = self.get_codec() diff --git a/benchs/bench_fw/optimize.py b/benchs/bench_fw/optimize.py index d4992b26d..43c0b57b5 100644 --- a/benchs/bench_fw/optimize.py +++ b/benchs/bench_fw/optimize.py @@ -9,7 +9,7 @@ from typing import Dict, List, Tuple import faiss # @manual=//faiss/python:pyfaiss -# from faiss.contrib.evaluation import ( # @manual=//faiss/contrib:faiss_contrib +# from faiss.contrib.evaluation import ( # @manual=//faiss/contrib:faiss_contrib # noqa: E501 # OperatingPoints, # ) @@ -162,7 +162,10 @@ class Optimizer: ) ivf_descs.append( IndexDescriptorClassic( - factory=f"{pretransform}IVF{nlist}({quantizer_desc.factory}),{fine_ivf}", + factory=( + f"{pretransform}IVF{nlist}" + f"({quantizer_desc.factory}),{fine_ivf}" + ), construction_params=construction_params, ) ) diff --git a/benchs/bench_fw_codecs.py b/benchs/bench_fw_codecs.py index 504e0a5ca..f6e53ee1c 100644 --- a/benchs/bench_fw_codecs.py +++ b/benchs/bench_fw_codecs.py @@ -147,9 +147,12 @@ def run_local(rp): training_size=training_size, search_params=search_params, ) - for factory, construction_params, training_size, search_params in factory_factory( - d - ) + for ( + factory, + construction_params, + training_size, + search_params, + ) in factory_factory(d) ], k=1, distance_metric=distance_metric, diff --git a/benchs/bench_ivfflat_cuvs.py b/benchs/bench_ivfflat_cuvs.py index f890b738a..8b4ecdd75 100644 --- a/benchs/bench_ivfflat_cuvs.py +++ b/benchs/bench_ivfflat_cuvs.py @@ -121,7 +121,9 @@ if args.bm_train: xt, d, args.nlist, False ) print( - "Method: IVFFlat, Operation: TRAIN, dim: %d, nlist %d, numTrain: %d, classical GPU train time: %.3f milliseconds, cuVS enabled GPU train time: %.3f milliseconds" + "Method: IVFFlat, Operation: TRAIN, dim: %d, nlist %d, numTrain: %d, " + "classical GPU train time: %.3f milliseconds, " + "cuVS enabled GPU train time: %.3f milliseconds" % (d, args.nlist, nt, classical_gpu_train_time, cuvs_gpu_train_time) ) @@ -149,7 +151,9 @@ if args.bm_add: cuvs_gpu_add_time = bench_add_milliseconds(xb, quantizer, True) classical_gpu_add_time = bench_add_milliseconds(xb, quantizer, False) print( - "Method: IVFFlat, Operation: ADD, dim: %d, nlist %d, numAdd: %d, classical GPU add time: %.3f milliseconds, cuVS enabled GPU add time: %.3f milliseconds" + "Method: IVFFlat, Operation: ADD, dim: %d, nlist %d, numAdd: %d, " + "classical GPU add time: %.3f milliseconds, " + "cuVS enabled GPU add time: %.3f milliseconds" % (d, args.nlist, nb, classical_gpu_add_time, cuvs_gpu_add_time) ) @@ -179,7 +183,10 @@ if args.bm_search: idx_cpu, xq, args.nprobe, args.k, False ) print( - "Method: IVFFlat, Operation: SEARCH, dim: %d, nlist: %d, numVecs: %d, numQuery: %d, nprobe: %d, k: %d, classical GPU search time: %.3f milliseconds, cuVS enabled GPU search time: %.3f milliseconds" + "Method: IVFFlat, Operation: SEARCH, dim: %d, nlist: %d, numVecs: %d, " + "numQuery: %d, nprobe: %d, k: %d, " + "classical GPU search time: %.3f milliseconds, " + "cuVS enabled GPU search time: %.3f milliseconds" % ( d, args.nlist, diff --git a/benchs/bench_ivfpq_cuvs.py b/benchs/bench_ivfpq_cuvs.py index ed9086c01..38aad02af 100644 --- a/benchs/bench_ivfpq_cuvs.py +++ b/benchs/bench_ivfpq_cuvs.py @@ -141,7 +141,9 @@ if args.bm_train: cuvs_gpu_train_time = bench_train_milliseconds(xt, True) classical_gpu_train_time = bench_train_milliseconds(xt, False) print( - "TRAIN, dim: %d, nlist %d, numTrain: %d, classical GPU train time: %.3f milliseconds, cuVS enabled GPU train time: %.3f milliseconds" + "TRAIN, dim: %d, nlist %d, numTrain: %d, " + "classical GPU train time: %.3f milliseconds, " + "cuVS enabled GPU train time: %.3f milliseconds" % (d, nlist, nt, classical_gpu_train_time, cuvs_gpu_train_time) ) @@ -167,7 +169,9 @@ if args.bm_add: cuvs_gpu_add_time = bench_add_milliseconds(xb, index_cpu, True) classical_gpu_add_time = bench_add_milliseconds(xb, index_cpu, False) print( - "ADD, dim: %d, nlist %d, numAdd: %d, classical GPU add time: %.3f milliseconds, cuVS enabled GPU add time: %.3f milliseconds" + "ADD, dim: %d, nlist %d, numAdd: %d, " + "classical GPU add time: %.3f milliseconds, " + "cuVS enabled GPU add time: %.3f milliseconds" % (d, nlist, nb, classical_gpu_add_time, cuvs_gpu_add_time) ) @@ -202,6 +206,8 @@ if args.bm_search: classical_gpu_indices, classical_gpu_search_time ) print( - "SEARCH, dim: %d, nlist: %d, numVecs: %d, numQuery: %d, nprobe: %d, k: %d, classical GPU qps: %.3f, cuVS enabled GPU qps: %.3f" + "SEARCH, dim: %d, nlist: %d, numVecs: %d, numQuery: %d, " + "nprobe: %d, k: %d, " + "classical GPU qps: %.3f, cuVS enabled GPU qps: %.3f" % (d, nlist, nb, nq, args.nprobe, args.k, classical_qps, cuvs_qps) ) diff --git a/benchs/bench_pq_transposed_centroid_table.py b/benchs/bench_pq_transposed_centroid_table.py index b586f759d..93c6921fc 100644 --- a/benchs/bench_pq_transposed_centroid_table.py +++ b/benchs/bench_pq_transposed_centroid_table.py @@ -46,7 +46,8 @@ def test_bigann10m(index_file, index_parameters): index_ivf, vec_transform = unwind_index_ivf(index) print( - "params regular transp_centroids regular R@1 R@10 R@100" + "params " + " regular transp_centroids regular R@1 R@10 R@100" ) for index_parameter in index_parameters: ps.set_index_parameters(index, index_parameter) @@ -92,7 +93,10 @@ if __name__ == "__main__": faiss.contrib.datasets.dataset_basedir = "/home/aguzhva/ANN_SIFT1B/" # represents OPQ32_128,IVF65536_HNSW32,PQ32 index - index_file_1 = "/home/aguzhva/ANN_SIFT1B/run_tests/bench_ivf/indexes/hnsw32/.faissindex" + index_file_1 = ( + "/home/aguzhva/ANN_SIFT1B/run_tests/bench_ivf/indexes/" + "hnsw32/.faissindex" + ) nprobe_values = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] quantizer_efsearch_values = [4, 8, 16, 32, 64, 128, 256, 512] @@ -194,9 +198,9 @@ if __name__ == "__main__": quantizer_nprobe = random.choice(quantizer_nprobe_values) ht = random.choice(ht_values) index_parameters_2.append( - "nprobe={},quantizer_k_factor_rf={},quantizer_nprobe={},ht={}".format( - nprobe, quantizer_k_factor_rf, quantizer_nprobe, ht - ) + ( + "nprobe={},quantizer_k_factor_rf={},quantizer_nprobe={},ht={}" + ).format(nprobe, quantizer_k_factor_rf, quantizer_nprobe, ht) ) test_bigann10m(index_file_2, index_parameters_2) diff --git a/benchs/bench_refine_panorama.py b/benchs/bench_refine_panorama.py index 0b9651f35..efecd73cb 100644 --- a/benchs/bench_refine_panorama.py +++ b/benchs/bench_refine_panorama.py @@ -73,7 +73,8 @@ kfactor_list = [1, 8, 64, 256, 1024] print(f"Benchmark on GIST1M with base '{factory}', k={k}, nq={nq}") print( - "nprobe k_factor recall_flat qps_flat recall_pano qps_pano dims_scanned(%) speedup(x)" + "nprobe k_factor recall_flat qps_flat recall_pano qps_pano " + "dims_scanned(%) speedup(x)" ) faiss.omp_set_num_threads(1) diff --git a/contrib/big_batch_search.py b/contrib/big_batch_search.py index af3c32c01..0b33f6c5d 100644 --- a/contrib/big_batch_search.py +++ b/contrib/big_batch_search.py @@ -284,7 +284,8 @@ def big_batch_search( checkpointing (only for threaded > 1): checkpoint: file where the checkpoints are stored - checkpoint_freq: when to perform checkpointing. Should be a multiple of threaded + checkpoint_freq: when to perform checkpointing. Should be a multiple of + threaded start_list, end_list: process only a subset of invlists """ diff --git a/contrib/clustering.py b/contrib/clustering.py index f0bbd4e17..9154561ef 100644 --- a/contrib/clustering.py +++ b/contrib/clustering.py @@ -41,7 +41,8 @@ def two_level_clustering( log = print if verbose else print_nop log( - f"2-level clustering of {xt.shape} nb 1st level clusters = {nc1} total {nc2}" + f"2-level clustering of {xt.shape} nb 1st level clusters = {nc1} " + f"total {nc2}" ) log("perform coarse training") @@ -58,7 +59,8 @@ def two_level_clustering( _, assign1 = km.assign(xt) bc = np.bincount(assign1, minlength=nc1) log( - f"done in {time.time() - t0:.2f} s. Sizes of clusters {min(bc)}-{max(bc)}" + f"done in {time.time() - t0:.2f} s. " + f"Sizes of clusters {min(bc)}-{max(bc)}" ) o = assign1.argsort() del km @@ -81,7 +83,8 @@ def two_level_clustering( for c1 in range(nc1): nc2 = int(all_nc2[c1]) log( - f"[{time.time() - t0:.2f} s] training sub-cluster {c1}/{nc1} nc2={nc2}\r", + f"[{time.time() - t0:.2f} s] training sub-cluster " + f"{c1}/{nc1} nc2={nc2}\r", end="", flush=True, ) @@ -171,7 +174,8 @@ def balanced_assignment_with_penalties( n = len(x) nopt = n / nc # targed bin sizes - # we assign to the top-maxk clusters. The final assignment will pick among these clusters. + # we assign to the top-maxk clusters. The final assignment will pick + # among these clusters. full_d2, full_assign = faiss.knn(x, centroids, maxk) # scalar penalty for each cluster @@ -184,7 +188,8 @@ def balanced_assignment_with_penalties( a0 = full_d2_penalized.argmin(axis=1) assign = np.take_along_axis(full_assign, a0[:, None], axis=1).ravel() binsizes = np.bincount(assign, minlength=nc) - # print(imbalance_factor(nc, assign), mse, int(binsizes.min()), int(binsizes.max())) + # print(imbalance_factor(nc, assign), mse, int(binsizes.min()), + # int(binsizes.max())) penalties *= (binsizes / nopt) ** alpha stats = dict( diff --git a/contrib/datasets.py b/contrib/datasets.py index 5360f2d07..abcc259af 100644 --- a/contrib/datasets.py +++ b/contrib/datasets.py @@ -291,7 +291,8 @@ class DatasetGlove(Dataset): if not loc: loc = dataset_basedir + "glove/glove-100-angular.hdf5" self.glove_h5py = h5py.File(loc, "r") - # IP and L2 are equivalent in this case, but it is traditionally seen as an IP dataset + # IP and L2 are equivalent in this case, but it is traditionally + # seen as an IP dataset self.metric = "IP" self.d, self.nt = 100, 0 self.nb = self.glove_h5py["train"].shape[0] @@ -379,9 +380,14 @@ class DatasetGIST1M(Dataset): class DatasetDINO10B(Dataset): """ Data from https://dl.fbaipublicfiles.com/large_objects/dino_vitl_10B/ - The dataset contains 10 billion 1024-d vectors extracted from image patches from the YFCC100M dataset, using a Dino-ViT-L 16 model (facebook/dinov3-vitl16-pretrain-lvd1689m). - The dataset is sharded in multiple chunked .bvecs files. Downloading instructions can be obtained with "wget https://dl.fbaipublicfiles.com/large_objects/dino_vitl_10B/README.md". - Supported sizes : 100k 200k 500k 1M ... 5B 10B listed in supported_nbs (see __init__). + The dataset contains 10 billion 1024-d vectors extracted from image + patches from the YFCC100M dataset, using a Dino-ViT-L 16 model + (facebook/dinov3-vitl16-pretrain-lvd1689m). + The dataset is sharded in multiple chunked .bvecs files. Downloading + instructions can be obtained with + "wget https://dl.fbaipublicfiles.com/large_objects/dino_vitl_10B/README.md". + Supported sizes : 100k 200k 500k 1M ... 5B 10B listed in supported_nbs + (see __init__). """ def __init__(self, nb, ignore_supported=False): @@ -406,11 +412,13 @@ class DatasetDINO10B(Dataset): ] if nb not in supported_nbs and not ignore_supported: raise ValueError( - f"Unsupported dataset size: {nb}, supported values are: {supported_nbs}" + f"Unsupported dataset size: {nb}, supported values are: " + f"{supported_nbs}" ) if not os.path.exists(dataset_basedir): raise ValueError( - f"Provided dataset base directory does not exist: {dataset_basedir}" + "Provided dataset base directory does not exist: " + f"{dataset_basedir}" ) self.basedir = dataset_basedir + "dino_vitl_10B/" self.indexdir = self.basedir + "chunked_base_10B" @@ -418,9 +426,10 @@ class DatasetDINO10B(Dataset): self.indexdir ), f"Index path should exist, check your dataset path: {self.indexdir}" self.queriesdir = self.basedir + "queries_clean.bvecs" - assert os.path.exists( - self.queriesdir - ), f"Queries path should exist as dataset size {nb} is supported: {self.queriesdir}" + assert os.path.exists(self.queriesdir), ( + f"Queries path should exist as dataset size {nb} is supported: " + f"{self.queriesdir}" + ) self.gtsdir = ( self.basedir + "gts/" @@ -445,7 +454,10 @@ class DatasetDINO10B(Dataset): """Get training query vectors as a single array""" if maxtrain is None or maxtrain > 10_000_000: raise NotImplementedError( - "The training set is potentially too large to fit in RAM (400 GB of data). Please use train_iterator or use maxtrain parameter below 10_000_000 to get the first maxtrain training vectors." + "The training set is potentially too large to fit in RAM " + "(400 GB of data). Please use train_iterator or use " + "maxtrain parameter below 10_000_000 to get the first " + "maxtrain training vectors." ) return sanitize(bvecs_mmap(self.train_queriesdir)[:maxtrain]) @@ -453,7 +465,9 @@ class DatasetDINO10B(Dataset): """Get all database vectors as a single array""" if self.nb > 10_000_000: raise NotImplementedError( - "The dataset is potentially too large to fit in RAM. Please use database_iterator or use a dataset size equal to or below 10_000_000." + "The dataset is potentially too large to fit in RAM. " + "Please use database_iterator or use a dataset size equal " + "to or below 10_000_000." ) else: return sanitize( @@ -461,7 +475,8 @@ class DatasetDINO10B(Dataset): ) def database_iterator(self, bs=10_000): - """Iterator over the database of size nb, corresponding to the first nb vectors in the .bvecs file""" + """Iterator over the database of size nb, corresponding to the + first nb vectors in the .bvecs file""" total_read = 0 for batch in bvecs_iter_chunked(self.indexdir, batch_size=bs): if total_read + batch.shape[0] > self.nb: diff --git a/contrib/evaluation.py b/contrib/evaluation.py index 6ed969b81..d7fb91319 100644 --- a/contrib/evaluation.py +++ b/contrib/evaluation.py @@ -312,11 +312,12 @@ class OperatingPoints: def compare_keys(self, k1, k2): """return -1 if k1 > k2, 1 if k2 > k1, 0 otherwise""" - raise NotImplemented + raise NotImplementedError def do_nothing_key(self): - """parameters to say we do nothing, takes 0 time and has 0 performance""" - raise NotImplemented + """parameters to say we do nothing, takes 0 time and has 0 + performance""" + raise NotImplementedError def is_pareto_optimal(self, perf_new, t_new): for _, perf, t in self.operating_points: @@ -496,5 +497,6 @@ class RepeatTimer: return np.std(self.times) * 1000 if len(self.times) > 1 else 0.0 def nruns(self): - """effective number of runs (may be lower than runs - warmup due to timeout)""" + """effective number of runs (may be lower than runs - warmup due + to timeout)""" return len(self.times) diff --git a/contrib/exhaustive_search.py b/contrib/exhaustive_search.py index f3dea6bf7..7fe4f0471 100644 --- a/contrib/exhaustive_search.py +++ b/contrib/exhaustive_search.py @@ -124,7 +124,8 @@ def range_search_gpu(xq, r2, index_gpu, index_cpu, gpu_k=1024): combiner.D_remain = sp(D_remain) combiner.lim_remain = sp(lim_remain.view("int64")) combiner.I_remain = sp(I_remain) - # combiner.set_range_result(sp(mask), sp(lim_remain.view("int64")), sp(D_remain), sp(I_remain)) + # combiner.set_range_result(sp(mask), + # sp(lim_remain.view("int64")), sp(D_remain), sp(I_remain)) L_res = np.empty(nq + 1, dtype="int64") combiner.compute_sizes(sp(L_res)) nres = L_res[-1] diff --git a/contrib/factory_tools.py b/contrib/factory_tools.py index d2b8e4389..1470d3433 100644 --- a/contrib/factory_tools.py +++ b/contrib/factory_tools.py @@ -155,7 +155,10 @@ def reverse_index_factory(index): return f"HNSW{get_hnsw_M(index)}" elif isinstance(index, faiss.IndexRefine): - return f"{reverse_index_factory(index.base_index)},Refine({reverse_index_factory(index.refine_index)})" + return ( + f"{reverse_index_factory(index.base_index)}," + f"Refine({reverse_index_factory(index.refine_index)})" + ) elif isinstance(index, faiss.IndexPQFastScan): return f"PQ{index.pq.M}x{index.pq.nbits}fs" diff --git a/contrib/rpc.py b/contrib/rpc.py index 7fb044d31..de6781ddf 100755 --- a/contrib/rpc.py +++ b/contrib/rpc.py @@ -144,7 +144,8 @@ class Server: ret = f(*args) except Exception as e: # due to a bug (in mod_python?), ServerException cannot be - # unpickled, so send the string and make the exception on the client side + # unpickled, so send the string and make the exception on the + # client side # st=ServerException( # "".join(traceback.format_tb(sys.exc_info()[2]))+ diff --git a/contrib/vecs_io.py b/contrib/vecs_io.py index a80e5a556..ab0978101 100644 --- a/contrib/vecs_io.py +++ b/contrib/vecs_io.py @@ -84,8 +84,9 @@ def bvecs_iter(filepath, batch_size=100_000): def bvecs_iter_chunked(chunk_folder, batch_size=100_000): """ Memory-mapped iterator over chunked .bvecs files. - Iterates through all chunk files in order (chunk_0000.bvecs, chunk_0001.bvecs, etc.) - and yields batches of vectors, handling cases where batches span multiple files. + Iterates through all chunk files in order (chunk_0000.bvecs, + chunk_0001.bvecs, etc.) and yields batches of vectors, handling cases + where batches span multiple files. Args: chunk_folder: path to folder containing chunk_XXXX.bvecs files @@ -127,7 +128,8 @@ def bvecs_iter_chunked(chunk_folder, batch_size=100_000): if sorted(chunk_numbers) != expected_chunks: missing = set(expected_chunks) - set(chunk_numbers) raise ValueError( - f"Gap detected in chunk sequence! Missing chunks: {sorted(missing)}\n" + f"Gap detected in chunk sequence! Missing chunks: " + f"{sorted(missing)}\n" f"Found chunks: {sorted(chunk_numbers)}\n" f"Expected continuous sequence from 0 to {len(chunk_numbers)-1}" ) diff --git a/demos/demo_auto_tune.py b/demos/demo_auto_tune.py index f36fd2ed9..50ba46f7a 100755 --- a/demos/demo_auto_tune.py +++ b/demos/demo_auto_tune.py @@ -118,9 +118,10 @@ use_gpu = False if use_gpu: # if this fails, it means that the GPU version was not compiled - assert ( - faiss.StandardGpuResources - ), "Faiss was not compiled with GPU support, or loading _swigfaiss_gpu.so failed" + assert faiss.StandardGpuResources, ( + "Faiss was not compiled with GPU support, " + "or loading _swigfaiss_gpu.so failed" + ) res = faiss.StandardGpuResources() dev_no = 0 @@ -178,7 +179,9 @@ for index_key in keys_to_test: pyplot.grid() for i2, opi2 in op_per_key: plot_OperatingPoints(opi2, crit.nq, label=i2, marker="o") - # plot_OperatingPoints(op, crit.nq, label = 'best', marker = 'o', color = 'r') + # plot_OperatingPoints( + # op, crit.nq, label = 'best', marker = 'o', color = 'r' + # ) pyplot.legend(loc=2) fig.savefig("tmp/demo_auto_tune.png") diff --git a/demos/offline_ivf/create_sharded_ssnpp_files.py b/demos/offline_ivf/create_sharded_ssnpp_files.py index 68593ccfa..cd03ec02e 100644 --- a/demos/offline_ivf/create_sharded_ssnpp_files.py +++ b/demos/offline_ivf/create_sharded_ssnpp_files.py @@ -11,7 +11,7 @@ import os def xbin_mmap(fname, dtype, maxn=-1): """ Code from - https://github.com/harsha-simhadri/big-ann-benchmarks/blob/main/benchmark/dataset_io.py#L94 + https://github.com/harsha-simhadri/big-ann-benchmarks/blob/main/benchmark/dataset_io.py#L94 # noqa: E501 mmap the competition file format for a given type of items """ n, d = map(int, np.fromfile(fname, dtype="uint32", count=2)) @@ -49,7 +49,10 @@ if __name__ == "__main__": "--filepath", dest="filepath", type=str, - default="/datasets01/big-ann-challenge-data/FB_ssnpp/FB_ssnpp_database.u8bin", + default=( + "/datasets01/big-ann-challenge-data/FB_ssnpp/" + "FB_ssnpp_database.u8bin" + ), help="path of 1B ssnpp database vectors' original file", ) parser.add_argument( diff --git a/demos/offline_ivf/offline_ivf.py b/demos/offline_ivf/offline_ivf.py index 8eac6b7f2..23b5754ed 100644 --- a/demos/offline_ivf/offline_ivf.py +++ b/demos/offline_ivf/offline_ivf.py @@ -80,9 +80,9 @@ class OfflineIVF: self.xq_index_shard_prefix = ( f"{xq_output_dir}/{self.index_factory_fn}.shard_" ) - self.index_file = ( # TODO: added back temporarily for evaluate, handle name of non-sharded index file and remove. - f"{xb_output_dir}/{self.index_factory_fn}.faissindex" - ) + # TODO: added back temporarily for evaluate, handle name of + # non-sharded index file and remove. + self.index_file = f"{xb_output_dir}/{self.index_factory_fn}.faissindex" self.xq_index_file = ( f"{xq_output_dir}/{self.index_factory_fn}.faissindex" ) @@ -140,7 +140,8 @@ class OfflineIVF: def input_stats(self): """ - Trains the index using a subsample of the first chunk of data in the database and saves it in the template file (with no vectors added). + Trains the index using a subsample of the first chunk of data in the + database and saves it in the template file (with no vectors added). """ xb_sample = self.xb_ds.get_first_n(self.training_sample, np.float32) logging.info(f"input shape: {xb_sample.shape}") @@ -193,7 +194,8 @@ class OfflineIVF: def train_index(self): """ - Trains the index using a subsample of the first chunk of data in the database and saves it in the template file (with no vectors added). + Trains the index using a subsample of the first chunk of data in the + database and saves it in the template file (with no vectors added). """ assert not os.path.exists(self.index_template_file), ( "The train command has been ran, the index template file already" @@ -652,7 +654,10 @@ class OfflineIVF: # quantizer = faiss.index_cpu_to_all_gpus(index_ivf.quantizer) for i in range(0, self.xq_ds.size, self.xq_bs): Ifn = f"{self.knn_dir}/I{(i):010}_{self.knn_output_file_suffix}" - Dfn = f"{self.knn_dir}/D_approx{(i):010}_{self.knn_output_file_suffix}" + Dfn = ( + f"{self.knn_dir}/D_approx{(i):010}_" + f"{self.knn_output_file_suffix}" + ) CPfn = f"{self.knn_dir}/CP{(i):010}_{self.knn_output_file_suffix}" if slurm_job_id: @@ -853,7 +858,10 @@ class OfflineIVF: logging.info("search results...") index_ivf.nprobe = self.nprobe for i in range(0, self.xq_ds.size, self.xq_bs): - Ifn = f"{self.knn_dir}/I{i:010}_{self.index_factory_fn}_np{self.nprobe}.npy" + Ifn = ( + f"{self.knn_dir}/I{i:010}_{self.index_factory_fn}_" + f"np{self.nprobe}.npy" + ) assert os.path.exists(Ifn) assert os.path.getsize(Ifn) > 0, f"The file {Ifn} is empty." logging.info(Ifn) @@ -863,7 +871,10 @@ class OfflineIVF: assert I.shape[0] == min(self.xq_bs, self.xq_ds.size - i) assert np.all(I[:, 1] >= 0) - Dfn = f"{self.knn_dir}/D_approx{i:010}_{self.index_factory_fn}_np{self.nprobe}.npy" + Dfn = ( + f"{self.knn_dir}/D_approx{i:010}_{self.index_factory_fn}_" + f"np{self.nprobe}.npy" + ) assert os.path.exists(Dfn) assert os.path.getsize(Dfn) > 0, f"The file {Dfn} is empty." logging.info(Dfn) @@ -872,14 +883,10 @@ class OfflineIVF: xq = next(self.xq_ds.iterate(i, SMALL_DATA_SAMPLE, np.float32)) D_online, I_online = index.search(xq, self.k) - assert ( - np.where(I[:SMALL_DATA_SAMPLE] == I_online)[0].size - / (self.k * SMALL_DATA_SAMPLE) - > 0.95 - ), ( - "the ratio is" - f" {np.where(I[:SMALL_DATA_SAMPLE] == I_online)[0].size / (self.k * SMALL_DATA_SAMPLE)}" + ratio = np.where(I[:SMALL_DATA_SAMPLE] == I_online)[0].size / ( + self.k * SMALL_DATA_SAMPLE ) + assert ratio > 0.95, f"the ratio is {ratio}" assert np.allclose( D[:SMALL_DATA_SAMPLE].sum(axis=1), D_online.sum(axis=1), diff --git a/demos/offline_ivf/run.py b/demos/offline_ivf/run.py index 7b5648784..5e872c075 100644 --- a/demos/offline_ivf/run.py +++ b/demos/offline_ivf/run.py @@ -16,7 +16,8 @@ import submitit def join_lists_in_dict(poss: List[str]) -> List[str]: """ - Joins two lists of prod and non-prod values, checking if the prod value is already included. + Joins two lists of prod and non-prod values, checking if the prod value is + already included. If there is no non-prod list, it returns the prod list. """ if "non-prod" in poss.keys(): @@ -40,9 +41,11 @@ def main( def process_options_and_run_jobs(args: argparse.Namespace) -> None: """ - If "--cluster_run", it launches an array of jobs to the cluster using the submitit library for all the index strings. In - the case of evaluate, it launches a job for each index string and nprobe pair. Otherwise, it launches a single job - that is ran locally with the prod values for index string and nprobe. + If "--cluster_run", it launches an array of jobs to the cluster using the + submitit library for all the index strings. In the case of evaluate, it + launches a job for each index string and nprobe pair. Otherwise, it + launches a single job that is ran locally with the prod values for index + string and nprobe. """ cfg = load_config(args.config) diff --git a/demos/offline_ivf/tests/test_iterate_input.py b/demos/offline_ivf/tests/test_iterate_input.py index c5be0db76..372c349c8 100644 --- a/demos/offline_ivf/tests/test_iterate_input.py +++ b/demos/offline_ivf/tests/test_iterate_input.py @@ -81,7 +81,8 @@ class TestUtilsMethods(unittest.TestCase): def test_get_vs_iterate(self) -> None: """ - Loads vectors with iterator and get, and checks that they match, non-aligned by file size case. + Loads vectors with iterator and get, and checks that they match, + non-aligned by file size case. """ with tempfile.TemporaryDirectory() as tmpdirname: data_creator = TestDataCreator( @@ -105,7 +106,8 @@ class TestUtilsMethods(unittest.TestCase): def test_iterate_back(self) -> None: """ - Loads vectors with iterator and get, and checks that they match, non-aligned by file size case. + Loads vectors with iterator and get, and checks that they match, + non-aligned by file size case. """ with tempfile.TemporaryDirectory() as tmpdirname: data_creator = TestDataCreator( diff --git a/demos/offline_ivf/tests/test_offline_ivf.py b/demos/offline_ivf/tests/test_offline_ivf.py index 4d866fd5e..7958a4d9f 100644 --- a/demos/offline_ivf/tests/test_offline_ivf.py +++ b/demos/offline_ivf/tests/test_offline_ivf.py @@ -37,7 +37,8 @@ A_INDEX_OPQ_FILES: List[str] = [ class TestOIVF(unittest.TestCase): """ - Unit tests for OIVF. Some of these unit tests first copy the required test data objects and puts them in the tempdir created by the context manager. + Unit tests for OIVF. Some of these unit tests first copy the required test + data objects and puts them in the tempdir created by the context manager. """ def assert_file_exists(self, filepath: str) -> None: @@ -46,7 +47,8 @@ class TestOIVF(unittest.TestCase): def test_consistency_check(self) -> None: """ - Test the OIVF consistency check step, that it throws if no other steps have been ran. + Test the OIVF consistency check step, that it throws if no other steps + have been ran. """ with tempfile.TemporaryDirectory() as tmpdirname: data_creator = TestDataCreator( @@ -69,7 +71,8 @@ class TestOIVF(unittest.TestCase): def test_train_index(self) -> None: """ - Test the OIVF train index step, that it correctly produces the empty.faissindex template file. + Test the OIVF train index step, that it correctly produces the + empty.faissindex template file. """ with tempfile.TemporaryDirectory() as tmpdirname: data_creator = TestDataCreator( @@ -98,7 +101,8 @@ class TestOIVF(unittest.TestCase): def test_index_shard_equal_file_sizes(self) -> None: """ - Test the case where the shard size is a divisor of the database size and it is equal to the first file size. + Test the case where the shard size is a divisor of the database size + and it is equal to the first file size. """ with tempfile.TemporaryDirectory() as tmpdirname: @@ -142,7 +146,8 @@ class TestOIVF(unittest.TestCase): def test_index_shard_unequal_file_sizes(self) -> None: """ - Test the case where the shard size is not a divisor of the database size and is greater than the first file size. + Test the case where the shard size is not a divisor of the database + size and is greater than the first file size. """ with tempfile.TemporaryDirectory() as tmpdirname: file_sizes = [20000, 15001, 13990] @@ -182,7 +187,8 @@ class TestOIVF(unittest.TestCase): def test_search(self) -> None: """ - Test search step using test data objects to bypass dependencies on previous steps. + Test search step using test data objects to bypass dependencies on + previous steps. """ with tempfile.TemporaryDirectory() as tmpdirname: num_files = 3 @@ -220,7 +226,8 @@ class TestOIVF(unittest.TestCase): def test_evaluate_without_margin(self) -> None: """ - Test evaluate step using test data objects, no margin evaluation, single index. + Test evaluate step using test data objects, no margin evaluation, + single index. """ with tempfile.TemporaryDirectory() as tmpdirname: data_creator = TestDataCreator( @@ -255,7 +262,8 @@ class TestOIVF(unittest.TestCase): def test_evaluate_without_margin_OPQ(self) -> None: """ - Test evaluate step using test data objects, no margin evaluation, single index. + Test evaluate step using test data objects, no margin evaluation, + single index. """ with tempfile.TemporaryDirectory() as tmpdirname: data_creator = TestDataCreator( diff --git a/demos/offline_ivf/tests/testing_utils.py b/demos/offline_ivf/tests/testing_utils.py index 0d2dc492b..427c35e68 100644 --- a/demos/offline_ivf/tests/testing_utils.py +++ b/demos/offline_ivf/tests/testing_utils.py @@ -157,9 +157,11 @@ class TestDataCreator: def _create_data_files(self, name_of_file="my_data") -> List[str]: """ - Creates a dataset "my_test_data" with number of files (num_files), using padding in the files - name. If self.with_queries is True, it adds an extra dataset "my_queries_data" with the same number of files - as the "my_test_data". The default name for embeddings files is "my_data" + .npy. + Creates a dataset "my_test_data" with number of files (num_files), + using padding in the files name. If self.with_queries is True, it adds + an extra dataset "my_queries_data" with the same number of files as the + "my_test_data". The default name for embeddings files is "my_data" + + .npy. """ filenames = [] for i, file_size in enumerate(self.file_sizes): diff --git a/demos/offline_ivf/utils.py b/demos/offline_ivf/utils.py index c1d91c970..3e59ebd51 100644 --- a/demos/offline_ivf/utils.py +++ b/demos/offline_ivf/utils.py @@ -37,10 +37,13 @@ def margin(sample, idx_a, idx_b, D_a_b, D_a, D_b, k, k_extract, threshold): idx_a - (np,) - query vector ids in xa idx_b - (np,) - query vector ids in xb D_a_b - (np,) - pairwise distances between xa[idx_a] and xb[idx_b] - D_a - (np, k) - distances between vectors xa[idx_a] and corresponding nearest neighbours in xb - D_b - (np, k) - distances between vectors xb[idx_b] and corresponding nearest neighbours in xa + D_a - (np, k) - distances between vectors xa[idx_a] and corresponding + nearest neighbours in xb + D_b - (np, k) - distances between vectors xb[idx_b] and corresponding + nearest neighbours in xa k - k nearest neighbours used for margin - k_extract - number of nearest neighbours of each query in xb we consider for margin calculation and filtering + k_extract - number of nearest neighbours of each query in xb we consider + for margin calculation and filtering threshold - margin threshold """ @@ -76,7 +79,8 @@ def get_intersection_cardinality_frequencies( I: np.ndarray, I_gt: np.ndarray ) -> Dict[int, int]: """ - Computes the frequencies for the cardinalities of the intersection of neighbour indices. + Computes the frequencies for the cardinalities of the intersection of + neighbour indices. """ nq = I.shape[0] res = [] diff --git a/faiss/python/__init__.py b/faiss/python/__init__.py index a2fab0f60..132aaf6cf 100644 --- a/faiss/python/__init__.py +++ b/faiss/python/__init__.py @@ -20,8 +20,8 @@ def _preload_gpu_libs(): These libs ship in nvidia-*-cuNN / libcuvs-cuNN wheels, off ld.so's search path, so we dlopen them before the SWIG extension loads libfaiss.so. Gated on the `faiss._gpu_build` marker (CMake writes it only for GPU builds); the - `_cuvs_build` marker selects the CUDA 13 cuVS variant (else CUDA 12) and adds - the cuVS stack. Missing wheels raise a fix-it. + `_cuvs_build` marker selects the CUDA 13 cuVS variant (else CUDA 12) and + adds the cuVS stack. Missing wheels raise a fix-it. """ try: from . import _gpu_build # noqa: F401 @@ -37,8 +37,8 @@ def _preload_gpu_libs(): ctypes.CDLL(path, mode=ctypes.RTLD_GLOBAL) except OSError as e: raise RuntimeError( - f"faiss-gpu: failed to load {os.path.basename(path)} from {path} " - f"— corrupt or incomplete nvidia CUDA wheel?" + f"faiss-gpu: failed to load {os.path.basename(path)} from " + f"{path} — corrupt or incomplete nvidia CUDA wheel?" ) from e # faiss-gpu-cuvs wheels carry the `_cuvs_build` marker and are built against @@ -82,7 +82,8 @@ def _preload_gpu_libs(): # CUDA 12 per-component layout: each nvidia-*-cu12 wheel exposes an # importable module whose lib/ dir holds the .so. def _nvidia_lib_dir(import_name, pip_spec): - """Return the lib/ dir of an nvidia-*-cu12 wheel, or raise a fix-it.""" + """Return the lib/ dir of an nvidia-*-cu12 wheel, or raise a + fix-it.""" try: mod = __import__( "nvidia." + import_name, fromlist=[import_name] @@ -93,7 +94,8 @@ def _preload_gpu_libs(): f"faiss-gpu installed but {pip_name} is missing — " f"pip install '{pip_spec}'" ) from e - # __path__[0] not __file__: PEP 420 namespace pkgs have __file__ = None. + # __path__[0] not __file__: PEP 420 namespace pkgs have + # __file__ = None. return os.path.join(mod.__path__[0], "lib") _cudart = _nvidia_lib_dir( @@ -109,14 +111,16 @@ def _preload_gpu_libs(): # faiss-gpu-cuvs wheels also need the cuVS stack. Delegate to RAPIDS' # load_library() (loads each .so RTLD_GLOBAL + its CUDA deps); order - # rmm -> raft -> cuvs makes every symbol global before the SWIG extension loads. + # rmm -> raft -> cuvs makes every symbol global before the SWIG extension + # loads. try: import libcuvs import libraft import librmm except ImportError as e: raise RuntimeError( - "faiss-gpu-cuvs installed but the cuVS runtime wheels are missing — " + "faiss-gpu-cuvs installed but the cuVS runtime wheels are " + "missing — " "pip install 'libcuvs-cu13>=26.06,<27' " "--extra-index-url https://pypi.nvidia.com" ) from e diff --git a/faiss/python/array_conversions.py b/faiss/python/array_conversions.py index 45c9c13e9..e53f32686 100644 --- a/faiss/python/array_conversions.py +++ b/faiss/python/array_conversions.py @@ -46,12 +46,16 @@ def _make_deprecated_swig_class(deprecated_name, base_name): base_class = globals()[base_name] def new_meth(cls, *args, **kwargs): - msg = f"The class faiss.{deprecated_name} is deprecated in favour of faiss.{base_name}!" + msg = ( + f"The class faiss.{deprecated_name} is deprecated in favour of " + f"faiss.{base_name}!" + ) warnings.warn(msg, DeprecationWarning, stacklevel=2) instance = super(base_class, cls).__new__(cls, *args, **kwargs) return instance - # three-argument version of "type" uses (name, tuple-of-bases, dict-of-attributes) + # three-argument version of "type" uses (name, tuple-of-bases, + # dict-of-attributes) klazz = type(deprecated_name, (base_class,), {"__new__": new_meth}) # this ends up adding the class to the "faiss" namespace, in a way that it @@ -87,7 +91,8 @@ for depr_prefix, base_prefix in deprecated_name_map.items(): ) # mapping from vector names in swigfaiss.swig and the numpy dtype names -# TODO: once deprecated classes are removed, remove the dict and just use .lower() below +# TODO: once deprecated classes are removed, remove the dict and just use +# .lower() below vector_name_map = { "Float32": "float32", "Float64": "float64", diff --git a/faiss/python/class_wrappers.py b/faiss/python/class_wrappers.py index f48f9ae00..dfe6e3f1a 100644 --- a/faiss/python/class_wrappers.py +++ b/faiss/python/class_wrappers.py @@ -53,7 +53,8 @@ def _numeric_to_str(numeric_type): return "int8" else: raise ValueError( - "numeric type must be either faiss.Float32, faiss.Float16, or faiss.Int8" + "numeric type must be either faiss.Float32, faiss.Float16, " + "or faiss.Int8" ) @@ -76,17 +77,20 @@ def replace_method(the_class, name, replacement, ignore_missing=False): def handle_Clustering(the_class): def replacement_train(self, x, index, weights=None): - """Perform clustering on a set of vectors. The index is used for assignment. + """Perform clustering on a set of vectors. The index is used for + assignment. Parameters ---------- x : array_like Training vectors, shape (n, self.d). `dtype` must be float32. index : faiss.Index - Index used for assignment. The dimension of the index should be `self.d`. + Index used for assignment. The dimension of the index + should be `self.d`. weights : array_like, optional - Per training sample weight (size n) used when computing the weighted - average to obtain the centroid (default is 1 for all training vectors). + Per training sample weight (size n) used when computing + the weighted average to obtain the centroid (default is + 1 for all training vectors). """ n, d = x.shape x = np.ascontiguousarray(x, dtype="float32") @@ -99,20 +103,24 @@ def handle_Clustering(the_class): self.train_c(n, swig_ptr(x), index) def replacement_train_encoded(self, x, codec, index, weights=None): - """Perform clustering on a set of compressed vectors. The index is used for assignment. + """Perform clustering on a set of compressed vectors. The index is + used for assignment. The decompression is performed on-the-fly. Parameters ---------- x : array_like - Training vectors, shape (n, codec.code_size()). `dtype` must be `uint8`. + Training vectors, shape (n, codec.code_size()). `dtype` must + be `uint8`. codec : faiss.Index Index used to decode the vectors. Should have dimension `self.d`. index : faiss.Index - Index used for assignment. The dimension of the index should be `self.d`. + Index used for assignment. The dimension of the index + should be `self.d`. weights : array_like, optional - Per training sample weight (size n) used when computing the weighted - average to obtain the centroid (default is 1 for all training vectors). + Per training sample weight (size n) used when computing + the weighted average to obtain the centroid (default is + 1 for all training vectors). """ n, d = x.shape x = _check_dtype_uint8(x) @@ -213,7 +221,8 @@ def handle_Quantizer(the_class): Returns ------- - Reconstructed vectors for each code, shape `(n, d)` and `dtype` float32. + Reconstructed vectors for each code, shape `(n, d)` and + `dtype` float32. """ n, cs = codes.shape codes = _check_dtype_uint8(codes) @@ -280,8 +289,9 @@ def handle_Index(the_class): def replacement_add(self, x, numeric_type=faiss.Float32): """Adds vectors to the index. The index must be trained before vectors can be added to it. - The vectors are implicitly numbered in sequence. When `n` vectors are - added to the index, they are given ids `ntotal`, `ntotal + 1`, ..., `ntotal + n - 1`. + The vectors are implicitly numbered in sequence. When `n` + vectors are added to the index, they are given ids `ntotal`, + `ntotal + 1`, ..., `ntotal + n - 1`. Parameters ---------- @@ -299,7 +309,8 @@ def handle_Index(the_class): self.add_ex(n, swig_ptr(x), numeric_type) def replacement_add_with_ids(self, x, ids, numeric_type=faiss.Float32): - """Adds vectors with arbitrary ids to the index (not all indexes support this). + """Adds vectors with arbitrary ids to the index (not all indexes + support this). The index must be trained before vectors can be added to it. Vector `i` is stored in `x[i]` and has id `ids[i]`. @@ -309,8 +320,9 @@ def handle_Index(the_class): Query vectors, shape (n, d) where d is appropriate for the index. `dtype` must be float32. ids : array_like - Array if ids of size n. The ids must be of type `int64`. Note that `-1` is reserved - in result lists to mean "not found" so it's better to not use it as an id. + Array if ids of size n. The ids must be of type `int64`. + Note that `-1` is reserved in result lists to mean "not + found" so it's better to not use it as an id. """ n, d = x.shape assert d == self.d @@ -419,7 +431,8 @@ def handle_Index(the_class): k : int Number of nearest neighbors. params : SearchParameters - Search parameters of the current search (overrides the class-level params) + Search parameters of the current search (overrides the + class-level params) D : array_like, optional Distance array to store the result. I : array_like, optional @@ -428,8 +441,9 @@ def handle_Index(the_class): Returns ------- D : array_like - Distances of the nearest neighbors, shape (n, k). When not enough results are found - the label is set to +Inf or -Inf. + Distances of the nearest neighbors, shape (n, k). When + not enough results are found the label is set to +Inf or + -Inf. I : array_like Labels of the nearest neighbors, shape (n, k). When not enough results are found, the label is set to -1 @@ -479,7 +493,8 @@ def handle_Index(the_class): k : int Number of nearest neighbors. params : SearchParameters - Search parameters of the current search (overrides the class-level params) + Search parameters of the current search (overrides the + class-level params) D : array_like, optional Distance array to store the result. I : array_like, optional @@ -490,13 +505,15 @@ def handle_Index(the_class): Returns ------- D : array_like - Distances of the nearest neighbors, shape (n, k). When not enough results are found - the label is set to +Inf or -Inf. + Distances of the nearest neighbors, shape (n, k). When + not enough results are found the label is set to +Inf or + -Inf. I : array_like - Labels of the nearest neighbors, shape (n, k). When not enough results are found, - the label is set to -1 + Labels of the nearest neighbors, shape (n, k). When not + enough results are found, the label is set to -1 R : array_like - Approximate (reconstructed) nearest neighbor vectors, shape (n, k, d). + Approximate (reconstructed) nearest neighbor vectors, + shape (n, k, d). """ n, d = x.shape assert d == self.d @@ -546,7 +563,8 @@ def handle_Index(the_class): k : int Number of nearest neighbors. params : SearchParameters - Search parameters of the current search (overrides the class-level params) + Search parameters of the current search (overrides the + class-level params) include_listnos : bool, optional whether to include the list ids in the first bytes of each code D : array_like, optional @@ -559,13 +577,15 @@ def handle_Index(the_class): Returns ------- D : array_like - Distances of the nearest neighbors, shape (n, k). When not enough results are found - the label is set to +Inf or -Inf. + Distances of the nearest neighbors, shape (n, k). When + not enough results are found the label is set to +Inf or + -Inf. I : array_like - Labels of the nearest neighbors, shape (n, k). When not enough results are found, - the label is set to -1 + Labels of the nearest neighbors, shape (n, k). When not + enough results are found, the label is set to -1 R : array_like - Approximate (reconstructed) nearest neighbor vectors, shape (n, k, d). + Approximate (reconstructed) nearest neighbor vectors, + shape (n, k, d). """ n, d = x.shape assert d == self.d @@ -679,7 +699,8 @@ def handle_Index(the_class): return x def replacement_reconstruct_n(self, n0=0, ni=-1, x=None): - """Approximate reconstruction of vectors `n0` ... `n0 + ni - 1` from the index. + """Approximate reconstruction of vectors `n0` ... `n0 + ni - 1` + from the index. Missing vectors trigger an exception. Parameters @@ -724,11 +745,13 @@ def handle_Index(the_class): Query vectors, shape (n, d) where d is appropriate for the index. `dtype` must be float32. thresh : float - Threshold to select neighbors. All elements within this radius are returned, - except for maximum inner product indexes, where the elements above the - threshold are returned + Threshold to select neighbors. All elements within this + radius are returned, except for maximum inner product + indexes, where the elements above the threshold are + returned params : SearchParameters - Search parameters of the current search (overrides the class-level params) + Search parameters of the current search (overrides the + class-level params) Returns @@ -736,8 +759,8 @@ def handle_Index(the_class): lims: array_like Starting index of the results for each query vector, size n+1. D : array_like - Distances of the nearest neighbors, shape `lims[n]`. The distances for - query i are in `D[lims[i]:lims[i+1]]`. + Distances of the nearest neighbors, shape `lims[n]`. The + distances for query i are in `D[lims[i]:lims[i+1]]`. I : array_like Labels of nearest neighbors, shape `lims[n]`. The labels for query i are in `I[lims[i]:lims[i+1]]`. @@ -776,7 +799,8 @@ def handle_Index(the_class): Nearest centroids, size (n, nprobe) params : SearchParameters - Search parameters of the current search (overrides the class-level params) + Search parameters of the current search (overrides the + class-level params) D : array_like, optional Distance array to store the result. I : array_like, optional @@ -785,8 +809,9 @@ def handle_Index(the_class): Returns ------- D : array_like - Distances of the nearest neighbors, shape (n, k). When not enough results are found - the label is set to +Inf or -Inf. + Distances of the nearest neighbors, shape (n, k). When + not enough results are found the label is set to +Inf or + -Inf. I : array_like Labels of the nearest neighbors, shape (n, k). When not enough results are found, the label is set to -1 @@ -839,15 +864,17 @@ def handle_Index(the_class): Query vectors, shape (n, d) where d is appropriate for the index. `dtype` must be float32. thresh : float - Threshold to select neighbors. All elements within this radius are returned, - except for maximum inner product indexes, where the elements above the - threshold are returned + Threshold to select neighbors. All elements within this + radius are returned, except for maximum inner product + indexes, where the elements above the threshold are + returned Iq : array_like, optional Nearest centroids, size (n, nprobe) Dq : array_like, optional Distance array to the centroids, size (n, nprobe) params : SearchParameters - Search parameters of the current search (overrides the class-level params) + Search parameters of the current search (overrides the + class-level params) Returns @@ -855,8 +882,8 @@ def handle_Index(the_class): lims: array_like Starting index of the results for each query vector, size n+1. D : array_like - Distances of the nearest neighbors, shape `lims[n]`. The distances for - query i are in `D[lims[i]:lims[i+1]]`. + Distances of the nearest neighbors, shape `lims[n]`. The + distances for query i are in `D[lims[i]:lims[i+1]]`. I : array_like Labels of nearest neighbors, shape `lims[n]`. The labels for query i are in `I[lims[i]:lims[i+1]]`. @@ -1378,7 +1405,8 @@ def add_to_referenced_objects(self, ref): class RememberSwigOwnership: """ SWIG's seattr transfers ownership of SWIG wrapped objects to the class - (btw this seems to contradict https://www.swig.org/Doc1.3/Python.html#Python_nn22 + (btw this seems to contradict + https://www.swig.org/Doc1.3/Python.html#Python_nn22 31.4.2) This interferes with how we manage ownership: with the referenced_objects table. Therefore, we reset the thisown field in this context manager. diff --git a/faiss/python/extra_wrappers.py b/faiss/python/extra_wrappers.py index b164a55dd..3752d042e 100644 --- a/faiss/python/extra_wrappers.py +++ b/faiss/python/extra_wrappers.py @@ -168,7 +168,8 @@ def bucket_sort(tab, nbucket=None, nt=0): lims : array_like cumulative sum of bucket sizes (size vmax + 1) perm : array_like - perm[lims[i] : lims[i + 1]] contains the indices of bucket #i (size tab.size) + perm[lims[i] : lims[i + 1]] contains the indices of bucket #i + (size tab.size) """ tab = np.ascontiguousarray(tab, dtype="int64") if nbucket is None: @@ -292,7 +293,8 @@ class ResultHeap: def merge_knn_results(Dall, Iall, keep_max=False): """ - Merge a set of sorted knn-results obtained from different shards in a dataset + Merge a set of sorted knn-results obtained from different shards in a + dataset Dall and Iall are of size (nshard, nq, k) each D[i, j] should be sorted returns D, I of size (nq, k) as the merged result set """ @@ -419,7 +421,8 @@ def knn(xq, xb, k, metric=METRIC_L2, metric_arg=0.0): def knn_hamming(xq, xb, k, variant="hc"): """ - Compute the k nearest neighbors of a set of vectors without constructing an index. + Compute the k nearest neighbors of a set of vectors without constructing + an index. Parameters ---------- @@ -480,7 +483,8 @@ def knn_hamming(xq, xb, k, variant="hc"): class Kmeans: """Object that performs k-means clustering and manages the centroids. - The `Kmeans` class is essentially a wrapper around the C++ `Clustering` object. + The `Kmeans` class is essentially a wrapper around the C++ `Clustering` + object. Parameters ---------- @@ -495,7 +499,8 @@ class Kmeans: progressive_dim_steps: use a progressive dimension clustering (with that number of steps) - Subsequent parameters are fields of the Clustring object. The most important are: + Subsequent parameters are fields of the Clustring object. The most + important are: niter: int, optional clustering iterations @@ -541,7 +546,8 @@ class Kmeans: v = get_num_gpus() self.gpu = v else: - # if this raises an exception, it means that it is a non-existent field + # if this raises an exception, it means that + # it is a non-existent field getattr(self.cp, k) setattr(self.cp, k, v) self.set_index() @@ -579,7 +585,8 @@ class Kmeans: - the centroids are in the centroids field of size (`k`, `d`). - - the objective value at each iteration is in the array obj (size `niter`) + - the objective value at each iteration is in the array obj (size + `niter`) - detailed optimization statistics are in the array iteration_stats. diff --git a/faiss/python/gpu_wrappers.py b/faiss/python/gpu_wrappers.py index 3cc8039c0..c8a2e0a8f 100644 --- a/faiss/python/gpu_wrappers.py +++ b/faiss/python/gpu_wrappers.py @@ -71,7 +71,8 @@ def knn_gpu( queriesMemoryLimit=0, ): """ - Compute the k nearest neighbors of a vector on one GPU without constructing an index + Compute the k nearest neighbors of a vector on one GPU without constructing + an index Parameters ---------- @@ -95,7 +96,8 @@ def knn_gpu( Which CUDA device in the system to run the search on. -1 indicates that the current thread-local device state (via cudaGetDevice) should be used (can also be set via torch.cuda.set_device in PyTorch) - Otherwise, an integer 0 <= device < numDevices indicates the GPU on which + Otherwise, an integer 0 <= device < numDevices indicates the GPU on + which the computation should be run vectorsMemoryLimit: int, optional queriesMemoryLimit: int, optional @@ -208,7 +210,8 @@ def knn_gpu( def pairwise_distance_gpu(res, xq, xb, D=None, metric=METRIC_L2, device=-1): """ - Compute all pairwise distances between xq and xb on one GPU without constructing an index + Compute all pairwise distances between xq and xb on one GPU without + constructing an index Parameters ---------- @@ -228,7 +231,8 @@ def pairwise_distance_gpu(res, xq, xb, D=None, metric=METRIC_L2, device=-1): Which CUDA device in the system to run the search on. -1 indicates that the current thread-local device state (via cudaGetDevice) should be used (can also be set via torch.cuda.set_device in PyTorch) - Otherwise, an integer 0 <= device < numDevices indicates the GPU on which + Otherwise, an integer 0 <= device < numDevices indicates the GPU on + which the computation should be run Returns diff --git a/faiss/python/loader.py b/faiss/python/loader.py index f1357f239..952712276 100644 --- a/faiss/python/loader.py +++ b/faiss/python/loader.py @@ -15,7 +15,7 @@ from packaging.version import Version def supported_instruction_sets(): """ Returns the set of supported CPU features, see - https://github.com/numpy/numpy/blob/master/numpy/core/src/common/npy_cpu_features.h + https://github.com/numpy/numpy/blob/master/numpy/core/src/common/npy_cpu_features.h # noqa: E501 for the list of features that this set may contain per architecture. Example: diff --git a/faiss/python/setup.py b/faiss/python/setup.py index 59d1db9a1..892e87a65 100644 --- a/faiss/python/setup.py +++ b/faiss/python/setup.py @@ -61,7 +61,9 @@ if platform.system() != "AIX": or found_faiss_example_external_module_lib ), ( f"Could not find {swigfaiss_generic_lib} or " - f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_avx512_spr_lib} or {swigfaiss_sve_lib} or {faiss_example_external_module_lib}. " + f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or " + f"{swigfaiss_avx512_spr_lib} or {swigfaiss_sve_lib} or " + f"{faiss_example_external_module_lib}. " f"Faiss may not be compiled yet." ) @@ -118,7 +120,10 @@ are implemented on the GPU. It is developed by Facebook AI Research. setup( name="faiss", version="1.14.3", - description="A library for efficient similarity search and clustering of dense vectors", + description=( + "A library for efficient similarity search and clustering of dense " + "vectors" + ), long_description=long_description, long_description_content_type="text/plain", url="https://github.com/facebookresearch/faiss", diff --git a/tests/test_clustering.py b/tests/test_clustering.py index 51b419876..ed57266d9 100644 --- a/tests/test_clustering.py +++ b/tests/test_clustering.py @@ -197,7 +197,8 @@ class TestCompositeClustering(unittest.TestCase): self.assertGreater(obj1, obj10) def test_redo_cosine(self): - # test redo with cosine distance (inner prod, so objectives are reversed) + # test redo with cosine distance (inner prod, so objectives are + # reversed) d = 64 n = 1000 diff --git a/tests/test_contrib.py b/tests/test_contrib.py index 8a89af0c0..acbcce6d4 100644 --- a/tests/test_contrib.py +++ b/tests/test_contrib.py @@ -881,7 +881,9 @@ class TestFactoryTools(unittest.TestCase): faiss.ScalarQuantizer.QT_fp16: "IVF32,SQfp16", faiss.ScalarQuantizer.QT_bf16: "IVF32,SQbf16", faiss.ScalarQuantizer.QT_8bit_direct: "IVF32,SQ8_direct", - faiss.ScalarQuantizer.QT_8bit_direct_signed: "IVF32,SQ8_direct_signed", + faiss.ScalarQuantizer.QT_8bit_direct_signed: ( + "IVF32,SQ8_direct_signed" + ), faiss.ScalarQuantizer.QT_0bit: "IVF32,SQ0", faiss.ScalarQuantizer.QT_1bit_tqmse: "IVF32,SQtqmse1", faiss.ScalarQuantizer.QT_2bit_tqmse: "IVF32,SQtqmse2", @@ -931,7 +933,8 @@ class TestFactoryTools(unittest.TestCase): def test_get_code_size_hnsw_non_default_m(self): d = 128 - # Non-default M values previously raised RuntimeError("cannot parse HNSW16") + # Non-default M values previously raised + # RuntimeError("cannot parse HNSW16") self.assertEqual( factory_tools.get_code_size(d, "HNSW16"), d * 4 + 16 * 2 * 4 ) @@ -948,7 +951,8 @@ class TestFactoryTools(unittest.TestCase): def test_get_code_size_ivf_hnsw_non_default_m(self): d = 128 - # IVF+HNSW coarse quantizer with non-default M: code size is inner type only + # IVF+HNSW coarse quantizer with non-default M: code size is inner + # type only self.assertEqual( factory_tools.get_code_size(d, "IVF64_HNSW16,Flat"), d * 4 ) diff --git a/tests/test_hnsw_panorama.py b/tests/test_hnsw_panorama.py index b5309a7ba..8f8f0ae1c 100644 --- a/tests/test_hnsw_panorama.py +++ b/tests/test_hnsw_panorama.py @@ -43,7 +43,8 @@ class TestIndexHNSWFlatPanorama(unittest.TestCase): return D, I def compute_recall(self, gt_I, test_I): - """Compute recall@k - fraction of ground truth results found in test results.""" + """Compute recall@k - fraction of ground truth results found in + test results.""" nq, k = gt_I.shape recalls = [np.isin(gt_I[i], test_I[i]).sum() for i in range(nq)] return sum(recalls) / (nq * k) @@ -122,7 +123,8 @@ class TestIndexHNSWFlatPanorama(unittest.TestCase): print(f"Recall@{k}: {recall}") # With efSearch=64, we should get reasonably good recall - # The threshold is lower than vanilla HNSW because of approximate distances + # The threshold is lower than vanilla HNSW because of approximate + # distances self.assertGreaterEqual(recall, 0.85) def test_different_panorama_levels(self): @@ -394,7 +396,8 @@ class TestIndexHNSWFlatPanorama(unittest.TestCase): recall = self.compute_recall(gt_I, I_after) print(f"Recall after adding more vectors: {recall}") - # Recall might be slightly lower than single-batch due to HNSW graph structure + # Recall might be slightly lower than single-batch due to HNSW + # graph structure self.assertGreaterEqual(recall, 0.80) # Verify that previously found neighbors can still be found @@ -409,7 +412,8 @@ class TestIndexHNSWFlatPanorama(unittest.TestCase): retention = float(found_count) / (nq * k) print(f"Retention of previous neighbors: {retention}") - # Should retain a reasonable number of previous neighbors (new ones might push some out) + # Should retain a reasonable number of previous neighbors (new ones + # might push some out) # The threshold is lower to account for the approximate nature of HNSW self.assertGreaterEqual(retention, 0.5) diff --git a/tests/test_index.py b/tests/test_index.py index dc438e02b..45a22ee99 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -459,9 +459,9 @@ class TestScalarQuantizer(unittest.TestCase): D, I = index.search(xq, 10) nok["flat"] = (I[:, 0] == I_ref[:, 0]).sum() - for ( - qname - ) in "QT_4bit QT_4bit_uniform QT_8bit QT_8bit_uniform QT_fp16 QT_bf16".split(): + for qname in ( + "QT_4bit QT_4bit_uniform QT_8bit QT_8bit_uniform QT_fp16 QT_bf16" + ).split(): qtype = getattr(faiss.ScalarQuantizer, qname) index = faiss.IndexIVFScalarQuantizer( quantizer, d, ncent, qtype, faiss.METRIC_L2 @@ -500,9 +500,9 @@ class TestScalarQuantizer(unittest.TestCase): nok = {} - for ( - qname - ) in "QT_4bit QT_4bit_uniform QT_8bit QT_8bit_uniform QT_fp16 QT_bf16".split(): + for qname in ( + "QT_4bit QT_4bit_uniform QT_8bit QT_8bit_uniform QT_fp16 QT_bf16" + ).split(): qtype = getattr(faiss.ScalarQuantizer, qname) index = faiss.IndexScalarQuantizer(d, qtype, faiss.METRIC_L2) index.train(xt) diff --git a/tests/test_index_composite.py b/tests/test_index_composite.py index f5b9c5a06..eb82c80a4 100644 --- a/tests/test_index_composite.py +++ b/tests/test_index_composite.py @@ -456,7 +456,8 @@ class TestIVFFlatDedup(unittest.TestCase): index_new.verbose = True # should display - # IndexIVFFlatDedup::train: train on 350 points after dedup (was 500 points) + # IndexIVFFlatDedup::train: train on 350 points after dedup + # (was 500 points) index_new.train(xt) index_ref = faiss.IndexIVFFlat(quantizer, d, 20) @@ -840,8 +841,8 @@ class TestIndependentQuantizer(unittest.TestCase): self.assertLess(perf_ref, perf_new) def test_precomputed_tables(self): - """see how precomputed tables behave with centroid distance estimates from a mismatching - coarse quantizer""" + """see how precomputed tables behave with centroid distance + estimates from a mismatching coarse quantizer""" ds = SyntheticDataset(48, 2000, 500, 250) gt = ds.get_groundtruth(10) diff --git a/tests/test_ivf_flat_panorama.py b/tests/test_ivf_flat_panorama.py index 4c250b04d..89d871944 100644 --- a/tests/test_ivf_flat_panorama.py +++ b/tests/test_ivf_flat_panorama.py @@ -120,7 +120,8 @@ class TestIndexIVFFlatPanorama(unittest.TestCase): otol=1e-3, rtol=1e-4, ): - """Compare range search results with tolerance for boundary differences.""" + """Compare range search results with tolerance for boundary + differences.""" total_matches = total_regular = 0 for i in range(nq): @@ -160,7 +161,8 @@ class TestIndexIVFFlatPanorama(unittest.TestCase): def validate_and_compare_range_results( self, metric, radius, lims_reg, D_reg, I_reg, lims_pan, D_pan, I_pan, nq ): - """Helper to validate range search results match between regular and panorama.""" + """Helper to validate range search results match between regular + and panorama.""" if metric == faiss.METRIC_L2: self.assertTrue( np.all(D_pan <= radius), @@ -473,7 +475,8 @@ class TestIndexIVFFlatPanorama(unittest.TestCase): # Batch size and edge case tests def test_batch_boundaries(self): - """Test correctness at various batch size boundaries (kDefaultBatchSize=128)""" + """Test correctness at various batch size boundaries + (kDefaultBatchSize=128)""" d, nlist, nlevels, nt, nq, k = 128, 64, 8, 10000, 200, 15 np.random.seed(987) xt = np.random.rand(nt, d).astype("float32") @@ -692,7 +695,8 @@ class TestIndexIVFFlatPanorama(unittest.TestCase): ) def test_update_vectors(self): - """Test update operations (single, batch, and interleaved with search)""" + """Test update operations (single, batch, and interleaved with + search)""" d, nb, nt, nq, nlist, nlevels, k = 128, 40000, 60000, 400, 256, 8, 15 xt, xb, xq = self.generate_data(d, nt, nb, nq, seed=1414) @@ -745,7 +749,8 @@ class TestIndexIVFFlatPanorama(unittest.TestCase): ) def test_serialization(self): - """Test that writing and reading Panorama indexes preserves search results""" + """Test that writing and reading Panorama indexes preserves + search results""" d, nb, nt, nq, nlist, nlevels, k = 128, 10000, 15000, 100, 128, 8, 20 xt, xb, xq = self.generate_data(d, nt, nb, nq, seed=2024) diff --git a/tests/test_rabitq_fastscan.py b/tests/test_rabitq_fastscan.py index 623933165..fd16e0bce 100644 --- a/tests/test_rabitq_fastscan.py +++ b/tests/test_rabitq_fastscan.py @@ -645,7 +645,8 @@ class TestMultiBitRaBitQFastScan(unittest.TestCase): self.assertEqual(index.code_size, expected_size) def test_ivf_construction(self): - """Test IndexIVFRaBitQFastScan construction with valid/invalid nb_bits.""" + """Test IndexIVFRaBitQFastScan construction with valid/invalid + nb_bits.""" d, nlist = 128, 16 # Valid nb_bits for nb_bits in [1, 2, 4, 8]: diff --git a/tests/test_refine.py b/tests/test_refine.py index feeb0ce3f..6bb1ed159 100644 --- a/tests/test_refine.py +++ b/tests/test_refine.py @@ -170,7 +170,8 @@ class TestIndexRefineRangeSearch(unittest.TestCase): self.assertAlmostEqual(recall_1, recall_2) - # validate: refined range_search() updates distances, and new distances are correct L2 distances + # validate: refined range_search() updates distances, and new + # distances are correct L2 distances for iq in range(0, ds.nq): start_lim = lims_2[iq] end_lim = lims_2[iq + 1] diff --git a/tests/test_residual_quantizer.py b/tests/test_residual_quantizer.py index aeaf05cd3..53a825a0e 100644 --- a/tests/test_residual_quantizer.py +++ b/tests/test_residual_quantizer.py @@ -226,8 +226,8 @@ class TestResidualQuantizer(unittest.TestCase): pq.train(xt) err_pq = eval_codec(pq, xb) - # in practice RQ is often better than PQ but it is not the case here, so just check - # that we are within some factor. + # in practice RQ is often better than PQ but it is not the case + # here, so just check that we are within some factor. self.assertLess(err_rq, err_pq * 1.2) def test_beam_size(self): @@ -382,7 +382,8 @@ def retrain_AQ_codebook(index, xt): ) # replace codebook - # faiss.copy_array_to_vector(B.astype('float32').ravel(), index.rq.codebooks) + # faiss.copy_array_to_vector( + # B.astype('float32').ravel(), index.rq.codebooks) # update codebook tables # index.rq.compute_codebook_tables() diff --git a/tests/test_search_params.py b/tests/test_search_params.py index 37fbb5a4f..bf4fd3a37 100644 --- a/tests/test_search_params.py +++ b/tests/test_search_params.py @@ -33,7 +33,8 @@ class TestSelector(unittest.TestCase): ): """Verify that the id selector returns the subset of results that are members according to the IDSelector. - Supports id_selector_type="batch", "bitmap", "range", "range_sorted", "and", "or", "xor" + Supports id_selector_type="batch", "bitmap", "range", + "range_sorted", "and", "or", "xor" """ d = 32 # make sure dimension is multiple of 8 for binary ds = datasets.SyntheticDataset(d, 1000, 100, 20) diff --git a/tests/test_simd_dispatch.py b/tests/test_simd_dispatch.py index 032a5e080..12851777e 100644 --- a/tests/test_simd_dispatch.py +++ b/tests/test_simd_dispatch.py @@ -80,7 +80,8 @@ class TestSIMDDispatch(unittest.TestCase): self.assertIsNotNone(result) def test_get_level_equals_get_dispatched_level(self): - """Verify get_level() and get_dispatched_level() return the same value.""" + """Verify get_level() and get_dispatched_level() return the same + value.""" try: import faiss except ImportError: diff --git a/tests/test_standalone_codec.py b/tests/test_standalone_codec.py index b21301a56..9a9df0dd4 100644 --- a/tests/test_standalone_codec.py +++ b/tests/test_standalone_codec.py @@ -510,7 +510,8 @@ class TestRefine(unittest.TestCase): np.testing.assert_allclose(x_decoded, x_decoded_ref) def test_equiv_rcq_rq(self): - """make sure that the codes generated by the standalone codec are the same + """make sure that the codes generated by the standalone codec are + the same between an IndexRefine with ResidualQuantizer and diff --git a/tests/torch_test_neural_net.py b/tests/torch_test_neural_net.py index 07aefa474..2d0c285c5 100644 --- a/tests/torch_test_neural_net.py +++ b/tests/torch_test_neural_net.py @@ -211,7 +211,8 @@ class QINCo(nn.Module): """ Encode a batch of vectors x to codes of length M. If this function is called from IVF-QINCo, codes are 1 index longer, - due to the first index being the IVF index, and codebook0 is the IVF codebook. + due to the first index being the IVF index, and codebook0 is the + IVF codebook. """ M = len(self.steps) + 1 bs, d = x.shape