diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e367a99b..cbf077ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -386,12 +386,12 @@ Changed - Improve naming due to codemod (#4063) - Improve naming due to codemod (#4064) - Improve naming due to codemod (#4065) -- separare the github build into two conditions (#4066) +- separate the github build into two conditions (#4066) - Improve naming due to codemod (#4070) - improve naming due to codemod (#4067) - improve naming due to codemod (#4071) - improve naming due to codemod (#4072) -- fix nightily build (#4080) +- fix nightly build (#4080) - Change github action workflows name (#4083) - Resolve Packaging Issues (#4044) - Update __init__.py (#4086) @@ -474,7 +474,7 @@ Deprecated - faster hnsw CPU index training (#3822) - Some small improvements. (#3692) - First attempt at LSH matching with nbits (#3679) -- Set verbosoe before train (#3619) +- Set verbose before train (#3619) - Remove duplicate NegativeDistanceComputer instances (#3450) - interrupt for NNDescent (#3432) - Get rid of redundant instructions in ScalarQuantizer (#3430) @@ -486,11 +486,11 @@ Deprecated - Skip HNSWPQ sdc init with new io flag (#3250) ### Fixed -- FIx a bug for a non-simdlib code of ResidualQuantizer (#3868) +- Fix a bug for a non-simdlib code of ResidualQuantizer (#3868) - assign_index should default to null (#3855) -- Fix an incorrectly counted the number of computed distances for HNSW (#3840) +- Fix incorrectly counted the number of computed distances for HNSW (#3840) - Add error for overflowing nbits during PQ construction (#3833) -- Fix radius search with HSNW and IP (#3698) +- Fix radius search with HNSW and IP (#3698) - fix algorithm of spreading vectors over shards (#3374) - Fix IndexBinary.assign Python method (#3384) - Few fixes in bench_fw to enable IndexFromCodec (#3383) @@ -558,7 +558,7 @@ Deprecated - Improved ResidualQuantizer vector encoding (pooling memory allocations, avoid r/w to a temporary buffer) ### Fixed -- HSNW bug fixed which improves the recall rate! Special thanks to zh Wang @hhy3 for this. +- HNSW bug fixed which improves the recall rate! Special thanks to zh Wang @hhy3 for this. - Faiss GPU IVF large query batch fix - Faiss + Torch fixes, re-enable k = 2048 - Fix the number of distance computations to match max_codes parameter @@ -599,7 +599,7 @@ Deprecated - Implementation of Local Search Quantization (by @KinglittleQ) ### Changed -- The order of xb an xq was different between `faiss.knn` and `faiss.knn_gpu`. +- The order of xb and xq was different between `faiss.knn` and `faiss.knn_gpu`. Also the metric argument was called distance_type. - The typed vectors (LongVector, LongLongVector, etc.) of the SWIG interface have been deprecated. They have been replaced with Int32Vector, Int64Vector, etc. (by h-vetinari) @@ -730,7 +730,7 @@ propagated. - Support for IMI2x16 (4B virtual centroids). - Support for k = 2048 search on GPU (instead of 1024). - Support for renaming an ondisk invertedlists. -- Support for nterrupting computations with interrupt signal (ctrl-C) in python. +- Support for interrupting computations with interrupt signal (ctrl-C) in python. - Simplified build system (with --with-cuda/--with-cuda-arch options). ### Changed diff --git a/benchs/bench_all_ivf/parse_bench_all_ivf.py b/benchs/bench_all_ivf/parse_bench_all_ivf.py index cb22f143e..8a5477f73 100644 --- a/benchs/bench_all_ivf/parse_bench_all_ivf.py +++ b/benchs/bench_all_ivf/parse_bench_all_ivf.py @@ -136,7 +136,7 @@ def collect_results_for(db='deep1M', prefix="autotune."): print("FAIL %s" % e) res = np.zeros((2, 0)) except Exception as e: - print("PARSE ERROR " + e) + print("PARSE ERROR " + str(e)) res = np.zeros((2, 0)) else: print(len(res), "results") diff --git a/benchs/bench_cppcontrib_sa_decode.cpp b/benchs/bench_cppcontrib_sa_decode.cpp index 42f98abd7..f47ef6cbd 100644 --- a/benchs/bench_cppcontrib_sa_decode.cpp +++ b/benchs/bench_cppcontrib_sa_decode.cpp @@ -221,9 +221,9 @@ static void verifyIndex2LevelDecoder( // generate a random order of points std::uniform_int_distribution un(0, n - 1); - std::vector pointIncidesToDecode(nIterations * n, 0); + std::vector pointIndicesToDecode(nIterations * n, 0); for (uint64_t i = 0; i < nIterations * n; i++) { - pointIncidesToDecode[i] = un(rng); + pointIndicesToDecode[i] = un(rng); } { @@ -234,7 +234,7 @@ static void verifyIndex2LevelDecoder( StopWatch swFaiss; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; index->sa_decode( 1, encodedData.data() + pointIdx * codeSize, @@ -247,7 +247,7 @@ static void verifyIndex2LevelDecoder( StopWatch swKernel; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; T::store( pqCoarseCentroidsQ, pqFineCentroidsQ, @@ -289,7 +289,7 @@ static void verifyIndex2LevelDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; index->sa_decode( 1, @@ -307,7 +307,7 @@ static void verifyIndex2LevelDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; T::accum( pqCoarseCentroidsQ, @@ -331,10 +331,10 @@ static void verifyIndex2LevelDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqCoarseCentroidsQ, @@ -360,10 +360,10 @@ static void verifyIndex2LevelDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqCoarseCentroidsQ, @@ -391,13 +391,13 @@ static void verifyIndex2LevelDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqCoarseCentroidsQ, @@ -425,13 +425,13 @@ static void verifyIndex2LevelDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqCoarseCentroidsQ, @@ -532,9 +532,9 @@ static void verifyMinMaxIndex2LevelDecoder( // generate a random order of points std::uniform_int_distribution un(0, n - 1); - std::vector pointIncidesToDecode(nIterations * n, 0); + std::vector pointIndicesToDecode(nIterations * n, 0); for (uint64_t i = 0; i < nIterations * n; i++) { - pointIncidesToDecode[i] = un(rng); + pointIndicesToDecode[i] = un(rng); } { @@ -545,7 +545,7 @@ static void verifyMinMaxIndex2LevelDecoder( StopWatch swFaiss; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; index->sa_decode( 1, encodedData.data() + pointIdx * codeSize, @@ -558,7 +558,7 @@ static void verifyMinMaxIndex2LevelDecoder( StopWatch swKernel; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; T::store( pqCoarseCentroidsQ, pqFineCentroidsQ, @@ -601,7 +601,7 @@ static void verifyMinMaxIndex2LevelDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; index->sa_decode( 1, @@ -620,7 +620,7 @@ static void verifyMinMaxIndex2LevelDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; T::accum( pqCoarseCentroidsQ, @@ -649,10 +649,10 @@ static void verifyMinMaxIndex2LevelDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqCoarseCentroidsQ, @@ -683,10 +683,10 @@ static void verifyMinMaxIndex2LevelDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqCoarseCentroidsQ, @@ -719,13 +719,13 @@ static void verifyMinMaxIndex2LevelDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqCoarseCentroidsQ, @@ -758,13 +758,13 @@ static void verifyMinMaxIndex2LevelDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqCoarseCentroidsQ, @@ -859,9 +859,9 @@ static void verifyIndexPQDecoder( // generate a random order of points std::uniform_int_distribution un(0, n - 1); - std::vector pointIncidesToDecode(nIterations * n, 0); + std::vector pointIndicesToDecode(nIterations * n, 0); for (uint64_t i = 0; i < nIterations * n; i++) { - pointIncidesToDecode[i] = un(rng); + pointIndicesToDecode[i] = un(rng); } { @@ -872,7 +872,7 @@ static void verifyIndexPQDecoder( StopWatch swFaiss; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; index->sa_decode( 1, encodedData.data() + pointIdx * codeSize, @@ -885,7 +885,7 @@ static void verifyIndexPQDecoder( StopWatch swKernel; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; T::store( pqFineCentroidsQ, encodedData.data() + pointIdx * codeSize, @@ -927,7 +927,7 @@ static void verifyIndexPQDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; index->sa_decode( 1, @@ -945,7 +945,7 @@ static void verifyIndexPQDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; T::accum( pqFineCentroidsQ, @@ -968,10 +968,10 @@ static void verifyIndexPQDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqFineCentroidsQ, @@ -996,10 +996,10 @@ static void verifyIndexPQDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqFineCentroidsQ, @@ -1025,13 +1025,13 @@ static void verifyIndexPQDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqFineCentroidsQ, @@ -1058,13 +1058,13 @@ static void verifyIndexPQDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqFineCentroidsQ, @@ -1157,9 +1157,9 @@ static void verifyMinMaxIndexPQDecoder( // generate a random order of points std::uniform_int_distribution un(0, n - 1); - std::vector pointIncidesToDecode(nIterations * n, 0); + std::vector pointIndicesToDecode(nIterations * n, 0); for (uint64_t i = 0; i < nIterations * n; i++) { - pointIncidesToDecode[i] = un(rng); + pointIndicesToDecode[i] = un(rng); } { @@ -1170,7 +1170,7 @@ static void verifyMinMaxIndexPQDecoder( StopWatch swFaiss; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; index->sa_decode( 1, encodedData.data() + pointIdx * codeSize, @@ -1183,7 +1183,7 @@ static void verifyMinMaxIndexPQDecoder( StopWatch swKernel; for (uint64_t iter = 0; iter < nIterations; iter++) { for (uint64_t i = 0; i < n; i++) { - const auto pointIdx = pointIncidesToDecode[i + iter * n]; + const auto pointIdx = pointIndicesToDecode[i + iter * n]; T::store( pqFineCentroidsQ, encodedData.data() + pointIdx * codeSize, @@ -1225,7 +1225,7 @@ static void verifyMinMaxIndexPQDecoder( for (uint64_t i = 0; i < n; i++) { for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; index->sa_decode( 1, @@ -1244,7 +1244,7 @@ static void verifyMinMaxIndexPQDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter++) { const auto pointIdx = - pointIncidesToDecode[i * nIterations + iter]; + pointIndicesToDecode[i * nIterations + iter]; const auto weight = weights[i * nIterations + iter]; T::accum( pqFineCentroidsQ, @@ -1272,10 +1272,10 @@ static void verifyMinMaxIndexPQDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqFineCentroidsQ, @@ -1305,10 +1305,10 @@ static void verifyMinMaxIndexPQDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 2) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; T::accum( pqFineCentroidsQ, @@ -1339,13 +1339,13 @@ static void verifyMinMaxIndexPQDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqFineCentroidsQ, @@ -1377,13 +1377,13 @@ static void verifyMinMaxIndexPQDecoder( float outputAccumMin = 0; for (uint64_t iter = 0; iter < nIterations; iter += 3) { const auto pointIdx0 = - pointIncidesToDecode[i * nIterations + iter + 0]; + pointIndicesToDecode[i * nIterations + iter + 0]; const auto weight0 = weights[i * nIterations + iter + 0]; const auto pointIdx1 = - pointIncidesToDecode[i * nIterations + iter + 1]; + pointIndicesToDecode[i * nIterations + iter + 1]; const auto weight1 = weights[i * nIterations + iter + 1]; const auto pointIdx2 = - pointIncidesToDecode[i * nIterations + iter + 2]; + pointIndicesToDecode[i * nIterations + iter + 2]; const auto weight2 = weights[i * nIterations + iter + 2]; T::accum( pqFineCentroidsQ, diff --git a/benchs/bench_fw/descriptors.py b/benchs/bench_fw/descriptors.py index 666ad1d3a..a344205ce 100644 --- a/benchs/bench_fw/descriptors.py +++ b/benchs/bench_fw/descriptors.py @@ -326,7 +326,7 @@ class IndexDescriptor(IndexBaseDescriptor): self.flat_desc_name = self.codec_desc.flat_name() + self.database_desc.get_filename(prefix=IndexDescriptor.FILENAME_PREFIX) return self.flat_desc_name - # alias is used to refer when index is uploaded to blobstore and refered again + # alias is used to refer when index is uploaded to blobstore and referred again def alias(self, benchmark_io: BenchmarkIO): if hasattr(benchmark_io, "bucket"): return IndexDescriptor(desc_name=self.get_name(), bucket=benchmark_io.bucket, path=self.get_path(benchmark_io), d=self.d, metric=self.metric) diff --git a/benchs/bench_fw/index.py b/benchs/bench_fw/index.py index b1252ad1b..b0d12c246 100644 --- a/benchs/bench_fw/index.py +++ b/benchs/bench_fw/index.py @@ -558,7 +558,7 @@ class IndexBase: "construction_params": self.get_construction_params(), "search_params": search_parameters, } - logger.info("range_seach: end") + logger.info("range_search: end") return lims, D, I, R, P, None diff --git a/benchs/bench_gpu_1bn.py b/benchs/bench_gpu_1bn.py index 04f6fa927..fc17e9de7 100644 --- a/benchs/bench_gpu_1bn.py +++ b/benchs/bench_gpu_1bn.py @@ -166,7 +166,7 @@ def rate_limited_imap(f, l): class IdentPreproc: - """a pre-processor is either a faiss.VectorTransform or an IndentPreproc""" + """a pre-processor is either a faiss.VectorTransform or an IdentPreproc""" def __init__(self, d): self.d_in = self.d_out = d @@ -229,7 +229,7 @@ elif dbname == 'Deep1B': xb = mmap_fvecs('deep1b/base.fvecs') xq = mmap_fvecs('deep1b/deep1B_queries.fvecs') xt = mmap_fvecs('deep1b/learn.fvecs') - # deep1B's train is is outrageously big + # deep1B's train is outrageously big xt = xt[:10 * 1000 * 1000] gt_I = ivecs_read('deep1b/deep1B_groundtruth.ivecs') @@ -740,11 +740,11 @@ def eval_dataset(index, preproc): print("1-R@%d: %.4f" % (rank, nok / float(nq)), end=' ') print() if I_fname: - I_fname_i = I_fname % I + I_fname_i = I_fname % nprobe print("storing", I_fname_i) np.save(I, I_fname_i) if D_fname: - D_fname_i = I_fname % I + D_fname_i = D_fname % nprobe print("storing", D_fname_i) np.save(D, D_fname_i) diff --git a/benchs/bench_hybrid_cpu_gpu.py b/benchs/bench_hybrid_cpu_gpu.py index 7fe3e2546..f6df7b8c4 100644 --- a/benchs/bench_hybrid_cpu_gpu.py +++ b/benchs/bench_hybrid_cpu_gpu.py @@ -282,7 +282,7 @@ def main(): group = parser.add_argument_group('dataset options') aa('--nq', type=int, default=int(10e5), - help="nb queries (queries will be duplicated if below that number") + help="nb queries (queries will be duplicated if below that number)") aa('--db', default='bigann10M', help='dataset') group = parser.add_argument_group('index options') @@ -311,7 +311,7 @@ def main(): help="how to search" ) aa('--ivf_quant_nlist', type=int, default=1024, - help="nb of invlists for IVF quantizer") + help="nb of inverted lists for IVF quantizer") aa('--batch_size', type=int, default=-1, help="batch size for tiled CPU / GPU computation (-1= no tiling)") aa('--n_autotune', type=int, default=300, @@ -540,7 +540,7 @@ def main(): op.all_experiments = [] op.platform = { "loadavg": open("/proc/loadavg", "r").readlines(), - "procesor": [l for l in open("/proc/cpuinfo") if "model name" in l][0], + "processor": [l for l in open("/proc/cpuinfo") if "model name" in l][0], "GPU": list(os.popen("nvidia-smi", "r")), "mem": open("/proc/meminfo", "r").readlines(), "pid": os.getpid() diff --git a/benchs/bench_ivf_flat_panorama.py b/benchs/bench_ivf_flat_panorama.py index a7fab3c03..85cf84059 100644 --- a/benchs/bench_ivf_flat_panorama.py +++ b/benchs/bench_ivf_flat_panorama.py @@ -93,6 +93,7 @@ eval_and_plot(f"IVF{nlist},Flat") eval_and_plot(f"PCA{d},IVF{nlist},FlatPanorama{nlevels}") plt.title("IVF Flat Indexes on GIST1M") +plt.title("Indices on GIST1M") plt.xlabel(f"Recall@{k}") plt.ylabel("QPS") plt.yscale("log") diff --git a/benchs/bench_partition.py b/benchs/bench_partition.py index 5a26584bb..04609e7de 100644 --- a/benchs/bench_partition.py +++ b/benchs/bench_partition.py @@ -59,7 +59,7 @@ def do_partition(n, qin, maxval=65536, seed=123, id_type='int64'): print( f"times {times.mean():.3f} µs (± {times.std():.4f} µs) nerr={nerr} " - f"bissect {stats.bissect_cycles / 1e6:.3f} Mcy " + f"bisect {stats.bissect_cycles / 1e6:.3f} Mcy " f"compress {stats.compress_cycles / 1e6:.3f} Mcy" ) diff --git a/benchs/bench_polysemous_1bn.py b/benchs/bench_polysemous_1bn.py index 8cbfe0e0d..a73722e09 100644 --- a/benchs/bench_polysemous_1bn.py +++ b/benchs/bench_polysemous_1bn.py @@ -69,7 +69,7 @@ elif dbname == 'Deep1B': xb = mmap_fvecs('deep1b/base.fvecs') xq = mmap_fvecs('deep1b/deep1B_queries.fvecs') xt = mmap_fvecs('deep1b/learn.fvecs') - # deep1B's train is is outrageously big + # deep1B's train is outrageously big xt = xt[:10 * 1000 * 1000] gt = ivecs_read('deep1b/deep1B_groundtruth.ivecs') diff --git a/benchs/bench_pq_tables.py b/benchs/bench_pq_tables.py index b22a68573..b7ea31cf1 100644 --- a/benchs/bench_pq_tables.py +++ b/benchs/bench_pq_tables.py @@ -73,5 +73,5 @@ for have_threads in False, True: res = run_bench(M * dsub, dsub, nbit, metric) allres.append(res) allres = np.array(allres) - print("formated result:") + print("formatted result:") print(format_tab(allres)) diff --git a/benchs/distributed_ondisk/combined_index.py b/benchs/distributed_ondisk/combined_index.py index a9449f6e0..45f3618d1 100755 --- a/benchs/distributed_ondisk/combined_index.py +++ b/benchs/distributed_ondisk/combined_index.py @@ -136,7 +136,6 @@ class CombinedIndex: for idx in self.indexes: il = faiss.downcast_InvertedLists( faiss.extract_index_ivf(idx).invlists) - il.prefetch_nthread il.prefetch_nthread = nt def set_omp_num_threads(self, nt): diff --git a/benchs/distributed_ondisk/run_on_cluster.bash b/benchs/distributed_ondisk/run_on_cluster.bash index 886fdcbba..c203d3645 100755 --- a/benchs/distributed_ondisk/run_on_cluster.bash +++ b/benchs/distributed_ondisk/run_on_cluster.bash @@ -42,11 +42,11 @@ elif [ $todo == test_kmeans_1 ]; then --k $k --gpu -1 elif [ $todo == test_kmeans_2 ]; then - # distrbuted run, with one local server per GPU + # distributed run, with one local server per GPU ngpu=$( echo /dev/nvidia? | wc -w ) baseport=12012 - # kill background porcesses on output of this script + # kill background processes on output of this script trap 'kill -HUP 0' 0 hostports='' diff --git a/demos/demo_auto_tune.py b/demos/demo_auto_tune.py index 7e98a0116..73ddc825f 100755 --- a/demos/demo_auto_tune.py +++ b/demos/demo_auto_tune.py @@ -101,7 +101,7 @@ use_gpu = False if use_gpu: - # if this fails, it means that the GPU version was not comp + # if this fails, it means that the GPU version was not compiled assert faiss.StandardGpuResources, \ "Faiss was not compiled with GPU support, or loading _swigfaiss_gpu.so failed" res = faiss.StandardGpuResources() diff --git a/demos/demo_distributed_kmeans_torch.py b/demos/demo_distributed_kmeans_torch.py index a0d78ad81..c3f9780af 100644 --- a/demos/demo_distributed_kmeans_torch.py +++ b/demos/demo_distributed_kmeans_torch.py @@ -113,7 +113,7 @@ class DatasetAssignDistributedGPU(clustering.DatasetAssign): torch.distributed.reduce(sum_per_centroid, 0) if rank == 0: - # gather deos not support tensors of different sizes + # gather does not support tensors of different sizes # should be implemented with point-to-point communication assert np.all(self.sizes == self.sizes[0]) device = self.device diff --git a/demos/demo_imi_flat.cpp b/demos/demo_imi_flat.cpp index a7f70bca1..ccf9072ee 100644 --- a/demos/demo_imi_flat.cpp +++ b/demos/demo_imi_flat.cpp @@ -63,14 +63,14 @@ int main() { ncentroids, nb); - // the coarse quantizer should not be dealloced before the index + // the coarse quantizer should not be deallocated before the index // 4 = nb of bytes per code (d must be a multiple of this) // 8 = nb of bits per sub-code (almost always 8) faiss::MetricType metric = faiss::METRIC_L2; // can be METRIC_INNER_PRODUCT faiss::IndexIVFFlat index(&coarse_quantizer, d, ncentroids, metric); index.quantizer_trains_alone = true; - // define the number of probes. 2048 is for high-dim, overkilled in practice + // define the number of probes. 2048 is for high-dim, overkill in practice // Use 4-1024 depending on the trade-off speed accuracy that you want index.nprobe = 2048; diff --git a/demos/demo_imi_pq.cpp b/demos/demo_imi_pq.cpp index 1e6387312..96019f08d 100644 --- a/demos/demo_imi_pq.cpp +++ b/demos/demo_imi_pq.cpp @@ -74,7 +74,7 @@ int main() { ncentroids, nb); - // the coarse quantizer should not be dealloced before the index + // the coarse quantizer should not be deallocated before the index // 4 = nb of bytes per code (d must be a multiple of this) // 8 = nb of bits per sub-code (almost always 8) faiss::IndexIVFPQ index( diff --git a/demos/demo_ivfpq_indexing.cpp b/demos/demo_ivfpq_indexing.cpp index aa84542cf..bf455dc54 100644 --- a/demos/demo_ivfpq_indexing.cpp +++ b/demos/demo_ivfpq_indexing.cpp @@ -41,7 +41,7 @@ int main() { // a reasonable number of centroids to index nb vectors int ncentroids = int(4 * sqrt(nb)); - // the coarse quantizer should not be dealloced before the index + // the coarse quantizer should not be deallocated before the index // 4 = nb of bytes per code (d must be a multiple of this) // 8 = nb of bits per sub-code (almost always 8) faiss::IndexIVFPQ index(&coarse_quantizer, d, ncentroids, 4, 8); diff --git a/demos/demo_residual_quantizer.cpp b/demos/demo_residual_quantizer.cpp index 508838143..cacc53783 100644 --- a/demos/demo_residual_quantizer.cpp +++ b/demos/demo_residual_quantizer.cpp @@ -76,7 +76,7 @@ int main() { faiss::ResidualQuantizer rq( d, 7, 6, faiss::AdditiveQuantizer::ST_norm_qint8); - // do cheap an inaccurate training + // do cheap and inaccurate training rq.cp.niter = 5; rq.max_beam_size = 5; rq.train_type = 0; diff --git a/demos/demo_sift1M.cpp b/demos/demo_sift1M.cpp index 99a2e9c5b..f9a0fb699 100644 --- a/demos/demo_sift1M.cpp +++ b/demos/demo_sift1M.cpp @@ -23,7 +23,7 @@ * * http://corpus-texmex.irisa.fr/ * - * and unzip it to the sudirectory sift1M. + * and unzip it to the subdirectory sift1M. **/ /***************************************************** diff --git a/demos/offline_ivf/README.md b/demos/offline_ivf/README.md index df848ba0a..5f8e926a2 100644 --- a/demos/offline_ivf/README.md +++ b/demos/offline_ivf/README.md @@ -48,5 +48,4 @@ b. A new dataset needs to be prepared (step 1) before passing it to the query ve 6. We can always run the consistency-check for sanity checks! -`python run.py --command consistency_check--config config_ssnpp.yaml --xb ssnpp_1B` - +`python run.py --command consistency_check --config config_ssnpp.yaml --xb ssnpp_1B` diff --git a/demos/offline_ivf/offline_ivf.py b/demos/offline_ivf/offline_ivf.py index 8fe7d6b39..75430d65c 100644 --- a/demos/offline_ivf/offline_ivf.py +++ b/demos/offline_ivf/offline_ivf.py @@ -284,7 +284,7 @@ class OfflineIVF: logging.info(jj) assert ( jj <= self.shard_size - ), f"jj {jj} and shard_zide {self.shard_size}" + ), f"jj {jj} and shard_size {self.shard_size}" if jj == self.shard_size: break logging.info(f"writing {sfn}...") diff --git a/demos/rocksdb_ivf/README.md b/demos/rocksdb_ivf/README.md index cf29ee2fd..fca65fcd2 100644 --- a/demos/rocksdb_ivf/README.md +++ b/demos/rocksdb_ivf/README.md @@ -19,5 +19,5 @@ make -C build -j$(nproc) ``` cd ~/faiss/demos/rocksdb_ivf/build -./rocksdb_ivf test_db +./demo_rocksdb_ivf test_db ``` diff --git a/faiss/gpu/utils/blockselect/BlockSelectFloatF512.cu b/faiss/gpu/utils/blockselect/BlockSelectFloatF512.cu index c8060af2f..b0a324427 100644 --- a/faiss/gpu/utils/blockselect/BlockSelectFloatF512.cu +++ b/faiss/gpu/utils/blockselect/BlockSelectFloatF512.cu @@ -12,5 +12,5 @@ namespace gpu { BLOCK_SELECT_IMPL(float, false, 512, 8); -} +} // namespace gpu } // namespace faiss diff --git a/faiss/gpu/utils/blockselect/BlockSelectFloatT1024.cu b/faiss/gpu/utils/blockselect/BlockSelectFloatT1024.cu index 3d0fb6589..14459f4b2 100644 --- a/faiss/gpu/utils/blockselect/BlockSelectFloatT1024.cu +++ b/faiss/gpu/utils/blockselect/BlockSelectFloatT1024.cu @@ -12,5 +12,5 @@ namespace gpu { BLOCK_SELECT_IMPL(float, true, 1024, 8); -} +} // namespace gpu } // namespace faiss diff --git a/faiss/gpu/utils/blockselect/BlockSelectFloatT512.cu b/faiss/gpu/utils/blockselect/BlockSelectFloatT512.cu index 485d13c26..92bc98d32 100644 --- a/faiss/gpu/utils/blockselect/BlockSelectFloatT512.cu +++ b/faiss/gpu/utils/blockselect/BlockSelectFloatT512.cu @@ -12,5 +12,5 @@ namespace gpu { BLOCK_SELECT_IMPL(float, true, 512, 8); -} +} // namespace gpu } // namespace faiss diff --git a/faiss/gpu/utils/warpselect/WarpSelectFloatF512.cu b/faiss/gpu/utils/warpselect/WarpSelectFloatF512.cu index c35dce5b8..19dbc514c 100644 --- a/faiss/gpu/utils/warpselect/WarpSelectFloatF512.cu +++ b/faiss/gpu/utils/warpselect/WarpSelectFloatF512.cu @@ -12,5 +12,5 @@ namespace gpu { WARP_SELECT_IMPL(float, false, 512, 8); -} +} // namespace gpu } // namespace faiss diff --git a/faiss/gpu/utils/warpselect/WarpSelectFloatT1024.cu b/faiss/gpu/utils/warpselect/WarpSelectFloatT1024.cu index f803b1b49..46f2a36ab 100644 --- a/faiss/gpu/utils/warpselect/WarpSelectFloatT1024.cu +++ b/faiss/gpu/utils/warpselect/WarpSelectFloatT1024.cu @@ -12,5 +12,5 @@ namespace gpu { WARP_SELECT_IMPL(float, true, 1024, 8); -} +} // namespace gpu } // namespace faiss diff --git a/faiss/gpu/utils/warpselect/WarpSelectFloatT512.cu b/faiss/gpu/utils/warpselect/WarpSelectFloatT512.cu index 10782f6f1..b436b2d92 100644 --- a/faiss/gpu/utils/warpselect/WarpSelectFloatT512.cu +++ b/faiss/gpu/utils/warpselect/WarpSelectFloatT512.cu @@ -12,5 +12,5 @@ namespace gpu { WARP_SELECT_IMPL(float, true, 512, 8); -} +} // namespace gpu } // namespace faiss diff --git a/faiss/impl/IDSelector.h b/faiss/impl/IDSelector.h index 5c24f1e69..be7391520 100644 --- a/faiss/impl/IDSelector.h +++ b/faiss/impl/IDSelector.h @@ -131,7 +131,7 @@ struct IDSelectorAll : IDSelector { virtual ~IDSelectorAll() {} }; -/// does an AND operation on the the two given IDSelector's is_membership +/// does an AND operation on the two given IDSelector's is_membership /// results. struct IDSelectorAnd : IDSelector { const IDSelector* lhs; @@ -144,7 +144,7 @@ struct IDSelectorAnd : IDSelector { virtual ~IDSelectorAnd() {} }; -/// does an OR operation on the the two given IDSelector's is_membership +/// does an OR operation on the two given IDSelector's is_membership /// results. struct IDSelectorOr : IDSelector { const IDSelector* lhs; @@ -157,7 +157,7 @@ struct IDSelectorOr : IDSelector { virtual ~IDSelectorOr() {} }; -/// does an XOR operation on the the two given IDSelector's is_membership +/// does an XOR operation on the two given IDSelector's is_membership /// results. struct IDSelectorXOr : IDSelector { const IDSelector* lhs; diff --git a/faiss/invlists/BlockInvertedLists.h b/faiss/invlists/BlockInvertedLists.h index 7f608a87a..fcbe06b92 100644 --- a/faiss/invlists/BlockInvertedLists.h +++ b/faiss/invlists/BlockInvertedLists.h @@ -20,7 +20,7 @@ struct IDSelector; /** Inverted Lists that are organized by blocks. * * Different from the regular inverted lists, the codes are organized by blocks - * of size block_size bytes that reprsent a set of n_per_block. Therefore, code + * of size block_size bytes that represent a set of n_per_block. Therefore, code * allocations are always rounded up to block_size bytes. The codes are also * aligned on 32-byte boundaries for use with SIMD. * diff --git a/faiss/invlists/DirectMap.cpp b/faiss/invlists/DirectMap.cpp index 50ff71177..a83a0e5e9 100644 --- a/faiss/invlists/DirectMap.cpp +++ b/faiss/invlists/DirectMap.cpp @@ -53,7 +53,7 @@ void DirectMap::set_type( for (long ofs = 0; ofs < list_size; ofs++) { FAISS_THROW_IF_NOT_MSG( 0 <= idlist[ofs] && idlist[ofs] < ntotal, - "direct map supported only for seuquential ids"); + "direct map supported only for sequential ids"); array[idlist[ofs]] = lo_build(key, ofs); } } else if (new_type == Hashtable) { diff --git a/faiss/invlists/InvertedLists.cpp b/faiss/invlists/InvertedLists.cpp index e079b6c42..448b96973 100644 --- a/faiss/invlists/InvertedLists.cpp +++ b/faiss/invlists/InvertedLists.cpp @@ -229,7 +229,7 @@ bool InvertedLists::is_empty(size_t list_no, void* inverted_list_context) } } -// implemnent iterator on top of get_codes / get_ids +// implement iterator on top of get_codes / get_ids namespace { struct CodeArrayIterator : InvertedListsIterator { diff --git a/faiss/invlists/OnDiskInvertedLists.cpp b/faiss/invlists/OnDiskInvertedLists.cpp index cc37fd1e2..a53668a6d 100644 --- a/faiss/invlists/OnDiskInvertedLists.cpp +++ b/faiss/invlists/OnDiskInvertedLists.cpp @@ -372,7 +372,7 @@ OnDiskInvertedLists::~OnDiskInvertedLists() { if (ptr != nullptr) { int err = munmap(ptr, totsize); if (err != 0) { - fprintf(stderr, "mumap error: %s", strerror(errno)); + fprintf(stderr, "munmap error: %s", strerror(errno)); } } delete locks; diff --git a/faiss/invlists/OnDiskInvertedLists.h b/faiss/invlists/OnDiskInvertedLists.h index 52953f9c0..b809d4c79 100644 --- a/faiss/invlists/OnDiskInvertedLists.h +++ b/faiss/invlists/OnDiskInvertedLists.h @@ -121,7 +121,7 @@ struct OnDiskInvertedLists : InvertedLists { LockLevels* locks; - // encapsulates the threads that are busy prefeteching + // encapsulates the threads that are busy prefetching struct OngoingPrefetch; OngoingPrefetch* pf; int prefetch_nthread; diff --git a/tutorial/cpp/7-PQFastScan.cpp b/tutorial/cpp/7-PQFastScan.cpp index 3e0894890..817993c6b 100644 --- a/tutorial/cpp/7-PQFastScan.cpp +++ b/tutorial/cpp/7-PQFastScan.cpp @@ -72,4 +72,4 @@ int main() { delete[] xq; return 0; -} // namespace facebook::detail +}