From 42483099145af4e6b005dfb1ef331a97ab535efa Mon Sep 17 00:00:00 2001 From: matthijs Date: Tue, 9 Jan 2018 06:44:43 -0800 Subject: [PATCH] sync with FB version 2018-01-09 - adding HNSW indexing method - simultaneous search and reconstruction for IndexIVFPQ --- docs/html/2-IVFFlat_8cpp_source.html | 2 +- docs/html/AutoTune_8cpp_source.html | 1702 ++++----- docs/html/AutoTune_8h_source.html | 38 +- docs/html/Clustering_8cpp_source.html | 339 +- docs/html/Clustering_8h_source.html | 2 +- docs/html/CompareIVFFlat_8cu_source.html | 4 +- docs/html/CompareIVFPQGrid_8cu_source.html | 2 +- docs/html/CompareIVFPQ_8cu_source.html | 2 +- docs/html/DeviceDefs_8cuh_source.html | 48 +- docs/html/DeviceTensor-inl_8cuh_source.html | 10 +- docs/html/DeviceTensor_8cuh_source.html | 6 +- docs/html/FlatIndex_8cu_source.html | 4 +- docs/html/Float16_8cu_source.html | 24 +- docs/html/Float16_8cuh_source.html | 2 +- docs/html/GpuAutoTune_8cpp_source.html | 24 +- docs/html/GpuIndexFlat_8cu_source.html | 2 +- docs/html/GpuIndexIVFFlat_8cu_source.html | 16 +- docs/html/GpuIndexIVFFlat_8h_source.html | 2 +- docs/html/GpuIndexIVFPQ_8cu_source.html | 24 +- docs/html/GpuIndexIVF_8cu_source.html | 485 +-- docs/html/GpuIndexIVF_8h_source.html | 16 +- docs/html/Heap_8h_source.html | 2 +- docs/html/HostTensor-inl_8cuh_source.html | 10 +- docs/html/HostTensor_8cuh_source.html | 6 +- docs/html/IVFFlatScan_8cu_source.html | 2 +- docs/html/IVFFlat_8cu_source.html | 693 ++-- docs/html/IVFFlat_8cuh_source.html | 8 +- docs/html/IVFPQ_8cu_source.html | 8 +- docs/html/IndexFlat_8cpp_source.html | 14 +- docs/html/IndexHNSW_8cpp_source.html | 2405 +++++++++++++ docs/html/IndexHNSW_8h_source.html | 473 +++ docs/html/IndexIVFPQ_8cpp_source.html | 3099 +++++++++-------- docs/html/IndexIVFPQ_8h_source.html | 568 +-- docs/html/IndexIVF_8cpp_source.html | 1409 ++++---- docs/html/IndexIVF_8h_source.html | 518 +-- docs/html/IndexLSH_8cpp_source.html | 2 +- docs/html/IndexLSH_8h_source.html | 2 +- docs/html/IndexPQ_8cpp_source.html | 1225 +++---- docs/html/IndexPQ_8h_source.html | 14 +- docs/html/IndexProxy_8cpp_source.html | 6 +- .../IndexScalarQuantizer_8cpp_source.html | 2065 ++++++----- docs/html/IndexScalarQuantizer_8h_source.html | 208 +- docs/html/Index_8cpp_source.html | 150 +- docs/html/Index_8h_source.html | 238 +- docs/html/Limits_8cuh_source.html | 88 +- docs/html/LoadStoreOperators_8cuh_source.html | 130 +- docs/html/MathOperators_8cuh_source.html | 542 +-- docs/html/MetaIndexes_8cpp_source.html | 10 +- docs/html/NoTypeTensor_8cuh_source.html | 6 +- ...ScanMultiPassNoPrecomputed_8cu_source.html | 213 +- docs/html/PolysemousTraining_8cpp_source.html | 2 +- docs/html/ProductQuantizer_8cpp_source.html | 4 +- docs/html/Select_8cuh_source.html | 826 ++--- docs/html/Tensor-inl_8cuh_source.html | 1368 ++++---- docs/html/Tensor_8cuh_source.html | 1242 ++++--- .../html/TestGpuIndexIVFFlat_8cpp_source.html | 2 +- docs/html/Transpose_8cuh_source.html | 6 +- docs/html/VectorTransform_8cpp_source.html | 1886 +++++----- docs/html/VectorTransform_8h_source.html | 511 +-- docs/html/WarpShuffles_8cuh_source.html | 179 +- docs/html/WriteIndex_8cpp_source.html | 2 +- docs/html/annotated.html | 436 +-- docs/html/classes.html | 129 +- ...sfaiss_1_1gpu_1_1DeviceTensor-members.html | 6 +- .../classfaiss_1_1gpu_1_1DeviceTensor.html | 24 +- ...classfaiss_1_1gpu_1_1GpuIndex-members.html | 11 +- docs/html/classfaiss_1_1gpu_1_1GpuIndex.html | 12 +- ...sfaiss_1_1gpu_1_1GpuIndexFlat-members.html | 19 +- .../classfaiss_1_1gpu_1_1GpuIndexFlat.html | 8 +- ...aiss_1_1gpu_1_1GpuIndexFlatIP-members.html | 19 +- .../classfaiss_1_1gpu_1_1GpuIndexFlatIP.html | 8 +- ...aiss_1_1gpu_1_1GpuIndexFlatL2-members.html | 19 +- .../classfaiss_1_1gpu_1_1GpuIndexFlatL2.html | 8 +- ...ssfaiss_1_1gpu_1_1GpuIndexIVF-members.html | 17 +- .../classfaiss_1_1gpu_1_1GpuIndexIVF.html | 14 +- ...iss_1_1gpu_1_1GpuIndexIVFFlat-members.html | 19 +- .../classfaiss_1_1gpu_1_1GpuIndexIVFFlat.html | 14 +- ...faiss_1_1gpu_1_1GpuIndexIVFPQ-members.html | 21 +- .../classfaiss_1_1gpu_1_1GpuIndexIVFPQ.html | 14 +- ...assfaiss_1_1gpu_1_1HostTensor-members.html | 6 +- .../html/classfaiss_1_1gpu_1_1HostTensor.html | 24 +- docs/html/classfaiss_1_1gpu_1_1IVFFlat.html | 6 +- ...assfaiss_1_1gpu_1_1IndexProxy-members.html | 11 +- .../html/classfaiss_1_1gpu_1_1IndexProxy.html | 10 +- .../classfaiss_1_1gpu_1_1Tensor-members.html | 6 +- docs/html/classfaiss_1_1gpu_1_1Tensor.html | 72 +- ...ssfaiss_1_1gpu_1_1detail_1_1SubTensor.html | 6 +- ...TensorType_00_010_00_01PtrTraits_01_4.html | 2 +- docs/html/demo__imi__flat_8cpp_source.html | 2 +- docs/html/demo__sift1M_8cpp_source.html | 14 +- docs/html/files.html | 42 +- docs/html/functions.html | 45 +- docs/html/functions_0x63.html | 53 +- docs/html/functions_0x65.html | 18 +- docs/html/functions_0x66.html | 3 + docs/html/functions_0x67.html | 14 +- docs/html/functions_0x68.html | 3 + docs/html/functions_0x69.html | 24 +- docs/html/functions_0x6c.html | 3 + docs/html/functions_0x6d.html | 9 +- docs/html/functions_0x6e.html | 19 +- docs/html/functions_0x6f.html | 18 +- docs/html/functions_0x70.html | 1 + docs/html/functions_0x71.html | 11 +- docs/html/functions_0x72.html | 83 +- docs/html/functions_0x73.html | 79 +- docs/html/functions_0x74.html | 35 +- docs/html/functions_0x75.html | 7 +- docs/html/functions_func.html | 45 +- docs/html/functions_func_0x63.html | 25 +- docs/html/functions_func_0x65.html | 3 + docs/html/functions_func_0x66.html | 3 + docs/html/functions_func_0x67.html | 20 +- docs/html/functions_func_0x68.html | 5 +- docs/html/functions_func_0x69.html | 12 +- docs/html/functions_func_0x6e.html | 6 + docs/html/functions_func_0x6f.html | 11 +- docs/html/functions_func_0x72.html | 87 +- docs/html/functions_func_0x73.html | 60 +- docs/html/functions_func_0x74.html | 37 +- docs/html/functions_type.html | 4 + docs/html/functions_vars.html | 626 +--- docs/html/functions_vars_0x62.html | 162 + docs/html/functions_vars_0x63.html | 186 + docs/html/functions_vars_0x64.html | 183 + docs/html/functions_vars_0x65.html | 150 + docs/html/functions_vars_0x66.html | 141 + docs/html/functions_vars_0x67.html | 144 + docs/html/functions_vars_0x68.html | 144 + docs/html/functions_vars_0x69.html | 165 + docs/html/functions_vars_0x6b.html | 152 + docs/html/functions_vars_0x6c.html | 153 + docs/html/functions_vars_0x6d.html | 185 + docs/html/functions_vars_0x6e.html | 192 + docs/html/functions_vars_0x6f.html | 150 + docs/html/functions_vars_0x70.html | 163 + docs/html/functions_vars_0x71.html | 148 + docs/html/functions_vars_0x72.html | 160 + docs/html/functions_vars_0x73.html | 179 + docs/html/functions_vars_0x74.html | 160 + docs/html/functions_vars_0x75.html | 166 + docs/html/functions_vars_0x76.html | 144 + docs/html/functions_vars_0x77.html | 141 + docs/html/functions_vars_0x78.html | 138 + docs/html/hierarchy.html | 357 +- docs/html/index__io_8cpp_source.html | 1609 +++++---- docs/html/index__io_8h_source.html | 4 +- docs/html/namespacefaiss.html | 87 +- docs/html/search/all_61.js | 6 +- docs/html/search/all_63.js | 19 +- docs/html/search/all_64.js | 7 +- docs/html/search/all_65.js | 4 + docs/html/search/all_66.js | 2 + docs/html/search/all_67.js | 11 +- docs/html/search/all_68.js | 3 + docs/html/search/all_69.js | 29 +- docs/html/search/all_6c.js | 4 +- docs/html/search/all_6d.js | 7 +- docs/html/search/all_6e.js | 5 +- docs/html/search/all_6f.js | 6 +- docs/html/search/all_70.js | 3 +- docs/html/search/all_71.js | 5 +- docs/html/search/all_72.js | 14 +- docs/html/search/all_73.js | 17 +- docs/html/search/all_74.js | 6 +- docs/html/search/all_75.js | 1 + docs/html/search/all_76.js | 5 +- docs/html/search/classes_64.js | 7 +- docs/html/search/classes_66.js | 3 +- docs/html/search/classes_67.js | 1 + docs/html/search/classes_68.js | 2 + docs/html/search/classes_69.js | 8 +- docs/html/search/classes_6c.js | 1 + docs/html/search/classes_70.js | 1 + docs/html/search/classes_72.js | 1 + docs/html/search/classes_73.js | 1 + docs/html/search/classes_76.js | 3 +- docs/html/search/functions_61.js | 5 +- docs/html/search/functions_63.js | 5 +- docs/html/search/functions_65.js | 1 + docs/html/search/functions_66.js | 1 + docs/html/search/functions_67.js | 2 + docs/html/search/functions_68.js | 1 + docs/html/search/functions_69.js | 4 +- docs/html/search/functions_6e.js | 2 + docs/html/search/functions_6f.js | 3 +- docs/html/search/functions_72.js | 11 +- docs/html/search/functions_73.js | 13 +- docs/html/search/functions_74.js | 6 +- docs/html/search/search.js | 2 +- docs/html/search/typedefs_69.js | 2 +- docs/html/search/typedefs_73.html | 26 + docs/html/search/typedefs_73.js | 4 + docs/html/search/variables_61.js | 3 +- docs/html/search/variables_63.js | 12 +- docs/html/search/variables_65.js | 5 +- docs/html/search/variables_69.js | 1 + docs/html/search/variables_6c.js | 1 + docs/html/search/variables_6d.js | 3 +- docs/html/search/variables_6e.js | 3 +- docs/html/search/variables_6f.js | 3 +- docs/html/search/variables_70.js | 2 +- docs/html/search/variables_71.js | 5 +- docs/html/search/variables_75.js | 1 + docs/html/structfaiss_1_1ArgSort.html | 2 +- .../structfaiss_1_1AutoTuneCriterion.html | 2 +- ...tfaiss_1_1BaseShiftDistanceCorrection.html | 2 +- docs/html/structfaiss_1_1CodeCmp.html | 2 +- ...structfaiss_1_1Distance2Level-members.html | 116 + docs/html/structfaiss_1_1Distance2Level.html | 167 + docs/html/structfaiss_1_1Distance2Level.png | Bin 0 -> 1414 bytes ...structfaiss_1_1Distance2xXPQ4-members.html | 120 + docs/html/structfaiss_1_1Distance2xXPQ4.html | 180 + docs/html/structfaiss_1_1Distance2xXPQ4.png | Bin 0 -> 1052 bytes .../structfaiss_1_1DistanceXPQ4-members.html | 120 + docs/html/structfaiss_1_1DistanceXPQ4.html | 180 + docs/html/structfaiss_1_1DistanceXPQ4.png | Bin 0 -> 1039 bytes .../structfaiss_1_1FlatL2Dis-members.html | 117 + docs/html/structfaiss_1_1FlatL2Dis.html | 165 + docs/html/structfaiss_1_1FlatL2Dis.png | Bin 0 -> 706 bytes ...ss_1_1GenericDistanceComputer-members.html | 115 + ...tructfaiss_1_1GenericDistanceComputer.html | 162 + ...structfaiss_1_1GenericDistanceComputer.png | Bin 0 -> 800 bytes docs/html/structfaiss_1_1HNSW-members.html | 131 + docs/html/structfaiss_1_1HNSW.html | 346 ++ .../structfaiss_1_1HNSWStats-members.html | 112 + docs/html/structfaiss_1_1HNSWStats.html | 140 + ...s_1_1HNSW_1_1DistanceComputer-members.html | 109 + ...ructfaiss_1_1HNSW_1_1DistanceComputer.html | 152 + ...tructfaiss_1_1HNSW_1_1DistanceComputer.png | Bin 0 -> 2716 bytes docs/html/structfaiss_1_1Index-members.html | 9 +- docs/html/structfaiss_1_1Index.html | 146 +- docs/html/structfaiss_1_1Index.png | Bin 5449 -> 6221 bytes .../structfaiss_1_1Index2Layer-members.html | 135 + docs/html/structfaiss_1_1Index2Layer.html | 418 +++ docs/html/structfaiss_1_1Index2Layer.png | Bin 0 -> 526 bytes .../structfaiss_1_1IndexFlat-members.html | 11 +- docs/html/structfaiss_1_1IndexFlat.html | 12 +- .../structfaiss_1_1IndexFlat1D-members.html | 13 +- docs/html/structfaiss_1_1IndexFlat1D.html | 12 +- .../structfaiss_1_1IndexFlatIP-members.html | 11 +- docs/html/structfaiss_1_1IndexFlatIP.html | 12 +- .../structfaiss_1_1IndexFlatL2-members.html | 11 +- docs/html/structfaiss_1_1IndexFlatL2.html | 12 +- ...faiss_1_1IndexFlatL2BaseShift-members.html | 13 +- .../structfaiss_1_1IndexFlatL2BaseShift.html | 12 +- .../structfaiss_1_1IndexHNSW-members.html | 140 + docs/html/structfaiss_1_1IndexHNSW.html | 415 +++ docs/html/structfaiss_1_1IndexHNSW.png | Bin 0 -> 1677 bytes ...tructfaiss_1_1IndexHNSW2Level-members.html | 143 + docs/html/structfaiss_1_1IndexHNSW2Level.html | 245 ++ docs/html/structfaiss_1_1IndexHNSW2Level.png | Bin 0 -> 879 bytes .../structfaiss_1_1IndexHNSWFlat-members.html | 142 + docs/html/structfaiss_1_1IndexHNSWFlat.html | 242 ++ docs/html/structfaiss_1_1IndexHNSWFlat.png | Bin 0 -> 798 bytes .../structfaiss_1_1IndexHNSWPQ-members.html | 142 + docs/html/structfaiss_1_1IndexHNSWPQ.html | 242 ++ docs/html/structfaiss_1_1IndexHNSWPQ.png | Bin 0 -> 746 bytes .../structfaiss_1_1IndexHNSWSQ-members.html | 142 + docs/html/structfaiss_1_1IndexHNSWSQ.html | 242 ++ docs/html/structfaiss_1_1IndexHNSWSQ.png | Bin 0 -> 748 bytes .../structfaiss_1_1IndexIDMap-members.html | 11 +- docs/html/structfaiss_1_1IndexIDMap.html | 14 +- .../structfaiss_1_1IndexIDMap2-members.html | 13 +- docs/html/structfaiss_1_1IndexIDMap2.html | 8 +- .../html/structfaiss_1_1IndexIVF-members.html | 45 +- docs/html/structfaiss_1_1IndexIVF.html | 337 +- docs/html/structfaiss_1_1IndexIVF.png | Bin 2179 -> 2402 bytes .../structfaiss_1_1IndexIVFFlat-members.html | 47 +- docs/html/structfaiss_1_1IndexIVFFlat.html | 129 +- docs/html/structfaiss_1_1IndexIVFFlat.png | Bin 721 -> 1196 bytes .../structfaiss_1_1IndexIVFPQ-members.html | 43 +- docs/html/structfaiss_1_1IndexIVFPQ.html | 251 +- docs/html/structfaiss_1_1IndexIVFPQ.png | Bin 1553 -> 1750 bytes ...uctfaiss_1_1IndexIVFPQCompact-members.html | 43 +- .../structfaiss_1_1IndexIVFPQCompact.html | 100 +- .../html/structfaiss_1_1IndexIVFPQCompact.png | Bin 1140 -> 1667 bytes .../structfaiss_1_1IndexIVFPQR-members.html | 47 +- docs/html/structfaiss_1_1IndexIVFPQR.html | 181 +- docs/html/structfaiss_1_1IndexIVFPQR.png | Bin 968 -> 1559 bytes docs/html/structfaiss_1_1IndexIVFPQStats.html | 2 +- ...ss_1_1IndexIVFScalarQuantizer-members.html | 47 +- ...tructfaiss_1_1IndexIVFScalarQuantizer.html | 140 +- ...structfaiss_1_1IndexIVFScalarQuantizer.png | Bin 910 -> 1365 bytes ...structfaiss_1_1IndexIVFStats-members.html} | 17 +- ...html => structfaiss_1_1IndexIVFStats.html} | 29 +- .../html/structfaiss_1_1IndexLSH-members.html | 19 +- docs/html/structfaiss_1_1IndexLSH.html | 14 +- docs/html/structfaiss_1_1IndexPQ-members.html | 27 +- docs/html/structfaiss_1_1IndexPQ.html | 18 +- ...uctfaiss_1_1IndexPreTransform-members.html | 10 +- .../structfaiss_1_1IndexPreTransform.html | 188 +- ...tructfaiss_1_1IndexRefineFlat-members.html | 11 +- docs/html/structfaiss_1_1IndexRefineFlat.html | 14 +- ...faiss_1_1IndexScalarQuantizer-members.html | 11 +- .../structfaiss_1_1IndexScalarQuantizer.html | 28 +- .../structfaiss_1_1IndexShards-members.html | 19 +- docs/html/structfaiss_1_1IndexShards.html | 14 +- ...uctfaiss_1_1IndexSplitVectors-members.html | 19 +- .../structfaiss_1_1IndexSplitVectors.html | 14 +- ...tructfaiss_1_1Level1Quantizer-members.html | 114 + docs/html/structfaiss_1_1Level1Quantizer.html | 182 + docs/html/structfaiss_1_1Level1Quantizer.png | Bin 0 -> 2276 bytes ...tructfaiss_1_1LinearTransform-members.html | 8 +- docs/html/structfaiss_1_1LinearTransform.html | 26 +- docs/html/structfaiss_1_1MinSumK.html | 4 +- ...tfaiss_1_1MultiIndexQuantizer-members.html | 9 +- .../structfaiss_1_1MultiIndexQuantizer.html | 22 +- .../structfaiss_1_1NopDistanceCorrection.html | 2 +- ...iss_1_1NormalizationTransform-members.html | 2 +- ...structfaiss_1_1NormalizationTransform.html | 8 +- .../structfaiss_1_1OPQMatrix-members.html | 20 +- docs/html/structfaiss_1_1OPQMatrix.html | 67 +- docs/html/structfaiss_1_1OperatingPoints.html | 4 +- .../structfaiss_1_1PCAMatrix-members.html | 20 +- docs/html/structfaiss_1_1PCAMatrix.html | 69 +- docs/html/structfaiss_1_1PQDis-members.html | 120 + docs/html/structfaiss_1_1PQDis.html | 174 + docs/html/structfaiss_1_1PQDis.png | Bin 0 -> 681 bytes docs/html/structfaiss_1_1ParameterSpace.html | 8 +- ...faiss_1_1RandomRotationMatrix-members.html | 12 +- .../structfaiss_1_1RandomRotationMatrix.html | 68 +- ...s_1_1ReconstructFromNeighbors-members.html | 124 + ...ructfaiss_1_1ReconstructFromNeighbors.html | 215 ++ ...ructfaiss_1_1RemapDimensionsTransform.html | 6 +- docs/html/structfaiss_1_1SQDis-members.html | 119 + docs/html/structfaiss_1_1SQDis.html | 171 + docs/html/structfaiss_1_1SQDis.png | Bin 0 -> 685 bytes ...tructfaiss_1_1ScalarQuantizer-members.html | 35 +- docs/html/structfaiss_1_1ScalarQuantizer.html | 9 + ...Quantizer_1_1DistanceComputer-members.html | 107 + ..._1ScalarQuantizer_1_1DistanceComputer.html | 122 + docs/html/structfaiss_1_1ScopeFileCloser.html | 2 +- docs/html/structfaiss_1_1SemiSortedArray.html | 2 +- docs/html/structfaiss_1_1SortedArray.html | 2 +- docs/html/structfaiss_1_1VectorTransform.html | 4 +- .../structfaiss_1_1VisitedTable-members.html | 110 + docs/html/structfaiss_1_1VisitedTable.html | 143 + .../structfaiss_1_1gpu_1_1BlockSelect.html | 4 +- ...1NumThreadQ_00_01ThreadsPerBlock_01_4.html | 2 +- ...ructfaiss_1_1gpu_1_1GpuParameterSpace.html | 2 +- ...imits_3_01Pair_3_01K_00_01V_01_4_01_4.html | 2 +- ...ctfaiss_1_1gpu_1_1Limits_3_01int_01_4.html | 2 +- .../html/structfaiss_1_1gpu_1_1LoadStore.html | 2 +- .../structfaiss_1_1gpu_1_1WarpSelect.html | 6 +- ...1NumThreadQ_00_01ThreadsPerBlock_01_4.html | 2 +- .../test__pairs__decoding_8cpp_source.html | 24 +- docs/html/utils_8cpp_source.html | 2058 +++++------ docs/html/utils_8h_source.html | 420 +-- 349 files changed, 28773 insertions(+), 14269 deletions(-) create mode 100644 docs/html/IndexHNSW_8cpp_source.html create mode 100644 docs/html/IndexHNSW_8h_source.html create mode 100644 docs/html/functions_vars_0x62.html create mode 100644 docs/html/functions_vars_0x63.html create mode 100644 docs/html/functions_vars_0x64.html create mode 100644 docs/html/functions_vars_0x65.html create mode 100644 docs/html/functions_vars_0x66.html create mode 100644 docs/html/functions_vars_0x67.html create mode 100644 docs/html/functions_vars_0x68.html create mode 100644 docs/html/functions_vars_0x69.html create mode 100644 docs/html/functions_vars_0x6b.html create mode 100644 docs/html/functions_vars_0x6c.html create mode 100644 docs/html/functions_vars_0x6d.html create mode 100644 docs/html/functions_vars_0x6e.html create mode 100644 docs/html/functions_vars_0x6f.html create mode 100644 docs/html/functions_vars_0x70.html create mode 100644 docs/html/functions_vars_0x71.html create mode 100644 docs/html/functions_vars_0x72.html create mode 100644 docs/html/functions_vars_0x73.html create mode 100644 docs/html/functions_vars_0x74.html create mode 100644 docs/html/functions_vars_0x75.html create mode 100644 docs/html/functions_vars_0x76.html create mode 100644 docs/html/functions_vars_0x77.html create mode 100644 docs/html/functions_vars_0x78.html create mode 100644 docs/html/search/typedefs_73.html create mode 100644 docs/html/search/typedefs_73.js create mode 100644 docs/html/structfaiss_1_1Distance2Level-members.html create mode 100644 docs/html/structfaiss_1_1Distance2Level.html create mode 100644 docs/html/structfaiss_1_1Distance2Level.png create mode 100644 docs/html/structfaiss_1_1Distance2xXPQ4-members.html create mode 100644 docs/html/structfaiss_1_1Distance2xXPQ4.html create mode 100644 docs/html/structfaiss_1_1Distance2xXPQ4.png create mode 100644 docs/html/structfaiss_1_1DistanceXPQ4-members.html create mode 100644 docs/html/structfaiss_1_1DistanceXPQ4.html create mode 100644 docs/html/structfaiss_1_1DistanceXPQ4.png create mode 100644 docs/html/structfaiss_1_1FlatL2Dis-members.html create mode 100644 docs/html/structfaiss_1_1FlatL2Dis.html create mode 100644 docs/html/structfaiss_1_1FlatL2Dis.png create mode 100644 docs/html/structfaiss_1_1GenericDistanceComputer-members.html create mode 100644 docs/html/structfaiss_1_1GenericDistanceComputer.html create mode 100644 docs/html/structfaiss_1_1GenericDistanceComputer.png create mode 100644 docs/html/structfaiss_1_1HNSW-members.html create mode 100644 docs/html/structfaiss_1_1HNSW.html create mode 100644 docs/html/structfaiss_1_1HNSWStats-members.html create mode 100644 docs/html/structfaiss_1_1HNSWStats.html create mode 100644 docs/html/structfaiss_1_1HNSW_1_1DistanceComputer-members.html create mode 100644 docs/html/structfaiss_1_1HNSW_1_1DistanceComputer.html create mode 100644 docs/html/structfaiss_1_1HNSW_1_1DistanceComputer.png create mode 100644 docs/html/structfaiss_1_1Index2Layer-members.html create mode 100644 docs/html/structfaiss_1_1Index2Layer.html create mode 100644 docs/html/structfaiss_1_1Index2Layer.png create mode 100644 docs/html/structfaiss_1_1IndexHNSW-members.html create mode 100644 docs/html/structfaiss_1_1IndexHNSW.html create mode 100644 docs/html/structfaiss_1_1IndexHNSW.png create mode 100644 docs/html/structfaiss_1_1IndexHNSW2Level-members.html create mode 100644 docs/html/structfaiss_1_1IndexHNSW2Level.html create mode 100644 docs/html/structfaiss_1_1IndexHNSW2Level.png create mode 100644 docs/html/structfaiss_1_1IndexHNSWFlat-members.html create mode 100644 docs/html/structfaiss_1_1IndexHNSWFlat.html create mode 100644 docs/html/structfaiss_1_1IndexHNSWFlat.png create mode 100644 docs/html/structfaiss_1_1IndexHNSWPQ-members.html create mode 100644 docs/html/structfaiss_1_1IndexHNSWPQ.html create mode 100644 docs/html/structfaiss_1_1IndexHNSWPQ.png create mode 100644 docs/html/structfaiss_1_1IndexHNSWSQ-members.html create mode 100644 docs/html/structfaiss_1_1IndexHNSWSQ.html create mode 100644 docs/html/structfaiss_1_1IndexHNSWSQ.png rename docs/html/{structfaiss_1_1IndexIVFFlatStats-members.html => structfaiss_1_1IndexIVFStats-members.html} (74%) rename docs/html/{structfaiss_1_1IndexIVFFlatStats.html => structfaiss_1_1IndexIVFStats.html} (83%) create mode 100644 docs/html/structfaiss_1_1Level1Quantizer-members.html create mode 100644 docs/html/structfaiss_1_1Level1Quantizer.html create mode 100644 docs/html/structfaiss_1_1Level1Quantizer.png create mode 100644 docs/html/structfaiss_1_1PQDis-members.html create mode 100644 docs/html/structfaiss_1_1PQDis.html create mode 100644 docs/html/structfaiss_1_1PQDis.png create mode 100644 docs/html/structfaiss_1_1ReconstructFromNeighbors-members.html create mode 100644 docs/html/structfaiss_1_1ReconstructFromNeighbors.html create mode 100644 docs/html/structfaiss_1_1SQDis-members.html create mode 100644 docs/html/structfaiss_1_1SQDis.html create mode 100644 docs/html/structfaiss_1_1SQDis.png create mode 100644 docs/html/structfaiss_1_1ScalarQuantizer_1_1DistanceComputer-members.html create mode 100644 docs/html/structfaiss_1_1ScalarQuantizer_1_1DistanceComputer.html create mode 100644 docs/html/structfaiss_1_1VisitedTable-members.html create mode 100644 docs/html/structfaiss_1_1VisitedTable.html diff --git a/docs/html/2-IVFFlat_8cpp_source.html b/docs/html/2-IVFFlat_8cpp_source.html index c3b17d682..34aa3a9b6 100644 --- a/docs/html/2-IVFFlat_8cpp_source.html +++ b/docs/html/2-IVFFlat_8cpp_source.html @@ -170,7 +170,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
81  return 0;
82 }
faiss::IndexFlatL2
Definition: IndexFlat.h:79
-
faiss::IndexIVFFlat
Definition: IndexIVF.h:181
+
faiss::IndexIVFFlat
Definition: IndexIVF.h:235
diff --git a/docs/html/AutoTune_8cpp_source.html b/docs/html/AutoTune_8cpp_source.html index 6e3a4b777..e8ffbf2e8 100644 --- a/docs/html/AutoTune_8cpp_source.html +++ b/docs/html/AutoTune_8cpp_source.html @@ -108,924 +108,940 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
23 #include "IndexIVFPQ.h"
24 #include "MetaIndexes.h"
25 #include "IndexScalarQuantizer.h"
-
26 
+
26 #include "IndexHNSW.h"
27 
-
28 namespace faiss {
-
29 
+
28 
+
29 namespace faiss {
30 
-
31 AutoTuneCriterion::AutoTuneCriterion (idx_t nq, idx_t nnn):
-
32  nq (nq), nnn (nnn), gt_nnn (0)
-
33 {}
-
34 
+
31 
+
32 AutoTuneCriterion::AutoTuneCriterion (idx_t nq, idx_t nnn):
+
33  nq (nq), nnn (nnn), gt_nnn (0)
+
34 {}
35 
-
36 void AutoTuneCriterion::set_groundtruth (
-
37  int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
-
38 {
-
39  this->gt_nnn = gt_nnn;
-
40  if (gt_D_in) { // allow null for this, as it is often not used
-
41  gt_D.resize (nq * gt_nnn);
-
42  memcpy (gt_D.data(), gt_D_in, sizeof (gt_D[0]) * nq * gt_nnn);
-
43  }
-
44  gt_I.resize (nq * gt_nnn);
-
45  memcpy (gt_I.data(), gt_I_in, sizeof (gt_I[0]) * nq * gt_nnn);
-
46 }
-
47 
+
36 
+
37 void AutoTuneCriterion::set_groundtruth (
+
38  int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
+
39 {
+
40  this->gt_nnn = gt_nnn;
+
41  if (gt_D_in) { // allow null for this, as it is often not used
+
42  gt_D.resize (nq * gt_nnn);
+
43  memcpy (gt_D.data(), gt_D_in, sizeof (gt_D[0]) * nq * gt_nnn);
+
44  }
+
45  gt_I.resize (nq * gt_nnn);
+
46  memcpy (gt_I.data(), gt_I_in, sizeof (gt_I[0]) * nq * gt_nnn);
+
47 }
48 
49 
-
50 OneRecallAtRCriterion::OneRecallAtRCriterion (idx_t nq, idx_t R):
-
51  AutoTuneCriterion(nq, R), R(R)
-
52 {}
-
53 
-
54 double OneRecallAtRCriterion::evaluate(const float* /*D*/, const idx_t* I)
-
55  const {
-
56  FAISS_THROW_IF_NOT_MSG(
-
57  (gt_I.size() == gt_nnn * nq && gt_nnn >= 1 && nnn >= R),
-
58  "ground truth not initialized");
-
59  idx_t n_ok = 0;
-
60  for (idx_t q = 0; q < nq; q++) {
-
61  idx_t gt_nn = gt_I[q * gt_nnn];
-
62  const idx_t* I_line = I + q * nnn;
-
63  for (int i = 0; i < R; i++) {
-
64  if (I_line[i] == gt_nn) {
-
65  n_ok++;
-
66  break;
-
67  }
-
68  }
-
69  }
-
70  return n_ok / double(nq);
-
71 }
-
72 
+
50 
+
51 OneRecallAtRCriterion::OneRecallAtRCriterion (idx_t nq, idx_t R):
+
52  AutoTuneCriterion(nq, R), R(R)
+
53 {}
+
54 
+
55 double OneRecallAtRCriterion::evaluate(const float* /*D*/, const idx_t* I)
+
56  const {
+
57  FAISS_THROW_IF_NOT_MSG(
+
58  (gt_I.size() == gt_nnn * nq && gt_nnn >= 1 && nnn >= R),
+
59  "ground truth not initialized");
+
60  idx_t n_ok = 0;
+
61  for (idx_t q = 0; q < nq; q++) {
+
62  idx_t gt_nn = gt_I[q * gt_nnn];
+
63  const idx_t* I_line = I + q * nnn;
+
64  for (int i = 0; i < R; i++) {
+
65  if (I_line[i] == gt_nn) {
+
66  n_ok++;
+
67  break;
+
68  }
+
69  }
+
70  }
+
71  return n_ok / double(nq);
+
72 }
73 
-
74 IntersectionCriterion::IntersectionCriterion (idx_t nq, idx_t R):
-
75  AutoTuneCriterion(nq, R), R(R)
-
76 {}
-
77 
-
78 double IntersectionCriterion::evaluate(const float* /*D*/, const idx_t* I)
-
79  const {
-
80  FAISS_THROW_IF_NOT_MSG(
-
81  (gt_I.size() == gt_nnn * nq && gt_nnn >= R && nnn >= R),
-
82  "ground truth not initialized");
-
83  long n_ok = 0;
-
84 #pragma omp parallel for reduction(+: n_ok)
-
85  for (idx_t q = 0; q < nq; q++) {
-
86  n_ok += ranklist_intersection_size (
-
87  R, &gt_I [q * gt_nnn],
-
88  R, I + q * nnn);
-
89  }
-
90  return n_ok / double (nq * R);
-
91 }
-
92 
-
93 /***************************************************************
-
94  * OperatingPoints
-
95  ***************************************************************/
-
96 
-
97 OperatingPoints::OperatingPoints ()
-
98 {
-
99  clear();
-
100 }
-
101 
-
102 void OperatingPoints::clear ()
-
103 {
-
104  all_pts.clear();
-
105  optimal_pts.clear();
-
106  /// default point: doing nothing gives 0 performance and takes 0 time
-
107  OperatingPoint op = {0, 0, "", -1};
-
108  optimal_pts.push_back(op);
-
109 }
-
110 
-
111 /// add a performance measure
-
112 bool OperatingPoints::add (double perf, double t, const std::string & key,
-
113  size_t cno)
-
114 {
-
115  OperatingPoint op = {perf, t, key, long(cno)};
-
116  all_pts.push_back (op);
-
117  if (perf == 0) {
-
118  return false; // no method for 0 accuracy is faster than doing nothing
-
119  }
-
120  std::vector<OperatingPoint> & a = optimal_pts;
-
121  if (perf > a.back().perf) {
-
122  // keep unconditionally
-
123  a.push_back (op);
-
124  } else if (perf == a.back().perf) {
-
125  if (t < a.back ().t) {
-
126  a.back() = op;
-
127  } else {
-
128  return false;
-
129  }
-
130  } else {
-
131  int i;
-
132  // stricto sensu this should be a bissection
-
133  for (i = 0; i < a.size(); i++) {
-
134  if (a[i].perf >= perf) break;
-
135  }
-
136  assert (i < a.size());
-
137  if (t < a[i].t) {
-
138  if (a[i].perf == perf) {
-
139  a[i] = op;
-
140  } else {
-
141  a.insert (a.begin() + i, op);
-
142  }
-
143  } else {
-
144  return false;
-
145  }
-
146  }
-
147  { // remove non-optimal points from array
-
148  int i = a.size() - 1;
-
149  while (i > 0) {
-
150  if (a[i].t < a[i - 1].t)
-
151  a.erase (a.begin() + (i - 1));
-
152  i--;
-
153  }
-
154  }
-
155  return true;
-
156 }
-
157 
+
74 
+
75 IntersectionCriterion::IntersectionCriterion (idx_t nq, idx_t R):
+
76  AutoTuneCriterion(nq, R), R(R)
+
77 {}
+
78 
+
79 double IntersectionCriterion::evaluate(const float* /*D*/, const idx_t* I)
+
80  const {
+
81  FAISS_THROW_IF_NOT_MSG(
+
82  (gt_I.size() == gt_nnn * nq && gt_nnn >= R && nnn >= R),
+
83  "ground truth not initialized");
+
84  long n_ok = 0;
+
85 #pragma omp parallel for reduction(+: n_ok)
+
86  for (idx_t q = 0; q < nq; q++) {
+
87  n_ok += ranklist_intersection_size (
+
88  R, &gt_I [q * gt_nnn],
+
89  R, I + q * nnn);
+
90  }
+
91  return n_ok / double (nq * R);
+
92 }
+
93 
+
94 /***************************************************************
+
95  * OperatingPoints
+
96  ***************************************************************/
+
97 
+
98 OperatingPoints::OperatingPoints ()
+
99 {
+
100  clear();
+
101 }
+
102 
+
103 void OperatingPoints::clear ()
+
104 {
+
105  all_pts.clear();
+
106  optimal_pts.clear();
+
107  /// default point: doing nothing gives 0 performance and takes 0 time
+
108  OperatingPoint op = {0, 0, "", -1};
+
109  optimal_pts.push_back(op);
+
110 }
+
111 
+
112 /// add a performance measure
+
113 bool OperatingPoints::add (double perf, double t, const std::string & key,
+
114  size_t cno)
+
115 {
+
116  OperatingPoint op = {perf, t, key, long(cno)};
+
117  all_pts.push_back (op);
+
118  if (perf == 0) {
+
119  return false; // no method for 0 accuracy is faster than doing nothing
+
120  }
+
121  std::vector<OperatingPoint> & a = optimal_pts;
+
122  if (perf > a.back().perf) {
+
123  // keep unconditionally
+
124  a.push_back (op);
+
125  } else if (perf == a.back().perf) {
+
126  if (t < a.back ().t) {
+
127  a.back() = op;
+
128  } else {
+
129  return false;
+
130  }
+
131  } else {
+
132  int i;
+
133  // stricto sensu this should be a bissection
+
134  for (i = 0; i < a.size(); i++) {
+
135  if (a[i].perf >= perf) break;
+
136  }
+
137  assert (i < a.size());
+
138  if (t < a[i].t) {
+
139  if (a[i].perf == perf) {
+
140  a[i] = op;
+
141  } else {
+
142  a.insert (a.begin() + i, op);
+
143  }
+
144  } else {
+
145  return false;
+
146  }
+
147  }
+
148  { // remove non-optimal points from array
+
149  int i = a.size() - 1;
+
150  while (i > 0) {
+
151  if (a[i].t < a[i - 1].t)
+
152  a.erase (a.begin() + (i - 1));
+
153  i--;
+
154  }
+
155  }
+
156  return true;
+
157 }
158 
-
159 int OperatingPoints::merge_with (const OperatingPoints &other,
-
160  const std::string & prefix)
-
161 {
-
162  int n_add = 0;
-
163  for (int i = 0; i < other.all_pts.size(); i++) {
-
164  const OperatingPoint & op = other.all_pts[i];
-
165  if (add (op.perf, op.t, prefix + op.key, op.cno))
-
166  n_add++;
-
167  }
-
168  return n_add;
-
169 }
-
170 
+
159 
+
160 int OperatingPoints::merge_with (const OperatingPoints &other,
+
161  const std::string & prefix)
+
162 {
+
163  int n_add = 0;
+
164  for (int i = 0; i < other.all_pts.size(); i++) {
+
165  const OperatingPoint & op = other.all_pts[i];
+
166  if (add (op.perf, op.t, prefix + op.key, op.cno))
+
167  n_add++;
+
168  }
+
169  return n_add;
+
170 }
171 
-
172 
-
173 /// get time required to obtain a given performance measure
-
174 double OperatingPoints::t_for_perf (double perf) const
-
175 {
-
176  const std::vector<OperatingPoint> & a = optimal_pts;
-
177  if (perf > a.back().perf) return 1e50;
-
178  int i0 = -1, i1 = a.size() - 1;
-
179  while (i0 + 1 < i1) {
-
180  int imed = (i0 + i1 + 1) / 2;
-
181  if (a[imed].perf < perf) i0 = imed;
-
182  else i1 = imed;
-
183  }
-
184  return a[i1].t;
-
185 }
-
186 
+
172 
+
173 
+
174 /// get time required to obtain a given performance measure
+
175 double OperatingPoints::t_for_perf (double perf) const
+
176 {
+
177  const std::vector<OperatingPoint> & a = optimal_pts;
+
178  if (perf > a.back().perf) return 1e50;
+
179  int i0 = -1, i1 = a.size() - 1;
+
180  while (i0 + 1 < i1) {
+
181  int imed = (i0 + i1 + 1) / 2;
+
182  if (a[imed].perf < perf) i0 = imed;
+
183  else i1 = imed;
+
184  }
+
185  return a[i1].t;
+
186 }
187 
-
188 void OperatingPoints::all_to_gnuplot (const char *fname) const
-
189 {
-
190  FILE *f = fopen(fname, "w");
-
191  if (!f) {
-
192  fprintf (stderr, "cannot open %s", fname);
-
193  perror("");
-
194  abort();
-
195  }
-
196  for (int i = 0; i < all_pts.size(); i++) {
-
197  const OperatingPoint & op = all_pts[i];
-
198  fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
-
199  }
-
200  fclose(f);
-
201 }
-
202 
-
203 void OperatingPoints::optimal_to_gnuplot (const char *fname) const
-
204 {
-
205  FILE *f = fopen(fname, "w");
-
206  if (!f) {
-
207  fprintf (stderr, "cannot open %s", fname);
-
208  perror("");
-
209  abort();
-
210  }
-
211  double prev_perf = 0.0;
-
212  for (int i = 0; i < optimal_pts.size(); i++) {
-
213  const OperatingPoint & op = optimal_pts[i];
-
214  fprintf (f, "%g %g\n", prev_perf, op.t);
-
215  fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
-
216  prev_perf = op.perf;
-
217  }
-
218  fclose(f);
-
219 }
-
220 
-
221 void OperatingPoints::display (bool only_optimal) const
-
222 {
-
223  const std::vector<OperatingPoint> &pts =
-
224  only_optimal ? optimal_pts : all_pts;
-
225  printf("Tested %ld operating points, %ld ones are optimal:\n",
-
226  all_pts.size(), optimal_pts.size());
-
227 
-
228  for (int i = 0; i < pts.size(); i++) {
-
229  const OperatingPoint & op = pts[i];
-
230  const char *star = "";
-
231  if (!only_optimal) {
-
232  for (int j = 0; j < optimal_pts.size(); j++) {
-
233  if (op.cno == optimal_pts[j].cno) {
-
234  star = "*";
-
235  break;
-
236  }
-
237  }
-
238  }
-
239  printf ("cno=%ld key=%s perf=%.4f t=%.3f %s\n",
-
240  op.cno, op.key.c_str(), op.perf, op.t, star);
-
241  }
-
242 
-
243 }
-
244 
-
245 /***************************************************************
-
246  * ParameterSpace
-
247  ***************************************************************/
-
248 
-
249 ParameterSpace::ParameterSpace ():
-
250  verbose (1), n_experiments (500),
-
251  batchsize (1<<30), thread_over_batches (false)
-
252 {
-
253 }
-
254 
-
255 /* not keeping this constructor as inheritors will call the parent
-
256  initialize()
-
257  */
-
258 
-
259 #if 0
-
260 ParameterSpace::ParameterSpace (Index *index):
-
261  verbose (1), n_experiments (500),
-
262  batchsize (1<<30), thread_over_batches (false)
-
263 {
-
264  initialize(index);
-
265 }
-
266 #endif
-
267 
-
268 size_t ParameterSpace::n_combinations () const
-
269 {
-
270  size_t n = 1;
-
271  for (int i = 0; i < parameter_ranges.size(); i++)
-
272  n *= parameter_ranges[i].values.size();
-
273  return n;
-
274 }
-
275 
-
276 /// get string representation of the combination
-
277 std::string ParameterSpace::combination_name (size_t cno) const {
-
278  char buf[1000], *wp = buf;
-
279  *wp = 0;
-
280  for (int i = 0; i < parameter_ranges.size(); i++) {
-
281  const ParameterRange & pr = parameter_ranges[i];
-
282  size_t j = cno % pr.values.size();
-
283  cno /= pr.values.size();
-
284  wp += snprintf (
-
285  wp, buf + 1000 - wp, "%s%s=%g", i == 0 ? "" : ",",
-
286  pr.name.c_str(), pr.values[j]);
-
287  }
-
288  return std::string (buf);
-
289 }
-
290 
+
188 
+
189 void OperatingPoints::all_to_gnuplot (const char *fname) const
+
190 {
+
191  FILE *f = fopen(fname, "w");
+
192  if (!f) {
+
193  fprintf (stderr, "cannot open %s", fname);
+
194  perror("");
+
195  abort();
+
196  }
+
197  for (int i = 0; i < all_pts.size(); i++) {
+
198  const OperatingPoint & op = all_pts[i];
+
199  fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
+
200  }
+
201  fclose(f);
+
202 }
+
203 
+
204 void OperatingPoints::optimal_to_gnuplot (const char *fname) const
+
205 {
+
206  FILE *f = fopen(fname, "w");
+
207  if (!f) {
+
208  fprintf (stderr, "cannot open %s", fname);
+
209  perror("");
+
210  abort();
+
211  }
+
212  double prev_perf = 0.0;
+
213  for (int i = 0; i < optimal_pts.size(); i++) {
+
214  const OperatingPoint & op = optimal_pts[i];
+
215  fprintf (f, "%g %g\n", prev_perf, op.t);
+
216  fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
+
217  prev_perf = op.perf;
+
218  }
+
219  fclose(f);
+
220 }
+
221 
+
222 void OperatingPoints::display (bool only_optimal) const
+
223 {
+
224  const std::vector<OperatingPoint> &pts =
+
225  only_optimal ? optimal_pts : all_pts;
+
226  printf("Tested %ld operating points, %ld ones are optimal:\n",
+
227  all_pts.size(), optimal_pts.size());
+
228 
+
229  for (int i = 0; i < pts.size(); i++) {
+
230  const OperatingPoint & op = pts[i];
+
231  const char *star = "";
+
232  if (!only_optimal) {
+
233  for (int j = 0; j < optimal_pts.size(); j++) {
+
234  if (op.cno == optimal_pts[j].cno) {
+
235  star = "*";
+
236  break;
+
237  }
+
238  }
+
239  }
+
240  printf ("cno=%ld key=%s perf=%.4f t=%.3f %s\n",
+
241  op.cno, op.key.c_str(), op.perf, op.t, star);
+
242  }
+
243 
+
244 }
+
245 
+
246 /***************************************************************
+
247  * ParameterSpace
+
248  ***************************************************************/
+
249 
+
250 ParameterSpace::ParameterSpace ():
+
251  verbose (1), n_experiments (500),
+
252  batchsize (1<<30), thread_over_batches (false)
+
253 {
+
254 }
+
255 
+
256 /* not keeping this constructor as inheritors will call the parent
+
257  initialize()
+
258  */
+
259 
+
260 #if 0
+
261 ParameterSpace::ParameterSpace (Index *index):
+
262  verbose (1), n_experiments (500),
+
263  batchsize (1<<30), thread_over_batches (false)
+
264 {
+
265  initialize(index);
+
266 }
+
267 #endif
+
268 
+
269 size_t ParameterSpace::n_combinations () const
+
270 {
+
271  size_t n = 1;
+
272  for (int i = 0; i < parameter_ranges.size(); i++)
+
273  n *= parameter_ranges[i].values.size();
+
274  return n;
+
275 }
+
276 
+
277 /// get string representation of the combination
+
278 std::string ParameterSpace::combination_name (size_t cno) const {
+
279  char buf[1000], *wp = buf;
+
280  *wp = 0;
+
281  for (int i = 0; i < parameter_ranges.size(); i++) {
+
282  const ParameterRange & pr = parameter_ranges[i];
+
283  size_t j = cno % pr.values.size();
+
284  cno /= pr.values.size();
+
285  wp += snprintf (
+
286  wp, buf + 1000 - wp, "%s%s=%g", i == 0 ? "" : ",",
+
287  pr.name.c_str(), pr.values[j]);
+
288  }
+
289  return std::string (buf);
+
290 }
291 
-
292 bool ParameterSpace::combination_ge (size_t c1, size_t c2) const
-
293 {
-
294  for (int i = 0; i < parameter_ranges.size(); i++) {
-
295  int nval = parameter_ranges[i].values.size();
-
296  size_t j1 = c1 % nval;
-
297  size_t j2 = c2 % nval;
-
298  if (!(j1 >= j2)) return false;
-
299  c1 /= nval;
-
300  c2 /= nval;
-
301  }
-
302  return true;
-
303 }
-
304 
+
292 
+
293 bool ParameterSpace::combination_ge (size_t c1, size_t c2) const
+
294 {
+
295  for (int i = 0; i < parameter_ranges.size(); i++) {
+
296  int nval = parameter_ranges[i].values.size();
+
297  size_t j1 = c1 % nval;
+
298  size_t j2 = c2 % nval;
+
299  if (!(j1 >= j2)) return false;
+
300  c1 /= nval;
+
301  c2 /= nval;
+
302  }
+
303  return true;
+
304 }
305 
306 
-
307 #define DC(classname) \
-
308  const classname *ix = dynamic_cast<const classname *>(index)
-
309 
-
310 static void init_pq_ParameterRange (const ProductQuantizer & pq,
-
311  ParameterRange & pr)
-
312 {
-
313  if (pq.code_size % 4 == 0) {
-
314  // Polysemous not supported for code sizes that are not a
-
315  // multiple of 4
-
316  for (int i = 2; i <= pq.code_size * 8 / 2; i+= 2)
-
317  pr.values.push_back(i);
-
318  }
-
319  pr.values.push_back (pq.code_size * 8);
-
320 }
-
321 
-
322 ParameterRange &ParameterSpace::add_range(const char * name)
-
323 {
-
324  parameter_ranges.push_back (ParameterRange ());
-
325  parameter_ranges.back ().name = name;
-
326  return parameter_ranges.back ();
-
327 }
-
328 
-
329 
-
330 /// initialize with reasonable parameters for the index
-
331 void ParameterSpace::initialize (const Index * index)
-
332 {
-
333  if (DC (IndexPreTransform)) {
-
334  index = ix->index;
-
335  }
-
336  if (DC (IndexRefineFlat)) {
-
337  ParameterRange & pr = add_range("k_factor_rf");
-
338  for (int i = 0; i <= 6; i++) {
-
339  pr.values.push_back (1 << i);
-
340  }
-
341  index = ix->base_index;
-
342  }
-
343  if (DC (IndexPreTransform)) {
-
344  index = ix->index;
-
345  }
-
346 
-
347  if (DC (IndexIVF)) {
-
348  {
-
349  ParameterRange & pr = add_range("nprobe");
-
350  for (int i = 0; i < 13; i++) {
-
351  size_t nprobe = 1 << i;
-
352  if (nprobe >= ix->nlist) break;
-
353  pr.values.push_back (nprobe);
-
354  }
-
355  }
-
356  }
-
357  if (DC (IndexPQ)) {
-
358  ParameterRange & pr = add_range("ht");
-
359  init_pq_ParameterRange (ix->pq, pr);
-
360  }
-
361  if (DC (IndexIVFPQ)) {
-
362  ParameterRange & pr = add_range("ht");
-
363  init_pq_ParameterRange (ix->pq, pr);
-
364 
-
365  const MultiIndexQuantizer *miq =
-
366  dynamic_cast<const MultiIndexQuantizer *> (ix->quantizer);
-
367  if (miq) {
-
368  ParameterRange & pr_max_codes = add_range("max_codes");
-
369  for (int i = 8; i < 20; i++) {
-
370  pr_max_codes.values.push_back (1 << i);
-
371  }
-
372  pr_max_codes.values.push_back (1.0 / 0.0);
-
373  }
-
374  }
-
375  if (DC (IndexIVFPQR)) {
-
376  ParameterRange & pr = add_range("k_factor");
-
377  for (int i = 0; i <= 6; i++) {
-
378  pr.values.push_back (1 << i);
-
379  }
-
380  }
-
381 }
-
382 
-
383 #undef DC
-
384 
-
385 // non-const version
-
386 #define DC(classname) classname *ix = dynamic_cast<classname *>(index)
-
387 
-
388 
-
389 /// set a combination of parameters on an index
-
390 void ParameterSpace::set_index_parameters (Index *index, size_t cno) const
-
391 {
-
392 
-
393  for (int i = 0; i < parameter_ranges.size(); i++) {
-
394  const ParameterRange & pr = parameter_ranges[i];
-
395  size_t j = cno % pr.values.size();
-
396  cno /= pr.values.size();
-
397  double val = pr.values [j];
-
398  set_index_parameter (index, pr.name, val);
-
399  }
-
400 }
-
401 
-
402 /// set a combination of parameters on an index
-
403 void ParameterSpace::set_index_parameters (
-
404  Index *index, const char *description_in) const
-
405 {
-
406  char description[strlen(description_in) + 1];
-
407  char *ptr;
-
408  memcpy (description, description_in, strlen(description_in) + 1);
-
409 
-
410  for (char *tok = strtok_r (description, " ,", &ptr);
-
411  tok;
-
412  tok = strtok_r (nullptr, " ,", &ptr)) {
-
413  char name[100];
-
414  double val;
-
415  int ret = sscanf (tok, "%100[^=]=%lf", name, &val);
-
416  FAISS_THROW_IF_NOT_FMT (
-
417  ret == 2, "could not interpret parameters %s", tok);
-
418  set_index_parameter (index, name, val);
+
307 
+
308 #define DC(classname) \
+
309  const classname *ix = dynamic_cast<const classname *>(index)
+
310 
+
311 static void init_pq_ParameterRange (const ProductQuantizer & pq,
+
312  ParameterRange & pr)
+
313 {
+
314  if (pq.code_size % 4 == 0) {
+
315  // Polysemous not supported for code sizes that are not a
+
316  // multiple of 4
+
317  for (int i = 2; i <= pq.code_size * 8 / 2; i+= 2)
+
318  pr.values.push_back(i);
+
319  }
+
320  pr.values.push_back (pq.code_size * 8);
+
321 }
+
322 
+
323 ParameterRange &ParameterSpace::add_range(const char * name)
+
324 {
+
325  for (auto & pr : parameter_ranges) {
+
326  if (pr.name == name) {
+
327  return pr;
+
328  }
+
329  }
+
330  parameter_ranges.push_back (ParameterRange ());
+
331  parameter_ranges.back ().name = name;
+
332  return parameter_ranges.back ();
+
333 }
+
334 
+
335 
+
336 /// initialize with reasonable parameters for the index
+
337 void ParameterSpace::initialize (const Index * index)
+
338 {
+
339  if (DC (IndexPreTransform)) {
+
340  index = ix->index;
+
341  }
+
342  if (DC (IndexRefineFlat)) {
+
343  ParameterRange & pr = add_range("k_factor_rf");
+
344  for (int i = 0; i <= 6; i++) {
+
345  pr.values.push_back (1 << i);
+
346  }
+
347  index = ix->base_index;
+
348  }
+
349  if (DC (IndexPreTransform)) {
+
350  index = ix->index;
+
351  }
+
352 
+
353  if (DC (IndexIVF)) {
+
354  {
+
355  ParameterRange & pr = add_range("nprobe");
+
356  for (int i = 0; i < 13; i++) {
+
357  size_t nprobe = 1 << i;
+
358  if (nprobe >= ix->nlist) break;
+
359  pr.values.push_back (nprobe);
+
360  }
+
361  }
+
362  if (dynamic_cast<const IndexHNSW*>(ix->quantizer)) {
+
363  ParameterRange & pr = add_range("efSearch");
+
364  for (int i = 2; i <= 9; i++) {
+
365  pr.values.push_back (1 << i);
+
366  }
+
367  }
+
368  }
+
369  if (DC (IndexPQ)) {
+
370  ParameterRange & pr = add_range("ht");
+
371  init_pq_ParameterRange (ix->pq, pr);
+
372  }
+
373  if (DC (IndexIVFPQ)) {
+
374  ParameterRange & pr = add_range("ht");
+
375  init_pq_ParameterRange (ix->pq, pr);
+
376  }
+
377 
+
378  if (DC (IndexIVF)) {
+
379  const MultiIndexQuantizer *miq =
+
380  dynamic_cast<const MultiIndexQuantizer *> (ix->quantizer);
+
381  if (miq) {
+
382  ParameterRange & pr_max_codes = add_range("max_codes");
+
383  for (int i = 8; i < 20; i++) {
+
384  pr_max_codes.values.push_back (1 << i);
+
385  }
+
386  pr_max_codes.values.push_back (1.0 / 0.0);
+
387  }
+
388  }
+
389  if (DC (IndexIVFPQR)) {
+
390  ParameterRange & pr = add_range("k_factor");
+
391  for (int i = 0; i <= 6; i++) {
+
392  pr.values.push_back (1 << i);
+
393  }
+
394  }
+
395  if (dynamic_cast<const IndexHNSW*>(index)) {
+
396  ParameterRange & pr = add_range("efSearch");
+
397  for (int i = 2; i <= 9; i++) {
+
398  pr.values.push_back (1 << i);
+
399  }
+
400  }
+
401 }
+
402 
+
403 #undef DC
+
404 
+
405 // non-const version
+
406 #define DC(classname) classname *ix = dynamic_cast<classname *>(index)
+
407 
+
408 
+
409 /// set a combination of parameters on an index
+
410 void ParameterSpace::set_index_parameters (Index *index, size_t cno) const
+
411 {
+
412 
+
413  for (int i = 0; i < parameter_ranges.size(); i++) {
+
414  const ParameterRange & pr = parameter_ranges[i];
+
415  size_t j = cno % pr.values.size();
+
416  cno /= pr.values.size();
+
417  double val = pr.values [j];
+
418  set_index_parameter (index, pr.name, val);
419  }
-
420 
-
421 }
-
422 
-
423 void ParameterSpace::set_index_parameter (
-
424  Index * index, const std::string & name, double val) const
+
420 }
+
421 
+
422 /// set a combination of parameters on an index
+
423 void ParameterSpace::set_index_parameters (
+
424  Index *index, const char *description_in) const
425 {
-
426  if (verbose > 1)
-
427  printf(" set %s=%g\n", name.c_str(), val);
-
428 
-
429  if (name == "verbose") {
-
430  index->verbose = int(val);
-
431  // and fall through to also enable it on sub-indexes
-
432  }
-
433  if (DC (IndexPreTransform)) {
-
434  index = ix->index;
-
435  }
-
436  if (DC (IndexShards)) {
-
437  // call on all sub-indexes
-
438  for (auto & shard_index : ix->shard_indexes) {
-
439  set_index_parameter (shard_index, name, val);
-
440  }
-
441  return;
-
442  }
-
443  if (name == "verbose") {
-
444  index->verbose = int(val);
-
445  // in case it was an IndexPreTransform
-
446  }
-
447  if (DC (IndexRefineFlat)) {
-
448  if (name == "k_factor_rf") {
-
449  ix->k_factor = int(val);
-
450  return;
-
451  }
-
452  index = ix->base_index;
-
453  }
-
454  if (DC (IndexPreTransform)) {
-
455  index = ix->index;
+
426  char description[strlen(description_in) + 1];
+
427  char *ptr;
+
428  memcpy (description, description_in, strlen(description_in) + 1);
+
429 
+
430  for (char *tok = strtok_r (description, " ,", &ptr);
+
431  tok;
+
432  tok = strtok_r (nullptr, " ,", &ptr)) {
+
433  char name[100];
+
434  double val;
+
435  int ret = sscanf (tok, "%100[^=]=%lf", name, &val);
+
436  FAISS_THROW_IF_NOT_FMT (
+
437  ret == 2, "could not interpret parameters %s", tok);
+
438  set_index_parameter (index, name, val);
+
439  }
+
440 
+
441 }
+
442 
+
443 void ParameterSpace::set_index_parameter (
+
444  Index * index, const std::string & name, double val) const
+
445 {
+
446  if (verbose > 1)
+
447  printf(" set %s=%g\n", name.c_str(), val);
+
448 
+
449  if (name == "verbose") {
+
450  index->verbose = int(val);
+
451  // and fall through to also enable it on sub-indexes
+
452  }
+
453  if (DC (IndexPreTransform)) {
+
454  set_index_parameter (ix->index, name, val);
+
455  return;
456  }
-
457  if (name == "verbose") {
-
458  index->verbose = int(val);
-
459  return; // last verbose that we could find
-
460  }
-
461  if (name == "nprobe") {
-
462  if ( DC(IndexIVF)) {
-
463  ix->nprobe = int(val);
-
464  return;
-
465  }
-
466  }
-
467  if (name == "ht") {
-
468  if (DC (IndexPQ)) {
-
469  if (val >= ix->pq.code_size * 8) {
-
470  ix->search_type = IndexPQ::ST_PQ;
-
471  } else {
-
472  ix->search_type = IndexPQ::ST_polysemous;
-
473  ix->polysemous_ht = int(val);
-
474  }
-
475  return;
-
476  } else if (DC (IndexIVFPQ)) {
-
477  if (val >= ix->pq.code_size * 8) {
-
478  ix->polysemous_ht = 0;
-
479  } else {
-
480  ix->polysemous_ht = int(val);
-
481  }
+
457  if (DC (IndexShards)) {
+
458  // call on all sub-indexes
+
459  for (auto & shard_index : ix->shard_indexes) {
+
460  set_index_parameter (shard_index, name, val);
+
461  }
+
462  return;
+
463  }
+
464  if (DC (IndexRefineFlat)) {
+
465  if (name == "k_factor_rf") {
+
466  ix->k_factor = int(val);
+
467  return;
+
468  }
+
469  // otherwise it is for the sub-index
+
470  set_index_parameter (&ix->refine_index, name, val);
+
471  return;
+
472  }
+
473 
+
474  if (name == "verbose") {
+
475  index->verbose = int(val);
+
476  return; // last verbose that we could find
+
477  }
+
478 
+
479  if (name == "nprobe") {
+
480  if ( DC(IndexIVF)) {
+
481  ix->nprobe = int(val);
482  return;
483  }
484  }
485 
-
486  if (name == "k_factor") {
-
487  if (DC (IndexIVFPQR)) {
-
488  ix->k_factor = val;
-
489  return;
-
490  }
-
491  }
-
492  if (name == "max_codes") {
-
493  if (DC (IndexIVFPQ)) {
-
494  ix->max_codes = finite(val) ? size_t(val) : 0;
-
495  return;
-
496  }
-
497  }
-
498  FAISS_THROW_FMT ("ParameterSpace::set_index_parameter:"
-
499  "could not set parameter %s",
-
500  name.c_str());
-
501 }
-
502 
-
503 void ParameterSpace::display () const
-
504 {
-
505  printf ("ParameterSpace, %ld parameters, %ld combinations:\n",
-
506  parameter_ranges.size (), n_combinations ());
-
507  for (int i = 0; i < parameter_ranges.size(); i++) {
-
508  const ParameterRange & pr = parameter_ranges[i];
-
509  printf (" %s: ", pr.name.c_str ());
-
510  char sep = '[';
-
511  for (int j = 0; j < pr.values.size(); j++) {
-
512  printf ("%c %g", sep, pr.values [j]);
-
513  sep = ',';
-
514  }
-
515  printf ("]\n");
+
486  if (name == "ht") {
+
487  if (DC (IndexPQ)) {
+
488  if (val >= ix->pq.code_size * 8) {
+
489  ix->search_type = IndexPQ::ST_PQ;
+
490  } else {
+
491  ix->search_type = IndexPQ::ST_polysemous;
+
492  ix->polysemous_ht = int(val);
+
493  }
+
494  return;
+
495  } else if (DC (IndexIVFPQ)) {
+
496  if (val >= ix->pq.code_size * 8) {
+
497  ix->polysemous_ht = 0;
+
498  } else {
+
499  ix->polysemous_ht = int(val);
+
500  }
+
501  return;
+
502  }
+
503  }
+
504 
+
505  if (name == "k_factor") {
+
506  if (DC (IndexIVFPQR)) {
+
507  ix->k_factor = val;
+
508  return;
+
509  }
+
510  }
+
511  if (name == "max_codes") {
+
512  if (DC (IndexIVF)) {
+
513  ix->max_codes = finite(val) ? size_t(val) : 0;
+
514  return;
+
515  }
516  }
-
517 }
-
518 
-
519 
-
520 
-
521 void ParameterSpace::update_bounds (size_t cno, const OperatingPoint & op,
-
522  double *upper_bound_perf,
-
523  double *lower_bound_t) const
-
524 {
-
525  if (combination_ge (cno, op.cno)) {
-
526  if (op.t > *lower_bound_t) *lower_bound_t = op.t;
-
527  }
-
528  if (combination_ge (op.cno, cno)) {
-
529  if (op.perf < *upper_bound_perf) *upper_bound_perf = op.perf;
-
530  }
-
531 }
-
532 
-
533 
-
534 
-
535 void ParameterSpace::explore (Index *index,
-
536  size_t nq, const float *xq,
-
537  const AutoTuneCriterion & crit,
-
538  OperatingPoints * ops) const
-
539 {
-
540  FAISS_THROW_IF_NOT_MSG (nq == crit.nq,
-
541  "criterion does not have the same nb of queries");
-
542 
-
543  size_t n_comb = n_combinations ();
-
544 
-
545  if (n_experiments == 0) {
-
546 
-
547  for (size_t cno = 0; cno < n_comb; cno++) {
-
548  set_index_parameters (index, cno);
-
549  std::vector<Index::idx_t> I(nq * crit.nnn);
-
550  std::vector<float> D(nq * crit.nnn);
+
517  FAISS_THROW_FMT ("ParameterSpace::set_index_parameter:"
+
518  "could not set parameter %s",
+
519  name.c_str());
+
520 }
+
521 
+
522 void ParameterSpace::display () const
+
523 {
+
524  printf ("ParameterSpace, %ld parameters, %ld combinations:\n",
+
525  parameter_ranges.size (), n_combinations ());
+
526  for (int i = 0; i < parameter_ranges.size(); i++) {
+
527  const ParameterRange & pr = parameter_ranges[i];
+
528  printf (" %s: ", pr.name.c_str ());
+
529  char sep = '[';
+
530  for (int j = 0; j < pr.values.size(); j++) {
+
531  printf ("%c %g", sep, pr.values [j]);
+
532  sep = ',';
+
533  }
+
534  printf ("]\n");
+
535  }
+
536 }
+
537 
+
538 
+
539 
+
540 void ParameterSpace::update_bounds (size_t cno, const OperatingPoint & op,
+
541  double *upper_bound_perf,
+
542  double *lower_bound_t) const
+
543 {
+
544  if (combination_ge (cno, op.cno)) {
+
545  if (op.t > *lower_bound_t) *lower_bound_t = op.t;
+
546  }
+
547  if (combination_ge (op.cno, cno)) {
+
548  if (op.perf < *upper_bound_perf) *upper_bound_perf = op.perf;
+
549  }
+
550 }
551 
-
552  double t0 = getmillisecs ();
-
553  index->search (nq, xq, crit.nnn, D.data(), I.data());
-
554  double t_search = (getmillisecs() - t0) / 1e3;
-
555 
-
556  double perf = crit.evaluate (D.data(), I.data());
-
557 
-
558  bool keep = ops->add (perf, t_search, combination_name (cno), cno);
-
559 
-
560  if (verbose)
-
561  printf(" %ld/%ld: %s perf=%.3f t=%.3f s %s\n", cno, n_comb,
-
562  combination_name (cno).c_str(), perf, t_search,
-
563  keep ? "*" : "");
-
564  }
-
565  return;
-
566  }
-
567 
-
568  int n_exp = n_experiments;
-
569 
-
570  if (n_exp > n_comb) n_exp = n_comb;
-
571  FAISS_THROW_IF_NOT (n_comb == 1 || n_exp > 2);
-
572  std::vector<int> perm (n_comb);
-
573  // make sure the slowest and fastest experiment are run
-
574  perm[0] = 0;
-
575  if (n_comb > 1) {
-
576  perm[1] = n_comb - 1;
-
577  rand_perm (&perm[2], n_comb - 2, 1234);
-
578  for (int i = 2; i < perm.size(); i++) perm[i] ++;
-
579  }
-
580 
-
581  for (size_t xp = 0; xp < n_exp; xp++) {
-
582  size_t cno = perm[xp];
-
583 
-
584  if (verbose)
-
585  printf(" %ld/%d: cno=%ld %s ", xp, n_exp, cno,
-
586  combination_name (cno).c_str());
-
587 
-
588  {
-
589  double lower_bound_t = 0.0;
-
590  double upper_bound_perf = 1.0;
-
591  for (int i = 0; i < ops->all_pts.size(); i++) {
-
592  update_bounds (cno, ops->all_pts[i],
-
593  &upper_bound_perf, &lower_bound_t);
-
594  }
-
595  double best_t = ops->t_for_perf (upper_bound_perf);
-
596  if (verbose)
-
597  printf ("bounds [perf<=%.3f t>=%.3f] %s",
-
598  upper_bound_perf, lower_bound_t,
-
599  best_t <= lower_bound_t ? "skip\n" : "");
-
600  if (best_t <= lower_bound_t) continue;
-
601  }
+
552 
+
553 
+
554 void ParameterSpace::explore (Index *index,
+
555  size_t nq, const float *xq,
+
556  const AutoTuneCriterion & crit,
+
557  OperatingPoints * ops) const
+
558 {
+
559  FAISS_THROW_IF_NOT_MSG (nq == crit.nq,
+
560  "criterion does not have the same nb of queries");
+
561 
+
562  size_t n_comb = n_combinations ();
+
563 
+
564  if (n_experiments == 0) {
+
565 
+
566  for (size_t cno = 0; cno < n_comb; cno++) {
+
567  set_index_parameters (index, cno);
+
568  std::vector<Index::idx_t> I(nq * crit.nnn);
+
569  std::vector<float> D(nq * crit.nnn);
+
570 
+
571  double t0 = getmillisecs ();
+
572  index->search (nq, xq, crit.nnn, D.data(), I.data());
+
573  double t_search = (getmillisecs() - t0) / 1e3;
+
574 
+
575  double perf = crit.evaluate (D.data(), I.data());
+
576 
+
577  bool keep = ops->add (perf, t_search, combination_name (cno), cno);
+
578 
+
579  if (verbose)
+
580  printf(" %ld/%ld: %s perf=%.3f t=%.3f s %s\n", cno, n_comb,
+
581  combination_name (cno).c_str(), perf, t_search,
+
582  keep ? "*" : "");
+
583  }
+
584  return;
+
585  }
+
586 
+
587  int n_exp = n_experiments;
+
588 
+
589  if (n_exp > n_comb) n_exp = n_comb;
+
590  FAISS_THROW_IF_NOT (n_comb == 1 || n_exp > 2);
+
591  std::vector<int> perm (n_comb);
+
592  // make sure the slowest and fastest experiment are run
+
593  perm[0] = 0;
+
594  if (n_comb > 1) {
+
595  perm[1] = n_comb - 1;
+
596  rand_perm (&perm[2], n_comb - 2, 1234);
+
597  for (int i = 2; i < perm.size(); i++) perm[i] ++;
+
598  }
+
599 
+
600  for (size_t xp = 0; xp < n_exp; xp++) {
+
601  size_t cno = perm[xp];
602 
-
603  set_index_parameters (index, cno);
-
604  std::vector<Index::idx_t> I(nq * crit.nnn);
-
605  std::vector<float> D(nq * crit.nnn);
+
603  if (verbose)
+
604  printf(" %ld/%d: cno=%ld %s ", xp, n_exp, cno,
+
605  combination_name (cno).c_str());
606 
-
607  double t0 = getmillisecs ();
-
608 
-
609  if (thread_over_batches) {
-
610 #pragma omp parallel for
-
611  for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
-
612  size_t q1 = q0 + batchsize;
-
613  if (q1 > nq) q1 = nq;
-
614  index->search (q1 - q0, xq + q0 * index->d,
-
615  crit.nnn,
-
616  D.data() + q0 * crit.nnn,
-
617  I.data() + q0 * crit.nnn);
-
618  }
-
619  } else {
-
620  for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
-
621  size_t q1 = q0 + batchsize;
-
622  if (q1 > nq) q1 = nq;
-
623  index->search (q1 - q0, xq + q0 * index->d,
-
624  crit.nnn,
-
625  D.data() + q0 * crit.nnn,
-
626  I.data() + q0 * crit.nnn);
-
627  }
-
628  }
-
629 
-
630  double t_search = (getmillisecs() - t0) / 1e3;
-
631 
-
632  double perf = crit.evaluate (D.data(), I.data());
-
633 
-
634  bool keep = ops->add (perf, t_search, combination_name (cno), cno);
-
635 
-
636  if (verbose)
-
637  printf(" perf %.3f t %.3f %s\n", perf, t_search,
-
638  keep ? "*" : "");
-
639  }
-
640 }
-
641 
-
642 /***************************************************************
-
643  * index_factory
-
644  ***************************************************************/
-
645 
-
646 namespace {
-
647 
-
648 struct VTChain {
-
649  std::vector<VectorTransform *> chain;
-
650  ~VTChain () {
-
651  for (int i = 0; i < chain.size(); i++) {
-
652  delete chain[i];
-
653  }
-
654  }
-
655 };
-
656 
-
657 
-
658 /// what kind of training does this coarse quantizer require?
-
659 char get_trains_alone(const Index *coarse_quantizer) {
-
660  return
-
661  dynamic_cast<const MultiIndexQuantizer*>(coarse_quantizer) ? 1 :
-
662  0;
-
663 }
+
607  {
+
608  double lower_bound_t = 0.0;
+
609  double upper_bound_perf = 1.0;
+
610  for (int i = 0; i < ops->all_pts.size(); i++) {
+
611  update_bounds (cno, ops->all_pts[i],
+
612  &upper_bound_perf, &lower_bound_t);
+
613  }
+
614  double best_t = ops->t_for_perf (upper_bound_perf);
+
615  if (verbose)
+
616  printf ("bounds [perf<=%.3f t>=%.3f] %s",
+
617  upper_bound_perf, lower_bound_t,
+
618  best_t <= lower_bound_t ? "skip\n" : "");
+
619  if (best_t <= lower_bound_t) continue;
+
620  }
+
621 
+
622  set_index_parameters (index, cno);
+
623  std::vector<Index::idx_t> I(nq * crit.nnn);
+
624  std::vector<float> D(nq * crit.nnn);
+
625 
+
626  double t0 = getmillisecs ();
+
627 
+
628  if (thread_over_batches) {
+
629 #pragma omp parallel for
+
630  for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
+
631  size_t q1 = q0 + batchsize;
+
632  if (q1 > nq) q1 = nq;
+
633  index->search (q1 - q0, xq + q0 * index->d,
+
634  crit.nnn,
+
635  D.data() + q0 * crit.nnn,
+
636  I.data() + q0 * crit.nnn);
+
637  }
+
638  } else {
+
639  for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
+
640  size_t q1 = q0 + batchsize;
+
641  if (q1 > nq) q1 = nq;
+
642  index->search (q1 - q0, xq + q0 * index->d,
+
643  crit.nnn,
+
644  D.data() + q0 * crit.nnn,
+
645  I.data() + q0 * crit.nnn);
+
646  }
+
647  }
+
648 
+
649  double t_search = (getmillisecs() - t0) / 1e3;
+
650 
+
651  double perf = crit.evaluate (D.data(), I.data());
+
652 
+
653  bool keep = ops->add (perf, t_search, combination_name (cno), cno);
+
654 
+
655  if (verbose)
+
656  printf(" perf %.3f t %.3f %s\n", perf, t_search,
+
657  keep ? "*" : "");
+
658  }
+
659 }
+
660 
+
661 /***************************************************************
+
662  * index_factory
+
663  ***************************************************************/
664 
-
665 
-
666 }
-
667 
-
668 Index *index_factory (int d, const char *description_in, MetricType metric)
-
669 {
-
670  VTChain vts;
-
671  Index *coarse_quantizer = nullptr;
-
672  Index *index = nullptr;
-
673  bool add_idmap = false;
-
674  bool make_IndexRefineFlat = false;
+
665 namespace {
+
666 
+
667 struct VTChain {
+
668  std::vector<VectorTransform *> chain;
+
669  ~VTChain () {
+
670  for (int i = 0; i < chain.size(); i++) {
+
671  delete chain[i];
+
672  }
+
673  }
+
674 };
675 
-
676  ScopeDeleter1<Index> del_coarse_quantizer, del_index;
-
677 
-
678  char description[strlen(description_in) + 1];
-
679  char *ptr;
-
680  memcpy (description, description_in, strlen(description_in) + 1);
-
681 
-
682  int ncentroids = -1;
+
676 
+
677 /// what kind of training does this coarse quantizer require?
+
678 char get_trains_alone(const Index *coarse_quantizer) {
+
679  return
+
680  dynamic_cast<const MultiIndexQuantizer*>(coarse_quantizer) ? 1 :
+
681  0;
+
682 }
683 
-
684  for (char *tok = strtok_r (description, " ,", &ptr);
-
685  tok;
-
686  tok = strtok_r (nullptr, " ,", &ptr)) {
-
687  int d_out, opq_M, nbit, M, M2;
-
688  char option[100];
-
689  std::string stok(tok);
-
690 
-
691  // to avoid mem leaks with exceptions:
-
692  // do all tests before any instanciation
-
693 
-
694  VectorTransform *vt_1 = nullptr;
-
695  Index *coarse_quantizer_1 = nullptr;
-
696  Index *index_1 = nullptr;
-
697 
-
698  // VectorTransforms
-
699  if (sscanf (tok, "PCA%d", &d_out) == 1) {
-
700  vt_1 = new PCAMatrix (d, d_out);
-
701  d = d_out;
-
702  } else if (sscanf (tok, "PCAR%d", &d_out) == 1) {
-
703  vt_1 = new PCAMatrix (d, d_out, 0, true);
-
704  d = d_out;
-
705  } else if (sscanf (tok, "PCAW%d", &d_out) == 1) {
-
706  vt_1 = new PCAMatrix (d, d_out, -0.5, false);
-
707  d = d_out;
-
708  } else if (sscanf (tok, "PCAWR%d", &d_out) == 1) {
-
709  vt_1 = new PCAMatrix (d, d_out, -0.5, true);
-
710  d = d_out;
-
711  } else if (sscanf (tok, "OPQ%d_%d", &opq_M, &d_out) == 2) {
-
712  vt_1 = new OPQMatrix (d, opq_M, d_out);
-
713  d = d_out;
-
714  } else if (sscanf (tok, "OPQ%d", &opq_M) == 1) {
-
715  vt_1 = new OPQMatrix (d, opq_M);
-
716  } else if (stok == "L2norm") {
-
717  vt_1 = new NormalizationTransform (d, 2.0);
-
718 
-
719 
-
720  } else if (!coarse_quantizer &&
-
721  sscanf (tok, "IVF%d", &ncentroids) == 1) {
-
722  if (metric == METRIC_L2) {
-
723  coarse_quantizer_1 = new IndexFlatL2 (d);
-
724  } else { // if (metric == METRIC_IP)
-
725  coarse_quantizer_1 = new IndexFlatIP (d);
-
726  }
-
727  } else if (!coarse_quantizer && sscanf (tok, "IMI2x%d", &nbit) == 1) {
-
728  FAISS_THROW_IF_NOT_MSG (metric == METRIC_L2,
-
729  "MultiIndex not implemented for inner prod search");
-
730  coarse_quantizer_1 = new MultiIndexQuantizer (d, 2, nbit);
-
731  ncentroids = 1 << (2 * nbit);
-
732  } else if (stok == "IDMap") {
-
733  add_idmap = true;
-
734 
-
735  // IVFs
-
736  } else if (!index && stok == "Flat") {
-
737  if (coarse_quantizer) {
-
738  // if there was an IVF in front, then it is an IVFFlat
-
739  IndexIVF *index_ivf = new IndexIVFFlat (
-
740  coarse_quantizer, d, ncentroids, metric);
-
741  index_ivf->quantizer_trains_alone =
-
742  get_trains_alone (coarse_quantizer);
-
743  index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
-
744  del_coarse_quantizer.release ();
-
745  index_ivf->own_fields = true;
-
746  index_1 = index_ivf;
-
747  } else {
-
748  index_1 = new IndexFlat (d, metric);
-
749  }
-
750  } else if (!index && (stok == "SQ8" || stok == "SQ4")) {
-
751  ScalarQuantizer::QuantizerType qt =
-
752  stok == "SQ8" ? ScalarQuantizer::QT_8bit :
-
753  stok == "SQ4" ? ScalarQuantizer::QT_4bit :
-
754  ScalarQuantizer::QT_4bit;
+
684 
+
685 }
+
686 
+
687 Index *index_factory (int d, const char *description_in, MetricType metric)
+
688 {
+
689  VTChain vts;
+
690  Index *coarse_quantizer = nullptr;
+
691  Index *index = nullptr;
+
692  bool add_idmap = false;
+
693  bool make_IndexRefineFlat = false;
+
694 
+
695  ScopeDeleter1<Index> del_coarse_quantizer, del_index;
+
696 
+
697  char description[strlen(description_in) + 1];
+
698  char *ptr;
+
699  memcpy (description, description_in, strlen(description_in) + 1);
+
700 
+
701  int ncentroids = -1;
+
702 
+
703  for (char *tok = strtok_r (description, " ,", &ptr);
+
704  tok;
+
705  tok = strtok_r (nullptr, " ,", &ptr)) {
+
706  int d_out, opq_M, nbit, M, M2, pq_m, ncent;
+
707  std::string stok(tok);
+
708 
+
709  // to avoid mem leaks with exceptions:
+
710  // do all tests before any instanciation
+
711 
+
712  VectorTransform *vt_1 = nullptr;
+
713  Index *coarse_quantizer_1 = nullptr;
+
714  Index *index_1 = nullptr;
+
715 
+
716  // VectorTransforms
+
717  if (sscanf (tok, "PCA%d", &d_out) == 1) {
+
718  vt_1 = new PCAMatrix (d, d_out);
+
719  d = d_out;
+
720  } else if (sscanf (tok, "PCAR%d", &d_out) == 1) {
+
721  vt_1 = new PCAMatrix (d, d_out, 0, true);
+
722  d = d_out;
+
723  } else if (sscanf (tok, "PCAW%d", &d_out) == 1) {
+
724  vt_1 = new PCAMatrix (d, d_out, -0.5, false);
+
725  d = d_out;
+
726  } else if (sscanf (tok, "PCAWR%d", &d_out) == 1) {
+
727  vt_1 = new PCAMatrix (d, d_out, -0.5, true);
+
728  d = d_out;
+
729  } else if (sscanf (tok, "OPQ%d_%d", &opq_M, &d_out) == 2) {
+
730  vt_1 = new OPQMatrix (d, opq_M, d_out);
+
731  d = d_out;
+
732  } else if (sscanf (tok, "OPQ%d", &opq_M) == 1) {
+
733  vt_1 = new OPQMatrix (d, opq_M);
+
734  } else if (stok == "L2norm") {
+
735  vt_1 = new NormalizationTransform (d, 2.0);
+
736 
+
737 
+
738  } else if (!coarse_quantizer &&
+
739  sscanf (tok, "IVF%d", &ncentroids) == 1) {
+
740  if (metric == METRIC_L2) {
+
741  coarse_quantizer_1 = new IndexFlatL2 (d);
+
742  } else { // if (metric == METRIC_IP)
+
743  coarse_quantizer_1 = new IndexFlatIP (d);
+
744  }
+
745  } else if (!coarse_quantizer && sscanf (tok, "IMI2x%d", &nbit) == 1) {
+
746  FAISS_THROW_IF_NOT_MSG (metric == METRIC_L2,
+
747  "MultiIndex not implemented for inner prod search");
+
748  coarse_quantizer_1 = new MultiIndexQuantizer (d, 2, nbit);
+
749  ncentroids = 1 << (2 * nbit);
+
750  } else if (stok == "IDMap") {
+
751  add_idmap = true;
+
752 
+
753  // IVFs
+
754  } else if (!index && stok == "Flat") {
755  if (coarse_quantizer) {
-
756  IndexIVFScalarQuantizer *index_ivf =
-
757  new IndexIVFScalarQuantizer (
-
758  coarse_quantizer, d, ncentroids, qt, metric);
-
759  index_ivf->quantizer_trains_alone =
+
756  // if there was an IVF in front, then it is an IVFFlat
+
757  IndexIVF *index_ivf = new IndexIVFFlat (
+
758  coarse_quantizer, d, ncentroids, metric);
+
759  index_ivf->quantizer_trains_alone =
760  get_trains_alone (coarse_quantizer);
-
761  del_coarse_quantizer.release ();
-
762  index_ivf->own_fields = true;
-
763  index_1 = index_ivf;
-
764  } else {
-
765  index_1 = new IndexScalarQuantizer (d, qt, metric);
-
766  }
-
767  } else if (!index && sscanf (tok, "PQ%d+%d", &M, &M2) == 2) {
-
768  FAISS_THROW_IF_NOT_MSG(coarse_quantizer,
-
769  "PQ with + works only with an IVF");
-
770  FAISS_THROW_IF_NOT_MSG(metric == METRIC_L2,
-
771  "IVFPQR not implemented for inner product search");
-
772  IndexIVFPQR *index_ivf = new IndexIVFPQR (
-
773  coarse_quantizer, d, ncentroids, M, 8, M2, 8);
-
774  index_ivf->quantizer_trains_alone =
-
775  get_trains_alone (coarse_quantizer);
-
776  del_coarse_quantizer.release ();
-
777  index_ivf->own_fields = true;
-
778  index_1 = index_ivf;
-
779  } else if (!index && sscanf (tok, "PQ%d%10s", &M, option) == 2) {
-
780  std::string soption = option;
-
781  // np to disable polysemous trainign
-
782  FAISS_THROW_IF_NOT(soption == "" || soption == "np");
-
783  if (coarse_quantizer) {
-
784  IndexIVFPQ *index_ivf = new IndexIVFPQ (
-
785  coarse_quantizer, d, ncentroids, M, 8);
-
786  index_ivf->quantizer_trains_alone =
-
787  get_trains_alone (coarse_quantizer);
-
788  index_ivf->metric_type = metric;
-
789  index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
-
790  del_coarse_quantizer.release ();
-
791  index_ivf->own_fields = true;
-
792  index_ivf->do_polysemous_training = soption != "np";
-
793  index_1 = index_ivf;
-
794  } else {
-
795  IndexPQ *index_pq = new IndexPQ (d, M, 8, metric);
-
796  index_pq->do_polysemous_training = soption != "np";
-
797  index_1 = index_pq;
-
798  }
-
799 
-
800  } else if (stok == "RFlat") {
-
801  make_IndexRefineFlat = true;
-
802  } else {
-
803  FAISS_THROW_FMT( "could not parse token \"%s\" in %s\n",
-
804  tok, description_in);
-
805  }
-
806 
-
807  if (index_1 && add_idmap) {
-
808  IndexIDMap *idmap = new IndexIDMap(index_1);
-
809  del_index.set (idmap);
-
810  idmap->own_fields = true;
-
811  index_1 = idmap;
-
812  add_idmap = false;
-
813  }
-
814 
-
815  if (vt_1) {
-
816  vts.chain.push_back (vt_1);
-
817  }
-
818 
-
819  if (coarse_quantizer_1) {
-
820  coarse_quantizer = coarse_quantizer_1;
-
821  del_coarse_quantizer.set (coarse_quantizer);
-
822  }
-
823 
-
824  if (index_1) {
-
825  index = index_1;
-
826  del_index.set (index);
-
827  }
-
828  }
-
829 
-
830  FAISS_THROW_IF_NOT_FMT(index, "descrption %s did not generate an index",
-
831  description_in);
-
832 
-
833  // nothing can go wrong now
-
834  del_index.release ();
-
835  del_coarse_quantizer.release ();
-
836 
-
837  if (add_idmap) {
-
838  fprintf(stderr, "index_factory: WARNING: "
-
839  "IDMap option not used\n");
-
840  }
-
841 
-
842  if (vts.chain.size() > 0) {
-
843  IndexPreTransform *index_pt = new IndexPreTransform (index);
-
844  index_pt->own_fields = true;
-
845  // add from back
-
846  while (vts.chain.size() > 0) {
-
847  index_pt->prepend_transform (vts.chain.back());
-
848  vts.chain.pop_back ();
-
849  }
-
850  index = index_pt;
-
851  }
+
761  index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
+
762  del_coarse_quantizer.release ();
+
763  index_ivf->own_fields = true;
+
764  index_1 = index_ivf;
+
765  } else {
+
766  index_1 = new IndexFlat (d, metric);
+
767  }
+
768  } else if (!index && (stok == "SQ8" || stok == "SQ4")) {
+
769  ScalarQuantizer::QuantizerType qt =
+
770  stok == "SQ8" ? ScalarQuantizer::QT_8bit :
+
771  stok == "SQ4" ? ScalarQuantizer::QT_4bit :
+
772  ScalarQuantizer::QT_4bit;
+
773  if (coarse_quantizer) {
+
774  IndexIVFScalarQuantizer *index_ivf =
+
775  new IndexIVFScalarQuantizer (
+
776  coarse_quantizer, d, ncentroids, qt, metric);
+
777  index_ivf->quantizer_trains_alone =
+
778  get_trains_alone (coarse_quantizer);
+
779  del_coarse_quantizer.release ();
+
780  index_ivf->own_fields = true;
+
781  index_1 = index_ivf;
+
782  } else {
+
783  index_1 = new IndexScalarQuantizer (d, qt, metric);
+
784  }
+
785  } else if (!index && sscanf (tok, "PQ%d+%d", &M, &M2) == 2) {
+
786  FAISS_THROW_IF_NOT_MSG(coarse_quantizer,
+
787  "PQ with + works only with an IVF");
+
788  FAISS_THROW_IF_NOT_MSG(metric == METRIC_L2,
+
789  "IVFPQR not implemented for inner product search");
+
790  IndexIVFPQR *index_ivf = new IndexIVFPQR (
+
791  coarse_quantizer, d, ncentroids, M, 8, M2, 8);
+
792  index_ivf->quantizer_trains_alone =
+
793  get_trains_alone (coarse_quantizer);
+
794  del_coarse_quantizer.release ();
+
795  index_ivf->own_fields = true;
+
796  index_1 = index_ivf;
+
797  } else if (!index && (sscanf (tok, "PQ%d", &M) == 1 ||
+
798  sscanf (tok, "PQ%dnp", &M) == 1)) {
+
799  bool do_polysemous_training = stok.find("np") == std::string::npos;
+
800  if (coarse_quantizer) {
+
801  IndexIVFPQ *index_ivf = new IndexIVFPQ (
+
802  coarse_quantizer, d, ncentroids, M, 8);
+
803  index_ivf->quantizer_trains_alone =
+
804  get_trains_alone (coarse_quantizer);
+
805  index_ivf->metric_type = metric;
+
806  index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
+
807  del_coarse_quantizer.release ();
+
808  index_ivf->own_fields = true;
+
809  index_ivf->do_polysemous_training = do_polysemous_training;
+
810  index_1 = index_ivf;
+
811  } else {
+
812  IndexPQ *index_pq = new IndexPQ (d, M, 8, metric);
+
813  index_pq->do_polysemous_training = do_polysemous_training;
+
814  index_1 = index_pq;
+
815  }
+
816  } else if (stok == "RFlat") {
+
817  make_IndexRefineFlat = true;
+
818  } else {
+
819  FAISS_THROW_FMT( "could not parse token \"%s\" in %s\n",
+
820  tok, description_in);
+
821  }
+
822 
+
823  if (index_1 && add_idmap) {
+
824  IndexIDMap *idmap = new IndexIDMap(index_1);
+
825  del_index.set (idmap);
+
826  idmap->own_fields = true;
+
827  index_1 = idmap;
+
828  add_idmap = false;
+
829  }
+
830 
+
831  if (vt_1) {
+
832  vts.chain.push_back (vt_1);
+
833  }
+
834 
+
835  if (coarse_quantizer_1) {
+
836  coarse_quantizer = coarse_quantizer_1;
+
837  del_coarse_quantizer.set (coarse_quantizer);
+
838  }
+
839 
+
840  if (index_1) {
+
841  index = index_1;
+
842  del_index.set (index);
+
843  }
+
844  }
+
845 
+
846  FAISS_THROW_IF_NOT_FMT(index, "descrption %s did not generate an index",
+
847  description_in);
+
848 
+
849  // nothing can go wrong now
+
850  del_index.release ();
+
851  del_coarse_quantizer.release ();
852 
-
853  if (make_IndexRefineFlat) {
-
854  IndexRefineFlat *index_rf = new IndexRefineFlat (index);
-
855  index_rf->own_fields = true;
-
856  index = index_rf;
-
857  }
-
858 
-
859  return index;
-
860 }
-
861 
-
862 
-
863 
-
864 
-
865 }; // namespace faiss
-
faiss::ParameterSpace::explore
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:535
- +
853  if (add_idmap) {
+
854  fprintf(stderr, "index_factory: WARNING: "
+
855  "IDMap option not used\n");
+
856  }
+
857 
+
858  if (vts.chain.size() > 0) {
+
859  IndexPreTransform *index_pt = new IndexPreTransform (index);
+
860  index_pt->own_fields = true;
+
861  // add from back
+
862  while (vts.chain.size() > 0) {
+
863  index_pt->prepend_transform (vts.chain.back ());
+
864  vts.chain.pop_back ();
+
865  }
+
866  index = index_pt;
+
867  }
+
868 
+
869  if (make_IndexRefineFlat) {
+
870  IndexRefineFlat *index_rf = new IndexRefineFlat (index);
+
871  index_rf->own_fields = true;
+
872  index = index_rf;
+
873  }
+
874 
+
875  return index;
+
876 }
+
877 
+
878 
+
879 
+
880 
+
881 }; // namespace faiss
+
faiss::ParameterSpace::explore
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:554
+
faiss::ParameterSpace::parameter_ranges
std::vector< ParameterRange > parameter_ranges
all tunable parameters
Definition: AutoTune.h:134
faiss::OperatingPoint::key
std::string key
key that identifies this op pt
Definition: AutoTune.h:89
faiss::OperatingPoint::cno
long cno
integer identifer
Definition: AutoTune.h:90
- +
faiss::IndexPQ::do_polysemous_training
bool do_polysemous_training
false = standard PQ
Definition: IndexPQ.h:69
- + - -
faiss::IndexIVF::quantizer_trains_alone
char quantizer_trains_alone
Definition: IndexIVF.h:56
-
faiss::OperatingPoints::display
void display(bool only_optimal=true) const
easy-to-read output
Definition: AutoTune.cpp:221
+ +
faiss::OperatingPoints::display
void display(bool only_optimal=true) const
easy-to-read output
Definition: AutoTune.cpp:222
faiss::OperatingPoint::perf
double perf
performance measure (output of a Criterion)
Definition: AutoTune.h:87
-
faiss::OperatingPoints::t_for_perf
double t_for_perf(double perf) const
get time required to obtain a given performance measure
Definition: AutoTune.cpp:174
+
faiss::OperatingPoints::t_for_perf
double t_for_perf(double perf) const
get time required to obtain a given performance measure
Definition: AutoTune.cpp:175
faiss::AutoTuneCriterion::nnn
idx_t nnn
nb of NNs that the query should request
Definition: AutoTune.h:29
- + -
faiss::OperatingPoints::add
bool add(double perf, double t, const std::string &key, size_t cno=0)
add a performance measure. Return whether it is an optimal point
Definition: AutoTune.cpp:112
- +
faiss::OperatingPoints::add
bool add(double perf, double t, const std::string &key, size_t cno=0)
add a performance measure. Return whether it is an optimal point
Definition: AutoTune.cpp:113
+
faiss::IndexIVFPQ::do_polysemous_training
bool do_polysemous_training
reorder PQ centroids after training?
Definition: IndexIVFPQ.h:34
faiss::ParameterSpace::batchsize
size_t batchsize
maximum number of queries to submit at a time.
Definition: AutoTune.h:145
faiss::AutoTuneCriterion::evaluate
virtual double evaluate(const float *D, const idx_t *I) const =0
- +
faiss::AutoTuneCriterion::nq
idx_t nq
nb of queries this criterion is evaluated on
Definition: AutoTune.h:28
- +
faiss::OperatingPoints::optimal_pts
std::vector< OperatingPoint > optimal_pts
optimal operating points, sorted by perf
Definition: AutoTune.h:98
-
faiss::AutoTuneCriterion::set_groundtruth
void set_groundtruth(int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
Definition: AutoTune.cpp:36
-
faiss::ParameterSpace::add_range
ParameterRange & add_range(const char *name)
add a new parameter
Definition: AutoTune.cpp:322
+
faiss::AutoTuneCriterion::set_groundtruth
void set_groundtruth(int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
Definition: AutoTune.cpp:37
+
faiss::ParameterSpace::add_range
ParameterRange & add_range(const char *name)
add a new parameter (or return it if it exists)
Definition: AutoTune.cpp:323
faiss::AutoTuneCriterion::gt_nnn
idx_t gt_nnn
nb of GT NNs required to evaluate crterion
Definition: AutoTune.h:30
-
faiss::OperatingPoints::all_to_gnuplot
void all_to_gnuplot(const char *fname) const
output to a format easy to digest by gnuplot
Definition: AutoTune.cpp:188
+
faiss::OperatingPoints::all_to_gnuplot
void all_to_gnuplot(const char *fname) const
output to a format easy to digest by gnuplot
Definition: AutoTune.cpp:189
faiss::IndexRefineFlat::own_fields
bool own_fields
should the base index be deallocated?
Definition: IndexFlat.h:110
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:34
-
faiss::IndexIVF::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:59
+ -
faiss::IndexIVF::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:57
faiss::OperatingPoints::all_pts
std::vector< OperatingPoint > all_pts
all operating points
Definition: AutoTune.h:95
-
faiss::ranklist_intersection_size
size_t ranklist_intersection_size(size_t k1, const long *v1, size_t k2, const long *v2_in)
Definition: utils.cpp:1574
+
faiss::ranklist_intersection_size
size_t ranklist_intersection_size(size_t k1, const long *v1, size_t k2, const long *v2_in)
Definition: utils.cpp:1576
+
faiss::Level1Quantizer::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:44
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
faiss::AutoTuneCriterion::gt_D
std::vector< float > gt_D
Ground-truth distances (size nq * gt_nnn)
Definition: AutoTune.h:32
- -
faiss::ParameterSpace::combination_name
std::string combination_name(size_t cno) const
get string representation of the combination
Definition: AutoTune.cpp:277
+ +
faiss::ParameterSpace::combination_name
std::string combination_name(size_t cno) const
get string representation of the combination
Definition: AutoTune.cpp:278
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
-
faiss::ParameterSpace::update_bounds
void update_bounds(size_t cno, const OperatingPoint &op, double *upper_bound_perf, double *lower_bound_t) const
Definition: AutoTune.cpp:521
+
faiss::ParameterSpace::update_bounds
void update_bounds(size_t cno, const OperatingPoint &op, double *upper_bound_perf, double *lower_bound_t) const
Definition: AutoTune.cpp:540
-
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:244
-
faiss::ParameterSpace::initialize
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:331
+
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:251
+
faiss::ParameterSpace::initialize
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:337
faiss::ParameterSpace::verbose
int verbose
verbosity during exploration
Definition: AutoTune.h:139
-
faiss::OperatingPoints::merge_with
int merge_with(const OperatingPoints &other, const std::string &prefix="")
add operating points from other to this, with a prefix to the keys
Definition: AutoTune.cpp:159
-
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:423
-
faiss::ParameterSpace::n_combinations
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:268
+
faiss::OperatingPoints::merge_with
int merge_with(const OperatingPoints &other, const std::string &prefix="")
add operating points from other to this, with a prefix to the keys
Definition: AutoTune.cpp:160
+
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:443
+
faiss::ParameterSpace::n_combinations
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:269
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
-
faiss::ParameterSpace::set_index_parameters
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:390
+
faiss::ParameterSpace::set_index_parameters
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:410
faiss::IndexPQ::ST_PQ
asymmetric product quantizer (default)
Definition: IndexPQ.h:76
-
faiss::ParameterSpace::display
void display() const
print a description on stdout
Definition: AutoTune.cpp:503
+
faiss::ParameterSpace::display
void display() const
print a description on stdout
Definition: AutoTune.cpp:522
faiss::IndexPQ::ST_polysemous
HE filter (using ht) + PQ combination.
Definition: IndexPQ.h:80
- -
faiss::ParameterSpace::combination_ge
bool combination_ge(size_t c1, size_t c2) const
returns whether combinations c1 &gt;= c2 in the tuple sense
Definition: AutoTune.cpp:292
+ +
faiss::ParameterSpace::combination_ge
bool combination_ge(size_t c1, size_t c2) const
returns whether combinations c1 &gt;= c2 in the tuple sense
Definition: AutoTune.cpp:293
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:29
+
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:42
faiss::ParameterRange
possible values of a parameter, sorted from least to most expensive/accurate
Definition: AutoTune.h:125
-
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:668
+
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:687
faiss::ParameterSpace::n_experiments
int n_experiments
nb of experiments during optimization (0 = try all combinations)
Definition: AutoTune.h:142
faiss::AutoTuneCriterion::gt_I
std::vector< idx_t > gt_I
Ground-truth indexes (size nq * gt_nnn)
Definition: AutoTune.h:33
faiss::OperatingPoint::t
double t
corresponding execution time (ms)
Definition: AutoTune.h:88
diff --git a/docs/html/AutoTune_8h_source.html b/docs/html/AutoTune_8h_source.html index c63aae318..f81ce9828 100644 --- a/docs/html/AutoTune_8h_source.html +++ b/docs/html/AutoTune_8h_source.html @@ -247,7 +247,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
162  /// print a description on stdout
163  void display () const;
164 
-
165  /// add a new parameter
+
165  /// add a new parameter (or return it if it exists)
166  ParameterRange &add_range(const char * name);
167 
168  /// initialize with reasonable parameters for the index
@@ -295,46 +295,46 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
210 
211 
212 #endif
-
faiss::ParameterSpace::explore
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:535
+
faiss::ParameterSpace::explore
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:554
faiss::ParameterSpace::parameter_ranges
std::vector< ParameterRange > parameter_ranges
all tunable parameters
Definition: AutoTune.h:134
faiss::OperatingPoint::key
std::string key
key that identifies this op pt
Definition: AutoTune.h:89
faiss::OperatingPoint::cno
long cno
integer identifer
Definition: AutoTune.h:90
- + -
faiss::OperatingPoints::display
void display(bool only_optimal=true) const
easy-to-read output
Definition: AutoTune.cpp:221
+
faiss::OperatingPoints::display
void display(bool only_optimal=true) const
easy-to-read output
Definition: AutoTune.cpp:222
faiss::OperatingPoint::perf
double perf
performance measure (output of a Criterion)
Definition: AutoTune.h:87
-
faiss::OperatingPoints::t_for_perf
double t_for_perf(double perf) const
get time required to obtain a given performance measure
Definition: AutoTune.cpp:174
+
faiss::OperatingPoints::t_for_perf
double t_for_perf(double perf) const
get time required to obtain a given performance measure
Definition: AutoTune.cpp:175
faiss::AutoTuneCriterion::nnn
idx_t nnn
nb of NNs that the query should request
Definition: AutoTune.h:29
-
faiss::OperatingPoints::add
bool add(double perf, double t, const std::string &key, size_t cno=0)
add a performance measure. Return whether it is an optimal point
Definition: AutoTune.cpp:112
+
faiss::OperatingPoints::add
bool add(double perf, double t, const std::string &key, size_t cno=0)
add a performance measure. Return whether it is an optimal point
Definition: AutoTune.cpp:113
faiss::ParameterSpace::batchsize
size_t batchsize
maximum number of queries to submit at a time.
Definition: AutoTune.h:145
faiss::AutoTuneCriterion::evaluate
virtual double evaluate(const float *D, const idx_t *I) const =0
faiss::AutoTuneCriterion::nq
idx_t nq
nb of queries this criterion is evaluated on
Definition: AutoTune.h:28
faiss::OperatingPoints::optimal_pts
std::vector< OperatingPoint > optimal_pts
optimal operating points, sorted by perf
Definition: AutoTune.h:98
-
faiss::AutoTuneCriterion::set_groundtruth
void set_groundtruth(int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
Definition: AutoTune.cpp:36
-
faiss::ParameterSpace::add_range
ParameterRange & add_range(const char *name)
add a new parameter
Definition: AutoTune.cpp:322
+
faiss::AutoTuneCriterion::set_groundtruth
void set_groundtruth(int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
Definition: AutoTune.cpp:37
+
faiss::ParameterSpace::add_range
ParameterRange & add_range(const char *name)
add a new parameter (or return it if it exists)
Definition: AutoTune.cpp:323
faiss::AutoTuneCriterion::gt_nnn
idx_t gt_nnn
nb of GT NNs required to evaluate crterion
Definition: AutoTune.h:30
-
faiss::OperatingPoints::all_to_gnuplot
void all_to_gnuplot(const char *fname) const
output to a format easy to digest by gnuplot
Definition: AutoTune.cpp:188
+
faiss::OperatingPoints::all_to_gnuplot
void all_to_gnuplot(const char *fname) const
output to a format easy to digest by gnuplot
Definition: AutoTune.cpp:189
faiss::OperatingPoints::all_pts
std::vector< OperatingPoint > all_pts
all operating points
Definition: AutoTune.h:95
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::AutoTuneCriterion::gt_D
std::vector< float > gt_D
Ground-truth distances (size nq * gt_nnn)
Definition: AutoTune.h:32
-
faiss::ParameterSpace::combination_name
std::string combination_name(size_t cno) const
get string representation of the combination
Definition: AutoTune.cpp:277
-
faiss::ParameterSpace::update_bounds
void update_bounds(size_t cno, const OperatingPoint &op, double *upper_bound_perf, double *lower_bound_t) const
Definition: AutoTune.cpp:521
-
faiss::ParameterSpace::initialize
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:331
+
faiss::ParameterSpace::combination_name
std::string combination_name(size_t cno) const
get string representation of the combination
Definition: AutoTune.cpp:278
+
faiss::ParameterSpace::update_bounds
void update_bounds(size_t cno, const OperatingPoint &op, double *upper_bound_perf, double *lower_bound_t) const
Definition: AutoTune.cpp:540
+
faiss::ParameterSpace::initialize
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:337
faiss::ParameterSpace::verbose
int verbose
verbosity during exploration
Definition: AutoTune.h:139
-
faiss::OperatingPoints::merge_with
int merge_with(const OperatingPoints &other, const std::string &prefix="")
add operating points from other to this, with a prefix to the keys
Definition: AutoTune.cpp:159
-
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:423
-
faiss::ParameterSpace::n_combinations
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:268
-
faiss::ParameterSpace::set_index_parameters
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:390
+
faiss::OperatingPoints::merge_with
int merge_with(const OperatingPoints &other, const std::string &prefix="")
add operating points from other to this, with a prefix to the keys
Definition: AutoTune.cpp:160
+
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:443
+
faiss::ParameterSpace::n_combinations
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:269
+
faiss::ParameterSpace::set_index_parameters
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:410
-
faiss::ParameterSpace::display
void display() const
print a description on stdout
Definition: AutoTune.cpp:503
-
faiss::ParameterSpace::combination_ge
bool combination_ge(size_t c1, size_t c2) const
returns whether combinations c1 &gt;= c2 in the tuple sense
Definition: AutoTune.cpp:292
+
faiss::ParameterSpace::display
void display() const
print a description on stdout
Definition: AutoTune.cpp:522
+
faiss::ParameterSpace::combination_ge
bool combination_ge(size_t c1, size_t c2) const
returns whether combinations c1 &gt;= c2 in the tuple sense
Definition: AutoTune.cpp:293
faiss::ParameterRange
possible values of a parameter, sorted from least to most expensive/accurate
Definition: AutoTune.h:125
-
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:668
+
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:687
faiss::ParameterSpace::n_experiments
int n_experiments
nb of experiments during optimization (0 = try all combinations)
Definition: AutoTune.h:142
faiss::AutoTuneCriterion::gt_I
std::vector< idx_t > gt_I
Ground-truth indexes (size nq * gt_nnn)
Definition: AutoTune.h:33
faiss::OperatingPoint::t
double t
corresponding execution time (ms)
Definition: AutoTune.h:88
diff --git a/docs/html/Clustering_8cpp_source.html b/docs/html/Clustering_8cpp_source.html index 7c0aa87b9..7151325f8 100644 --- a/docs/html/Clustering_8cpp_source.html +++ b/docs/html/Clustering_8cpp_source.html @@ -150,177 +150,192 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
65 
66 
67 void Clustering::train (idx_t nx, const float *x_in, Index & index) {
-
68  FAISS_THROW_IF_NOT_MSG (nx >= k,
-
69  "need at least as many training points as clusters");
-
70 
-
71  double t0 = getmillisecs();
-
72 
-
73  // yes it is the user's responsibility, but it may spare us some
-
74  // hard-to-debug reports.
-
75  for (size_t i = 0; i < nx * d; i++) {
-
76  FAISS_THROW_IF_NOT_MSG (finite (x_in[i]),
-
77  "input contains NaN's or Inf's");
-
78  }
-
79 
-
80  const float *x = x_in;
-
81  ScopeDeleter<float> del1;
-
82 
-
83  if (nx > k * max_points_per_centroid) {
-
84  if (verbose)
-
85  printf("Sampling a subset of %ld / %ld for training\n",
-
86  k * max_points_per_centroid, nx);
-
87  std::vector<int> perm (nx);
-
88  rand_perm (perm.data (), nx, seed);
-
89  nx = k * max_points_per_centroid;
-
90  float * x_new = new float [nx * d];
-
91  for (idx_t i = 0; i < nx; i++)
-
92  memcpy (x_new + i * d, x + perm[i] * d, sizeof(x_new[0]) * d);
-
93  x = x_new;
-
94  del1.set (x);
-
95  } else if (nx < k * min_points_per_centroid) {
-
96  fprintf (stderr,
-
97  "WARNING clustering %ld points to %ld centroids: "
-
98  "please provide at least %ld training points\n",
-
99  nx, k, idx_t(k) * min_points_per_centroid);
-
100  }
-
101 
+
68  FAISS_THROW_IF_NOT_FMT (nx >= k,
+
69  "Number of training points (%ld) should be at least "
+
70  "as large as number of clusters (%ld)", nx, k);
+
71 
+
72  double t0 = getmillisecs();
+
73 
+
74  // yes it is the user's responsibility, but it may spare us some
+
75  // hard-to-debug reports.
+
76  for (size_t i = 0; i < nx * d; i++) {
+
77  FAISS_THROW_IF_NOT_MSG (finite (x_in[i]),
+
78  "input contains NaN's or Inf's");
+
79  }
+
80 
+
81  const float *x = x_in;
+
82  ScopeDeleter<float> del1;
+
83 
+
84  if (nx > k * max_points_per_centroid) {
+
85  if (verbose)
+
86  printf("Sampling a subset of %ld / %ld for training\n",
+
87  k * max_points_per_centroid, nx);
+
88  std::vector<int> perm (nx);
+
89  rand_perm (perm.data (), nx, seed);
+
90  nx = k * max_points_per_centroid;
+
91  float * x_new = new float [nx * d];
+
92  for (idx_t i = 0; i < nx; i++)
+
93  memcpy (x_new + i * d, x + perm[i] * d, sizeof(x_new[0]) * d);
+
94  x = x_new;
+
95  del1.set (x);
+
96  } else if (nx < k * min_points_per_centroid) {
+
97  fprintf (stderr,
+
98  "WARNING clustering %ld points to %ld centroids: "
+
99  "please provide at least %ld training points\n",
+
100  nx, k, idx_t(k) * min_points_per_centroid);
+
101  }
102 
-
103  if (verbose)
-
104  printf("Clustering %d points in %ldD to %ld clusters, "
-
105  "redo %d times, %d iterations\n",
-
106  int(nx), d, k, nredo, niter);
-
107 
-
108 
-
109  idx_t * assign = new idx_t[nx];
-
110  ScopeDeleter<idx_t> del (assign);
-
111  float * dis = new float[nx];
-
112  ScopeDeleter<float> del2(dis);
-
113 
-
114  // for redo
-
115  float best_err = 1e50;
-
116  std::vector<float> best_obj;
-
117  std::vector<float> best_centroids;
-
118 
-
119  // support input centroids
+
103 
+
104  if (nx == k) {
+
105  if (verbose) {
+
106  printf("Number of training points (%ld) same as number of "
+
107  "clusters, just copying\n", nx);
+
108  }
+
109  // this is a corner case, just copy training set to clusters
+
110  centroids.resize (d * k);
+
111  memcpy (centroids.data(), x_in, sizeof (*x_in) * d * k);
+
112  return;
+
113  }
+
114 
+
115 
+
116  if (verbose)
+
117  printf("Clustering %d points in %ldD to %ld clusters, "
+
118  "redo %d times, %d iterations\n",
+
119  int(nx), d, k, nredo, niter);
120 
-
121  FAISS_THROW_IF_NOT_MSG (
-
122  centroids.size() % d == 0,
-
123  "size of provided input centroids not a multiple of dimension");
-
124 
-
125  size_t n_input_centroids = centroids.size() / d;
-
126 
-
127  if (verbose && n_input_centroids > 0) {
-
128  printf (" Using %zd centroids provided as input (%sfrozen)\n",
-
129  n_input_centroids, frozen_centroids ? "" : "not ");
-
130  }
-
131 
-
132  double t_search_tot = 0;
-
133  if (verbose) {
-
134  printf(" Preprocessing in %.2f s\n",
-
135  (getmillisecs() - t0)/1000.);
-
136  }
-
137  t0 = getmillisecs();
-
138 
-
139  for (int redo = 0; redo < nredo; redo++) {
-
140 
-
141  if (verbose && nredo > 1) {
-
142  printf("Outer iteration %d / %d\n", redo, nredo);
-
143  }
-
144 
-
145 
-
146  // initialize remaining centroids with random points from the dataset
-
147  centroids.resize (d * k);
-
148  std::vector<int> perm (nx);
-
149 
-
150  rand_perm (perm.data(), nx, seed + 1 + redo * 15486557L);
-
151  for (int i = n_input_centroids; i < k ; i++)
-
152  memcpy (&centroids[i * d], x + perm[i] * d,
-
153  d * sizeof (float));
-
154 
-
155  if (spherical)
-
156  fvec_renorm_L2 (d, k, centroids.data());
-
157 
-
158  if (!index.is_trained)
-
159  index.train (k, centroids.data());
+
121 
+
122 
+
123 
+
124  idx_t * assign = new idx_t[nx];
+
125  ScopeDeleter<idx_t> del (assign);
+
126  float * dis = new float[nx];
+
127  ScopeDeleter<float> del2(dis);
+
128 
+
129  // for redo
+
130  float best_err = 1e50;
+
131  std::vector<float> best_obj;
+
132  std::vector<float> best_centroids;
+
133 
+
134  // support input centroids
+
135 
+
136  FAISS_THROW_IF_NOT_MSG (
+
137  centroids.size() % d == 0,
+
138  "size of provided input centroids not a multiple of dimension");
+
139 
+
140  size_t n_input_centroids = centroids.size() / d;
+
141 
+
142  if (verbose && n_input_centroids > 0) {
+
143  printf (" Using %zd centroids provided as input (%sfrozen)\n",
+
144  n_input_centroids, frozen_centroids ? "" : "not ");
+
145  }
+
146 
+
147  double t_search_tot = 0;
+
148  if (verbose) {
+
149  printf(" Preprocessing in %.2f s\n",
+
150  (getmillisecs() - t0)/1000.);
+
151  }
+
152  t0 = getmillisecs();
+
153 
+
154  for (int redo = 0; redo < nredo; redo++) {
+
155 
+
156  if (verbose && nredo > 1) {
+
157  printf("Outer iteration %d / %d\n", redo, nredo);
+
158  }
+
159 
160 
-
161  FAISS_THROW_IF_NOT (index.ntotal == 0);
-
162  index.add (k, centroids.data());
-
163  float err = 0;
-
164  for (int i = 0; i < niter; i++) {
-
165  double t0s = getmillisecs();
-
166  index.search (nx, x, 1, dis, assign);
-
167  t_search_tot += getmillisecs() - t0s;
-
168 
-
169  err = 0;
-
170  for (int j = 0; j < nx; j++)
-
171  err += dis[j];
-
172  obj.push_back (err);
-
173 
-
174  int nsplit = km_update_centroids (
-
175  x, centroids.data(),
-
176  assign, d, k, nx, frozen_centroids ? n_input_centroids : 0);
-
177 
-
178  if (verbose) {
-
179  printf (" Iteration %d (%.2f s, search %.2f s): "
-
180  "objective=%g imbalance=%.3f nsplit=%d \r",
-
181  i, (getmillisecs() - t0) / 1000.0,
-
182  t_search_tot / 1000,
-
183  err, imbalance_factor (nx, k, assign),
-
184  nsplit);
-
185  fflush (stdout);
-
186  }
-
187 
-
188  if (spherical)
-
189  fvec_renorm_L2 (d, k, centroids.data());
-
190 
-
191  index.reset ();
-
192  if (update_index)
-
193  index.train (k, centroids.data());
-
194 
-
195  assert (index.ntotal == 0);
-
196  index.add (k, centroids.data());
-
197  }
-
198  if (verbose) printf("\n");
-
199  if (nredo > 1) {
-
200  if (err < best_err) {
-
201  if (verbose)
-
202  printf ("Objective improved: keep new clusters\n");
-
203  best_centroids = centroids;
-
204  best_obj = obj;
-
205  best_err = err;
-
206  }
-
207  index.reset ();
-
208  }
-
209  }
-
210  if (nredo > 1) {
-
211  centroids = best_centroids;
-
212  obj = best_obj;
-
213  }
-
214 
-
215 }
-
216 
-
217 float kmeans_clustering (size_t d, size_t n, size_t k,
-
218  const float *x,
-
219  float *centroids)
-
220 {
-
221  Clustering clus (d, k);
-
222  clus.verbose = d * n * k > (1L << 30);
-
223  // display logs if > 1Gflop per iteration
-
224  IndexFlatL2 index (d);
-
225  clus.train (n, x, index);
-
226  memcpy(centroids, clus.centroids.data(), sizeof(*centroids) * d * k);
-
227  return clus.obj.back();
-
228 }
+
161  // initialize remaining centroids with random points from the dataset
+
162  centroids.resize (d * k);
+
163  std::vector<int> perm (nx);
+
164 
+
165  rand_perm (perm.data(), nx, seed + 1 + redo * 15486557L);
+
166  for (int i = n_input_centroids; i < k ; i++)
+
167  memcpy (&centroids[i * d], x + perm[i] * d,
+
168  d * sizeof (float));
+
169 
+
170  if (spherical)
+
171  fvec_renorm_L2 (d, k, centroids.data());
+
172 
+
173  if (!index.is_trained)
+
174  index.train (k, centroids.data());
+
175 
+
176  FAISS_THROW_IF_NOT (index.ntotal == 0);
+
177  index.add (k, centroids.data());
+
178  float err = 0;
+
179  for (int i = 0; i < niter; i++) {
+
180  double t0s = getmillisecs();
+
181  index.search (nx, x, 1, dis, assign);
+
182  t_search_tot += getmillisecs() - t0s;
+
183 
+
184  err = 0;
+
185  for (int j = 0; j < nx; j++)
+
186  err += dis[j];
+
187  obj.push_back (err);
+
188 
+
189  int nsplit = km_update_centroids (
+
190  x, centroids.data(),
+
191  assign, d, k, nx, frozen_centroids ? n_input_centroids : 0);
+
192 
+
193  if (verbose) {
+
194  printf (" Iteration %d (%.2f s, search %.2f s): "
+
195  "objective=%g imbalance=%.3f nsplit=%d \r",
+
196  i, (getmillisecs() - t0) / 1000.0,
+
197  t_search_tot / 1000,
+
198  err, imbalance_factor (nx, k, assign),
+
199  nsplit);
+
200  fflush (stdout);
+
201  }
+
202 
+
203  if (spherical)
+
204  fvec_renorm_L2 (d, k, centroids.data());
+
205 
+
206  index.reset ();
+
207  if (update_index)
+
208  index.train (k, centroids.data());
+
209 
+
210  assert (index.ntotal == 0);
+
211  index.add (k, centroids.data());
+
212  }
+
213  if (verbose) printf("\n");
+
214  if (nredo > 1) {
+
215  if (err < best_err) {
+
216  if (verbose)
+
217  printf ("Objective improved: keep new clusters\n");
+
218  best_centroids = centroids;
+
219  best_obj = obj;
+
220  best_err = err;
+
221  }
+
222  index.reset ();
+
223  }
+
224  }
+
225  if (nredo > 1) {
+
226  centroids = best_centroids;
+
227  obj = best_obj;
+
228  }
229 
-
230 } // namespace faiss
-
faiss::km_update_centroids
int km_update_centroids(const float *x, float *centroids, long *assign, size_t d, size_t k, size_t n, size_t k_frozen)
Definition: utils.cpp:1399
+
230 }
+
231 
+
232 float kmeans_clustering (size_t d, size_t n, size_t k,
+
233  const float *x,
+
234  float *centroids)
+
235 {
+
236  Clustering clus (d, k);
+
237  clus.verbose = d * n * k > (1L << 30);
+
238  // display logs if > 1Gflop per iteration
+
239  IndexFlatL2 index (d);
+
240  clus.train (n, x, index);
+
241  memcpy(centroids, clus.centroids.data(), sizeof(*centroids) * d * k);
+
242  return clus.obj.back();
+
243 }
+
244 
+
245 } // namespace faiss
+
faiss::km_update_centroids
int km_update_centroids(const float *x, float *centroids, long *assign, size_t d, size_t k, size_t n, size_t k_frozen)
Definition: utils.cpp:1401
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:25
faiss::ClusteringParameters::nredo
int nredo
redo clustering this many times and keep best
Definition: Clustering.h:26
faiss::ClusteringParameters::ClusteringParameters
ClusteringParameters()
sets reasonable defaults
Definition: Clustering.cpp:27
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
faiss::Clustering::Clustering
Clustering(int d, int k)
the only mandatory parameters are k and d
Definition: Clustering.cpp:40
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
faiss::Clustering::k
size_t k
nb of centroids
Definition: Clustering.h:60
faiss::ClusteringParameters::seed
int seed
seed for the random number generator
Definition: Clustering.h:36
@@ -329,7 +344,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::Clustering::obj
std::vector< float > obj
Definition: Clustering.h:67
-
faiss::kmeans_clustering
float kmeans_clustering(size_t d, size_t n, size_t k, const float *x, float *centroids)
Definition: Clustering.cpp:217
+
faiss::kmeans_clustering
float kmeans_clustering(size_t d, size_t n, size_t k, const float *x, float *centroids)
Definition: Clustering.cpp:232
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
faiss::Clustering::centroids
std::vector< float > centroids
centroids (k * d)
Definition: Clustering.h:63
diff --git a/docs/html/Clustering_8h_source.html b/docs/html/Clustering_8h_source.html index 5be366be2..ab066c4a6 100644 --- a/docs/html/Clustering_8h_source.html +++ b/docs/html/Clustering_8h_source.html @@ -192,7 +192,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::Clustering::obj
std::vector< float > obj
Definition: Clustering.h:67
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
-
faiss::kmeans_clustering
float kmeans_clustering(size_t d, size_t n, size_t k, const float *x, float *centroids)
Definition: Clustering.cpp:217
+
faiss::kmeans_clustering
float kmeans_clustering(size_t d, size_t n, size_t k, const float *x, float *centroids)
Definition: Clustering.cpp:232
faiss::Clustering::centroids
std::vector< float > centroids
centroids (k * d)
Definition: Clustering.h:63
faiss::Clustering::d
size_t d
dimension of the vectors
Definition: Clustering.h:59
faiss::ClusteringParameters::update_index
bool update_index
update index after each iteration?
Definition: Clustering.h:30
diff --git a/docs/html/CompareIVFFlat_8cu_source.html b/docs/html/CompareIVFFlat_8cu_source.html index a23c0f6f9..a3ead2591 100644 --- a/docs/html/CompareIVFFlat_8cu_source.html +++ b/docs/html/CompareIVFFlat_8cu_source.html @@ -240,11 +240,11 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
151  return 0;
152 }
faiss::gpu::CpuTimer::elapsedMilliseconds
float elapsedMilliseconds()
Returns elapsed time in milliseconds.
Definition: Timer.cpp:52
-
faiss::read_index
Index * read_index(FILE *f, bool try_mmap)
Definition: index_io.cpp:536
+
faiss::read_index
Index * read_index(FILE *f, bool try_mmap)
Definition: index_io.cpp:592
faiss::gpu::CpuTimer
CPU wallclock elapsed timer.
Definition: Timer.h:42
- +
faiss::gpu::GpuIndexFlatConfig::useFloat16
bool useFloat16
Whether or not data is stored as float16.
Definition: GpuIndexFlat.h:35
faiss::gpu::GpuIndexConfig::device
int device
GPU device on which the index is resident.
Definition: GpuIndex.h:27
faiss::gpu::GpuIndexIVFConfig::flatConfig
GpuIndexFlatConfig flatConfig
Configuration for the coarse quantizer object.
Definition: GpuIndexIVF.h:34
diff --git a/docs/html/CompareIVFPQGrid_8cu_source.html b/docs/html/CompareIVFPQGrid_8cu_source.html index ab4be88b5..77a53b4b4 100644 --- a/docs/html/CompareIVFPQGrid_8cu_source.html +++ b/docs/html/CompareIVFPQGrid_8cu_source.html @@ -284,7 +284,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
195 }
faiss::gpu::CpuTimer::elapsedMilliseconds
float elapsedMilliseconds()
Returns elapsed time in milliseconds.
Definition: Timer.cpp:52
-
faiss::read_index
Index * read_index(FILE *f, bool try_mmap)
Definition: index_io.cpp:536
+
faiss::read_index
Index * read_index(FILE *f, bool try_mmap)
Definition: index_io.cpp:592
faiss::gpu::CpuTimer
CPU wallclock elapsed timer.
Definition: Timer.h:42
diff --git a/docs/html/CompareIVFPQ_8cu_source.html b/docs/html/CompareIVFPQ_8cu_source.html index f563b17cf..443118b8f 100644 --- a/docs/html/CompareIVFPQ_8cu_source.html +++ b/docs/html/CompareIVFPQ_8cu_source.html @@ -248,7 +248,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
159 }
faiss::gpu::CpuTimer::elapsedMilliseconds
float elapsedMilliseconds()
Returns elapsed time in milliseconds.
Definition: Timer.cpp:52
-
faiss::read_index
Index * read_index(FILE *f, bool try_mmap)
Definition: index_io.cpp:536
+
faiss::read_index
Index * read_index(FILE *f, bool try_mmap)
Definition: index_io.cpp:592
faiss::gpu::CpuTimer
CPU wallclock elapsed timer.
Definition: Timer.h:42
diff --git a/docs/html/DeviceDefs_8cuh_source.html b/docs/html/DeviceDefs_8cuh_source.html index 7d0eb5cb1..cc8c908aa 100644 --- a/docs/html/DeviceDefs_8cuh_source.html +++ b/docs/html/DeviceDefs_8cuh_source.html @@ -102,7 +102,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
13 namespace faiss { namespace gpu {
14 
15 #ifdef __CUDA_ARCH__
-
16 #if __CUDA_ARCH__ <= 620
+
16 #if __CUDA_ARCH__ <= 700
17 constexpr int kWarpSize = 32;
18 #else
19 #error Unknown __CUDA_ARCH__; please define parameters for compute capability
@@ -114,40 +114,18 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
25 constexpr int kWarpSize = 32;
26 #endif // !__CUDA_ARCH__
27 
-
28 __forceinline__ __device__ void warpFence() {
-
29  // Technically, memory barriers are required via the CUDA
-
30  // programming model, since warp synchronous programming no longer
-
31  // is guaranteed.
-
32  //
-
33  // There are two components to it:
-
34  // -a barrier known to the compiler such that the compiler will not
-
35  // schedule loads and stores across the barrier;
-
36  // -a HW-level barrier that guarantees that writes are seen in the
-
37  // proper order
-
38  //
-
39  // However, __threadfence_block() is a stronger constraint than what
-
40  // we really want out of the hardware: a warp-wide barrier.
-
41  //
-
42  // In current hardware, it appears that warp synchronous programming
-
43  // is a reality; by all tests it appears safe and race-free.
-
44  //
-
45  // However, understandably it may not be in the future (based on
-
46  // what Nvidia says in the Kepler guide, it may change depending
-
47  // upon compiler/toolchain issues or future hardware).
-
48  //
-
49  // Removing the fence results in 10%+ faster performance.
-
50  // However, we are judicious as to where we insert the fence, so if
-
51  // this reality ever changes, uncommenting this will result in CUDA
-
52  // programming model-safe ordering again.
-
53  //
-
54  // FIXME: we should probably qualify as volatile as well, since the
-
55  // compiler could technically preserve values across loops? This
-
56  // seems very impractical for the compiler to do, however.
-
57 
-
58  // __threadfence_block();
-
59 }
-
60 
-
61 } } // namespace
+
28 // This is a memory barrier for intra-warp writes to shared memory.
+
29 __forceinline__ __device__ void warpFence() {
+
30 
+
31 #if __CUDA_ARCH__ >= 700
+
32  __syncwarp();
+
33 #else
+
34  // For the time being, assume synchronicity.
+
35  // __threadfence_block();
+
36 #endif
+
37 }
+
38 
+
39 } } // namespace
diff --git a/docs/html/DeviceTensor-inl_8cuh_source.html b/docs/html/DeviceTensor-inl_8cuh_source.html index 11d79c74b..c1f92ed85 100644 --- a/docs/html/DeviceTensor-inl_8cuh_source.html +++ b/docs/html/DeviceTensor-inl_8cuh_source.html @@ -131,7 +131,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
42  CUDA_VERIFY(cudaFree(this->data_));
43  }
44 
-
45  this->Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator=(
+
45  this->Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator=(
46  std::move(t));
47 
48  this->state_ = t.state_; t.state_ = AllocState::NotOwner;
@@ -319,16 +319,16 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
230 } } // namespace
faiss::gpu::DeviceTensor::~DeviceTensor
__host__ ~DeviceTensor()
Destructor.
Definition: DeviceTensor-inl.cuh:58
faiss::gpu::DeviceTensor::zero
__host__ DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > & zero(cudaStream_t stream)
Call to zero out memory.
Definition: DeviceTensor-inl.cuh:217
-
faiss::gpu::Tensor::data_
DataPtrType data_
Raw pointer to where the tensor data begins.
Definition: Tensor.cuh:346
+
faiss::gpu::Tensor::data_
DataPtrType data_
Raw pointer to where the tensor data begins.
Definition: Tensor.cuh:344
faiss::gpu::DeviceMemory::getMemory
virtual DeviceMemoryReservation getMemory(cudaStream_t stream, size_t size)=0
-
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:101
+
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:132
+
faiss::gpu::Tensor::operator=
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)
Assignment.
Definition: Tensor-inl.cuh:50
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
faiss::gpu::DeviceTensor::DeviceTensor
__host__ DeviceTensor()
Default constructor.
Definition: DeviceTensor-inl.cuh:18
faiss::gpu::DeviceTensor::operator=
__host__ DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)
Move assignment.
Definition: DeviceTensor-inl.cuh:39
-
faiss::gpu::Tensor::operator=
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default
Assignment.
faiss::gpu::DeviceMemory
Manages temporary memory allocations on a GPU device.
Definition: DeviceMemory.h:46
-
faiss::gpu::Tensor::getSizeInBytes
__host__ __device__ size_t getSizeInBytes() const
Definition: Tensor.cuh:242
+
faiss::gpu::Tensor::getSizeInBytes
__host__ __device__ size_t getSizeInBytes() const
Definition: Tensor.cuh:240
diff --git a/docs/html/DeviceTensor_8cuh_source.html b/docs/html/DeviceTensor_8cuh_source.html index 95cc2c415..fd3448ff2 100644 --- a/docs/html/DeviceTensor_8cuh_source.html +++ b/docs/html/DeviceTensor_8cuh_source.html @@ -204,9 +204,9 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
115 #include "DeviceTensor-inl.cuh"
faiss::gpu::DeviceTensor::~DeviceTensor
__host__ ~DeviceTensor()
Destructor.
Definition: DeviceTensor-inl.cuh:58
faiss::gpu::DeviceTensor::zero
__host__ DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > & zero(cudaStream_t stream)
Call to zero out memory.
Definition: DeviceTensor-inl.cuh:217
-
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:247
-
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:252
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:245
+
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:250
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
faiss::gpu::DeviceTensor::DeviceTensor
__host__ DeviceTensor()
Default constructor.
Definition: DeviceTensor-inl.cuh:18
faiss::gpu::DeviceTensor::operator=
__host__ DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)
Move assignment.
Definition: DeviceTensor-inl.cuh:39
diff --git a/docs/html/FlatIndex_8cu_source.html b/docs/html/FlatIndex_8cu_source.html index 57cae51e5..d049cd129 100644 --- a/docs/html/FlatIndex_8cu_source.html +++ b/docs/html/FlatIndex_8cu_source.html @@ -369,8 +369,8 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::GpuResources::getMemoryManagerCurrentDevice
DeviceMemory & getMemoryManagerCurrentDevice()
Calls getMemoryManager for the current device.
Definition: GpuResources.cpp:35
faiss::gpu::FlatIndex::reserve
void reserve(size_t numVecs, cudaStream_t stream)
Reserve storage that can contain at least this many vectors.
Definition: FlatIndex.cu:68
faiss::gpu::FlatIndex::add
void add(const float *data, int numVecs, cudaStream_t stream)
Definition: FlatIndex.cu:195
-
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
-
faiss::gpu::Tensor::copyTo
__host__ void copyTo(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies ourselves into a tensor; sizes must match.
Definition: Tensor-inl.cuh:140
+
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
+
faiss::gpu::Tensor::copyTo
__host__ void copyTo(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies ourselves into a tensor; sizes must match.
Definition: Tensor-inl.cuh:171
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
faiss::gpu::FlatIndex::getVectorsFloat32Ref
Tensor< float, 2, true > & getVectorsFloat32Ref()
Returns a reference to our vectors currently in use.
Definition: FlatIndex.cu:79
faiss::gpu::FlatIndex::reset
void reset()
Free all storage.
Definition: FlatIndex.cu:269
diff --git a/docs/html/Float16_8cu_source.html b/docs/html/Float16_8cu_source.html index 8c9a58719..a431fabc1 100644 --- a/docs/html/Float16_8cu_source.html +++ b/docs/html/Float16_8cu_source.html @@ -137,15 +137,21 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
48  in, in + num, out, HalfToFloat());
49 }
50 
-
51 half hostFloat2Half(float a) {
-
52  half h;
-
53  h.x = cpu_float2half_rn(a).x;
-
54  return h;
-
55 }
-
56 
-
57 } } // namespace
-
58 
-
59 #endif // FAISS_USE_FLOAT16
+
51 __half hostFloat2Half(float a) {
+
52 #if CUDA_VERSION >= 9000
+
53  __half_raw raw;
+
54  raw.x = cpu_float2half_rn(a).x;
+
55  return __half(raw);
+
56 #else
+
57  __half h;
+
58  h.x = cpu_float2half_rn(a).x;
+
59  return h;
+
60 #endif
+
61 }
+
62 
+
63 } } // namespace
+
64 
+
65 #endif // FAISS_USE_FLOAT16
diff --git a/docs/html/Float16_8cuh_source.html b/docs/html/Float16_8cuh_source.html index e532f79b2..7c41d4f33 100644 --- a/docs/html/Float16_8cuh_source.html +++ b/docs/html/Float16_8cuh_source.html @@ -238,7 +238,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
149  return out;
150 }
151 
-
152 half hostFloat2Half(float v);
+
152 __half hostFloat2Half(float v);
153 
154 #endif // FAISS_USE_FLOAT16
155 
diff --git a/docs/html/GpuAutoTune_8cpp_source.html b/docs/html/GpuAutoTune_8cpp_source.html index d29001b2e..d8e5c1ee5 100644 --- a/docs/html/GpuAutoTune_8cpp_source.html +++ b/docs/html/GpuAutoTune_8cpp_source.html @@ -470,20 +470,20 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
381  set_index_parameter (ix->at(i), name, val);
382  return;
383  }
-
384  if (DC (GpuIndexIVF)) {
-
385  if (name == "nprobe") {
+
384  if (name == "nprobe") {
+
385  if (DC (GpuIndexIVF)) {
386  ix->setNumProbes (int (val));
387  return;
388  }
389  }
-
390  if(DC (GpuIndexIVFPQ)) {
-
391  if (name == "use_precomputed_table") {
+
390  if (name == "use_precomputed_table") {
+
391  if (DC (GpuIndexIVFPQ)) {
392  ix->setPrecomputedCodes(bool (val));
393  return;
394  }
395  }
396 
-
397  // maybe norma lindex parameters apply?
+
397  // maybe normal index parameters apply?
398  ParameterSpace::set_index_parameter (index, name, val);
399 }
400 
@@ -491,15 +491,15 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
402 
403 
404 } } // namespace
- +
faiss::gpu::GpuClonerOptions::storeTransposed
bool storeTransposed
For GpuIndexFlat, store data in transposed layout?
Definition: GpuClonerOptions.h:38
-
faiss::IndexIVF::copy_subset_to
virtual void copy_subset_to(IndexIVF &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:272
-
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:47
+
faiss::IndexIVF::copy_subset_to
virtual void copy_subset_to(IndexIVF &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:375
+
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:78
faiss::gpu::GpuParameterSpace::initialize
void initialize(const faiss::Index *index) override
initialize with reasonable parameters for the index
Definition: GpuAutoTune.cpp:343
faiss::gpu::GpuClonerOptions
set some options on how to copy to GPU
Definition: GpuClonerOptions.h:17
@@ -507,12 +507,12 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::ProductQuantizer::dsub
size_t dsub
dimensionality of each subvector
Definition: ProductQuantizer.h:32
faiss::gpu::GpuClonerOptions::usePrecomputed
bool usePrecomputed
use precomputed tables?
Definition: GpuClonerOptions.h:32
- + - + -
faiss::ParameterSpace::add_range
ParameterRange & add_range(const char *name)
add a new parameter
Definition: AutoTune.cpp:322
+
faiss::ParameterSpace::add_range
ParameterRange & add_range(const char *name)
add a new parameter (or return it if it exists)
Definition: AutoTune.cpp:323
faiss::gpu::GpuIndexFlatConfig::useFloat16
bool useFloat16
Whether or not data is stored as float16.
Definition: GpuIndexFlat.h:35
faiss::gpu::GpuIndexConfig::device
int device
GPU device on which the index is resident.
Definition: GpuIndex.h:27
@@ -536,7 +536,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::IndexProxy::addIndex
void addIndex(faiss::Index *index)
Definition: IndexProxy.cpp:32
faiss::gpu::GpuIndexIVFPQ
IVFPQ index for the GPU.
Definition: GpuIndexIVFPQ.h:40
-
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:423
+
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:443
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
faiss::IndexIVFPQ::pq
ProductQuantizer pq
produces the codes
Definition: IndexIVFPQ.h:32
faiss::ProductQuantizer::M
size_t M
number of subquantizers
Definition: ProductQuantizer.h:28
diff --git a/docs/html/GpuIndexFlat_8cu_source.html b/docs/html/GpuIndexFlat_8cu_source.html index c75f9323a..be17e4495 100644 --- a/docs/html/GpuIndexFlat_8cu_source.html +++ b/docs/html/GpuIndexFlat_8cu_source.html @@ -709,7 +709,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::GpuIndexFlatL2::GpuIndexFlatL2
GpuIndexFlatL2(GpuResources *resources, faiss::IndexFlatL2 *index, GpuIndexFlatConfig config=GpuIndexFlatConfig())
Definition: GpuIndexFlat.cu:549
faiss::gpu::GpuResources::getDefaultStream
virtual cudaStream_t getDefaultStream(int device)=0
-
faiss::gpu::Tensor::narrowOutermost
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrowOutermost(IndexT start, IndexT size)
Definition: Tensor-inl.cuh:625
+
faiss::gpu::Tensor::narrowOutermost
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrowOutermost(IndexT start, IndexT size)
Definition: Tensor-inl.cuh:656
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
diff --git a/docs/html/GpuIndexIVFFlat_8cu_source.html b/docs/html/GpuIndexIVFFlat_8cu_source.html index 5e5d665ab..927669eed 100644 --- a/docs/html/GpuIndexIVFFlat_8cu_source.html +++ b/docs/html/GpuIndexIVFFlat_8cu_source.html @@ -375,32 +375,32 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); - +
faiss::gpu::GpuIndexIVFFlat::addImpl_
void addImpl_(faiss::Index::idx_t n, const float *x, const faiss::Index::idx_t *ids) override
Called from GpuIndex for add/add_with_ids.
Definition: GpuIndexIVFFlat.cu:207
faiss::gpu::GpuIndexIVFFlat::train
void train(Index::idx_t n, const float *x) override
Definition: GpuIndexIVFFlat.cu:177
-
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
+
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::gpu::IVFFlat::classifyAndAddVectors
int classifyAndAddVectors(Tensor< float, 2, true > &vecs, Tensor< long, 1, true > &indices)
Definition: IVFFlat.cu:130
+
faiss::gpu::IVFFlat::classifyAndAddVectors
int classifyAndAddVectors(Tensor< float, 2, true > &vecs, Tensor< long, 1, true > &indices)
Definition: IVFFlat.cu:129
faiss::gpu::GpuIndexIVFFlat::copyTo
void copyTo(faiss::IndexIVFFlat *index) const
Definition: GpuIndexIVFFlat.cu:127
faiss::gpu::GpuIndexIVF::nprobe_
int nprobe_
Number of inverted list probes per query.
Definition: GpuIndexIVF.h:91
faiss::gpu::GpuIndexIVFFlat::reserveMemory
void reserveMemory(size_t numVecs)
Reserve GPU memory in our inverted lists for this number of vectors.
Definition: GpuIndexIVFFlat.cu:71
faiss::gpu::GpuIndex::device_
const int device_
The GPU device we are resident on.
Definition: GpuIndex.h:94
faiss::gpu::GpuIndex::resources_
GpuResources * resources_
Manages streans, cuBLAS handles and scratch memory for devices.
Definition: GpuIndex.h:91
-
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:148
+
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:149
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::gpu::GpuIndexIVF::nlist_
int nlist_
Number of inverted lists that we manage.
Definition: GpuIndexIVF.h:88
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
-
faiss::gpu::IVFFlat::addCodeVectorsFromCpu
void addCodeVectorsFromCpu(int listId, const float *vecs, const long *indices, size_t numVecs)
Definition: IVFFlat.cu:58
+
faiss::gpu::IVFFlat::addCodeVectorsFromCpu
void addCodeVectorsFromCpu(int listId, const float *vecs, const long *indices, size_t numVecs)
Definition: IVFFlat.cu:54
faiss::gpu::GpuIndex::memorySpace_
const MemorySpace memorySpace_
The memory space of our primary storage on the GPU.
Definition: GpuIndex.h:97
faiss::gpu::GpuIndexIVF::quantizer_
GpuIndexFlat * quantizer_
Quantizer for inverted lists.
Definition: GpuIndexIVF.h:94
-
faiss::gpu::IVFFlat::query
void query(Tensor< float, 2, true > &queries, int nprobe, int k, Tensor< float, 2, true > &outDistances, Tensor< long, 2, true > &outIndices)
Definition: IVFFlat.cu:288
-
faiss::gpu::IVFFlat::getListVectors
std::vector< float > getListVectors(int listId) const
Return the vectors of a particular list back to the CPU.
Definition: IVFFlat.cu:354
+
faiss::gpu::IVFFlat::query
void query(Tensor< float, 2, true > &queries, int nprobe, int k, Tensor< float, 2, true > &outDistances, Tensor< long, 2, true > &outIndices)
Definition: IVFFlat.cu:287
+
faiss::gpu::IVFFlat::getListVectors
std::vector< float > getListVectors(int listId) const
Return the vectors of a particular list back to the CPU.
Definition: IVFFlat.cu:353
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
faiss::gpu::IVFBase::getListIndices
std::vector< long > getListIndices(int listId) const
Return the list indices of a particular list back to the CPU.
Definition: IVFBase.cu:207
-
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:80
+
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:81
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
faiss::gpu::GpuIndexIVFConfig::indicesOptions
IndicesOptions indicesOptions
Index storage options for the GPU.
Definition: GpuIndexIVF.h:31
diff --git a/docs/html/GpuIndexIVFFlat_8h_source.html b/docs/html/GpuIndexIVFFlat_8h_source.html index 0a0b1860d..6fcb63c9a 100644 --- a/docs/html/GpuIndexIVFFlat_8h_source.html +++ b/docs/html/GpuIndexIVFFlat_8h_source.html @@ -191,7 +191,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); - +
faiss::gpu::GpuIndexIVFFlat::addImpl_
void addImpl_(faiss::Index::idx_t n, const float *x, const faiss::Index::idx_t *ids) override
Called from GpuIndex for add/add_with_ids.
Definition: GpuIndexIVFFlat.cu:207
faiss::gpu::GpuIndexIVFFlat::train
void train(Index::idx_t n, const float *x) override
Definition: GpuIndexIVFFlat.cu:177
diff --git a/docs/html/GpuIndexIVFPQ_8cu_source.html b/docs/html/GpuIndexIVFPQ_8cu_source.html index 06d9dcbba..1f413a7b8 100644 --- a/docs/html/GpuIndexIVFPQ_8cu_source.html +++ b/docs/html/GpuIndexIVFPQ_8cu_source.html @@ -149,7 +149,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
60  reserveMemoryVecs_(0),
61  index_(nullptr) {
62 #ifndef FAISS_USE_FLOAT16
-
63  FAISS_ASSERT(!useFloat16LookupTables_);
+
63  FAISS_ASSERT(!config.useFloat16LookupTables);
64 #endif
65 
66  verifySettings_();
@@ -252,7 +252,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
163  index->polysemous_training = nullptr;
164 
165  index->scan_table_threshold = 0;
-
166  index->max_codes = 0;
+
166  index->max_codes = 0;
167  index->polysemous_ht = 0;
168  index->codes.clear();
169  index->codes.resize(nlist_);
@@ -569,7 +569,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
480 } } // namespace
faiss::gpu::GpuIndexIVFPQ::getListIndices
std::vector< long > getListIndices(int listId) const
Definition: GpuIndexIVFPQ.cu:407
faiss::gpu::GpuIndexIVFPQ::searchImpl_
void searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const override
Called from GpuIndex for search.
Definition: GpuIndexIVFPQ.cu:348
-
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:356
+
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:344
faiss::ProductQuantizer::nbits
size_t nbits
number of bits per quantization index
Definition: ProductQuantizer.h:29
faiss::gpu::GpuResources::getDefaultStreamCurrentDevice
cudaStream_t getDefaultStreamCurrentDevice()
Calls getDefaultStream with the current device.
Definition: GpuResources.cpp:25
@@ -580,22 +580,21 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::IVFBase::getDim
int getDim() const
Return the number of dimensions we are indexing.
Definition: IVFBase.cu:100
faiss::gpu::IVFBase::getListLength
int getListLength(int listId) const
Definition: IVFBase.cu:200
faiss::gpu::GpuIndexFlat::getGpuData
FlatIndex * getGpuData()
For internal access.
Definition: GpuIndexFlat.h:120
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
faiss::gpu::IVFBase::reserveMemory
void reserveMemory(size_t numVecs)
Reserve GPU memory in our inverted lists for this number of vectors.
Definition: IVFBase.cu:45
faiss::gpu::GpuIndexIVFPQ::getListLength
int getListLength(int listId) const
Definition: GpuIndexIVFPQ.cu:393
faiss::IndexIVFPQ::do_polysemous_training
bool do_polysemous_training
reorder PQ centroids after training?
Definition: IndexIVFPQ.h:34
faiss::IndexIVFPQ::scan_table_threshold
size_t scan_table_threshold
use table computation or on-the-fly?
Definition: IndexIVFPQ.h:38
-
faiss::IndexIVFPQ::precomputed_table
std::vector< float > precomputed_table
Definition: IndexIVFPQ.h:45
-
faiss::IndexIVFPQ::polysemous_ht
int polysemous_ht
Hamming thresh for polysemous filtering.
Definition: IndexIVFPQ.h:40
+
faiss::IndexIVFPQ::precomputed_table
std::vector< float > precomputed_table
Definition: IndexIVFPQ.h:44
+
faiss::IndexIVFPQ::polysemous_ht
int polysemous_ht
Hamming thresh for polysemous filtering.
Definition: IndexIVFPQ.h:39
faiss::gpu::GpuIndexIVFPQ::getBitsPerCode
int getBitsPerCode() const
Return the number of bits per PQ code.
Definition: GpuIndexIVFPQ.cu:224
faiss::gpu::GpuResources::getDefaultStream
virtual cudaStream_t getDefaultStream(int device)=0
-
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
+
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::gpu::GpuIndexIVFPQ::train
void train(Index::idx_t n, const float *x) override
Definition: GpuIndexIVFPQ.cu:300
-
faiss::IndexIVFPQ::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVFPQ.h:39
faiss::gpu::IVFPQ::isSupportedPQCodeLength
static bool isSupportedPQCodeLength(int size)
Returns true if we support PQ in this size.
Definition: IVFPQ.cu:72
@@ -608,7 +607,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::GpuIndexIVFPQ::copyFrom
void copyFrom(const faiss::IndexIVFPQ *index)
Definition: GpuIndexIVFPQ.cu:80
faiss::gpu::IVFPQ::getPQCentroids
Tensor< float, 3, true > getPQCentroids()
Definition: IVFPQ.cu:591
faiss::gpu::GpuIndex::resources_
GpuResources * resources_
Manages streans, cuBLAS handles and scratch memory for devices.
Definition: GpuIndex.h:91
-
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:148
+
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:149
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::gpu::GpuIndexIVF::nlist_
int nlist_
Number of inverted lists that we manage.
Definition: GpuIndexIVF.h:88
faiss::gpu::GpuIndexIVFPQ::addImpl_
void addImpl_(faiss::Index::idx_t n, const float *x, const faiss::Index::idx_t *ids) override
Called from GpuIndex for add/add_with_ids.
Definition: GpuIndexIVFPQ.cu:319
@@ -630,14 +629,15 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::GpuIndexIVFPQ::getCentroidsPerSubQuantizer
int getCentroidsPerSubQuantizer() const
Return the number of centroids per PQ code (2^bits per code)
Definition: GpuIndexIVFPQ.cu:229
faiss::gpu::GpuIndexIVFPQ::setPrecomputedCodes
void setPrecomputedCodes(bool enable)
Enable or disable pre-computed codes.
Definition: GpuIndexIVFPQ.cu:203
-
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:80
+
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:81
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:56
+
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:86
+
faiss::IndexIVF::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVF.h:79
faiss::gpu::GpuIndexIVFPQ::getPrecomputedCodes
bool getPrecomputedCodes() const
Are pre-computed codes enabled?
Definition: GpuIndexIVFPQ.cu:214
faiss::gpu::GpuIndexIVFConfig::indicesOptions
IndicesOptions indicesOptions
Index storage options for the GPU.
Definition: GpuIndexIVF.h:31
faiss::gpu::IVFBase::reclaimMemory
size_t reclaimMemory()
Definition: IVFBase.cu:105
faiss::gpu::IVFPQ
Implementing class for IVFPQ on the GPU.
Definition: IVFPQ.cuh:19
-
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:64
+
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:83
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
faiss::IndexIVFPQ::use_precomputed_table
int use_precomputed_table
if by_residual, build precompute tables
Definition: IndexIVFPQ.h:31
faiss::gpu::GpuIndexIVFPQ::getListCodes
std::vector< unsigned char > getListCodes(int listId) const
Definition: GpuIndexIVFPQ.cu:399
diff --git a/docs/html/GpuIndexIVF_8cu_source.html b/docs/html/GpuIndexIVF_8cu_source.html index bdb0d1247..46707c206 100644 --- a/docs/html/GpuIndexIVF_8cu_source.html +++ b/docs/html/GpuIndexIVF_8cu_source.html @@ -118,258 +118,257 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
29  nprobe_(1),
30  quantizer_(nullptr) {
31 #ifndef FAISS_USE_FLOAT16
-
32  FAISS_THROW_IF_NOT_MSG(!ivfConfig_.flatConfig.useFloat16CoarseQuantizer,
-
33  "float16 unsupported; need CUDA SDK >= 7.5");
-
34 #endif
-
35 
-
36  init_();
-
37 }
-
38 
-
39 void
-
40 GpuIndexIVF::init_() {
-
41  FAISS_ASSERT(nlist_ > 0);
-
42 
-
43  // Spherical by default if the metric is inner_product
-
44  if (this->metric_type == faiss::METRIC_INNER_PRODUCT) {
-
45  this->cp.spherical = true;
-
46  }
-
47 
-
48  // here we set a low # iterations because this is typically used
-
49  // for large clusterings
-
50  this->cp.niter = 10;
-
51  this->cp.verbose = this->verbose;
-
52 
-
53  if (!quantizer_) {
-
54  // Construct an empty quantizer
-
55  GpuIndexFlatConfig config = ivfConfig_.flatConfig;
-
56  // FIXME: inherit our same device
-
57  config.device = device_;
-
58 
-
59  if (this->metric_type == faiss::METRIC_L2) {
-
60  quantizer_ = new GpuIndexFlatL2(resources_, this->d, config);
-
61  } else if (this->metric_type == faiss::METRIC_INNER_PRODUCT) {
-
62  quantizer_ = new GpuIndexFlatIP(resources_, this->d, config);
-
63  } else {
-
64  // unknown metric type
-
65  FAISS_ASSERT_MSG(false, "unknown metric type");
-
66  }
-
67  }
-
68 }
-
69 
-
70 GpuIndexIVF::~GpuIndexIVF() {
-
71  delete quantizer_;
-
72 }
-
73 
-
74 GpuIndexFlat*
-
75 GpuIndexIVF::getQuantizer() {
-
76  return quantizer_;
-
77 }
-
78 
-
79 void
-
80 GpuIndexIVF::copyFrom(const faiss::IndexIVF* index) {
-
81  DeviceScope scope(device_);
-
82 
-
83  this->d = index->d;
-
84  this->metric_type = index->metric_type;
-
85 
-
86  FAISS_ASSERT(index->nlist > 0);
-
87  FAISS_THROW_IF_NOT_FMT(index->nlist <=
-
88  (faiss::Index::idx_t) std::numeric_limits<int>::max(),
-
89  "GPU index only supports %zu inverted lists",
-
90  (size_t) std::numeric_limits<int>::max());
-
91  nlist_ = index->nlist;
-
92  nprobe_ = index->nprobe;
-
93 
-
94  // The metric type may have changed as well, so we might have to
-
95  // change our quantizer
-
96  delete quantizer_;
-
97  quantizer_ = nullptr;
-
98 
-
99  // Construct an empty quantizer
-
100  GpuIndexFlatConfig config = ivfConfig_.flatConfig;
-
101  // FIXME: inherit our same device
-
102  config.device = device_;
-
103 
-
104  if (index->metric_type == faiss::METRIC_L2) {
-
105  // FIXME: 2 different float16 options?
-
106  quantizer_ = new GpuIndexFlatL2(resources_, this->d, config);
-
107  } else if (index->metric_type == faiss::METRIC_INNER_PRODUCT) {
-
108  // FIXME: 2 different float16 options?
-
109  quantizer_ = new GpuIndexFlatIP(resources_, this->d, config);
-
110  } else {
-
111  // unknown metric type
-
112  FAISS_ASSERT(false);
-
113  }
-
114 
-
115  if (!index->is_trained) {
-
116  this->is_trained = false;
-
117  this->ntotal = 0;
-
118  return;
-
119  }
-
120 
-
121  // Otherwise, we can populate ourselves from the other index
-
122  this->is_trained = true;
-
123 
-
124  // ntotal can exceed max int, but the number of vectors per inverted
-
125  // list cannot exceed this. We check this in the subclasses.
-
126  this->ntotal = index->ntotal;
-
127 
-
128  // Since we're trained, the quantizer must have data
-
129  FAISS_ASSERT(index->quantizer->ntotal > 0);
-
130 
-
131  if (index->metric_type == faiss::METRIC_L2) {
-
132  auto q = dynamic_cast<faiss::IndexFlatL2*>(index->quantizer);
-
133  FAISS_ASSERT(q);
-
134 
-
135  quantizer_->copyFrom(q);
-
136  } else if (index->metric_type == faiss::METRIC_INNER_PRODUCT) {
-
137  auto q = dynamic_cast<faiss::IndexFlatIP*>(index->quantizer);
-
138  FAISS_ASSERT(q);
-
139 
-
140  quantizer_->copyFrom(q);
-
141  } else {
-
142  // unknown metric type
-
143  FAISS_ASSERT(false);
-
144  }
-
145 }
-
146 
-
147 void
-
148 GpuIndexIVF::copyTo(faiss::IndexIVF* index) const {
-
149  DeviceScope scope(device_);
-
150 
-
151  //
-
152  // Index information
-
153  //
-
154  index->ntotal = this->ntotal;
-
155  index->d = this->d;
-
156  index->metric_type = this->metric_type;
-
157  index->is_trained = this->is_trained;
-
158 
-
159  //
-
160  // IndexIVF information
-
161  //
-
162  index->nlist = nlist_;
-
163  index->nprobe = nprobe_;
-
164 
-
165  // Construct and copy the appropriate quantizer
-
166  faiss::IndexFlat* q = nullptr;
-
167 
-
168  if (this->metric_type == faiss::METRIC_L2) {
-
169  q = new faiss::IndexFlatL2(this->d);
-
170 
-
171  } else if (this->metric_type == faiss::METRIC_INNER_PRODUCT) {
-
172  q = new faiss::IndexFlatIP(this->d);
-
173 
-
174  } else {
-
175  // unknown metric type
-
176  FAISS_ASSERT(false);
-
177  }
-
178 
-
179  FAISS_ASSERT(quantizer_);
-
180  quantizer_->copyTo(q);
-
181 
-
182  if (index->own_fields) {
-
183  delete index->quantizer;
-
184  }
-
185 
-
186  index->quantizer = q;
-
187  index->quantizer_trains_alone = 0;
-
188  index->own_fields = true;
-
189  index->cp = this->cp;
-
190  index->ids.clear();
-
191  index->ids.resize(nlist_);
-
192  index->maintain_direct_map = false;
-
193  index->direct_map.clear();
-
194 }
-
195 
-
196 int
-
197 GpuIndexIVF::getNumLists() const {
-
198  return nlist_;
-
199 }
-
200 
-
201 void
-
202 GpuIndexIVF::setNumProbes(int nprobe) {
-
203  FAISS_THROW_IF_NOT_FMT(nprobe > 0 && nprobe <= 1024,
-
204  "nprobe must be from 1 to 1024; passed %d",
-
205  nprobe);
-
206  nprobe_ = nprobe;
-
207 }
-
208 
-
209 int
-
210 GpuIndexIVF::getNumProbes() const {
-
211  return nprobe_;
-
212 }
-
213 
-
214 void
-
215 GpuIndexIVF::add(Index::idx_t n, const float* x) {
-
216  // FIXME: GPU-ize
-
217  std::vector<Index::idx_t> ids(n);
-
218  for (Index::idx_t i = 0; i < n; ++i) {
-
219  ids[i] = this->ntotal + i;
-
220  }
-
221 
-
222  add_with_ids(n, x, ids.data());
-
223 }
-
224 
-
225 void
-
226 GpuIndexIVF::trainQuantizer_(faiss::Index::idx_t n, const float* x) {
-
227  if (n == 0) {
-
228  // nothing to do
-
229  return;
-
230  }
-
231 
-
232  if (quantizer_->is_trained && (quantizer_->ntotal == nlist_)) {
-
233  if (this->verbose) {
-
234  printf ("IVF quantizer does not need training.\n");
-
235  }
-
236 
-
237  return;
-
238  }
-
239 
-
240  if (this->verbose) {
-
241  printf ("Training IVF quantizer on %ld vectors in %dD\n", n, d);
-
242  }
-
243 
-
244  DeviceScope scope(device_);
-
245 
-
246  // leverage the CPU-side k-means code, which works for the GPU
-
247  // flat index as well
-
248  quantizer_->reset();
-
249  Clustering clus(this->d, nlist_, this->cp);
-
250  clus.verbose = verbose;
-
251  clus.train(n, x, *quantizer_);
-
252  quantizer_->is_trained = true;
-
253 
-
254  FAISS_ASSERT(quantizer_->ntotal == nlist_);
-
255 }
-
256 
-
257 } } // namespace
- -
faiss::gpu::GpuIndexIVF::getNumProbes
int getNumProbes() const
Returns our current number of list probes per query.
Definition: GpuIndexIVF.cu:210
+
32  FAISS_THROW_IF_NOT_MSG(!ivfConfig_.flatConfig.useFloat16 &&
+
33  !ivfConfig_.flatConfig.useFloat16Accumulator,
+
34  "float16 unsupported; need CUDA SDK >= 7.5");
+
35 #endif
+
36 
+
37  init_();
+
38 }
+
39 
+
40 void
+
41 GpuIndexIVF::init_() {
+
42  FAISS_ASSERT(nlist_ > 0);
+
43 
+
44  // Spherical by default if the metric is inner_product
+
45  if (this->metric_type == faiss::METRIC_INNER_PRODUCT) {
+
46  this->cp.spherical = true;
+
47  }
+
48 
+
49  // here we set a low # iterations because this is typically used
+
50  // for large clusterings
+
51  this->cp.niter = 10;
+
52  this->cp.verbose = this->verbose;
+
53 
+
54  if (!quantizer_) {
+
55  // Construct an empty quantizer
+
56  GpuIndexFlatConfig config = ivfConfig_.flatConfig;
+
57  // FIXME: inherit our same device
+
58  config.device = device_;
+
59 
+
60  if (this->metric_type == faiss::METRIC_L2) {
+
61  quantizer_ = new GpuIndexFlatL2(resources_, this->d, config);
+
62  } else if (this->metric_type == faiss::METRIC_INNER_PRODUCT) {
+
63  quantizer_ = new GpuIndexFlatIP(resources_, this->d, config);
+
64  } else {
+
65  // unknown metric type
+
66  FAISS_ASSERT_MSG(false, "unknown metric type");
+
67  }
+
68  }
+
69 }
+
70 
+
71 GpuIndexIVF::~GpuIndexIVF() {
+
72  delete quantizer_;
+
73 }
+
74 
+
75 GpuIndexFlat*
+
76 GpuIndexIVF::getQuantizer() {
+
77  return quantizer_;
+
78 }
+
79 
+
80 void
+
81 GpuIndexIVF::copyFrom(const faiss::IndexIVF* index) {
+
82  DeviceScope scope(device_);
+
83 
+
84  this->d = index->d;
+
85  this->metric_type = index->metric_type;
+
86 
+
87  FAISS_ASSERT(index->nlist > 0);
+
88  FAISS_THROW_IF_NOT_FMT(index->nlist <=
+
89  (faiss::Index::idx_t) std::numeric_limits<int>::max(),
+
90  "GPU index only supports %zu inverted lists",
+
91  (size_t) std::numeric_limits<int>::max());
+
92  nlist_ = index->nlist;
+
93  nprobe_ = index->nprobe;
+
94 
+
95  // The metric type may have changed as well, so we might have to
+
96  // change our quantizer
+
97  delete quantizer_;
+
98  quantizer_ = nullptr;
+
99 
+
100  // Construct an empty quantizer
+
101  GpuIndexFlatConfig config = ivfConfig_.flatConfig;
+
102  // FIXME: inherit our same device
+
103  config.device = device_;
+
104 
+
105  if (index->metric_type == faiss::METRIC_L2) {
+
106  // FIXME: 2 different float16 options?
+
107  quantizer_ = new GpuIndexFlatL2(resources_, this->d, config);
+
108  } else if (index->metric_type == faiss::METRIC_INNER_PRODUCT) {
+
109  // FIXME: 2 different float16 options?
+
110  quantizer_ = new GpuIndexFlatIP(resources_, this->d, config);
+
111  } else {
+
112  // unknown metric type
+
113  FAISS_ASSERT(false);
+
114  }
+
115 
+
116  if (!index->is_trained) {
+
117  this->is_trained = false;
+
118  this->ntotal = 0;
+
119  return;
+
120  }
+
121 
+
122  // Otherwise, we can populate ourselves from the other index
+
123  this->is_trained = true;
+
124 
+
125  // ntotal can exceed max int, but the number of vectors per inverted
+
126  // list cannot exceed this. We check this in the subclasses.
+
127  this->ntotal = index->ntotal;
+
128 
+
129  // Since we're trained, the quantizer must have data
+
130  FAISS_ASSERT(index->quantizer->ntotal > 0);
+
131 
+
132  if (index->metric_type == faiss::METRIC_L2) {
+
133  auto q = dynamic_cast<faiss::IndexFlatL2*>(index->quantizer);
+
134  FAISS_ASSERT(q);
+
135 
+
136  quantizer_->copyFrom(q);
+
137  } else if (index->metric_type == faiss::METRIC_INNER_PRODUCT) {
+
138  auto q = dynamic_cast<faiss::IndexFlatIP*>(index->quantizer);
+
139  FAISS_ASSERT(q);
+
140 
+
141  quantizer_->copyFrom(q);
+
142  } else {
+
143  // unknown metric type
+
144  FAISS_ASSERT(false);
+
145  }
+
146 }
+
147 
+
148 void
+
149 GpuIndexIVF::copyTo(faiss::IndexIVF* index) const {
+
150  DeviceScope scope(device_);
+
151 
+
152  //
+
153  // Index information
+
154  //
+
155  index->ntotal = this->ntotal;
+
156  index->d = this->d;
+
157  index->metric_type = this->metric_type;
+
158  index->is_trained = this->is_trained;
+
159 
+
160  //
+
161  // IndexIVF information
+
162  //
+
163  index->nlist = nlist_;
+
164  index->nprobe = nprobe_;
+
165 
+
166  // Construct and copy the appropriate quantizer
+
167  faiss::IndexFlat* q = nullptr;
+
168 
+
169  if (this->metric_type == faiss::METRIC_L2) {
+
170  q = new faiss::IndexFlatL2(this->d);
+
171 
+
172  } else if (this->metric_type == faiss::METRIC_INNER_PRODUCT) {
+
173  q = new faiss::IndexFlatIP(this->d);
+
174 
+
175  } else {
+
176  // unknown metric type
+
177  FAISS_ASSERT(false);
+
178  }
+
179 
+
180  FAISS_ASSERT(quantizer_);
+
181  quantizer_->copyTo(q);
+
182 
+
183  if (index->own_fields) {
+
184  delete index->quantizer;
+
185  }
+
186 
+
187  index->quantizer = q;
+
188  index->quantizer_trains_alone = 0;
+
189  index->own_fields = true;
+
190  index->cp = this->cp;
+
191  index->ids.clear();
+
192  index->ids.resize(nlist_);
+
193  index->maintain_direct_map = false;
+
194  index->direct_map.clear();
+
195 }
+
196 
+
197 int
+
198 GpuIndexIVF::getNumLists() const {
+
199  return nlist_;
+
200 }
+
201 
+
202 void
+
203 GpuIndexIVF::setNumProbes(int nprobe) {
+
204  FAISS_THROW_IF_NOT_FMT(nprobe > 0 && nprobe <= 1024,
+
205  "nprobe must be from 1 to 1024; passed %d",
+
206  nprobe);
+
207  nprobe_ = nprobe;
+
208 }
+
209 
+
210 int
+
211 GpuIndexIVF::getNumProbes() const {
+
212  return nprobe_;
+
213 }
+
214 
+
215 void
+
216 GpuIndexIVF::add(Index::idx_t n, const float* x) {
+
217  // FIXME: GPU-ize
+
218  std::vector<Index::idx_t> ids(n);
+
219  for (Index::idx_t i = 0; i < n; ++i) {
+
220  ids[i] = this->ntotal + i;
+
221  }
+
222 
+
223  add_with_ids(n, x, ids.data());
+
224 }
+
225 
+
226 void
+
227 GpuIndexIVF::trainQuantizer_(faiss::Index::idx_t n, const float* x) {
+
228  if (n == 0) {
+
229  // nothing to do
+
230  return;
+
231  }
+
232 
+
233  if (quantizer_->is_trained && (quantizer_->ntotal == nlist_)) {
+
234  if (this->verbose) {
+
235  printf ("IVF quantizer does not need training.\n");
+
236  }
+
237 
+
238  return;
+
239  }
+
240 
+
241  if (this->verbose) {
+
242  printf ("Training IVF quantizer on %ld vectors in %dD\n", n, d);
+
243  }
+
244 
+
245  DeviceScope scope(device_);
+
246 
+
247  // leverage the CPU-side k-means code, which works for the GPU
+
248  // flat index as well
+
249  quantizer_->reset();
+
250  Clustering clus(this->d, nlist_, this->cp);
+
251  clus.verbose = verbose;
+
252  clus.train(n, x, *quantizer_);
+
253  quantizer_->is_trained = true;
+
254 
+
255  FAISS_ASSERT(quantizer_->ntotal == nlist_);
+
256 }
+
257 
+
258 } } // namespace
+ +
faiss::gpu::GpuIndexIVF::getNumProbes
int getNumProbes() const
Returns our current number of list probes per query.
Definition: GpuIndexIVF.cu:211
-
faiss::gpu::GpuIndexIVF::setNumProbes
void setNumProbes(int nprobe)
Sets the number of list probes per query.
Definition: GpuIndexIVF.cu:202
+
faiss::gpu::GpuIndexIVF::setNumProbes
void setNumProbes(int nprobe)
Sets the number of list probes per query.
Definition: GpuIndexIVF.cu:203
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:25
-
faiss::gpu::GpuIndexIVF::getNumLists
int getNumLists() const
Returns the number of inverted lists we&#39;re managing.
Definition: GpuIndexIVF.cu:197
+
faiss::gpu::GpuIndexIVF::getNumLists
int getNumLists() const
Returns the number of inverted lists we&#39;re managing.
Definition: GpuIndexIVF.cu:198
faiss::gpu::GpuIndexFlat::copyTo
void copyTo(faiss::IndexFlat *index) const
Definition: GpuIndexFlat.cu:120
-
faiss::IndexIVF::quantizer_trains_alone
char quantizer_trains_alone
Definition: IndexIVF.h:56
-
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:47
+
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:78
faiss::gpu::GpuIndexConfig::device
int device
GPU device on which the index is resident.
Definition: GpuIndex.h:27
-
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
+
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
faiss::gpu::GpuIndex::add_with_ids
void add_with_ids(Index::idx_t n, const float *x, const Index::idx_t *ids) override
Definition: GpuIndex.cu:66
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
faiss::gpu::GpuIndexIVFConfig::flatConfig
GpuIndexFlatConfig flatConfig
Configuration for the coarse quantizer object.
Definition: GpuIndexIVF.h:34
-
faiss::gpu::GpuIndexIVF::getQuantizer
GpuIndexFlat * getQuantizer()
Return the quantizer we&#39;re using.
Definition: GpuIndexIVF.cu:75
-
faiss::IndexIVF::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:59
+
faiss::gpu::GpuIndexIVF::getQuantizer
GpuIndexFlat * getQuantizer()
Return the quantizer we&#39;re using.
Definition: GpuIndexIVF.cu:76
faiss::gpu::GpuIndexIVF::nprobe_
int nprobe_
Number of inverted list probes per query.
Definition: GpuIndexIVF.h:91
-
faiss::IndexIVF::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:57
+
faiss::gpu::GpuIndex::device_
const int device_
The GPU device we are resident on.
Definition: GpuIndex.h:94
faiss::gpu::GpuIndex::resources_
GpuResources * resources_
Manages streans, cuBLAS handles and scratch memory for devices.
Definition: GpuIndex.h:91
-
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:148
+
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:149
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
+
faiss::Level1Quantizer::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:44
faiss::gpu::GpuIndexIVF::nlist_
int nlist_
Number of inverted lists that we manage.
Definition: GpuIndexIVF.h:88
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
@@ -379,14 +378,16 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::GpuIndexIVF::quantizer_
GpuIndexFlat * quantizer_
Quantizer for inverted lists.
Definition: GpuIndexIVF.h:94
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
-
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
faiss::gpu::GpuIndexFlat::reset
void reset() override
Clears all vectors from this index.
Definition: GpuIndexFlat.cu:148
-
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:80
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
+
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:81
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::gpu::GpuIndexIVF::add
void add(Index::idx_t n, const float *x) override
Definition: GpuIndexIVF.cu:215
-
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:68
+
faiss::gpu::GpuIndexIVF::add
void add(Index::idx_t n, const float *x) override
Definition: GpuIndexIVF.cu:216
+
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:87
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:29
+
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:42
faiss::gpu::GpuIndexIVF::cp
ClusteringParameters cp
Definition: GpuIndexIVF.h:82
+
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
diff --git a/docs/html/GpuIndexIVF_8h_source.html b/docs/html/GpuIndexIVF_8h_source.html index f6bde8b20..63126442d 100644 --- a/docs/html/GpuIndexIVF_8h_source.html +++ b/docs/html/GpuIndexIVF_8h_source.html @@ -184,27 +184,27 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
95 };
96 
97 } } // namespace
- -
faiss::gpu::GpuIndexIVF::getNumProbes
int getNumProbes() const
Returns our current number of list probes per query.
Definition: GpuIndexIVF.cu:210
-
faiss::gpu::GpuIndexIVF::setNumProbes
void setNumProbes(int nprobe)
Sets the number of list probes per query.
Definition: GpuIndexIVF.cu:202
-
faiss::gpu::GpuIndexIVF::getNumLists
int getNumLists() const
Returns the number of inverted lists we&#39;re managing.
Definition: GpuIndexIVF.cu:197
+ +
faiss::gpu::GpuIndexIVF::getNumProbes
int getNumProbes() const
Returns our current number of list probes per query.
Definition: GpuIndexIVF.cu:211
+
faiss::gpu::GpuIndexIVF::setNumProbes
void setNumProbes(int nprobe)
Sets the number of list probes per query.
Definition: GpuIndexIVF.cu:203
+
faiss::gpu::GpuIndexIVF::getNumLists
int getNumLists() const
Returns the number of inverted lists we&#39;re managing.
Definition: GpuIndexIVF.cu:198
faiss::gpu::GpuIndexIVFConfig::flatConfig
GpuIndexFlatConfig flatConfig
Configuration for the coarse quantizer object.
Definition: GpuIndexIVF.h:34
-
faiss::gpu::GpuIndexIVF::getQuantizer
GpuIndexFlat * getQuantizer()
Return the quantizer we&#39;re using.
Definition: GpuIndexIVF.cu:75
+
faiss::gpu::GpuIndexIVF::getQuantizer
GpuIndexFlat * getQuantizer()
Return the quantizer we&#39;re using.
Definition: GpuIndexIVF.cu:76
faiss::gpu::GpuIndexIVF::nprobe_
int nprobe_
Number of inverted list probes per query.
Definition: GpuIndexIVF.h:91
-
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:148
+
faiss::gpu::GpuIndexIVF::copyTo
void copyTo(faiss::IndexIVF *index) const
Copy what we have to the CPU equivalent.
Definition: GpuIndexIVF.cu:149
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::gpu::GpuIndexIVF::nlist_
int nlist_
Number of inverted lists that we manage.
Definition: GpuIndexIVF.h:88
faiss::gpu::GpuIndexIVF::quantizer_
GpuIndexFlat * quantizer_
Quantizer for inverted lists.
Definition: GpuIndexIVF.h:94
-
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:80
-
faiss::gpu::GpuIndexIVF::add
void add(Index::idx_t n, const float *x) override
Definition: GpuIndexIVF.cu:215
+
faiss::gpu::GpuIndexIVF::copyFrom
void copyFrom(const faiss::IndexIVF *index)
Copy what we need from the CPU equivalent.
Definition: GpuIndexIVF.cu:81
+
faiss::gpu::GpuIndexIVF::add
void add(Index::idx_t n, const float *x) override
Definition: GpuIndexIVF.cu:216
faiss::gpu::GpuIndexIVF::cp
ClusteringParameters cp
Definition: GpuIndexIVF.h:82
faiss::gpu::GpuIndexIVFConfig::indicesOptions
IndicesOptions indicesOptions
Index storage options for the GPU.
Definition: GpuIndexIVF.h:31
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
diff --git a/docs/html/Heap_8h_source.html b/docs/html/Heap_8h_source.html index 40d4c9774..7dfb07846 100644 --- a/docs/html/Heap_8h_source.html +++ b/docs/html/Heap_8h_source.html @@ -264,7 +264,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
179  * Heap initialization
180  *******************************************************************/
181 
-
182 /* Initialization phase for the heap (with inconditionnal pushes).
+
182 /* Initialization phase for the heap (with unconditionnal pushes).
183  * Store k0 elements in a heap containing up to k values. Note that
184  * (bh_val, bh_ids) can be the same as (x, ids) */
185 template <class C> inline
diff --git a/docs/html/HostTensor-inl_8cuh_source.html b/docs/html/HostTensor-inl_8cuh_source.html index 2bf0fb404..297e672e6 100644 --- a/docs/html/HostTensor-inl_8cuh_source.html +++ b/docs/html/HostTensor-inl_8cuh_source.html @@ -239,14 +239,14 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
150 }
151 
152 } } // namespace
-
faiss::gpu::Tensor::numElements
__host__ __device__ size_t numElements() const
Definition: Tensor-inl.cuh:352
+
faiss::gpu::Tensor::numElements
__host__ __device__ size_t numElements() const
Definition: Tensor-inl.cuh:383
faiss::gpu::HostTensor::~HostTensor
__host__ ~HostTensor()
Destructor.
Definition: HostTensor-inl.cuh:24
faiss::gpu::HostTensor::zero
__host__ HostTensor< T, Dim, InnerContig, IndexT, PtrTraits > & zero()
Call to zero out memory.
Definition: HostTensor-inl.cuh:105
-
faiss::gpu::Tensor::data_
DataPtrType data_
Raw pointer to where the tensor data begins.
Definition: Tensor.cuh:346
-
faiss::gpu::Tensor::isContiguous
__host__ __device__ bool isContiguous() const
Definition: Tensor-inl.cuh:365
-
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:101
+
faiss::gpu::Tensor::data_
DataPtrType data_
Raw pointer to where the tensor data begins.
Definition: Tensor.cuh:344
+
faiss::gpu::Tensor::isContiguous
__host__ __device__ bool isContiguous() const
Definition: Tensor-inl.cuh:396
+
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:132
faiss::gpu::HostTensor::HostTensor
__host__ HostTensor()
Default constructor.
Definition: HostTensor-inl.cuh:16
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
faiss::gpu::HostTensor::maxDiff
__host__ T maxDiff(const HostTensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) const
Returns the maximum difference seen between two tensors.
Definition: HostTensor-inl.cuh:119
diff --git a/docs/html/HostTensor_8cuh_source.html b/docs/html/HostTensor_8cuh_source.html index 4d05ba9fe..acd7dd12f 100644 --- a/docs/html/HostTensor_8cuh_source.html +++ b/docs/html/HostTensor_8cuh_source.html @@ -175,10 +175,10 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
86 #include "HostTensor-inl.cuh"
faiss::gpu::HostTensor::~HostTensor
__host__ ~HostTensor()
Destructor.
Definition: HostTensor-inl.cuh:24
faiss::gpu::HostTensor::zero
__host__ HostTensor< T, Dim, InnerContig, IndexT, PtrTraits > & zero()
Call to zero out memory.
Definition: HostTensor-inl.cuh:105
-
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:247
+
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:245
faiss::gpu::HostTensor::HostTensor
__host__ HostTensor()
Default constructor.
Definition: HostTensor-inl.cuh:16
-
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:252
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:250
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
faiss::gpu::HostTensor::equal
__host__ bool equal(const HostTensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) const
Are the two tensors exactly equal?
Definition: HostTensor.cuh:66
diff --git a/docs/html/IVFFlatScan_8cu_source.html b/docs/html/IVFFlatScan_8cu_source.html index ea7499443..e783f0be0 100644 --- a/docs/html/IVFFlatScan_8cu_source.html +++ b/docs/html/IVFFlatScan_8cu_source.html @@ -994,7 +994,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
905 }
906 
907 } } // namespace
- +
faiss::gpu::Math::reduceAdd
static __device__ T reduceAdd(T v)
For a vector type, this is a horizontal add, returning sum(v_i)
Definition: MathOperators.cuh:44
faiss::gpu::IVFFlatScan
The class that we use to provide scan specializations.
Definition: IVFFlatScan.cu:46
diff --git a/docs/html/IVFFlat_8cu_source.html b/docs/html/IVFFlat_8cu_source.html index 1a41308de..519da3af5 100644 --- a/docs/html/IVFFlat_8cu_source.html +++ b/docs/html/IVFFlat_8cu_source.html @@ -134,370 +134,369 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
45  space),
46  l2Distance_(l2Distance),
47  useFloat16_(useFloat16) {
-
48 #ifndef FAISS_USE_FLOAT16
-
49  FAISS_ASSERT_MSG(!useFloat16, "float16 unsupported");
-
50  useFloat16_ = false;
-
51 #endif
-
52 }
-
53 
-
54 IVFFlat::~IVFFlat() {
-
55 }
-
56 
-
57 void
-
58 IVFFlat::addCodeVectorsFromCpu(int listId,
-
59  const float* vecs,
-
60  const long* indices,
-
61  size_t numVecs) {
-
62  // This list must already exist
-
63  FAISS_ASSERT(listId < deviceListData_.size());
-
64  auto stream = resources_->getDefaultStreamCurrentDevice();
-
65 
-
66  // If there's nothing to add, then there's nothing we have to do
-
67  if (numVecs == 0) {
-
68  return;
-
69  }
-
70 
-
71  size_t lengthInBytes = numVecs * bytesPerVector_;
-
72 
-
73  auto& listData = deviceListData_[listId];
-
74  auto prevData = listData->data();
-
75 
-
76  // We only have int32 length representations on the GPU per each
-
77  // list; the length is in sizeof(char)
-
78  FAISS_ASSERT(listData->size() + lengthInBytes <=
-
79  (size_t) std::numeric_limits<int>::max());
-
80 
-
81  if (useFloat16_) {
-
82 #ifdef FAISS_USE_FLOAT16
-
83  // We have to convert data to the half format.
-
84  // Make sure the source data is on our device first; it is not
-
85  // guaranteed before function entry to avoid unnecessary h2d copies
-
86  auto floatData =
-
87  toDevice<float, 1>(resources_,
-
88  getCurrentDevice(),
-
89  (float*) vecs,
-
90  stream,
-
91  {(int) numVecs * dim_});
-
92  auto halfData = toHalf<1>(resources_, stream, floatData);
-
93 
-
94  listData->append((unsigned char*) halfData.data(),
-
95  lengthInBytes,
-
96  stream,
-
97  true /* exact reserved size */);
-
98 #endif
-
99  } else {
-
100  listData->append((unsigned char*) vecs,
-
101  lengthInBytes,
-
102  stream,
-
103  true /* exact reserved size */);
-
104  }
-
105 
-
106  // Handle the indices as well
-
107  addIndicesFromCpu_(listId, indices, numVecs);
-
108 
-
109  // This list address may have changed due to vector resizing, but
-
110  // only bother updating it on the device if it has changed
-
111  if (prevData != listData->data()) {
-
112  deviceListDataPointers_[listId] = listData->data();
-
113  }
-
114 
-
115  // And our size has changed too
-
116  int listLength = listData->size() / bytesPerVector_;
-
117  deviceListLengths_[listId] = listLength;
-
118 
-
119  // We update this as well, since the multi-pass algorithm uses it
-
120  maxListLength_ = std::max(maxListLength_, listLength);
-
121 
-
122  // device_vector add is potentially happening on a different stream
-
123  // than our default stream
-
124  if (stream != 0) {
-
125  streamWait({stream}, {0});
-
126  }
-
127 }
-
128 
-
129 int
-
130 IVFFlat::classifyAndAddVectors(Tensor<float, 2, true>& vecs,
-
131  Tensor<long, 1, true>& indices) {
-
132  FAISS_ASSERT(vecs.getSize(0) == indices.getSize(0));
-
133  FAISS_ASSERT(vecs.getSize(1) == dim_);
-
134 
-
135  auto& mem = resources_->getMemoryManagerCurrentDevice();
-
136  auto stream = resources_->getDefaultStreamCurrentDevice();
-
137 
-
138  // Number of valid vectors that we actually add; we return this
-
139  int numAdded = 0;
-
140 
-
141  // We don't actually need this
-
142  DeviceTensor<float, 2, true> listDistance(mem, {vecs.getSize(0), 1}, stream);
-
143  // We use this
-
144  DeviceTensor<int, 2, true> listIds2d(mem, {vecs.getSize(0), 1}, stream);
-
145  auto listIds = listIds2d.view<1>({vecs.getSize(0)});
-
146 
-
147  quantizer_->query(vecs, 1, listDistance, listIds2d, false);
-
148 
-
149  // Copy the lists that we wish to append to back to the CPU
-
150  // FIXME: really this can be into pinned memory and a true async
-
151  // copy on a different stream; we can start the copy early, but it's
-
152  // tiny
-
153  HostTensor<int, 1, true> listIdsHost(listIds, stream);
-
154 
-
155  // Now we add the encoded vectors to the individual lists
-
156  // First, make sure that there is space available for adding the new
-
157  // encoded vectors and indices
-
158 
-
159  // list id -> # being added
-
160  std::unordered_map<int, int> assignCounts;
-
161 
-
162  // vector id -> offset in list
-
163  // (we already have vector id -> list id in listIds)
-
164  HostTensor<int, 1, true> listOffsetHost({listIdsHost.getSize(0)});
-
165 
-
166  for (int i = 0; i < listIds.getSize(0); ++i) {
-
167  int listId = listIdsHost[i];
-
168 
-
169  // Add vector could be invalid (contains NaNs etc)
-
170  if (listId < 0) {
-
171  listOffsetHost[i] = -1;
-
172  continue;
-
173  }
-
174 
-
175  FAISS_ASSERT(listId < numLists_);
-
176  ++numAdded;
-
177 
-
178  int offset = deviceListData_[listId]->size() / bytesPerVector_;
-
179 
-
180  auto it = assignCounts.find(listId);
-
181  if (it != assignCounts.end()) {
-
182  offset += it->second;
-
183  it->second++;
-
184  } else {
-
185  assignCounts[listId] = 1;
-
186  }
-
187 
-
188  listOffsetHost[i] = offset;
-
189  }
-
190 
-
191  // If we didn't add anything (all invalid vectors), no need to
-
192  // continue
-
193  if (numAdded == 0) {
-
194  return 0;
-
195  }
-
196 
-
197  // We need to resize the data structures for the inverted lists on
-
198  // the GPUs, which means that they might need reallocation, which
-
199  // means that their base address may change. Figure out the new base
-
200  // addresses, and update those in a batch on the device
-
201  {
-
202  for (auto& counts : assignCounts) {
-
203  auto& data = deviceListData_[counts.first];
-
204  data->resize(data->size() + counts.second * bytesPerVector_,
-
205  stream);
-
206  int newNumVecs = (int) (data->size() / bytesPerVector_);
-
207 
-
208  auto& indices = deviceListIndices_[counts.first];
-
209  if ((indicesOptions_ == INDICES_32_BIT) ||
-
210  (indicesOptions_ == INDICES_64_BIT)) {
-
211  size_t indexSize =
-
212  (indicesOptions_ == INDICES_32_BIT) ? sizeof(int) : sizeof(long);
-
213 
-
214  indices->resize(indices->size() + counts.second * indexSize, stream);
-
215  } else if (indicesOptions_ == INDICES_CPU) {
-
216  // indices are stored on the CPU side
-
217  FAISS_ASSERT(counts.first < listOffsetToUserIndex_.size());
-
218 
-
219  auto& userIndices = listOffsetToUserIndex_[counts.first];
-
220  userIndices.resize(newNumVecs);
-
221  } else {
-
222  // indices are not stored on the GPU or CPU side
-
223  FAISS_ASSERT(indicesOptions_ == INDICES_IVF);
-
224  }
-
225 
-
226  // This is used by the multi-pass query to decide how much scratch
-
227  // space to allocate for intermediate results
-
228  maxListLength_ = std::max(maxListLength_, newNumVecs);
-
229  }
-
230 
-
231  // Update all pointers to the lists on the device that may have
-
232  // changed
-
233  {
-
234  std::vector<int> listIds(assignCounts.size());
-
235  int i = 0;
-
236  for (auto& counts : assignCounts) {
-
237  listIds[i++] = counts.first;
-
238  }
-
239 
-
240  updateDeviceListInfo_(listIds, stream);
-
241  }
-
242  }
-
243 
-
244  // If we're maintaining the indices on the CPU side, update our
-
245  // map. We already resized our map above.
-
246  if (indicesOptions_ == INDICES_CPU) {
-
247  // We need to maintain the indices on the CPU side
-
248  HostTensor<long, 1, true> hostIndices(indices, stream);
-
249 
-
250  for (int i = 0; i < hostIndices.getSize(0); ++i) {
-
251  int listId = listIdsHost[i];
-
252 
-
253  // Add vector could be invalid (contains NaNs etc)
-
254  if (listId < 0) {
-
255  continue;
-
256  }
-
257 
-
258  int offset = listOffsetHost[i];
-
259 
-
260  FAISS_ASSERT(listId < listOffsetToUserIndex_.size());
-
261  auto& userIndices = listOffsetToUserIndex_[listId];
-
262 
-
263  FAISS_ASSERT(offset < userIndices.size());
-
264  userIndices[offset] = hostIndices[i];
-
265  }
-
266  }
-
267 
-
268  // We similarly need to actually append the new vectors
-
269  {
-
270  DeviceTensor<int, 1, true> listOffset(mem, listOffsetHost, stream);
-
271 
-
272  // Now, for each list to which a vector is being assigned, write it
-
273  runIVFFlatInvertedListAppend(listIds,
-
274  listOffset,
-
275  vecs,
-
276  indices,
-
277  useFloat16_,
-
278  deviceListDataPointers_,
-
279  deviceListIndexPointers_,
-
280  indicesOptions_,
-
281  stream);
-
282  }
-
283 
-
284  return numAdded;
-
285 }
-
286 
-
287 void
-
288 IVFFlat::query(Tensor<float, 2, true>& queries,
-
289  int nprobe,
-
290  int k,
-
291  Tensor<float, 2, true>& outDistances,
-
292  Tensor<long, 2, true>& outIndices) {
-
293  auto& mem = resources_->getMemoryManagerCurrentDevice();
-
294  auto stream = resources_->getDefaultStreamCurrentDevice();
-
295 
-
296  // Validate these at a top level
-
297  FAISS_ASSERT(nprobe <= 1024);
-
298  FAISS_ASSERT(k <= 1024);
-
299  nprobe = std::min(nprobe, quantizer_->getSize());
-
300 
-
301  FAISS_ASSERT(queries.getSize(1) == dim_);
-
302 
-
303  FAISS_ASSERT(outDistances.getSize(0) == queries.getSize(0));
-
304  FAISS_ASSERT(outIndices.getSize(0) == queries.getSize(0));
-
305 
-
306  // Reserve space for the quantized information
-
307  DeviceTensor<float, 2, true>
-
308  coarseDistances(mem, {queries.getSize(0), nprobe}, stream);
-
309  DeviceTensor<int, 2, true>
-
310  coarseIndices(mem, {queries.getSize(0), nprobe}, stream);
-
311 
-
312  // Find the `nprobe` closest lists; we can use int indices both
-
313  // internally and externally
-
314  quantizer_->query(queries,
-
315  nprobe,
-
316  coarseDistances,
-
317  coarseIndices,
-
318  false);
-
319 
-
320  runIVFFlatScan(queries,
-
321  coarseIndices,
-
322  deviceListDataPointers_,
-
323  deviceListIndexPointers_,
-
324  indicesOptions_,
-
325  deviceListLengths_,
-
326  maxListLength_,
-
327  k,
-
328  l2Distance_,
-
329  useFloat16_,
-
330  outDistances,
-
331  outIndices,
-
332  resources_);
-
333 
-
334  // If the GPU isn't storing indices (they are on the CPU side), we
-
335  // need to perform the re-mapping here
-
336  // FIXME: we might ultimately be calling this function with inputs
-
337  // from the CPU, these are unnecessary copies
-
338  if (indicesOptions_ == INDICES_CPU) {
-
339  HostTensor<long, 2, true> hostOutIndices(outIndices, stream);
-
340 
-
341  ivfOffsetToUserIndex(hostOutIndices.data(),
-
342  numLists_,
-
343  hostOutIndices.getSize(0),
-
344  hostOutIndices.getSize(1),
-
345  listOffsetToUserIndex_);
-
346 
-
347  // Copy back to GPU, since the input to this function is on the
-
348  // GPU
-
349  outIndices.copyFrom(hostOutIndices, stream);
-
350  }
-
351 }
-
352 
-
353 std::vector<float>
-
354 IVFFlat::getListVectors(int listId) const {
-
355  FAISS_ASSERT(listId < deviceListData_.size());
-
356  auto& encVecs = *deviceListData_[listId];
-
357 
-
358  auto stream = resources_->getDefaultStreamCurrentDevice();
-
359 
-
360  if (useFloat16_) {
-
361 #ifdef FAISS_USE_FLOAT16
-
362  size_t num = encVecs.size() / sizeof(half);
-
363 
-
364  Tensor<half, 1, true> devHalf((half*) encVecs.data(), {(int) num});
-
365  auto devFloat = fromHalf(resources_, stream, devHalf);
-
366 
-
367  std::vector<float> out(num);
-
368  HostTensor<float, 1, true> hostFloat(out.data(), {(int) num});
-
369  hostFloat.copyFrom(devFloat, stream);
-
370 
-
371  return out;
-
372 #endif
-
373  }
-
374 
-
375  size_t num = encVecs.size() / sizeof(float);
-
376 
-
377  Tensor<float, 1, true> devFloat((float*) encVecs.data(), {(int) num});
-
378 
-
379  std::vector<float> out(num);
-
380  HostTensor<float, 1, true> hostFloat(out.data(), {(int) num});
-
381  hostFloat.copyFrom(devFloat, stream);
-
382 
-
383  return out;
-
384 }
-
385 
-
386 } } // namespace
+
48 }
+
49 
+
50 IVFFlat::~IVFFlat() {
+
51 }
+
52 
+
53 void
+
54 IVFFlat::addCodeVectorsFromCpu(int listId,
+
55  const float* vecs,
+
56  const long* indices,
+
57  size_t numVecs) {
+
58  // This list must already exist
+
59  FAISS_ASSERT(listId < deviceListData_.size());
+
60  auto stream = resources_->getDefaultStreamCurrentDevice();
+
61 
+
62  // If there's nothing to add, then there's nothing we have to do
+
63  if (numVecs == 0) {
+
64  return;
+
65  }
+
66 
+
67  size_t lengthInBytes = numVecs * bytesPerVector_;
+
68 
+
69  auto& listData = deviceListData_[listId];
+
70  auto prevData = listData->data();
+
71 
+
72  // We only have int32 length representations on the GPU per each
+
73  // list; the length is in sizeof(char)
+
74  FAISS_ASSERT(listData->size() + lengthInBytes <=
+
75  (size_t) std::numeric_limits<int>::max());
+
76 
+
77  if (useFloat16_) {
+
78 #ifdef FAISS_USE_FLOAT16
+
79  // We have to convert data to the half format.
+
80  // Make sure the source data is on our device first; it is not
+
81  // guaranteed before function entry to avoid unnecessary h2d copies
+
82  auto floatData =
+
83  toDevice<float, 1>(resources_,
+
84  getCurrentDevice(),
+
85  (float*) vecs,
+
86  stream,
+
87  {(int) numVecs * dim_});
+
88  auto halfData = toHalf<1>(resources_, stream, floatData);
+
89 
+
90  listData->append((unsigned char*) halfData.data(),
+
91  lengthInBytes,
+
92  stream,
+
93  true /* exact reserved size */);
+
94 #else
+
95  // we are not compiling with float16 support
+
96  FAISS_ASSERT(false);
+
97 #endif
+
98  } else {
+
99  listData->append((unsigned char*) vecs,
+
100  lengthInBytes,
+
101  stream,
+
102  true /* exact reserved size */);
+
103  }
+
104 
+
105  // Handle the indices as well
+
106  addIndicesFromCpu_(listId, indices, numVecs);
+
107 
+
108  // This list address may have changed due to vector resizing, but
+
109  // only bother updating it on the device if it has changed
+
110  if (prevData != listData->data()) {
+
111  deviceListDataPointers_[listId] = listData->data();
+
112  }
+
113 
+
114  // And our size has changed too
+
115  int listLength = listData->size() / bytesPerVector_;
+
116  deviceListLengths_[listId] = listLength;
+
117 
+
118  // We update this as well, since the multi-pass algorithm uses it
+
119  maxListLength_ = std::max(maxListLength_, listLength);
+
120 
+
121  // device_vector add is potentially happening on a different stream
+
122  // than our default stream
+
123  if (stream != 0) {
+
124  streamWait({stream}, {0});
+
125  }
+
126 }
+
127 
+
128 int
+
129 IVFFlat::classifyAndAddVectors(Tensor<float, 2, true>& vecs,
+
130  Tensor<long, 1, true>& indices) {
+
131  FAISS_ASSERT(vecs.getSize(0) == indices.getSize(0));
+
132  FAISS_ASSERT(vecs.getSize(1) == dim_);
+
133 
+
134  auto& mem = resources_->getMemoryManagerCurrentDevice();
+
135  auto stream = resources_->getDefaultStreamCurrentDevice();
+
136 
+
137  // Number of valid vectors that we actually add; we return this
+
138  int numAdded = 0;
+
139 
+
140  // We don't actually need this
+
141  DeviceTensor<float, 2, true> listDistance(mem, {vecs.getSize(0), 1}, stream);
+
142  // We use this
+
143  DeviceTensor<int, 2, true> listIds2d(mem, {vecs.getSize(0), 1}, stream);
+
144  auto listIds = listIds2d.view<1>({vecs.getSize(0)});
+
145 
+
146  quantizer_->query(vecs, 1, listDistance, listIds2d, false);
+
147 
+
148  // Copy the lists that we wish to append to back to the CPU
+
149  // FIXME: really this can be into pinned memory and a true async
+
150  // copy on a different stream; we can start the copy early, but it's
+
151  // tiny
+
152  HostTensor<int, 1, true> listIdsHost(listIds, stream);
+
153 
+
154  // Now we add the encoded vectors to the individual lists
+
155  // First, make sure that there is space available for adding the new
+
156  // encoded vectors and indices
+
157 
+
158  // list id -> # being added
+
159  std::unordered_map<int, int> assignCounts;
+
160 
+
161  // vector id -> offset in list
+
162  // (we already have vector id -> list id in listIds)
+
163  HostTensor<int, 1, true> listOffsetHost({listIdsHost.getSize(0)});
+
164 
+
165  for (int i = 0; i < listIds.getSize(0); ++i) {
+
166  int listId = listIdsHost[i];
+
167 
+
168  // Add vector could be invalid (contains NaNs etc)
+
169  if (listId < 0) {
+
170  listOffsetHost[i] = -1;
+
171  continue;
+
172  }
+
173 
+
174  FAISS_ASSERT(listId < numLists_);
+
175  ++numAdded;
+
176 
+
177  int offset = deviceListData_[listId]->size() / bytesPerVector_;
+
178 
+
179  auto it = assignCounts.find(listId);
+
180  if (it != assignCounts.end()) {
+
181  offset += it->second;
+
182  it->second++;
+
183  } else {
+
184  assignCounts[listId] = 1;
+
185  }
+
186 
+
187  listOffsetHost[i] = offset;
+
188  }
+
189 
+
190  // If we didn't add anything (all invalid vectors), no need to
+
191  // continue
+
192  if (numAdded == 0) {
+
193  return 0;
+
194  }
+
195 
+
196  // We need to resize the data structures for the inverted lists on
+
197  // the GPUs, which means that they might need reallocation, which
+
198  // means that their base address may change. Figure out the new base
+
199  // addresses, and update those in a batch on the device
+
200  {
+
201  for (auto& counts : assignCounts) {
+
202  auto& data = deviceListData_[counts.first];
+
203  data->resize(data->size() + counts.second * bytesPerVector_,
+
204  stream);
+
205  int newNumVecs = (int) (data->size() / bytesPerVector_);
+
206 
+
207  auto& indices = deviceListIndices_[counts.first];
+
208  if ((indicesOptions_ == INDICES_32_BIT) ||
+
209  (indicesOptions_ == INDICES_64_BIT)) {
+
210  size_t indexSize =
+
211  (indicesOptions_ == INDICES_32_BIT) ? sizeof(int) : sizeof(long);
+
212 
+
213  indices->resize(indices->size() + counts.second * indexSize, stream);
+
214  } else if (indicesOptions_ == INDICES_CPU) {
+
215  // indices are stored on the CPU side
+
216  FAISS_ASSERT(counts.first < listOffsetToUserIndex_.size());
+
217 
+
218  auto& userIndices = listOffsetToUserIndex_[counts.first];
+
219  userIndices.resize(newNumVecs);
+
220  } else {
+
221  // indices are not stored on the GPU or CPU side
+
222  FAISS_ASSERT(indicesOptions_ == INDICES_IVF);
+
223  }
+
224 
+
225  // This is used by the multi-pass query to decide how much scratch
+
226  // space to allocate for intermediate results
+
227  maxListLength_ = std::max(maxListLength_, newNumVecs);
+
228  }
+
229 
+
230  // Update all pointers to the lists on the device that may have
+
231  // changed
+
232  {
+
233  std::vector<int> listIds(assignCounts.size());
+
234  int i = 0;
+
235  for (auto& counts : assignCounts) {
+
236  listIds[i++] = counts.first;
+
237  }
+
238 
+
239  updateDeviceListInfo_(listIds, stream);
+
240  }
+
241  }
+
242 
+
243  // If we're maintaining the indices on the CPU side, update our
+
244  // map. We already resized our map above.
+
245  if (indicesOptions_ == INDICES_CPU) {
+
246  // We need to maintain the indices on the CPU side
+
247  HostTensor<long, 1, true> hostIndices(indices, stream);
+
248 
+
249  for (int i = 0; i < hostIndices.getSize(0); ++i) {
+
250  int listId = listIdsHost[i];
+
251 
+
252  // Add vector could be invalid (contains NaNs etc)
+
253  if (listId < 0) {
+
254  continue;
+
255  }
+
256 
+
257  int offset = listOffsetHost[i];
+
258 
+
259  FAISS_ASSERT(listId < listOffsetToUserIndex_.size());
+
260  auto& userIndices = listOffsetToUserIndex_[listId];
+
261 
+
262  FAISS_ASSERT(offset < userIndices.size());
+
263  userIndices[offset] = hostIndices[i];
+
264  }
+
265  }
+
266 
+
267  // We similarly need to actually append the new vectors
+
268  {
+
269  DeviceTensor<int, 1, true> listOffset(mem, listOffsetHost, stream);
+
270 
+
271  // Now, for each list to which a vector is being assigned, write it
+
272  runIVFFlatInvertedListAppend(listIds,
+
273  listOffset,
+
274  vecs,
+
275  indices,
+
276  useFloat16_,
+
277  deviceListDataPointers_,
+
278  deviceListIndexPointers_,
+
279  indicesOptions_,
+
280  stream);
+
281  }
+
282 
+
283  return numAdded;
+
284 }
+
285 
+
286 void
+
287 IVFFlat::query(Tensor<float, 2, true>& queries,
+
288  int nprobe,
+
289  int k,
+
290  Tensor<float, 2, true>& outDistances,
+
291  Tensor<long, 2, true>& outIndices) {
+
292  auto& mem = resources_->getMemoryManagerCurrentDevice();
+
293  auto stream = resources_->getDefaultStreamCurrentDevice();
+
294 
+
295  // Validate these at a top level
+
296  FAISS_ASSERT(nprobe <= 1024);
+
297  FAISS_ASSERT(k <= 1024);
+
298  nprobe = std::min(nprobe, quantizer_->getSize());
+
299 
+
300  FAISS_ASSERT(queries.getSize(1) == dim_);
+
301 
+
302  FAISS_ASSERT(outDistances.getSize(0) == queries.getSize(0));
+
303  FAISS_ASSERT(outIndices.getSize(0) == queries.getSize(0));
+
304 
+
305  // Reserve space for the quantized information
+
306  DeviceTensor<float, 2, true>
+
307  coarseDistances(mem, {queries.getSize(0), nprobe}, stream);
+
308  DeviceTensor<int, 2, true>
+
309  coarseIndices(mem, {queries.getSize(0), nprobe}, stream);
+
310 
+
311  // Find the `nprobe` closest lists; we can use int indices both
+
312  // internally and externally
+
313  quantizer_->query(queries,
+
314  nprobe,
+
315  coarseDistances,
+
316  coarseIndices,
+
317  false);
+
318 
+
319  runIVFFlatScan(queries,
+
320  coarseIndices,
+
321  deviceListDataPointers_,
+
322  deviceListIndexPointers_,
+
323  indicesOptions_,
+
324  deviceListLengths_,
+
325  maxListLength_,
+
326  k,
+
327  l2Distance_,
+
328  useFloat16_,
+
329  outDistances,
+
330  outIndices,
+
331  resources_);
+
332 
+
333  // If the GPU isn't storing indices (they are on the CPU side), we
+
334  // need to perform the re-mapping here
+
335  // FIXME: we might ultimately be calling this function with inputs
+
336  // from the CPU, these are unnecessary copies
+
337  if (indicesOptions_ == INDICES_CPU) {
+
338  HostTensor<long, 2, true> hostOutIndices(outIndices, stream);
+
339 
+
340  ivfOffsetToUserIndex(hostOutIndices.data(),
+
341  numLists_,
+
342  hostOutIndices.getSize(0),
+
343  hostOutIndices.getSize(1),
+
344  listOffsetToUserIndex_);
+
345 
+
346  // Copy back to GPU, since the input to this function is on the
+
347  // GPU
+
348  outIndices.copyFrom(hostOutIndices, stream);
+
349  }
+
350 }
+
351 
+
352 std::vector<float>
+
353 IVFFlat::getListVectors(int listId) const {
+
354  FAISS_ASSERT(listId < deviceListData_.size());
+
355  auto& encVecs = *deviceListData_[listId];
+
356 
+
357  auto stream = resources_->getDefaultStreamCurrentDevice();
+
358 
+
359  if (useFloat16_) {
+
360 #ifdef FAISS_USE_FLOAT16
+
361  size_t num = encVecs.size() / sizeof(half);
+
362 
+
363  Tensor<half, 1, true> devHalf((half*) encVecs.data(), {(int) num});
+
364  auto devFloat = fromHalf(resources_, stream, devHalf);
+
365 
+
366  std::vector<float> out(num);
+
367  HostTensor<float, 1, true> hostFloat(out.data(), {(int) num});
+
368  hostFloat.copyFrom(devFloat, stream);
+
369 
+
370  return out;
+
371 #endif
+
372  }
+
373 
+
374  size_t num = encVecs.size() / sizeof(float);
+
375 
+
376  Tensor<float, 1, true> devFloat((float*) encVecs.data(), {(int) num});
+
377 
+
378  std::vector<float> out(num);
+
379  HostTensor<float, 1, true> hostFloat(out.data(), {(int) num});
+
380  hostFloat.copyFrom(devFloat, stream);
+
381 
+
382  return out;
+
383 }
+
384 
+
385 } } // namespace
faiss::gpu::IVFBase::numLists_
const int numLists_
Number of inverted lists we maintain.
Definition: IVFBase.cuh:91
faiss::gpu::IVFBase::maxListLength_
int maxListLength_
Maximum list length seen.
Definition: IVFBase.cuh:115
faiss::gpu::GpuResources::getDefaultStreamCurrentDevice
cudaStream_t getDefaultStreamCurrentDevice()
Calls getDefaultStream with the current device.
Definition: GpuResources.cpp:25
faiss::gpu::FlatIndex::getSize
int getSize() const
Returns the number of vectors we contain.
Definition: FlatIndex.cu:47
faiss::gpu::IVFBase::listOffsetToUserIndex_
std::vector< std::vector< long > > listOffsetToUserIndex_
Definition: IVFBase.cuh:127
faiss::gpu::FlatIndex
Holder of GPU resources for a particular flat index.
Definition: FlatIndex.cuh:23
-
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view(DataPtrType at)
Definition: Tensor-inl.cuh:598
+
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view(DataPtrType at)
Definition: Tensor-inl.cuh:629
faiss::gpu::IVFBase
Base inverted list functionality for IVFFlat and IVFPQ.
Definition: IVFBase.cuh:27
faiss::gpu::IVFFlat::IVFFlat
IVFFlat(GpuResources *resources, FlatIndex *quantizer, bool l2Distance, bool useFloat16, IndicesOptions indicesOptions, MemorySpace space)
Construct from a quantizer that has elemen.
Definition: IVFFlat.cu:29
faiss::gpu::IVFBase::deviceListLengths_
thrust::device_vector< int > deviceListLengths_
Definition: IVFBase.cuh:112
faiss::gpu::IVFBase::deviceListIndexPointers_
thrust::device_vector< void * > deviceListIndexPointers_
Definition: IVFBase.cuh:108
-
faiss::gpu::IVFFlat::classifyAndAddVectors
int classifyAndAddVectors(Tensor< float, 2, true > &vecs, Tensor< long, 1, true > &indices)
Definition: IVFFlat.cu:130
+
faiss::gpu::IVFFlat::classifyAndAddVectors
int classifyAndAddVectors(Tensor< float, 2, true > &vecs, Tensor< long, 1, true > &indices)
Definition: IVFFlat.cu:129
faiss::gpu::GpuResources::getMemoryManagerCurrentDevice
DeviceMemory & getMemoryManagerCurrentDevice()
Calls getMemoryManager for the current device.
Definition: GpuResources.cpp:35
-
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:101
+
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:132
faiss::gpu::IVFBase::quantizer_
FlatIndex * quantizer_
Quantizer object.
Definition: IVFBase.cuh:85
-
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
+
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
faiss::gpu::IVFBase::deviceListDataPointers_
thrust::device_vector< void * > deviceListDataPointers_
Definition: IVFBase.cuh:104
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::IVFBase::resources_
GpuResources * resources_
Collection of GPU resources that we use.
Definition: IVFBase.cuh:82
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
-
faiss::gpu::IVFFlat::addCodeVectorsFromCpu
void addCodeVectorsFromCpu(int listId, const float *vecs, const long *indices, size_t numVecs)
Definition: IVFFlat.cu:58
+
faiss::gpu::IVFFlat::addCodeVectorsFromCpu
void addCodeVectorsFromCpu(int listId, const float *vecs, const long *indices, size_t numVecs)
Definition: IVFFlat.cu:54
faiss::gpu::IVFBase::bytesPerVector_
const int bytesPerVector_
Number of bytes per vector in the list.
Definition: IVFBase.cuh:94
-
faiss::gpu::IVFFlat::query
void query(Tensor< float, 2, true > &queries, int nprobe, int k, Tensor< float, 2, true > &outDistances, Tensor< long, 2, true > &outIndices)
Definition: IVFFlat.cu:288
-
faiss::gpu::IVFFlat::getListVectors
std::vector< float > getListVectors(int listId) const
Return the vectors of a particular list back to the CPU.
Definition: IVFFlat.cu:354
+
faiss::gpu::IVFFlat::query
void query(Tensor< float, 2, true > &queries, int nprobe, int k, Tensor< float, 2, true > &outDistances, Tensor< long, 2, true > &outIndices)
Definition: IVFFlat.cu:287
+
faiss::gpu::IVFFlat::getListVectors
std::vector< float > getListVectors(int listId) const
Return the vectors of a particular list back to the CPU.
Definition: IVFFlat.cu:353
faiss::gpu::IVFBase::updateDeviceListInfo_
void updateDeviceListInfo_(cudaStream_t stream)
Update all device-side list pointer and size information.
Definition: IVFBase.cu:138
faiss::gpu::IVFBase::indicesOptions_
const IndicesOptions indicesOptions_
How are user indices stored on the GPU?
Definition: IVFBase.cuh:97
diff --git a/docs/html/IVFFlat_8cuh_source.html b/docs/html/IVFFlat_8cuh_source.html index 967039c87..6c20d6f28 100644 --- a/docs/html/IVFFlat_8cuh_source.html +++ b/docs/html/IVFFlat_8cuh_source.html @@ -159,11 +159,11 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::IVFBase
Base inverted list functionality for IVFFlat and IVFPQ.
Definition: IVFBase.cuh:27
faiss::gpu::IVFFlat::IVFFlat
IVFFlat(GpuResources *resources, FlatIndex *quantizer, bool l2Distance, bool useFloat16, IndicesOptions indicesOptions, MemorySpace space)
Construct from a quantizer that has elemen.
Definition: IVFFlat.cu:29
-
faiss::gpu::IVFFlat::classifyAndAddVectors
int classifyAndAddVectors(Tensor< float, 2, true > &vecs, Tensor< long, 1, true > &indices)
Definition: IVFFlat.cu:130
+
faiss::gpu::IVFFlat::classifyAndAddVectors
int classifyAndAddVectors(Tensor< float, 2, true > &vecs, Tensor< long, 1, true > &indices)
Definition: IVFFlat.cu:129
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
-
faiss::gpu::IVFFlat::addCodeVectorsFromCpu
void addCodeVectorsFromCpu(int listId, const float *vecs, const long *indices, size_t numVecs)
Definition: IVFFlat.cu:58
-
faiss::gpu::IVFFlat::query
void query(Tensor< float, 2, true > &queries, int nprobe, int k, Tensor< float, 2, true > &outDistances, Tensor< long, 2, true > &outIndices)
Definition: IVFFlat.cu:288
-
faiss::gpu::IVFFlat::getListVectors
std::vector< float > getListVectors(int listId) const
Return the vectors of a particular list back to the CPU.
Definition: IVFFlat.cu:354
+
faiss::gpu::IVFFlat::addCodeVectorsFromCpu
void addCodeVectorsFromCpu(int listId, const float *vecs, const long *indices, size_t numVecs)
Definition: IVFFlat.cu:54
+
faiss::gpu::IVFFlat::query
void query(Tensor< float, 2, true > &queries, int nprobe, int k, Tensor< float, 2, true > &outDistances, Tensor< long, 2, true > &outIndices)
Definition: IVFFlat.cu:287
+
faiss::gpu::IVFFlat::getListVectors
std::vector< float > getListVectors(int listId) const
Return the vectors of a particular list back to the CPU.
Definition: IVFFlat.cu:353
diff --git a/docs/html/IVFPQ_8cu_source.html b/docs/html/IVFPQ_8cu_source.html index ba32e34dd..8f142183f 100644 --- a/docs/html/IVFPQ_8cu_source.html +++ b/docs/html/IVFPQ_8cu_source.html @@ -802,7 +802,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::FlatIndex::getSize
int getSize() const
Returns the number of vectors we contain.
Definition: FlatIndex.cu:47
faiss::gpu::IVFBase::listOffsetToUserIndex_
std::vector< std::vector< long > > listOffsetToUserIndex_
Definition: IVFBase.cuh:127
faiss::gpu::FlatIndex
Holder of GPU resources for a particular flat index.
Definition: FlatIndex.cuh:23
-
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view(DataPtrType at)
Definition: Tensor-inl.cuh:598
+
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view(DataPtrType at)
Definition: Tensor-inl.cuh:629
faiss::gpu::IVFBase
Base inverted list functionality for IVFFlat and IVFPQ.
Definition: IVFBase.cuh:27
faiss::gpu::IVFBase::deviceListLengths_
thrust::device_vector< int > deviceListLengths_
Definition: IVFBase.cuh:112
@@ -811,15 +811,15 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::GpuResources::getBlasHandleCurrentDevice
cublasHandle_t getBlasHandleCurrentDevice()
Calls getBlasHandle with the current device.
Definition: GpuResources.cpp:20
faiss::gpu::GpuResources::getMemoryManagerCurrentDevice
DeviceMemory & getMemoryManagerCurrentDevice()
Calls getMemoryManager for the current device.
Definition: GpuResources.cpp:35
faiss::gpu::IVFPQ::classifyAndAddVectors
int classifyAndAddVectors(Tensor< float, 2, true > &vecs, Tensor< long, 1, true > &indices)
Definition: IVFPQ.cu:120
-
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:101
+
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:132
faiss::gpu::IVFPQ::query
void query(Tensor< float, 2, true > &queries, int nprobe, int k, Tensor< float, 2, true > &outDistances, Tensor< long, 2, true > &outIndices)
Definition: IVFPQ.cu:516
faiss::gpu::IVFPQ::getPQCentroids
Tensor< float, 3, true > getPQCentroids()
Definition: IVFPQ.cu:591
faiss::gpu::IVFBase::quantizer_
FlatIndex * quantizer_
Quantizer object.
Definition: IVFBase.cuh:85
faiss::gpu::IVFPQ::setPrecomputedCodes
void setPrecomputedCodes(bool enable)
Enable or disable pre-computed codes.
Definition: IVFPQ.cu:102
faiss::gpu::IVFPQ::getListCodes
std::vector< unsigned char > getListCodes(int listId) const
Return the list codes of a particular list back to the CPU.
Definition: IVFPQ.cu:583
-
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
+
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
faiss::gpu::IVFBase::deviceListDataPointers_
thrust::device_vector< void * > deviceListDataPointers_
Definition: IVFBase.cuh:104
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::IVFBase::resources_
GpuResources * resources_
Collection of GPU resources that we use.
Definition: IVFBase.cuh:82
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
faiss::gpu::IVFBase::bytesPerVector_
const int bytesPerVector_
Number of bytes per vector in the list.
Definition: IVFBase.cuh:94
diff --git a/docs/html/IndexFlat_8cpp_source.html b/docs/html/IndexFlat_8cpp_source.html index 39294b4c3..c26736b98 100644 --- a/docs/html/IndexFlat_8cpp_source.html +++ b/docs/html/IndexFlat_8cpp_source.html @@ -255,7 +255,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
170 
171 void IndexRefineFlat::train (idx_t n, const float *x)
172 {
-
173  base_index->train (n, x);
+
173  base_index->train (n, x);
174  is_trained = true;
175 }
176 
@@ -483,18 +483,18 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
398 
399 
400 } // namespace faiss
-
faiss::knn_L2sqr_base_shift
void knn_L2sqr_base_shift(const float *x, const float *y, size_t d, size_t nx, size_t ny, float_maxheap_array_t *res, const float *base_shift)
Definition: utils.cpp:986
+
faiss::knn_L2sqr_base_shift
void knn_L2sqr_base_shift(const float *x, const float *y, size_t d, size_t nx, size_t ny, float_maxheap_array_t *res, const float *base_shift)
Definition: utils.cpp:988
faiss::IndexFlatL2BaseShift::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexFlat.cpp:133
faiss::IndexRefineFlat::reset
void reset() override
removes all elements from the database.
Definition: IndexFlat.cpp:184
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
faiss::IndexFlat1D::continuous_update
bool continuous_update
is the permutation updated continuously?
Definition: IndexFlat.h:139
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
faiss::IndexFlat::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexFlat.cpp:42
faiss::IndexFlat::reset
void reset() override
removes all elements from the database.
Definition: IndexFlat.cpp:36
faiss::IndexFlat1D::update_permutation
void update_permutation()
Definition: IndexFlat.cpp:284
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
faiss::IndexFlat::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexFlat.cpp:118
faiss::IndexFlat1D::add
void add(idx_t n, const float *x) override
Definition: IndexFlat.cpp:294
faiss::IndexFlat::remove_ids
long remove_ids(const IDSelector &sel) override
Definition: IndexFlat.cpp:95
@@ -509,18 +509,18 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
-
faiss::range_search_inner_product
void range_search_inner_product(const float *x, const float *y, size_t d, size_t nx, size_t ny, float radius, RangeSearchResult *res)
same as range_search_L2sqr for the inner product similarity
Definition: utils.cpp:1280
+
faiss::range_search_inner_product
void range_search_inner_product(const float *x, const float *y, size_t d, size_t nx, size_t ny, float radius, RangeSearchResult *res)
same as range_search_L2sqr for the inner product similarity
Definition: utils.cpp:1282
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
-
faiss::knn_inner_product
void knn_inner_product(const float *x, const float *y, size_t d, size_t nx, size_t ny, float_minheap_array_t *res)
Definition: utils.cpp:946
+
faiss::knn_inner_product
void knn_inner_product(const float *x, const float *y, size_t d, size_t nx, size_t ny, float_minheap_array_t *res)
Definition: utils.cpp:948
faiss::IndexFlat::add
void add(idx_t n, const float *x) override
Definition: IndexFlat.cpp:30
faiss::IndexFlat1D::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Warn: the distances returned are L1 not L2.
Definition: IndexFlat.cpp:307
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
-
faiss::range_search_L2sqr
void range_search_L2sqr(const float *x, const float *y, size_t d, size_t nx, size_t ny, float radius, RangeSearchResult *res)
Definition: utils.cpp:1265
+
faiss::range_search_L2sqr
void range_search_L2sqr(const float *x, const float *y, size_t d, size_t nx, size_t ny, float radius, RangeSearchResult *res)
Definition: utils.cpp:1267
faiss::IndexFlat::compute_distance_subset
void compute_distance_subset(idx_t n, const float *x, idx_t k, float *distances, const idx_t *labels) const
Definition: IndexFlat.cpp:73
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
-
faiss::knn_L2sqr
void knn_L2sqr(const float *x, const float *y, size_t d, size_t nx, size_t ny, float_maxheap_array_t *res)
Definition: utils.cpp:966
+
faiss::knn_L2sqr
void knn_L2sqr(const float *x, const float *y, size_t d, size_t nx, size_t ny, float_maxheap_array_t *res)
Definition: utils.cpp:968
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
faiss::IndexFlat::xb
std::vector< float > xb
database vectors, size ntotal * d
Definition: IndexFlat.h:25
diff --git a/docs/html/IndexHNSW_8cpp_source.html b/docs/html/IndexHNSW_8cpp_source.html new file mode 100644 index 000000000..29412b4c9 --- /dev/null +++ b/docs/html/IndexHNSW_8cpp_source.html @@ -0,0 +1,2405 @@ + + + + + + +Faiss: /data/users/matthijs/github_faiss/faiss/IndexHNSW.cpp Source File + + + + + + + + + +
+
+ + + + + + +
+
Faiss +
+
+
+ + + + + +
+ + + + +
+ +
+ +
+
+
/data/users/matthijs/github_faiss/faiss/IndexHNSW.cpp
+
+
+
1 /**
+
2  * Copyright (c) 2015-present, Facebook, Inc.
+
3  * All rights reserved.
+
4  *
+
5  * This source code is licensed under the BSD+Patents license found in the
+
6  * LICENSE file in the root directory of this source tree.
+
7  */
+
8 
+
9 #include "IndexHNSW.h"
+
10 
+
11 
+
12 #include <cstdlib>
+
13 #include <cassert>
+
14 #include <cstring>
+
15 #include <cstdio>
+
16 #include <cmath>
+
17 #include <omp.h>
+
18 
+
19 #include <unordered_set>
+
20 #include <queue>
+
21 
+
22 #include <sys/types.h>
+
23 #include <sys/stat.h>
+
24 #include <unistd.h>
+
25 #include <stdint.h>
+
26 
+
27 #include <immintrin.h>
+
28 
+
29 #include "utils.h"
+
30 #include "Heap.h"
+
31 #include "FaissAssert.h"
+
32 #include "IndexFlat.h"
+
33 #include "IndexIVFPQ.h"
+
34 
+
35 
+
36 extern "C" {
+
37 
+
38 /* declare BLAS functions, see http://www.netlib.org/clapack/cblas/ */
+
39 
+
40 int sgemm_ (const char *transa, const char *transb, FINTEGER *m, FINTEGER *
+
41  n, FINTEGER *k, const float *alpha, const float *a,
+
42  FINTEGER *lda, const float *b, FINTEGER *
+
43  ldb, float *beta, float *c, FINTEGER *ldc);
+
44 
+
45 }
+
46 
+
47 namespace faiss {
+
48 
+
49 /**************************************************************
+
50  * Auxiliary structures
+
51  **************************************************************/
+
52 
+
53 /// set implementation optimized for fast access.
+
54 struct VisitedTable {
+
55  std::vector<uint8_t> visited;
+
56  int visno;
+
57 
+
58  VisitedTable(int size):
+
59  visited(size), visno(1)
+
60  {}
+
61  /// set flog #no to true
+
62  void set(int no) {
+
63  visited[no] = visno;
+
64  }
+
65  /// get flag #no
+
66  bool get(int no) const {
+
67  return visited[no] == visno;
+
68  }
+
69  /// reset all flags to false
+
70  void advance() {
+
71  visno++;
+
72  if (visno == 250) {
+
73  // 250 rather than 255 because sometimes we use visno and visno+1
+
74  memset (visited.data(), 0, sizeof(visited[0]) * visited.size());
+
75  visno = 1;
+
76  }
+
77  }
+
78 };
+
79 
+
80 
+
81 namespace {
+
82 
+
83 typedef HNSW::idx_t idx_t;
+
84 typedef HNSW::storage_idx_t storage_idx_t;
+
85 typedef HNSW::DistanceComputer DistanceComputer;
+
86  // typedef ::faiss::VisitedTable VisitedTable;
+
87 
+
88 /// to sort pairs of (id, distance) from nearest to fathest or the reverse
+
89 struct NodeDistCloser {
+
90  float d;
+
91  int id;
+
92  NodeDistCloser(float d, int id): d(d), id(id) {}
+
93  bool operator<(const NodeDistCloser &obj1) const { return d < obj1.d; }
+
94 };
+
95 
+
96 struct NodeDistFarther {
+
97  float d;
+
98  int id;
+
99  NodeDistFarther(float d, int id): d(d), id(id) {}
+
100  bool operator<(const NodeDistFarther &obj1) const { return d > obj1.d; }
+
101 };
+
102 
+
103 
+
104 /** Heap structure that allows fast */
+
105 
+
106 struct MinimaxHeap {
+
107  int n;
+
108  int k;
+
109  int nvalid;
+
110 
+
111  std::vector<storage_idx_t> ids;
+
112  std::vector<float> dis;
+
113  typedef faiss::CMax<float, storage_idx_t> HC;
+
114 
+
115  explicit MinimaxHeap(int n): n(n), k(0), nvalid(0), ids(n), dis(n) {}
+
116 
+
117  void push(storage_idx_t i, float v)
+
118  {
+
119  if (k == n) {
+
120  if (v >= dis[0]) return;
+
121  faiss::heap_pop<HC> (k--, dis.data(), ids.data());
+
122  nvalid--;
+
123  }
+
124  faiss::heap_push<HC> (++k, dis.data(), ids.data(), v, i);
+
125  nvalid++;
+
126  }
+
127 
+
128  float max() const
+
129  {
+
130  return dis[0];
+
131  }
+
132 
+
133  int size() const {return nvalid;}
+
134 
+
135  void clear() {nvalid = k = 0; }
+
136 
+
137  int pop_min(float *vmin_out = nullptr)
+
138  {
+
139  assert(k > 0);
+
140  // returns min. This is an O(n) operation
+
141  int i = k - 1;
+
142  while (i >= 0) {
+
143  if (ids[i] != -1) break;
+
144  i--;
+
145  }
+
146  if (i == -1) return -1;
+
147  int imin = i;
+
148  float vmin = dis[i];
+
149  i--;
+
150  while(i >= 0) {
+
151  if (ids[i] != -1 && dis[i] < vmin) {
+
152  vmin = dis[i];
+
153  imin = i;
+
154  }
+
155  i--;
+
156  }
+
157  if (vmin_out) *vmin_out = vmin;
+
158  int ret = ids[imin];
+
159  ids[imin] = -1;
+
160  nvalid --;
+
161  return ret;
+
162  }
+
163 
+
164  int count_below(float thresh) {
+
165  float n_below = 0;
+
166  for(int i = 0; i < k; i++) {
+
167  if (dis[i] < thresh)
+
168  n_below++;
+
169  }
+
170  return n_below;
+
171  }
+
172 
+
173 };
+
174 
+
175 
+
176 /**************************************************************
+
177  * Addition subroutines
+
178  **************************************************************/
+
179 
+
180 /** Enumerate vertices from farthest to nearest from query, keep a
+
181  * neighbor only if there is no previous neighbor that is closer to
+
182  * that vertex than the query.
+
183  */
+
184 void shrink_neighbor_list(DistanceComputer & qdis,
+
185  std::priority_queue<NodeDistFarther> &input,
+
186  std::vector<NodeDistFarther> &output,
+
187  int max_size)
+
188 {
+
189  while (input.size() > 0) {
+
190  NodeDistFarther v1 = input.top();
+
191  input.pop();
+
192  float dist_v1_q = v1.d;
+
193 
+
194  bool good = true;
+
195  for (NodeDistFarther v2 : output) {
+
196  float dist_v1_v2 = qdis.symmetric_dis(v2.id, v1.id);
+
197 
+
198  if (dist_v1_v2 < dist_v1_q) {
+
199  good = false;
+
200  break;
+
201  }
+
202  }
+
203  if (good) {
+
204  output.push_back(v1);
+
205  if (output.size() >= max_size)
+
206  return;
+
207  }
+
208  }
+
209 }
+
210 
+
211 
+
212 
+
213 
+
214 /// remove neighbors from the list to make it smaller than max_size
+
215 void shrink_neighbor_list(DistanceComputer & qdis,
+
216  std::priority_queue<NodeDistCloser> &resultSet1,
+
217  int max_size)
+
218 {
+
219 
+
220  if (resultSet1.size() < max_size) {
+
221  return;
+
222  }
+
223  std::priority_queue<NodeDistFarther> resultSet;
+
224  std::vector<NodeDistFarther> returnlist;
+
225 
+
226  while (resultSet1.size() > 0) {
+
227  resultSet.emplace(resultSet1.top().d, resultSet1.top().id);
+
228  resultSet1.pop();
+
229  }
+
230 
+
231  shrink_neighbor_list (qdis, resultSet, returnlist, max_size);
+
232 
+
233  for (NodeDistFarther curen2 : returnlist) {
+
234  resultSet1.emplace(curen2.d, curen2.id);
+
235  }
+
236 
+
237 }
+
238 
+
239 
+
240 /// add a link between two elements, possibly shrinking the list
+
241 /// of links to make room for it.
+
242 void add_link(HNSW & hnsw,
+
243  DistanceComputer & qdis,
+
244  storage_idx_t src, storage_idx_t dest,
+
245  int level)
+
246 {
+
247  size_t begin, end;
+
248  hnsw.neighbor_range(src, level, &begin, &end);
+
249  if (hnsw.neighbors[end - 1] == -1) {
+
250  // there is enough room, find a slot to add it
+
251  size_t i = end;
+
252  while(i > begin) {
+
253  if (hnsw.neighbors[i - 1] != -1) break;
+
254  i--;
+
255  }
+
256  hnsw.neighbors[i] = dest;
+
257  return;
+
258  }
+
259 
+
260  // otherwise we let them fight out which to keep
+
261 
+
262  // copy to resultSet...
+
263  std::priority_queue<NodeDistCloser> resultSet;
+
264  resultSet.emplace(qdis.symmetric_dis(src, dest), dest);
+
265  for (size_t i = begin; i < end; i++) { // HERE WAS THE BUG
+
266  storage_idx_t neigh = hnsw.neighbors[i];
+
267  resultSet.emplace(qdis.symmetric_dis(src, neigh), neigh);
+
268  }
+
269 
+
270  shrink_neighbor_list(qdis, resultSet, end - begin);
+
271 
+
272  // ...and back
+
273  size_t i = begin;
+
274  while (resultSet.size()) {
+
275  hnsw.neighbors[i++] = resultSet.top().id;
+
276  resultSet.pop();
+
277  }
+
278  // they may have shrunk more than just by 1 element
+
279  while(i < end) {
+
280  hnsw.neighbors[i++] = -1;
+
281  }
+
282 }
+
283 
+
284 /// search neighbors on a single level, starting from an entry point
+
285 void search_neighbors_to_add(HNSW & hnsw,
+
286  DistanceComputer &qdis,
+
287  std::priority_queue<NodeDistCloser> &results,
+
288  int entry_point,
+
289  float d_entry_point,
+
290  int level,
+
291  VisitedTable &vt)
+
292 {
+
293  // top is nearest candidate
+
294  std::priority_queue<NodeDistFarther> candidates;
+
295 
+
296  NodeDistFarther ev(d_entry_point, entry_point);
+
297  candidates.push(ev);
+
298  results.emplace(d_entry_point, entry_point);
+
299  vt.set(entry_point);
+
300 
+
301  while (!candidates.empty()) {
+
302  // get nearest
+
303  const NodeDistFarther &currEv = candidates.top();
+
304 
+
305  if (currEv.d > results.top().d) {
+
306  break;
+
307  }
+
308  int currNode = currEv.id;
+
309  candidates.pop();
+
310 
+
311  // loop over neighbors
+
312  size_t begin, end;
+
313  hnsw.neighbor_range(currNode, level, &begin, &end);
+
314  for(size_t i = begin; i < end; i++) {
+
315  storage_idx_t nodeId = hnsw.neighbors[i];
+
316  if (nodeId < 0) break;
+
317  if (vt.get(nodeId)) continue;
+
318  vt.set(nodeId);
+
319 
+
320  float dis = qdis(nodeId);
+
321  NodeDistFarther evE1(dis, nodeId);
+
322 
+
323  if (results.size() < hnsw.efConstruction ||
+
324  results.top().d > dis) {
+
325 
+
326  results.emplace(dis, nodeId);
+
327  candidates.emplace(dis, nodeId);
+
328  if (results.size() > hnsw.efConstruction) {
+
329  results.pop();
+
330  }
+
331  }
+
332  }
+
333  }
+
334  vt.advance();
+
335 }
+
336 
+
337 
+
338 /// Finds neighbors and builds links with them, starting from an entry
+
339 /// point. The own neighbor list is assumed to be locked.
+
340 void add_links_starting_from(HNSW & hnsw,
+
341  DistanceComputer &ptdis,
+
342  storage_idx_t pt_id,
+
343  storage_idx_t nearest,
+
344  float d_nearest,
+
345  int level,
+
346  omp_lock_t * locks,
+
347  VisitedTable &vt)
+
348 {
+
349 
+
350  std::priority_queue<NodeDistCloser> link_targets;
+
351 
+
352  search_neighbors_to_add(
+
353  hnsw, ptdis, link_targets, nearest, d_nearest,
+
354  level, vt);
+
355 
+
356  // but we can afford only this many neighbors
+
357  int M = hnsw.nb_neighbors(level);
+
358 
+
359  shrink_neighbor_list(ptdis, link_targets, M);
+
360 
+
361  while (!link_targets.empty()) {
+
362  int other_id = link_targets.top().id;
+
363 
+
364  omp_set_lock(&locks[other_id]);
+
365  add_link(hnsw, ptdis, other_id, pt_id, level);
+
366  omp_unset_lock(&locks[other_id]);
+
367 
+
368  add_link(hnsw, ptdis, pt_id, other_id, level);
+
369 
+
370  link_targets.pop();
+
371  }
+
372 }
+
373 
+
374 /**************************************************************
+
375  * Searching subroutines
+
376  **************************************************************/
+
377 
+
378 /// greedily update a nearest vector at a given level
+
379 void greedy_update_nearest(const HNSW & hnsw,
+
380  DistanceComputer & qdis,
+
381  int level,
+
382  storage_idx_t & nearest,
+
383  float & d_nearest)
+
384 {
+
385  for(;;) {
+
386  storage_idx_t prev_nearest = nearest;
+
387 
+
388  size_t begin, end;
+
389  hnsw.neighbor_range(nearest, level, &begin, &end);
+
390  for(size_t i = begin; i < end; i++) {
+
391  storage_idx_t v = hnsw.neighbors[i];
+
392  if (v < 0) break;
+
393  float dis = qdis(v);
+
394  if (dis < d_nearest) {
+
395  nearest = v;
+
396  d_nearest = dis;
+
397  }
+
398  }
+
399  if (nearest == prev_nearest) {
+
400  return;
+
401  }
+
402  }
+
403 
+
404 }
+
405 
+
406 
+
407 /** Do a BFS on the candidates list */
+
408 
+
409 int search_from_candidates(const HNSW & hnsw,
+
410  DistanceComputer & qdis, int k,
+
411  idx_t *I, float * D,
+
412  MinimaxHeap &candidates,
+
413  VisitedTable &vt,
+
414  int level, int nres_in = 0)
+
415 {
+
416  int nres = nres_in;
+
417  int ndis = 0;
+
418  for (int i = 0; i < candidates.size(); i++) {
+
419  idx_t v1 = candidates.ids[i];
+
420  float d = candidates.dis[i];
+
421  FAISS_ASSERT(v1 >= 0);
+
422  if (nres < k) {
+
423  faiss::maxheap_push (++nres, D, I, d, v1);
+
424  } else if (d < D[0]) {
+
425  faiss::maxheap_pop (nres--, D, I);
+
426  faiss::maxheap_push (++nres, D, I, d, v1);
+
427  }
+
428  vt.set(v1);
+
429  }
+
430 
+
431  bool do_dis_check = hnsw.check_relative_distance;
+
432  int nstep = 0;
+
433 
+
434  while (candidates.size() > 0) {
+
435  float d0 = 0;
+
436  int v0 = candidates.pop_min(&d0);
+
437 
+
438  if (do_dis_check) {
+
439  // tricky stopping condition: there are more that ef
+
440  // distances that are processed already that are smaller
+
441  // than d0
+
442 
+
443  int n_dis_below = candidates.count_below(d0);
+
444  if(n_dis_below >= hnsw.efSearch) {
+
445  break;
+
446  }
+
447  }
+
448  size_t begin, end;
+
449  hnsw.neighbor_range(v0, level, &begin, &end);
+
450 
+
451  for (size_t j = begin; j < end; j++) {
+
452  int v1 = hnsw.neighbors[j];
+
453  if (v1 < 0) break;
+
454  if (vt.get(v1)) {
+
455  continue;
+
456  }
+
457  vt.set(v1);
+
458  ndis++;
+
459  float d = qdis(v1);
+
460  if (nres < k) {
+
461  faiss::maxheap_push (++nres, D, I, d, v1);
+
462  } else if (d < D[0]) {
+
463  faiss::maxheap_pop (nres--, D, I);
+
464  faiss::maxheap_push (++nres, D, I, d, v1);
+
465  }
+
466  candidates.push(v1, d);
+
467  }
+
468 
+
469  nstep++;
+
470  if (!do_dis_check && nstep > hnsw.efSearch) {
+
471  break;
+
472  }
+
473  }
+
474 
+
475  if (level == 0) {
+
476 #pragma omp critical
+
477  {
+
478  hnsw_stats.n1 ++;
+
479  if (candidates.size() == 0)
+
480  hnsw_stats.n2 ++;
+
481  hnsw_stats.n3 += ndis;
+
482  }
+
483  }
+
484 
+
485  return nres;
+
486 }
+
487 
+
488 
+
489 } // anonymous namespace
+
490 
+
491 
+
492 /**************************************************************
+
493  * HNSW structure implementation
+
494  **************************************************************/
+
495 
+
496 
+
497 int HNSW::nb_neighbors(int layer_no) const
+
498 {
+
499  return cum_nneighbor_per_level[layer_no + 1] -
+
500  cum_nneighbor_per_level[layer_no];
+
501 }
+
502 
+
503 void HNSW::set_nb_neighbors(int level_no, int n)
+
504 {
+
505  FAISS_THROW_IF_NOT(levels.size() == 0);
+
506  int cur_n = nb_neighbors(level_no);
+
507  for (int i = level_no + 1; i < cum_nneighbor_per_level.size(); i++) {
+
508  cum_nneighbor_per_level[i] += n - cur_n;
+
509  }
+
510 }
+
511 
+
512 int HNSW::cum_nb_neighbors(int layer_no) const
+
513 {
+
514  return cum_nneighbor_per_level[layer_no];
+
515 }
+
516 
+
517 void HNSW::neighbor_range(idx_t no, int layer_no,
+
518  size_t * begin, size_t * end) const
+
519 {
+
520  size_t o = offsets[no];
+
521  *begin = o + cum_nb_neighbors(layer_no);
+
522  *end = o + cum_nb_neighbors(layer_no + 1);
+
523 }
+
524 
+
525 
+
526 
+
527 HNSW::HNSW(int M): rng(12345) {
+
528  set_default_probas(M, 1.0 / log(M));
+
529  max_level = -1;
+
530  entry_point = -1;
+
531  efSearch = 16;
+
532  check_relative_distance = true;
+
533  efConstruction = 40;
+
534  upper_beam = 1;
+
535  offsets.push_back(0);
+
536 }
+
537 
+
538 
+
539 int HNSW::random_level()
+
540 {
+
541  double f = rng.rand_float();
+
542  // could be a bit faster with bissection
+
543  for (int level = 0; level < assign_probas.size(); level++) {
+
544  if (f < assign_probas[level]) {
+
545  return level;
+
546  }
+
547  f -= assign_probas[level];
+
548  }
+
549  // happens with exponentially low probability
+
550  return assign_probas.size() - 1;
+
551 }
+
552 
+
553 void HNSW::set_default_probas(int M, float levelMult)
+
554 {
+
555  int nn = 0;
+
556  cum_nneighbor_per_level.push_back (0);
+
557  for (int level = 0; ;level++) {
+
558  float proba = exp(-level / levelMult) * (1 - exp(-1 / levelMult));
+
559  if (proba < 1e-9) break;
+
560  assign_probas.push_back(proba);
+
561  nn += level == 0 ? M * 2 : M;
+
562  cum_nneighbor_per_level.push_back (nn);
+
563  }
+
564 }
+
565 
+
566 void HNSW::clear_neighbor_tables(int level)
+
567 {
+
568  for (int i = 0; i < levels.size(); i++) {
+
569  size_t begin, end;
+
570  neighbor_range(i, level, &begin, &end);
+
571  for (size_t j = begin; j < end; j++)
+
572  neighbors[j] = -1;
+
573  }
+
574 }
+
575 
+
576 
+
577 void HNSW::reset() {
+
578  max_level = -1;
+
579  entry_point = -1;
+
580  offsets.clear();
+
581  offsets.push_back(0);
+
582  levels.clear();
+
583  neighbors.clear();
+
584 }
+
585 
+
586 
+
587 
+
588 void HNSW::print_neighbor_stats(int level) const
+
589 {
+
590  FAISS_THROW_IF_NOT (level < cum_nneighbor_per_level.size());
+
591  printf("stats on level %d, max %d neighbors per vertex:\n",
+
592  level, nb_neighbors(level));
+
593  size_t tot_neigh = 0, tot_common = 0, tot_reciprocal = 0, n_node = 0;
+
594 #pragma omp parallel for reduction(+: tot_neigh) reduction(+: tot_common) \
+
595  reduction(+: tot_reciprocal) reduction(+: n_node)
+
596  for (int i = 0; i < levels.size(); i++) {
+
597  if (levels[i] > level) {
+
598  n_node++;
+
599  size_t begin, end;
+
600  neighbor_range(i, level, &begin, &end);
+
601  std::unordered_set<int> neighset;
+
602  for (size_t j = begin; j < end; j++) {
+
603  if (neighbors [j] < 0) break;
+
604  neighset.insert(neighbors[j]);
+
605  }
+
606  int n_neigh = neighset.size();
+
607  int n_common = 0;
+
608  int n_reciprocal = 0;
+
609  for (size_t j = begin; j < end; j++) {
+
610  storage_idx_t i2 = neighbors[j];
+
611  if (i2 < 0) break;
+
612  FAISS_ASSERT(i2 != i);
+
613  size_t begin2, end2;
+
614  neighbor_range(i2, level, &begin2, &end2);
+
615  for (size_t j2 = begin2; j2 < end2; j2++) {
+
616  storage_idx_t i3 = neighbors[j2];
+
617  if (i3 < 0) break;
+
618  if (i3 == i) {
+
619  n_reciprocal++;
+
620  continue;
+
621  }
+
622  if (neighset.count(i3)) {
+
623  neighset.erase(i3);
+
624  n_common++;
+
625  }
+
626  }
+
627  }
+
628  tot_neigh += n_neigh;
+
629  tot_common += n_common;
+
630  tot_reciprocal += n_reciprocal;
+
631  }
+
632  }
+
633  float normalizer = n_node;
+
634  printf(" nb of nodes at that level %ld\n", n_node);
+
635  printf(" neighbors per node: %.2f (%ld)\n", tot_neigh / normalizer, tot_neigh);
+
636  printf(" nb of reciprocal neighbors: %.2f\n", tot_reciprocal / normalizer);
+
637  printf(" nb of neighbors that are also neighbor-of-neighbors: %.2f (%ld)\n",
+
638  tot_common / normalizer, tot_common);
+
639 
+
640 
+
641 
+
642 }
+
643 
+
644 HNSWStats hnsw_stats;
+
645 
+
646 void HNSWStats::reset ()
+
647 {
+
648  memset(this, 0, sizeof(*this));
+
649 }
+
650 
+
651 
+
652 
+
653 /**************************************************************
+
654  * Building, parallel
+
655  **************************************************************/
+
656 
+
657 void HNSW::add_with_locks(
+
658  DistanceComputer & ptdis, int pt_level, int pt_id,
+
659  std::vector<omp_lock_t> & locks,
+
660  VisitedTable &vt)
+
661 {
+
662  // greedy search on upper levels
+
663 
+
664  storage_idx_t nearest;
+
665 #pragma omp critical
+
666  {
+
667  nearest = entry_point;
+
668 
+
669  if (nearest == -1) {
+
670  max_level = pt_level;
+
671  entry_point = pt_id;
+
672  }
+
673  }
+
674 
+
675  if (nearest < 0) {
+
676  return;
+
677  }
+
678 
+
679  omp_set_lock(&locks[pt_id]);
+
680 
+
681  int level = max_level; // level at which we start adding neighbors
+
682  float d_nearest = ptdis(nearest);
+
683 
+
684  for(; level > pt_level; level--) {
+
685  greedy_update_nearest(*this, ptdis, level, nearest, d_nearest);
+
686  }
+
687 
+
688  for(; level >= 0; level--) {
+
689  add_links_starting_from(*this, ptdis, pt_id, nearest, d_nearest,
+
690  level, locks.data(), vt);
+
691  }
+
692 
+
693  omp_unset_lock(&locks[pt_id]);
+
694 
+
695  if (pt_level > max_level) {
+
696  max_level = pt_level;
+
697  entry_point = pt_id;
+
698  }
+
699 
+
700 }
+
701 
+
702 
+
703 
+
704 /**************************************************************
+
705  * Searching
+
706  **************************************************************/
+
707 
+
708 
+
709 
+
710 
+
711 void HNSW::search(DistanceComputer & qdis,
+
712  int k, idx_t *I, float * D,
+
713  VisitedTable &vt) const
+
714 {
+
715 
+
716  if (upper_beam == 1) {
+
717 
+
718  // greedy search on upper levels
+
719  storage_idx_t nearest = entry_point;
+
720  float d_nearest = qdis(nearest);
+
721 
+
722  for(int level = max_level; level >= 1; level--) {
+
723  greedy_update_nearest(*this, qdis, level, nearest, d_nearest);
+
724  }
+
725 
+
726  int candidates_size = std::max(efSearch, k);
+
727  MinimaxHeap candidates(candidates_size);
+
728 
+
729  candidates.push(nearest, d_nearest);
+
730 
+
731  search_from_candidates (
+
732  *this, qdis, k, I, D, candidates, vt, 0);
+
733  vt.advance();
+
734 
+
735  } else {
+
736 
+
737  int candidates_size = upper_beam;
+
738  MinimaxHeap candidates(candidates_size);
+
739 
+
740  std::vector<idx_t> I_to_next(candidates_size);
+
741  std::vector<float> D_to_next(candidates_size);
+
742 
+
743  int nres = 1;
+
744  I_to_next[0] = entry_point;
+
745  D_to_next[0] = qdis(entry_point);
+
746 
+
747  for(int level = max_level; level >= 0; level--) {
+
748 
+
749  // copy I, D -> candidates
+
750 
+
751  candidates.clear();
+
752 
+
753  for (int i = 0; i < nres; i++) {
+
754  candidates.push(I_to_next[i], D_to_next[i]);
+
755  }
+
756 
+
757  if (level == 0) {
+
758  nres = search_from_candidates (
+
759  *this, qdis, k, I, D, candidates, vt, 0);
+
760  } else {
+
761  nres = search_from_candidates (
+
762  *this, qdis, candidates_size,
+
763  I_to_next.data(), D_to_next.data(),
+
764  candidates, vt, level);
+
765  }
+
766  vt.advance();
+
767  }
+
768 
+
769  }
+
770 }
+
771 
+
772 /**************************************************************
+
773  * add / search blocks of descriptors
+
774  **************************************************************/
+
775 
+
776 namespace {
+
777 
+
778 int prepare_level_tab (HNSW & hnsw, size_t n, bool preset_levels = false)
+
779 {
+
780  size_t n0 = hnsw.offsets.size() - 1;
+
781 
+
782  if (preset_levels) {
+
783  FAISS_ASSERT (n0 + n == hnsw.levels.size());
+
784  } else {
+
785  FAISS_ASSERT (n0 == hnsw.levels.size());
+
786  for (int i = 0; i < n; i++) {
+
787  int pt_level = hnsw.random_level();
+
788  hnsw.levels.push_back(pt_level + 1);
+
789  }
+
790  }
+
791 
+
792  int max_level = 0;
+
793  for (int i = 0; i < n; i++) {
+
794  int pt_level = hnsw.levels[i + n0] - 1;
+
795  if (pt_level > max_level) max_level = pt_level;
+
796  hnsw.offsets.push_back(hnsw.offsets.back() +
+
797  hnsw.cum_nb_neighbors(pt_level + 1));
+
798  hnsw.neighbors.resize(hnsw.offsets.back(), -1);
+
799  }
+
800  return max_level;
+
801 }
+
802 
+
803 void hnsw_add_vertices(IndexHNSW &index_hnsw,
+
804  size_t n0,
+
805  size_t n, const float *x,
+
806  bool verbose,
+
807  bool preset_levels = false) {
+
808  HNSW & hnsw = index_hnsw.hnsw;
+
809  size_t ntotal = n0 + n;
+
810  double t0 = getmillisecs();
+
811  if (verbose) {
+
812  printf("hnsw_add_vertices: adding %ld elements on top of %ld "
+
813  "(preset_levels=%d)\n",
+
814  n, n0, int(preset_levels));
+
815  }
+
816 
+
817  int max_level = prepare_level_tab (index_hnsw.hnsw, n, preset_levels);
+
818 
+
819  if (verbose) {
+
820  printf(" max_level = %d\n", max_level);
+
821  }
+
822 
+
823  std::vector<omp_lock_t> locks(ntotal);
+
824  for(int i = 0; i < ntotal; i++)
+
825  omp_init_lock(&locks[i]);
+
826 
+
827  // add vectors from highest to lowest level
+
828  std::vector<int> hist;
+
829  std::vector<int> order(n);
+
830 
+
831  { // make buckets with vectors of the same level
+
832 
+
833  // build histogram
+
834  for (int i = 0; i < n; i++) {
+
835  storage_idx_t pt_id = i + n0;
+
836  int pt_level = hnsw.levels[pt_id] - 1;
+
837  while (pt_level >= hist.size())
+
838  hist.push_back(0);
+
839  hist[pt_level] ++;
+
840  }
+
841 
+
842  // accumulate
+
843  std::vector<int> offsets(hist.size() + 1, 0);
+
844  for (int i = 0; i < hist.size() - 1; i++) {
+
845  offsets[i + 1] = offsets[i] + hist[i];
+
846  }
+
847 
+
848  // bucket sort
+
849  for (int i = 0; i < n; i++) {
+
850  storage_idx_t pt_id = i + n0;
+
851  int pt_level = hnsw.levels[pt_id] - 1;
+
852  order[offsets[pt_level]++] = pt_id;
+
853  }
+
854  }
+
855 
+
856  { // perform add
+
857  RandomGenerator rng2(789);
+
858 
+
859  int i1 = n;
+
860 
+
861  for (int pt_level = hist.size() - 1; pt_level >= 0; pt_level--) {
+
862  int i0 = i1 - hist[pt_level];
+
863 
+
864  if (verbose) {
+
865  printf("Adding %d elements at level %d\n",
+
866  i1 - i0, pt_level);
+
867  }
+
868 
+
869  // random permutation to get rid of dataset order bias
+
870  for (int j = i0; j < i1; j++)
+
871  std::swap(order[j], order[j + rng2.rand_int(i1 - j)]);
+
872 
+
873 #pragma omp parallel
+
874  {
+
875  VisitedTable vt (ntotal);
+
876 
+
877  DistanceComputer *dis = index_hnsw.get_distance_computer();
+
878  ScopeDeleter1<DistanceComputer> del(dis);
+
879  int prev_display = verbose && omp_get_thread_num() == 0 ? 0 : -1;
+
880 
+
881 #pragma omp for schedule(dynamic)
+
882  for (int i = i0; i < i1; i++) {
+
883  storage_idx_t pt_id = order[i];
+
884  dis->set_query (x + (pt_id - n0) * dis->d);
+
885 
+
886  hnsw.add_with_locks (
+
887  *dis, pt_level, pt_id, locks,
+
888  vt);
+
889 
+
890  if (prev_display >= 0 && i - i0 > prev_display + 10000) {
+
891  prev_display = i - i0;
+
892  printf(" %d / %d\r", i - i0, i1 - i0);
+
893  fflush(stdout);
+
894  }
+
895  }
+
896  }
+
897  i1 = i0;
+
898  }
+
899  FAISS_ASSERT(i1 == 0);
+
900  }
+
901  if (verbose)
+
902  printf("Done in %.3f ms\n", getmillisecs() - t0);
+
903 
+
904  for(int i = 0; i < ntotal; i++)
+
905  omp_destroy_lock(&locks[i]);
+
906 
+
907 }
+
908 
+
909 
+
910 } // anonymous namespace
+
911 
+
912 
+
913 void HNSW::fill_with_random_links(size_t n)
+
914 {
+
915  int max_level = prepare_level_tab (*this, n);
+
916  RandomGenerator rng2(456);
+
917 
+
918  for (int level = max_level - 1; level >= 0; level++) {
+
919  std::vector<int> elts;
+
920  for (int i = 0; i < n; i++) {
+
921  if (levels[i] > level) {
+
922  elts.push_back(i);
+
923  }
+
924  }
+
925  printf ("linking %ld elements in level %d\n",
+
926  elts.size(), level);
+
927 
+
928  if (elts.size() == 1) continue;
+
929 
+
930  for (int ii = 0; ii < elts.size(); ii++) {
+
931  int i = elts[ii];
+
932  size_t begin, end;
+
933  neighbor_range(i, 0, &begin, &end);
+
934  for (size_t j = begin; j < end; j++) {
+
935  int other = 0;
+
936  do {
+
937  other = elts[rng2.rand_int(elts.size())];
+
938  } while(other == i);
+
939 
+
940  neighbors[j] = other;
+
941  }
+
942 
+
943  }
+
944 
+
945  }
+
946 
+
947 }
+
948 
+
949 
+
950 
+
951 
+
952 /**************************************************************
+
953  * IndexHNSW implementation
+
954  **************************************************************/
+
955 
+
956 IndexHNSW::IndexHNSW(int d, int M):
+
957  Index(d, METRIC_L2),
+
958  hnsw(M),
+
959  own_fields(false),
+
960  storage(nullptr),
+
961  reconstruct_from_neighbors(nullptr)
+
962 {}
+
963 
+
964 IndexHNSW::IndexHNSW(Index *storage, int M):
+
965  Index(storage->d, METRIC_L2),
+
966  hnsw(M),
+
967  own_fields(false),
+
968  storage(storage),
+
969  reconstruct_from_neighbors(nullptr)
+
970 {}
+
971 
+
972 IndexHNSW::~IndexHNSW() {
+
973  if (own_fields) {
+
974  delete storage;
+
975  }
+
976 }
+
977 
+
978 void IndexHNSW::train(idx_t n, const float* x)
+
979 {
+
980  // hnsw structure does not require training
+
981  storage->train (n, x);
+
982  is_trained = true;
+
983 }
+
984 
+
985 void IndexHNSW::search (idx_t n, const float *x, idx_t k,
+
986  float *distances, idx_t *labels) const
+
987 
+
988 {
+
989 
+
990 #pragma omp parallel
+
991  {
+
992  VisitedTable vt (ntotal);
+
993  DistanceComputer *dis = get_distance_computer();
+
994  ScopeDeleter1<DistanceComputer> del(dis);
+
995  size_t nreorder = 0;
+
996 
+
997 #pragma omp for
+
998  for(int i = 0; i < n; i++) {
+
999  idx_t * idxi = labels + i * k;
+
1000  float * simi = distances + i * k;
+
1001  dis->set_query(x + i * d);
+
1002 
+
1003  maxheap_heapify (k, simi, idxi);
+
1004  hnsw.search (*dis, k, idxi, simi, vt);
+
1005 
+
1006  maxheap_reorder (k, simi, idxi);
+
1007 
+
1008  if (reconstruct_from_neighbors &&
+
1009  reconstruct_from_neighbors->k_reorder != 0) {
+
1010  int k_reorder = reconstruct_from_neighbors->k_reorder;
+
1011  if (k_reorder == -1 || k_reorder > k) k_reorder = k;
+
1012 
+
1013  nreorder += reconstruct_from_neighbors->compute_distances(
+
1014  k_reorder, idxi, x + i * d, simi);
+
1015 
+
1016  // sort top k_reorder
+
1017  maxheap_heapify (k_reorder, simi, idxi, simi, idxi, k_reorder);
+
1018  maxheap_reorder (k_reorder, simi, idxi);
+
1019  }
+
1020  }
+
1021 #pragma omp critical
+
1022  {
+
1023  hnsw_stats.nreorder += nreorder;
+
1024  }
+
1025  }
+
1026 
+
1027 
+
1028 }
+
1029 
+
1030 
+
1031 void IndexHNSW::add(idx_t n, const float *x)
+
1032 {
+
1033  FAISS_THROW_IF_NOT(is_trained);
+
1034  int n0 = ntotal;
+
1035  storage->add(n, x);
+
1036  ntotal = storage->ntotal;
+
1037 
+
1038  hnsw_add_vertices (*this, n0, n, x, verbose,
+
1039  hnsw.levels.size() == ntotal);
+
1040 
+
1041 }
+
1042 
+
1043 void IndexHNSW::reset()
+
1044 {
+
1045  hnsw.reset();
+
1046  storage->reset();
+
1047  ntotal = 0;
+
1048 }
+
1049 
+
1050 void IndexHNSW::reconstruct (idx_t key, float* recons) const
+
1051 {
+
1052  storage->reconstruct(key, recons);
+
1053 }
+
1054 
+
1055 void IndexHNSW::shrink_level_0_neighbors(int new_size)
+
1056 {
+
1057 #pragma omp parallel
+
1058  {
+
1059  DistanceComputer *dis = get_distance_computer();
+
1060  ScopeDeleter1<DistanceComputer> del(dis);
+
1061 
+
1062 #pragma omp for
+
1063  for (idx_t i = 0; i < ntotal; i++) {
+
1064 
+
1065  size_t begin, end;
+
1066  hnsw.neighbor_range(i, 0, &begin, &end);
+
1067 
+
1068  std::priority_queue<NodeDistFarther> initial_list;
+
1069 
+
1070  for (size_t j = begin; j < end; j++) {
+
1071  int v1 = hnsw.neighbors[j];
+
1072  if (v1 < 0) break;
+
1073  initial_list.emplace(dis->symmetric_dis(i, v1), v1);
+
1074 
+
1075  // initial_list.emplace(qdis(v1), v1);
+
1076  }
+
1077 
+
1078  std::vector<NodeDistFarther> shrunk_list;
+
1079  shrink_neighbor_list (*dis, initial_list, shrunk_list, new_size);
+
1080 
+
1081  for (size_t j = begin; j < end; j++) {
+
1082  if (j - begin < shrunk_list.size())
+
1083  hnsw.neighbors[j] = shrunk_list[j - begin].id;
+
1084  else
+
1085  hnsw.neighbors[j] = -1;
+
1086  }
+
1087  }
+
1088  }
+
1089 
+
1090 }
+
1091 
+
1092 void IndexHNSW::search_level_0(
+
1093  idx_t n, const float *x, idx_t k,
+
1094  const storage_idx_t *nearest, const float *nearest_d,
+
1095  float *distances, idx_t *labels, int nprobe,
+
1096  int search_type) const
+
1097 {
+
1098 
+
1099  storage_idx_t ntotal = hnsw.levels.size();
+
1100 #pragma omp parallel
+
1101  {
+
1102  DistanceComputer *qdis = get_distance_computer();
+
1103  ScopeDeleter1<DistanceComputer> del(qdis);
+
1104 
+
1105  VisitedTable vt (ntotal);
+
1106 
+
1107 #pragma omp for
+
1108  for(idx_t i = 0; i < n; i++) {
+
1109  idx_t * idxi = labels + i * k;
+
1110  float * simi = distances + i * k;
+
1111 
+
1112  qdis->set_query(x + i * d);
+
1113  maxheap_heapify (k, simi, idxi);
+
1114 
+
1115  if (search_type == 1) {
+
1116 
+
1117  int nres = 0;
+
1118 
+
1119  for(int j = 0; j < nprobe; j++) {
+
1120  storage_idx_t cj = nearest[i * nprobe + j];
+
1121 
+
1122  if (cj < 0) break;
+
1123 
+
1124  if (vt.get(cj)) continue;
+
1125 
+
1126  int candidates_size = std::max(hnsw.efSearch, int(k));
+
1127  MinimaxHeap candidates(candidates_size);
+
1128 
+
1129  candidates.push(cj, nearest_d[i * nprobe + j]);
+
1130 
+
1131  nres = search_from_candidates (
+
1132  hnsw, *qdis, k, idxi, simi,
+
1133  candidates, vt, 0, nres);
+
1134  }
+
1135  } else if (search_type == 2) {
+
1136 
+
1137  int candidates_size = std::max(hnsw.efSearch, int(k));
+
1138  candidates_size = std::max(candidates_size, nprobe);
+
1139 
+
1140  MinimaxHeap candidates(candidates_size);
+
1141  for(int j = 0; j < nprobe; j++) {
+
1142  storage_idx_t cj = nearest[i * nprobe + j];
+
1143 
+
1144  if (cj < 0) break;
+
1145  candidates.push(cj, nearest_d[i * nprobe + j]);
+
1146  }
+
1147  search_from_candidates (
+
1148  hnsw, *qdis, k, idxi, simi,
+
1149  candidates, vt, 0);
+
1150 
+
1151  }
+
1152  vt.advance();
+
1153 
+
1154  maxheap_reorder (k, simi, idxi);
+
1155 
+
1156  }
+
1157  }
+
1158 
+
1159 
+
1160 }
+
1161 
+
1162 void IndexHNSW::init_level_0_from_knngraph(
+
1163  int k, const float *D, const idx_t *I)
+
1164 {
+
1165  int dest_size = hnsw.nb_neighbors (0);
+
1166 
+
1167 #pragma omp parallel for
+
1168  for (idx_t i = 0; i < ntotal; i++) {
+
1169  DistanceComputer *qdis = get_distance_computer();
+
1170  float vec[d];
+
1171  storage->reconstruct(i, vec);
+
1172  qdis->set_query(vec);
+
1173 
+
1174  std::priority_queue<NodeDistFarther> initial_list;
+
1175 
+
1176  for (size_t j = 0; j < k; j++) {
+
1177  int v1 = I[i * k + j];
+
1178  if (v1 == i) continue;
+
1179  if (v1 < 0) break;
+
1180  initial_list.emplace(D[i * k + j], v1);
+
1181  }
+
1182 
+
1183  std::vector<NodeDistFarther> shrunk_list;
+
1184  shrink_neighbor_list (*qdis, initial_list, shrunk_list, dest_size);
+
1185 
+
1186  size_t begin, end;
+
1187  hnsw.neighbor_range(i, 0, &begin, &end);
+
1188 
+
1189  for (size_t j = begin; j < end; j++) {
+
1190  if (j - begin < shrunk_list.size())
+
1191  hnsw.neighbors[j] = shrunk_list[j - begin].id;
+
1192  else
+
1193  hnsw.neighbors[j] = -1;
+
1194  }
+
1195  }
+
1196 }
+
1197 
+
1198 
+
1199 
+
1200 void IndexHNSW::init_level_0_from_entry_points(
+
1201  int n, const storage_idx_t *points,
+
1202  const storage_idx_t *nearests)
+
1203 {
+
1204 
+
1205  std::vector<omp_lock_t> locks(ntotal);
+
1206  for(int i = 0; i < ntotal; i++)
+
1207  omp_init_lock(&locks[i]);
+
1208 
+
1209 #pragma omp parallel
+
1210  {
+
1211  VisitedTable vt (ntotal);
+
1212 
+
1213  DistanceComputer *dis = get_distance_computer();
+
1214  ScopeDeleter1<DistanceComputer> del(dis);
+
1215  float vec[storage->d];
+
1216 
+
1217 #pragma omp for schedule(dynamic)
+
1218  for (int i = 0; i < n; i++) {
+
1219  storage_idx_t pt_id = points[i];
+
1220  storage_idx_t nearest = nearests[i];
+
1221  storage->reconstruct (pt_id, vec);
+
1222  dis->set_query (vec);
+
1223 
+
1224  add_links_starting_from(hnsw, *dis, pt_id, nearest, (*dis)(nearest),
+
1225  0, locks.data(), vt);
+
1226 
+
1227  if (verbose && i % 10000 == 0) {
+
1228  printf(" %d / %d\r", i, n);
+
1229  fflush(stdout);
+
1230  }
+
1231  }
+
1232  }
+
1233  if (verbose) {
+
1234  printf("\n");
+
1235  }
+
1236 
+
1237  for(int i = 0; i < ntotal; i++)
+
1238  omp_destroy_lock(&locks[i]);
+
1239 }
+
1240 
+
1241 void IndexHNSW::reorder_links()
+
1242 {
+
1243  int M = hnsw.nb_neighbors(0);
+
1244 
+
1245 #pragma omp parallel
+
1246  {
+
1247  std::vector<float> distances (M);
+
1248  std::vector<size_t> order (M);
+
1249  std::vector<storage_idx_t> tmp (M);
+
1250  DistanceComputer *dis = get_distance_computer();
+
1251  ScopeDeleter1<DistanceComputer> del(dis);
+
1252 
+
1253 #pragma omp for
+
1254  for(storage_idx_t i = 0; i < ntotal; i++) {
+
1255 
+
1256  size_t begin, end;
+
1257  hnsw.neighbor_range(i, 0, &begin, &end);
+
1258 
+
1259  for (size_t j = begin; j < end; j++) {
+
1260  storage_idx_t nj = hnsw.neighbors[j];
+
1261  if (nj < 0) {
+
1262  end = j;
+
1263  break;
+
1264  }
+
1265  distances[j - begin] = dis->symmetric_dis(i, nj);
+
1266  tmp [j - begin] = nj;
+
1267  }
+
1268 
+
1269  fvec_argsort (end - begin, distances.data(), order.data());
+
1270  for (size_t j = begin; j < end; j++) {
+
1271  hnsw.neighbors[j] = tmp[order[j - begin]];
+
1272  }
+
1273  }
+
1274 
+
1275  }
+
1276 }
+
1277 
+
1278 
+
1279 void IndexHNSW::link_singletons()
+
1280 {
+
1281  printf("search for singletons\n");
+
1282 
+
1283  std::vector<bool> seen(ntotal);
+
1284 
+
1285  for (size_t i = 0; i < ntotal; i++) {
+
1286  size_t begin, end;
+
1287  hnsw.neighbor_range(i, 0, &begin, &end);
+
1288  for (size_t j = begin; j < end; j++) {
+
1289  storage_idx_t ni = hnsw.neighbors[j];
+
1290  if (ni >= 0) seen[ni] = true;
+
1291  }
+
1292  }
+
1293 
+
1294  int n_sing = 0, n_sing_l1 = 0;
+
1295  std::vector<storage_idx_t> singletons;
+
1296  for (storage_idx_t i = 0; i < ntotal; i++) {
+
1297  if (!seen[i]) {
+
1298  singletons.push_back(i);
+
1299  n_sing++;
+
1300  if (hnsw.levels[i] > 1)
+
1301  n_sing_l1++;
+
1302  }
+
1303  }
+
1304 
+
1305  printf(" Found %d / %ld singletons (%d appear in a level above)\n",
+
1306  n_sing, ntotal, n_sing_l1);
+
1307 
+
1308  std::vector<float>recons(singletons.size() * d);
+
1309  for (int i = 0; i < singletons.size(); i++) {
+
1310 
+
1311  FAISS_ASSERT(!"not implemented");
+
1312 
+
1313  }
+
1314 
+
1315 
+
1316 }
+
1317 
+
1318 
+
1319 
+
1320 
+
1321 // storage that explicitly reconstructs vectors before computing distances
+
1322 struct GenericDistanceComputer: HNSW::DistanceComputer {
+
1323 
+
1324  const Index & storage;
+
1325  std::vector<float> buf;
+
1326  const float *q;
+
1327 
+
1328  GenericDistanceComputer(const Index & storage): storage(storage)
+
1329  {
+
1330  d = storage.d;
+
1331  buf.resize(d * 2);
+
1332  }
+
1333 
+
1334  float operator () (storage_idx_t i) override
+
1335  {
+
1336  storage.reconstruct(i, buf.data());
+
1337  return fvec_L2sqr(q, buf.data(), d);
+
1338  }
+
1339 
+
1340  float symmetric_dis(storage_idx_t i, storage_idx_t j) override
+
1341  {
+
1342  storage.reconstruct(i, buf.data());
+
1343  storage.reconstruct(j, buf.data() + d);
+
1344  return fvec_L2sqr(buf.data() + d, buf.data(), d);
+
1345  }
+
1346 
+
1347  void set_query(const float *x) override {
+
1348  q = x;
+
1349  }
+
1350 
+
1351 
+
1352 };
+
1353 
+
1354 HNSW::DistanceComputer * IndexHNSW::get_distance_computer () const
+
1355 {
+
1356  return new GenericDistanceComputer (*storage);
+
1357 }
+
1358 
+
1359 
+
1360 /**************************************************************
+
1361  * ReconstructFromNeighbors implementation
+
1362  **************************************************************/
+
1363 
+
1364 
+
1365 ReconstructFromNeighbors::ReconstructFromNeighbors(
+
1366  const IndexHNSW & index, size_t k, size_t nsq):
+
1367  index(index), k(k), nsq(nsq) {
+
1368  M = index.hnsw.nb_neighbors(0);
+
1369  FAISS_ASSERT(k <= 256);
+
1370  code_size = k == 1 ? 0 : nsq;
+
1371  ntotal = 0;
+
1372  d = index.d;
+
1373  FAISS_ASSERT(d % nsq == 0);
+
1374  dsub = d / nsq;
+
1375  k_reorder = -1;
+
1376 }
+
1377 
+
1378 void ReconstructFromNeighbors::reconstruct(storage_idx_t i, float *x, float *tmp) const
+
1379 {
+
1380 
+
1381 
+
1382  const HNSW & hnsw = index.hnsw;
+
1383  size_t begin, end;
+
1384  hnsw.neighbor_range(i, 0, &begin, &end);
+
1385 
+
1386  if (k == 1 || nsq == 1) {
+
1387  const float * beta;
+
1388  if (k == 1) {
+
1389  beta = codebook.data();
+
1390  } else {
+
1391  int idx = codes[i];
+
1392  beta = codebook.data() + idx * (M + 1);
+
1393  }
+
1394 
+
1395  float w0 = beta[0]; // weight of image itself
+
1396  index.storage->reconstruct(i, tmp);
+
1397 
+
1398  for (int l = 0; l < d; l++)
+
1399  x[l] = w0 * tmp[l];
+
1400 
+
1401  for (size_t j = begin; j < end; j++) {
+
1402 
+
1403  storage_idx_t ji = hnsw.neighbors[j];
+
1404  if (ji < 0) ji = i;
+
1405  float w = beta[j - begin + 1];
+
1406  index.storage->reconstruct(ji, tmp);
+
1407  for (int l = 0; l < d; l++)
+
1408  x[l] += w * tmp[l];
+
1409  }
+
1410  } else if (nsq == 2) {
+
1411  int idx0 = codes[2 * i];
+
1412  int idx1 = codes[2 * i + 1];
+
1413 
+
1414  const float *beta0 = codebook.data() + idx0 * (M + 1);
+
1415  const float *beta1 = codebook.data() + (idx1 + k) * (M + 1);
+
1416 
+
1417  index.storage->reconstruct(i, tmp);
+
1418 
+
1419  float w0;
+
1420 
+
1421  w0 = beta0[0];
+
1422  for (int l = 0; l < dsub; l++)
+
1423  x[l] = w0 * tmp[l];
+
1424 
+
1425  w0 = beta1[0];
+
1426  for (int l = dsub; l < d; l++)
+
1427  x[l] = w0 * tmp[l];
+
1428 
+
1429  for (size_t j = begin; j < end; j++) {
+
1430  storage_idx_t ji = hnsw.neighbors[j];
+
1431  if (ji < 0) ji = i;
+
1432  index.storage->reconstruct(ji, tmp);
+
1433  float w;
+
1434  w = beta0[j - begin + 1];
+
1435  for (int l = 0; l < dsub; l++)
+
1436  x[l] += w * tmp[l];
+
1437 
+
1438  w = beta1[j - begin + 1];
+
1439  for (int l = dsub; l < d; l++)
+
1440  x[l] += w * tmp[l];
+
1441  }
+
1442  } else {
+
1443  const float *betas[nsq];
+
1444  {
+
1445  const float *b = codebook.data();
+
1446  const uint8_t *c = &codes[i * code_size];
+
1447  for (int sq = 0; sq < nsq; sq++) {
+
1448  betas[sq] = b + (*c++) * (M + 1);
+
1449  b += (M + 1) * k;
+
1450  }
+
1451  }
+
1452 
+
1453  index.storage->reconstruct(i, tmp);
+
1454  {
+
1455  int d0 = 0;
+
1456  for (int sq = 0; sq < nsq; sq++) {
+
1457  float w = *(betas[sq]++);
+
1458  int d1 = d0 + dsub;
+
1459  for (int l = d0; l < d1; l++) {
+
1460  x[l] = w * tmp[l];
+
1461  }
+
1462  d0 = d1;
+
1463  }
+
1464  }
+
1465 
+
1466  for (size_t j = begin; j < end; j++) {
+
1467  storage_idx_t ji = hnsw.neighbors[j];
+
1468  if (ji < 0) ji = i;
+
1469 
+
1470  index.storage->reconstruct(ji, tmp);
+
1471  int d0 = 0;
+
1472  for (int sq = 0; sq < nsq; sq++) {
+
1473  float w = *(betas[sq]++);
+
1474  int d1 = d0 + dsub;
+
1475  for (int l = d0; l < d1; l++) {
+
1476  x[l] += w * tmp[l];
+
1477  }
+
1478  d0 = d1;
+
1479  }
+
1480  }
+
1481  }
+
1482 }
+
1483 
+
1484 void ReconstructFromNeighbors::reconstruct_n(storage_idx_t n0,
+
1485  storage_idx_t ni,
+
1486  float *x) const
+
1487 {
+
1488 #pragma omp parallel
+
1489  {
+
1490  std::vector<float> tmp(index.d);
+
1491 #pragma omp for
+
1492  for (storage_idx_t i = 0; i < ni; i++) {
+
1493  reconstruct(n0 + i, x + i * index.d, tmp.data());
+
1494  }
+
1495  }
+
1496 }
+
1497 
+
1498 size_t ReconstructFromNeighbors::compute_distances(size_t n, const idx_t *shortlist,
+
1499  const float *query, float *distances) const
+
1500 {
+
1501  std::vector<float> tmp(2 * index.d);
+
1502  size_t ncomp = 0;
+
1503  for (int i = 0; i < n; i++) {
+
1504  if (shortlist[i] < 0) break;
+
1505  reconstruct(shortlist[i], tmp.data(), tmp.data() + index.d);
+
1506  distances[i] = fvec_L2sqr(query, tmp.data(), index.d);
+
1507  ncomp++;
+
1508  }
+
1509  return ncomp;
+
1510 }
+
1511 
+
1512 void ReconstructFromNeighbors::get_neighbor_table(storage_idx_t i, float *tmp1) const
+
1513 {
+
1514  const HNSW & hnsw = index.hnsw;
+
1515  size_t begin, end;
+
1516  hnsw.neighbor_range(i, 0, &begin, &end);
+
1517  size_t d = index.d;
+
1518 
+
1519  index.storage->reconstruct(i, tmp1);
+
1520 
+
1521  for (size_t j = begin; j < end; j++) {
+
1522  storage_idx_t ji = hnsw.neighbors[j];
+
1523  if (ji < 0) ji = i;
+
1524  index.storage->reconstruct(ji, tmp1 + (j - begin + 1) * d);
+
1525  }
+
1526 
+
1527 }
+
1528 
+
1529 
+
1530 /// called by add_codes
+
1531 void ReconstructFromNeighbors::estimate_code(
+
1532  const float *x, storage_idx_t i, uint8_t *code) const
+
1533 {
+
1534 
+
1535  // fill in tmp table with the neighbor values
+
1536  float *tmp1 = new float[d * (M + 1) + (d * k)];
+
1537  float *tmp2 = tmp1 + d * (M + 1);
+
1538  ScopeDeleter<float> del(tmp1);
+
1539 
+
1540  // collect coordinates of base
+
1541  get_neighbor_table (i, tmp1);
+
1542 
+
1543  for (int sq = 0; sq < nsq; sq++) {
+
1544  int d0 = sq * dsub;
+
1545  int d1 = d0 + dsub;
+
1546 
+
1547  {
+
1548  FINTEGER ki = k, di = d, m1 = M + 1;
+
1549  FINTEGER dsubi = dsub;
+
1550  float zero = 0, one = 1;
+
1551 
+
1552  sgemm_ ("N", "N", &dsubi, &ki, &m1, &one,
+
1553  tmp1 + d0, &di,
+
1554  codebook.data() + sq * (m1 * k), &m1,
+
1555  &zero, tmp2, &dsubi);
+
1556  }
+
1557 
+
1558  float min = HUGE_VAL;
+
1559  int argmin = -1;
+
1560  for (int j = 0; j < k; j++) {
+
1561  float dis = fvec_L2sqr(x + d0, tmp2 + j * dsub, dsub);
+
1562  if (dis < min) {
+
1563  min = dis;
+
1564  argmin = j;
+
1565  }
+
1566  }
+
1567  code[sq] = argmin;
+
1568  }
+
1569 
+
1570 }
+
1571 
+
1572 void ReconstructFromNeighbors::add_codes(size_t n, const float *x)
+
1573 {
+
1574  if (k == 1) { // nothing to encode
+
1575  ntotal += n;
+
1576  return;
+
1577  }
+
1578  codes.resize(codes.size() + code_size * n);
+
1579 #pragma omp parallel for
+
1580  for (int i = 0; i < n; i++) {
+
1581  estimate_code(x + i * index.d, ntotal + i,
+
1582  codes.data() + (ntotal + i) * code_size);
+
1583  }
+
1584  ntotal += n;
+
1585  FAISS_ASSERT (codes.size() == ntotal * code_size);
+
1586 }
+
1587 
+
1588 
+
1589 /**************************************************************
+
1590  * IndexHNSWFlat implementation
+
1591  **************************************************************/
+
1592 
+
1593 
+
1594 struct FlatL2Dis: HNSW::DistanceComputer {
+
1595  Index::idx_t nb;
+
1596  const float *q;
+
1597  const float *b;
+
1598  size_t ndis;
+
1599 
+
1600  float operator () (storage_idx_t i) override
+
1601  {
+
1602  ndis++;
+
1603  return (fvec_L2sqr(q, b + i * d, d));
+
1604  }
+
1605 
+
1606  float symmetric_dis(storage_idx_t i, storage_idx_t j) override
+
1607  {
+
1608  return (fvec_L2sqr(b + j * d, b + i * d, d));
+
1609  }
+
1610 
+
1611 
+
1612  FlatL2Dis(const IndexFlatL2 & storage, const float *q = nullptr):
+
1613  q(q)
+
1614  {
+
1615  nb = storage.ntotal;
+
1616  d = storage.d;
+
1617  b = storage.xb.data();
+
1618  ndis = 0;
+
1619  }
+
1620 
+
1621  void set_query(const float *x) override {
+
1622  q = x;
+
1623  }
+
1624 
+
1625  virtual ~FlatL2Dis () {
+
1626 #pragma omp critical
+
1627  {
+
1628  hnsw_stats.ndis += ndis;
+
1629  }
+
1630  }
+
1631 };
+
1632 
+
1633 
+
1634 IndexHNSWFlat::IndexHNSWFlat()
+
1635 {
+
1636  is_trained = true;
+
1637 }
+
1638 
+
1639 
+
1640 IndexHNSWFlat::IndexHNSWFlat(int d, int M):
+
1641  IndexHNSW(new IndexFlatL2(d), M)
+
1642 {
+
1643  own_fields = true;
+
1644  is_trained = true;
+
1645 }
+
1646 
+
1647 
+
1648 HNSW::DistanceComputer * IndexHNSWFlat::get_distance_computer () const
+
1649 {
+
1650  return new FlatL2Dis (*dynamic_cast<IndexFlatL2*> (storage));
+
1651 }
+
1652 
+
1653 
+
1654 
+
1655 
+
1656 /**************************************************************
+
1657  * IndexHNSWPQ implementation
+
1658  **************************************************************/
+
1659 
+
1660 
+
1661 struct PQDis: HNSW::DistanceComputer {
+
1662  Index::idx_t nb;
+
1663  const uint8_t *codes;
+
1664  size_t code_size;
+
1665  const ProductQuantizer & pq;
+
1666  const float *sdc;
+
1667  std::vector<float> precomputed_table;
+
1668  size_t ndis;
+
1669 
+
1670  float operator () (storage_idx_t i) override
+
1671  {
+
1672  const uint8_t *code = codes + i * code_size;
+
1673  const float *dt = precomputed_table.data();
+
1674  float accu = 0;
+
1675  for (int j = 0; j < pq.M; j++) {
+
1676  accu += dt[*code++];
+
1677  dt += 256;
+
1678  }
+
1679  ndis++;
+
1680  return accu;
+
1681  }
+
1682 
+
1683  float symmetric_dis(storage_idx_t i, storage_idx_t j) override
+
1684  {
+
1685  const float * sdci = sdc;
+
1686  float accu = 0;
+
1687  const uint8_t *codei = codes + i * code_size;
+
1688  const uint8_t *codej = codes + j * code_size;
+
1689 
+
1690  for (int l = 0; l < pq.M; l++) {
+
1691  accu += sdci[(*codei++) + (*codej++) * 256];
+
1692  sdci += 256 * 256;
+
1693  }
+
1694  return accu;
+
1695  }
+
1696 
+
1697 
+
1698  PQDis(const IndexPQ & storage, const float *q = nullptr):
+
1699  pq(storage.pq)
+
1700  {
+
1701  precomputed_table.resize(pq.M * pq.ksub);
+
1702  nb = storage.ntotal;
+
1703  d = storage.d;
+
1704  codes = storage.codes.data();
+
1705  code_size = pq.code_size;
+
1706  FAISS_ASSERT(pq.ksub == 256);
+
1707  FAISS_ASSERT(pq.sdc_table.size() == pq.ksub * pq.ksub * pq.M);
+
1708  sdc = pq.sdc_table.data();
+
1709  ndis = 0;
+
1710  }
+
1711 
+
1712  void set_query(const float *x) override {
+
1713  pq.compute_distance_table(x, precomputed_table.data());
+
1714  }
+
1715 
+
1716  virtual ~PQDis () {
+
1717 #pragma omp critical
+
1718  {
+
1719  hnsw_stats.ndis += ndis;
+
1720  }
+
1721  }
+
1722 };
+
1723 
+
1724 IndexHNSWPQ::IndexHNSWPQ() {}
+
1725 
+
1726 IndexHNSWPQ::IndexHNSWPQ(int d, int pq_m, int M):
+
1727  IndexHNSW(new IndexPQ(d, pq_m, 8), M)
+
1728 {
+
1729  own_fields = true;
+
1730  is_trained = false;
+
1731 }
+
1732 
+
1733 void IndexHNSWPQ::train(idx_t n, const float* x)
+
1734 {
+
1735  IndexHNSW::train (n, x);
+
1736  (dynamic_cast<IndexPQ*> (storage))->pq.compute_sdc_table();
+
1737 }
+
1738 
+
1739 
+
1740 
+
1741 HNSW::DistanceComputer * IndexHNSWPQ::get_distance_computer () const
+
1742 {
+
1743  return new PQDis (*dynamic_cast<IndexPQ*> (storage));
+
1744 }
+
1745 
+
1746 
+
1747 /**************************************************************
+
1748  * IndexHNSWSQ implementation
+
1749  **************************************************************/
+
1750 
+
1751 
+
1752 struct SQDis: HNSW::DistanceComputer {
+
1753  Index::idx_t nb;
+
1754  const uint8_t *codes;
+
1755  size_t code_size;
+
1756  const ScalarQuantizer & sq;
+
1757  const float *q;
+
1758  ScalarQuantizer::DistanceComputer * dc;
+
1759 
+
1760  float operator () (storage_idx_t i) override
+
1761  {
+
1762  const uint8_t *code = codes + i * code_size;
+
1763 
+
1764  return dc->compute_distance (q, code);
+
1765  }
+
1766 
+
1767  float symmetric_dis(storage_idx_t i, storage_idx_t j) override
+
1768  {
+
1769  const uint8_t *codei = codes + i * code_size;
+
1770  const uint8_t *codej = codes + j * code_size;
+
1771  return dc->compute_code_distance (codei, codej);
+
1772  }
+
1773 
+
1774 
+
1775  SQDis(const IndexScalarQuantizer & storage, const float *q = nullptr):
+
1776  sq(storage.sq)
+
1777  {
+
1778  nb = storage.ntotal;
+
1779  d = storage.d;
+
1780  codes = storage.codes.data();
+
1781  code_size = sq.code_size;
+
1782  dc = sq.get_distance_computer();
+
1783  }
+
1784 
+
1785  void set_query(const float *x) override {
+
1786  q = x;
+
1787  }
+
1788 
+
1789  virtual ~SQDis () {
+
1790  delete dc;
+
1791  }
+
1792 };
+
1793 
+
1794 IndexHNSWSQ::IndexHNSWSQ(int d, ScalarQuantizer::QuantizerType qtype, int M):
+
1795  IndexHNSW (new IndexScalarQuantizer (d, qtype), M)
+
1796 {
+
1797  own_fields = true;
+
1798 }
+
1799 
+
1800 IndexHNSWSQ::IndexHNSWSQ() {}
+
1801 
+
1802 HNSW::DistanceComputer * IndexHNSWSQ::get_distance_computer () const
+
1803 {
+
1804  return new SQDis (*dynamic_cast<IndexScalarQuantizer*> (storage));
+
1805 }
+
1806 
+
1807 
+
1808 
+
1809 
+
1810 /**************************************************************
+
1811  * IndexHNSW2Level implementation
+
1812  **************************************************************/
+
1813 
+
1814 
+
1815 
+
1816 IndexHNSW2Level::IndexHNSW2Level(Index *quantizer, size_t nlist, int m_pq, int M):
+
1817  IndexHNSW (new Index2Layer (quantizer, nlist, m_pq), M)
+
1818 {
+
1819  own_fields = true;
+
1820  is_trained = false;
+
1821 }
+
1822 
+
1823 IndexHNSW2Level::IndexHNSW2Level() {}
+
1824 
+
1825 struct Distance2Level: HNSW::DistanceComputer {
+
1826 
+
1827  const Index2Layer & storage;
+
1828  std::vector<float> buf;
+
1829  const float *q;
+
1830 
+
1831  const float *pq_l1_tab, *pq_l2_tab;
+
1832 
+
1833  Distance2Level(const Index2Layer & storage): storage(storage)
+
1834  {
+
1835  d = storage.d;
+
1836  FAISS_ASSERT(storage.pq.dsub == 4);
+
1837  pq_l2_tab = storage.pq.centroids.data();
+
1838  buf.resize(2 * d);
+
1839  }
+
1840 
+
1841  float symmetric_dis(storage_idx_t i, storage_idx_t j) override
+
1842  {
+
1843  storage.reconstruct(i, buf.data());
+
1844  storage.reconstruct(j, buf.data() + d);
+
1845  return fvec_L2sqr(buf.data() + d, buf.data(), d);
+
1846  }
+
1847 
+
1848  void set_query(const float *x) override {
+
1849  q = x;
+
1850  }
+
1851 
+
1852 
+
1853 };
+
1854 
+
1855 
+
1856 // well optimized for xNN+PQNN
+
1857 struct DistanceXPQ4: Distance2Level {
+
1858 
+
1859  int M, k;
+
1860 
+
1861  DistanceXPQ4(const Index2Layer & storage):
+
1862  Distance2Level (storage)
+
1863  {
+
1864  const IndexFlat *quantizer =
+
1865  dynamic_cast<IndexFlat*> (storage.q1.quantizer);
+
1866 
+
1867  FAISS_ASSERT(quantizer);
+
1868  M = storage.pq.M;
+
1869  pq_l1_tab = quantizer->xb.data();
+
1870  }
+
1871 
+
1872  float operator () (storage_idx_t i) override
+
1873  {
+
1874  const uint8_t *code = storage.codes.data() + i * storage.code_size;
+
1875  long key = 0;
+
1876  memcpy (&key, code, storage.code_size_1);
+
1877  code += storage.code_size_1;
+
1878 
+
1879  // walking pointers
+
1880  const float *qa = q;
+
1881  const __m128 *l1_t = (const __m128 *)(pq_l1_tab + d * key);
+
1882  const __m128 *pq_l2_t = (const __m128 *)pq_l2_tab;
+
1883  __m128 accu = _mm_setzero_ps();
+
1884 
+
1885  for (int m = 0; m < M; m++) {
+
1886  __m128 qi = _mm_loadu_ps(qa);
+
1887  __m128 recons = l1_t[m] + pq_l2_t[*code++];
+
1888  __m128 diff = qi - recons;
+
1889  accu += diff * diff;
+
1890  pq_l2_t += 256;
+
1891  qa += 4;
+
1892  }
+
1893 
+
1894  accu = _mm_hadd_ps (accu, accu);
+
1895  accu = _mm_hadd_ps (accu, accu);
+
1896  return _mm_cvtss_f32 (accu);
+
1897  }
+
1898 
+
1899 };
+
1900 
+
1901 // well optimized for 2xNN+PQNN
+
1902 struct Distance2xXPQ4: Distance2Level {
+
1903 
+
1904  int M_2, mi_nbits;
+
1905 
+
1906  Distance2xXPQ4(const Index2Layer & storage):
+
1907  Distance2Level (storage)
+
1908  {
+
1909  const MultiIndexQuantizer *mi =
+
1910  dynamic_cast<MultiIndexQuantizer*> (storage.q1.quantizer);
+
1911 
+
1912  FAISS_ASSERT(mi);
+
1913  FAISS_ASSERT(storage.pq.M % 2 == 0);
+
1914  M_2 = storage.pq.M / 2;
+
1915  mi_nbits = mi->pq.nbits;
+
1916  pq_l1_tab = mi->pq.centroids.data();
+
1917  }
+
1918 
+
1919  float operator () (storage_idx_t i) override
+
1920  {
+
1921  const uint8_t *code = storage.codes.data() + i * storage.code_size;
+
1922  long key01 = 0;
+
1923  memcpy (&key01, code, storage.code_size_1);
+
1924  code += storage.code_size_1;
+
1925 
+
1926  // walking pointers
+
1927  const float *qa = q;
+
1928  const __m128 *pq_l1_t = (const __m128 *)pq_l1_tab;
+
1929  const __m128 *pq_l2_t = (const __m128 *)pq_l2_tab;
+
1930  __m128 accu = _mm_setzero_ps();
+
1931 
+
1932  for (int mi_m = 0; mi_m < 2; mi_m++) {
+
1933  long l1_idx = key01 & ((1L << mi_nbits) - 1);
+
1934  const __m128 * pq_l1 = pq_l1_t + M_2 * l1_idx;
+
1935 
+
1936  for (int m = 0; m < M_2; m++) {
+
1937  __m128 qi = _mm_loadu_ps(qa);
+
1938  __m128 recons = pq_l1[m] + pq_l2_t[*code++];
+
1939  __m128 diff = qi - recons;
+
1940  accu += diff * diff;
+
1941  pq_l2_t += 256;
+
1942  qa += 4;
+
1943  }
+
1944  pq_l1_t += M_2 << mi_nbits;
+
1945  key01 >>= mi_nbits;
+
1946  }
+
1947  accu = _mm_hadd_ps (accu, accu);
+
1948  accu = _mm_hadd_ps (accu, accu);
+
1949  return _mm_cvtss_f32 (accu);
+
1950  }
+
1951 
+
1952 };
+
1953 
+
1954 
+
1955 
+
1956 HNSW::DistanceComputer * IndexHNSW2Level::get_distance_computer () const
+
1957 {
+
1958  const Index2Layer *storage2l =
+
1959  dynamic_cast<Index2Layer*>(storage);
+
1960 
+
1961  if (storage2l) {
+
1962 
+
1963  const MultiIndexQuantizer *mi =
+
1964  dynamic_cast<MultiIndexQuantizer*> (storage2l->q1.quantizer);
+
1965 
+
1966  if (mi && storage2l->pq.M % 2 == 0 && storage2l->pq.dsub == 4) {
+
1967  return new Distance2xXPQ4(*storage2l);
+
1968  }
+
1969 
+
1970  const IndexFlat *fl =
+
1971  dynamic_cast<IndexFlat*> (storage2l->q1.quantizer);
+
1972 
+
1973  if (fl && storage2l->pq.dsub == 4) {
+
1974  return new DistanceXPQ4(*storage2l);
+
1975  }
+
1976  }
+
1977 
+
1978  // IVFPQ and cases not handled above
+
1979  return new GenericDistanceComputer (*storage);
+
1980 
+
1981 }
+
1982 
+
1983 
+
1984 namespace {
+
1985 
+
1986 
+
1987 // same as search_from_candidates but uses v
+
1988 // visno -> is in result list
+
1989 // visno + 1 -> in result list + in candidates
+
1990 int search_from_candidates_2(const HNSW & hnsw,
+
1991  DistanceComputer & qdis, int k,
+
1992  idx_t *I, float * D,
+
1993  MinimaxHeap &candidates,
+
1994  VisitedTable &vt,
+
1995  int level, int nres_in = 0)
+
1996 {
+
1997  int nres = nres_in;
+
1998  int ndis = 0;
+
1999  for (int i = 0; i < candidates.size(); i++) {
+
2000  idx_t v1 = candidates.ids[i];
+
2001  float d = candidates.dis[i];
+
2002  FAISS_ASSERT(v1 >= 0);
+
2003  vt.visited[v1] = vt.visno + 1;
+
2004  }
+
2005 
+
2006  bool do_dis_check = hnsw.check_relative_distance;
+
2007  int nstep = 0;
+
2008 
+
2009  while (candidates.size() > 0) {
+
2010  float d0 = 0;
+
2011  int v0 = candidates.pop_min(&d0);
+
2012 
+
2013  if (do_dis_check) {
+
2014  // tricky stopping condition: there are more that ef
+
2015  // distances that are processed already that are smaller
+
2016  // than d0
+
2017 
+
2018  int n_dis_below = candidates.count_below(d0);
+
2019  if(n_dis_below >= hnsw.efSearch) {
+
2020  break;
+
2021  }
+
2022  }
+
2023  size_t begin, end;
+
2024  hnsw.neighbor_range(v0, level, &begin, &end);
+
2025 
+
2026  for (size_t j = begin; j < end; j++) {
+
2027  int v1 = hnsw.neighbors[j];
+
2028  if (v1 < 0) break;
+
2029  if (vt.visited[v1] == vt.visno + 1) {
+
2030  // nothing to do
+
2031  } else {
+
2032  ndis++;
+
2033  float d = qdis(v1);
+
2034  candidates.push(v1, d);
+
2035 
+
2036  // never seen before --> add to heap
+
2037  if (vt.visited[v1] < vt.visno) {
+
2038  if (nres < k) {
+
2039  faiss::maxheap_push (++nres, D, I, d, v1);
+
2040  } else if (d < D[0]) {
+
2041  faiss::maxheap_pop (nres--, D, I);
+
2042  faiss::maxheap_push (++nres, D, I, d, v1);
+
2043  }
+
2044  }
+
2045  vt.visited[v1] = vt.visno + 1;
+
2046  }
+
2047  }
+
2048 
+
2049  nstep++;
+
2050  if (!do_dis_check && nstep > hnsw.efSearch) {
+
2051  break;
+
2052  }
+
2053  }
+
2054 
+
2055  if (level == 0) {
+
2056 #pragma omp critical
+
2057  {
+
2058  hnsw_stats.n1 ++;
+
2059  if (candidates.size() == 0)
+
2060  hnsw_stats.n2 ++;
+
2061  }
+
2062  }
+
2063 
+
2064 
+
2065  return nres;
+
2066 }
+
2067 
+
2068 
+
2069 } // anonymous namespace
+
2070 
+
2071 void IndexHNSW2Level::search (idx_t n, const float *x, idx_t k,
+
2072  float *distances, idx_t *labels) const
+
2073 {
+
2074  if (dynamic_cast<const Index2Layer*>(storage)) {
+
2075  IndexHNSW::search (n, x, k, distances, labels);
+
2076 
+
2077  } else { // "mixed" search
+
2078 
+
2079  const IndexIVFPQ *index_ivfpq =
+
2080  dynamic_cast<const IndexIVFPQ*>(storage);
+
2081 
+
2082  int nprobe = index_ivfpq->nprobe;
+
2083 
+
2084  long * coarse_assign = new long [n * nprobe];
+
2085  ScopeDeleter<long> del (coarse_assign);
+
2086  float * coarse_dis = new float [n * nprobe];
+
2087  ScopeDeleter<float> del2 (coarse_dis);
+
2088 
+
2089  index_ivfpq->quantizer->search (n, x, nprobe, coarse_dis, coarse_assign);
+
2090 
+
2091  index_ivfpq->search_preassigned (
+
2092  n, x, k, coarse_assign, coarse_dis, distances, labels, false);
+
2093 
+
2094 #pragma omp parallel
+
2095  {
+
2096  VisitedTable vt (ntotal);
+
2097  DistanceComputer *dis = get_distance_computer();
+
2098  ScopeDeleter1<DistanceComputer> del(dis);
+
2099 
+
2100  int candidates_size = hnsw.upper_beam;
+
2101  MinimaxHeap candidates(candidates_size);
+
2102 
+
2103 #pragma omp for
+
2104  for(int i = 0; i < n; i++) {
+
2105  idx_t * idxi = labels + i * k;
+
2106  float * simi = distances + i * k;
+
2107  dis->set_query(x + i * d);
+
2108 
+
2109  // mark all inverted list elements as visited
+
2110 
+
2111  for (int j = 0; j < nprobe; j++) {
+
2112  idx_t key = coarse_assign[j + i * nprobe];
+
2113  if (key < 0) break;
+
2114  const std::vector<idx_t> & ids = index_ivfpq->ids[key];
+
2115  for (int jj = 0; jj < ids.size(); jj++) {
+
2116  vt.set (ids[jj]);
+
2117  }
+
2118  }
+
2119 
+
2120  candidates.clear();
+
2121  // copy the upper_beam elements to candidates list
+
2122 
+
2123  int search_policy = 2;
+
2124 
+
2125  if (search_policy == 1) {
+
2126 
+
2127  for (int j = 0 ; j < hnsw.upper_beam && j < k; j++) {
+
2128  if (idxi[j] < 0) break;
+
2129  candidates.push (idxi[j], simi[j]);
+
2130  // search_from_candidates adds them back
+
2131  idxi[j] = -1;
+
2132  simi[j] = HUGE_VAL;
+
2133  }
+
2134 
+
2135  // reorder from sorted to heap
+
2136  maxheap_heapify (k, simi, idxi, simi, idxi, k);
+
2137 
+
2138  search_from_candidates (
+
2139  hnsw, *dis, k, idxi, simi,
+
2140  candidates, vt, 0, k);
+
2141 
+
2142  vt.advance();
+
2143 
+
2144  } else if (search_policy == 2) {
+
2145 
+
2146  for (int j = 0 ; j < hnsw.upper_beam && j < k; j++) {
+
2147  if (idxi[j] < 0) break;
+
2148  candidates.push (idxi[j], simi[j]);
+
2149  }
+
2150 
+
2151  // reorder from sorted to heap
+
2152  maxheap_heapify (k, simi, idxi, simi, idxi, k);
+
2153 
+
2154  search_from_candidates_2 (
+
2155  hnsw, *dis, k, idxi, simi,
+
2156  candidates, vt, 0, k);
+
2157  vt.advance ();
+
2158  vt.advance ();
+
2159 
+
2160  }
+
2161 
+
2162  maxheap_reorder (k, simi, idxi);
+
2163  }
+
2164  }
+
2165  }
+
2166 
+
2167 
+
2168 }
+
2169 
+
2170 
+
2171 void IndexHNSW2Level::flip_to_ivf ()
+
2172 {
+
2173  Index2Layer *storage2l =
+
2174  dynamic_cast<Index2Layer*>(storage);
+
2175 
+
2176  FAISS_THROW_IF_NOT (storage2l);
+
2177 
+
2178  IndexIVFPQ * index_ivfpq =
+
2179  new IndexIVFPQ (storage2l->q1.quantizer,
+
2180  d, storage2l->q1.nlist,
+
2181  storage2l->pq.M, 8);
+
2182  index_ivfpq->pq = storage2l->pq;
+
2183  index_ivfpq->is_trained = storage2l->is_trained;
+
2184  index_ivfpq->precompute_table();
+
2185  index_ivfpq->own_fields = storage2l->q1.own_fields;
+
2186  storage2l->transfer_to_IVFPQ(*index_ivfpq);
+
2187  index_ivfpq->make_direct_map (true);
+
2188 
+
2189  storage = index_ivfpq;
+
2190  delete storage2l;
+
2191 
+
2192 }
+
2193 
+
2194 
+
2195 } // namespace faiss
+
faiss::RandomGenerator
random generator that can be used in multithreaded contexts
Definition: utils.h:48
+
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:344
+
faiss::HNSW::add_with_locks
void add_with_locks(DistanceComputer &ptdis, int pt_level, int pt_id, std::vector< omp_lock_t > &locks, VisitedTable &vt)
Definition: IndexHNSW.cpp:657
+
faiss::IndexPQ::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexPQ.h:34
+
faiss::Index2Layer::transfer_to_IVFPQ
void transfer_to_IVFPQ(IndexIVFPQ &other) const
transfer the flat codes to an IVFPQ index
Definition: IndexIVFPQ.cpp:1416
+
faiss::ScalarQuantizer::code_size
size_t code_size
bytes per vector
Definition: IndexScalarQuantizer.h:60
+ + +
faiss::HNSW::neighbor_range
void neighbor_range(idx_t no, int layer_no, size_t *begin, size_t *end) const
range of entries in the neighbors table of vertex no at layer_no
Definition: IndexHNSW.cpp:517
+
faiss::ProductQuantizer::nbits
size_t nbits
number of bits per quantization index
Definition: ProductQuantizer.h:29
+
faiss::VisitedTable::get
bool get(int no) const
get flag #no
Definition: IndexHNSW.cpp:66
+
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils.cpp:574
+ +
faiss::IndexHNSWPQ::train
void train(idx_t n, const float *x) override
Trains the storage if needed.
Definition: IndexHNSW.cpp:1733
+ +
faiss::HNSW::nb_neighbors
int nb_neighbors(int layer_no) const
nb of neighbors for this level
Definition: IndexHNSW.cpp:497
+
faiss::IndexHNSW2Level::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
entry point for search
Definition: IndexHNSW.cpp:2071
+ +
faiss::Index2Layer::code_size
size_t code_size
code_size_1 + code_size_2
Definition: IndexIVFPQ.h:256
+
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
+
faiss::HNSW::entry_point
storage_idx_t entry_point
entry point in the search structure (one of the points with maximum level
Definition: IndexHNSW.h:88
+
faiss::HNSW::cum_nb_neighbors
int cum_nb_neighbors(int layer_no) const
cumumlative nb up to (and excluding) this level
Definition: IndexHNSW.cpp:512
+ +
faiss::ScalarQuantizer::DistanceComputer::compute_code_distance
virtual float compute_code_distance(const uint8_t *code1, const uint8_t *code2)=0
code-to-code distance computation
+ +
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:78
+ +
faiss::HNSW::assign_probas
std::vector< double > assign_probas
assignment probability to each layer (sum=1)
Definition: IndexHNSW.h:70
+
faiss::ProductQuantizer::sdc_table
std::vector< float > sdc_table
Symmetric Distance Table.
Definition: ProductQuantizer.h:155
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
+ +
faiss::RandomGenerator::rand_float
float rand_float()
between 0 and 1
Definition: utils.cpp:211
+ +
faiss::HNSW::cum_nneighbor_per_level
std::vector< int > cum_nneighbor_per_level
Definition: IndexHNSW.h:74
+ + +
faiss::ProductQuantizer::dsub
size_t dsub
dimensionality of each subvector
Definition: ProductQuantizer.h:32
+
faiss::Index2Layer::q1
Level1Quantizer q1
first level quantizer
Definition: IndexIVFPQ.h:241
+
faiss::Index2Layer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * code_size.
Definition: IndexIVFPQ.h:247
+ +
faiss::VisitedTable::advance
void advance()
reset all flags to false
Definition: IndexHNSW.cpp:70
+ +
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
+
faiss::ReconstructFromNeighbors::get_neighbor_table
void get_neighbor_table(storage_idx_t i, float *out) const
get the M+1 -by-d table for neighbor coordinates for vector i
Definition: IndexHNSW.cpp:1512
+
faiss::Distance2Level::set_query
void set_query(const float *x) override
called before computing distances
Definition: IndexHNSW.cpp:1848
+
faiss::IndexHNSW::train
void train(idx_t n, const float *x) override
Trains the storage if needed.
Definition: IndexHNSW.cpp:978
+
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
+
faiss::ProductQuantizer::compute_distance_table
void compute_distance_table(const float *x, float *dis_table) const
Definition: ProductQuantizer.cpp:410
+
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:34
+
faiss::HNSW::offsets
std::vector< size_t > offsets
Definition: IndexHNSW.h:81
+
faiss::IndexScalarQuantizer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexScalarQuantizer.h:104
+
faiss::VisitedTable
set implementation optimized for fast access.
Definition: IndexHNSW.cpp:54
+ +
faiss::IndexHNSW::add
void add(idx_t n, const float *x) override
Definition: IndexHNSW.cpp:1031
+
faiss::SQDis::set_query
void set_query(const float *x) override
called before computing distances
Definition: IndexHNSW.cpp:1785
+ +
faiss::ReconstructFromNeighbors::reconstruct
void reconstruct(storage_idx_t i, float *x, float *tmp) const
called by compute_distances
Definition: IndexHNSW.cpp:1378
+
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
+
faiss::RandomGenerator::rand_int
int rand_int()
random 31-bit positive integer
+
faiss::IndexScalarQuantizer::sq
ScalarQuantizer sq
Used to encode the vectors.
Definition: IndexScalarQuantizer.h:101
+
faiss::ProductQuantizer::ksub
size_t ksub
number of centroids for each subquantizer
Definition: ProductQuantizer.h:35
+
faiss::HNSW::efSearch
int efSearch
expansion factor at search time
Definition: IndexHNSW.h:99
+
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
+
faiss::HNSW::check_relative_distance
bool check_relative_distance
during search: do we check whether the next best distance is good enough?
Definition: IndexHNSW.h:102
+
faiss::HNSW::HNSW
HNSW(int M=32)
only mandatory parameter: nb of neighbors
Definition: IndexHNSW.cpp:527
+
faiss::IndexPQ::pq
ProductQuantizer pq
The product quantizer used to encode the vectors.
Definition: IndexPQ.h:31
+
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
+
faiss::HNSW::upper_beam
int upper_beam
number of entry points in levels &gt; 0.
Definition: IndexHNSW.h:105
+
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
+
faiss::HNSW::set_nb_neighbors
void set_nb_neighbors(int level_no, int n)
set nb of neighbors for this level (before adding anything)
Definition: IndexHNSW.cpp:503
+
faiss::ReconstructFromNeighbors::estimate_code
void estimate_code(const float *x, storage_idx_t i, uint8_t *code) const
called by add_codes
Definition: IndexHNSW.cpp:1531
+
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
+ + +
faiss::HNSW::idx_t
faiss::Index::idx_t idx_t
Faiss results are 64-bit.
Definition: IndexHNSW.h:46
+
faiss::ReconstructFromNeighbors::add_codes
void add_codes(size_t n, const float *x)
Definition: IndexHNSW.cpp:1572
+
faiss::IndexIVF::make_direct_map
void make_direct_map(bool new_maintain_direct_map=true)
Definition: IndexIVF.cpp:139
+
faiss::GenericDistanceComputer::set_query
void set_query(const float *x) override
called before computing distances
Definition: IndexHNSW.cpp:1347
+
faiss::HNSW::random_level
int random_level()
pick a random level for a new point
Definition: IndexHNSW.cpp:539
+
faiss::HNSW::set_default_probas
void set_default_probas(int M, float levelMult)
Definition: IndexHNSW.cpp:553
+
faiss::IndexIVFPQ::pq
ProductQuantizer pq
produces the codes
Definition: IndexIVFPQ.h:32
+
faiss::IndexHNSW::reset
void reset() override
removes all elements from the database.
Definition: IndexHNSW.cpp:1043
+
faiss::ProductQuantizer::M
size_t M
number of subquantizers
Definition: ProductQuantizer.h:28
+
faiss::Index2Layer::code_size_1
size_t code_size_1
size of the code for the first level (ceil(log8(q1.nlist)))
Definition: IndexIVFPQ.h:250
+
faiss::HNSW::search
void search(DistanceComputer &qdis, int k, idx_t *I, float *D, VisitedTable &vt) const
search interface
Definition: IndexHNSW.cpp:711
+ + +
faiss::IndexHNSW::init_level_0_from_entry_points
void init_level_0_from_entry_points(int npt, const storage_idx_t *points, const storage_idx_t *nearests)
alternative graph building
Definition: IndexHNSW.cpp:1200
+
faiss::FlatL2Dis::set_query
void set_query(const float *x) override
called before computing distances
Definition: IndexHNSW.cpp:1621
+
faiss::IndexHNSW::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
entry point for search
Definition: IndexHNSW.cpp:985
+
faiss::IndexIVFPQ::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:919
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
+
faiss::IndexHNSW::init_level_0_from_knngraph
void init_level_0_from_knngraph(int k, const float *D, const idx_t *I)
alternative graph building
Definition: IndexHNSW.cpp:1162
+ +
faiss::IndexHNSW::search_level_0
void search_level_0(idx_t n, const float *x, idx_t k, const storage_idx_t *nearest, const float *nearest_d, float *distances, idx_t *labels, int nprobe=1, int search_type=1) const
Definition: IndexHNSW.cpp:1092
+
faiss::HNSW::fill_with_random_links
void fill_with_random_links(size_t n)
add n random levels to table (for debugging...)
Definition: IndexHNSW.cpp:913
+
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
+
faiss::HNSW::neighbors
std::vector< storage_idx_t > neighbors
Definition: IndexHNSW.h:85
+
faiss::IndexHNSW::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexHNSW.cpp:1050
+ +
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:54
+
faiss::Index2Layer::pq
ProductQuantizer pq
second level quantizer is always a PQ
Definition: IndexIVFPQ.h:244
+
faiss::HNSW::efConstruction
int efConstruction
expansion factor at construction time
Definition: IndexHNSW.h:96
+
faiss::HNSW::storage_idx_t
int storage_idx_t
internal storage of vectors (32 bits: this is expensive)
Definition: IndexHNSW.h:43
+ +
faiss::IndexFlat::xb
std::vector< float > xb
database vectors, size ntotal * d
Definition: IndexFlat.h:25
+
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:42
+
faiss::VisitedTable::set
void set(int no)
set flog #no to true
Definition: IndexHNSW.cpp:62
+ +
faiss::PQDis::set_query
void set_query(const float *x) override
called before computing distances
Definition: IndexHNSW.cpp:1712
+
faiss::Index2Layer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVFPQ.cpp:1442
+
faiss::HNSW::levels
std::vector< int > levels
level of each vector (base level = 1), size = ntotal
Definition: IndexHNSW.h:77
+
faiss::HNSW::max_level
int max_level
maximum level
Definition: IndexHNSW.h:93
+
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
+
faiss::ScalarQuantizer::DistanceComputer::compute_distance
virtual float compute_distance(const float *x, const uint8_t *code)=0
vector-to-code distance computation
+
faiss::ProductQuantizer::centroids
std::vector< float > centroids
Centroid table, size M * ksub * dsub.
Definition: ProductQuantizer.h:52
+
+ +
+Generated by   +doxygen + 1.8.5 +
+ + diff --git a/docs/html/IndexHNSW_8h_source.html b/docs/html/IndexHNSW_8h_source.html new file mode 100644 index 000000000..87dd75deb --- /dev/null +++ b/docs/html/IndexHNSW_8h_source.html @@ -0,0 +1,473 @@ + + + + + + +Faiss: /data/users/matthijs/github_faiss/faiss/IndexHNSW.h Source File + + + + + + + + + +
+
+ + + + + + +
+
Faiss +
+
+
+ + + + + +
+ + + + +
+ +
+ +
+
+
/data/users/matthijs/github_faiss/faiss/IndexHNSW.h
+
+
+
1 /**
+
2  * Copyright (c) 2015-present, Facebook, Inc.
+
3  * All rights reserved.
+
4  *
+
5  * This source code is licensed under the BSD+Patents license found in the
+
6  * LICENSE file in the root directory of this source tree.
+
7  */
+
8 
+
9 #pragma once
+
10 
+
11 #include <vector>
+
12 #include <omp.h>
+
13 
+
14 #include "IndexFlat.h"
+
15 #include "IndexPQ.h"
+
16 #include "IndexScalarQuantizer.h"
+
17 #include "utils.h"
+
18 
+
19 
+
20 namespace faiss {
+
21 
+
22 /** Implementation of the Hierarchical Navigable Small World
+
23  * datastructure.
+
24  *
+
25  * Efficient and robust approximate nearest neighbor search using
+
26  * Hierarchical Navigable Small World graphs
+
27  *
+
28  * Yu. A. Malkov, D. A. Yashunin, arXiv 2017
+
29  *
+
30  * This implmentation is heavily influenced by the NMSlib
+
31  * implementation by Yury Malkov and Leonid Boystov
+
32  * (https://github.com/searchivarius/nmslib)
+
33  *
+
34  * The HNSW object stores only the neighbor link structure, see
+
35  * IndexHNSW below for the full index object.
+
36  */
+
37 
+
38 struct VisitedTable;
+
39 
+
40 struct HNSW {
+
41 
+
42  /// internal storage of vectors (32 bits: this is expensive)
+
43  typedef int storage_idx_t;
+
44 
+
45  /// Faiss results are 64-bit
+
46  typedef faiss::Index::idx_t idx_t;
+
47 
+
48  /** The HNSW structure does not store vectors, it only accesses
+
49  * them through this class.
+
50  *
+
51  * Functions are guaranteed to be be accessed only from 1 thread. */
+
52  struct DistanceComputer {
+
53 
+
54  idx_t d;
+
55 
+
56  /// called before computing distances
+
57  virtual void set_query (const float *x) = 0;
+
58 
+
59  /// compute distance of vector i to current query
+
60  virtual float operator () (storage_idx_t i) = 0;
+
61 
+
62  /// compute distance between two stored vectors
+
63  virtual float symmetric_dis(storage_idx_t i, storage_idx_t j) = 0;
+
64 
+
65  virtual ~DistanceComputer () {}
+
66 
+
67  };
+
68 
+
69  /// assignment probability to each layer (sum=1)
+
70  std::vector<double> assign_probas;
+
71 
+
72  /// number of neighbors stored per layer (cumulative), should not
+
73  /// be changed after first add
+
74  std::vector<int> cum_nneighbor_per_level;
+
75 
+
76  /// level of each vector (base level = 1), size = ntotal
+
77  std::vector<int> levels;
+
78 
+
79  /// offsets[i] is the offset in the neighbors array where vector i is stored
+
80  /// size ntotal + 1
+
81  std::vector<size_t> offsets;
+
82 
+
83  /// neighbors[offsets[i]:offsets[i+1]] is the list of neighbors of vector i
+
84  /// for all levels. this is where all storage goes.
+
85  std::vector<storage_idx_t> neighbors;
+
86 
+
87  /// entry point in the search structure (one of the points with maximum level
+
88  storage_idx_t entry_point;
+
89 
+
90  faiss::RandomGenerator rng;
+
91 
+
92  /// maximum level
+
93  int max_level;
+
94 
+
95  /// expansion factor at construction time
+
96  int efConstruction;
+
97 
+
98  /// expansion factor at search time
+
99  int efSearch;
+
100 
+
101  /// during search: do we check whether the next best distance is good enough?
+
102  bool check_relative_distance;
+
103 
+
104  /// number of entry points in levels > 0.
+
105  int upper_beam;
+
106 
+
107  // methods that initialize the tree sizes
+
108 
+
109  /// initialize the assign_probas and cum_nneighbor_per_level to
+
110  /// have 2*M links on level 0 and M links on levels > 0
+
111  void set_default_probas(int M, float levelMult);
+
112 
+
113  /// set nb of neighbors for this level (before adding anything)
+
114  void set_nb_neighbors(int level_no, int n);
+
115 
+
116  // methods that access the tree sizes
+
117 
+
118  /// nb of neighbors for this level
+
119  int nb_neighbors(int layer_no) const;
+
120 
+
121  /// cumumlative nb up to (and excluding) this level
+
122  int cum_nb_neighbors(int layer_no) const;
+
123 
+
124  /// range of entries in the neighbors table of vertex no at layer_no
+
125  void neighbor_range(idx_t no, int layer_no,
+
126  size_t * begin, size_t * end) const;
+
127 
+
128  /// only mandatory parameter: nb of neighbors
+
129  explicit HNSW(int M = 32);
+
130 
+
131  /// pick a random level for a new point
+
132  int random_level();
+
133 
+
134  /// add n random levels to table (for debugging...)
+
135  void fill_with_random_links(size_t n);
+
136 
+
137  /** add point pt_id on all levels <= pt_level and build the link
+
138  * structure for them. */
+
139  void add_with_locks(DistanceComputer & ptdis, int pt_level, int pt_id,
+
140  std::vector<omp_lock_t> & locks,
+
141  VisitedTable &vt);
+
142 
+
143 
+
144  /// search interface
+
145  void search(DistanceComputer & qdis, int k,
+
146  idx_t *I, float * D,
+
147  VisitedTable &vt) const;
+
148 
+
149  void reset();
+
150 
+
151  void clear_neighbor_tables(int level);
+
152  void print_neighbor_stats(int level) const;
+
153 };
+
154 
+
155 
+
156 struct HNSWStats {
+
157  size_t n1, n2, n3;
+
158  size_t ndis;
+
159  size_t nreorder;
+
160  bool view;
+
161 
+
162  HNSWStats () {reset (); }
+
163  void reset ();
+
164 };
+
165 
+
166 // global var that collects them all
+
167 extern HNSWStats hnsw_stats;
+
168 
+
169 class IndexHNSW;
+
170 
+
171 struct ReconstructFromNeighbors {
+
172  typedef Index::idx_t idx_t;
+
173  typedef HNSW::storage_idx_t storage_idx_t;
+
174 
+
175  const IndexHNSW & index;
+
176  size_t M; // number of neighbors
+
177  size_t k; // number of codebook entries
+
178  size_t nsq; // number of subvectors
+
179  size_t code_size;
+
180  int k_reorder; // nb to reorder. -1 = all
+
181 
+
182  std::vector<float> codebook; // size nsq * k * (M + 1)
+
183 
+
184  std::vector<uint8_t> codes; // size ntotal * code_size
+
185  size_t ntotal;
+
186  size_t d, dsub; // derived values
+
187 
+
188  ReconstructFromNeighbors(const IndexHNSW & index,
+
189  size_t k=256, size_t nsq=1);
+
190 
+
191  /// codes must be added in the correct order and the IndexHNSW
+
192  /// must be populated and sorted
+
193  void add_codes(size_t n, const float *x);
+
194 
+
195  size_t compute_distances(size_t n, const idx_t *shortlist,
+
196  const float *query, float *distances) const;
+
197 
+
198  /// called by add_codes
+
199  void estimate_code(const float *x, storage_idx_t i, uint8_t *code) const;
+
200 
+
201  /// called by compute_distances
+
202  void reconstruct(storage_idx_t i, float *x, float *tmp) const;
+
203 
+
204  void reconstruct_n(storage_idx_t n0, storage_idx_t ni, float *x) const;
+
205 
+
206  /// get the M+1 -by-d table for neighbor coordinates for vector i
+
207  void get_neighbor_table(storage_idx_t i, float *out) const;
+
208 
+
209 };
+
210 
+
211 
+
212 /** The HNSW index is a normal random-access index with a HNSW
+
213  * link structure built on top */
+
214 
+
215 struct IndexHNSW: Index {
+
216 
+
217  typedef HNSW::storage_idx_t storage_idx_t;
+
218 
+
219  // the link strcuture
+
220  HNSW hnsw;
+
221 
+
222  // the sequential storage
+
223  bool own_fields;
+
224  Index * storage;
+
225 
+
226  ReconstructFromNeighbors *reconstruct_from_neighbors;
+
227 
+
228  explicit IndexHNSW (int d = 0, int M = 32);
+
229  explicit IndexHNSW (Index * storage, int M = 32);
+
230 
+
231  ~IndexHNSW() override;
+
232 
+
233  // get a DistanceComputer object for this kind of storage
+
234  virtual HNSW::DistanceComputer * get_distance_computer() const = 0;
+
235 
+
236  void add(idx_t n, const float *x) override;
+
237 
+
238  /// Trains the storage if needed
+
239  void train(idx_t n, const float* x) override;
+
240 
+
241  /// entry point for search
+
242  void search (idx_t n, const float *x, idx_t k,
+
243  float *distances, idx_t *labels) const override;
+
244 
+
245  void reconstruct(idx_t key, float* recons) const override;
+
246 
+
247  void reset () override;
+
248 
+
249  void shrink_level_0_neighbors(int size);
+
250 
+
251  /** Perform search only on level 0, given the starting points for
+
252  * each vertex.
+
253  *
+
254  * @param search_type 1:perform one search per nprobe, 2: enqueue
+
255  * all entry points
+
256  */
+
257  void search_level_0(idx_t n, const float *x, idx_t k,
+
258  const storage_idx_t *nearest, const float *nearest_d,
+
259  float *distances, idx_t *labels, int nprobe = 1,
+
260  int search_type = 1) const;
+
261 
+
262  /// alternative graph building
+
263  void init_level_0_from_knngraph(
+
264  int k, const float *D, const idx_t *I);
+
265 
+
266  /// alternative graph building
+
267  void init_level_0_from_entry_points(
+
268  int npt, const storage_idx_t *points,
+
269  const storage_idx_t *nearests);
+
270 
+
271  // reorder links from nearest to farthest
+
272  void reorder_links();
+
273 
+
274  void link_singletons();
+
275 };
+
276 
+
277 
+
278 
+
279 /** Flat index topped with with a HNSW structure to access elements
+
280  * more efficiently.
+
281  */
+
282 
+
283 struct IndexHNSWFlat: IndexHNSW {
+
284  IndexHNSWFlat();
+
285  IndexHNSWFlat(int d, int M);
+
286  HNSW::DistanceComputer * get_distance_computer() const override;
+
287 };
+
288 
+
289 /** PQ index topped with with a HNSW structure to access elements
+
290  * more efficiently.
+
291  */
+
292 struct IndexHNSWPQ: IndexHNSW {
+
293  IndexHNSWPQ();
+
294  IndexHNSWPQ(int d, int pq_m, int M);
+
295  void train(idx_t n, const float* x) override;
+
296  HNSW::DistanceComputer * get_distance_computer() const override;
+
297 };
+
298 
+
299 /** SQ index topped with with a HNSW structure to access elements
+
300  * more efficiently.
+
301  */
+
302 struct IndexHNSWSQ: IndexHNSW {
+
303  IndexHNSWSQ();
+
304  IndexHNSWSQ(int d, ScalarQuantizer::QuantizerType qtype, int M);
+
305  HNSW::DistanceComputer * get_distance_computer() const override;
+
306 };
+
307 
+
308 /** 2-level code structure with fast random access
+
309  */
+
310 struct IndexHNSW2Level: IndexHNSW {
+
311  IndexHNSW2Level();
+
312  IndexHNSW2Level(Index *quantizer, size_t nlist, int m_pq, int M);
+
313  HNSW::DistanceComputer * get_distance_computer() const override;
+
314  void flip_to_ivf();
+
315 
+
316  /// entry point for search
+
317  void search (idx_t n, const float *x, idx_t k,
+
318  float *distances, idx_t *labels) const override;
+
319 
+
320 };
+
321 
+
322 
+
323 
+
324 };
+
faiss::RandomGenerator
random generator that can be used in multithreaded contexts
Definition: utils.h:48
+
faiss::HNSW::add_with_locks
void add_with_locks(DistanceComputer &ptdis, int pt_level, int pt_id, std::vector< omp_lock_t > &locks, VisitedTable &vt)
Definition: IndexHNSW.cpp:657
+ + +
faiss::HNSW::neighbor_range
void neighbor_range(idx_t no, int layer_no, size_t *begin, size_t *end) const
range of entries in the neighbors table of vertex no at layer_no
Definition: IndexHNSW.cpp:517
+
faiss::IndexHNSWPQ::train
void train(idx_t n, const float *x) override
Trains the storage if needed.
Definition: IndexHNSW.cpp:1733
+
faiss::HNSW::DistanceComputer::operator()
virtual float operator()(storage_idx_t i)=0
compute distance of vector i to current query
+
faiss::HNSW::nb_neighbors
int nb_neighbors(int layer_no) const
nb of neighbors for this level
Definition: IndexHNSW.cpp:497
+
faiss::IndexHNSW2Level::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
entry point for search
Definition: IndexHNSW.cpp:2071
+
faiss::HNSW::entry_point
storage_idx_t entry_point
entry point in the search structure (one of the points with maximum level
Definition: IndexHNSW.h:88
+
faiss::HNSW::cum_nb_neighbors
int cum_nb_neighbors(int layer_no) const
cumumlative nb up to (and excluding) this level
Definition: IndexHNSW.cpp:512
+
faiss::HNSW::DistanceComputer::symmetric_dis
virtual float symmetric_dis(storage_idx_t i, storage_idx_t j)=0
compute distance between two stored vectors
+ +
faiss::HNSW::assign_probas
std::vector< double > assign_probas
assignment probability to each layer (sum=1)
Definition: IndexHNSW.h:70
+
faiss::HNSW::cum_nneighbor_per_level
std::vector< int > cum_nneighbor_per_level
Definition: IndexHNSW.h:74
+
faiss::ReconstructFromNeighbors::get_neighbor_table
void get_neighbor_table(storage_idx_t i, float *out) const
get the M+1 -by-d table for neighbor coordinates for vector i
Definition: IndexHNSW.cpp:1512
+ +
faiss::IndexHNSW::train
void train(idx_t n, const float *x) override
Trains the storage if needed.
Definition: IndexHNSW.cpp:978
+
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
+ +
faiss::HNSW::offsets
std::vector< size_t > offsets
Definition: IndexHNSW.h:81
+
faiss::VisitedTable
set implementation optimized for fast access.
Definition: IndexHNSW.cpp:54
+
faiss::IndexHNSW::add
void add(idx_t n, const float *x) override
Definition: IndexHNSW.cpp:1031
+
faiss::ReconstructFromNeighbors::reconstruct
void reconstruct(storage_idx_t i, float *x, float *tmp) const
called by compute_distances
Definition: IndexHNSW.cpp:1378
+
faiss::HNSW::efSearch
int efSearch
expansion factor at search time
Definition: IndexHNSW.h:99
+
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
+
faiss::HNSW::DistanceComputer::set_query
virtual void set_query(const float *x)=0
called before computing distances
+
faiss::HNSW::check_relative_distance
bool check_relative_distance
during search: do we check whether the next best distance is good enough?
Definition: IndexHNSW.h:102
+
faiss::HNSW::HNSW
HNSW(int M=32)
only mandatory parameter: nb of neighbors
Definition: IndexHNSW.cpp:527
+
faiss::HNSW::upper_beam
int upper_beam
number of entry points in levels &gt; 0.
Definition: IndexHNSW.h:105
+
faiss::HNSW::set_nb_neighbors
void set_nb_neighbors(int level_no, int n)
set nb of neighbors for this level (before adding anything)
Definition: IndexHNSW.cpp:503
+
faiss::ReconstructFromNeighbors::estimate_code
void estimate_code(const float *x, storage_idx_t i, uint8_t *code) const
called by add_codes
Definition: IndexHNSW.cpp:1531
+ +
faiss::HNSW::idx_t
faiss::Index::idx_t idx_t
Faiss results are 64-bit.
Definition: IndexHNSW.h:46
+ +
faiss::ReconstructFromNeighbors::add_codes
void add_codes(size_t n, const float *x)
Definition: IndexHNSW.cpp:1572
+ +
faiss::HNSW::random_level
int random_level()
pick a random level for a new point
Definition: IndexHNSW.cpp:539
+
faiss::HNSW::set_default_probas
void set_default_probas(int M, float levelMult)
Definition: IndexHNSW.cpp:553
+
faiss::IndexHNSW::reset
void reset() override
removes all elements from the database.
Definition: IndexHNSW.cpp:1043
+ +
faiss::HNSW::search
void search(DistanceComputer &qdis, int k, idx_t *I, float *D, VisitedTable &vt) const
search interface
Definition: IndexHNSW.cpp:711
+ +
faiss::IndexHNSW::init_level_0_from_entry_points
void init_level_0_from_entry_points(int npt, const storage_idx_t *points, const storage_idx_t *nearests)
alternative graph building
Definition: IndexHNSW.cpp:1200
+
faiss::IndexHNSW::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
entry point for search
Definition: IndexHNSW.cpp:985
+
faiss::IndexHNSW::init_level_0_from_knngraph
void init_level_0_from_knngraph(int k, const float *D, const idx_t *I)
alternative graph building
Definition: IndexHNSW.cpp:1162
+
faiss::IndexHNSW::search_level_0
void search_level_0(idx_t n, const float *x, idx_t k, const storage_idx_t *nearest, const float *nearest_d, float *distances, idx_t *labels, int nprobe=1, int search_type=1) const
Definition: IndexHNSW.cpp:1092
+
faiss::HNSW::fill_with_random_links
void fill_with_random_links(size_t n)
add n random levels to table (for debugging...)
Definition: IndexHNSW.cpp:913
+
faiss::HNSW::neighbors
std::vector< storage_idx_t > neighbors
Definition: IndexHNSW.h:85
+
faiss::IndexHNSW::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexHNSW.cpp:1050
+
faiss::HNSW::efConstruction
int efConstruction
expansion factor at construction time
Definition: IndexHNSW.h:96
+
faiss::HNSW::storage_idx_t
int storage_idx_t
internal storage of vectors (32 bits: this is expensive)
Definition: IndexHNSW.h:43
+ +
faiss::HNSW::levels
std::vector< int > levels
level of each vector (base level = 1), size = ntotal
Definition: IndexHNSW.h:77
+
faiss::HNSW::max_level
int max_level
maximum level
Definition: IndexHNSW.h:93
+
+ +
+Generated by   +doxygen + 1.8.5 +
+ + diff --git a/docs/html/IndexIVFPQ_8cpp_source.html b/docs/html/IndexIVFPQ_8cpp_source.html index 0080c778a..dcde964d1 100644 --- a/docs/html/IndexIVFPQ_8cpp_source.html +++ b/docs/html/IndexIVFPQ_8cpp_source.html @@ -138,1593 +138,1734 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
53  by_residual = true;
54  use_precomputed_table = 0;
55  scan_table_threshold = 0;
-
56  max_codes = 0; // means unlimited
-
57 
-
58  polysemous_training = nullptr;
-
59  do_polysemous_training = false;
-
60  polysemous_ht = 0;
-
61 
-
62 }
+
56 
+
57  polysemous_training = nullptr;
+
58  do_polysemous_training = false;
+
59  polysemous_ht = 0;
+
60 
+
61 }
+
62 
63 
-
64 
-
65 void IndexIVFPQ::train_residual (idx_t n, const float *x)
-
66 {
-
67  train_residual_o (n, x, nullptr);
-
68 }
+
64 void IndexIVFPQ::train_residual (idx_t n, const float *x)
+
65 {
+
66  train_residual_o (n, x, nullptr);
+
67 }
+
68 
69 
-
70 
-
71 void IndexIVFPQ::train_residual_o (idx_t n, const float *x, float *residuals_2)
-
72 {
-
73  const float * x_in = x;
-
74 
-
75  x = fvecs_maybe_subsample (
-
76  d, (size_t*)&n, pq.cp.max_points_per_centroid * pq.ksub,
-
77  x, verbose, pq.cp.seed);
-
78 
-
79  ScopeDeleter<float> del_x (x_in == x ? nullptr : x);
-
80 
-
81  const float *trainset;
-
82  ScopeDeleter<float> del_residuals;
-
83  if (by_residual) {
-
84  if(verbose) printf("computing residuals\n");
-
85  idx_t * assign = new idx_t [n]; // assignement to coarse centroids
-
86  ScopeDeleter<idx_t> del (assign);
-
87  quantizer->assign (n, x, assign);
-
88  float *residuals = new float [n * d];
-
89  del_residuals.set (residuals);
-
90  for (idx_t i = 0; i < n; i++)
-
91  quantizer->compute_residual (x + i * d, residuals+i*d, assign[i]);
-
92 
-
93  trainset = residuals;
-
94  } else {
-
95  trainset = x;
-
96  }
-
97  if (verbose)
-
98  printf ("training %zdx%zd product quantizer on %ld vectors in %dD\n",
-
99  pq.M, pq.ksub, n, d);
-
100  pq.verbose = verbose;
-
101  pq.train (n, trainset);
-
102 
-
103  if (do_polysemous_training) {
-
104  if (verbose)
-
105  printf("doing polysemous training for PQ\n");
-
106  PolysemousTraining default_pt;
-
107  PolysemousTraining *pt = polysemous_training;
-
108  if (!pt) pt = &default_pt;
-
109  pt->optimize_pq_for_hamming (pq, n, trainset);
-
110  }
-
111 
-
112  // prepare second-level residuals for refine PQ
-
113  if (residuals_2) {
-
114  uint8_t *train_codes = new uint8_t [pq.code_size * n];
-
115  ScopeDeleter<uint8_t> del (train_codes);
-
116  pq.compute_codes (trainset, train_codes, n);
-
117 
-
118  for (idx_t i = 0; i < n; i++) {
-
119  const float *xx = trainset + i * d;
-
120  float * res = residuals_2 + i * d;
-
121  pq.decode (train_codes + i * pq.code_size, res);
-
122  for (int j = 0; j < d; j++)
-
123  res[j] = xx[j] - res[j];
-
124  }
-
125 
-
126  }
-
127 
-
128  if (by_residual) {
-
129  precompute_table ();
-
130  }
-
131 
-
132 }
+
70 void IndexIVFPQ::train_residual_o (idx_t n, const float *x, float *residuals_2)
+
71 {
+
72  const float * x_in = x;
+
73 
+
74  x = fvecs_maybe_subsample (
+
75  d, (size_t*)&n, pq.cp.max_points_per_centroid * pq.ksub,
+
76  x, verbose, pq.cp.seed);
+
77 
+
78  ScopeDeleter<float> del_x (x_in == x ? nullptr : x);
+
79 
+
80  const float *trainset;
+
81  ScopeDeleter<float> del_residuals;
+
82  if (by_residual) {
+
83  if(verbose) printf("computing residuals\n");
+
84  idx_t * assign = new idx_t [n]; // assignement to coarse centroids
+
85  ScopeDeleter<idx_t> del (assign);
+
86  quantizer->assign (n, x, assign);
+
87  float *residuals = new float [n * d];
+
88  del_residuals.set (residuals);
+
89  for (idx_t i = 0; i < n; i++)
+
90  quantizer->compute_residual (x + i * d, residuals+i*d, assign[i]);
+
91 
+
92  trainset = residuals;
+
93  } else {
+
94  trainset = x;
+
95  }
+
96  if (verbose)
+
97  printf ("training %zdx%zd product quantizer on %ld vectors in %dD\n",
+
98  pq.M, pq.ksub, n, d);
+
99  pq.verbose = verbose;
+
100  pq.train (n, trainset);
+
101 
+
102  if (do_polysemous_training) {
+
103  if (verbose)
+
104  printf("doing polysemous training for PQ\n");
+
105  PolysemousTraining default_pt;
+
106  PolysemousTraining *pt = polysemous_training;
+
107  if (!pt) pt = &default_pt;
+
108  pt->optimize_pq_for_hamming (pq, n, trainset);
+
109  }
+
110 
+
111  // prepare second-level residuals for refine PQ
+
112  if (residuals_2) {
+
113  uint8_t *train_codes = new uint8_t [pq.code_size * n];
+
114  ScopeDeleter<uint8_t> del (train_codes);
+
115  pq.compute_codes (trainset, train_codes, n);
+
116 
+
117  for (idx_t i = 0; i < n; i++) {
+
118  const float *xx = trainset + i * d;
+
119  float * res = residuals_2 + i * d;
+
120  pq.decode (train_codes + i * pq.code_size, res);
+
121  for (int j = 0; j < d; j++)
+
122  res[j] = xx[j] - res[j];
+
123  }
+
124 
+
125  }
+
126 
+
127  if (by_residual) {
+
128  precompute_table ();
+
129  }
+
130 
+
131 }
+
132 
133 
-
134 
-
135 /* produce a binary signature based on the residual vector */
-
136 void IndexIVFPQ::encode (long key, const float * x, uint8_t * code) const
-
137 {
-
138  if (by_residual) {
-
139  float residual_vec[d];
-
140  quantizer->compute_residual (x, residual_vec, key);
-
141  pq.compute_code (residual_vec, code);
-
142  }
-
143  else pq.compute_code (x, code);
-
144 }
+
134 /* produce a binary signature based on the residual vector */
+
135 void IndexIVFPQ::encode (long key, const float * x, uint8_t * code) const
+
136 {
+
137  if (by_residual) {
+
138  float residual_vec[d];
+
139  quantizer->compute_residual (x, residual_vec, key);
+
140  pq.compute_code (residual_vec, code);
+
141  }
+
142  else pq.compute_code (x, code);
+
143 }
+
144 
145 
146 
147 
148 
-
149 
-
150 void IndexIVFPQ::encode_multiple (size_t n, long *keys,
-
151  const float * x, uint8_t * xcodes,
-
152  bool compute_keys) const
-
153 {
-
154  if (compute_keys)
-
155  quantizer->assign (n, x, keys);
-
156 
-
157  if (by_residual) {
-
158  float *residuals = new float [n * d];
-
159  ScopeDeleter<float> del (residuals);
-
160  // TODO: parallelize?
-
161  for (size_t i = 0; i < n; i++)
-
162  quantizer->compute_residual (x + i * d, residuals + i * d, keys[i]);
-
163  pq.compute_codes (residuals, xcodes, n);
-
164  } else {
-
165  pq.compute_codes (x, xcodes, n);
-
166  }
-
167 }
-
168 
-
169 void IndexIVFPQ::decode_multiple (size_t n, const long *keys,
-
170  const uint8_t * xcodes, float * x) const
-
171 {
-
172  pq.decode (xcodes, x, n);
-
173  if (by_residual) {
-
174  std::vector<float> centroid (d);
-
175  for (size_t i = 0; i < n; i++) {
-
176  quantizer->reconstruct (keys[i], centroid.data());
-
177  float *xi = x + i * d;
-
178  for (size_t j = 0; j < d; j++) {
-
179  xi [j] += centroid [j];
-
180  }
-
181  }
-
182  }
-
183 }
+
149 void IndexIVFPQ::encode_multiple (size_t n, long *keys,
+
150  const float * x, uint8_t * xcodes,
+
151  bool compute_keys) const
+
152 {
+
153  if (compute_keys)
+
154  quantizer->assign (n, x, keys);
+
155 
+
156  if (by_residual) {
+
157  float *residuals = new float [n * d];
+
158  ScopeDeleter<float> del (residuals);
+
159  // TODO: parallelize?
+
160  for (size_t i = 0; i < n; i++)
+
161  quantizer->compute_residual (x + i * d, residuals + i * d, keys[i]);
+
162  pq.compute_codes (residuals, xcodes, n);
+
163  } else {
+
164  pq.compute_codes (x, xcodes, n);
+
165  }
+
166 }
+
167 
+
168 void IndexIVFPQ::decode_multiple (size_t n, const long *keys,
+
169  const uint8_t * xcodes, float * x) const
+
170 {
+
171  pq.decode (xcodes, x, n);
+
172  if (by_residual) {
+
173  std::vector<float> centroid (d);
+
174  for (size_t i = 0; i < n; i++) {
+
175  quantizer->reconstruct (keys[i], centroid.data());
+
176  float *xi = x + i * d;
+
177  for (size_t j = 0; j < d; j++) {
+
178  xi [j] += centroid [j];
+
179  }
+
180  }
+
181  }
+
182 }
+
183 
184 
-
185 
-
186 void IndexIVFPQ::add_with_ids (idx_t n, const float * x, const long *xids)
-
187 {
-
188  add_core_o (n, x, xids, nullptr);
-
189 }
+
185 void IndexIVFPQ::add_with_ids (idx_t n, const float * x, const long *xids)
+
186 {
+
187  add_core_o (n, x, xids, nullptr);
+
188 }
+
189 
190 
-
191 
-
192 void IndexIVFPQ::add_core_o (idx_t n, const float * x, const long *xids,
-
193  float *residuals_2, const long *precomputed_idx)
-
194 {
-
195  FAISS_THROW_IF_NOT (is_trained);
-
196  double t0 = getmillisecs ();
-
197  const long * idx;
-
198  ScopeDeleter<long> del_idx;
-
199 
-
200  if (precomputed_idx) {
-
201  idx = precomputed_idx;
-
202  } else {
-
203  long * idx0 = new long [n];
-
204  del_idx.set (idx0);
-
205  quantizer->assign (n, x, idx0);
-
206  idx = idx0;
-
207  }
-
208 
-
209  double t1 = getmillisecs ();
-
210  uint8_t * xcodes = new uint8_t [n * code_size];
-
211  ScopeDeleter<uint8_t> del_xcodes (xcodes);
-
212 
-
213  const float *to_encode = nullptr;
-
214  ScopeDeleter<float> del_to_encode;
+
191 void IndexIVFPQ::add_core_o (idx_t n, const float * x, const long *xids,
+
192  float *residuals_2, const long *precomputed_idx)
+
193 {
+
194 
+
195  idx_t bs = 32768;
+
196  if (n > bs) {
+
197  for (idx_t i0 = 0; i0 < n; i0 += bs) {
+
198  idx_t i1 = std::min(i0 + bs, n);
+
199  if (verbose) {
+
200  printf("IndexIVFPQ::add_core_o: adding %ld:%ld / %ld\n",
+
201  i0, i1, n);
+
202  }
+
203  add_core_o (i1 - i0, x + i0 * d,
+
204  xids ? xids + i0 : nullptr,
+
205  residuals_2 ? residuals_2 + i0 * d : nullptr,
+
206  precomputed_idx ? precomputed_idx + i0 : nullptr);
+
207  }
+
208  return;
+
209  }
+
210 
+
211  FAISS_THROW_IF_NOT (is_trained);
+
212  double t0 = getmillisecs ();
+
213  const long * idx;
+
214  ScopeDeleter<long> del_idx;
215 
-
216  if (by_residual) {
-
217  float *residuals = new float [n * d];
-
218  // TODO: parallelize?
-
219  for (size_t i = 0; i < n; i++) {
-
220  if (idx[i] < 0)
-
221  memset (residuals + i * d, 0, sizeof(*residuals) * d);
-
222  else
-
223  quantizer->compute_residual (
-
224  x + i * d, residuals + i * d, idx[i]);
-
225  }
-
226  to_encode = residuals;
-
227  del_to_encode.set (to_encode);
-
228  } else {
-
229  to_encode = x;
-
230  }
-
231  pq.compute_codes (to_encode, xcodes, n);
-
232 
-
233  double t2 = getmillisecs ();
-
234  // TODO: parallelize?
-
235  size_t n_ignore = 0;
-
236  for (size_t i = 0; i < n; i++) {
-
237  idx_t key = idx[i];
-
238  if (key < 0) {
-
239  n_ignore ++;
-
240  if (residuals_2)
-
241  memset (residuals_2, 0, sizeof(*residuals_2) * d);
-
242  continue;
-
243  }
-
244  idx_t id = xids ? xids[i] : ntotal + i;
-
245  ids[key].push_back (id);
-
246  uint8_t *code = xcodes + i * code_size;
-
247  for (size_t j = 0; j < code_size; j++)
-
248  codes[key].push_back (code[j]);
-
249 
-
250  if (residuals_2) {
-
251  float *res2 = residuals_2 + i * d;
-
252  const float *xi = to_encode + i * d;
-
253  pq.decode (code, res2);
-
254  for (int j = 0; j < d; j++)
-
255  res2[j] = xi[j] - res2[j];
-
256  }
-
257 
-
258  if (maintain_direct_map)
-
259  direct_map.push_back (key << 32 | (ids[key].size() - 1));
-
260  }
-
261 
-
262 
-
263  double t3 = getmillisecs ();
-
264  if(verbose) {
-
265  char comment[100] = {0};
-
266  if (n_ignore > 0)
-
267  snprintf (comment, 100, "(%ld vectors ignored)", n_ignore);
-
268  printf(" add_core times: %.3f %.3f %.3f %s\n",
-
269  t1 - t0, t2 - t1, t3 - t2, comment);
-
270  }
-
271  ntotal += n;
-
272 }
+
216  if (precomputed_idx) {
+
217  idx = precomputed_idx;
+
218  } else {
+
219  long * idx0 = new long [n];
+
220  del_idx.set (idx0);
+
221  quantizer->assign (n, x, idx0);
+
222  idx = idx0;
+
223  }
+
224 
+
225  double t1 = getmillisecs ();
+
226  uint8_t * xcodes = new uint8_t [n * code_size];
+
227  ScopeDeleter<uint8_t> del_xcodes (xcodes);
+
228 
+
229  const float *to_encode = nullptr;
+
230  ScopeDeleter<float> del_to_encode;
+
231 
+
232  if (by_residual) {
+
233  float *residuals = new float [n * d];
+
234  // TODO: parallelize?
+
235  for (size_t i = 0; i < n; i++) {
+
236  if (idx[i] < 0)
+
237  memset (residuals + i * d, 0, sizeof(*residuals) * d);
+
238  else
+
239  quantizer->compute_residual (
+
240  x + i * d, residuals + i * d, idx[i]);
+
241  }
+
242  to_encode = residuals;
+
243  del_to_encode.set (to_encode);
+
244  } else {
+
245  to_encode = x;
+
246  }
+
247  pq.compute_codes (to_encode, xcodes, n);
+
248 
+
249  double t2 = getmillisecs ();
+
250  // TODO: parallelize?
+
251  size_t n_ignore = 0;
+
252  for (size_t i = 0; i < n; i++) {
+
253  idx_t key = idx[i];
+
254  if (key < 0) {
+
255  n_ignore ++;
+
256  if (residuals_2)
+
257  memset (residuals_2, 0, sizeof(*residuals_2) * d);
+
258  continue;
+
259  }
+
260  idx_t id = xids ? xids[i] : ntotal + i;
+
261  ids[key].push_back (id);
+
262  uint8_t *code = xcodes + i * code_size;
+
263  for (size_t j = 0; j < code_size; j++)
+
264  codes[key].push_back (code[j]);
+
265 
+
266  if (residuals_2) {
+
267  float *res2 = residuals_2 + i * d;
+
268  const float *xi = to_encode + i * d;
+
269  pq.decode (code, res2);
+
270  for (int j = 0; j < d; j++)
+
271  res2[j] = xi[j] - res2[j];
+
272  }
273 
-
274 void IndexIVFPQ::reconstruct_n (idx_t i0, idx_t ni, float *recons) const
-
275 {
-
276  FAISS_THROW_IF_NOT (ni == 0 || (i0 >= 0 && i0 + ni <= ntotal));
+
274  if (maintain_direct_map)
+
275  direct_map.push_back (key << 32 | (ids[key].size() - 1));
+
276  }
277 
-
278  std::vector<float> centroid (d);
-
279 
-
280  for (int key = 0; key < nlist; key++) {
-
281  const std::vector<long> & idlist = ids[key];
-
282  const uint8_t * code_line = codes[key].data();
-
283 
-
284  for (long ofs = 0; ofs < idlist.size(); ofs++) {
-
285  long id = idlist[ofs];
-
286  if (!(id >= i0 && id < i0 + ni)) continue;
-
287  float *r = recons + d * (id - i0);
-
288  if (by_residual) {
-
289  quantizer->reconstruct (key, centroid.data());
-
290  pq.decode (code_line + ofs * pq.code_size, r);
-
291  for (int j = 0; j < d; j++) {
-
292  r[j] += centroid[j];
-
293  }
-
294  } else {
-
295  pq.decode (code_line + ofs * pq.code_size, r);
-
296  }
-
297  }
-
298  }
-
299 }
-
300 
-
301 
-
302 void IndexIVFPQ::reconstruct (idx_t key, float * recons) const
-
303 {
-
304  FAISS_THROW_IF_NOT (direct_map.size() == ntotal);
-
305 
-
306  int list_no = direct_map[key] >> 32;
-
307  int ofs = direct_map[key] & 0xffffffff;
+
278 
+
279  double t3 = getmillisecs ();
+
280  if(verbose) {
+
281  char comment[100] = {0};
+
282  if (n_ignore > 0)
+
283  snprintf (comment, 100, "(%ld vectors ignored)", n_ignore);
+
284  printf(" add_core times: %.3f %.3f %.3f %s\n",
+
285  t1 - t0, t2 - t1, t3 - t2, comment);
+
286  }
+
287  ntotal += n;
+
288 }
+
289 
+
290 
+
291 void IndexIVFPQ::reconstruct_from_offset (long list_no, long offset,
+
292  float* recons) const
+
293 {
+
294  const uint8_t* code = &(codes[list_no][offset * code_size]);
+
295 
+
296  if (by_residual) {
+
297  std::vector<float> centroid(d);
+
298  quantizer->reconstruct (list_no, centroid.data());
+
299 
+
300  pq.decode (code, recons);
+
301  for (int i = 0; i < d; ++i) {
+
302  recons[i] += centroid[i];
+
303  }
+
304  } else {
+
305  pq.decode (code, recons);
+
306  }
+
307 }
308 
-
309  quantizer->reconstruct (list_no, recons);
-
310  const uint8_t * code = &(codes[list_no][ofs * pq.code_size]);
+
309 
+
310 
311 
-
312  for (size_t m = 0; m < pq.M; m++) {
-
313  float * out = recons + m * pq.dsub;
-
314  const float * cent = pq.get_centroids (m, code[m]);
-
315  for (size_t i = 0; i < pq.dsub; i++) {
-
316  out[i] += cent[i];
-
317  }
-
318  }
-
319 }
-
320 
-
321 
-
322 
-
323 
-
324 
-
325 
-
326 /** Precomputed tables for residuals
-
327  *
-
328  * During IVFPQ search with by_residual, we compute
-
329  *
-
330  * d = || x - y_C - y_R ||^2
-
331  *
-
332  * where x is the query vector, y_C the coarse centroid, y_R the
-
333  * refined PQ centroid. The expression can be decomposed as:
+
312 
+
313 
+
314 /** Precomputed tables for residuals
+
315  *
+
316  * During IVFPQ search with by_residual, we compute
+
317  *
+
318  * d = || x - y_C - y_R ||^2
+
319  *
+
320  * where x is the query vector, y_C the coarse centroid, y_R the
+
321  * refined PQ centroid. The expression can be decomposed as:
+
322  *
+
323  * d = || x - y_C ||^2 + || y_R ||^2 + 2 * (y_C|y_R) - 2 * (x|y_R)
+
324  * --------------- --------------------------- -------
+
325  * term 1 term 2 term 3
+
326  *
+
327  * When using multiprobe, we use the following decomposition:
+
328  * - term 1 is the distance to the coarse centroid, that is computed
+
329  * during the 1st stage search.
+
330  * - term 2 can be precomputed, as it does not involve x. However,
+
331  * because of the PQ, it needs nlist * M * ksub storage. This is why
+
332  * use_precomputed_table is off by default
+
333  * - term 3 is the classical non-residual distance table.
334  *
-
335  * d = || x - y_C ||^2 + || y_R ||^2 + 2 * (y_C|y_R) - 2 * (x|y_R)
-
336  * --------------- --------------------------- -------
-
337  * term 1 term 2 term 3
-
338  *
-
339  * When using multiprobe, we use the following decomposition:
-
340  * - term 1 is the distance to the coarse centroid, that is computed
-
341  * during the 1st stage search.
-
342  * - term 2 can be precomputed, as it does not involve x. However,
-
343  * because of the PQ, it needs nlist * M * ksub storage. This is why
-
344  * use_precomputed_table is off by default
-
345  * - term 3 is the classical non-residual distance table.
-
346  *
-
347  * Since y_R defined by a product quantizer, it is split across
-
348  * subvectors and stored separately for each subvector. If the coarse
-
349  * quantizer is a MultiIndexQuantizer then the table can be stored
-
350  * more compactly.
-
351  *
-
352  * At search time, the tables for term 2 and term 3 are added up. This
-
353  * is faster when the length of the lists is > ksub * M.
-
354  */
-
355 
-
356 void IndexIVFPQ::precompute_table ()
-
357 {
-
358  if (use_precomputed_table == 0) { // then choose the type of table
-
359  if (quantizer->metric_type == METRIC_INNER_PRODUCT) {
-
360  fprintf(stderr, "IndexIVFPQ::precompute_table: WARN precomputed "
-
361  "tables not needed for inner product quantizers\n");
-
362  return;
-
363  }
-
364  const MultiIndexQuantizer *miq =
-
365  dynamic_cast<const MultiIndexQuantizer *> (quantizer);
-
366  if (miq && pq.M % miq->pq.M == 0)
-
367  use_precomputed_table = 2;
-
368  else
-
369  use_precomputed_table = 1;
-
370  } // otherwise assume user has set appropriate flag on input
+
335  * Since y_R defined by a product quantizer, it is split across
+
336  * subvectors and stored separately for each subvector. If the coarse
+
337  * quantizer is a MultiIndexQuantizer then the table can be stored
+
338  * more compactly.
+
339  *
+
340  * At search time, the tables for term 2 and term 3 are added up. This
+
341  * is faster when the length of the lists is > ksub * M.
+
342  */
+
343 
+
344 void IndexIVFPQ::precompute_table ()
+
345 {
+
346  if (use_precomputed_table == 0) { // then choose the type of table
+
347  if (quantizer->metric_type == METRIC_INNER_PRODUCT) {
+
348  fprintf(stderr, "IndexIVFPQ::precompute_table: WARN precomputed "
+
349  "tables not needed for inner product quantizers\n");
+
350  return;
+
351  }
+
352  const MultiIndexQuantizer *miq =
+
353  dynamic_cast<const MultiIndexQuantizer *> (quantizer);
+
354  if (miq && pq.M % miq->pq.M == 0)
+
355  use_precomputed_table = 2;
+
356  else
+
357  use_precomputed_table = 1;
+
358  } // otherwise assume user has set appropriate flag on input
+
359 
+
360  if (verbose) {
+
361  printf ("precomputing IVFPQ tables type %d\n",
+
362  use_precomputed_table);
+
363  }
+
364 
+
365  // squared norms of the PQ centroids
+
366  std::vector<float> r_norms (pq.M * pq.ksub, NAN);
+
367  for (int m = 0; m < pq.M; m++)
+
368  for (int j = 0; j < pq.ksub; j++)
+
369  r_norms [m * pq.ksub + j] =
+
370  fvec_norm_L2sqr (pq.get_centroids (m, j), pq.dsub);
371 
-
372  if (verbose) {
-
373  printf ("precomputing IVFPQ tables type %d\n",
-
374  use_precomputed_table);
-
375  }
+
372  if (use_precomputed_table == 1) {
+
373 
+
374  precomputed_table.resize (nlist * pq.M * pq.ksub);
+
375  std::vector<float> centroid (d);
376 
-
377  // squared norms of the PQ centroids
-
378  std::vector<float> r_norms (pq.M * pq.ksub, NAN);
-
379  for (int m = 0; m < pq.M; m++)
-
380  for (int j = 0; j < pq.ksub; j++)
-
381  r_norms [m * pq.ksub + j] =
-
382  fvec_norm_L2sqr (pq.get_centroids (m, j), pq.dsub);
-
383 
-
384  if (use_precomputed_table == 1) {
-
385 
-
386  precomputed_table.resize (nlist * pq.M * pq.ksub);
-
387  std::vector<float> centroid (d);
-
388 
-
389  for (size_t i = 0; i < nlist; i++) {
-
390  quantizer->reconstruct (i, centroid.data());
-
391 
-
392  float *tab = &precomputed_table[i * pq.M * pq.ksub];
-
393  pq.compute_inner_prod_table (centroid.data(), tab);
-
394  fvec_madd (pq.M * pq.ksub, r_norms.data(), 2.0, tab, tab);
-
395  }
-
396  } else if (use_precomputed_table == 2) {
-
397  const MultiIndexQuantizer *miq =
-
398  dynamic_cast<const MultiIndexQuantizer *> (quantizer);
-
399  FAISS_THROW_IF_NOT (miq);
-
400  const ProductQuantizer &cpq = miq->pq;
-
401  FAISS_THROW_IF_NOT (pq.M % cpq.M == 0);
-
402 
-
403  precomputed_table.resize(cpq.ksub * pq.M * pq.ksub);
-
404 
-
405  // reorder PQ centroid table
-
406  std::vector<float> centroids (d * cpq.ksub, NAN);
-
407 
-
408  for (int m = 0; m < cpq.M; m++) {
-
409  for (size_t i = 0; i < cpq.ksub; i++) {
-
410  memcpy (centroids.data() + i * d + m * cpq.dsub,
-
411  cpq.get_centroids (m, i),
-
412  sizeof (*centroids.data()) * cpq.dsub);
-
413  }
-
414  }
-
415 
-
416  pq.compute_inner_prod_tables (cpq.ksub, centroids.data (),
-
417  precomputed_table.data ());
-
418 
-
419  for (size_t i = 0; i < cpq.ksub; i++) {
-
420  float *tab = &precomputed_table[i * pq.M * pq.ksub];
-
421  fvec_madd (pq.M * pq.ksub, r_norms.data(), 2.0, tab, tab);
-
422  }
-
423 
-
424  }
-
425 }
-
426 
-
427 namespace {
+
377  for (size_t i = 0; i < nlist; i++) {
+
378  quantizer->reconstruct (i, centroid.data());
+
379 
+
380  float *tab = &precomputed_table[i * pq.M * pq.ksub];
+
381  pq.compute_inner_prod_table (centroid.data(), tab);
+
382  fvec_madd (pq.M * pq.ksub, r_norms.data(), 2.0, tab, tab);
+
383  }
+
384  } else if (use_precomputed_table == 2) {
+
385  const MultiIndexQuantizer *miq =
+
386  dynamic_cast<const MultiIndexQuantizer *> (quantizer);
+
387  FAISS_THROW_IF_NOT (miq);
+
388  const ProductQuantizer &cpq = miq->pq;
+
389  FAISS_THROW_IF_NOT (pq.M % cpq.M == 0);
+
390 
+
391  precomputed_table.resize(cpq.ksub * pq.M * pq.ksub);
+
392 
+
393  // reorder PQ centroid table
+
394  std::vector<float> centroids (d * cpq.ksub, NAN);
+
395 
+
396  for (int m = 0; m < cpq.M; m++) {
+
397  for (size_t i = 0; i < cpq.ksub; i++) {
+
398  memcpy (centroids.data() + i * d + m * cpq.dsub,
+
399  cpq.get_centroids (m, i),
+
400  sizeof (*centroids.data()) * cpq.dsub);
+
401  }
+
402  }
+
403 
+
404  pq.compute_inner_prod_tables (cpq.ksub, centroids.data (),
+
405  precomputed_table.data ());
+
406 
+
407  for (size_t i = 0; i < cpq.ksub; i++) {
+
408  float *tab = &precomputed_table[i * pq.M * pq.ksub];
+
409  fvec_madd (pq.M * pq.ksub, r_norms.data(), 2.0, tab, tab);
+
410  }
+
411 
+
412  }
+
413 }
+
414 
+
415 namespace {
+
416 
+
417 static uint64_t get_cycles () {
+
418  uint32_t high, low;
+
419  asm volatile("rdtsc \n\t"
+
420  : "=a" (low),
+
421  "=d" (high));
+
422  return ((uint64_t)high << 32) | (low);
+
423 }
+
424 
+
425 #define TIC t0 = get_cycles()
+
426 #define TOC get_cycles () - t0
+
427 
428 
-
429 static uint64_t get_cycles () {
-
430  uint32_t high, low;
-
431  asm volatile("rdtsc \n\t"
-
432  : "=a" (low),
-
433  "=d" (high));
-
434  return ((uint64_t)high << 32) | (low);
-
435 }
-
436 
-
437 #define TIC t0 = get_cycles()
-
438 #define TOC get_cycles () - t0
-
439 
-
440 
-
441 
-
442 /** QueryTables manages the various ways of searching an
-
443  * IndexIVFPQ. The code contains a lot of branches, depending on:
-
444  * - metric_type: are we computing L2 or Inner product similarity?
-
445  * - by_residual: do we encode raw vectors or residuals?
-
446  * - use_precomputed_table: are x_R|x_C tables precomputed?
-
447  * - polysemous_ht: are we filtering with polysemous codes?
-
448  */
-
449 struct QueryTables {
-
450 
-
451  /*****************************************************
-
452  * General data from the IVFPQ
-
453  *****************************************************/
-
454 
-
455  const IndexIVFPQ & ivfpq;
-
456 
-
457  // copied from IndexIVFPQ for easier access
-
458  int d;
-
459  const ProductQuantizer & pq;
-
460  MetricType metric_type;
-
461  bool by_residual;
-
462  int use_precomputed_table;
-
463 
-
464  // pre-allocated data buffers
-
465  float * sim_table, * sim_table_2;
-
466  float * residual_vec, *decoded_vec;
-
467 
-
468  // single data buffer
-
469  std::vector<float> mem;
-
470 
-
471  // for table pointers
-
472  std::vector<const float *> sim_table_ptrs;
-
473 
-
474  explicit QueryTables (const IndexIVFPQ & ivfpq):
-
475  ivfpq(ivfpq),
-
476  d(ivfpq.d),
-
477  pq (ivfpq.pq),
-
478  metric_type (ivfpq.metric_type),
-
479  by_residual (ivfpq.by_residual),
-
480  use_precomputed_table (ivfpq.use_precomputed_table)
-
481  {
-
482  mem.resize (pq.ksub * pq.M * 2 + d *2);
-
483  sim_table = mem.data();
-
484  sim_table_2 = sim_table + pq.ksub * pq.M;
-
485  residual_vec = sim_table_2 + pq.ksub * pq.M;
-
486  decoded_vec = residual_vec + d;
+
429 
+
430 /** QueryTables manages the various ways of searching an
+
431  * IndexIVFPQ. The code contains a lot of branches, depending on:
+
432  * - metric_type: are we computing L2 or Inner product similarity?
+
433  * - by_residual: do we encode raw vectors or residuals?
+
434  * - use_precomputed_table: are x_R|x_C tables precomputed?
+
435  * - polysemous_ht: are we filtering with polysemous codes?
+
436  */
+
437 struct QueryTables {
+
438 
+
439  /*****************************************************
+
440  * General data from the IVFPQ
+
441  *****************************************************/
+
442 
+
443  const IndexIVFPQ & ivfpq;
+
444 
+
445  // copied from IndexIVFPQ for easier access
+
446  int d;
+
447  const ProductQuantizer & pq;
+
448  MetricType metric_type;
+
449  bool by_residual;
+
450  int use_precomputed_table;
+
451 
+
452  // pre-allocated data buffers
+
453  float * sim_table, * sim_table_2;
+
454  float * residual_vec, *decoded_vec;
+
455 
+
456  // single data buffer
+
457  std::vector<float> mem;
+
458 
+
459  // for table pointers
+
460  std::vector<const float *> sim_table_ptrs;
+
461 
+
462  explicit QueryTables (const IndexIVFPQ & ivfpq):
+
463  ivfpq(ivfpq),
+
464  d(ivfpq.d),
+
465  pq (ivfpq.pq),
+
466  metric_type (ivfpq.metric_type),
+
467  by_residual (ivfpq.by_residual),
+
468  use_precomputed_table (ivfpq.use_precomputed_table)
+
469  {
+
470  mem.resize (pq.ksub * pq.M * 2 + d *2);
+
471  sim_table = mem.data();
+
472  sim_table_2 = sim_table + pq.ksub * pq.M;
+
473  residual_vec = sim_table_2 + pq.ksub * pq.M;
+
474  decoded_vec = residual_vec + d;
+
475 
+
476  // for polysemous
+
477  if (ivfpq.polysemous_ht != 0) {
+
478  q_code.resize (pq.code_size);
+
479  }
+
480  init_list_cycles = 0;
+
481  sim_table_ptrs.resize (pq.M);
+
482  }
+
483 
+
484  /*****************************************************
+
485  * What we do when query is known
+
486  *****************************************************/
487 
-
488  // for polysemous
-
489  if (ivfpq.polysemous_ht != 0) {
-
490  q_code.resize (pq.code_size);
-
491  }
-
492  init_list_cycles = 0;
-
493  sim_table_ptrs.resize (pq.M);
-
494  }
-
495 
-
496  /*****************************************************
-
497  * What we do when query is known
-
498  *****************************************************/
-
499 
-
500  // field specific to query
-
501  const float * qi;
-
502 
-
503  // query-specific intialization
-
504  void init_query (const float * qi) {
-
505  this->qi = qi;
-
506  if (metric_type == METRIC_INNER_PRODUCT)
-
507  init_query_IP ();
-
508  else
-
509  init_query_L2 ();
-
510  if (!by_residual && ivfpq.polysemous_ht != 0)
-
511  pq.compute_code (qi, q_code.data());
-
512  }
-
513 
-
514  void init_query_IP () {
-
515  // precompute some tables specific to the query qi
-
516  pq.compute_inner_prod_table (qi, sim_table);
-
517  // we compute negated inner products for use with the maxheap
-
518  for (int i = 0; i < pq.ksub * pq.M; i++) {
-
519  sim_table[i] = - sim_table[i];
-
520  }
-
521  }
+
488  // field specific to query
+
489  const float * qi;
+
490 
+
491  // query-specific intialization
+
492  void init_query (const float * qi) {
+
493  this->qi = qi;
+
494  if (metric_type == METRIC_INNER_PRODUCT)
+
495  init_query_IP ();
+
496  else
+
497  init_query_L2 ();
+
498  if (!by_residual && ivfpq.polysemous_ht != 0)
+
499  pq.compute_code (qi, q_code.data());
+
500  }
+
501 
+
502  void init_query_IP () {
+
503  // precompute some tables specific to the query qi
+
504  pq.compute_inner_prod_table (qi, sim_table);
+
505  // we compute negated inner products for use with the maxheap
+
506  for (int i = 0; i < pq.ksub * pq.M; i++) {
+
507  sim_table[i] = - sim_table[i];
+
508  }
+
509  }
+
510 
+
511  void init_query_L2 () {
+
512  if (!by_residual) {
+
513  pq.compute_distance_table (qi, sim_table);
+
514  } else if (use_precomputed_table) {
+
515  pq.compute_inner_prod_table (qi, sim_table_2);
+
516  }
+
517  }
+
518 
+
519  /*****************************************************
+
520  * When inverted list is known: prepare computations
+
521  *****************************************************/
522 
-
523  void init_query_L2 () {
-
524  if (!by_residual) {
-
525  pq.compute_distance_table (qi, sim_table);
-
526  } else if (use_precomputed_table) {
-
527  pq.compute_inner_prod_table (qi, sim_table_2);
-
528  }
-
529  }
-
530 
-
531  /*****************************************************
-
532  * When inverted list is known: prepare computations
-
533  *****************************************************/
-
534 
-
535  // fields specific to list
-
536  Index::idx_t key;
-
537  float coarse_dis;
-
538  std::vector<uint8_t> q_code;
-
539 
-
540  uint64_t init_list_cycles;
-
541 
-
542  /// once we know the query and the centroid, we can prepare the
-
543  /// sim_table that will be used for accumulation
-
544  /// and dis0, the initial value
-
545  float precompute_list_tables () {
-
546  float dis0 = 0;
-
547  uint64_t t0; TIC;
-
548  if (by_residual) {
-
549  if (metric_type == METRIC_INNER_PRODUCT)
-
550  dis0 = precompute_list_tables_IP ();
-
551  else
-
552  dis0 = precompute_list_tables_L2 ();
-
553  }
-
554  init_list_cycles += TOC;
-
555  return dis0;
-
556  }
-
557 
-
558  float precompute_list_table_pointers () {
-
559  float dis0 = 0;
-
560  uint64_t t0; TIC;
-
561  if (by_residual) {
-
562  if (metric_type == METRIC_INNER_PRODUCT)
-
563  FAISS_THROW_MSG ("not implemented");
-
564  else
-
565  dis0 = precompute_list_table_pointers_L2 ();
-
566  }
-
567  init_list_cycles += TOC;
-
568  return dis0;
-
569  }
+
523  // fields specific to list
+
524  Index::idx_t key;
+
525  float coarse_dis;
+
526  std::vector<uint8_t> q_code;
+
527 
+
528  uint64_t init_list_cycles;
+
529 
+
530  /// once we know the query and the centroid, we can prepare the
+
531  /// sim_table that will be used for accumulation
+
532  /// and dis0, the initial value
+
533  float precompute_list_tables () {
+
534  float dis0 = 0;
+
535  uint64_t t0; TIC;
+
536  if (by_residual) {
+
537  if (metric_type == METRIC_INNER_PRODUCT)
+
538  dis0 = precompute_list_tables_IP ();
+
539  else
+
540  dis0 = precompute_list_tables_L2 ();
+
541  }
+
542  init_list_cycles += TOC;
+
543  return dis0;
+
544  }
+
545 
+
546  float precompute_list_table_pointers () {
+
547  float dis0 = 0;
+
548  uint64_t t0; TIC;
+
549  if (by_residual) {
+
550  if (metric_type == METRIC_INNER_PRODUCT)
+
551  FAISS_THROW_MSG ("not implemented");
+
552  else
+
553  dis0 = precompute_list_table_pointers_L2 ();
+
554  }
+
555  init_list_cycles += TOC;
+
556  return dis0;
+
557  }
+
558 
+
559  /*****************************************************
+
560  * compute tables for inner prod
+
561  *****************************************************/
+
562 
+
563  float precompute_list_tables_IP ()
+
564  {
+
565  // prepare the sim_table that will be used for accumulation
+
566  // and dis0, the initial value
+
567  ivfpq.quantizer->reconstruct (key, decoded_vec);
+
568  // decoded_vec = centroid
+
569  float dis0 = -fvec_inner_product (qi, decoded_vec, d);
570 
-
571  /*****************************************************
-
572  * compute tables for inner prod
-
573  *****************************************************/
-
574 
-
575  float precompute_list_tables_IP ()
-
576  {
-
577  // prepare the sim_table that will be used for accumulation
-
578  // and dis0, the initial value
-
579  ivfpq.quantizer->reconstruct (key, decoded_vec);
-
580  // decoded_vec = centroid
-
581  float dis0 = -fvec_inner_product (qi, decoded_vec, d);
-
582 
-
583  if (ivfpq.polysemous_ht) {
-
584  for (int i = 0; i < d; i++) {
-
585  residual_vec [i] = qi[i] - decoded_vec[i];
-
586  }
-
587  pq.compute_code (residual_vec, q_code.data());
-
588  }
-
589  return dis0;
-
590  }
-
591 
-
592 
-
593  /*****************************************************
-
594  * compute tables for L2 distance
-
595  *****************************************************/
-
596 
-
597  float precompute_list_tables_L2 ()
-
598  {
-
599  float dis0 = 0;
-
600 
-
601  if (use_precomputed_table == 0) {
-
602  ivfpq.quantizer->compute_residual (qi, residual_vec, key);
-
603  pq.compute_distance_table (residual_vec, sim_table);
-
604  } else if (use_precomputed_table == 1) {
-
605  dis0 = coarse_dis;
-
606 
-
607  fvec_madd (pq.M * pq.ksub,
-
608  &ivfpq.precomputed_table [key * pq.ksub * pq.M],
-
609  -2.0, sim_table_2,
-
610  sim_table);
-
611  } else if (use_precomputed_table == 2) {
-
612  dis0 = coarse_dis;
-
613 
-
614  const MultiIndexQuantizer *miq =
-
615  dynamic_cast<const MultiIndexQuantizer *> (ivfpq.quantizer);
-
616  FAISS_THROW_IF_NOT (miq);
-
617  const ProductQuantizer &cpq = miq->pq;
-
618  int Mf = pq.M / cpq.M;
-
619 
-
620  const float *qtab = sim_table_2; // query-specific table
-
621  float *ltab = sim_table; // (output) list-specific table
+
571  if (ivfpq.polysemous_ht) {
+
572  for (int i = 0; i < d; i++) {
+
573  residual_vec [i] = qi[i] - decoded_vec[i];
+
574  }
+
575  pq.compute_code (residual_vec, q_code.data());
+
576  }
+
577  return dis0;
+
578  }
+
579 
+
580 
+
581  /*****************************************************
+
582  * compute tables for L2 distance
+
583  *****************************************************/
+
584 
+
585  float precompute_list_tables_L2 ()
+
586  {
+
587  float dis0 = 0;
+
588 
+
589  if (use_precomputed_table == 0) {
+
590  ivfpq.quantizer->compute_residual (qi, residual_vec, key);
+
591  pq.compute_distance_table (residual_vec, sim_table);
+
592  } else if (use_precomputed_table == 1) {
+
593  dis0 = coarse_dis;
+
594 
+
595  fvec_madd (pq.M * pq.ksub,
+
596  &ivfpq.precomputed_table [key * pq.ksub * pq.M],
+
597  -2.0, sim_table_2,
+
598  sim_table);
+
599  } else if (use_precomputed_table == 2) {
+
600  dis0 = coarse_dis;
+
601 
+
602  const MultiIndexQuantizer *miq =
+
603  dynamic_cast<const MultiIndexQuantizer *> (ivfpq.quantizer);
+
604  FAISS_THROW_IF_NOT (miq);
+
605  const ProductQuantizer &cpq = miq->pq;
+
606  int Mf = pq.M / cpq.M;
+
607 
+
608  const float *qtab = sim_table_2; // query-specific table
+
609  float *ltab = sim_table; // (output) list-specific table
+
610 
+
611  long k = key;
+
612  for (int cm = 0; cm < cpq.M; cm++) {
+
613  // compute PQ index
+
614  int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
+
615  k >>= cpq.nbits;
+
616 
+
617  // get corresponding table
+
618  const float *pc = &ivfpq.precomputed_table
+
619  [(ki * pq.M + cm * Mf) * pq.ksub];
+
620 
+
621  if (ivfpq.polysemous_ht == 0) {
622 
-
623  long k = key;
-
624  for (int cm = 0; cm < cpq.M; cm++) {
-
625  // compute PQ index
-
626  int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
-
627  k >>= cpq.nbits;
-
628 
-
629  // get corresponding table
-
630  const float *pc = &ivfpq.precomputed_table
-
631  [(ki * pq.M + cm * Mf) * pq.ksub];
-
632 
-
633  if (ivfpq.polysemous_ht == 0) {
-
634 
-
635  // sum up with query-specific table
-
636  fvec_madd (Mf * pq.ksub,
-
637  pc,
-
638  -2.0, qtab,
-
639  ltab);
-
640  ltab += Mf * pq.ksub;
-
641  qtab += Mf * pq.ksub;
-
642  } else {
-
643  for (int m = cm * Mf; m < (cm + 1) * Mf; m++) {
-
644  q_code[m] = fvec_madd_and_argmin
-
645  (pq.ksub, pc, -2, qtab, ltab);
-
646  pc += pq.ksub;
-
647  ltab += pq.ksub;
-
648  qtab += pq.ksub;
-
649  }
-
650  }
-
651 
-
652  }
-
653  }
-
654 
-
655  return dis0;
-
656  }
-
657 
-
658  float precompute_list_table_pointers_L2 ()
-
659  {
-
660  float dis0 = 0;
-
661 
-
662  if (use_precomputed_table == 1) {
-
663  dis0 = coarse_dis;
-
664 
-
665  const float * s = &ivfpq.precomputed_table [key * pq.ksub * pq.M];
-
666  for (int m = 0; m < pq.M; m++) {
-
667  sim_table_ptrs [m] = s;
-
668  s += pq.ksub;
-
669  }
-
670  } else if (use_precomputed_table == 2) {
-
671  dis0 = coarse_dis;
+
623  // sum up with query-specific table
+
624  fvec_madd (Mf * pq.ksub,
+
625  pc,
+
626  -2.0, qtab,
+
627  ltab);
+
628  ltab += Mf * pq.ksub;
+
629  qtab += Mf * pq.ksub;
+
630  } else {
+
631  for (int m = cm * Mf; m < (cm + 1) * Mf; m++) {
+
632  q_code[m] = fvec_madd_and_argmin
+
633  (pq.ksub, pc, -2, qtab, ltab);
+
634  pc += pq.ksub;
+
635  ltab += pq.ksub;
+
636  qtab += pq.ksub;
+
637  }
+
638  }
+
639 
+
640  }
+
641  }
+
642 
+
643  return dis0;
+
644  }
+
645 
+
646  float precompute_list_table_pointers_L2 ()
+
647  {
+
648  float dis0 = 0;
+
649 
+
650  if (use_precomputed_table == 1) {
+
651  dis0 = coarse_dis;
+
652 
+
653  const float * s = &ivfpq.precomputed_table [key * pq.ksub * pq.M];
+
654  for (int m = 0; m < pq.M; m++) {
+
655  sim_table_ptrs [m] = s;
+
656  s += pq.ksub;
+
657  }
+
658  } else if (use_precomputed_table == 2) {
+
659  dis0 = coarse_dis;
+
660 
+
661  const MultiIndexQuantizer *miq =
+
662  dynamic_cast<const MultiIndexQuantizer *> (ivfpq.quantizer);
+
663  FAISS_THROW_IF_NOT (miq);
+
664  const ProductQuantizer &cpq = miq->pq;
+
665  int Mf = pq.M / cpq.M;
+
666 
+
667  long k = key;
+
668  int m0 = 0;
+
669  for (int cm = 0; cm < cpq.M; cm++) {
+
670  int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
+
671  k >>= cpq.nbits;
672 
-
673  const MultiIndexQuantizer *miq =
-
674  dynamic_cast<const MultiIndexQuantizer *> (ivfpq.quantizer);
-
675  FAISS_THROW_IF_NOT (miq);
-
676  const ProductQuantizer &cpq = miq->pq;
-
677  int Mf = pq.M / cpq.M;
-
678 
-
679  long k = key;
-
680  int m0 = 0;
-
681  for (int cm = 0; cm < cpq.M; cm++) {
-
682  int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
-
683  k >>= cpq.nbits;
-
684 
-
685  const float *pc = &ivfpq.precomputed_table
-
686  [(ki * pq.M + cm * Mf) * pq.ksub];
-
687 
-
688  for (int m = m0; m < m0 + Mf; m++) {
-
689  sim_table_ptrs [m] = pc;
-
690  pc += pq.ksub;
-
691  }
-
692  m0 += Mf;
-
693  }
-
694  } else {
-
695  FAISS_THROW_MSG ("need precomputed tables");
-
696  }
-
697 
-
698  if (ivfpq.polysemous_ht) {
-
699  FAISS_THROW_MSG ("not implemented");
-
700  // Not clear that it makes sense to implemente this,
-
701  // because it costs M * ksub, which is what we wanted to
-
702  // avoid with the tables pointers.
-
703  }
-
704 
-
705  return dis0;
-
706  }
+
673  const float *pc = &ivfpq.precomputed_table
+
674  [(ki * pq.M + cm * Mf) * pq.ksub];
+
675 
+
676  for (int m = m0; m < m0 + Mf; m++) {
+
677  sim_table_ptrs [m] = pc;
+
678  pc += pq.ksub;
+
679  }
+
680  m0 += Mf;
+
681  }
+
682  } else {
+
683  FAISS_THROW_MSG ("need precomputed tables");
+
684  }
+
685 
+
686  if (ivfpq.polysemous_ht) {
+
687  FAISS_THROW_MSG ("not implemented");
+
688  // Not clear that it makes sense to implemente this,
+
689  // because it costs M * ksub, which is what we wanted to
+
690  // avoid with the tables pointers.
+
691  }
+
692 
+
693  return dis0;
+
694  }
+
695 
+
696 
+
697 };
+
698 
+
699 
+
700 /*****************************************************
+
701  * Scaning the codes.
+
702  * The scanning functions call their favorite precompute_*
+
703  * function to precompute the tables they need.
+
704  *****************************************************/
+
705 template <typename IDType>
+
706 struct InvertedListScanner: QueryTables {
707 
-
708 
-
709 };
-
710 
+
708  const uint8_t * __restrict list_codes;
+
709  const IDType * list_ids;
+
710  size_t list_size;
711 
-
712 /*****************************************************
-
713  * Scaning the codes.
-
714  * The scanning functions call their favorite precompute_*
-
715  * function to precompute the tables they need.
-
716  *****************************************************/
-
717 template <typename IDType>
-
718 struct InvertedListScanner: QueryTables {
-
719 
-
720  const uint8_t * __restrict list_codes;
-
721  const IDType * list_ids;
-
722  size_t list_size;
-
723 
-
724  explicit InvertedListScanner (const IndexIVFPQ & ivfpq):
-
725  QueryTables (ivfpq)
-
726  {
-
727  FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
-
728  n_hamming_pass = 0;
-
729  }
-
730 
-
731  /// list_specific intialization
-
732  void init_list (Index::idx_t key, float coarse_dis,
-
733  size_t list_size_in, const IDType *list_ids_in,
-
734  const uint8_t *list_codes_in) {
-
735  this->key = key;
-
736  this->coarse_dis = coarse_dis;
-
737  list_size = list_size_in;
-
738  list_codes = list_codes_in;
-
739  list_ids = list_ids_in;
-
740  }
+
712  explicit InvertedListScanner (const IndexIVFPQ & ivfpq):
+
713  QueryTables (ivfpq)
+
714  {
+
715  FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
+
716  n_hamming_pass = 0;
+
717  }
+
718 
+
719  /// list_specific intialization
+
720  void init_list (Index::idx_t key, float coarse_dis,
+
721  size_t list_size_in, const IDType *list_ids_in,
+
722  const uint8_t *list_codes_in) {
+
723  this->key = key;
+
724  this->coarse_dis = coarse_dis;
+
725  list_size = list_size_in;
+
726  list_codes = list_codes_in;
+
727  list_ids = list_ids_in;
+
728  }
+
729 
+
730  /*****************************************************
+
731  * Scaning the codes: simple PQ scan.
+
732  *****************************************************/
+
733 
+
734  /// version of the scan where we use precomputed tables
+
735  void scan_list_with_table (
+
736  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
+
737  {
+
738  float dis0 = precompute_list_tables ();
+
739 
+
740  for (size_t j = 0; j < list_size; j++) {
741 
-
742  /*****************************************************
-
743  * Scaning the codes: simple PQ scan.
-
744  *****************************************************/
-
745 
-
746  /// version of the scan where we use precomputed tables
-
747  void scan_list_with_table (
-
748  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
-
749  {
-
750  float dis0 = precompute_list_tables ();
-
751 
-
752  for (size_t j = 0; j < list_size; j++) {
-
753 
-
754  float dis = dis0;
-
755  const float *tab = sim_table;
-
756 
-
757  for (size_t m = 0; m < pq.M; m++) {
-
758  dis += tab[*list_codes++];
-
759  tab += pq.ksub;
-
760  }
-
761 
-
762  if (dis < heap_sim[0]) {
-
763  maxheap_pop (k, heap_sim, heap_ids);
-
764  long id = store_pairs ? (key << 32 | j) : list_ids[j];
-
765  maxheap_push (k, heap_sim, heap_ids, dis, id);
-
766  }
-
767  }
-
768  }
-
769 
-
770 
-
771  /// tables are not precomputed, but pointers are provided to the
-
772  /// relevant X_c|x_r tables
-
773  void scan_list_with_pointer (
-
774  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
-
775  {
-
776 
-
777  float dis0 = precompute_list_table_pointers ();
-
778 
-
779  for (size_t j = 0; j < list_size; j++) {
-
780 
-
781  float dis = dis0;
-
782  const float *tab = sim_table_2;
-
783 
-
784  for (size_t m = 0; m < pq.M; m++) {
-
785  int ci = *list_codes++;
-
786  dis += sim_table_ptrs [m][ci] - 2 * tab [ci];
-
787  tab += pq.ksub;
-
788  }
-
789 
-
790  if (dis < heap_sim[0]) {
-
791  maxheap_pop (k, heap_sim, heap_ids);
-
792  long id = store_pairs ? (key << 32 | j) : list_ids[j];
-
793  maxheap_push (k, heap_sim, heap_ids, dis, id);
-
794  }
+
742  float dis = dis0;
+
743  const float *tab = sim_table;
+
744 
+
745  for (size_t m = 0; m < pq.M; m++) {
+
746  dis += tab[*list_codes++];
+
747  tab += pq.ksub;
+
748  }
+
749 
+
750  if (dis < heap_sim[0]) {
+
751  maxheap_pop (k, heap_sim, heap_ids);
+
752  long id = store_pairs ? (key << 32 | j) : list_ids[j];
+
753  maxheap_push (k, heap_sim, heap_ids, dis, id);
+
754  }
+
755  }
+
756  }
+
757 
+
758 
+
759  /// tables are not precomputed, but pointers are provided to the
+
760  /// relevant X_c|x_r tables
+
761  void scan_list_with_pointer (
+
762  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
+
763  {
+
764 
+
765  float dis0 = precompute_list_table_pointers ();
+
766 
+
767  for (size_t j = 0; j < list_size; j++) {
+
768 
+
769  float dis = dis0;
+
770  const float *tab = sim_table_2;
+
771 
+
772  for (size_t m = 0; m < pq.M; m++) {
+
773  int ci = *list_codes++;
+
774  dis += sim_table_ptrs [m][ci] - 2 * tab [ci];
+
775  tab += pq.ksub;
+
776  }
+
777 
+
778  if (dis < heap_sim[0]) {
+
779  maxheap_pop (k, heap_sim, heap_ids);
+
780  long id = store_pairs ? (key << 32 | j) : list_ids[j];
+
781  maxheap_push (k, heap_sim, heap_ids, dis, id);
+
782  }
+
783  }
+
784 
+
785  }
+
786 
+
787  /// nothing is precomputed: access residuals on-the-fly
+
788  void scan_on_the_fly_dist (
+
789  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
+
790  {
+
791 
+
792  if (by_residual && use_precomputed_table) {
+
793  scan_list_with_pointer (k, heap_sim, heap_ids, store_pairs);
+
794  return;
795  }
796 
-
797  }
-
798 
-
799  /// nothing is precomputed: access residuals on-the-fly
-
800  void scan_on_the_fly_dist (
-
801  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
-
802  {
-
803 
-
804  if (by_residual && use_precomputed_table) {
-
805  scan_list_with_pointer (k, heap_sim, heap_ids, store_pairs);
-
806  return;
-
807  }
-
808 
-
809  const float *dvec;
-
810  float dis0 = 0;
-
811 
-
812  if (by_residual) {
-
813  if (metric_type == METRIC_INNER_PRODUCT) {
-
814  ivfpq.quantizer->reconstruct (key, residual_vec);
-
815  dis0 = fvec_inner_product (residual_vec, qi, d);
-
816  } else {
-
817  ivfpq.quantizer->compute_residual (qi, residual_vec, key);
-
818  }
-
819  dvec = residual_vec;
-
820  } else {
-
821  dvec = qi;
-
822  dis0 = 0;
-
823  }
+
797  const float *dvec;
+
798  float dis0 = 0;
+
799 
+
800  if (by_residual) {
+
801  if (metric_type == METRIC_INNER_PRODUCT) {
+
802  ivfpq.quantizer->reconstruct (key, residual_vec);
+
803  dis0 = fvec_inner_product (residual_vec, qi, d);
+
804  } else {
+
805  ivfpq.quantizer->compute_residual (qi, residual_vec, key);
+
806  }
+
807  dvec = residual_vec;
+
808  } else {
+
809  dvec = qi;
+
810  dis0 = 0;
+
811  }
+
812 
+
813  for (size_t j = 0; j < list_size; j++) {
+
814 
+
815  pq.decode (list_codes, decoded_vec);
+
816  list_codes += pq.code_size;
+
817 
+
818  float dis;
+
819  if (metric_type == METRIC_INNER_PRODUCT) {
+
820  dis = -dis0 - fvec_inner_product (decoded_vec, qi, d);
+
821  } else {
+
822  dis = fvec_L2sqr (decoded_vec, dvec, d);
+
823  }
824 
-
825  for (size_t j = 0; j < list_size; j++) {
-
826 
-
827  pq.decode (list_codes, decoded_vec);
-
828  list_codes += pq.code_size;
-
829 
-
830  float dis;
-
831  if (metric_type == METRIC_INNER_PRODUCT) {
-
832  dis = -dis0 - fvec_inner_product (decoded_vec, qi, d);
-
833  } else {
-
834  dis = fvec_L2sqr (decoded_vec, dvec, d);
-
835  }
+
825  if (dis < heap_sim[0]) {
+
826  maxheap_pop (k, heap_sim, heap_ids);
+
827  long id = store_pairs ? (key << 32 | j) : list_ids[j];
+
828  maxheap_push (k, heap_sim, heap_ids, dis, id);
+
829  }
+
830  }
+
831  }
+
832 
+
833  /*****************************************************
+
834  * Scanning codes with polysemous filtering
+
835  *****************************************************/
836 
-
837  if (dis < heap_sim[0]) {
-
838  maxheap_pop (k, heap_sim, heap_ids);
-
839  long id = store_pairs ? (key << 32 | j) : list_ids[j];
-
840  maxheap_push (k, heap_sim, heap_ids, dis, id);
-
841  }
-
842  }
-
843  }
-
844 
-
845  /*****************************************************
-
846  * Scanning codes with polysemous filtering
-
847  *****************************************************/
-
848 
-
849  // code for the query
-
850  size_t n_hamming_pass;
+
837  // code for the query
+
838  size_t n_hamming_pass;
+
839 
+
840 
+
841  template <class HammingComputer>
+
842  void scan_list_polysemous_hc (
+
843  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
+
844  {
+
845  float dis0 = precompute_list_tables ();
+
846  int ht = ivfpq.polysemous_ht;
+
847 
+
848  int code_size = pq.code_size;
+
849 
+
850  HammingComputer hc (q_code.data(), code_size);
851 
-
852 
-
853  template <class HammingComputer>
-
854  void scan_list_polysemous_hc (
-
855  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
-
856  {
-
857  float dis0 = precompute_list_tables ();
-
858  int ht = ivfpq.polysemous_ht;
-
859 
-
860  int code_size = pq.code_size;
-
861 
-
862  HammingComputer hc (q_code.data(), code_size);
-
863 
-
864  for (size_t j = 0; j < list_size; j++) {
-
865  const uint8_t *b_code = list_codes;
-
866  int hd = hc.hamming (b_code);
-
867  if (hd < ht) {
-
868  n_hamming_pass ++;
-
869 
-
870  float dis = dis0;
-
871  const float *tab = sim_table;
-
872 
-
873  for (size_t m = 0; m < pq.M; m++) {
-
874  dis += tab[*b_code++];
-
875  tab += pq.ksub;
-
876  }
-
877 
-
878  if (dis < heap_sim[0]) {
-
879  maxheap_pop (k, heap_sim, heap_ids);
-
880  long id = store_pairs ? (key << 32 | j) : list_ids[j];
-
881  maxheap_push (k, heap_sim, heap_ids, dis, id);
-
882  }
-
883  }
-
884  list_codes += code_size;
-
885  }
-
886  }
-
887 
-
888  void scan_list_polysemous (
-
889  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
-
890  {
-
891  switch (pq.code_size) {
-
892 #define HANDLE_CODE_SIZE(cs) \
-
893  case cs: \
-
894  scan_list_polysemous_hc <HammingComputer ## cs> \
-
895  (k, heap_sim, heap_ids, store_pairs); \
-
896  break
-
897  HANDLE_CODE_SIZE(4);
-
898  HANDLE_CODE_SIZE(8);
-
899  HANDLE_CODE_SIZE(16);
-
900  HANDLE_CODE_SIZE(20);
-
901  HANDLE_CODE_SIZE(32);
-
902  HANDLE_CODE_SIZE(64);
-
903 #undef HANDLE_CODE_SIZE
-
904  default:
-
905  if (pq.code_size % 8 == 0)
-
906  scan_list_polysemous_hc <HammingComputerM8>
-
907  (k, heap_sim, heap_ids, store_pairs);
-
908  else
-
909  scan_list_polysemous_hc <HammingComputerM4>
-
910  (k, heap_sim, heap_ids, store_pairs);
-
911  break;
-
912  }
-
913  }
-
914 
-
915 };
+
852  for (size_t j = 0; j < list_size; j++) {
+
853  const uint8_t *b_code = list_codes;
+
854  int hd = hc.hamming (b_code);
+
855  if (hd < ht) {
+
856  n_hamming_pass ++;
+
857 
+
858  float dis = dis0;
+
859  const float *tab = sim_table;
+
860 
+
861  for (size_t m = 0; m < pq.M; m++) {
+
862  dis += tab[*b_code++];
+
863  tab += pq.ksub;
+
864  }
+
865 
+
866  if (dis < heap_sim[0]) {
+
867  maxheap_pop (k, heap_sim, heap_ids);
+
868  long id = store_pairs ? (key << 32 | j) : list_ids[j];
+
869  maxheap_push (k, heap_sim, heap_ids, dis, id);
+
870  }
+
871  }
+
872  list_codes += code_size;
+
873  }
+
874  }
+
875 
+
876  void scan_list_polysemous (
+
877  size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
+
878  {
+
879  switch (pq.code_size) {
+
880 #define HANDLE_CODE_SIZE(cs) \
+
881  case cs: \
+
882  scan_list_polysemous_hc <HammingComputer ## cs> \
+
883  (k, heap_sim, heap_ids, store_pairs); \
+
884  break
+
885  HANDLE_CODE_SIZE(4);
+
886  HANDLE_CODE_SIZE(8);
+
887  HANDLE_CODE_SIZE(16);
+
888  HANDLE_CODE_SIZE(20);
+
889  HANDLE_CODE_SIZE(32);
+
890  HANDLE_CODE_SIZE(64);
+
891 #undef HANDLE_CODE_SIZE
+
892  default:
+
893  if (pq.code_size % 8 == 0)
+
894  scan_list_polysemous_hc <HammingComputerM8>
+
895  (k, heap_sim, heap_ids, store_pairs);
+
896  else
+
897  scan_list_polysemous_hc <HammingComputerM4>
+
898  (k, heap_sim, heap_ids, store_pairs);
+
899  break;
+
900  }
+
901  }
+
902 
+
903 };
+
904 
+
905 
+
906 
+
907 
+
908 } // anonymous namespace
+
909 
+
910 
+
911 IndexIVFPQStats indexIVFPQ_stats;
+
912 
+
913 void IndexIVFPQStats::reset () {
+
914  memset (this, 0, sizeof (*this));
+
915 }
916 
917 
918 
-
919 
-
920 } // anonymous namespace
-
921 
-
922 
-
923 IndexIVFPQStats indexIVFPQ_stats;
-
924 
-
925 void IndexIVFPQStats::reset () {
-
926  memset (this, 0, sizeof (*this));
-
927 }
-
928 
+
919 void IndexIVFPQ::search_preassigned (idx_t nx, const float *qx, idx_t k,
+
920  const idx_t *keys,
+
921  const float *coarse_dis,
+
922  float *distances, idx_t *labels,
+
923  bool store_pairs) const
+
924 {
+
925  float_maxheap_array_t res = {
+
926  size_t(nx), size_t(k),
+
927  labels, distances
+
928  };
929 
-
930 
-
931 void IndexIVFPQ::search_preassigned (idx_t nx, const float *qx, idx_t k,
-
932  const idx_t *keys,
-
933  const float *coarse_dis,
-
934  float *distances, idx_t *labels,
-
935  bool store_pairs) const
-
936 {
-
937  float_maxheap_array_t res = {
-
938  size_t(nx), size_t(k),
-
939  labels, distances
-
940  };
-
941 
-
942 #pragma omp parallel
-
943  {
-
944  InvertedListScanner<long> qt (*this);
-
945  size_t stats_nlist = 0;
-
946  size_t stats_ncode = 0;
-
947  uint64_t init_query_cycles = 0;
-
948  uint64_t scan_cycles = 0;
-
949  uint64_t heap_cycles = 0;
-
950 
-
951 #pragma omp for
-
952  for (size_t i = 0; i < nx; i++) {
-
953  const float *qi = qx + i * d;
-
954  const long * keysi = keys + i * nprobe;
-
955  const float *coarse_dis_i = coarse_dis + i * nprobe;
-
956  float * heap_sim = res.get_val (i);
-
957  long * heap_ids = res.get_ids (i);
-
958 
-
959  uint64_t t0;
-
960  TIC;
-
961  maxheap_heapify (k, heap_sim, heap_ids);
-
962  heap_cycles += TOC;
-
963 
-
964  TIC;
-
965  qt.init_query (qi);
-
966  init_query_cycles += TOC;
-
967 
-
968  size_t nscan = 0;
-
969 
-
970  for (size_t ik = 0; ik < nprobe; ik++) {
-
971  long key = keysi[ik]; /* select the list */
-
972  if (key < 0) {
-
973  // not enough centroids for multiprobe
-
974  continue;
-
975  }
-
976  FAISS_THROW_IF_NOT_FMT (
-
977  key < (long) nlist,
-
978  "Invalid key=%ld at ik=%ld nlist=%ld\n",
-
979  key, ik, nlist);
-
980 
-
981  size_t list_size = ids[key].size();
-
982  stats_nlist ++;
-
983  nscan += list_size;
-
984 
-
985  if (list_size == 0) continue;
-
986 
-
987  qt.init_list (key, coarse_dis_i[ik],
-
988  list_size, ids[key].data(),
-
989  codes[key].data());
-
990 
-
991  TIC;
-
992  if (polysemous_ht > 0) {
-
993  qt.scan_list_polysemous
-
994  (k, heap_sim, heap_ids, store_pairs);
-
995  } else if (list_size > scan_table_threshold) {
-
996  qt.scan_list_with_table (k, heap_sim, heap_ids, store_pairs);
-
997  } else {
-
998  qt.scan_on_the_fly_dist (k, heap_sim, heap_ids, store_pairs);
-
999  }
-
1000  scan_cycles += TOC;
-
1001 
-
1002  if (max_codes && nscan >= max_codes) break;
-
1003  }
-
1004  stats_ncode += nscan;
-
1005  TIC;
-
1006  maxheap_reorder (k, heap_sim, heap_ids);
-
1007 
-
1008  if (metric_type == METRIC_INNER_PRODUCT) {
-
1009  for (size_t j = 0; j < k; j++)
-
1010  heap_sim[j] = -heap_sim[j];
-
1011  }
-
1012  heap_cycles += TOC;
+
930 #pragma omp parallel
+
931  {
+
932  InvertedListScanner<long> qt (*this);
+
933  size_t stats_nlist = 0;
+
934  size_t stats_ncode = 0;
+
935  uint64_t init_query_cycles = 0;
+
936  uint64_t scan_cycles = 0;
+
937  uint64_t heap_cycles = 0;
+
938 
+
939 #pragma omp for
+
940  for (size_t i = 0; i < nx; i++) {
+
941  const float *qi = qx + i * d;
+
942  const long * keysi = keys + i * nprobe;
+
943  const float *coarse_dis_i = coarse_dis + i * nprobe;
+
944  float * heap_sim = res.get_val (i);
+
945  long * heap_ids = res.get_ids (i);
+
946 
+
947  uint64_t t0;
+
948  TIC;
+
949  maxheap_heapify (k, heap_sim, heap_ids);
+
950  heap_cycles += TOC;
+
951 
+
952  TIC;
+
953  qt.init_query (qi);
+
954  init_query_cycles += TOC;
+
955 
+
956  size_t nscan = 0;
+
957 
+
958  for (size_t ik = 0; ik < nprobe; ik++) {
+
959  long key = keysi[ik]; /* select the list */
+
960  if (key < 0) {
+
961  // not enough centroids for multiprobe
+
962  continue;
+
963  }
+
964  FAISS_THROW_IF_NOT_FMT (
+
965  key < (long) nlist,
+
966  "Invalid key=%ld at ik=%ld nlist=%ld\n",
+
967  key, ik, nlist);
+
968 
+
969  size_t list_size = ids[key].size();
+
970  stats_nlist ++;
+
971  nscan += list_size;
+
972 
+
973  if (list_size == 0) continue;
+
974 
+
975  qt.init_list (key, coarse_dis_i[ik],
+
976  list_size, ids[key].data(),
+
977  codes[key].data());
+
978 
+
979  TIC;
+
980  if (polysemous_ht > 0) {
+
981  qt.scan_list_polysemous
+
982  (k, heap_sim, heap_ids, store_pairs);
+
983  } else if (list_size > scan_table_threshold) {
+
984  qt.scan_list_with_table (k, heap_sim, heap_ids, store_pairs);
+
985  } else {
+
986  qt.scan_on_the_fly_dist (k, heap_sim, heap_ids, store_pairs);
+
987  }
+
988  scan_cycles += TOC;
+
989 
+
990  if (max_codes && nscan >= max_codes) break;
+
991  }
+
992  stats_ncode += nscan;
+
993  TIC;
+
994  maxheap_reorder (k, heap_sim, heap_ids);
+
995 
+
996  if (metric_type == METRIC_INNER_PRODUCT) {
+
997  for (size_t j = 0; j < k; j++)
+
998  heap_sim[j] = -heap_sim[j];
+
999  }
+
1000  heap_cycles += TOC;
+
1001  }
+
1002 
+
1003 #pragma omp critical
+
1004  {
+
1005  indexIVFPQ_stats.n_hamming_pass += qt.n_hamming_pass;
+
1006  indexIVFPQ_stats.nlist += stats_nlist;
+
1007  indexIVFPQ_stats.ncode += stats_ncode;
+
1008 
+
1009  indexIVFPQ_stats.init_query_cycles += init_query_cycles;
+
1010  indexIVFPQ_stats.init_list_cycles += qt.init_list_cycles;
+
1011  indexIVFPQ_stats.scan_cycles += scan_cycles - qt.init_list_cycles;
+
1012  indexIVFPQ_stats.heap_cycles += heap_cycles;
1013  }
1014 
-
1015 #pragma omp critical
-
1016  {
-
1017  indexIVFPQ_stats.n_hamming_pass += qt.n_hamming_pass;
-
1018  indexIVFPQ_stats.nlist += stats_nlist;
-
1019  indexIVFPQ_stats.ncode += stats_ncode;
-
1020 
-
1021  indexIVFPQ_stats.init_query_cycles += init_query_cycles;
-
1022  indexIVFPQ_stats.init_list_cycles += qt.init_list_cycles;
-
1023  indexIVFPQ_stats.scan_cycles += scan_cycles - qt.init_list_cycles;
-
1024  indexIVFPQ_stats.heap_cycles += heap_cycles;
-
1025  }
-
1026 
-
1027  }
-
1028  indexIVFPQ_stats.nq += nx;
-
1029 }
+
1015  }
+
1016  indexIVFPQ_stats.nq += nx;
+
1017 }
+
1018 
+
1019 
+
1020 IndexIVFPQ::IndexIVFPQ ()
+
1021 {
+
1022  // initialize some runtime values
+
1023  use_precomputed_table = 0;
+
1024  scan_table_threshold = 0;
+
1025  do_polysemous_training = false;
+
1026  polysemous_ht = 0;
+
1027  polysemous_training = nullptr;
+
1028 }
+
1029 
1030 
-
1031 
-
1032 void IndexIVFPQ::search_and_reconstruct (idx_t n, const float *x, idx_t k,
-
1033  float *distances, idx_t *labels,
-
1034  float *reconstructed)
-
1035 {
-
1036  long * idx = new long [n * nprobe];
-
1037  ScopeDeleter<long> del (idx);
-
1038  float * coarse_dis = new float [n * nprobe];
-
1039  ScopeDeleter<float> del2 (coarse_dis);
-
1040 
-
1041  quantizer->search (n, x, nprobe, coarse_dis, idx);
+
1031 struct CodeCmp {
+
1032  const uint8_t *tab;
+
1033  size_t code_size;
+
1034  bool operator () (int a, int b) const {
+
1035  return cmp (a, b) > 0;
+
1036  }
+
1037  int cmp (int a, int b) const {
+
1038  return memcmp (tab + a * code_size, tab + b * code_size,
+
1039  code_size);
+
1040  }
+
1041 };
1042 
-
1043  search_preassigned (n, x, k, idx, coarse_dis,
-
1044  distances, labels, true);
-
1045 
-
1046  for (long i = 0; i < n; i++) {
-
1047  for (long j = 0; j < k; j++) {
-
1048  long ij = i * k + j;
-
1049  idx_t res = labels[ij];
-
1050  float *recons = reconstructed + d * (ij);
-
1051  if (res < 0) {
-
1052  // fill with NaNs
-
1053  memset(recons, -1, sizeof(*recons) * d);
-
1054  } else {
-
1055  int list_no = res >> 32;
-
1056  int ofs = res & 0xffffffff;
-
1057  labels[ij] = ids[list_no][ofs];
-
1058 
-
1059  quantizer->reconstruct (list_no, recons);
-
1060  const uint8_t * code = &(codes[list_no][ofs * pq.code_size]);
-
1061 
-
1062  for (size_t m = 0; m < pq.M; m++) {
-
1063  float * out = recons + m * pq.dsub;
-
1064  const float * cent = pq.get_centroids (m, code[m]);
-
1065  for (size_t l = 0; l < pq.dsub; l++) {
-
1066  out[l] += cent[l];
-
1067  }
-
1068  }
-
1069  }
-
1070  }
-
1071  }
-
1072 
-
1073 }
+
1043 
+
1044 size_t IndexIVFPQ::find_duplicates (idx_t *dup_ids, size_t *lims) const
+
1045 {
+
1046  size_t ngroup = 0;
+
1047  lims[0] = 0;
+
1048  for (size_t list_no = 0; list_no < nlist; list_no++) {
+
1049  size_t n = ids[list_no].size();
+
1050  std::vector<int> ord (n);
+
1051  for (int i = 0; i < n; i++) ord[i] = i;
+
1052  CodeCmp cs = { codes[list_no].data(), code_size };
+
1053  std::sort (ord.begin(), ord.end(), cs);
+
1054 
+
1055  const idx_t *list_ids = ids[list_no].data();
+
1056  int prev = -1; // all elements from prev to i-1 are equal
+
1057  for (int i = 0; i < n; i++) {
+
1058  if (prev >= 0 && cs.cmp (ord [prev], ord [i]) == 0) {
+
1059  // same as previous => remember
+
1060  if (prev + 1 == i) { // start new group
+
1061  ngroup++;
+
1062  lims[ngroup] = lims[ngroup - 1];
+
1063  dup_ids [lims [ngroup]++] = list_ids [ord [prev]];
+
1064  }
+
1065  dup_ids [lims [ngroup]++] = list_ids [ord [i]];
+
1066  } else { // not same as previous.
+
1067  prev = i;
+
1068  }
+
1069  }
+
1070  }
+
1071  return ngroup;
+
1072 }
+
1073 
1074 
1075 
1076 
-
1077 
-
1078 
-
1079 IndexIVFPQ::IndexIVFPQ ()
-
1080 {
-
1081  // initialize some runtime values
-
1082  use_precomputed_table = 0;
-
1083  scan_table_threshold = 0;
-
1084  do_polysemous_training = false;
-
1085  polysemous_ht = 0;
-
1086  max_codes = 0;
-
1087  polysemous_training = nullptr;
-
1088 }
-
1089 
-
1090 
-
1091 struct CodeCmp {
-
1092  const uint8_t *tab;
-
1093  size_t code_size;
-
1094  bool operator () (int a, int b) const {
-
1095  return cmp (a, b) > 0;
-
1096  }
-
1097  int cmp (int a, int b) const {
-
1098  return memcmp (tab + a * code_size, tab + b * code_size,
-
1099  code_size);
-
1100  }
-
1101 };
-
1102 
-
1103 
-
1104 size_t IndexIVFPQ::find_duplicates (idx_t *dup_ids, size_t *lims) const
-
1105 {
-
1106  size_t ngroup = 0;
-
1107  lims[0] = 0;
-
1108  for (size_t list_no = 0; list_no < nlist; list_no++) {
-
1109  size_t n = ids[list_no].size();
-
1110  std::vector<int> ord (n);
-
1111  for (int i = 0; i < n; i++) ord[i] = i;
-
1112  CodeCmp cs = { codes[list_no].data(), code_size };
-
1113  std::sort (ord.begin(), ord.end(), cs);
+
1077 /*****************************************
+
1078  * IndexIVFPQR implementation
+
1079  ******************************************/
+
1080 
+
1081 IndexIVFPQR::IndexIVFPQR (
+
1082  Index * quantizer, size_t d, size_t nlist,
+
1083  size_t M, size_t nbits_per_idx,
+
1084  size_t M_refine, size_t nbits_per_idx_refine):
+
1085  IndexIVFPQ (quantizer, d, nlist, M, nbits_per_idx),
+
1086  refine_pq (d, M_refine, nbits_per_idx_refine),
+
1087  k_factor (4)
+
1088 {
+
1089  by_residual = true;
+
1090 }
+
1091 
+
1092 IndexIVFPQR::IndexIVFPQR ():
+
1093  k_factor (1)
+
1094 {
+
1095  by_residual = true;
+
1096 }
+
1097 
+
1098 
+
1099 
+
1100 void IndexIVFPQR::reset()
+
1101 {
+
1102  IndexIVFPQ::reset();
+
1103  refine_codes.clear();
+
1104 }
+
1105 
+
1106 
+
1107 
+
1108 
+
1109 void IndexIVFPQR::train_residual (idx_t n, const float *x)
+
1110 {
+
1111 
+
1112  float * residual_2 = new float [n * d];
+
1113  ScopeDeleter <float> del(residual_2);
1114 
-
1115  const idx_t *list_ids = ids[list_no].data();
-
1116  int prev = -1; // all elements from prev to i-1 are equal
-
1117  for (int i = 0; i < n; i++) {
-
1118  if (prev >= 0 && cs.cmp (ord [prev], ord [i]) == 0) {
-
1119  // same as previous => remember
-
1120  if (prev + 1 == i) { // start new group
-
1121  ngroup++;
-
1122  lims[ngroup] = lims[ngroup - 1];
-
1123  dup_ids [lims [ngroup]++] = list_ids [ord [prev]];
-
1124  }
-
1125  dup_ids [lims [ngroup]++] = list_ids [ord [i]];
-
1126  } else { // not same as previous.
-
1127  prev = i;
-
1128  }
-
1129  }
-
1130  }
-
1131  return ngroup;
-
1132 }
-
1133 
-
1134 
+
1115  train_residual_o (n, x, residual_2);
+
1116 
+
1117  if (verbose)
+
1118  printf ("training %zdx%zd 2nd level PQ quantizer on %ld %dD-vectors\n",
+
1119  refine_pq.M, refine_pq.ksub, n, d);
+
1120 
+
1121  refine_pq.cp.max_points_per_centroid = 1000;
+
1122  refine_pq.cp.verbose = verbose;
+
1123 
+
1124  refine_pq.train (n, residual_2);
+
1125 
+
1126 }
+
1127 
+
1128 
+
1129 void IndexIVFPQR::add_with_ids (idx_t n, const float *x, const long *xids) {
+
1130  add_core (n, x, xids, nullptr);
+
1131 }
+
1132 
+
1133 void IndexIVFPQR::add_core (idx_t n, const float *x, const long *xids,
+
1134  const long *precomputed_idx) {
1135 
-
1136 
-
1137 /*****************************************
-
1138  * IndexIVFPQR implementation
-
1139  ******************************************/
+
1136  float * residual_2 = new float [n * d];
+
1137  ScopeDeleter <float> del(residual_2);
+
1138 
+
1139  idx_t n0 = ntotal;
1140 
-
1141 IndexIVFPQR::IndexIVFPQR (
-
1142  Index * quantizer, size_t d, size_t nlist,
-
1143  size_t M, size_t nbits_per_idx,
-
1144  size_t M_refine, size_t nbits_per_idx_refine):
-
1145  IndexIVFPQ (quantizer, d, nlist, M, nbits_per_idx),
-
1146  refine_pq (d, M_refine, nbits_per_idx_refine),
-
1147  k_factor (4)
-
1148 {
-
1149  by_residual = true;
-
1150 }
+
1141  add_core_o (n, x, xids, residual_2, precomputed_idx);
+
1142 
+
1143  refine_codes.resize (ntotal * refine_pq.code_size);
+
1144 
+
1145  refine_pq.compute_codes (
+
1146  residual_2, &refine_codes[n0 * refine_pq.code_size], n);
+
1147 
+
1148 
+
1149 }
+
1150 
1151 
-
1152 IndexIVFPQR::IndexIVFPQR ():
-
1153  k_factor (1)
-
1154 {
-
1155  by_residual = true;
-
1156 }
-
1157 
-
1158 
-
1159 
-
1160 void IndexIVFPQR::reset()
-
1161 {
-
1162  IndexIVFPQ::reset();
-
1163  refine_codes.clear();
-
1164 }
-
1165 
+
1152 void IndexIVFPQR::search_preassigned (idx_t n, const float *x, idx_t k,
+
1153  const idx_t *idx,
+
1154  const float *L1_dis,
+
1155  float *distances, idx_t *labels,
+
1156  bool store_pairs) const
+
1157 {
+
1158  uint64_t t0;
+
1159  TIC;
+
1160  size_t k_coarse = long(k * k_factor);
+
1161  idx_t *coarse_labels = new idx_t [k_coarse * n];
+
1162  ScopeDeleter<idx_t> del1 (coarse_labels);
+
1163  { // query with quantizer levels 1 and 2.
+
1164  float *coarse_distances = new float [k_coarse * n];
+
1165  ScopeDeleter<float> del(coarse_distances);
1166 
-
1167 
-
1168 
-
1169 void IndexIVFPQR::train_residual (idx_t n, const float *x)
-
1170 {
+
1167  IndexIVFPQ::search_preassigned (n, x, k_coarse,
+
1168  idx, L1_dis, coarse_distances, coarse_labels,
+
1169  true);
+
1170  }
1171 
-
1172  float * residual_2 = new float [n * d];
-
1173  ScopeDeleter <float> del(residual_2);
+
1172 
+
1173  indexIVFPQ_stats.search_cycles += TOC;
1174 
-
1175  train_residual_o (n, x, residual_2);
+
1175  TIC;
1176 
-
1177  if (verbose)
-
1178  printf ("training %zdx%zd 2nd level PQ quantizer on %ld %dD-vectors\n",
-
1179  refine_pq.M, refine_pq.ksub, n, d);
-
1180 
-
1181  refine_pq.cp.max_points_per_centroid = 1000;
-
1182  refine_pq.cp.verbose = verbose;
-
1183 
-
1184  refine_pq.train (n, residual_2);
-
1185 
-
1186 }
-
1187 
-
1188 
-
1189 void IndexIVFPQR::add_with_ids (idx_t n, const float *x, const long *xids) {
-
1190  add_core (n, x, xids, nullptr);
-
1191 }
+
1177  // 3rd level refinement
+
1178  size_t n_refine = 0;
+
1179 #pragma omp parallel reduction(+ : n_refine)
+
1180  {
+
1181  // tmp buffers
+
1182  float *residual_1 = new float [2 * d];
+
1183  ScopeDeleter<float> del (residual_1);
+
1184  float *residual_2 = residual_1 + d;
+
1185 #pragma omp for
+
1186  for (idx_t i = 0; i < n; i++) {
+
1187  const float *xq = x + i * d;
+
1188  const long * shortlist = coarse_labels + k_coarse * i;
+
1189  float * heap_sim = distances + k * i;
+
1190  long * heap_ids = labels + k * i;
+
1191  maxheap_heapify (k, heap_sim, heap_ids);
1192 
-
1193 void IndexIVFPQR::add_core (idx_t n, const float *x, const long *xids,
-
1194  const long *precomputed_idx) {
+
1193  for (int j = 0; j < k_coarse; j++) {
+
1194  long sl = shortlist[j];
1195 
-
1196  float * residual_2 = new float [n * d];
-
1197  ScopeDeleter <float> del(residual_2);
-
1198 
-
1199  idx_t n0 = ntotal;
+
1196  if (sl == -1) continue;
+
1197 
+
1198  int list_no = sl >> 32;
+
1199  int ofs = sl & 0xffffffff;
1200 
-
1201  add_core_o (n, x, xids, residual_2, precomputed_idx);
-
1202 
-
1203  refine_codes.resize (ntotal * refine_pq.code_size);
-
1204 
-
1205  refine_pq.compute_codes (
-
1206  residual_2, &refine_codes[n0 * refine_pq.code_size], n);
-
1207 
-
1208 
-
1209 }
-
1210 
-
1211 
-
1212 void IndexIVFPQR::search (
-
1213  idx_t n, const float *x, idx_t k,
-
1214  float *distances, idx_t *labels) const
-
1215 {
-
1216  FAISS_THROW_IF_NOT (is_trained);
-
1217  long * idx = new long [n * nprobe];
-
1218  ScopeDeleter<long> del (idx);
-
1219  float * L1_dis = new float [n * nprobe];
-
1220  ScopeDeleter<float> del2 (L1_dis);
-
1221  uint64_t t0;
-
1222  TIC;
-
1223  quantizer->search (n, x, nprobe, L1_dis, idx);
-
1224  indexIVFPQ_stats.assign_cycles += TOC;
-
1225 
-
1226  TIC;
-
1227  size_t k_coarse = long(k * k_factor);
-
1228  idx_t *coarse_labels = new idx_t [k_coarse * n];
-
1229  ScopeDeleter<idx_t> del3 (coarse_labels);
-
1230  { // query with quantizer levels 1 and 2.
-
1231  float *coarse_distances = new float [k_coarse * n];
-
1232  ScopeDeleter<float> del(coarse_distances);
-
1233 
-
1234  search_preassigned (n, x, k_coarse,
-
1235  idx, L1_dis, coarse_distances, coarse_labels,
-
1236  true);
-
1237  }
-
1238 
+
1201  assert (list_no >= 0 && list_no < nlist);
+
1202  assert (ofs >= 0 && ofs < ids[list_no].size());
+
1203 
+
1204  // 1st level residual
+
1205  quantizer->compute_residual (xq, residual_1, list_no);
+
1206 
+
1207  // 2nd level residual
+
1208  const uint8_t * l2code = &codes[list_no][ofs * pq.code_size];
+
1209  pq.decode (l2code, residual_2);
+
1210  for (int l = 0; l < d; l++)
+
1211  residual_2[l] = residual_1[l] - residual_2[l];
+
1212 
+
1213  // 3rd level residual's approximation
+
1214  idx_t id = ids[list_no][ofs];
+
1215  assert (0 <= id && id < ntotal);
+
1216  refine_pq.decode (&refine_codes [id * refine_pq.code_size],
+
1217  residual_1);
+
1218 
+
1219  float dis = fvec_L2sqr (residual_1, residual_2, d);
+
1220 
+
1221  if (dis < heap_sim[0]) {
+
1222  maxheap_pop (k, heap_sim, heap_ids);
+
1223  long id_or_pair = store_pairs ? sl : id;
+
1224  maxheap_push (k, heap_sim, heap_ids, dis, id_or_pair);
+
1225  }
+
1226  n_refine ++;
+
1227  }
+
1228  maxheap_reorder (k, heap_sim, heap_ids);
+
1229  }
+
1230  }
+
1231  indexIVFPQ_stats.nrefine += n_refine;
+
1232  indexIVFPQ_stats.refine_cycles += TOC;
+
1233 }
+
1234 
+
1235 void IndexIVFPQR::reconstruct_from_offset (long list_no, long offset,
+
1236  float* recons) const
+
1237 {
+
1238  IndexIVFPQ::reconstruct_from_offset (list_no, offset, recons);
1239 
-
1240  indexIVFPQ_stats.search_cycles += TOC;
-
1241 
-
1242  TIC;
-
1243 
-
1244  // 3rd level refinement
-
1245  size_t n_refine = 0;
-
1246 #pragma omp parallel reduction(+ : n_refine)
-
1247  {
-
1248  // tmp buffers
-
1249  float *residual_1 = new float [2 * d];
-
1250  ScopeDeleter<float> del (residual_1);
-
1251  float *residual_2 = residual_1 + d;
-
1252 #pragma omp for
-
1253  for (idx_t i = 0; i < n; i++) {
-
1254  const float *xq = x + i * d;
-
1255  const long * shortlist = coarse_labels + k_coarse * i;
-
1256  float * heap_sim = distances + k * i;
-
1257  long * heap_ids = labels + k * i;
-
1258  maxheap_heapify (k, heap_sim, heap_ids);
-
1259 
-
1260  for (int j = 0; j < k_coarse; j++) {
-
1261  long sl = shortlist[j];
+
1240  idx_t id = ids[list_no][offset];
+
1241  assert (0 <= id && id < ntotal);
+
1242 
+
1243  std::vector<float> r3(d);
+
1244  refine_pq.decode (&refine_codes [id * refine_pq.code_size], r3.data());
+
1245  for (int i = 0; i < d; ++i) {
+
1246  recons[i] += r3[i];
+
1247  }
+
1248 }
+
1249 
+
1250 void IndexIVFPQR::merge_from (IndexIVF &other_in, idx_t add_id)
+
1251 {
+
1252  IndexIVFPQR *other = dynamic_cast<IndexIVFPQR *> (&other_in);
+
1253  FAISS_THROW_IF_NOT(other);
+
1254 
+
1255  IndexIVF::merge_from (other_in, add_id);
+
1256 
+
1257  refine_codes.insert (refine_codes.end(),
+
1258  other->refine_codes.begin(),
+
1259  other->refine_codes.end());
+
1260  other->refine_codes.clear();
+
1261 }
1262 
-
1263  if (sl == -1) continue;
-
1264 
-
1265  int list_no = sl >> 32;
-
1266  int ofs = sl & 0xffffffff;
+
1263 long IndexIVFPQR::remove_ids(const IDSelector& /*sel*/) {
+
1264  FAISS_THROW_MSG("not implemented");
+
1265  return 0;
+
1266 }
1267 
-
1268  assert (list_no >= 0 && list_no < nlist);
-
1269  assert (ofs >= 0 && ofs < ids[list_no].size());
-
1270 
-
1271  // 1st level residual
-
1272  quantizer->compute_residual (xq, residual_1, list_no);
-
1273 
-
1274  // 2nd level residual
-
1275  const uint8_t * l2code = &codes[list_no][ofs * pq.code_size];
-
1276  pq.decode (l2code, residual_2);
-
1277  for (int l = 0; l < d; l++)
-
1278  residual_2[l] = residual_1[l] - residual_2[l];
-
1279 
-
1280  // 3rd level residual's approximation
-
1281  idx_t id = ids[list_no][ofs];
-
1282  assert (0 <= id && id < ntotal);
-
1283  refine_pq.decode (&refine_codes [id * refine_pq.code_size],
-
1284  residual_1);
-
1285 
-
1286  float dis = fvec_L2sqr (residual_1, residual_2, d);
-
1287 
-
1288  if (dis < heap_sim[0]) {
-
1289  maxheap_pop (k, heap_sim, heap_ids);
-
1290  maxheap_push (k, heap_sim, heap_ids, dis, id);
-
1291  }
-
1292  n_refine ++;
-
1293  }
-
1294  maxheap_reorder (k, heap_sim, heap_ids);
-
1295  }
-
1296  }
-
1297  indexIVFPQ_stats.nrefine += n_refine;
-
1298  indexIVFPQ_stats.refine_cycles += TOC;
-
1299 }
-
1300 
-
1301 void IndexIVFPQR::reconstruct_n (idx_t i0, idx_t ni, float *recons) const
-
1302 {
-
1303  std::vector<float> r3 (d);
-
1304 
-
1305  IndexIVFPQ::reconstruct_n (i0, ni, recons);
-
1306 
-
1307  for (idx_t i = i0; i < i0 + ni; i++) {
-
1308  float *r = recons + i * d;
-
1309  refine_pq.decode (&refine_codes [i * refine_pq.code_size], r3.data());
-
1310 
-
1311  for (int j = 0; j < d; j++)
-
1312  r[j] += r3[j];
+
1268 /*************************************
+
1269  * Index2Layer implementation
+
1270  *************************************/
+
1271 
+
1272 
+
1273 Index2Layer::Index2Layer (Index * quantizer, size_t nlist,
+
1274  int M,
+
1275  MetricType metric):
+
1276  Index (quantizer->d, metric),
+
1277  q1 (quantizer, nlist),
+
1278  pq (quantizer->d, M, 8)
+
1279 {
+
1280  is_trained = false;
+
1281  for (int nbyte = 0; nbyte < 7; nbyte++) {
+
1282  if ((1L << (8 * nbyte)) >= nlist) {
+
1283  code_size_1 = nbyte;
+
1284  break;
+
1285  }
+
1286  }
+
1287  code_size_2 = pq.code_size;
+
1288  code_size = code_size_1 + code_size_2;
+
1289 }
+
1290 
+
1291 Index2Layer::Index2Layer ()
+
1292 {
+
1293  code_size = code_size_1 = code_size_2 = 0;
+
1294 }
+
1295 
+
1296 Index2Layer::~Index2Layer ()
+
1297 {}
+
1298 
+
1299 void Index2Layer::train(idx_t n, const float* x)
+
1300 {
+
1301  if (verbose) {
+
1302  printf ("training level-1 quantizer %ld vectors in %dD\n",
+
1303  n, d);
+
1304  }
+
1305 
+
1306  q1.train_q1 (n, x, verbose, metric_type);
+
1307 
+
1308  if (verbose) {
+
1309  printf("computing residuals\n");
+
1310  }
+
1311 
+
1312  const float * x_in = x;
1313 
-
1314  }
-
1315 
-
1316 }
+
1314  x = fvecs_maybe_subsample (
+
1315  d, (size_t*)&n, pq.cp.max_points_per_centroid * pq.ksub,
+
1316  x, verbose, pq.cp.seed);
1317 
-
1318 
+
1318  ScopeDeleter<float> del_x (x_in == x ? nullptr : x);
1319 
-
1320 void IndexIVFPQR::merge_from (IndexIVF &other_in, idx_t add_id)
-
1321 {
-
1322  IndexIVFPQR *other = dynamic_cast<IndexIVFPQR *> (&other_in);
-
1323  FAISS_THROW_IF_NOT(other);
-
1324 
-
1325  IndexIVF::merge_from (other_in, add_id);
-
1326 
-
1327  refine_codes.insert (refine_codes.end(),
-
1328  other->refine_codes.begin(),
-
1329  other->refine_codes.end());
-
1330  other->refine_codes.clear();
-
1331 }
-
1332 
-
1333 long IndexIVFPQR::remove_ids(const IDSelector& /*sel*/) {
-
1334  FAISS_THROW_MSG("not implemented");
-
1335  return 0;
-
1336 }
-
1337 
-
1338 /*****************************************
-
1339  * IndexIVFPQCompact implementation
-
1340  ******************************************/
-
1341 
-
1342 IndexIVFPQCompact::IndexIVFPQCompact ()
-
1343 {
-
1344  alloc_type = Alloc_type_none;
-
1345  limits = nullptr;
-
1346  compact_ids = nullptr;
-
1347  compact_codes = nullptr;
-
1348 }
-
1349 
-
1350 
-
1351 IndexIVFPQCompact::IndexIVFPQCompact (const IndexIVFPQ &other)
-
1352 {
-
1353  FAISS_THROW_IF_NOT_MSG (other.ntotal < (1UL << 31),
-
1354  "IndexIVFPQCompact cannot store more than 2G images");
-
1355 
-
1356  // here it would be more convenient to just use the
-
1357  // copy-constructor, but it would copy the lists as well: too much
-
1358  // overhead...
-
1359 
-
1360  // copy fields from Index
-
1361  d = other.d;
-
1362  ntotal = other.ntotal;
-
1363  verbose = other.verbose;
-
1364  is_trained = other.is_trained;
-
1365  metric_type = other.metric_type;
-
1366 
-
1367  // copy fields from IndexIVF (except ids)
-
1368  nlist = other.nlist;
-
1369  nprobe = other.nprobe;
-
1370  quantizer = other.quantizer;
-
1371  quantizer_trains_alone = other.quantizer_trains_alone;
-
1372  own_fields = false;
-
1373  direct_map = other.direct_map;
-
1374 
-
1375  // copy fields from IndexIVFPQ (except codes)
-
1376  by_residual = other.by_residual;
-
1377  use_precomputed_table = other.use_precomputed_table;
-
1378  precomputed_table = other.precomputed_table;
-
1379  code_size = other.code_size;
-
1380  pq = other.pq;
-
1381  do_polysemous_training = other.do_polysemous_training;
-
1382  polysemous_training = nullptr;
-
1383 
-
1384  scan_table_threshold = other.scan_table_threshold;
-
1385  max_codes = other.max_codes;
-
1386  polysemous_ht = other.polysemous_ht;
-
1387 
-
1388  //allocate
-
1389  alloc_type = Alloc_type_new;
-
1390  limits = new uint32_t [nlist + 1];
-
1391  compact_ids = new uint32_t [ntotal];
-
1392  compact_codes = new uint8_t [ntotal * code_size];
-
1393 
+
1320  std::vector<idx_t> assign(n); // assignement to coarse centroids
+
1321  q1.quantizer->assign (n, x, assign.data());
+
1322  std::vector<float> residuals(n * d);
+
1323  for (idx_t i = 0; i < n; i++) {
+
1324  q1.quantizer->compute_residual (
+
1325  x + i * d, residuals.data() + i * d, assign[i]);
+
1326  }
+
1327 
+
1328  if (verbose)
+
1329  printf ("training %zdx%zd product quantizer on %ld vectors in %dD\n",
+
1330  pq.M, pq.ksub, n, d);
+
1331  pq.verbose = verbose;
+
1332  pq.train (n, residuals.data());
+
1333 
+
1334  is_trained = true;
+
1335 }
+
1336 
+
1337 void Index2Layer::add(idx_t n, const float* x)
+
1338 {
+
1339  idx_t bs = 32768;
+
1340  if (n > bs) {
+
1341  for (idx_t i0 = 0; i0 < n; i0 += bs) {
+
1342  idx_t i1 = std::min(i0 + bs, n);
+
1343  if (verbose) {
+
1344  printf("Index2Layer::add: adding %ld:%ld / %ld\n",
+
1345  i0, i1, n);
+
1346  }
+
1347  add (i1 - i0, x + i0 * d);
+
1348  }
+
1349  return;
+
1350  }
+
1351 
+
1352  std::vector<idx_t> codes1 (n);
+
1353  q1.quantizer->assign (n, x, codes1.data());
+
1354  std::vector<float> residuals(n * d);
+
1355  for (idx_t i = 0; i < n; i++) {
+
1356  q1.quantizer->compute_residual (
+
1357  x + i * d, residuals.data() + i * d, codes1[i]);
+
1358  }
+
1359  std::vector<uint8_t> codes2 (n * code_size_2);
+
1360 
+
1361  pq.compute_codes (residuals.data(), codes2.data(), n);
+
1362 
+
1363  codes.resize ((ntotal + n) * code_size);
+
1364  uint8_t *wp = &codes[ntotal * code_size];
+
1365 
+
1366  {
+
1367  int i = 0x11223344;
+
1368  const char *ip = (char*)&i;
+
1369  FAISS_THROW_IF_NOT_MSG (ip[0] == 0x44,
+
1370  "works only on a little-endian CPU");
+
1371  }
+
1372 
+
1373  // copy to output table
+
1374  for (idx_t i = 0; i < n; i++) {
+
1375  memcpy (wp, &codes1[i], code_size_1);
+
1376  wp += code_size_1;
+
1377  memcpy (wp, &codes2[i * code_size_2], code_size_2);
+
1378  wp += code_size_2;
+
1379  }
+
1380 
+
1381  ntotal += n;
+
1382 
+
1383 }
+
1384 
+
1385 void Index2Layer::search(
+
1386  idx_t n,
+
1387  const float* x,
+
1388  idx_t k,
+
1389  float* distances,
+
1390  idx_t* labels) const
+
1391 {
+
1392  FAISS_THROW_MSG ("not implemented");
+
1393 }
1394 
-
1395  // copy content from other
-
1396  size_t ofs = 0;
-
1397  for (size_t i = 0; i < nlist; i++) {
-
1398  limits [i] = ofs;
-
1399  const std::vector<long> &other_ids = other.ids[i];
-
1400  for (size_t j = 0; j < other_ids.size(); j++) {
-
1401  long id = other_ids[j];
-
1402  FAISS_THROW_IF_NOT_MSG (id < (1UL << 31),
-
1403  "IndexIVFPQCompact cannot store ids > 2G");
-
1404  compact_ids[ofs + j] = id;
-
1405  }
-
1406  memcpy (compact_codes + ofs * code_size,
-
1407  other.codes[i].data(),
-
1408  other.codes[i].size());
-
1409  ofs += other_ids.size();
-
1410  }
-
1411  FAISS_THROW_IF_NOT (ofs == ntotal);
-
1412  limits [nlist] = ofs;
-
1413 
+
1395 
+
1396 void Index2Layer::reconstruct_n(idx_t i0, idx_t ni, float* recons) const
+
1397 {
+
1398  float recons1[d];
+
1399  FAISS_THROW_IF_NOT (i0 >= 0 && i0 + ni <= ntotal);
+
1400  const uint8_t *rp = &codes[i0 * code_size];
+
1401 
+
1402  for (idx_t i = 0; i < ni; i++) {
+
1403  idx_t key = 0;
+
1404  memcpy (&key, rp, code_size_1);
+
1405  q1.quantizer->reconstruct (key, recons1);
+
1406  rp += code_size_1;
+
1407  pq.decode (rp, recons);
+
1408  for (idx_t j = 0; j < d; j++) {
+
1409  recons[j] += recons1[j];
+
1410  }
+
1411  rp += code_size_2;
+
1412  recons += d;
+
1413  }
1414 }
1415 
-
1416 void IndexIVFPQCompact::add (idx_t, const float *) {
-
1417  FAISS_THROW_MSG ("cannot add to an IndexIVFPQCompact");
-
1418 }
-
1419 
-
1420 void IndexIVFPQCompact::reset () {
-
1421  FAISS_THROW_MSG ("cannot reset an IndexIVFPQCompact");
-
1422 }
+
1416 void Index2Layer::transfer_to_IVFPQ (IndexIVFPQ & other) const
+
1417 {
+
1418  FAISS_THROW_IF_NOT (other.nlist == q1.nlist);
+
1419  FAISS_THROW_IF_NOT (other.code_size == code_size_2);
+
1420  FAISS_THROW_IF_NOT (other.ntotal == 0);
+
1421 
+
1422  const uint8_t *rp = codes.data();
1423 
-
1424 void IndexIVFPQCompact::train (idx_t, const float *) {
-
1425  FAISS_THROW_MSG ("cannot train an IndexIVFPQCompact");
-
1426 }
-
1427 
-
1428 
-
1429 
-
1430 
-
1431 IndexIVFPQCompact::~IndexIVFPQCompact ()
-
1432 {
-
1433  if (alloc_type == Alloc_type_new) {
-
1434  delete [] limits;
-
1435  delete [] compact_codes;
-
1436  delete [] compact_ids;
-
1437  } else if (alloc_type == Alloc_type_mmap) {
-
1438  munmap (mmap_buffer, mmap_length);
+
1424  for (idx_t i = 0; i < ntotal; i++) {
+
1425  idx_t key = 0;
+
1426  memcpy (&key, rp, code_size_1);
+
1427  other.ids[key].push_back (i);
+
1428  rp += code_size_1;
+
1429  std::vector<uint8_t> & list = other.codes[key];
+
1430  size_t len = list.size();
+
1431  list.resize(len + code_size_2);
+
1432  memcpy (&list[len], rp, code_size_2);
+
1433  rp += code_size_2;
+
1434  }
+
1435 
+
1436  other.ntotal = ntotal;
+
1437 
+
1438 }
1439 
-
1440  }
+
1440 
1441 
-
1442 }
-
1443 
-
1444 
-
1445 
-
1446 void IndexIVFPQCompact::search_preassigned (idx_t nx, const float *qx, idx_t k,
-
1447  const idx_t *keys,
-
1448  const float *coarse_dis,
-
1449  float *distances, idx_t *labels,
-
1450  bool store_pairs) const
-
1451 {
-
1452  float_maxheap_array_t res = {
-
1453  size_t(nx), size_t(k),
-
1454  labels, distances
-
1455  };
+
1442 void Index2Layer::reconstruct(idx_t key, float* recons) const
+
1443 {
+
1444  reconstruct_n (key, 1, recons);
+
1445 }
+
1446 
+
1447 void Index2Layer::reset()
+
1448 {
+
1449  ntotal = 0;
+
1450  codes.clear ();
+
1451 }
+
1452 
+
1453 
+
1454 
+
1455 
1456 
-
1457 #pragma omp parallel
-
1458  {
-
1459  InvertedListScanner<uint32_t> qt (*this);
-
1460  size_t stats_nlist = 0;
-
1461  size_t stats_ncode = 0;
-
1462  uint64_t init_query_cycles = 0;
-
1463  uint64_t scan_cycles = 0;
-
1464  uint64_t heap_cycles = 0;
+
1457 
+
1458 
+
1459 
+
1460 
+
1461 
+
1462 
+
1463 
+
1464 
1465 
-
1466 #pragma omp for
-
1467  for (size_t i = 0; i < nx; i++) {
-
1468  const float *qi = qx + i * d;
-
1469  const long * keysi = keys + i * nprobe;
-
1470  const float *coarse_dis_i = coarse_dis + i * nprobe;
-
1471  float * heap_sim = res.get_val (i);
-
1472  long * heap_ids = res.get_ids (i);
-
1473 
-
1474  uint64_t t0;
-
1475  TIC;
-
1476  maxheap_heapify (k, heap_sim, heap_ids);
-
1477  heap_cycles += TOC;
-
1478 
-
1479  TIC;
-
1480  qt.init_query (qi);
-
1481  init_query_cycles += TOC;
-
1482 
-
1483  size_t nscan = 0;
-
1484 
-
1485  for (size_t ik = 0; ik < nprobe; ik++) {
-
1486  long key = keysi[ik]; /* select the list */
-
1487  if (key < 0) {
-
1488  // not enough centroids for multiprobe
-
1489  continue;
-
1490  }
-
1491  if (key >= (long) nlist) {
-
1492  fprintf (stderr, "Invalid key=%ld nlist=%ld\n", key, nlist);
-
1493  throw;
-
1494  }
-
1495  size_t list_size = limits[key + 1] - limits[key];
-
1496  stats_nlist ++;
-
1497  nscan += list_size;
-
1498 
-
1499  if (list_size == 0) continue;
-
1500 
-
1501  qt.init_list (key, coarse_dis_i[ik],
-
1502  list_size, compact_ids + limits[key],
-
1503  compact_codes + limits[key] * code_size);
+
1466 
+
1467 
+
1468 /*****************************************
+
1469  * IndexIVFPQCompact implementation
+
1470  ******************************************/
+
1471 
+
1472 IndexIVFPQCompact::IndexIVFPQCompact ()
+
1473 {
+
1474  alloc_type = Alloc_type_none;
+
1475  limits = nullptr;
+
1476  compact_ids = nullptr;
+
1477  compact_codes = nullptr;
+
1478 }
+
1479 
+
1480 
+
1481 IndexIVFPQCompact::IndexIVFPQCompact (const IndexIVFPQ &other)
+
1482 {
+
1483  FAISS_THROW_IF_NOT_MSG (other.ntotal < (1UL << 31),
+
1484  "IndexIVFPQCompact cannot store more than 2G images");
+
1485 
+
1486  // here it would be more convenient to just use the
+
1487  // copy-constructor, but it would copy the lists as well: too much
+
1488  // overhead...
+
1489 
+
1490  // copy fields from Index
+
1491  d = other.d;
+
1492  ntotal = other.ntotal;
+
1493  verbose = other.verbose;
+
1494  is_trained = other.is_trained;
+
1495  metric_type = other.metric_type;
+
1496 
+
1497  // copy fields from IndexIVF (except ids)
+
1498  nlist = other.nlist;
+
1499  nprobe = other.nprobe;
+
1500  quantizer = other.quantizer;
+
1501  quantizer_trains_alone = other.quantizer_trains_alone;
+
1502  own_fields = false;
+
1503  direct_map = other.direct_map;
1504 
-
1505  TIC;
-
1506  if (polysemous_ht > 0) {
-
1507  qt.scan_list_polysemous
-
1508  (k, heap_sim, heap_ids, store_pairs);
-
1509  } else if (list_size > scan_table_threshold) {
-
1510  qt.scan_list_with_table (k, heap_sim, heap_ids, store_pairs);
-
1511  } else {
-
1512  qt.scan_on_the_fly_dist (k, heap_sim, heap_ids, store_pairs);
-
1513  }
-
1514  scan_cycles += TOC;
-
1515 
-
1516  if (max_codes && nscan >= max_codes) break;
-
1517  }
-
1518  stats_ncode += nscan;
-
1519  TIC;
-
1520  maxheap_reorder (k, heap_sim, heap_ids);
-
1521 
-
1522  if (metric_type == METRIC_INNER_PRODUCT) {
-
1523  for (size_t j = 0; j < k; j++) {
-
1524  heap_sim[i] = -heap_sim[i];
-
1525  }
-
1526  }
-
1527  heap_cycles += TOC;
-
1528  }
-
1529 
-
1530 #pragma omp critical
-
1531  {
-
1532  indexIVFPQ_stats.n_hamming_pass += qt.n_hamming_pass;
-
1533  indexIVFPQ_stats.nlist += stats_nlist;
-
1534  indexIVFPQ_stats.ncode += stats_ncode;
-
1535 
-
1536  indexIVFPQ_stats.init_query_cycles += init_query_cycles;
-
1537  indexIVFPQ_stats.init_list_cycles += qt.init_list_cycles;
-
1538  indexIVFPQ_stats.scan_cycles += scan_cycles - qt.init_list_cycles;
-
1539  indexIVFPQ_stats.heap_cycles += heap_cycles;
-
1540  }
-
1541 
-
1542  }
-
1543  indexIVFPQ_stats.nq += nx;
+
1505  // copy fields from IndexIVFPQ (except codes)
+
1506  by_residual = other.by_residual;
+
1507  use_precomputed_table = other.use_precomputed_table;
+
1508  precomputed_table = other.precomputed_table;
+
1509  code_size = other.code_size;
+
1510  pq = other.pq;
+
1511  do_polysemous_training = other.do_polysemous_training;
+
1512  polysemous_training = nullptr;
+
1513 
+
1514  scan_table_threshold = other.scan_table_threshold;
+
1515  max_codes = other.max_codes;
+
1516  polysemous_ht = other.polysemous_ht;
+
1517 
+
1518  //allocate
+
1519  alloc_type = Alloc_type_new;
+
1520  limits = new uint32_t [nlist + 1];
+
1521  compact_ids = new uint32_t [ntotal];
+
1522  compact_codes = new uint8_t [ntotal * code_size];
+
1523 
+
1524 
+
1525  // copy content from other
+
1526  size_t ofs = 0;
+
1527  for (size_t i = 0; i < nlist; i++) {
+
1528  limits [i] = ofs;
+
1529  const std::vector<long> &other_ids = other.ids[i];
+
1530  for (size_t j = 0; j < other_ids.size(); j++) {
+
1531  long id = other_ids[j];
+
1532  FAISS_THROW_IF_NOT_MSG (id < (1UL << 31),
+
1533  "IndexIVFPQCompact cannot store ids > 2G");
+
1534  compact_ids[ofs + j] = id;
+
1535  }
+
1536  memcpy (compact_codes + ofs * code_size,
+
1537  other.codes[i].data(),
+
1538  other.codes[i].size());
+
1539  ofs += other_ids.size();
+
1540  }
+
1541  FAISS_THROW_IF_NOT (ofs == ntotal);
+
1542  limits [nlist] = ofs;
+
1543 
1544 }
1545 
-
1546 
-
1547 
-
1548 } // namespace faiss
-
faiss::IndexIVFPQCompact::compact_ids
uint32_t * compact_ids
size ntotal
Definition: IndexIVFPQ.h:227
- -
faiss::IndexIVFPQCompact::compact_codes
uint8_t * compact_codes
size ntotal * code_size
Definition: IndexIVFPQ.h:228
-
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:356
-
faiss::IndexIVFPQR::merge_from
void merge_from(IndexIVF &other, idx_t add_id) override
Definition: IndexIVFPQ.cpp:1320
-
faiss::IndexIVFPQ::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVFPQ.cpp:302
- +
1546 void IndexIVFPQCompact::add (idx_t, const float *) {
+
1547  FAISS_THROW_MSG ("cannot add to an IndexIVFPQCompact");
+
1548 }
+
1549 
+
1550 void IndexIVFPQCompact::reset () {
+
1551  FAISS_THROW_MSG ("cannot reset an IndexIVFPQCompact");
+
1552 }
+
1553 
+
1554 void IndexIVFPQCompact::train (idx_t, const float *) {
+
1555  FAISS_THROW_MSG ("cannot train an IndexIVFPQCompact");
+
1556 }
+
1557 
+
1558 
+
1559 
+
1560 
+
1561 IndexIVFPQCompact::~IndexIVFPQCompact ()
+
1562 {
+
1563  if (alloc_type == Alloc_type_new) {
+
1564  delete [] limits;
+
1565  delete [] compact_codes;
+
1566  delete [] compact_ids;
+
1567  } else if (alloc_type == Alloc_type_mmap) {
+
1568  munmap (mmap_buffer, mmap_length);
+
1569 
+
1570  }
+
1571 
+
1572 }
+
1573 
+
1574 
+
1575 
+
1576 void IndexIVFPQCompact::search_preassigned (idx_t nx, const float *qx, idx_t k,
+
1577  const idx_t *keys,
+
1578  const float *coarse_dis,
+
1579  float *distances, idx_t *labels,
+
1580  bool store_pairs) const
+
1581 {
+
1582  float_maxheap_array_t res = {
+
1583  size_t(nx), size_t(k),
+
1584  labels, distances
+
1585  };
+
1586 
+
1587 #pragma omp parallel
+
1588  {
+
1589  InvertedListScanner<uint32_t> qt (*this);
+
1590  size_t stats_nlist = 0;
+
1591  size_t stats_ncode = 0;
+
1592  uint64_t init_query_cycles = 0;
+
1593  uint64_t scan_cycles = 0;
+
1594  uint64_t heap_cycles = 0;
+
1595 
+
1596 #pragma omp for
+
1597  for (size_t i = 0; i < nx; i++) {
+
1598  const float *qi = qx + i * d;
+
1599  const long * keysi = keys + i * nprobe;
+
1600  const float *coarse_dis_i = coarse_dis + i * nprobe;
+
1601  float * heap_sim = res.get_val (i);
+
1602  long * heap_ids = res.get_ids (i);
+
1603 
+
1604  uint64_t t0;
+
1605  TIC;
+
1606  maxheap_heapify (k, heap_sim, heap_ids);
+
1607  heap_cycles += TOC;
+
1608 
+
1609  TIC;
+
1610  qt.init_query (qi);
+
1611  init_query_cycles += TOC;
+
1612 
+
1613  size_t nscan = 0;
+
1614 
+
1615  for (size_t ik = 0; ik < nprobe; ik++) {
+
1616  long key = keysi[ik]; /* select the list */
+
1617  if (key < 0) {
+
1618  // not enough centroids for multiprobe
+
1619  continue;
+
1620  }
+
1621  if (key >= (long) nlist) {
+
1622  fprintf (stderr, "Invalid key=%ld nlist=%ld\n", key, nlist);
+
1623  throw;
+
1624  }
+
1625  size_t list_size = limits[key + 1] - limits[key];
+
1626  stats_nlist ++;
+
1627  nscan += list_size;
+
1628 
+
1629  if (list_size == 0) continue;
+
1630 
+
1631  qt.init_list (key, coarse_dis_i[ik],
+
1632  list_size, compact_ids + limits[key],
+
1633  compact_codes + limits[key] * code_size);
+
1634 
+
1635  TIC;
+
1636  if (polysemous_ht > 0) {
+
1637  qt.scan_list_polysemous
+
1638  (k, heap_sim, heap_ids, store_pairs);
+
1639  } else if (list_size > scan_table_threshold) {
+
1640  qt.scan_list_with_table (k, heap_sim, heap_ids, store_pairs);
+
1641  } else {
+
1642  qt.scan_on_the_fly_dist (k, heap_sim, heap_ids, store_pairs);
+
1643  }
+
1644  scan_cycles += TOC;
+
1645 
+
1646  if (max_codes && nscan >= max_codes) break;
+
1647  }
+
1648  stats_ncode += nscan;
+
1649  TIC;
+
1650  maxheap_reorder (k, heap_sim, heap_ids);
+
1651 
+
1652  if (metric_type == METRIC_INNER_PRODUCT) {
+
1653  for (size_t j = 0; j < k; j++) {
+
1654  heap_sim[i] = -heap_sim[i];
+
1655  }
+
1656  }
+
1657  heap_cycles += TOC;
+
1658  }
+
1659 
+
1660 #pragma omp critical
+
1661  {
+
1662  indexIVFPQ_stats.n_hamming_pass += qt.n_hamming_pass;
+
1663  indexIVFPQ_stats.nlist += stats_nlist;
+
1664  indexIVFPQ_stats.ncode += stats_ncode;
+
1665 
+
1666  indexIVFPQ_stats.init_query_cycles += init_query_cycles;
+
1667  indexIVFPQ_stats.init_list_cycles += qt.init_list_cycles;
+
1668  indexIVFPQ_stats.scan_cycles += scan_cycles - qt.init_list_cycles;
+
1669  indexIVFPQ_stats.heap_cycles += heap_cycles;
+
1670  }
+
1671 
+
1672  }
+
1673  indexIVFPQ_stats.nq += nx;
+
1674 }
+
1675 
+
1676 
+
1677 
+
1678 } // namespace faiss
+
faiss::IndexIVFPQCompact::compact_ids
uint32_t * compact_ids
size ntotal
Definition: IndexIVFPQ.h:208
+ +
faiss::IndexIVFPQCompact::compact_codes
uint8_t * compact_codes
size ntotal * code_size
Definition: IndexIVFPQ.h:209
+
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:344
+
faiss::IndexIVFPQR::merge_from
void merge_from(IndexIVF &other, idx_t add_id) override
Definition: IndexIVFPQ.cpp:1250
+
faiss::Index2Layer::transfer_to_IVFPQ
void transfer_to_IVFPQ(IndexIVFPQ &other) const
transfer the flat codes to an IVFPQ index
Definition: IndexIVFPQ.cpp:1416
+
faiss::Index2Layer::code_size_2
size_t code_size_2
size of the code for the second level
Definition: IndexIVFPQ.h:253
+
faiss::IndexIVFPQR::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVFPQ.cpp:1235
+
faiss::IndexIVFPQR::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:1152
+
faiss::ProductQuantizer::decode
void decode(const uint8_t *code, float *x) const
decode a vector from a given code (or n vectors if third argument)
Definition: ProductQuantizer.cpp:338
-
faiss::IndexIVFPQR::refine_pq
ProductQuantizer refine_pq
3rd level quantizer
Definition: IndexIVFPQ.h:169
+
faiss::IndexIVFPQR::refine_pq
ProductQuantizer refine_pq
3rd level quantizer
Definition: IndexIVFPQ.h:150
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils.cpp:574
faiss::IndexIVFPQ::polysemous_training
PolysemousTraining * polysemous_training
if NULL, use default
Definition: IndexIVFPQ.h:35
faiss::HeapArray::get_val
T * get_val(size_t key)
Return the list of values for a heap.
Definition: Heap.h:360
-
faiss::IndexIVFPQCompact::add
void add(idx_t, const float *) override
the three following functions will fail at runtime
Definition: IndexIVFPQ.cpp:1416
-
faiss::IndexIVFPQR::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVFPQ.cpp:1301
-
faiss::IndexIVFPQCompact::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:1446
-
faiss::IndexIVF::quantizer_trains_alone
char quantizer_trains_alone
Definition: IndexIVF.h:56
-
faiss::fvecs_maybe_subsample
const float * fvecs_maybe_subsample(size_t d, size_t *n, size_t nmax, const float *x, bool verbose, long seed)
Definition: utils.cpp:1973
-
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:47
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
+
faiss::Index2Layer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVFPQ.cpp:1396
+
faiss::Index2Layer::code_size
size_t code_size
code_size_1 + code_size_2
Definition: IndexIVFPQ.h:256
+
faiss::IndexIVFPQCompact::add
void add(idx_t, const float *) override
the three following functions will fail at runtime
Definition: IndexIVFPQ.cpp:1546
+
faiss::IndexIVFPQCompact::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:1576
+
faiss::fvecs_maybe_subsample
const float * fvecs_maybe_subsample(size_t d, size_t *n, size_t nmax, const float *x, bool verbose, long seed)
Definition: utils.cpp:1975
+
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:78
+
faiss::Index2Layer::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1447
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
- -
faiss::IndexIVFPQ::decode_multiple
void decode_multiple(size_t n, const long *keys, const uint8_t *xcodes, float *x) const
inverse of encode_multiple
Definition: IndexIVFPQ.cpp:169
-
faiss::IndexIVFPQ::train_residual_o
void train_residual_o(idx_t n, const float *x, float *residuals_2)
same as train_residual, also output 2nd level residuals
Definition: IndexIVFPQ.cpp:71
+ +
faiss::IndexIVFPQ::decode_multiple
void decode_multiple(size_t n, const long *keys, const uint8_t *xcodes, float *x) const
inverse of encode_multiple
Definition: IndexIVFPQ.cpp:168
+
faiss::IndexIVFPQ::train_residual_o
void train_residual_o(idx_t n, const float *x, float *residuals_2)
same as train_residual, also output 2nd level residuals
Definition: IndexIVFPQ.cpp:70
faiss::IndexIVFPQ::do_polysemous_training
bool do_polysemous_training
reorder PQ centroids after training?
Definition: IndexIVFPQ.h:34
faiss::IndexIVFPQ::scan_table_threshold
size_t scan_table_threshold
use table computation or on-the-fly?
Definition: IndexIVFPQ.h:38
-
faiss::IndexIVFPQR::train_residual
void train_residual(idx_t n, const float *x) override
trains the two product quantizers
Definition: IndexIVFPQ.cpp:1169
-
faiss::IndexIVFPQR::add_core
void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx=nullptr)
same as add_with_ids, but optionally use the precomputed list ids
Definition: IndexIVFPQ.cpp:1193
+
faiss::IndexIVFPQR::train_residual
void train_residual(idx_t n, const float *x) override
trains the two product quantizers
Definition: IndexIVFPQ.cpp:1109
+
faiss::IndexIVFPQR::add_core
void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx=nullptr)
same as add_with_ids, but optionally use the precomputed list ids
Definition: IndexIVFPQ.cpp:1133
-
faiss::IndexIVFPQCompact::limits
uint32_t * limits
size nlist + 1
Definition: IndexIVFPQ.h:226
+
faiss::IndexIVFPQCompact::limits
uint32_t * limits
size nlist + 1
Definition: IndexIVFPQ.h:207
faiss::ProductQuantizer::dsub
size_t dsub
dimensionality of each subvector
Definition: ProductQuantizer.h:32
faiss::ClusteringParameters::seed
int seed
seed for the random number generator
Definition: Clustering.h:36
-
faiss::IndexIVFPQ::precomputed_table
std::vector< float > precomputed_table
Definition: IndexIVFPQ.h:45
-
faiss::fvec_madd
void fvec_madd(size_t n, const float *a, float bf, const float *b, float *c)
Definition: utils.cpp:1887
-
faiss::IndexIVFPQ::polysemous_ht
int polysemous_ht
Hamming thresh for polysemous filtering.
Definition: IndexIVFPQ.h:40
-
faiss::IndexIVFPQR::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1160
-
faiss::IndexIVFPQ::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids=nullptr) override
Definition: IndexIVFPQ.cpp:186
-
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
+
faiss::IndexIVFPQ::precomputed_table
std::vector< float > precomputed_table
Definition: IndexIVFPQ.h:44
+
faiss::Index2Layer::q1
Level1Quantizer q1
first level quantizer
Definition: IndexIVFPQ.h:241
+
faiss::fvec_madd
void fvec_madd(size_t n, const float *a, float bf, const float *b, float *c)
Definition: utils.cpp:1889
+
faiss::IndexIVFPQ::polysemous_ht
int polysemous_ht
Hamming thresh for polysemous filtering.
Definition: IndexIVFPQ.h:39
+
faiss::IndexIVFPQR::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1100
+
faiss::Index2Layer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * code_size.
Definition: IndexIVFPQ.h:247
+
faiss::IndexIVFPQ::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids=nullptr) override
Definition: IndexIVFPQ.cpp:185
+
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
faiss::ProductQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: ProductQuantizer.cpp:385
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
-
faiss::IndexIVFPQ::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVFPQ.h:39
- -
faiss::IndexIVFPQR::refine_codes
std::vector< uint8_t > refine_codes
corresponding codes
Definition: IndexIVFPQ.h:170
+ +
faiss::IndexIVFPQR::refine_codes
std::vector< uint8_t > refine_codes
corresponding codes
Definition: IndexIVFPQ.h:151
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:34
-
faiss::IndexIVFPQ::train_residual
void train_residual(idx_t n, const float *x) override
trains the product quantizer
Definition: IndexIVFPQ.cpp:65
+
faiss::IndexIVFPQ::train_residual
void train_residual(idx_t n, const float *x) override
trains the product quantizer
Definition: IndexIVFPQ.cpp:64
+
faiss::IndexIVFPQ::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVFPQ.cpp:291
-
faiss::IndexIVFPQ::encode_multiple
void encode_multiple(size_t n, long *keys, const float *x, uint8_t *codes, bool compute_keys=false) const
Definition: IndexIVFPQ.cpp:150
-
faiss::IndexIVFPQCompact::train
void train(idx_t, const float *) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVFPQ.cpp:1424
-
faiss::IndexIVF::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:57
+
faiss::IndexIVFPQ::encode_multiple
void encode_multiple(size_t n, long *keys, const float *x, uint8_t *codes, bool compute_keys=false) const
Definition: IndexIVFPQ.cpp:149
+
faiss::IndexIVFPQCompact::train
void train(idx_t, const float *) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVFPQ.cpp:1554
+
faiss::ProductQuantizer::ksub
size_t ksub
number of centroids for each subquantizer
Definition: ProductQuantizer.h:35
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::ProductQuantizer::compute_code
void compute_code(const float *x, uint8_t *code) const
Quantize one vector with the product quantizer.
Definition: ProductQuantizer.cpp:311
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
+
faiss::Index2Layer::train
void train(idx_t n, const float *x) override
Definition: IndexIVFPQ.cpp:1299
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
-
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:112
+
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:252
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
faiss::PolysemousTraining
optimizes the order of indices in a ProductQuantizer
Definition: PolysemousTraining.h:125
-
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
+
faiss::Index2Layer::add
void add(idx_t n, const float *x) override
Definition: IndexIVFPQ.cpp:1337
faiss::fvec_norm_L2sqr
float fvec_norm_L2sqr(const float *x, size_t d)
Definition: utils.cpp:632
+
faiss::Level1Quantizer::train_q1
void train_q1(size_t n, const float *x, bool verbose, MetricType metric_type)
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:53
faiss::ProductQuantizer::cp
ClusteringParameters cp
parameters used during clustering
Definition: ProductQuantizer.h:49
+
faiss::Index2Layer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
not implemented
Definition: IndexIVFPQ.cpp:1385
faiss::IndexIVFPQ::by_residual
bool by_residual
Encode residual or plain vector?
Definition: IndexIVFPQ.h:30
faiss::HeapArray::get_ids
TI * get_ids(size_t key)
Correspponding identifiers.
Definition: Heap.h:363
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
- -
faiss::IndexIVFPQ::search_and_reconstruct
void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *reconstructed)
Definition: IndexIVFPQ.cpp:1032
+
faiss::IndexIVFPQ::pq
ProductQuantizer pq
produces the codes
Definition: IndexIVFPQ.h:32
faiss::ProductQuantizer::M
size_t M
number of subquantizers
Definition: ProductQuantizer.h:28
-
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
+
faiss::Index2Layer::code_size_1
size_t code_size_1
size of the code for the first level (ceil(log8(q1.nlist)))
Definition: IndexIVFPQ.h:250
-
faiss::IndexIVFPQ::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVFPQ.cpp:274
-
faiss::IndexIVFPQ::add_core_o
void add_core_o(idx_t n, const float *x, const long *xids, float *residuals_2, const long *precomputed_idx=nullptr)
Definition: IndexIVFPQ.cpp:192
-
faiss::fvec_madd_and_argmin
int fvec_madd_and_argmin(size_t n, const float *a, float bf, const float *b, float *c)
Definition: utils.cpp:1961
-
faiss::IndexIVFPQR::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVFPQ.cpp:1333
-
faiss::IndexIVFPQ::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:931
+
faiss::IndexIVFPQ::add_core_o
void add_core_o(idx_t n, const float *x, const long *xids, float *residuals_2, const long *precomputed_idx=nullptr)
Definition: IndexIVFPQ.cpp:191
+
faiss::fvec_madd_and_argmin
int fvec_madd_and_argmin(size_t n, const float *a, float bf, const float *b, float *c)
Definition: utils.cpp:1963
+
faiss::IndexIVFPQR::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVFPQ.cpp:1263
+
faiss::IndexIVFPQ::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:919
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
-
faiss::IndexIVFPQCompact::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1420
+
faiss::IndexIVFPQCompact::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1550
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:56
+
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:86
faiss::ProductQuantizer::get_centroids
float * get_centroids(size_t m, size_t i)
return the centroids associated with subvector m
Definition: ProductQuantizer.h:55
-
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:43
- -
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:68
+
faiss::IndexIVF::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVF.h:79
+
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:54
+
faiss::Index2Layer::pq
ProductQuantizer pq
second level quantizer is always a PQ
Definition: IndexIVFPQ.h:244
+ +
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:87
+
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:42
faiss::PolysemousTraining::optimize_pq_for_hamming
void optimize_pq_for_hamming(ProductQuantizer &pq, size_t n, const float *x) const
Definition: PolysemousTraining.cpp:933
faiss::ClusteringParameters::max_points_per_centroid
int max_points_per_centroid
to limit size of dataset
Definition: Clustering.h:34
faiss::ProductQuantizer::verbose
bool verbose
verbose during training?
Definition: ProductQuantizer.h:36
-
faiss::IndexIVFPQR::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexIVFPQ.cpp:1189
-
faiss::IndexIVFPQR::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVFPQ.cpp:1212
-
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:245
-
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:64
-
faiss::IndexIVFPQ::find_duplicates
size_t find_duplicates(idx_t *ids, size_t *lims) const
Definition: IndexIVFPQ.cpp:1104
+
faiss::IndexIVFPQR::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexIVFPQ.cpp:1129
+
faiss::Index2Layer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVFPQ.cpp:1442
+
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:348
+
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
+
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:83
+
faiss::IndexIVFPQ::find_duplicates
size_t find_duplicates(idx_t *ids, size_t *lims) const
Definition: IndexIVFPQ.cpp:1044
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
-
faiss::IndexIVFPQR::k_factor
float k_factor
factor between k requested in search and the k requested from the IVFPQ
Definition: IndexIVFPQ.h:173
+
faiss::IndexIVFPQR::k_factor
float k_factor
factor between k requested in search and the k requested from the IVFPQ
Definition: IndexIVFPQ.h:154
faiss::IndexIVFPQ::use_precomputed_table
int use_precomputed_table
if by_residual, build precompute tables
Definition: IndexIVFPQ.h:31
diff --git a/docs/html/IndexIVFPQ_8h_source.html b/docs/html/IndexIVFPQ_8h_source.html index 6f94826e5..ef50d21e7 100644 --- a/docs/html/IndexIVFPQ_8h_source.html +++ b/docs/html/IndexIVFPQ_8h_source.html @@ -121,285 +121,333 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
36 
37  // search-time parameters
38  size_t scan_table_threshold; ///< use table computation or on-the-fly?
-
39  size_t max_codes; ///< max nb of codes to visit to do a query
-
40  int polysemous_ht; ///< Hamming thresh for polysemous filtering
+
39  int polysemous_ht; ///< Hamming thresh for polysemous filtering
+
40 
41 
-
42 
-
43  /// if use_precompute_table
-
44  /// size nlist * pq.M * pq.ksub
-
45  std::vector <float> precomputed_table;
-
46 
-
47  IndexIVFPQ (
-
48  Index * quantizer, size_t d, size_t nlist,
-
49  size_t M, size_t nbits_per_idx);
-
50 
-
51  void add_with_ids(idx_t n, const float* x, const long* xids = nullptr)
-
52  override;
-
53 
-
54  /// same as add_core, also:
-
55  /// - output 2nd level residuals if residuals_2 != NULL
-
56  /// - use precomputed list numbers if precomputed_idx != NULL
-
57  void add_core_o (idx_t n, const float *x,
-
58  const long *xids, float *residuals_2,
-
59  const long *precomputed_idx = nullptr);
-
60 
-
61  /// trains the product quantizer
-
62  void train_residual(idx_t n, const float* x) override;
-
63 
-
64  /// same as train_residual, also output 2nd level residuals
-
65  void train_residual_o (idx_t n, const float *x, float *residuals_2);
-
66 
-
67 
-
68  /** Reconstruct a subset of the indexed vectors
-
69  *
-
70  * @param i0 first vector to reconstruct
-
71  * @param ni nb of vectors to reconstruct
-
72  * @param recons output array of reconstructed vectors, size ni * d
-
73  */
-
74  void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;
-
75 
-
76  void reconstruct(idx_t key, float* recons) const override;
-
77 
-
78  /** Find exact duplicates in the dataset.
-
79  *
-
80  * the duplicates are returned in pre-allocated arrays (see the
-
81  * max sizes).
-
82  *
-
83  * @params lims limits between groups of duplicates
-
84  * (max size ntotal / 2 + 1)
-
85  * @params ids ids[lims[i]] : ids[lims[i+1]-1] is a group of
-
86  * duplicates (max size ntotal)
-
87  * @return n number of groups found
-
88  */
-
89  size_t find_duplicates (idx_t *ids, size_t *lims) const;
-
90 
-
91  // map a vector to a binary code knowning the index
-
92  void encode (long key, const float * x, uint8_t * code) const;
-
93 
-
94  /** Encode multiple vectors
-
95  *
-
96  * @param n nb vectors to encode
-
97  * @param keys posting list ids for those vectors (size n)
-
98  * @param x vectors (size n * d)
-
99  * @param codes output codes (size n * code_size)
-
100  * @param compute_keys if false, assume keys are precomputed,
-
101  * otherwise compute them
-
102  */
-
103  void encode_multiple (size_t n, long *keys,
-
104  const float * x, uint8_t * codes,
-
105  bool compute_keys = false) const;
-
106 
-
107  /// inverse of encode_multiple
-
108  void decode_multiple (size_t n, const long *keys,
-
109  const uint8_t * xcodes, float * x) const;
+
42  /// if use_precompute_table
+
43  /// size nlist * pq.M * pq.ksub
+
44  std::vector <float> precomputed_table;
+
45 
+
46  IndexIVFPQ (
+
47  Index * quantizer, size_t d, size_t nlist,
+
48  size_t M, size_t nbits_per_idx);
+
49 
+
50  void add_with_ids(idx_t n, const float* x, const long* xids = nullptr)
+
51  override;
+
52 
+
53  /// same as add_core, also:
+
54  /// - output 2nd level residuals if residuals_2 != NULL
+
55  /// - use precomputed list numbers if precomputed_idx != NULL
+
56  void add_core_o (idx_t n, const float *x,
+
57  const long *xids, float *residuals_2,
+
58  const long *precomputed_idx = nullptr);
+
59 
+
60  /// trains the product quantizer
+
61  void train_residual(idx_t n, const float* x) override;
+
62 
+
63  /// same as train_residual, also output 2nd level residuals
+
64  void train_residual_o (idx_t n, const float *x, float *residuals_2);
+
65 
+
66  void reconstruct_from_offset (long list_no, long offset,
+
67  float* recons) const override;
+
68 
+
69  /** Find exact duplicates in the dataset.
+
70  *
+
71  * the duplicates are returned in pre-allocated arrays (see the
+
72  * max sizes).
+
73  *
+
74  * @params lims limits between groups of duplicates
+
75  * (max size ntotal / 2 + 1)
+
76  * @params ids ids[lims[i]] : ids[lims[i+1]-1] is a group of
+
77  * duplicates (max size ntotal)
+
78  * @return n number of groups found
+
79  */
+
80  size_t find_duplicates (idx_t *ids, size_t *lims) const;
+
81 
+
82  // map a vector to a binary code knowning the index
+
83  void encode (long key, const float * x, uint8_t * code) const;
+
84 
+
85  /** Encode multiple vectors
+
86  *
+
87  * @param n nb vectors to encode
+
88  * @param keys posting list ids for those vectors (size n)
+
89  * @param x vectors (size n * d)
+
90  * @param codes output codes (size n * code_size)
+
91  * @param compute_keys if false, assume keys are precomputed,
+
92  * otherwise compute them
+
93  */
+
94  void encode_multiple (size_t n, long *keys,
+
95  const float * x, uint8_t * codes,
+
96  bool compute_keys = false) const;
+
97 
+
98  /// inverse of encode_multiple
+
99  void decode_multiple (size_t n, const long *keys,
+
100  const uint8_t * xcodes, float * x) const;
+
101 
+
102  void search_preassigned (idx_t n, const float *x, idx_t k,
+
103  const idx_t *assign,
+
104  const float *centroid_dis,
+
105  float *distances, idx_t *labels,
+
106  bool store_pairs) const override;
+
107 
+
108  /// build precomputed table
+
109  void precompute_table ();
110 
-
111  void search_preassigned (idx_t n, const float *x, idx_t k,
-
112  const idx_t *assign,
-
113  const float *centroid_dis,
-
114  float *distances, idx_t *labels,
-
115  bool store_pairs) const override;
-
116 
-
117  /** Same as the search function, but also reconstruct approximate
-
118  * vectors for the search results
-
119  *
-
120  * @param reconstructed size (n, k, d)
-
121  **/
-
122  void search_and_reconstruct (idx_t n, const float *x, idx_t k,
-
123  float *distances, idx_t *labels,
-
124  float *reconstructed);
-
125 
-
126 
-
127  /// build precomputed table
-
128  void precompute_table ();
-
129 
-
130  IndexIVFPQ ();
-
131 
-
132 };
-
133 
-
134 
-
135 /// statistics are robust to internal threading, but not if
-
136 /// IndexIVFPQ::search_preassigned is called by multiple threads
-
137 struct IndexIVFPQStats {
-
138  size_t nq; // nb of queries run
-
139  size_t nlist; // nb of inverted lists scanned
-
140  size_t ncode; // nb of codes visited
-
141  size_t nrefine; // nb of refines (IVFPQR)
+
111  IndexIVFPQ ();
+
112 
+
113 };
+
114 
+
115 
+
116 /// statistics are robust to internal threading, but not if
+
117 /// IndexIVFPQ::search_preassigned is called by multiple threads
+
118 struct IndexIVFPQStats {
+
119  size_t nq; // nb of queries run
+
120  size_t nlist; // nb of inverted lists scanned
+
121  size_t ncode; // nb of codes visited
+
122  size_t nrefine; // nb of refines (IVFPQR)
+
123 
+
124  size_t n_hamming_pass;
+
125  // nb of passed Hamming distance tests (for polysemous)
+
126 
+
127  // timings measured with the CPU RTC
+
128  // on all threads
+
129  size_t assign_cycles;
+
130  size_t search_cycles;
+
131  size_t refine_cycles; // only for IVFPQR
+
132 
+
133  // single thread (double-counted with search_cycles)
+
134  size_t init_query_cycles;
+
135  size_t init_list_cycles;
+
136  size_t scan_cycles;
+
137  size_t heap_cycles;
+
138 
+
139  IndexIVFPQStats () {reset (); }
+
140  void reset ();
+
141 };
142 
-
143  size_t n_hamming_pass;
-
144  // nb of passed Hamming distance tests (for polysemous)
+
143 // global var that collects them all
+
144 extern IndexIVFPQStats indexIVFPQ_stats;
145 
-
146  // timings measured with the CPU RTC
-
147  // on all threads
-
148  size_t assign_cycles;
-
149  size_t search_cycles;
-
150  size_t refine_cycles; // only for IVFPQR
-
151 
-
152  // single thread (double-counted with search_cycles)
-
153  size_t init_query_cycles;
-
154  size_t init_list_cycles;
-
155  size_t scan_cycles;
-
156  size_t heap_cycles;
-
157 
-
158  IndexIVFPQStats () {reset (); }
-
159  void reset ();
-
160 };
-
161 
-
162 // global var that collects them all
-
163 extern IndexIVFPQStats indexIVFPQ_stats;
-
164 
-
165 
-
166 
-
167 /** Index with an additional level of PQ refinement */
-
168 struct IndexIVFPQR: IndexIVFPQ {
-
169  ProductQuantizer refine_pq; ///< 3rd level quantizer
-
170  std::vector <uint8_t> refine_codes; ///< corresponding codes
-
171 
-
172  /// factor between k requested in search and the k requested from the IVFPQ
-
173  float k_factor;
-
174 
-
175  IndexIVFPQR (
-
176  Index * quantizer, size_t d, size_t nlist,
-
177  size_t M, size_t nbits_per_idx,
-
178  size_t M_refine, size_t nbits_per_idx_refine);
+
146 
+
147 
+
148 /** Index with an additional level of PQ refinement */
+
149 struct IndexIVFPQR: IndexIVFPQ {
+
150  ProductQuantizer refine_pq; ///< 3rd level quantizer
+
151  std::vector <uint8_t> refine_codes; ///< corresponding codes
+
152 
+
153  /// factor between k requested in search and the k requested from the IVFPQ
+
154  float k_factor;
+
155 
+
156  IndexIVFPQR (
+
157  Index * quantizer, size_t d, size_t nlist,
+
158  size_t M, size_t nbits_per_idx,
+
159  size_t M_refine, size_t nbits_per_idx_refine);
+
160 
+
161  void reset() override;
+
162 
+
163  long remove_ids(const IDSelector& sel) override;
+
164 
+
165  /// trains the two product quantizers
+
166  void train_residual(idx_t n, const float* x) override;
+
167 
+
168  void add_with_ids(idx_t n, const float* x, const long* xids) override;
+
169 
+
170  /// same as add_with_ids, but optionally use the precomputed list ids
+
171  void add_core (idx_t n, const float *x, const long *xids,
+
172  const long *precomputed_idx = nullptr);
+
173 
+
174  void reconstruct_from_offset (long list_no, long offset,
+
175  float* recons) const override;
+
176 
+
177  void merge_from (IndexIVF &other, idx_t add_id) override;
+
178 
179 
-
180  void reset() override;
-
181 
-
182  long remove_ids(const IDSelector& sel) override;
-
183 
-
184  /// trains the two product quantizers
-
185  void train_residual(idx_t n, const float* x) override;
-
186 
-
187  void add_with_ids(idx_t n, const float* x, const long* xids) override;
-
188 
-
189  /// same as add_with_ids, but optionally use the precomputed list ids
-
190  void add_core (idx_t n, const float *x, const long *xids,
-
191  const long *precomputed_idx = nullptr);
-
192 
-
193  void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;
-
194 
-
195  void merge_from (IndexIVF &other, idx_t add_id) override;
-
196 
-
197 
-
198  void search(
-
199  idx_t n,
-
200  const float* x,
-
201  idx_t k,
-
202  float* distances,
-
203  idx_t* labels) const override;
+
180  void search_preassigned (idx_t n, const float *x, idx_t k,
+
181  const idx_t *assign,
+
182  const float *centroid_dis,
+
183  float *distances, idx_t *labels,
+
184  bool store_pairs) const override;
+
185 
+
186  IndexIVFPQR();
+
187 };
+
188 
+
189 
+
190 /** Index with 32-bit ids and flat tables. Must be constructed from an
+
191  * exisiting IndexIVFPQ. Cannot be copy-constructed/assigned. The
+
192  * actual data is stored in the compact_* tables, the ids and codes
+
193  * tables are not used. */
+
194 struct IndexIVFPQCompact: IndexIVFPQ {
+
195 
+
196  explicit IndexIVFPQCompact (const IndexIVFPQ &other);
+
197 
+
198  /// how were the compact tables allocated?
+
199  enum Alloc_type_t {
+
200  Alloc_type_none, ///< alloc from outside
+
201  Alloc_type_new, ///< was allocated with new
+
202  Alloc_type_mmap ///< was mmapped
+
203  };
204 
-
205  IndexIVFPQR();
-
206 };
-
207 
-
208 
-
209 /** Index with 32-bit ids and flat tables. Must be constructed from an
-
210  * exisiting IndexIVFPQ. Cannot be copy-constructed/assigned. The
-
211  * actual data is stored in the compact_* tables, the ids and codes
-
212  * tables are not used. */
-
213 struct IndexIVFPQCompact: IndexIVFPQ {
-
214 
-
215  explicit IndexIVFPQCompact (const IndexIVFPQ &other);
-
216 
-
217  /// how were the compact tables allocated?
-
218  enum Alloc_type_t {
-
219  Alloc_type_none, ///< alloc from outside
-
220  Alloc_type_new, ///< was allocated with new
-
221  Alloc_type_mmap ///< was mmapped
-
222  };
-
223 
-
224  Alloc_type_t alloc_type;
-
225 
-
226  uint32_t *limits; ///< size nlist + 1
-
227  uint32_t *compact_ids; ///< size ntotal
-
228  uint8_t *compact_codes; ///< size ntotal * code_size
-
229 
-
230  // file and buffer this was mmapped (will be unmapped when object
-
231  // is deleted)
-
232  char * mmap_buffer;
-
233  long mmap_length;
-
234 
-
235  void search_preassigned (idx_t n, const float *x, idx_t k,
-
236  const idx_t *assign,
-
237  const float *centroid_dis,
-
238  float *distances, idx_t *labels,
-
239  bool store_pairs) const override;
-
240 
-
241  /// the three following functions will fail at runtime
-
242  void add(idx_t, const float*) override;
-
243  void reset() override;
-
244  void train(idx_t, const float*) override;
-
245 
-
246  ~IndexIVFPQCompact() override;
-
247 
-
248  IndexIVFPQCompact ();
-
249 
-
250 };
-
251 
-
252 
-
253 
-
254 } // namespace faiss
-
255 
-
256 
+
205  Alloc_type_t alloc_type;
+
206 
+
207  uint32_t *limits; ///< size nlist + 1
+
208  uint32_t *compact_ids; ///< size ntotal
+
209  uint8_t *compact_codes; ///< size ntotal * code_size
+
210 
+
211  // file and buffer this was mmapped (will be unmapped when object
+
212  // is deleted)
+
213  char * mmap_buffer;
+
214  long mmap_length;
+
215 
+
216  void search_preassigned (idx_t n, const float *x, idx_t k,
+
217  const idx_t *assign,
+
218  const float *centroid_dis,
+
219  float *distances, idx_t *labels,
+
220  bool store_pairs) const override;
+
221 
+
222  /// the three following functions will fail at runtime
+
223  void add(idx_t, const float*) override;
+
224  void reset() override;
+
225  void train(idx_t, const float*) override;
+
226 
+
227  ~IndexIVFPQCompact() override;
+
228 
+
229  IndexIVFPQCompact ();
+
230 
+
231 };
+
232 
+
233 
+
234 /** Same as an IndexIVFPQ without the inverted lists: codes are stored sequentially
+
235  *
+
236  * The class is mainly inteded to store encoded vectors that can be
+
237  * accessed randomly, the search function is not implemented.
+
238  */
+
239 struct Index2Layer: Index {
+
240  /// first level quantizer
+
241  Level1Quantizer q1;
+
242 
+
243  /// second level quantizer is always a PQ
+
244  ProductQuantizer pq;
+
245 
+
246  /// Codes. Size ntotal * code_size.
+
247  std::vector<uint8_t> codes;
+
248 
+
249  /// size of the code for the first level (ceil(log8(q1.nlist)))
+
250  size_t code_size_1;
+
251 
+
252  /// size of the code for the second level
+
253  size_t code_size_2;
+
254 
+
255  /// code_size_1 + code_size_2
+
256  size_t code_size;
257 
-
258 
-
259 
-
260 #endif
-
faiss::IndexIVFPQCompact::compact_ids
uint32_t * compact_ids
size ntotal
Definition: IndexIVFPQ.h:227
- -
faiss::IndexIVFPQCompact::compact_codes
uint8_t * compact_codes
size ntotal * code_size
Definition: IndexIVFPQ.h:228
-
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:356
-
faiss::IndexIVFPQR::merge_from
void merge_from(IndexIVF &other, idx_t add_id) override
Definition: IndexIVFPQ.cpp:1320
-
faiss::IndexIVFPQ::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVFPQ.cpp:302
- -
faiss::IndexIVFPQR::refine_pq
ProductQuantizer refine_pq
3rd level quantizer
Definition: IndexIVFPQ.h:169
+
258  Index2Layer (Index * quantizer, size_t nlist,
+
259  int M, MetricType metric = METRIC_L2);
+
260 
+
261  Index2Layer ();
+
262  ~Index2Layer ();
+
263 
+
264  void train(idx_t n, const float* x) override;
+
265 
+
266  void add(idx_t n, const float* x) override;
+
267 
+
268  /// not implemented
+
269  void search(
+
270  idx_t n,
+
271  const float* x,
+
272  idx_t k,
+
273  float* distances,
+
274  idx_t* labels) const override;
+
275 
+
276  void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;
+
277 
+
278  void reconstruct(idx_t key, float* recons) const override;
+
279 
+
280  void reset() override;
+
281 
+
282  /// transfer the flat codes to an IVFPQ index
+
283  void transfer_to_IVFPQ(IndexIVFPQ & other) const;
+
284 
+
285 };
+
286 
+
287 
+
288 
+
289 } // namespace faiss
+
290 
+
291 
+
292 
+
293 
+
294 
+
295 #endif
+
faiss::IndexIVFPQCompact::compact_ids
uint32_t * compact_ids
size ntotal
Definition: IndexIVFPQ.h:208
+ +
faiss::IndexIVFPQCompact::compact_codes
uint8_t * compact_codes
size ntotal * code_size
Definition: IndexIVFPQ.h:209
+
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:344
+
faiss::IndexIVFPQR::merge_from
void merge_from(IndexIVF &other, idx_t add_id) override
Definition: IndexIVFPQ.cpp:1250
+
faiss::Index2Layer::transfer_to_IVFPQ
void transfer_to_IVFPQ(IndexIVFPQ &other) const
transfer the flat codes to an IVFPQ index
Definition: IndexIVFPQ.cpp:1416
+
faiss::Index2Layer::code_size_2
size_t code_size_2
size of the code for the second level
Definition: IndexIVFPQ.h:253
+
faiss::IndexIVFPQR::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVFPQ.cpp:1235
+ +
faiss::IndexIVFPQR::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:1152
+ +
faiss::IndexIVFPQR::refine_pq
ProductQuantizer refine_pq
3rd level quantizer
Definition: IndexIVFPQ.h:150
faiss::IndexIVFPQ::polysemous_training
PolysemousTraining * polysemous_training
if NULL, use default
Definition: IndexIVFPQ.h:35
-
faiss::IndexIVFPQCompact::add
void add(idx_t, const float *) override
the three following functions will fail at runtime
Definition: IndexIVFPQ.cpp:1416
- -
faiss::IndexIVFPQR::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVFPQ.cpp:1301
-
faiss::IndexIVFPQCompact::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:1446
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
+
faiss::Index2Layer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVFPQ.cpp:1396
+
faiss::Index2Layer::code_size
size_t code_size
code_size_1 + code_size_2
Definition: IndexIVFPQ.h:256
+
faiss::IndexIVFPQCompact::add
void add(idx_t, const float *) override
the three following functions will fail at runtime
Definition: IndexIVFPQ.cpp:1546
+ +
faiss::IndexIVFPQCompact::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:1576
+
faiss::Index2Layer::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1447
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
- -
faiss::IndexIVFPQ::decode_multiple
void decode_multiple(size_t n, const long *keys, const uint8_t *xcodes, float *x) const
inverse of encode_multiple
Definition: IndexIVFPQ.cpp:169
-
faiss::IndexIVFPQ::train_residual_o
void train_residual_o(idx_t n, const float *x, float *residuals_2)
same as train_residual, also output 2nd level residuals
Definition: IndexIVFPQ.cpp:71
+ +
faiss::IndexIVFPQ::decode_multiple
void decode_multiple(size_t n, const long *keys, const uint8_t *xcodes, float *x) const
inverse of encode_multiple
Definition: IndexIVFPQ.cpp:168
+
faiss::IndexIVFPQ::train_residual_o
void train_residual_o(idx_t n, const float *x, float *residuals_2)
same as train_residual, also output 2nd level residuals
Definition: IndexIVFPQ.cpp:70
faiss::IndexIVFPQ::do_polysemous_training
bool do_polysemous_training
reorder PQ centroids after training?
Definition: IndexIVFPQ.h:34
faiss::IndexIVFPQ::scan_table_threshold
size_t scan_table_threshold
use table computation or on-the-fly?
Definition: IndexIVFPQ.h:38
-
faiss::IndexIVFPQR::train_residual
void train_residual(idx_t n, const float *x) override
trains the two product quantizers
Definition: IndexIVFPQ.cpp:1169
-
faiss::IndexIVFPQR::add_core
void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx=nullptr)
same as add_with_ids, but optionally use the precomputed list ids
Definition: IndexIVFPQ.cpp:1193
-
faiss::IndexIVFPQCompact::limits
uint32_t * limits
size nlist + 1
Definition: IndexIVFPQ.h:226
-
faiss::IndexIVFPQ::precomputed_table
std::vector< float > precomputed_table
Definition: IndexIVFPQ.h:45
-
faiss::IndexIVFPQ::polysemous_ht
int polysemous_ht
Hamming thresh for polysemous filtering.
Definition: IndexIVFPQ.h:40
-
faiss::IndexIVFPQR::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1160
-
faiss::IndexIVFPQ::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids=nullptr) override
Definition: IndexIVFPQ.cpp:186
- -
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
+
faiss::IndexIVFPQR::train_residual
void train_residual(idx_t n, const float *x) override
trains the two product quantizers
Definition: IndexIVFPQ.cpp:1109
+
faiss::IndexIVFPQR::add_core
void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx=nullptr)
same as add_with_ids, but optionally use the precomputed list ids
Definition: IndexIVFPQ.cpp:1133
+
faiss::IndexIVFPQCompact::limits
uint32_t * limits
size nlist + 1
Definition: IndexIVFPQ.h:207
+
faiss::IndexIVFPQ::precomputed_table
std::vector< float > precomputed_table
Definition: IndexIVFPQ.h:44
+
faiss::Index2Layer::q1
Level1Quantizer q1
first level quantizer
Definition: IndexIVFPQ.h:241
+
faiss::IndexIVFPQ::polysemous_ht
int polysemous_ht
Hamming thresh for polysemous filtering.
Definition: IndexIVFPQ.h:39
+
faiss::IndexIVFPQR::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1100
+
faiss::Index2Layer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * code_size.
Definition: IndexIVFPQ.h:247
+
faiss::IndexIVFPQ::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids=nullptr) override
Definition: IndexIVFPQ.cpp:185
+ + +
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
-
faiss::IndexIVFPQ::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVFPQ.h:39
-
faiss::IndexIVFPQCompact::Alloc_type_t
Alloc_type_t
how were the compact tables allocated?
Definition: IndexIVFPQ.h:218
-
faiss::IndexIVFPQR::refine_codes
std::vector< uint8_t > refine_codes
corresponding codes
Definition: IndexIVFPQ.h:170
-
faiss::IndexIVFPQ::train_residual
void train_residual(idx_t n, const float *x) override
trains the product quantizer
Definition: IndexIVFPQ.cpp:65
+
faiss::IndexIVFPQCompact::Alloc_type_t
Alloc_type_t
how were the compact tables allocated?
Definition: IndexIVFPQ.h:199
+
faiss::IndexIVFPQR::refine_codes
std::vector< uint8_t > refine_codes
corresponding codes
Definition: IndexIVFPQ.h:151
+
faiss::IndexIVFPQ::train_residual
void train_residual(idx_t n, const float *x) override
trains the product quantizer
Definition: IndexIVFPQ.cpp:64
+
faiss::IndexIVFPQ::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVFPQ.cpp:291
-
faiss::IndexIVFPQ::encode_multiple
void encode_multiple(size_t n, long *keys, const float *x, uint8_t *codes, bool compute_keys=false) const
Definition: IndexIVFPQ.cpp:150
-
faiss::IndexIVFPQCompact::train
void train(idx_t, const float *) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVFPQ.cpp:1424
+
faiss::IndexIVFPQ::encode_multiple
void encode_multiple(size_t n, long *keys, const float *x, uint8_t *codes, bool compute_keys=false) const
Definition: IndexIVFPQ.cpp:149
+
faiss::IndexIVFPQCompact::train
void train(idx_t, const float *) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVFPQ.cpp:1554
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
+
faiss::Index2Layer::train
void train(idx_t n, const float *x) override
Definition: IndexIVFPQ.cpp:1299
faiss::PolysemousTraining
optimizes the order of indices in a ProductQuantizer
Definition: PolysemousTraining.h:125
+
faiss::Index2Layer::add
void add(idx_t n, const float *x) override
Definition: IndexIVFPQ.cpp:1337
+
faiss::Index2Layer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
not implemented
Definition: IndexIVFPQ.cpp:1385
faiss::IndexIVFPQ::by_residual
bool by_residual
Encode residual or plain vector?
Definition: IndexIVFPQ.h:30
- -
faiss::IndexIVFPQ::search_and_reconstruct
void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *reconstructed)
Definition: IndexIVFPQ.cpp:1032
+
faiss::IndexIVFPQ::pq
ProductQuantizer pq
produces the codes
Definition: IndexIVFPQ.h:32
-
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
+
faiss::Index2Layer::code_size_1
size_t code_size_1
size of the code for the first level (ceil(log8(q1.nlist)))
Definition: IndexIVFPQ.h:250
-
faiss::IndexIVFPQ::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVFPQ.cpp:274
-
faiss::IndexIVFPQ::add_core_o
void add_core_o(idx_t n, const float *x, const long *xids, float *residuals_2, const long *precomputed_idx=nullptr)
Definition: IndexIVFPQ.cpp:192
-
faiss::IndexIVFPQR::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVFPQ.cpp:1333
-
faiss::IndexIVFPQ::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:931
+
faiss::IndexIVFPQ::add_core_o
void add_core_o(idx_t n, const float *x, const long *xids, float *residuals_2, const long *precomputed_idx=nullptr)
Definition: IndexIVFPQ.cpp:191
+
faiss::IndexIVFPQR::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVFPQ.cpp:1263
+
faiss::IndexIVFPQ::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:919
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
-
faiss::IndexIVFPQCompact::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1420
- -
faiss::IndexIVFPQR::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexIVFPQ.cpp:1189
-
faiss::IndexIVFPQR::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVFPQ.cpp:1212
-
faiss::IndexIVFPQ::find_duplicates
size_t find_duplicates(idx_t *ids, size_t *lims) const
Definition: IndexIVFPQ.cpp:1104
-
faiss::IndexIVFPQR::k_factor
float k_factor
factor between k requested in search and the k requested from the IVFPQ
Definition: IndexIVFPQ.h:173
+
faiss::IndexIVFPQCompact::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1550
+
faiss::Index2Layer::pq
ProductQuantizer pq
second level quantizer is always a PQ
Definition: IndexIVFPQ.h:244
+ +
faiss::IndexIVFPQR::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexIVFPQ.cpp:1129
+
faiss::Index2Layer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVFPQ.cpp:1442
+
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
+
faiss::IndexIVFPQ::find_duplicates
size_t find_duplicates(idx_t *ids, size_t *lims) const
Definition: IndexIVFPQ.cpp:1044
+
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
+
faiss::IndexIVFPQR::k_factor
float k_factor
factor between k requested in search and the k requested from the IVFPQ
Definition: IndexIVFPQ.h:154
faiss::IndexIVFPQ::use_precomputed_table
int use_precomputed_table
if by_residual, build precompute tables
Definition: IndexIVFPQ.h:31
diff --git a/docs/html/IndexIVF_8cpp_source.html b/docs/html/IndexIVF_8cpp_source.html index 1e63f5743..25e6c686a 100644 --- a/docs/html/IndexIVF_8cpp_source.html +++ b/docs/html/IndexIVF_8cpp_source.html @@ -109,696 +109,803 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
24 namespace faiss {
25 
26 /*****************************************
-
27  * IndexIVF implementation
+
27  * Level1Quantizer implementation
28  ******************************************/
29 
30 
-
31 IndexIVF::IndexIVF (Index * quantizer, size_t d, size_t nlist,
-
32  MetricType metric):
-
33  Index (d, metric),
-
34  nlist (nlist),
-
35  nprobe (1),
-
36  quantizer (quantizer),
-
37  quantizer_trains_alone (0),
-
38  own_fields (false),
-
39  clustering_index (nullptr),
-
40  ids (nlist),
-
41  maintain_direct_map (false)
-
42 {
-
43  FAISS_THROW_IF_NOT (d == quantizer->d);
-
44  is_trained = quantizer->is_trained && (quantizer->ntotal == nlist);
-
45  // Spherical by default if the metric is inner_product
-
46  if (metric_type == METRIC_INNER_PRODUCT) {
-
47  cp.spherical = true;
-
48  }
-
49  // here we set a low # iterations because this is typically used
-
50  // for large clusterings (nb this is not used for the MultiIndex,
-
51  // for which quantizer_trains_alone = true)
-
52  cp.niter = 10;
-
53  cp.verbose = verbose;
-
54  code_size = 0; // let sub-classes set this
-
55  codes.resize(nlist);
-
56 }
-
57 
-
58 IndexIVF::IndexIVF ():
-
59  nlist (0), nprobe (1), quantizer (nullptr),
-
60  quantizer_trains_alone (0), own_fields (false),
-
61  clustering_index (nullptr),
-
62  maintain_direct_map (false)
-
63 {}
-
64 
-
65 
-
66 void IndexIVF::add (idx_t n, const float * x)
-
67 {
-
68  add_with_ids (n, x, nullptr);
-
69 }
+
31 Level1Quantizer::Level1Quantizer (Index * quantizer, size_t nlist):
+
32  quantizer (quantizer),
+
33  nlist (nlist),
+
34  quantizer_trains_alone (0),
+
35  own_fields (false),
+
36  clustering_index (nullptr)
+
37 {
+
38  cp.niter = 10;
+
39 }
+
40 
+
41 Level1Quantizer::Level1Quantizer ():
+
42  quantizer (nullptr),
+
43  nlist (0),
+
44  quantizer_trains_alone (0), own_fields (false),
+
45  clustering_index (nullptr)
+
46 {}
+
47 
+
48 Level1Quantizer::~Level1Quantizer ()
+
49 {
+
50  if (own_fields) delete quantizer;
+
51 }
+
52 
+
53 void Level1Quantizer::train_q1 (size_t n, const float *x, bool verbose, MetricType metric_type)
+
54 {
+
55  size_t d = quantizer->d;
+
56  if (quantizer->is_trained && (quantizer->ntotal == nlist)) {
+
57  if (verbose)
+
58  printf ("IVF quantizer does not need training.\n");
+
59  } else if (quantizer_trains_alone == 1) {
+
60  if (verbose)
+
61  printf ("IVF quantizer trains alone...\n");
+
62  quantizer->train (n, x);
+
63  quantizer->verbose = verbose;
+
64  FAISS_THROW_IF_NOT_MSG (quantizer->ntotal == nlist,
+
65  "nlist not consistent with quantizer size");
+
66  } else if (quantizer_trains_alone == 0) {
+
67  if (verbose)
+
68  printf ("Training level-1 quantizer on %ld vectors in %ldD\n",
+
69  n, d);
70 
-
71 void IndexIVF::make_direct_map (bool new_maintain_direct_map)
-
72 {
-
73  // nothing to do
-
74  if (new_maintain_direct_map == maintain_direct_map)
-
75  return;
-
76 
-
77  if (new_maintain_direct_map) {
-
78  direct_map.resize (ntotal, -1);
-
79  for (size_t key = 0; key < nlist; key++) {
-
80  const std::vector<long> & idlist = ids[key];
-
81 
-
82  for (long ofs = 0; ofs < idlist.size(); ofs++) {
-
83  FAISS_THROW_IF_NOT_MSG (
-
84  0 <= idlist [ofs] && idlist[ofs] < ntotal,
-
85  "direct map supported only for seuquential ids");
-
86  direct_map [idlist [ofs]] = key << 32 | ofs;
-
87  }
-
88  }
-
89  } else {
-
90  direct_map.clear ();
-
91  }
-
92  maintain_direct_map = new_maintain_direct_map;
-
93 }
-
94 
-
95 
-
96 void IndexIVF::search (idx_t n, const float *x, idx_t k,
-
97  float *distances, idx_t *labels) const
-
98 {
-
99  long * idx = new long [n * nprobe];
-
100  ScopeDeleter<long> del (idx);
-
101  float * coarse_dis = new float [n * nprobe];
-
102  ScopeDeleter<float> del2 (coarse_dis);
-
103 
-
104  quantizer->search (n, x, nprobe, coarse_dis, idx);
+
71  Clustering clus (d, nlist, cp);
+
72  quantizer->reset();
+
73  if (clustering_index) {
+
74  clus.train (n, x, *clustering_index);
+
75  quantizer->add (nlist, clus.centroids.data());
+
76  } else {
+
77  clus.train (n, x, *quantizer);
+
78  }
+
79  quantizer->is_trained = true;
+
80  } else if (quantizer_trains_alone == 2) {
+
81  if (verbose)
+
82  printf (
+
83  "Training L2 quantizer on %ld vectors in %ldD%s\n",
+
84  n, d,
+
85  clustering_index ? "(user provided index)" : "");
+
86  FAISS_THROW_IF_NOT (metric_type == METRIC_L2);
+
87  Clustering clus (d, nlist, cp);
+
88  if (!clustering_index) {
+
89  IndexFlatL2 assigner (d);
+
90  clus.train(n, x, assigner);
+
91  } else {
+
92  clus.train(n, x, *clustering_index);
+
93  }
+
94  if (verbose)
+
95  printf ("Adding centroids to quantizer\n");
+
96  quantizer->add (nlist, clus.centroids.data());
+
97  }
+
98 }
+
99 
+
100 
+
101 /*****************************************
+
102  * IndexIVF implementation
+
103  ******************************************/
+
104 
105 
-
106  search_preassigned (n, x, k, idx, coarse_dis,
-
107  distances, labels, false);
-
108 
-
109 }
-
110 
-
111 
-
112 void IndexIVF::reset ()
+
106 IndexIVF::IndexIVF (Index * quantizer, size_t d, size_t nlist,
+
107  MetricType metric):
+
108  Index (d, metric),
+
109  Level1Quantizer (quantizer, nlist),
+
110  nprobe (1),
+
111  max_codes (0),
+
112  maintain_direct_map (false)
113 {
-
114  ntotal = 0;
-
115  direct_map.clear();
-
116  for (size_t i = 0; i < ids.size(); i++) {
-
117  ids[i].clear();
-
118  codes[i].clear();
+
114  FAISS_THROW_IF_NOT (d == quantizer->d);
+
115  is_trained = quantizer->is_trained && (quantizer->ntotal == nlist);
+
116  // Spherical by default if the metric is inner_product
+
117  if (metric_type == METRIC_INNER_PRODUCT) {
+
118  cp.spherical = true;
119  }
-
120 }
-
121 
-
122 
-
123 long IndexIVF::remove_ids (const IDSelector & sel)
-
124 {
-
125  FAISS_THROW_IF_NOT_MSG (!maintain_direct_map,
-
126  "direct map remove not implemented");
-
127  long nremove = 0;
-
128 #pragma omp parallel for reduction(+: nremove)
-
129  for (long i = 0; i < nlist; i++) {
-
130  std::vector<idx_t> & idsi = ids[i];
-
131  uint8_t * codesi = codes[i].data();
+
120  // here we set a low # iterations because this is typically used
+
121  // for large clusterings (nb this is not used for the MultiIndex,
+
122  // for which quantizer_trains_alone = true)
+
123  code_size = 0; // let sub-classes set this
+
124  ids.resize (nlist);
+
125  codes.resize (nlist);
+
126 }
+
127 
+
128 IndexIVF::IndexIVF ():
+
129  nprobe (1), max_codes (0),
+
130  maintain_direct_map (false)
+
131 {}
132 
-
133  long l = idsi.size(), j = 0;
-
134  while (j < l) {
-
135  if (sel.is_member (idsi[j])) {
-
136  l--;
-
137  idsi [j] = idsi [l];
-
138  memmove (codesi + j * code_size,
-
139  codesi + l * code_size, code_size);
-
140  } else {
-
141  j++;
-
142  }
-
143  }
-
144  if (l < idsi.size()) {
-
145  nremove += idsi.size() - l;
-
146  idsi.resize (l);
-
147  codes[i].resize (l * code_size);
-
148  }
-
149  }
-
150  ntotal -= nremove;
-
151  return nremove;
-
152 }
-
153 
-
154 
-
155 
-
156 
-
157 void IndexIVF::train (idx_t n, const float *x)
-
158 {
-
159  if (quantizer->is_trained && (quantizer->ntotal == nlist)) {
-
160  if (verbose)
-
161  printf ("IVF quantizer does not need training.\n");
-
162  } else if (quantizer_trains_alone == 1) {
-
163  if (verbose)
-
164  printf ("IVF quantizer trains alone...\n");
-
165  quantizer->train (n, x);
-
166  quantizer->verbose = verbose;
-
167  FAISS_THROW_IF_NOT_MSG (quantizer->ntotal == nlist,
-
168  "nlist not consistent with quantizer size");
-
169  } else if (quantizer_trains_alone == 0) {
-
170  if (verbose)
-
171  printf ("Training IVF quantizer on %ld vectors in %dD\n",
-
172  n, d);
+
133 
+
134 void IndexIVF::add (idx_t n, const float * x)
+
135 {
+
136  add_with_ids (n, x, nullptr);
+
137 }
+
138 
+
139 void IndexIVF::make_direct_map (bool new_maintain_direct_map)
+
140 {
+
141  // nothing to do
+
142  if (new_maintain_direct_map == maintain_direct_map)
+
143  return;
+
144 
+
145  if (new_maintain_direct_map) {
+
146  direct_map.resize (ntotal, -1);
+
147  for (size_t key = 0; key < nlist; key++) {
+
148  const std::vector<long> & idlist = ids[key];
+
149 
+
150  for (long ofs = 0; ofs < idlist.size(); ofs++) {
+
151  FAISS_THROW_IF_NOT_MSG (
+
152  0 <= idlist [ofs] && idlist[ofs] < ntotal,
+
153  "direct map supported only for seuquential ids");
+
154  direct_map [idlist [ofs]] = key << 32 | ofs;
+
155  }
+
156  }
+
157  } else {
+
158  direct_map.clear ();
+
159  }
+
160  maintain_direct_map = new_maintain_direct_map;
+
161 }
+
162 
+
163 
+
164 void IndexIVF::search (idx_t n, const float *x, idx_t k,
+
165  float *distances, idx_t *labels) const
+
166 {
+
167  long * idx = new long [n * nprobe];
+
168  ScopeDeleter<long> del (idx);
+
169  float * coarse_dis = new float [n * nprobe];
+
170  ScopeDeleter<float> del2 (coarse_dis);
+
171 
+
172  quantizer->search (n, x, nprobe, coarse_dis, idx);
173 
-
174  Clustering clus (d, nlist, cp);
-
175  quantizer->reset();
-
176  if (clustering_index) {
-
177  clus.train (n, x, *clustering_index);
-
178  quantizer->add (nlist, clus.centroids.data());
-
179  } else {
-
180  clus.train (n, x, *quantizer);
-
181  }
-
182  quantizer->is_trained = true;
-
183  } else if (quantizer_trains_alone == 2) {
-
184  if (verbose)
-
185  printf (
-
186  "Training L2 quantizer on %ld vectors in %dD%s\n",
-
187  n, d,
-
188  clustering_index ? "(user provided index)" : "");
-
189  FAISS_THROW_IF_NOT (metric_type == METRIC_L2);
-
190  Clustering clus (d, nlist, cp);
-
191  if (!clustering_index) {
-
192  IndexFlatL2 assigner (d);
-
193  clus.train(n, x, assigner);
-
194  } else {
-
195  clus.train(n, x, *clustering_index);
-
196  }
-
197  if (verbose)
-
198  printf ("Adding centroids to quantizer\n");
-
199  quantizer->add (nlist, clus.centroids.data());
-
200  }
-
201  if (verbose)
-
202  printf ("Training IVF residual\n");
-
203 
-
204  train_residual (n, x);
-
205  is_trained = true;
-
206 }
-
207 
-
208 void IndexIVF::train_residual(idx_t /*n*/, const float* /*x*/) {
-
209  if (verbose)
-
210  printf("IndexIVF: no residual training\n");
-
211  // does nothing by default
-
212 }
-
213 
-
214 
-
215 
-
216 double IndexIVF::imbalance_factor () const
-
217 {
-
218  std::vector<int> hist (nlist);
-
219  for (int i = 0; i < nlist; i++) {
-
220  hist[i] = ids[i].size();
-
221  }
-
222  return faiss::imbalance_factor (nlist, hist.data());
-
223 }
-
224 
-
225 void IndexIVF::print_stats () const
-
226 {
-
227  std::vector<int> sizes(40);
-
228  for (int i = 0; i < nlist; i++) {
-
229  for (int j = 0; j < sizes.size(); j++) {
-
230  if ((ids[i].size() >> j) == 0) {
-
231  sizes[j]++;
-
232  break;
-
233  }
-
234  }
-
235  }
-
236  for (int i = 0; i < sizes.size(); i++) {
-
237  if (sizes[i]) {
-
238  printf ("list size in < %d: %d instances\n",
-
239  1 << i, sizes[i]);
-
240  }
-
241  }
-
242 
-
243 }
-
244 
-
245 void IndexIVF::merge_from (IndexIVF &other, idx_t add_id)
-
246 {
-
247  // minimal sanity checks
-
248  FAISS_THROW_IF_NOT (other.d == d);
-
249  FAISS_THROW_IF_NOT (other.nlist == nlist);
-
250  FAISS_THROW_IF_NOT_MSG ((!maintain_direct_map &&
-
251  !other.maintain_direct_map),
-
252  "direct map copy not implemented");
-
253  FAISS_THROW_IF_NOT_MSG (typeid (*this) == typeid (other),
-
254  "can only merge indexes of the same type");
-
255  for (long i = 0; i < nlist; i++) {
-
256  std::vector<idx_t> & src = other.ids[i];
-
257  std::vector<idx_t> & dest = ids[i];
-
258  for (long j = 0; j < src.size(); j++)
-
259  dest.push_back (src[j] + add_id);
-
260  src.clear();
-
261  codes[i].insert (codes[i].end(),
-
262  other.codes[i].begin(),
-
263  other.codes[i].end());
-
264  other.codes[i].clear();
-
265  }
-
266 
-
267  ntotal += other.ntotal;
-
268  other.ntotal = 0;
-
269 }
-
270 
-
271 
-
272 void IndexIVF::copy_subset_to (IndexIVF & other, int subset_type,
-
273  long a1, long a2) const
-
274 {
-
275  FAISS_THROW_IF_NOT (nlist == other.nlist);
-
276  FAISS_THROW_IF_NOT (!other.maintain_direct_map);
-
277  FAISS_THROW_IF_NOT_FMT (
-
278  subset_type == 0 || subset_type == 1 || subset_type == 2,
-
279  "subset type %d not implemented", subset_type);
-
280 
-
281  size_t accu_n = 0;
-
282  size_t accu_a1 = 0;
-
283  size_t accu_a2 = 0;
-
284 
-
285  for (long list_no = 0; list_no < nlist; list_no++) {
-
286  const std::vector<idx_t> & ids_in = ids[list_no];
-
287  std::vector<idx_t> & ids_out = other.ids[list_no];
-
288  const std::vector<uint8_t> & codes_in = codes[list_no];
-
289  std::vector<uint8_t> & codes_out = other.codes[list_no];
-
290  size_t n = ids_in.size();
-
291 
-
292  if (subset_type == 0) {
-
293  for (long i = 0; i < n; i++) {
-
294  idx_t id = ids_in[i];
-
295  if (a1 <= id && id < a2) {
-
296  ids_out.push_back (id);
-
297  codes_out.insert (codes_out.end(),
-
298  codes_in.begin() + i * code_size,
-
299  codes_in.begin() + (i + 1) * code_size);
-
300  other.ntotal++;
-
301  }
-
302  }
-
303  } else if (subset_type == 1) {
-
304  for (long i = 0; i < n; i++) {
-
305  idx_t id = ids_in[i];
-
306  if (id % a1 == a2) {
-
307  ids_out.push_back (id);
-
308  codes_out.insert (codes_out.end(),
-
309  codes_in.begin() + i * code_size,
-
310  codes_in.begin() + (i + 1) * code_size);
-
311  other.ntotal++;
-
312  }
-
313  }
-
314  } else if (subset_type == 2) {
-
315  // see what is allocated to a1 and to a2
-
316  size_t next_accu_n = accu_n + n;
-
317  size_t next_accu_a1 = next_accu_n * a1 / ntotal;
-
318  size_t i1 = next_accu_a1 - accu_a1;
-
319  size_t next_accu_a2 = next_accu_n * a2 / ntotal;
-
320  size_t i2 = next_accu_a2 - accu_a2;
-
321  ids_out.insert(ids_out.end(),
-
322  ids_in.begin() + i1,
-
323  ids_in.begin() + i2);
-
324  codes_out.insert (codes_out.end(),
-
325  codes_in.begin() + i1 * code_size,
-
326  codes_in.begin() + i2 * code_size);
-
327  other.ntotal += i2 - i1;
-
328  accu_a1 = next_accu_a1;
-
329  accu_a2 = next_accu_a2;
-
330  }
-
331  accu_n += n;
-
332  }
-
333  FAISS_ASSERT(accu_n == ntotal);
-
334 }
-
335 
-
336 
-
337 
-
338 IndexIVF::~IndexIVF()
-
339 {
-
340  if (own_fields) delete quantizer;
-
341 }
-
342 
-
343 
-
344 
-
345 /*****************************************
-
346  * IndexIVFFlat implementation
-
347  ******************************************/
-
348 
-
349 IndexIVFFlat::IndexIVFFlat (Index * quantizer,
-
350  size_t d, size_t nlist, MetricType metric):
-
351  IndexIVF (quantizer, d, nlist, metric)
-
352 {
-
353  code_size = sizeof(float) * d;
-
354 }
-
355 
-
356 
-
357 
-
358 
-
359 
-
360 
-
361 void IndexIVFFlat::add_with_ids (idx_t n, const float * x, const long *xids)
-
362 {
-
363  add_core (n, x, xids, nullptr);
-
364 }
-
365 
-
366 void IndexIVFFlat::add_core (idx_t n, const float * x, const long *xids,
-
367  const long *precomputed_idx)
-
368 
-
369 {
-
370  FAISS_THROW_IF_NOT (is_trained);
-
371  FAISS_THROW_IF_NOT_MSG (!(maintain_direct_map && xids),
-
372  "cannot have direct map and add with ids");
-
373  const long * idx;
-
374  ScopeDeleter<long> del;
-
375 
-
376  if (precomputed_idx) {
-
377  idx = precomputed_idx;
-
378  } else {
-
379  long * idx0 = new long [n];
-
380  quantizer->assign (n, x, idx0);
-
381  idx = idx0;
-
382  del.set (idx);
-
383  }
-
384  long n_add = 0;
-
385  for (size_t i = 0; i < n; i++) {
-
386  long id = xids ? xids[i] : ntotal + i;
-
387  long list_no = idx [i];
-
388  if (list_no < 0)
-
389  continue;
-
390  assert (list_no < nlist);
-
391 
-
392  ids[list_no].push_back (id);
-
393  const float *xi = x + i * d;
-
394  /* store the vectors */
-
395  size_t ofs = codes[list_no].size();
-
396  codes[list_no].resize(ofs + code_size);
-
397  memcpy(codes[list_no].data() + ofs,
-
398  xi, code_size);
-
399 
-
400  if (maintain_direct_map)
-
401  direct_map.push_back (list_no << 32 | (ids[list_no].size() - 1));
-
402  n_add++;
-
403  }
-
404  if (verbose) {
-
405  printf("IndexIVFFlat::add_core: added %ld / %ld vectors\n",
-
406  n_add, n);
-
407  }
-
408  ntotal += n_add;
-
409 }
-
410 
-
411 void IndexIVFFlatStats::reset()
-
412 {
-
413  memset ((void*)this, 0, sizeof (*this));
-
414 }
-
415 
-
416 
-
417 IndexIVFFlatStats indexIVFFlat_stats;
-
418 
-
419 namespace {
-
420 
-
421 void search_knn_inner_product (const IndexIVFFlat & ivf,
-
422  size_t nx,
-
423  const float * x,
-
424  const long * keys,
-
425  float_minheap_array_t * res,
-
426  bool store_pairs)
-
427 {
-
428 
-
429  const size_t k = res->k;
-
430  size_t nlistv = 0, ndis = 0;
-
431  size_t d = ivf.d;
-
432 
-
433 #pragma omp parallel for reduction(+: nlistv, ndis)
-
434  for (size_t i = 0; i < nx; i++) {
-
435  const float * xi = x + i * d;
-
436  const long * keysi = keys + i * ivf.nprobe;
-
437  float * __restrict simi = res->get_val (i);
-
438  long * __restrict idxi = res->get_ids (i);
-
439  minheap_heapify (k, simi, idxi);
+
174  search_preassigned (n, x, k, idx, coarse_dis,
+
175  distances, labels, false);
+
176 
+
177 }
+
178 
+
179 
+
180 void IndexIVF::reconstruct (idx_t key, float* recons) const
+
181 {
+
182  FAISS_THROW_IF_NOT_MSG (direct_map.size() == ntotal,
+
183  "direct map is not initialized");
+
184  long list_no = direct_map[key] >> 32;
+
185  long offset = direct_map[key] & 0xffffffff;
+
186  reconstruct_from_offset (list_no, offset, recons);
+
187 }
+
188 
+
189 
+
190 void IndexIVF::reconstruct_n (idx_t i0, idx_t ni, float* recons) const
+
191 {
+
192  FAISS_THROW_IF_NOT (ni == 0 || (i0 >= 0 && i0 + ni <= ntotal));
+
193 
+
194  for (long list_no = 0; list_no < nlist; list_no++) {
+
195  const std::vector<long>& idlist = ids[list_no];
+
196 
+
197  for (long offset = 0; offset < idlist.size(); offset++) {
+
198  long id = idlist[offset];
+
199  if (!(id >= i0 && id < i0 + ni)) {
+
200  continue;
+
201  }
+
202 
+
203  float* reconstructed = recons + (id - i0) * d;
+
204  reconstruct_from_offset (list_no, offset, reconstructed);
+
205  }
+
206  }
+
207 }
+
208 
+
209 
+
210 void IndexIVF::search_and_reconstruct (idx_t n, const float *x, idx_t k,
+
211  float *distances, idx_t *labels,
+
212  float *recons) const
+
213 {
+
214  long * idx = new long [n * nprobe];
+
215  ScopeDeleter<long> del (idx);
+
216  float * coarse_dis = new float [n * nprobe];
+
217  ScopeDeleter<float> del2 (coarse_dis);
+
218 
+
219  quantizer->search (n, x, nprobe, coarse_dis, idx);
+
220 
+
221  // search_preassigned() with `store_pairs` enabled to obtain the list_no
+
222  // and offset into `codes` for reconstruction
+
223  search_preassigned (n, x, k, idx, coarse_dis,
+
224  distances, labels, true /* store_pairs */);
+
225  for (idx_t i = 0; i < n; ++i) {
+
226  for (idx_t j = 0; j < k; ++j) {
+
227  idx_t ij = i * k + j;
+
228  idx_t key = labels[ij];
+
229  float* reconstructed = recons + ij * d;
+
230  if (key < 0) {
+
231  // Fill with NaNs
+
232  memset(reconstructed, -1, sizeof(*reconstructed) * d);
+
233  } else {
+
234  int list_no = key >> 32;
+
235  int offset = key & 0xffffffff;
+
236 
+
237  // Update label to the actual id
+
238  labels[ij] = ids[list_no][offset];
+
239 
+
240  reconstruct_from_offset (list_no, offset, reconstructed);
+
241  }
+
242  }
+
243  }
+
244 }
+
245 
+
246 void IndexIVF::reconstruct_from_offset (long list_no, long offset,
+
247  float* recons) const
+
248 {
+
249  FAISS_THROW_MSG ("reconstruct_from_offset not implemented");
+
250 }
+
251 
+
252 void IndexIVF::reset ()
+
253 {
+
254  ntotal = 0;
+
255  direct_map.clear();
+
256  for (size_t i = 0; i < ids.size(); i++) {
+
257  ids[i].clear();
+
258  codes[i].clear();
+
259  }
+
260 }
+
261 
+
262 
+
263 long IndexIVF::remove_ids (const IDSelector & sel)
+
264 {
+
265  FAISS_THROW_IF_NOT_MSG (!maintain_direct_map,
+
266  "direct map remove not implemented");
+
267  long nremove = 0;
+
268 #pragma omp parallel for reduction(+: nremove)
+
269  for (long i = 0; i < nlist; i++) {
+
270  std::vector<idx_t> & idsi = ids[i];
+
271  uint8_t * codesi = codes[i].data();
+
272 
+
273  long l = idsi.size(), j = 0;
+
274  while (j < l) {
+
275  if (sel.is_member (idsi[j])) {
+
276  l--;
+
277  idsi [j] = idsi [l];
+
278  memmove (codesi + j * code_size,
+
279  codesi + l * code_size, code_size);
+
280  } else {
+
281  j++;
+
282  }
+
283  }
+
284  if (l < idsi.size()) {
+
285  nremove += idsi.size() - l;
+
286  idsi.resize (l);
+
287  codes[i].resize (l * code_size);
+
288  }
+
289  }
+
290  ntotal -= nremove;
+
291  return nremove;
+
292 }
+
293 
+
294 
+
295 
+
296 
+
297 void IndexIVF::train (idx_t n, const float *x)
+
298 {
+
299  if (verbose)
+
300  printf ("Training level-1 quantizer\n");
+
301 
+
302  train_q1 (n, x, verbose, metric_type);
+
303 
+
304  if (verbose)
+
305  printf ("Training IVF residual\n");
+
306 
+
307  train_residual (n, x);
+
308  is_trained = true;
+
309 }
+
310 
+
311 void IndexIVF::train_residual(idx_t /*n*/, const float* /*x*/) {
+
312  if (verbose)
+
313  printf("IndexIVF: no residual training\n");
+
314  // does nothing by default
+
315 }
+
316 
+
317 
+
318 
+
319 double IndexIVF::imbalance_factor () const
+
320 {
+
321  std::vector<int> hist (nlist);
+
322  for (int i = 0; i < nlist; i++) {
+
323  hist[i] = ids[i].size();
+
324  }
+
325  return faiss::imbalance_factor (nlist, hist.data());
+
326 }
+
327 
+
328 void IndexIVF::print_stats () const
+
329 {
+
330  std::vector<int> sizes(40);
+
331  for (int i = 0; i < nlist; i++) {
+
332  for (int j = 0; j < sizes.size(); j++) {
+
333  if ((ids[i].size() >> j) == 0) {
+
334  sizes[j]++;
+
335  break;
+
336  }
+
337  }
+
338  }
+
339  for (int i = 0; i < sizes.size(); i++) {
+
340  if (sizes[i]) {
+
341  printf ("list size in < %d: %d instances\n",
+
342  1 << i, sizes[i]);
+
343  }
+
344  }
+
345 
+
346 }
+
347 
+
348 void IndexIVF::merge_from (IndexIVF &other, idx_t add_id)
+
349 {
+
350  // minimal sanity checks
+
351  FAISS_THROW_IF_NOT (other.d == d);
+
352  FAISS_THROW_IF_NOT (other.nlist == nlist);
+
353  FAISS_THROW_IF_NOT_MSG ((!maintain_direct_map &&
+
354  !other.maintain_direct_map),
+
355  "direct map copy not implemented");
+
356  FAISS_THROW_IF_NOT_MSG (typeid (*this) == typeid (other),
+
357  "can only merge indexes of the same type");
+
358  for (long i = 0; i < nlist; i++) {
+
359  std::vector<idx_t> & src = other.ids[i];
+
360  std::vector<idx_t> & dest = ids[i];
+
361  for (long j = 0; j < src.size(); j++)
+
362  dest.push_back (src[j] + add_id);
+
363  src.clear();
+
364  codes[i].insert (codes[i].end(),
+
365  other.codes[i].begin(),
+
366  other.codes[i].end());
+
367  other.codes[i].clear();
+
368  }
+
369 
+
370  ntotal += other.ntotal;
+
371  other.ntotal = 0;
+
372 }
+
373 
+
374 
+
375 void IndexIVF::copy_subset_to (IndexIVF & other, int subset_type,
+
376  long a1, long a2) const
+
377 {
+
378  FAISS_THROW_IF_NOT (nlist == other.nlist);
+
379  FAISS_THROW_IF_NOT (!other.maintain_direct_map);
+
380  FAISS_THROW_IF_NOT_FMT (
+
381  subset_type == 0 || subset_type == 1 || subset_type == 2,
+
382  "subset type %d not implemented", subset_type);
+
383 
+
384  size_t accu_n = 0;
+
385  size_t accu_a1 = 0;
+
386  size_t accu_a2 = 0;
+
387 
+
388  for (long list_no = 0; list_no < nlist; list_no++) {
+
389  const std::vector<idx_t> & ids_in = ids[list_no];
+
390  std::vector<idx_t> & ids_out = other.ids[list_no];
+
391  const std::vector<uint8_t> & codes_in = codes[list_no];
+
392  std::vector<uint8_t> & codes_out = other.codes[list_no];
+
393  size_t n = ids_in.size();
+
394 
+
395  if (subset_type == 0) {
+
396  for (long i = 0; i < n; i++) {
+
397  idx_t id = ids_in[i];
+
398  if (a1 <= id && id < a2) {
+
399  ids_out.push_back (id);
+
400  codes_out.insert (codes_out.end(),
+
401  codes_in.begin() + i * code_size,
+
402  codes_in.begin() + (i + 1) * code_size);
+
403  other.ntotal++;
+
404  }
+
405  }
+
406  } else if (subset_type == 1) {
+
407  for (long i = 0; i < n; i++) {
+
408  idx_t id = ids_in[i];
+
409  if (id % a1 == a2) {
+
410  ids_out.push_back (id);
+
411  codes_out.insert (codes_out.end(),
+
412  codes_in.begin() + i * code_size,
+
413  codes_in.begin() + (i + 1) * code_size);
+
414  other.ntotal++;
+
415  }
+
416  }
+
417  } else if (subset_type == 2) {
+
418  // see what is allocated to a1 and to a2
+
419  size_t next_accu_n = accu_n + n;
+
420  size_t next_accu_a1 = next_accu_n * a1 / ntotal;
+
421  size_t i1 = next_accu_a1 - accu_a1;
+
422  size_t next_accu_a2 = next_accu_n * a2 / ntotal;
+
423  size_t i2 = next_accu_a2 - accu_a2;
+
424  ids_out.insert(ids_out.end(),
+
425  ids_in.begin() + i1,
+
426  ids_in.begin() + i2);
+
427  codes_out.insert (codes_out.end(),
+
428  codes_in.begin() + i1 * code_size,
+
429  codes_in.begin() + i2 * code_size);
+
430  other.ntotal += i2 - i1;
+
431  accu_a1 = next_accu_a1;
+
432  accu_a2 = next_accu_a2;
+
433  }
+
434  accu_n += n;
+
435  }
+
436  FAISS_ASSERT(accu_n == ntotal);
+
437 }
+
438 
+
439 
440 
-
441  for (size_t ik = 0; ik < ivf.nprobe; ik++) {
-
442  long key = keysi[ik]; /* select the list */
-
443  if (key < 0) {
-
444  // not enough centroids for multiprobe
-
445  continue;
-
446  }
-
447  FAISS_THROW_IF_NOT_FMT (
-
448  key < (long) ivf.nlist,
-
449  "Invalid key=%ld at ik=%ld nlist=%ld\n",
-
450  key, ik, ivf.nlist);
-
451 
-
452  nlistv++;
-
453  const size_t list_size = ivf.ids[key].size();
-
454  const float * list_vecs = (const float*)(ivf.codes[key].data());
-
455 
-
456  for (size_t j = 0; j < list_size; j++) {
-
457  const float * yj = list_vecs + d * j;
-
458  float ip = fvec_inner_product (xi, yj, d);
-
459  if (ip > simi[0]) {
-
460  minheap_pop (k, simi, idxi);
-
461  long id = store_pairs ? (key << 32 | j) : ivf.ids[key][j];
-
462  minheap_push (k, simi, idxi, ip, id);
-
463  }
-
464  }
-
465  ndis += list_size;
-
466  }
-
467  minheap_reorder (k, simi, idxi);
-
468  }
-
469  indexIVFFlat_stats.nq += nx;
-
470  indexIVFFlat_stats.nlist += nlistv;
-
471  indexIVFFlat_stats.ndis += ndis;
-
472 }
-
473 
-
474 
-
475 void search_knn_L2sqr (const IndexIVFFlat &ivf,
-
476  size_t nx,
-
477  const float * x,
-
478  const long * keys,
-
479  float_maxheap_array_t * res,
-
480  bool store_pairs)
-
481 {
-
482  const size_t k = res->k;
-
483  size_t nlistv = 0, ndis = 0;
-
484  size_t d = ivf.d;
-
485 #pragma omp parallel for reduction(+: nlistv, ndis)
-
486  for (size_t i = 0; i < nx; i++) {
-
487  const float * xi = x + i * d;
-
488  const long * keysi = keys + i * ivf.nprobe;
-
489  float * __restrict disi = res->get_val (i);
-
490  long * __restrict idxi = res->get_ids (i);
-
491  maxheap_heapify (k, disi, idxi);
-
492 
-
493  for (size_t ik = 0; ik < ivf.nprobe; ik++) {
-
494  long key = keysi[ik]; /* select the list */
-
495  if (key < 0) {
-
496  // not enough centroids for multiprobe
-
497  continue;
-
498  }
-
499  FAISS_THROW_IF_NOT_FMT (
-
500  key < (long) ivf.nlist,
-
501  "Invalid key=%ld at ik=%ld nlist=%ld\n",
-
502  key, ik, ivf.nlist);
-
503 
-
504  nlistv++;
-
505  const size_t list_size = ivf.ids[key].size();
-
506  const float * list_vecs = (const float*)(ivf.codes[key].data());
-
507 
-
508  for (size_t j = 0; j < list_size; j++) {
-
509  const float * yj = list_vecs + d * j;
-
510  float disij = fvec_L2sqr (xi, yj, d);
-
511  if (disij < disi[0]) {
-
512  maxheap_pop (k, disi, idxi);
-
513  long id = store_pairs ? (key << 32 | j) : ivf.ids[key][j];
-
514  maxheap_push (k, disi, idxi, disij, id);
-
515  }
-
516  }
-
517  ndis += list_size;
-
518  }
-
519  maxheap_reorder (k, disi, idxi);
-
520  }
-
521  indexIVFFlat_stats.nq += nx;
-
522  indexIVFFlat_stats.nlist += nlistv;
-
523  indexIVFFlat_stats.ndis += ndis;
-
524 }
-
525 
-
526 
-
527 } // anonymous namespace
-
528 
-
529 void IndexIVFFlat::search_preassigned (idx_t n, const float *x, idx_t k,
-
530  const idx_t *idx,
-
531  const float * /* coarse_dis */,
-
532  float *distances, idx_t *labels,
-
533  bool store_pairs) const
-
534 {
-
535  if (metric_type == METRIC_INNER_PRODUCT) {
-
536  float_minheap_array_t res = {
-
537  size_t(n), size_t(k), labels, distances};
-
538  search_knn_inner_product (*this, n, x, idx, &res, store_pairs);
-
539 
-
540  } else if (metric_type == METRIC_L2) {
-
541  float_maxheap_array_t res = {
-
542  size_t(n), size_t(k), labels, distances};
-
543  search_knn_L2sqr (*this, n, x, idx, &res, store_pairs);
-
544  }
-
545 }
-
546 
-
547 
-
548 void IndexIVFFlat::range_search (idx_t nx, const float *x, float radius,
-
549  RangeSearchResult *result) const
-
550 {
-
551  idx_t * keys = new idx_t [nx * nprobe];
-
552  ScopeDeleter<idx_t> del (keys);
-
553  quantizer->assign (nx, x, keys, nprobe);
+
441 IndexIVF::~IndexIVF()
+
442 {
+
443 }
+
444 
+
445 
+
446 
+
447 /*****************************************
+
448  * IndexIVFFlat implementation
+
449  ******************************************/
+
450 
+
451 IndexIVFFlat::IndexIVFFlat (Index * quantizer,
+
452  size_t d, size_t nlist, MetricType metric):
+
453  IndexIVF (quantizer, d, nlist, metric)
+
454 {
+
455  code_size = sizeof(float) * d;
+
456 }
+
457 
+
458 
+
459 
+
460 
+
461 
+
462 
+
463 void IndexIVFFlat::add_with_ids (idx_t n, const float * x, const long *xids)
+
464 {
+
465  add_core (n, x, xids, nullptr);
+
466 }
+
467 
+
468 void IndexIVFFlat::add_core (idx_t n, const float * x, const long *xids,
+
469  const long *precomputed_idx)
+
470 
+
471 {
+
472  FAISS_THROW_IF_NOT (is_trained);
+
473  FAISS_THROW_IF_NOT_MSG (!(maintain_direct_map && xids),
+
474  "cannot have direct map and add with ids");
+
475  const long * idx;
+
476  ScopeDeleter<long> del;
+
477 
+
478  if (precomputed_idx) {
+
479  idx = precomputed_idx;
+
480  } else {
+
481  long * idx0 = new long [n];
+
482  quantizer->assign (n, x, idx0);
+
483  idx = idx0;
+
484  del.set (idx);
+
485  }
+
486  long n_add = 0;
+
487  for (size_t i = 0; i < n; i++) {
+
488  long id = xids ? xids[i] : ntotal + i;
+
489  long list_no = idx [i];
+
490  if (list_no < 0)
+
491  continue;
+
492  assert (list_no < nlist);
+
493 
+
494  ids[list_no].push_back (id);
+
495  const float *xi = x + i * d;
+
496  /* store the vectors */
+
497  size_t ofs = codes[list_no].size();
+
498  codes[list_no].resize(ofs + code_size);
+
499  memcpy(codes[list_no].data() + ofs,
+
500  xi, code_size);
+
501 
+
502  if (maintain_direct_map)
+
503  direct_map.push_back (list_no << 32 | (ids[list_no].size() - 1));
+
504  n_add++;
+
505  }
+
506  if (verbose) {
+
507  printf("IndexIVFFlat::add_core: added %ld / %ld vectors\n",
+
508  n_add, n);
+
509  }
+
510  ntotal += n_add;
+
511 }
+
512 
+
513 void IndexIVFStats::reset()
+
514 {
+
515  memset ((void*)this, 0, sizeof (*this));
+
516 }
+
517 
+
518 
+
519 IndexIVFStats indexIVF_stats;
+
520 
+
521 namespace {
+
522 
+
523 void search_knn_inner_product (const IndexIVFFlat & ivf,
+
524  size_t nx,
+
525  const float * x,
+
526  const long * keys,
+
527  float_minheap_array_t * res,
+
528  bool store_pairs)
+
529 {
+
530 
+
531  const size_t k = res->k;
+
532  size_t nlistv = 0, ndis = 0;
+
533  size_t d = ivf.d;
+
534 
+
535 #pragma omp parallel for reduction(+: nlistv, ndis)
+
536  for (size_t i = 0; i < nx; i++) {
+
537  const float * xi = x + i * d;
+
538  const long * keysi = keys + i * ivf.nprobe;
+
539  float * __restrict simi = res->get_val (i);
+
540  long * __restrict idxi = res->get_ids (i);
+
541  minheap_heapify (k, simi, idxi);
+
542  size_t nscan = 0;
+
543 
+
544  for (size_t ik = 0; ik < ivf.nprobe; ik++) {
+
545  long key = keysi[ik]; /* select the list */
+
546  if (key < 0) {
+
547  // not enough centroids for multiprobe
+
548  continue;
+
549  }
+
550  FAISS_THROW_IF_NOT_FMT (
+
551  key < (long) ivf.nlist,
+
552  "Invalid key=%ld at ik=%ld nlist=%ld\n",
+
553  key, ik, ivf.nlist);
554 
-
555 #pragma omp parallel
-
556  {
-
557  RangeSearchPartialResult pres(result);
+
555  nlistv++;
+
556  const size_t list_size = ivf.ids[key].size();
+
557  const float * list_vecs = (const float*)(ivf.codes[key].data());
558 
-
559  for (size_t i = 0; i < nx; i++) {
-
560  const float * xi = x + i * d;
-
561  const long * keysi = keys + i * nprobe;
-
562 
-
563  RangeSearchPartialResult::QueryResult & qres =
-
564  pres.new_result (i);
-
565 
-
566  for (size_t ik = 0; ik < nprobe; ik++) {
-
567  long key = keysi[ik]; /* select the list */
-
568  if (key < 0 || key >= (long) nlist) {
-
569  fprintf (stderr, "Invalid key=%ld at ik=%ld nlist=%ld\n",
-
570  key, ik, nlist);
-
571  throw;
-
572  }
-
573 
-
574  const size_t list_size = ids[key].size();
-
575  const float * list_vecs = (const float *)(codes[key].data());
-
576 
-
577  for (size_t j = 0; j < list_size; j++) {
-
578  const float * yj = list_vecs + d * j;
-
579  if (metric_type == METRIC_L2) {
-
580  float disij = fvec_L2sqr (xi, yj, d);
-
581  if (disij < radius) {
-
582  qres.add (disij, ids[key][j]);
-
583  }
-
584  } else if (metric_type == METRIC_INNER_PRODUCT) {
-
585  float disij = fvec_inner_product(xi, yj, d);
-
586  if (disij > radius) {
-
587  qres.add (disij, ids[key][j]);
-
588  }
-
589  }
-
590  }
-
591  }
-
592  }
-
593 
-
594  pres.finalize ();
-
595  }
-
596 }
-
597 
-
598 void IndexIVFFlat::update_vectors (int n, idx_t *new_ids, const float *x)
-
599 {
-
600  FAISS_THROW_IF_NOT (maintain_direct_map);
-
601  FAISS_THROW_IF_NOT (is_trained);
-
602  std::vector<idx_t> assign (n);
-
603  quantizer->assign (n, x, assign.data());
-
604 
-
605  for (int i = 0; i < n; i++) {
-
606  idx_t id = new_ids[i];
-
607  FAISS_THROW_IF_NOT_MSG (0 <= id && id < ntotal,
-
608  "id to update out of range");
-
609  { // remove old one
-
610  long dm = direct_map[id];
-
611  long ofs = dm & 0xffffffff;
-
612  long il = dm >> 32;
-
613  size_t l = ids[il].size();
-
614  if (ofs != l - 1) {
-
615  long id2 = ids[il].back();
-
616  ids[il][ofs] = id2;
-
617  direct_map[id2] = (il << 32) | ofs;
-
618  float * vecs = (float*)codes[il].data();
-
619  memcpy (vecs + ofs * d,
-
620  vecs + (l - 1) * d,
-
621  d * sizeof(float));
-
622  }
-
623  ids[il].pop_back();
-
624  codes[il].resize((l - 1) * code_size);
-
625  }
-
626  { // insert new one
-
627  long il = assign[i];
-
628  size_t l = ids[il].size();
-
629  long dm = (il << 32) | l;
-
630  direct_map[id] = dm;
-
631  ids[il].push_back (id);
-
632  codes[il].resize((l + 1) * code_size);
-
633  float * vecs = (float*)codes[il].data();
-
634  memcpy (vecs + l * d,
-
635  x + i * d,
-
636  d * sizeof(float));
-
637  }
-
638  }
+
559  for (size_t j = 0; j < list_size; j++) {
+
560  const float * yj = list_vecs + d * j;
+
561  float ip = fvec_inner_product (xi, yj, d);
+
562  if (ip > simi[0]) {
+
563  minheap_pop (k, simi, idxi);
+
564  long id = store_pairs ? (key << 32 | j) : ivf.ids[key][j];
+
565  minheap_push (k, simi, idxi, ip, id);
+
566  }
+
567  }
+
568  nscan += list_size;
+
569  if (ivf.max_codes && nscan >= ivf.max_codes)
+
570  break;
+
571  }
+
572  ndis += nscan;
+
573  minheap_reorder (k, simi, idxi);
+
574  }
+
575  indexIVF_stats.nq += nx;
+
576  indexIVF_stats.nlist += nlistv;
+
577  indexIVF_stats.ndis += ndis;
+
578 }
+
579 
+
580 
+
581 void search_knn_L2sqr (const IndexIVFFlat &ivf,
+
582  size_t nx,
+
583  const float * x,
+
584  const long * keys,
+
585  float_maxheap_array_t * res,
+
586  bool store_pairs)
+
587 {
+
588  const size_t k = res->k;
+
589  size_t nlistv = 0, ndis = 0;
+
590  size_t d = ivf.d;
+
591 #pragma omp parallel for reduction(+: nlistv, ndis)
+
592  for (size_t i = 0; i < nx; i++) {
+
593  const float * xi = x + i * d;
+
594  const long * keysi = keys + i * ivf.nprobe;
+
595  float * __restrict disi = res->get_val (i);
+
596  long * __restrict idxi = res->get_ids (i);
+
597  maxheap_heapify (k, disi, idxi);
+
598 
+
599  size_t nscan = 0;
+
600 
+
601  for (size_t ik = 0; ik < ivf.nprobe; ik++) {
+
602  long key = keysi[ik]; /* select the list */
+
603  if (key < 0) {
+
604  // not enough centroids for multiprobe
+
605  continue;
+
606  }
+
607  FAISS_THROW_IF_NOT_FMT (
+
608  key < (long) ivf.nlist,
+
609  "Invalid key=%ld at ik=%ld nlist=%ld\n",
+
610  key, ik, ivf.nlist);
+
611 
+
612  nlistv++;
+
613  const size_t list_size = ivf.ids[key].size();
+
614  const float * list_vecs = (const float*)(ivf.codes[key].data());
+
615 
+
616  for (size_t j = 0; j < list_size; j++) {
+
617  const float * yj = list_vecs + d * j;
+
618  float disij = fvec_L2sqr (xi, yj, d);
+
619  if (disij < disi[0]) {
+
620  maxheap_pop (k, disi, idxi);
+
621  long id = store_pairs ? (key << 32 | j) : ivf.ids[key][j];
+
622  maxheap_push (k, disi, idxi, disij, id);
+
623  }
+
624  }
+
625  nscan += list_size;
+
626  if (ivf.max_codes && nscan >= ivf.max_codes)
+
627  break;
+
628  }
+
629  ndis += nscan;
+
630  maxheap_reorder (k, disi, idxi);
+
631  }
+
632  indexIVF_stats.nq += nx;
+
633  indexIVF_stats.nlist += nlistv;
+
634  indexIVF_stats.ndis += ndis;
+
635 }
+
636 
+
637 
+
638 } // anonymous namespace
639 
-
640 }
-
641 
-
642 
-
643 
-
644 
-
645 
-
646 void IndexIVFFlat::reconstruct (idx_t key, float * recons) const
-
647 {
-
648  FAISS_THROW_IF_NOT_MSG (direct_map.size() == ntotal,
-
649  "direct map is not initialized");
-
650  int list_no = direct_map[key] >> 32;
-
651  int ofs = direct_map[key] & 0xffffffff;
-
652  memcpy (recons, &codes[list_no][ofs * code_size], d * sizeof(recons[0]));
-
653 }
-
654 
-
655 
-
656 
+
640 void IndexIVFFlat::search_preassigned (idx_t n, const float *x, idx_t k,
+
641  const idx_t *idx,
+
642  const float * /* coarse_dis */,
+
643  float *distances, idx_t *labels,
+
644  bool store_pairs) const
+
645 {
+
646  if (metric_type == METRIC_INNER_PRODUCT) {
+
647  float_minheap_array_t res = {
+
648  size_t(n), size_t(k), labels, distances};
+
649  search_knn_inner_product (*this, n, x, idx, &res, store_pairs);
+
650 
+
651  } else if (metric_type == METRIC_L2) {
+
652  float_maxheap_array_t res = {
+
653  size_t(n), size_t(k), labels, distances};
+
654  search_knn_L2sqr (*this, n, x, idx, &res, store_pairs);
+
655  }
+
656 }
657 
-
658 } // namespace faiss
+
658 
+
659 void IndexIVFFlat::range_search (idx_t nx, const float *x, float radius,
+
660  RangeSearchResult *result) const
+
661 {
+
662  idx_t * keys = new idx_t [nx * nprobe];
+
663  ScopeDeleter<idx_t> del (keys);
+
664  quantizer->assign (nx, x, keys, nprobe);
+
665 
+
666 #pragma omp parallel
+
667  {
+
668  RangeSearchPartialResult pres(result);
+
669 
+
670  for (size_t i = 0; i < nx; i++) {
+
671  const float * xi = x + i * d;
+
672  const long * keysi = keys + i * nprobe;
+
673 
+
674  RangeSearchPartialResult::QueryResult & qres =
+
675  pres.new_result (i);
+
676 
+
677  for (size_t ik = 0; ik < nprobe; ik++) {
+
678  long key = keysi[ik]; /* select the list */
+
679  if (key < 0 || key >= (long) nlist) {
+
680  fprintf (stderr, "Invalid key=%ld at ik=%ld nlist=%ld\n",
+
681  key, ik, nlist);
+
682  throw;
+
683  }
+
684 
+
685  const size_t list_size = ids[key].size();
+
686  const float * list_vecs = (const float *)(codes[key].data());
+
687 
+
688  for (size_t j = 0; j < list_size; j++) {
+
689  const float * yj = list_vecs + d * j;
+
690  if (metric_type == METRIC_L2) {
+
691  float disij = fvec_L2sqr (xi, yj, d);
+
692  if (disij < radius) {
+
693  qres.add (disij, ids[key][j]);
+
694  }
+
695  } else if (metric_type == METRIC_INNER_PRODUCT) {
+
696  float disij = fvec_inner_product(xi, yj, d);
+
697  if (disij > radius) {
+
698  qres.add (disij, ids[key][j]);
+
699  }
+
700  }
+
701  }
+
702  }
+
703  }
+
704 
+
705  pres.finalize ();
+
706  }
+
707 }
+
708 
+
709 void IndexIVFFlat::update_vectors (int n, idx_t *new_ids, const float *x)
+
710 {
+
711  FAISS_THROW_IF_NOT (maintain_direct_map);
+
712  FAISS_THROW_IF_NOT (is_trained);
+
713  std::vector<idx_t> assign (n);
+
714  quantizer->assign (n, x, assign.data());
+
715 
+
716  for (int i = 0; i < n; i++) {
+
717  idx_t id = new_ids[i];
+
718  FAISS_THROW_IF_NOT_MSG (0 <= id && id < ntotal,
+
719  "id to update out of range");
+
720  { // remove old one
+
721  long dm = direct_map[id];
+
722  long ofs = dm & 0xffffffff;
+
723  long il = dm >> 32;
+
724  size_t l = ids[il].size();
+
725  if (ofs != l - 1) {
+
726  long id2 = ids[il].back();
+
727  ids[il][ofs] = id2;
+
728  direct_map[id2] = (il << 32) | ofs;
+
729  float * vecs = (float*)codes[il].data();
+
730  memcpy (vecs + ofs * d,
+
731  vecs + (l - 1) * d,
+
732  d * sizeof(float));
+
733  }
+
734  ids[il].pop_back();
+
735  codes[il].resize((l - 1) * code_size);
+
736  }
+
737  { // insert new one
+
738  long il = assign[i];
+
739  size_t l = ids[il].size();
+
740  long dm = (il << 32) | l;
+
741  direct_map[id] = dm;
+
742  ids[il].push_back (id);
+
743  codes[il].resize((l + 1) * code_size);
+
744  float * vecs = (float*)codes[il].data();
+
745  memcpy (vecs + l * d,
+
746  x + i * d,
+
747  d * sizeof(float));
+
748  }
+
749  }
+
750 
+
751 }
+
752 
+
753 void IndexIVFFlat::reconstruct_from_offset (long list_no, long offset,
+
754  float* recons) const
+
755 {
+
756  memcpy (recons, &codes[list_no][offset * code_size], d * sizeof(recons[0]));
+
757 }
+
758 
+
759 
+
760 } // namespace faiss
faiss::IndexIVF::search_preassigned
virtual void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const =0
- -
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:25
+ +
faiss::RangeSearchPartialResult::QueryResult
result structure for a single query
Definition: AuxIndexStructures.h:145
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils.cpp:574
-
faiss::IndexIVF::imbalance_factor
double imbalance_factor() const
1= perfectly balanced, &gt;1: imbalanced
Definition: IndexIVF.cpp:216
+
faiss::IndexIVF::imbalance_factor
double imbalance_factor() const
1= perfectly balanced, &gt;1: imbalanced
Definition: IndexIVF.cpp:319
+
faiss::IndexIVF::search_and_reconstruct
void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const override
Definition: IndexIVF.cpp:210
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
-
faiss::IndexIVF::copy_subset_to
virtual void copy_subset_to(IndexIVF &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:272
-
faiss::IndexIVF::quantizer_trains_alone
char quantizer_trains_alone
Definition: IndexIVF.h:56
-
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:47
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
-
faiss::IndexIVFFlat::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVF.cpp:646
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
+
faiss::IndexIVF::copy_subset_to
virtual void copy_subset_to(IndexIVF &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:375
+
faiss::IndexIVF::reconstruct_from_offset
virtual void reconstruct_from_offset(long list_no, long offset, float *recons) const
Definition: IndexIVF.cpp:246
+
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:78
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
-
faiss::IndexIVFFlat::range_search
void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override
Definition: IndexIVF.cpp:548
-
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:30
-
faiss::IndexIVF::train_residual
virtual void train_residual(idx_t n, const float *x)
Definition: IndexIVF.cpp:208
+
faiss::IndexIVFFlat::range_search
void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override
Definition: IndexIVF.cpp:659
+
faiss::IndexIVF::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVF.cpp:180
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
+
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:41
+
faiss::IndexIVF::train_residual
virtual void train_residual(idx_t n, const float *x)
Definition: IndexIVF.cpp:311
-
faiss::imbalance_factor
double imbalance_factor(int n, int k, const long *assign)
a balanced assignment has a IF of 1
Definition: utils.cpp:1625
-
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
+
faiss::imbalance_factor
double imbalance_factor(int n, int k, const long *assign)
a balanced assignment has a IF of 1
Definition: utils.cpp:1627
+
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
-
faiss::IndexIVF::train
void train(idx_t n, const float *x) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:157
-
faiss::IndexIVF::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:59
-
faiss::IndexIVFFlat::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
implemented for all IndexIVF* classes
Definition: IndexIVF.cpp:361
-
faiss::IndexIVF::clustering_index
Index * clustering_index
to override index used during clustering
Definition: IndexIVF.h:60
+
faiss::IndexIVF::train
void train(idx_t n, const float *x) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:297
+
faiss::IndexIVF::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVF.cpp:190
+
faiss::IndexIVFFlat::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
implemented for all IndexIVF* classes
Definition: IndexIVF.cpp:463
+
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
-
faiss::IndexIVF::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:57
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
+
faiss::Level1Quantizer::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:44
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
-
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:112
+
faiss::IndexIVFFlat::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVF.cpp:753
+
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:252
faiss::Clustering::centroids
std::vector< float > centroids
centroids (k * d)
Definition: Clustering.h:63
faiss::RangeSearchPartialResult::new_result
QueryResult & new_result(idx_t qno)
begin a new result
Definition: AuxIndexStructures.cpp:117
-
faiss::IndexIVFFlat::update_vectors
void update_vectors(int nv, idx_t *idx, const float *v)
Definition: IndexIVF.cpp:598
+
faiss::IndexIVFFlat::update_vectors
void update_vectors(int nv, idx_t *idx, const float *v)
Definition: IndexIVF.cpp:709
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
+
faiss::Level1Quantizer::clustering_index
Index * clustering_index
to override index used during clustering
Definition: IndexIVF.h:45
+
faiss::Level1Quantizer::train_q1
void train_q1(size_t n, const float *x, bool verbose, MetricType metric_type)
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:53
faiss::RangeSearchPartialResult
the entries in the buffers are split per query
Definition: AuxIndexStructures.h:139
-
faiss::IndexIVF::make_direct_map
void make_direct_map(bool new_maintain_direct_map=true)
Definition: IndexIVF.cpp:71
+
faiss::IndexIVF::make_direct_map
void make_direct_map(bool new_maintain_direct_map=true)
Definition: IndexIVF.cpp:139
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
-
faiss::IndexIVF::print_stats
void print_stats() const
display some stats about the inverted lists
Definition: IndexIVF.cpp:225
-
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
+
faiss::IndexIVF::print_stats
void print_stats() const
display some stats about the inverted lists
Definition: IndexIVF.cpp:328
-
faiss::IndexIVF::add
void add(idx_t n, const float *x) override
Quantizes x and calls add_with_key.
Definition: IndexIVF.cpp:66
+
faiss::IndexIVF::add
void add(idx_t n, const float *x) override
Quantizes x and calls add_with_key.
Definition: IndexIVF.cpp:134
faiss::Clustering::train
virtual void train(idx_t n, const float *x, faiss::Index &index)
Index is used during the assignment stage.
Definition: Clustering.cpp:67
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::IndexIVF::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVF.cpp:123
-
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:68
+
faiss::IndexIVF::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVF.cpp:263
+
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:87
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:29
-
faiss::IndexIVF::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVF.cpp:96
-
faiss::IndexIVFFlat::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVF.cpp:529
-
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:245
-
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:64
+
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:42
+
faiss::IndexIVF::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVF.cpp:164
+
faiss::IndexIVFFlat::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVF.cpp:640
+
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:348
+
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
+
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:83
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
-
faiss::IndexIVFFlat::add_core
virtual void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx)
same as add_with_ids, with precomputed coarse quantizer
Definition: IndexIVF.cpp:366
+
faiss::IndexIVFFlat::add_core
virtual void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx)
same as add_with_ids, with precomputed coarse quantizer
Definition: IndexIVF.cpp:468
diff --git a/docs/html/IndexIVF_8h_source.html b/docs/html/IndexIVF_8h_source.html index d17eaceff..8538565ff 100644 --- a/docs/html/IndexIVF_8h_source.html +++ b/docs/html/IndexIVF_8h_source.html @@ -108,249 +108,311 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
23 
24 namespace faiss {
25 
-
26 
-
27 
-
28 /** Index based on a inverted file (IVF)
-
29  *
-
30  * In the inverted file, the quantizer (an Index instance) provides a
-
31  * quantization index for each vector to be added. The quantization
-
32  * index maps to a list (aka inverted list or posting list), where the
-
33  * id of the vector is then stored.
-
34  *
-
35  * At search time, the vector to be searched is also quantized, and
-
36  * only the list corresponding to the quantization index is
-
37  * searched. This speeds up the search by making it
-
38  * non-exhaustive. This can be relaxed using multi-probe search: a few
-
39  * (nprobe) quantization indices are selected and several inverted
-
40  * lists are visited.
-
41  *
-
42  * Sub-classes implement a post-filtering of the index that refines
-
43  * the distance estimation from the query to databse vectors.
-
44  */
-
45 struct IndexIVF: Index {
-
46  size_t nlist; ///< number of possible key values
-
47  size_t nprobe; ///< number of probes at query time
-
48 
-
49  Index * quantizer; ///< quantizer that maps vectors to inverted lists
-
50 
-
51  /**
-
52  * = 0: use the quantizer as index in a kmeans training
-
53  * = 1: just pass on the training set to the train() of the quantizer
-
54  * = 2: kmeans training on a flat index + add the centroids to the quantizer
-
55  */
-
56  char quantizer_trains_alone;
-
57  bool own_fields; ///< whether object owns the quantizer
-
58 
-
59  ClusteringParameters cp; ///< to override default clustering params
-
60  Index *clustering_index; ///< to override index used during clustering
-
61 
-
62  std::vector < std::vector<long> > ids; ///< Inverted lists for indexes
-
63 
-
64  size_t code_size; ///< code size per vector in bytes
-
65  std::vector < std::vector<uint8_t> > codes; // binary codes, size nlist
-
66 
-
67  /// map for direct access to the elements. Enables reconstruct().
-
68  bool maintain_direct_map;
-
69  std::vector <long> direct_map;
-
70 
-
71  /** The Inverted file takes a quantizer (an Index) on input,
-
72  * which implements the function mapping a vector to a list
-
73  * identifier. The pointer is borrowed: the quantizer should not
-
74  * be deleted while the IndexIVF is in use.
-
75  */
-
76  IndexIVF (Index * quantizer, size_t d, size_t nlist,
-
77  MetricType metric = METRIC_INNER_PRODUCT);
-
78 
-
79  void reset() override;
+
26 
+
27 /** Encapsulates a quantizer object for the IndexIVF
+
28  *
+
29  * The class isolates the fields that are independent of the storage
+
30  * of the lists (especially training)
+
31  */
+
32 struct Level1Quantizer {
+
33  Index * quantizer; ///< quantizer that maps vectors to inverted lists
+
34  size_t nlist; ///< number of possible key values
+
35 
+
36  /**
+
37  * = 0: use the quantizer as index in a kmeans training
+
38  * = 1: just pass on the training set to the train() of the quantizer
+
39  * = 2: kmeans training on a flat index + add the centroids to the quantizer
+
40  */
+
41  char quantizer_trains_alone;
+
42  bool own_fields; ///< whether object owns the quantizer
+
43 
+
44  ClusteringParameters cp; ///< to override default clustering params
+
45  Index *clustering_index; ///< to override index used during clustering
+
46 
+
47  /// Trains the quantizer and calls train_residual to train sub-quantizers
+
48  void train_q1 (size_t n, const float *x, bool verbose,
+
49  MetricType metric_type);
+
50 
+
51  Level1Quantizer (Index * quantizer, size_t nlist);
+
52 
+
53  Level1Quantizer ();
+
54 
+
55  ~Level1Quantizer ();
+
56 
+
57 };
+
58 
+
59 
+
60 /** Index based on a inverted file (IVF)
+
61  *
+
62  * In the inverted file, the quantizer (an Index instance) provides a
+
63  * quantization index for each vector to be added. The quantization
+
64  * index maps to a list (aka inverted list or posting list), where the
+
65  * id of the vector is then stored.
+
66  *
+
67  * At search time, the vector to be searched is also quantized, and
+
68  * only the list corresponding to the quantization index is
+
69  * searched. This speeds up the search by making it
+
70  * non-exhaustive. This can be relaxed using multi-probe search: a few
+
71  * (nprobe) quantization indices are selected and several inverted
+
72  * lists are visited.
+
73  *
+
74  * Sub-classes implement a post-filtering of the index that refines
+
75  * the distance estimation from the query to databse vectors.
+
76  */
+
77 struct IndexIVF: Index, Level1Quantizer {
+
78  size_t nprobe; ///< number of probes at query time
+
79  size_t max_codes; ///< max nb of codes to visit to do a query
80 
-
81  /// Trains the quantizer and calls train_residual to train sub-quantizers
-
82  void train(idx_t n, const float* x) override;
-
83 
-
84  /// Quantizes x and calls add_with_key
-
85  void add(idx_t n, const float* x) override;
-
86 
-
87  /// Sub-classes that encode the residuals can train their encoders here
-
88  /// does nothing by default
-
89  virtual void train_residual (idx_t n, const float *x);
-
90 
-
91 
-
92  /** search a set of vectors, that are pre-quantized by the IVF
-
93  * quantizer. Fill in the corresponding heaps with the query
-
94  * results. search() calls this.
-
95  *
-
96  * @param n nb of vectors to query
-
97  * @param x query vectors, size nx * d
-
98  * @param assign coarse quantization indices, size nx * nprobe
-
99  * @param centroid_dis
-
100  * distances to coarse centroids, size nx * nprobe
-
101  * @param distance
-
102  * output distances, size n * k
-
103  * @param labels output labels, size n * k
-
104  * @param store_pairs store inv list index + inv list offset
-
105  * instead in upper/lower 32 bit of result,
-
106  * instead of ids (used for reranking).
-
107  */
-
108  virtual void search_preassigned (idx_t n, const float *x, idx_t k,
-
109  const idx_t *assign,
-
110  const float *centroid_dis,
-
111  float *distances, idx_t *labels,
-
112  bool store_pairs) const = 0;
-
113 
-
114  /** assign the vectors, then call search_preassign */
-
115  virtual void search (idx_t n, const float *x, idx_t k,
-
116  float *distances, idx_t *labels) const override;
-
117 
-
118 
-
119  /// Dataset manipulation functions
-
120 
-
121  long remove_ids(const IDSelector& sel) override;
-
122 
-
123  /** moves the entries from another dataset to self. On output,
-
124  * other is empty. add_id is added to all moved ids (for
-
125  * sequential ids, this would be this->ntotal */
-
126  virtual void merge_from (IndexIVF &other, idx_t add_id);
-
127 
-
128  /** copy a subset of the entries index to the other index
-
129  *
-
130  * if subset_type == 0: copies ids in [a1, a2)
-
131  * if subset_type == 1: copies ids if id % a1 == a2
-
132  * if subset_type == 2: copies inverted lists such that a1
-
133  * elements are left before and a2 elements are after
-
134  */
-
135  virtual void copy_subset_to (IndexIVF & other, int subset_type,
-
136  long a1, long a2) const;
-
137 
-
138  ~IndexIVF() override;
-
139 
-
140  size_t get_list_size (size_t list_no) const
-
141  { return ids[list_no].size(); }
-
142 
-
143  /** intialize a direct map
-
144  *
-
145  * @param new_maintain_direct_map if true, create a direct map,
-
146  * else clear it
+
81  std::vector < std::vector<long> > ids; ///< Inverted lists for indexes
+
82 
+
83  size_t code_size; ///< code size per vector in bytes
+
84  std::vector < std::vector<uint8_t> > codes; // binary codes, size nlist
+
85 
+
86  /// map for direct access to the elements. Enables reconstruct().
+
87  bool maintain_direct_map;
+
88  std::vector <long> direct_map;
+
89 
+
90  /** The Inverted file takes a quantizer (an Index) on input,
+
91  * which implements the function mapping a vector to a list
+
92  * identifier. The pointer is borrowed: the quantizer should not
+
93  * be deleted while the IndexIVF is in use.
+
94  */
+
95  IndexIVF (Index * quantizer, size_t d, size_t nlist,
+
96  MetricType metric = METRIC_L2);
+
97 
+
98  void reset() override;
+
99 
+
100  /// Trains the quantizer and calls train_residual to train sub-quantizers
+
101  void train(idx_t n, const float* x) override;
+
102 
+
103  /// Quantizes x and calls add_with_key
+
104  void add(idx_t n, const float* x) override;
+
105 
+
106  /// Sub-classes that encode the residuals can train their encoders here
+
107  /// does nothing by default
+
108  virtual void train_residual (idx_t n, const float *x);
+
109 
+
110 
+
111  /** search a set of vectors, that are pre-quantized by the IVF
+
112  * quantizer. Fill in the corresponding heaps with the query
+
113  * results. search() calls this.
+
114  *
+
115  * @param n nb of vectors to query
+
116  * @param x query vectors, size nx * d
+
117  * @param assign coarse quantization indices, size nx * nprobe
+
118  * @param centroid_dis
+
119  * distances to coarse centroids, size nx * nprobe
+
120  * @param distance
+
121  * output distances, size n * k
+
122  * @param labels output labels, size n * k
+
123  * @param store_pairs store inv list index + inv list offset
+
124  * instead in upper/lower 32 bit of result,
+
125  * instead of ids (used for reranking).
+
126  */
+
127  virtual void search_preassigned (idx_t n, const float *x, idx_t k,
+
128  const idx_t *assign,
+
129  const float *centroid_dis,
+
130  float *distances, idx_t *labels,
+
131  bool store_pairs) const = 0;
+
132 
+
133  /** assign the vectors, then call search_preassign */
+
134  virtual void search (idx_t n, const float *x, idx_t k,
+
135  float *distances, idx_t *labels) const override;
+
136 
+
137  void reconstruct (idx_t key, float* recons) const override;
+
138 
+
139  /** Reconstruct a subset of the indexed vectors.
+
140  *
+
141  * Overrides default implementation to bypass reconstruct() which requires
+
142  * direct_map to be maintained.
+
143  *
+
144  * @param i0 first vector to reconstruct
+
145  * @param ni nb of vectors to reconstruct
+
146  * @param recons output array of reconstructed vectors, size ni * d
147  */
-
148  void make_direct_map (bool new_maintain_direct_map=true);
+
148  void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;
149 
-
150  /// 1= perfectly balanced, >1: imbalanced
-
151  double imbalance_factor () const;
-
152 
-
153  /// display some stats about the inverted lists
-
154  void print_stats () const;
-
155 
-
156  IndexIVF ();
-
157 };
-
158 
-
159 
-
160 struct IndexIVFFlatStats {
-
161  size_t nq; // nb of queries run
-
162  size_t nlist; // nb of inverted lists scanned
-
163  size_t ndis; // nb of distancs computed
-
164  size_t npartial; // nb of bound computations (IndexIVFFlatIPBounds)
-
165 
-
166  IndexIVFFlatStats () {reset (); }
-
167  void reset ();
-
168 };
-
169 
-
170 // global var that collects them all
-
171 extern IndexIVFFlatStats indexIVFFlat_stats;
+
150  /** Similar to search, but also reconstructs the stored vectors (or an
+
151  * approximation in the case of lossy coding) for the search results.
+
152  *
+
153  * Overrides default implementation to avoid having to maintain direct_map
+
154  * and instead fetch the code offsets through the `store_pairs` flag in
+
155  * search_preassigned().
+
156  *
+
157  * @param recons reconstructed vectors size (n, k, d)
+
158  */
+
159  void search_and_reconstruct (idx_t n, const float *x, idx_t k,
+
160  float *distances, idx_t *labels,
+
161  float *recons) const override;
+
162 
+
163  /** Reconstruct a vector given the location in terms of (inv list index +
+
164  * inv list offset) instead of the id.
+
165  *
+
166  * Useful for reconstructing when the direct_map is not maintained and
+
167  * the inv list offset is computed by search_preassigned() with
+
168  * `store_pairs` set.
+
169  */
+
170  virtual void reconstruct_from_offset (long list_no, long offset,
+
171  float* recons) const;
172 
-
173 
-
174 
-
175 
-
176 
-
177 /** Inverted file with stored vectors. Here the inverted file
-
178  * pre-selects the vectors to be searched, but they are not otherwise
-
179  * encoded, the code array just contains the raw float entries.
-
180  */
-
181 struct IndexIVFFlat: IndexIVF {
-
182 
-
183  IndexIVFFlat (
-
184  Index * quantizer, size_t d, size_t nlist_,
-
185  MetricType = METRIC_INNER_PRODUCT);
-
186 
-
187  /// same as add_with_ids, with precomputed coarse quantizer
-
188  virtual void add_core (idx_t n, const float * x, const long *xids,
-
189  const long *precomputed_idx);
-
190 
-
191  /// implemented for all IndexIVF* classes
-
192  void add_with_ids(idx_t n, const float* x, const long* xids) override;
-
193 
-
194  void search_preassigned (idx_t n, const float *x, idx_t k,
-
195  const idx_t *assign,
-
196  const float *centroid_dis,
-
197  float *distances, idx_t *labels,
-
198  bool store_pairs) const override;
-
199 
-
200  void range_search(
-
201  idx_t n,
-
202  const float* x,
-
203  float radius,
-
204  RangeSearchResult* result) const override;
-
205 
-
206  /** Update a subset of vectors.
-
207  *
-
208  * The index must have a direct_map
-
209  *
-
210  * @param nv nb of vectors to update
-
211  * @param idx vector indices to update, size nv
-
212  * @param v vectors of new values, size nv*d
-
213  */
-
214  void update_vectors (int nv, idx_t *idx, const float *v);
-
215 
-
216  void reconstruct(idx_t key, float* recons) const override;
-
217 
-
218  IndexIVFFlat () {}
-
219 };
-
220 
-
221 
-
222 
-
223 } // namespace faiss
-
224 
-
225 
+
173 
+
174  /// Dataset manipulation functions
+
175 
+
176  long remove_ids(const IDSelector& sel) override;
+
177 
+
178  /** moves the entries from another dataset to self. On output,
+
179  * other is empty. add_id is added to all moved ids (for
+
180  * sequential ids, this would be this->ntotal */
+
181  virtual void merge_from (IndexIVF &other, idx_t add_id);
+
182 
+
183  /** copy a subset of the entries index to the other index
+
184  *
+
185  * if subset_type == 0: copies ids in [a1, a2)
+
186  * if subset_type == 1: copies ids if id % a1 == a2
+
187  * if subset_type == 2: copies inverted lists such that a1
+
188  * elements are left before and a2 elements are after
+
189  */
+
190  virtual void copy_subset_to (IndexIVF & other, int subset_type,
+
191  long a1, long a2) const;
+
192 
+
193  ~IndexIVF() override;
+
194 
+
195  size_t get_list_size (size_t list_no) const
+
196  { return ids[list_no].size(); }
+
197 
+
198  /** intialize a direct map
+
199  *
+
200  * @param new_maintain_direct_map if true, create a direct map,
+
201  * else clear it
+
202  */
+
203  void make_direct_map (bool new_maintain_direct_map=true);
+
204 
+
205  /// 1= perfectly balanced, >1: imbalanced
+
206  double imbalance_factor () const;
+
207 
+
208  /// display some stats about the inverted lists
+
209  void print_stats () const;
+
210 
+
211  IndexIVF ();
+
212 };
+
213 
+
214 
+
215 struct IndexIVFStats {
+
216  size_t nq; // nb of queries run
+
217  size_t nlist; // nb of inverted lists scanned
+
218  size_t ndis; // nb of distancs computed
+
219 
+
220  IndexIVFStats () {reset (); }
+
221  void reset ();
+
222 };
+
223 
+
224 // global var that collects them all
+
225 extern IndexIVFStats indexIVF_stats;
226 
227 
228 
-
229 #endif
+
229 
+
230 
+
231 /** Inverted file with stored vectors. Here the inverted file
+
232  * pre-selects the vectors to be searched, but they are not otherwise
+
233  * encoded, the code array just contains the raw float entries.
+
234  */
+
235 struct IndexIVFFlat: IndexIVF {
+
236 
+
237  IndexIVFFlat (
+
238  Index * quantizer, size_t d, size_t nlist_,
+
239  MetricType = METRIC_L2);
+
240 
+
241  /// same as add_with_ids, with precomputed coarse quantizer
+
242  virtual void add_core (idx_t n, const float * x, const long *xids,
+
243  const long *precomputed_idx);
+
244 
+
245  /// implemented for all IndexIVF* classes
+
246  void add_with_ids(idx_t n, const float* x, const long* xids) override;
+
247 
+
248  void search_preassigned (idx_t n, const float *x, idx_t k,
+
249  const idx_t *assign,
+
250  const float *centroid_dis,
+
251  float *distances, idx_t *labels,
+
252  bool store_pairs) const override;
+
253 
+
254  void range_search(
+
255  idx_t n,
+
256  const float* x,
+
257  float radius,
+
258  RangeSearchResult* result) const override;
+
259 
+
260  /** Update a subset of vectors.
+
261  *
+
262  * The index must have a direct_map
+
263  *
+
264  * @param nv nb of vectors to update
+
265  * @param idx vector indices to update, size nv
+
266  * @param v vectors of new values, size nv*d
+
267  */
+
268  void update_vectors (int nv, idx_t *idx, const float *v);
+
269 
+
270  void reconstruct_from_offset (long list_no, long offset,
+
271  float* recons) const override;
+
272 
+
273  IndexIVFFlat () {}
+
274 };
+
275 
+
276 
+
277 
+
278 } // namespace faiss
+
279 
+
280 
+
281 
+
282 
+
283 
+
284 #endif
faiss::IndexIVF::search_preassigned
virtual void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const =0
- -
faiss::IndexIVF::imbalance_factor
double imbalance_factor() const
1= perfectly balanced, &gt;1: imbalanced
Definition: IndexIVF.cpp:216
-
faiss::IndexIVF::copy_subset_to
virtual void copy_subset_to(IndexIVF &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:272
-
faiss::IndexIVF::quantizer_trains_alone
char quantizer_trains_alone
Definition: IndexIVF.h:56
-
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:47
-
faiss::IndexIVFFlat::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVF.cpp:646
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
+ + +
faiss::IndexIVF::imbalance_factor
double imbalance_factor() const
1= perfectly balanced, &gt;1: imbalanced
Definition: IndexIVF.cpp:319
+
faiss::IndexIVF::search_and_reconstruct
void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const override
Definition: IndexIVF.cpp:210
+
faiss::IndexIVF::copy_subset_to
virtual void copy_subset_to(IndexIVF &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:375
+
faiss::IndexIVF::reconstruct_from_offset
virtual void reconstruct_from_offset(long list_no, long offset, float *recons) const
Definition: IndexIVF.cpp:246
+
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:78
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
-
faiss::IndexIVFFlat::range_search
void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override
Definition: IndexIVF.cpp:548
-
faiss::IndexIVF::train_residual
virtual void train_residual(idx_t n, const float *x)
Definition: IndexIVF.cpp:208
- -
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
- + +
faiss::IndexIVFFlat::range_search
void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override
Definition: IndexIVF.cpp:659
+
faiss::IndexIVF::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVF.cpp:180
+
faiss::IndexIVF::train_residual
virtual void train_residual(idx_t n, const float *x)
Definition: IndexIVF.cpp:311
+ +
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
-
faiss::IndexIVF::train
void train(idx_t n, const float *x) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:157
-
faiss::IndexIVF::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:59
-
faiss::IndexIVFFlat::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
implemented for all IndexIVF* classes
Definition: IndexIVF.cpp:361
-
faiss::IndexIVF::clustering_index
Index * clustering_index
to override index used during clustering
Definition: IndexIVF.h:60
-
faiss::IndexIVF::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:57
+
faiss::IndexIVF::train
void train(idx_t n, const float *x) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:297
+
faiss::IndexIVF::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVF.cpp:190
+
faiss::IndexIVFFlat::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
implemented for all IndexIVF* classes
Definition: IndexIVF.cpp:463
+
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
-
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:112
-
faiss::IndexIVFFlat::update_vectors
void update_vectors(int nv, idx_t *idx, const float *v)
Definition: IndexIVF.cpp:598
+
faiss::Level1Quantizer::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:44
+
faiss::IndexIVFFlat::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVF.cpp:753
+
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:252
+
faiss::IndexIVFFlat::update_vectors
void update_vectors(int nv, idx_t *idx, const float *v)
Definition: IndexIVF.cpp:709
+
faiss::Level1Quantizer::clustering_index
Index * clustering_index
to override index used during clustering
Definition: IndexIVF.h:45
+
faiss::Level1Quantizer::train_q1
void train_q1(size_t n, const float *x, bool verbose, MetricType metric_type)
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:53
-
faiss::IndexIVF::make_direct_map
void make_direct_map(bool new_maintain_direct_map=true)
Definition: IndexIVF.cpp:71
+
faiss::IndexIVF::make_direct_map
void make_direct_map(bool new_maintain_direct_map=true)
Definition: IndexIVF.cpp:139
-
faiss::IndexIVF::print_stats
void print_stats() const
display some stats about the inverted lists
Definition: IndexIVF.cpp:225
-
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
+
faiss::IndexIVF::print_stats
void print_stats() const
display some stats about the inverted lists
Definition: IndexIVF.cpp:328
-
faiss::IndexIVF::add
void add(idx_t n, const float *x) override
Quantizes x and calls add_with_key.
Definition: IndexIVF.cpp:66
-
faiss::IndexIVF::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVF.cpp:123
-
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:68
-
faiss::IndexIVF::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVF.cpp:96
-
faiss::IndexIVFFlat::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVF.cpp:529
-
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:245
-
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:64
+
faiss::IndexIVF::add
void add(idx_t n, const float *x) override
Quantizes x and calls add_with_key.
Definition: IndexIVF.cpp:134
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
+
faiss::IndexIVF::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVF.cpp:263
+
faiss::IndexIVF::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVF.h:79
+
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:87
+
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:42
+
faiss::IndexIVF::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVF.cpp:164
+
faiss::IndexIVFFlat::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVF.cpp:640
+
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:348
+
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
+
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:83
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
-
faiss::IndexIVFFlat::add_core
virtual void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx)
same as add_with_ids, with precomputed coarse quantizer
Definition: IndexIVF.cpp:366
+
faiss::IndexIVFFlat::add_core
virtual void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx)
same as add_with_ids, with precomputed coarse quantizer
Definition: IndexIVF.cpp:468
diff --git a/docs/html/IndexLSH_8cpp_source.html b/docs/html/IndexLSH_8cpp_source.html index 11f1c2f5c..b7bfae862 100644 --- a/docs/html/IndexLSH_8cpp_source.html +++ b/docs/html/IndexLSH_8cpp_source.html @@ -268,7 +268,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::IndexLSH::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexLSH.cpp:128
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::LinearTransform::b
std::vector< float > b
bias vector, size d_out
Definition: VectorTransform.h:85
+
faiss::LinearTransform::b
std::vector< float > b
bias vector, size d_out
Definition: VectorTransform.h:88
faiss::IndexLSH::rrot
RandomRotationMatrix rrot
optional random rotation
Definition: IndexLSH.h:32
faiss::IndexLSH::transfer_thresholds
void transfer_thresholds(LinearTransform *vt)
Definition: IndexLSH.cpp:160
diff --git a/docs/html/IndexLSH_8h_source.html b/docs/html/IndexLSH_8h_source.html index 40de5e530..07582816b 100644 --- a/docs/html/IndexLSH_8h_source.html +++ b/docs/html/IndexLSH_8h_source.html @@ -167,7 +167,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
82 
83 
84 #endif
-
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:107
+
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:117
faiss::IndexLSH::bytes_per_vec
int bytes_per_vec
nb of 8-bits per encoded vector
Definition: IndexLSH.h:28
faiss::IndexLSH::thresholds
std::vector< float > thresholds
thresholds to compare with
Definition: IndexLSH.h:34
diff --git a/docs/html/IndexPQ_8cpp_source.html b/docs/html/IndexPQ_8cpp_source.html index 1a38943bf..ce329fe28 100644 --- a/docs/html/IndexPQ_8cpp_source.html +++ b/docs/html/IndexPQ_8cpp_source.html @@ -345,189 +345,189 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
260 void IndexPQ::search_core_polysemous (idx_t n, const float *x, idx_t k,
261  float *distances, idx_t *labels) const
262 {
-
263  FAISS_THROW_IF_NOT (pq.code_size % 8 == 0);
-
264  FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
-
265 
-
266  // PQ distance tables
-
267  float * dis_tables = new float [n * pq.ksub * pq.M];
-
268  ScopeDeleter<float> del (dis_tables);
-
269  pq.compute_distance_tables (n, x, dis_tables);
-
270 
-
271  // Hamming embedding queries
-
272  uint8_t * q_codes = new uint8_t [n * pq.code_size];
-
273  ScopeDeleter<uint8_t> del2 (q_codes);
-
274 
-
275  if (false) {
-
276  pq.compute_codes (x, q_codes, n);
-
277  } else {
-
278 #pragma omp parallel for
-
279  for (idx_t qi = 0; qi < n; qi++) {
-
280  pq.compute_code_from_distance_table
-
281  (dis_tables + qi * pq.M * pq.ksub,
-
282  q_codes + qi * pq.code_size);
-
283  }
-
284  }
-
285 
-
286  size_t n_pass = 0;
-
287 
-
288 #pragma omp parallel for reduction (+: n_pass)
-
289  for (idx_t qi = 0; qi < n; qi++) {
-
290  const uint8_t * q_code = q_codes + qi * pq.code_size;
-
291 
-
292  const float * dis_table_qi = dis_tables + qi * pq.M * pq.ksub;
-
293 
-
294  long * heap_ids = labels + qi * k;
-
295  float *heap_dis = distances + qi * k;
-
296  maxheap_heapify (k, heap_dis, heap_ids);
-
297 
-
298  if (search_type == ST_polysemous) {
-
299 
-
300  switch (pq.code_size) {
-
301  case 4:
-
302  n_pass += polysemous_inner_loop<HammingComputer4>
-
303  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
304  break;
-
305  case 8:
-
306  n_pass += polysemous_inner_loop<HammingComputer8>
-
307  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
308  break;
-
309  case 16:
-
310  n_pass += polysemous_inner_loop<HammingComputer16>
-
311  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
312  break;
-
313  case 32:
-
314  n_pass += polysemous_inner_loop<HammingComputer32>
-
315  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
316  break;
-
317  case 20:
-
318  n_pass += polysemous_inner_loop<HammingComputer20>
-
319  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
320  break;
-
321  default:
-
322  if (pq.code_size % 8 == 0)
-
323  n_pass += polysemous_inner_loop<HammingComputerM8>
-
324  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
325  else
-
326  n_pass += polysemous_inner_loop<HammingComputerM4>
-
327  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
328  break;
-
329  }
-
330  } else {
-
331  switch (pq.code_size) {
-
332  case 8:
-
333  n_pass += polysemous_inner_loop<GenHammingComputer8>
-
334  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
-
335  break;
-
336  case 16:
-
337  n_pass += polysemous_inner_loop<GenHammingComputer16>
+
263  FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
+
264 
+
265  // PQ distance tables
+
266  float * dis_tables = new float [n * pq.ksub * pq.M];
+
267  ScopeDeleter<float> del (dis_tables);
+
268  pq.compute_distance_tables (n, x, dis_tables);
+
269 
+
270  // Hamming embedding queries
+
271  uint8_t * q_codes = new uint8_t [n * pq.code_size];
+
272  ScopeDeleter<uint8_t> del2 (q_codes);
+
273 
+
274  if (false) {
+
275  pq.compute_codes (x, q_codes, n);
+
276  } else {
+
277 #pragma omp parallel for
+
278  for (idx_t qi = 0; qi < n; qi++) {
+
279  pq.compute_code_from_distance_table
+
280  (dis_tables + qi * pq.M * pq.ksub,
+
281  q_codes + qi * pq.code_size);
+
282  }
+
283  }
+
284 
+
285  size_t n_pass = 0;
+
286 
+
287 #pragma omp parallel for reduction (+: n_pass)
+
288  for (idx_t qi = 0; qi < n; qi++) {
+
289  const uint8_t * q_code = q_codes + qi * pq.code_size;
+
290 
+
291  const float * dis_table_qi = dis_tables + qi * pq.M * pq.ksub;
+
292 
+
293  long * heap_ids = labels + qi * k;
+
294  float *heap_dis = distances + qi * k;
+
295  maxheap_heapify (k, heap_dis, heap_ids);
+
296 
+
297  if (search_type == ST_polysemous) {
+
298 
+
299  switch (pq.code_size) {
+
300  case 4:
+
301  n_pass += polysemous_inner_loop<HammingComputer4>
+
302  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
303  break;
+
304  case 8:
+
305  n_pass += polysemous_inner_loop<HammingComputer8>
+
306  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
307  break;
+
308  case 16:
+
309  n_pass += polysemous_inner_loop<HammingComputer16>
+
310  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
311  break;
+
312  case 32:
+
313  n_pass += polysemous_inner_loop<HammingComputer32>
+
314  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
315  break;
+
316  case 20:
+
317  n_pass += polysemous_inner_loop<HammingComputer20>
+
318  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
319  break;
+
320  default:
+
321  if (pq.code_size % 8 == 0) {
+
322  n_pass += polysemous_inner_loop<HammingComputerM8>
+
323  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
324  } else if (pq.code_size % 4 == 0) {
+
325  n_pass += polysemous_inner_loop<HammingComputerM4>
+
326  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
327  } else {
+
328  FAISS_THROW_FMT(
+
329  "code size %zd not supported for polysemous",
+
330  pq.code_size);
+
331  }
+
332  break;
+
333  }
+
334  } else {
+
335  switch (pq.code_size) {
+
336  case 8:
+
337  n_pass += polysemous_inner_loop<GenHammingComputer8>
338  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
339  break;
-
340  case 32:
-
341  n_pass += polysemous_inner_loop<GenHammingComputer32>
+
340  case 16:
+
341  n_pass += polysemous_inner_loop<GenHammingComputer16>
342  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
343  break;
-
344  default:
-
345  n_pass += polysemous_inner_loop<GenHammingComputerM8>
+
344  case 32:
+
345  n_pass += polysemous_inner_loop<GenHammingComputer32>
346  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
347  break;
-
348  }
-
349  }
-
350  maxheap_reorder (k, heap_dis, heap_ids);
-
351  }
-
352 
-
353  indexPQ_stats.nq += n;
-
354  indexPQ_stats.ncode += n * ntotal;
-
355  indexPQ_stats.n_hamming_pass += n_pass;
-
356 
-
357 
-
358 }
-
359 
-
360 
-
361 
+
348  default:
+
349  if (pq.code_size % 8 == 0) {
+
350  n_pass += polysemous_inner_loop<GenHammingComputerM8>
+
351  (*this, dis_table_qi, q_code, k, heap_dis, heap_ids);
+
352  } else {
+
353  FAISS_THROW_FMT(
+
354  "code size %zd not supported for polysemous",
+
355  pq.code_size);
+
356  }
+
357  break;
+
358  }
+
359  }
+
360  maxheap_reorder (k, heap_dis, heap_ids);
+
361  }
362 
-
363 /*****************************************
-
364  * Stats of IndexPQ codes
-
365  ******************************************/
+
363  indexPQ_stats.nq += n;
+
364  indexPQ_stats.ncode += n * ntotal;
+
365  indexPQ_stats.n_hamming_pass += n_pass;
366 
367 
-
368 
+
368 }
369 
-
370 void IndexPQ::hamming_distance_table (idx_t n, const float *x,
-
371  int32_t *dis) const
-
372 {
-
373  uint8_t * q_codes = new uint8_t [n * pq.code_size];
-
374  ScopeDeleter<uint8_t> del (q_codes);
-
375 
-
376  pq.compute_codes (x, q_codes, n);
+
370 
+
371 
+
372 
+
373 /*****************************************
+
374  * Stats of IndexPQ codes
+
375  ******************************************/
+
376 
377 
-
378  hammings (q_codes, codes.data(), n, ntotal, pq.code_size, dis);
-
379 }
-
380 
-
381 
-
382 void IndexPQ::hamming_distance_histogram (idx_t n, const float *x,
-
383  idx_t nb, const float *xb,
-
384  long *hist)
-
385 {
-
386  FAISS_THROW_IF_NOT (metric_type == METRIC_L2);
-
387  FAISS_THROW_IF_NOT (pq.code_size % 8 == 0);
-
388  FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
-
389 
-
390  // Hamming embedding queries
-
391  uint8_t * q_codes = new uint8_t [n * pq.code_size];
-
392  ScopeDeleter <uint8_t> del (q_codes);
-
393  pq.compute_codes (x, q_codes, n);
-
394 
-
395  uint8_t * b_codes ;
-
396  ScopeDeleter <uint8_t> del_b_codes;
-
397 
-
398  if (xb) {
-
399  b_codes = new uint8_t [nb * pq.code_size];
-
400  del_b_codes.set (b_codes);
-
401  pq.compute_codes (xb, b_codes, nb);
-
402  } else {
-
403  nb = ntotal;
-
404  b_codes = codes.data();
-
405  }
-
406  int nbits = pq.M * pq.nbits;
-
407  memset (hist, 0, sizeof(*hist) * (nbits + 1));
-
408  size_t bs = 256;
-
409 
-
410 #pragma omp parallel
-
411  {
-
412  std::vector<long> histi (nbits + 1);
-
413  hamdis_t *distances = new hamdis_t [nb * bs];
-
414  ScopeDeleter<hamdis_t> del (distances);
-
415 #pragma omp for
-
416  for (size_t q0 = 0; q0 < n; q0 += bs) {
-
417  // printf ("dis stats: %ld/%ld\n", q0, n);
-
418  size_t q1 = q0 + bs;
-
419  if (q1 > n) q1 = n;
-
420 
-
421  hammings (q_codes + q0 * pq.code_size, b_codes,
-
422  q1 - q0, nb,
-
423  pq.code_size, distances);
-
424 
-
425  for (size_t i = 0; i < nb * (q1 - q0); i++)
-
426  histi [distances [i]]++;
-
427  }
-
428 #pragma omp critical
-
429  {
-
430  for (int i = 0; i <= nbits; i++)
-
431  hist[i] += histi[i];
-
432  }
-
433  }
+
378 
+
379 
+
380 void IndexPQ::hamming_distance_table (idx_t n, const float *x,
+
381  int32_t *dis) const
+
382 {
+
383  uint8_t * q_codes = new uint8_t [n * pq.code_size];
+
384  ScopeDeleter<uint8_t> del (q_codes);
+
385 
+
386  pq.compute_codes (x, q_codes, n);
+
387 
+
388  hammings (q_codes, codes.data(), n, ntotal, pq.code_size, dis);
+
389 }
+
390 
+
391 
+
392 void IndexPQ::hamming_distance_histogram (idx_t n, const float *x,
+
393  idx_t nb, const float *xb,
+
394  long *hist)
+
395 {
+
396  FAISS_THROW_IF_NOT (metric_type == METRIC_L2);
+
397  FAISS_THROW_IF_NOT (pq.code_size % 8 == 0);
+
398  FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
+
399 
+
400  // Hamming embedding queries
+
401  uint8_t * q_codes = new uint8_t [n * pq.code_size];
+
402  ScopeDeleter <uint8_t> del (q_codes);
+
403  pq.compute_codes (x, q_codes, n);
+
404 
+
405  uint8_t * b_codes ;
+
406  ScopeDeleter <uint8_t> del_b_codes;
+
407 
+
408  if (xb) {
+
409  b_codes = new uint8_t [nb * pq.code_size];
+
410  del_b_codes.set (b_codes);
+
411  pq.compute_codes (xb, b_codes, nb);
+
412  } else {
+
413  nb = ntotal;
+
414  b_codes = codes.data();
+
415  }
+
416  int nbits = pq.M * pq.nbits;
+
417  memset (hist, 0, sizeof(*hist) * (nbits + 1));
+
418  size_t bs = 256;
+
419 
+
420 #pragma omp parallel
+
421  {
+
422  std::vector<long> histi (nbits + 1);
+
423  hamdis_t *distances = new hamdis_t [nb * bs];
+
424  ScopeDeleter<hamdis_t> del (distances);
+
425 #pragma omp for
+
426  for (size_t q0 = 0; q0 < n; q0 += bs) {
+
427  // printf ("dis stats: %ld/%ld\n", q0, n);
+
428  size_t q1 = q0 + bs;
+
429  if (q1 > n) q1 = n;
+
430 
+
431  hammings (q_codes + q0 * pq.code_size, b_codes,
+
432  q1 - q0, nb,
+
433  pq.code_size, distances);
434 
-
435 }
-
436 
-
437 
-
438 
-
439 
-
440 
-
441 
-
442 
-
443 
+
435  for (size_t i = 0; i < nb * (q1 - q0); i++)
+
436  histi [distances [i]]++;
+
437  }
+
438 #pragma omp critical
+
439  {
+
440  for (int i = 0; i <= nbits; i++)
+
441  hist[i] += histi[i];
+
442  }
+
443  }
444 
-
445 
+
445 }
446 
447 
448 
@@ -538,451 +538,462 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
453 
454 
455 
-
456 /*****************************************
-
457  * MultiIndexQuantizer
-
458  ******************************************/
+
456 
+
457 
+
458 
459 
460 
461 
-
462 template <typename T>
-
463 struct ArgSort {
-
464  const T * x;
-
465  bool operator() (size_t i, size_t j) {
-
466  return x[i] < x[j];
-
467  }
-
468 };
+
462 
+
463 
+
464 
+
465 
+
466 /*****************************************
+
467  * MultiIndexQuantizer
+
468  ******************************************/
469 
-
470 
-
471 /** Array that maintains a permutation of its elements so that the
-
472  * array's elements are sorted
-
473  */
-
474 template <typename T>
-
475 struct SortedArray {
-
476  const T * x;
-
477  int N;
-
478  std::vector<int> perm;
+
470 
+
471 
+
472 template <typename T>
+
473 struct ArgSort {
+
474  const T * x;
+
475  bool operator() (size_t i, size_t j) {
+
476  return x[i] < x[j];
+
477  }
+
478 };
479 
-
480  explicit SortedArray (int N) {
-
481  this->N = N;
-
482  perm.resize (N);
-
483  }
-
484 
-
485  void init (const T*x) {
-
486  this->x = x;
-
487  for (int n = 0; n < N; n++)
-
488  perm[n] = n;
-
489  ArgSort<T> cmp = {x };
-
490  std::sort (perm.begin(), perm.end(), cmp);
-
491  }
-
492 
-
493  // get smallest value
-
494  T get_0 () {
-
495  return x[perm[0]];
-
496  }
-
497 
-
498  // get delta between n-smallest and n-1 -smallest
-
499  T get_diff (int n) {
-
500  return x[perm[n]] - x[perm[n - 1]];
+
480 
+
481 /** Array that maintains a permutation of its elements so that the
+
482  * array's elements are sorted
+
483  */
+
484 template <typename T>
+
485 struct SortedArray {
+
486  const T * x;
+
487  int N;
+
488  std::vector<int> perm;
+
489 
+
490  explicit SortedArray (int N) {
+
491  this->N = N;
+
492  perm.resize (N);
+
493  }
+
494 
+
495  void init (const T*x) {
+
496  this->x = x;
+
497  for (int n = 0; n < N; n++)
+
498  perm[n] = n;
+
499  ArgSort<T> cmp = {x };
+
500  std::sort (perm.begin(), perm.end(), cmp);
501  }
502 
-
503  // remap orders counted from smallest to indices in array
-
504  int get_ord (int n) {
-
505  return perm[n];
+
503  // get smallest value
+
504  T get_0 () {
+
505  return x[perm[0]];
506  }
-
507 };
-
508 
-
509 
-
510 
-
511 /** Array has n values. Sort the k first ones and copy the other ones
-
512  * into elements k..n-1
-
513  */
-
514 template <class C>
-
515 void partial_sort (int k, int n,
-
516  const typename C::T * vals, typename C::TI * perm) {
-
517  // insert first k elts in heap
-
518  for (int i = 1; i < k; i++) {
-
519  indirect_heap_push<C> (i + 1, vals, perm, perm[i]);
-
520  }
-
521 
-
522  // insert next n - k elts in heap
-
523  for (int i = k; i < n; i++) {
-
524  typename C::TI id = perm[i];
-
525  typename C::TI top = perm[0];
-
526 
-
527  if (C::cmp(vals[top], vals[id])) {
-
528  indirect_heap_pop<C> (k, vals, perm);
-
529  indirect_heap_push<C> (k, vals, perm, id);
-
530  perm[i] = top;
-
531  } else {
-
532  // nothing, elt at i is good where it is.
-
533  }
-
534  }
-
535 
-
536  // order the k first elements in heap
-
537  for (int i = k - 1; i > 0; i--) {
-
538  typename C::TI top = perm[0];
-
539  indirect_heap_pop<C> (i + 1, vals, perm);
-
540  perm[i] = top;
-
541  }
-
542 }
-
543 
-
544 /** same as SortedArray, but only the k first elements are sorted */
-
545 template <typename T>
-
546 struct SemiSortedArray {
-
547  const T * x;
-
548  int N;
-
549 
-
550  // type of the heap: CMax = sort ascending
-
551  typedef CMax<T, int> HC;
-
552  std::vector<int> perm;
-
553 
-
554  int k; // k elements are sorted
-
555 
-
556  int initial_k, k_factor;
-
557 
-
558  explicit SemiSortedArray (int N) {
-
559  this->N = N;
-
560  perm.resize (N);
-
561  perm.resize (N);
-
562  initial_k = 3;
-
563  k_factor = 4;
-
564  }
+
507 
+
508  // get delta between n-smallest and n-1 -smallest
+
509  T get_diff (int n) {
+
510  return x[perm[n]] - x[perm[n - 1]];
+
511  }
+
512 
+
513  // remap orders counted from smallest to indices in array
+
514  int get_ord (int n) {
+
515  return perm[n];
+
516  }
+
517 };
+
518 
+
519 
+
520 
+
521 /** Array has n values. Sort the k first ones and copy the other ones
+
522  * into elements k..n-1
+
523  */
+
524 template <class C>
+
525 void partial_sort (int k, int n,
+
526  const typename C::T * vals, typename C::TI * perm) {
+
527  // insert first k elts in heap
+
528  for (int i = 1; i < k; i++) {
+
529  indirect_heap_push<C> (i + 1, vals, perm, perm[i]);
+
530  }
+
531 
+
532  // insert next n - k elts in heap
+
533  for (int i = k; i < n; i++) {
+
534  typename C::TI id = perm[i];
+
535  typename C::TI top = perm[0];
+
536 
+
537  if (C::cmp(vals[top], vals[id])) {
+
538  indirect_heap_pop<C> (k, vals, perm);
+
539  indirect_heap_push<C> (k, vals, perm, id);
+
540  perm[i] = top;
+
541  } else {
+
542  // nothing, elt at i is good where it is.
+
543  }
+
544  }
+
545 
+
546  // order the k first elements in heap
+
547  for (int i = k - 1; i > 0; i--) {
+
548  typename C::TI top = perm[0];
+
549  indirect_heap_pop<C> (i + 1, vals, perm);
+
550  perm[i] = top;
+
551  }
+
552 }
+
553 
+
554 /** same as SortedArray, but only the k first elements are sorted */
+
555 template <typename T>
+
556 struct SemiSortedArray {
+
557  const T * x;
+
558  int N;
+
559 
+
560  // type of the heap: CMax = sort ascending
+
561  typedef CMax<T, int> HC;
+
562  std::vector<int> perm;
+
563 
+
564  int k; // k elements are sorted
565 
-
566  void init (const T*x) {
-
567  this->x = x;
-
568  for (int n = 0; n < N; n++)
-
569  perm[n] = n;
-
570  k = 0;
-
571  grow (initial_k);
-
572  }
-
573 
-
574  /// grow the sorted part of the array to size next_k
-
575  void grow (int next_k) {
-
576  if (next_k < N) {
-
577  partial_sort<HC> (next_k - k, N - k, x, &perm[k]);
-
578  k = next_k;
-
579  } else { // full sort of remainder of array
-
580  ArgSort<T> cmp = {x };
-
581  std::sort (perm.begin() + k, perm.end(), cmp);
-
582  k = N;
-
583  }
-
584  }
-
585 
-
586  // get smallest value
-
587  T get_0 () {
-
588  return x[perm[0]];
-
589  }
-
590 
-
591  // get delta between n-smallest and n-1 -smallest
-
592  T get_diff (int n) {
-
593  if (n >= k) {
-
594  // want to keep powers of 2 - 1
-
595  int next_k = (k + 1) * k_factor - 1;
-
596  grow (next_k);
-
597  }
-
598  return x[perm[n]] - x[perm[n - 1]];
+
566  int initial_k, k_factor;
+
567 
+
568  explicit SemiSortedArray (int N) {
+
569  this->N = N;
+
570  perm.resize (N);
+
571  perm.resize (N);
+
572  initial_k = 3;
+
573  k_factor = 4;
+
574  }
+
575 
+
576  void init (const T*x) {
+
577  this->x = x;
+
578  for (int n = 0; n < N; n++)
+
579  perm[n] = n;
+
580  k = 0;
+
581  grow (initial_k);
+
582  }
+
583 
+
584  /// grow the sorted part of the array to size next_k
+
585  void grow (int next_k) {
+
586  if (next_k < N) {
+
587  partial_sort<HC> (next_k - k, N - k, x, &perm[k]);
+
588  k = next_k;
+
589  } else { // full sort of remainder of array
+
590  ArgSort<T> cmp = {x };
+
591  std::sort (perm.begin() + k, perm.end(), cmp);
+
592  k = N;
+
593  }
+
594  }
+
595 
+
596  // get smallest value
+
597  T get_0 () {
+
598  return x[perm[0]];
599  }
600 
-
601  // remap orders counted from smallest to indices in array
-
602  int get_ord (int n) {
-
603  assert (n < k);
-
604  return perm[n];
-
605  }
-
606 };
-
607 
-
608 
-
609 
-
610 /*****************************************
-
611  * Find the k smallest sums of M terms, where each term is taken in a
-
612  * table x of n values.
-
613  *
-
614  * A combination of terms is encoded as a scalar 0 <= t < n^M. The
-
615  * combination t0 ... t(M-1) that correspond to the sum
-
616  *
-
617  * sum = x[0, t0] + x[1, t1] + .... + x[M-1, t(M-1)]
-
618  *
-
619  * is encoded as
-
620  *
-
621  * t = t0 + t1 * n + t2 * n^2 + ... + t(M-1) * n^(M-1)
-
622  *
-
623  * MinSumK is an object rather than a function, so that storage can be
-
624  * re-used over several computations with the same sizes. use_seen is
-
625  * good when there may be ties in the x array and it is a concern if
-
626  * occasionally several t's are returned.
-
627  *
-
628  * @param x size M * n, values to add up
-
629  * @parms k nb of results to retrieve
-
630  * @param M nb of terms
-
631  * @param n nb of distinct values
-
632  * @param sums output, size k, sorted
-
633  * @prarm terms output, size k, with encoding as above
-
634  *
-
635  ******************************************/
-
636 template <typename T, class SSA, bool use_seen>
-
637 struct MinSumK {
-
638  int K; ///< nb of sums to return
-
639  int M; ///< nb of elements to sum up
-
640  int N; ///< nb of possible elements for each of the M terms
-
641 
-
642  /** the heap.
-
643  * We use a heap to maintain a queue of sums, with the associated
-
644  * terms involved in the sum.
-
645  */
-
646  typedef CMin<T, long> HC;
-
647  size_t heap_capacity, heap_size;
-
648  T *bh_val;
-
649  long *bh_ids;
-
650 
-
651  std::vector <SSA> ssx;
-
652  std::vector <long> weights;
-
653 
-
654  // all results get pushed several times. When there are ties, they
-
655  // are popped interleaved with others, so it is not easy to
-
656  // identify them. Therefore, this bit array just marks elements
-
657  // that were seen before.
-
658  std::vector <uint8_t> seen;
-
659 
-
660  MinSumK (int K, int M, int N): K(K), M(M), N(N) {
-
661  heap_capacity = K * M;
-
662  // we'll do k steps, each step pushes at most M vals
-
663  bh_val = new T[heap_capacity];
-
664  bh_ids = new long[heap_capacity];
-
665 
-
666  weights.push_back (1);
-
667  for (int m = 1; m < M; m++)
-
668  weights.push_back(weights[m - 1] * N);
+
601  // get delta between n-smallest and n-1 -smallest
+
602  T get_diff (int n) {
+
603  if (n >= k) {
+
604  // want to keep powers of 2 - 1
+
605  int next_k = (k + 1) * k_factor - 1;
+
606  grow (next_k);
+
607  }
+
608  return x[perm[n]] - x[perm[n - 1]];
+
609  }
+
610 
+
611  // remap orders counted from smallest to indices in array
+
612  int get_ord (int n) {
+
613  assert (n < k);
+
614  return perm[n];
+
615  }
+
616 };
+
617 
+
618 
+
619 
+
620 /*****************************************
+
621  * Find the k smallest sums of M terms, where each term is taken in a
+
622  * table x of n values.
+
623  *
+
624  * A combination of terms is encoded as a scalar 0 <= t < n^M. The
+
625  * combination t0 ... t(M-1) that correspond to the sum
+
626  *
+
627  * sum = x[0, t0] + x[1, t1] + .... + x[M-1, t(M-1)]
+
628  *
+
629  * is encoded as
+
630  *
+
631  * t = t0 + t1 * n + t2 * n^2 + ... + t(M-1) * n^(M-1)
+
632  *
+
633  * MinSumK is an object rather than a function, so that storage can be
+
634  * re-used over several computations with the same sizes. use_seen is
+
635  * good when there may be ties in the x array and it is a concern if
+
636  * occasionally several t's are returned.
+
637  *
+
638  * @param x size M * n, values to add up
+
639  * @parms k nb of results to retrieve
+
640  * @param M nb of terms
+
641  * @param n nb of distinct values
+
642  * @param sums output, size k, sorted
+
643  * @prarm terms output, size k, with encoding as above
+
644  *
+
645  ******************************************/
+
646 template <typename T, class SSA, bool use_seen>
+
647 struct MinSumK {
+
648  int K; ///< nb of sums to return
+
649  int M; ///< nb of elements to sum up
+
650  int N; ///< nb of possible elements for each of the M terms
+
651 
+
652  /** the heap.
+
653  * We use a heap to maintain a queue of sums, with the associated
+
654  * terms involved in the sum.
+
655  */
+
656  typedef CMin<T, long> HC;
+
657  size_t heap_capacity, heap_size;
+
658  T *bh_val;
+
659  long *bh_ids;
+
660 
+
661  std::vector <SSA> ssx;
+
662  std::vector <long> weights;
+
663 
+
664  // all results get pushed several times. When there are ties, they
+
665  // are popped interleaved with others, so it is not easy to
+
666  // identify them. Therefore, this bit array just marks elements
+
667  // that were seen before.
+
668  std::vector <uint8_t> seen;
669 
-
670  if (use_seen) {
-
671  long n_ids = weights.back() * N;
-
672  seen.resize ((n_ids + 7) / 8);
-
673  }
-
674 
-
675  for (int m = 0; m < M; m++)
-
676  ssx.push_back (SSA(N));
-
677 
-
678  }
+
670  MinSumK (int K, int M, int N): K(K), M(M), N(N) {
+
671  heap_capacity = K * M;
+
672  // we'll do k steps, each step pushes at most M vals
+
673  bh_val = new T[heap_capacity];
+
674  bh_ids = new long[heap_capacity];
+
675 
+
676  weights.push_back (1);
+
677  for (int m = 1; m < M; m++)
+
678  weights.push_back(weights[m - 1] * N);
679 
-
680  bool is_seen (long i) {
-
681  return (seen[i >> 3] >> (i & 7)) & 1;
-
682  }
-
683 
-
684  void mark_seen (long i) {
-
685  if (use_seen)
-
686  seen [i >> 3] |= 1 << (i & 7);
-
687  }
-
688 
-
689  void run (const T *x, T * sums, long * terms) {
-
690  heap_size = 0;
-
691 
-
692  for (int m = 0; m < M; m++)
-
693  ssx[m].init(x + N * m);
-
694 
-
695  { // intial result: take min for all elements
-
696  T sum = 0;
-
697  terms[0] = 0;
-
698  mark_seen (0);
-
699  for (int m = 0; m < M; m++) {
-
700  sum += ssx[m].get_0();
-
701  }
-
702  sums[0] = sum;
-
703  for (int m = 0; m < M; m++) {
-
704  heap_push<HC> (++heap_size, bh_val, bh_ids,
-
705  sum + ssx[m].get_diff(1),
-
706  weights[m]);
-
707  }
-
708  }
-
709 
-
710  for (int k = 1; k < K; k++) {
-
711  // pop smallest value from heap
-
712  if (use_seen) {// skip already seen elements
-
713  while (is_seen (bh_ids[0])) {
-
714  assert (heap_size > 0);
-
715  heap_pop<HC> (heap_size--, bh_val, bh_ids);
-
716  }
+
680  if (use_seen) {
+
681  long n_ids = weights.back() * N;
+
682  seen.resize ((n_ids + 7) / 8);
+
683  }
+
684 
+
685  for (int m = 0; m < M; m++)
+
686  ssx.push_back (SSA(N));
+
687 
+
688  }
+
689 
+
690  bool is_seen (long i) {
+
691  return (seen[i >> 3] >> (i & 7)) & 1;
+
692  }
+
693 
+
694  void mark_seen (long i) {
+
695  if (use_seen)
+
696  seen [i >> 3] |= 1 << (i & 7);
+
697  }
+
698 
+
699  void run (const T *x, T * sums, long * terms) {
+
700  heap_size = 0;
+
701 
+
702  for (int m = 0; m < M; m++)
+
703  ssx[m].init(x + N * m);
+
704 
+
705  { // intial result: take min for all elements
+
706  T sum = 0;
+
707  terms[0] = 0;
+
708  mark_seen (0);
+
709  for (int m = 0; m < M; m++) {
+
710  sum += ssx[m].get_0();
+
711  }
+
712  sums[0] = sum;
+
713  for (int m = 0; m < M; m++) {
+
714  heap_push<HC> (++heap_size, bh_val, bh_ids,
+
715  sum + ssx[m].get_diff(1),
+
716  weights[m]);
717  }
-
718  assert (heap_size > 0);
+
718  }
719 
-
720  T sum = sums[k] = bh_val[0];
-
721  long ti = terms[k] = bh_ids[0];
-
722 
-
723  if (use_seen) {
-
724  mark_seen (ti);
-
725  heap_pop<HC> (heap_size--, bh_val, bh_ids);
-
726  } else {
-
727  do {
-
728  heap_pop<HC> (heap_size--, bh_val, bh_ids);
-
729  } while (heap_size > 0 && bh_ids[0] == ti);
-
730  }
-
731 
-
732  // enqueue followers
-
733  long ii = ti;
-
734  for (int m = 0; m < M; m++) {
-
735  long n = ii % N;
-
736  ii /= N;
-
737  if (n + 1 >= N) continue;
-
738 
-
739  enqueue_follower (ti, m, n, sum);
+
720  for (int k = 1; k < K; k++) {
+
721  // pop smallest value from heap
+
722  if (use_seen) {// skip already seen elements
+
723  while (is_seen (bh_ids[0])) {
+
724  assert (heap_size > 0);
+
725  heap_pop<HC> (heap_size--, bh_val, bh_ids);
+
726  }
+
727  }
+
728  assert (heap_size > 0);
+
729 
+
730  T sum = sums[k] = bh_val[0];
+
731  long ti = terms[k] = bh_ids[0];
+
732 
+
733  if (use_seen) {
+
734  mark_seen (ti);
+
735  heap_pop<HC> (heap_size--, bh_val, bh_ids);
+
736  } else {
+
737  do {
+
738  heap_pop<HC> (heap_size--, bh_val, bh_ids);
+
739  } while (heap_size > 0 && bh_ids[0] == ti);
740  }
-
741  }
-
742 
-
743  /*
-
744  for (int k = 0; k < K; k++)
-
745  for (int l = k + 1; l < K; l++)
-
746  assert (terms[k] != terms[l]);
-
747  */
+
741 
+
742  // enqueue followers
+
743  long ii = ti;
+
744  for (int m = 0; m < M; m++) {
+
745  long n = ii % N;
+
746  ii /= N;
+
747  if (n + 1 >= N) continue;
748 
-
749  // convert indices by applying permutation
-
750  for (int k = 0; k < K; k++) {
-
751  long ii = terms[k];
-
752  if (use_seen) {
-
753  // clear seen for reuse at next loop
-
754  seen[ii >> 3] = 0;
-
755  }
-
756  long ti = 0;
-
757  for (int m = 0; m < M; m++) {
-
758  long n = ii % N;
-
759  ti += weights[m] * ssx[m].get_ord(n);
-
760  ii /= N;
-
761  }
-
762  terms[k] = ti;
-
763  }
-
764  }
-
765 
-
766 
-
767  void enqueue_follower (long ti, int m, int n, T sum) {
-
768  T next_sum = sum + ssx[m].get_diff(n + 1);
-
769  long next_ti = ti + weights[m];
-
770  heap_push<HC> (++heap_size, bh_val, bh_ids, next_sum, next_ti);
-
771  }
-
772 
-
773 
-
774  ~MinSumK () {
-
775  delete [] bh_ids;
-
776  delete [] bh_val;
-
777  }
-
778 };
-
779 
-
780 
-
781 
+
749  enqueue_follower (ti, m, n, sum);
+
750  }
+
751  }
+
752 
+
753  /*
+
754  for (int k = 0; k < K; k++)
+
755  for (int l = k + 1; l < K; l++)
+
756  assert (terms[k] != terms[l]);
+
757  */
+
758 
+
759  // convert indices by applying permutation
+
760  for (int k = 0; k < K; k++) {
+
761  long ii = terms[k];
+
762  if (use_seen) {
+
763  // clear seen for reuse at next loop
+
764  seen[ii >> 3] = 0;
+
765  }
+
766  long ti = 0;
+
767  for (int m = 0; m < M; m++) {
+
768  long n = ii % N;
+
769  ti += weights[m] * ssx[m].get_ord(n);
+
770  ii /= N;
+
771  }
+
772  terms[k] = ti;
+
773  }
+
774  }
+
775 
+
776 
+
777  void enqueue_follower (long ti, int m, int n, T sum) {
+
778  T next_sum = sum + ssx[m].get_diff(n + 1);
+
779  long next_ti = ti + weights[m];
+
780  heap_push<HC> (++heap_size, bh_val, bh_ids, next_sum, next_ti);
+
781  }
782 
-
783 MultiIndexQuantizer::MultiIndexQuantizer (int d,
-
784  size_t M,
-
785  size_t nbits):
-
786  Index(d, METRIC_L2), pq(d, M, nbits)
-
787 {
-
788  is_trained = false;
-
789  pq.verbose = verbose;
-
790 }
+
783 
+
784  ~MinSumK () {
+
785  delete [] bh_ids;
+
786  delete [] bh_val;
+
787  }
+
788 };
+
789 
+
790 
791 
792 
-
793 
-
794 void MultiIndexQuantizer::train(idx_t n, const float *x)
-
795 {
-
796  pq.train (n, x);
-
797  is_trained = true;
-
798  // count virtual elements in index
-
799  ntotal = 1;
-
800  for (int m = 0; m < pq.M; m++)
-
801  ntotal *= pq.ksub;
-
802 }
+
793 MultiIndexQuantizer::MultiIndexQuantizer (int d,
+
794  size_t M,
+
795  size_t nbits):
+
796  Index(d, METRIC_L2), pq(d, M, nbits)
+
797 {
+
798  is_trained = false;
+
799  pq.verbose = verbose;
+
800 }
+
801 
+
802 
803 
-
804 
-
805 void MultiIndexQuantizer::search (idx_t n, const float *x, idx_t k,
-
806  float *distances, idx_t *labels) const {
-
807  if (n == 0) return;
-
808 
-
809  float * dis_tables = new float [n * pq.ksub * pq.M];
-
810  ScopeDeleter<float> del (dis_tables);
-
811 
-
812  pq.compute_distance_tables (n, x, dis_tables);
-
813 
-
814  if (k == 1) {
-
815  // simple version that just finds the min in each table
-
816 
-
817 #pragma omp parallel for
-
818  for (int i = 0; i < n; i++) {
-
819  const float * dis_table = dis_tables + i * pq.ksub * pq.M;
-
820  float dis = 0;
-
821  idx_t label = 0;
+
804 void MultiIndexQuantizer::train(idx_t n, const float *x)
+
805 {
+
806  pq.verbose = verbose;
+
807  pq.train (n, x);
+
808  is_trained = true;
+
809  // count virtual elements in index
+
810  ntotal = 1;
+
811  for (int m = 0; m < pq.M; m++)
+
812  ntotal *= pq.ksub;
+
813 }
+
814 
+
815 
+
816 void MultiIndexQuantizer::search (idx_t n, const float *x, idx_t k,
+
817  float *distances, idx_t *labels) const {
+
818  if (n == 0) return;
+
819 
+
820  float * dis_tables = new float [n * pq.ksub * pq.M];
+
821  ScopeDeleter<float> del (dis_tables);
822 
-
823  for (int s = 0; s < pq.M; s++) {
-
824  float vmin = HUGE_VALF;
-
825  idx_t lmin = -1;
-
826 
-
827  for (idx_t j = 0; j < pq.ksub; j++) {
-
828  if (dis_table[j] < vmin) {
-
829  vmin = dis_table[j];
-
830  lmin = j;
-
831  }
-
832  }
-
833  dis += vmin;
-
834  label |= lmin << (s * pq.nbits);
-
835  dis_table += pq.ksub;
-
836  }
+
823  pq.compute_distance_tables (n, x, dis_tables);
+
824 
+
825  if (k == 1) {
+
826  // simple version that just finds the min in each table
+
827 
+
828 #pragma omp parallel for
+
829  for (int i = 0; i < n; i++) {
+
830  const float * dis_table = dis_tables + i * pq.ksub * pq.M;
+
831  float dis = 0;
+
832  idx_t label = 0;
+
833 
+
834  for (int s = 0; s < pq.M; s++) {
+
835  float vmin = HUGE_VALF;
+
836  idx_t lmin = -1;
837 
-
838  distances [i] = dis;
-
839  labels [i] = label;
-
840  }
-
841 
-
842 
-
843  } else {
-
844 
-
845 #pragma omp parallel if(n > 1)
-
846  {
-
847  MinSumK <float, SemiSortedArray<float>, false>
-
848  msk(k, pq.M, pq.ksub);
-
849 #pragma omp for
-
850  for (int i = 0; i < n; i++) {
-
851  msk.run (dis_tables + i * pq.ksub * pq.M,
-
852  distances + i * k, labels + i * k);
+
838  for (idx_t j = 0; j < pq.ksub; j++) {
+
839  if (dis_table[j] < vmin) {
+
840  vmin = dis_table[j];
+
841  lmin = j;
+
842  }
+
843  }
+
844  dis += vmin;
+
845  label |= lmin << (s * pq.nbits);
+
846  dis_table += pq.ksub;
+
847  }
+
848 
+
849  distances [i] = dis;
+
850  labels [i] = label;
+
851  }
+
852 
853 
-
854  }
-
855  }
-
856  }
-
857 
-
858 }
-
859 
-
860 
-
861 void MultiIndexQuantizer::reconstruct (idx_t key, float * recons) const
-
862 {
-
863  if (pq.byte_per_idx == 1) {
-
864  uint8_t code[pq.M];
-
865  long jj = key;
-
866  for (int m = 0; m < pq.M; m++) {
-
867  long n = jj % pq.ksub;
-
868  jj /= pq.ksub;
-
869  code[m] = n;
-
870  }
-
871  pq.decode (code, recons);
-
872  } else if (pq.byte_per_idx == 2) {
-
873  uint16_t code[pq.M];
-
874  long jj = key;
-
875  for (int m = 0; m < pq.M; m++) {
-
876  long n = jj % pq.ksub;
-
877  jj /= pq.ksub;
-
878  code[m] = n;
-
879  }
-
880  pq.decode ((uint8_t*)code, recons);
-
881  } else FAISS_THROW_MSG( "only 1 or 2 bytes per index supported");
-
882 }
-
883 
-
884 void MultiIndexQuantizer::add(idx_t /*n*/, const float* /*x*/) {
-
885  FAISS_THROW_MSG(
-
886  "This index has virtual elements, "
-
887  "it does not support add");
-
888 }
-
889 
-
890 void MultiIndexQuantizer::reset ()
-
891 {
-
892  FAISS_THROW_MSG ( "This index has virtual elements, "
-
893  "it does not support reset");
-
894 }
-
895 
-
896 
-
897 
-
898 
-
899 } // END namespace faiss
-
faiss::MinSumK::M
int M
nb of elements to sum up
Definition: IndexPQ.cpp:639
+
854  } else {
+
855 
+
856 #pragma omp parallel if(n > 1)
+
857  {
+
858  MinSumK <float, SemiSortedArray<float>, false>
+
859  msk(k, pq.M, pq.ksub);
+
860 #pragma omp for
+
861  for (int i = 0; i < n; i++) {
+
862  msk.run (dis_tables + i * pq.ksub * pq.M,
+
863  distances + i * k, labels + i * k);
+
864 
+
865  }
+
866  }
+
867  }
+
868 
+
869 }
+
870 
+
871 
+
872 void MultiIndexQuantizer::reconstruct (idx_t key, float * recons) const
+
873 {
+
874  if (pq.byte_per_idx == 1) {
+
875  uint8_t code[pq.M];
+
876  long jj = key;
+
877  for (int m = 0; m < pq.M; m++) {
+
878  long n = jj % pq.ksub;
+
879  jj /= pq.ksub;
+
880  code[m] = n;
+
881  }
+
882  pq.decode (code, recons);
+
883  } else if (pq.byte_per_idx == 2) {
+
884  uint16_t code[pq.M];
+
885  long jj = key;
+
886  for (int m = 0; m < pq.M; m++) {
+
887  long n = jj % pq.ksub;
+
888  jj /= pq.ksub;
+
889  code[m] = n;
+
890  }
+
891  pq.decode ((uint8_t*)code, recons);
+
892  } else FAISS_THROW_MSG( "only 1 or 2 bytes per index supported");
+
893 }
+
894 
+
895 void MultiIndexQuantizer::add(idx_t /*n*/, const float* /*x*/) {
+
896  FAISS_THROW_MSG(
+
897  "This index has virtual elements, "
+
898  "it does not support add");
+
899 }
+
900 
+
901 void MultiIndexQuantizer::reset ()
+
902 {
+
903  FAISS_THROW_MSG ( "This index has virtual elements, "
+
904  "it does not support reset");
+
905 }
+
906 
+
907 
+
908 
+
909 
+
910 } // END namespace faiss
+
faiss::MinSumK::M
int M
nb of elements to sum up
Definition: IndexPQ.cpp:649
faiss::IndexPQ::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexPQ.h:34
faiss::ProductQuantizer::nbits
size_t nbits
number of bits per quantization index
Definition: ProductQuantizer.h:29
faiss::ProductQuantizer::decode
void decode(const uint8_t *code, float *x) const
decode a vector from a given code (or n vectors if third argument)
Definition: ProductQuantizer.cpp:338
@@ -990,23 +1001,23 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::IndexPQ::do_polysemous_training
bool do_polysemous_training
false = standard PQ
Definition: IndexPQ.h:69
faiss::IndexPQ::train
void train(idx_t n, const float *x) override
Definition: IndexPQ.cpp:54
faiss::ProductQuantizer::byte_per_idx
size_t byte_per_idx
nb bytes per code component (1 or 2)
Definition: ProductQuantizer.h:33
-
faiss::MultiIndexQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexPQ.cpp:890
+
faiss::MultiIndexQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexPQ.cpp:901
-
faiss::partial_sort
void partial_sort(int k, int n, const typename C::T *vals, typename C::TI *perm)
Definition: IndexPQ.cpp:515
-
faiss::MultiIndexQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexPQ.cpp:794
- -
faiss::MinSumK::HC
CMin< T, long > HC
Definition: IndexPQ.cpp:646
- -
faiss::SemiSortedArray::grow
void grow(int next_k)
grow the sorted part of the array to size next_k
Definition: IndexPQ.cpp:575
- +
faiss::partial_sort
void partial_sort(int k, int n, const typename C::T *vals, typename C::TI *perm)
Definition: IndexPQ.cpp:525
+
faiss::MultiIndexQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexPQ.cpp:804
+ +
faiss::MinSumK::HC
CMin< T, long > HC
Definition: IndexPQ.cpp:656
+ +
faiss::SemiSortedArray::grow
void grow(int next_k)
grow the sorted part of the array to size next_k
Definition: IndexPQ.cpp:585
+
faiss::ProductQuantizer::compute_distance_tables
void compute_distance_tables(size_t nx, const float *x, float *dis_tables) const
Definition: ProductQuantizer.cpp:439
- +
faiss::generalized_hammings_knn
void generalized_hammings_knn(int_maxheap_array_t *ha, const uint8_t *a, const uint8_t *b, size_t nb, size_t code_size, int ordered)
Definition: hamming.cpp:639
faiss::ProductQuantizer::compute_code_from_distance_table
void compute_code_from_distance_table(const float *tab, uint8_t *code) const
Definition: ProductQuantizer.cpp:363
faiss::ProductQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: ProductQuantizer.cpp:385
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexPQ::hamming_distance_histogram
void hamming_distance_histogram(idx_t n, const float *x, idx_t nb, const float *xb, long *dist_histogram)
Definition: IndexPQ.cpp:382
+
faiss::IndexPQ::hamming_distance_histogram
void hamming_distance_histogram(idx_t n, const float *x, idx_t nb, const float *xb, long *dist_histogram)
Definition: IndexPQ.cpp:392
faiss::ProductQuantizer::search
void search(const float *x, size_t nx, const uint8_t *codes, const size_t ncodes, float_maxheap_array_t *res, bool init_finalize_heap=true) const
Definition: ProductQuantizer.cpp:540
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:34
faiss::IndexPQ::ST_polysemous_generalize
Filter on generalized Hamming.
Definition: IndexPQ.h:81
@@ -1020,19 +1031,19 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
faiss::IndexPQ::add
void add(idx_t n, const float *x) override
Definition: IndexPQ.cpp:78
-
faiss::MinSumK::K
int K
nb of sums to return
Definition: IndexPQ.cpp:638
-
faiss::IndexPQ::hamming_distance_table
void hamming_distance_table(idx_t n, const float *x, int32_t *dis) const
Definition: IndexPQ.cpp:370
-
faiss::MultiIndexQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexPQ.cpp:805
+
faiss::MinSumK::K
int K
nb of sums to return
Definition: IndexPQ.cpp:648
+
faiss::IndexPQ::hamming_distance_table
void hamming_distance_table(idx_t n, const float *x, int32_t *dis) const
Definition: IndexPQ.cpp:380
+
faiss::MultiIndexQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexPQ.cpp:816
faiss::IndexPQ::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexPQ.cpp:104
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
faiss::ProductQuantizer::M
size_t M
number of subquantizers
Definition: ProductQuantizer.h:28
-
faiss::MinSumK::N
int N
nb of possible elements for each of the M terms
Definition: IndexPQ.cpp:640
+
faiss::MinSumK::N
int N
nb of possible elements for each of the M terms
Definition: IndexPQ.cpp:650
faiss::IndexPQ::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexPQ.cpp:94
faiss::IndexPQ::ST_PQ
asymmetric product quantizer (default)
Definition: IndexPQ.h:76
-
faiss::MultiIndexQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexPQ.cpp:861
+
faiss::MultiIndexQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexPQ.cpp:872
faiss::IndexPQ::ST_polysemous
HE filter (using ht) + PQ combination.
Definition: IndexPQ.h:80
-
faiss::MultiIndexQuantizer::add
void add(idx_t n, const float *x) override
add and reset will crash at runtime
Definition: IndexPQ.cpp:884
+
faiss::MultiIndexQuantizer::add
void add(idx_t n, const float *x) override
add and reset will crash at runtime
Definition: IndexPQ.cpp:895
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
faiss::IndexPQ::reset
void reset() override
removes all elements from the database.
Definition: IndexPQ.cpp:88
faiss::PolysemousTraining::optimize_pq_for_hamming
void optimize_pq_for_hamming(ProductQuantizer &pq, size_t n, const float *x) const
Definition: PolysemousTraining.cpp:933
diff --git a/docs/html/IndexPQ_8h_source.html b/docs/html/IndexPQ_8h_source.html index 99eb14846..1c35dd397 100644 --- a/docs/html/IndexPQ_8h_source.html +++ b/docs/html/IndexPQ_8h_source.html @@ -253,11 +253,11 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::IndexPQ::ST_HE
Hamming distance on codes.
Definition: IndexPQ.h:77
faiss::IndexPQ::do_polysemous_training
bool do_polysemous_training
false = standard PQ
Definition: IndexPQ.h:69
faiss::IndexPQ::train
void train(idx_t n, const float *x) override
Definition: IndexPQ.cpp:54
-
faiss::MultiIndexQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexPQ.cpp:890
-
faiss::MultiIndexQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexPQ.cpp:794
+
faiss::MultiIndexQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexPQ.cpp:901
+
faiss::MultiIndexQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexPQ.cpp:804
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexPQ::hamming_distance_histogram
void hamming_distance_histogram(idx_t n, const float *x, idx_t nb, const float *xb, long *dist_histogram)
Definition: IndexPQ.cpp:382
+
faiss::IndexPQ::hamming_distance_histogram
void hamming_distance_histogram(idx_t n, const float *x, idx_t nb, const float *xb, long *dist_histogram)
Definition: IndexPQ.cpp:392
faiss::IndexPQ::Search_type_t
Search_type_t
how to perform the search in search_core
Definition: IndexPQ.h:75
faiss::IndexPQ::ST_polysemous_generalize
Filter on generalized Hamming.
Definition: IndexPQ.h:81
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
@@ -265,17 +265,17 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::IndexPQ::add
void add(idx_t n, const float *x) override
Definition: IndexPQ.cpp:78
faiss::PolysemousTraining
optimizes the order of indices in a ProductQuantizer
Definition: PolysemousTraining.h:125
-
faiss::IndexPQ::hamming_distance_table
void hamming_distance_table(idx_t n, const float *x, int32_t *dis) const
Definition: IndexPQ.cpp:370
-
faiss::MultiIndexQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexPQ.cpp:805
+
faiss::IndexPQ::hamming_distance_table
void hamming_distance_table(idx_t n, const float *x, int32_t *dis) const
Definition: IndexPQ.cpp:380
+
faiss::MultiIndexQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexPQ.cpp:816
faiss::IndexPQ::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexPQ.cpp:104
faiss::IndexPQ::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexPQ.cpp:94
faiss::IndexPQ::ST_PQ
asymmetric product quantizer (default)
Definition: IndexPQ.h:76
-
faiss::MultiIndexQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexPQ.cpp:861
+
faiss::MultiIndexQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexPQ.cpp:872
faiss::IndexPQ::ST_polysemous
HE filter (using ht) + PQ combination.
Definition: IndexPQ.h:80
-
faiss::MultiIndexQuantizer::add
void add(idx_t n, const float *x) override
add and reset will crash at runtime
Definition: IndexPQ.cpp:884
+
faiss::MultiIndexQuantizer::add
void add(idx_t n, const float *x) override
add and reset will crash at runtime
Definition: IndexPQ.cpp:895
faiss::IndexPQ::reset
void reset() override
removes all elements from the database.
Definition: IndexPQ.cpp:88
faiss::IndexPQ::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexPQ.cpp:124
faiss::IndexPQ::ST_SDC
symmetric product quantizer (SDC)
Definition: IndexPQ.h:79
diff --git a/docs/html/IndexProxy_8cpp_source.html b/docs/html/IndexProxy_8cpp_source.html index 09a8fd939..facf8aee8 100644 --- a/docs/html/IndexProxy_8cpp_source.html +++ b/docs/html/IndexProxy_8cpp_source.html @@ -193,7 +193,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
104 
105 void
106 IndexProxy::train(Index::idx_t n, const float* x) {
-
107  runOnIndex([n, x](faiss::Index* index){ index->train(n, x); });
+
107  runOnIndex([n, x](faiss::Index* index){ index->train(n, x); });
108 }
109 
110 void
@@ -298,7 +298,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
209  }
210  index = &proxy;
211  }
-
212  clus.train (n, x, *index);
+
212  clus.train (n, x, *index);
213 
214  memcpy(centroids, clus.centroids.data(), sizeof(*centroids) * d * k);
215  return clus.obj.back();
@@ -314,7 +314,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::gpu::IndexProxy::runOnIndex
void runOnIndex(std::function< void(faiss::Index *)> f)
Definition: IndexProxy.cpp:85
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
faiss::gpu::GpuIndexFlatConfig::useFloat16
bool useFloat16
Whether or not data is stored as float16.
Definition: GpuIndexFlat.h:35
faiss::gpu::GpuIndexConfig::device
int device
GPU device on which the index is resident.
Definition: GpuIndex.h:27
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
diff --git a/docs/html/IndexScalarQuantizer_8cpp_source.html b/docs/html/IndexScalarQuantizer_8cpp_source.html index f023c0216..34c2095d7 100644 --- a/docs/html/IndexScalarQuantizer_8cpp_source.html +++ b/docs/html/IndexScalarQuantizer_8cpp_source.html @@ -96,1033 +96,1166 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
11 #include <cstdio>
12 #include <algorithm>
13 
-
14 #include <omp.h>
+
14 #include <malloc.h>
15 
-
16 #include <immintrin.h>
+
16 #include <omp.h>
17 
-
18 #include "utils.h"
+
18 #include <immintrin.h>
19 
-
20 #include "FaissAssert.h"
+
20 #include "utils.h"
21 
-
22 namespace faiss {
+
22 #include "FaissAssert.h"
23 
-
24 /*******************************************************************
-
25  * ScalarQuantizer implementation
-
26  *
-
27  * The main source of complexity is to support combinations of 4
-
28  * variants without incurring runtime tests or virtual function calls:
-
29  *
-
30  * - 4 / 8 bits per code component
-
31  * - uniform / non-uniform
-
32  * - IP / L2 distance search
-
33  * - scalar / AVX distance computation
-
34  *
-
35  * The appropriate Quantizer object is returned via select_quantizer
-
36  * that hides the template mess.
-
37  ********************************************************************/
-
38 
-
39 #ifdef __AVX__
-
40 #define USE_AVX
-
41 #endif
-
42 
-
43 
-
44 namespace {
+
24 namespace faiss {
+
25 
+
26 /*******************************************************************
+
27  * ScalarQuantizer implementation
+
28  *
+
29  * The main source of complexity is to support combinations of 4
+
30  * variants without incurring runtime tests or virtual function calls:
+
31  *
+
32  * - 4 / 8 bits per code component
+
33  * - uniform / non-uniform
+
34  * - IP / L2 distance search
+
35  * - scalar / AVX distance computation
+
36  *
+
37  * The appropriate Quantizer object is returned via select_quantizer
+
38  * that hides the template mess.
+
39  ********************************************************************/
+
40 
+
41 #ifdef __AVX__
+
42 #define USE_AVX
+
43 #endif
+
44 
45 
-
46 typedef Index::idx_t idx_t;
-
47 typedef ScalarQuantizer::QuantizerType QuantizerType;
-
48 typedef ScalarQuantizer::RangeStat RangeStat;
-
49 
-
50 
-
51 /*******************************************************************
-
52  * Codec: converts between values in [0, 1] and an index in a code
-
53  * array. The "i" parameter is the vector component index (not byte
-
54  * index).
-
55  */
-
56 
-
57 struct Codec8bit {
-
58 
-
59  static void encode_component (float x, uint8_t *code, int i) {
-
60  code[i] = (int)(255 * x);
-
61  }
-
62 
-
63  static float decode_component (const uint8_t *code, int i) {
-
64  return (code[i] + 0.5f) / 255.0f;
-
65  }
-
66 
-
67 #ifdef USE_AVX
-
68  static __m256 decode_8_components (const uint8_t *code, int i) {
-
69  uint64_t c8 = *(uint64_t*)(code + i);
-
70  __m128i c4lo = _mm_cvtepu8_epi32 (_mm_set1_epi32(c8));
-
71  __m128i c4hi = _mm_cvtepu8_epi32 (_mm_set1_epi32(c8 >> 32));
-
72  // __m256i i8 = _mm256_set_m128i(c4lo, c4hi);
-
73  __m256i i8 = _mm256_castsi128_si256 (c4lo);
-
74  i8 = _mm256_insertf128_si256 (i8, c4hi, 1);
-
75  __m256 f8 = _mm256_cvtepi32_ps (i8);
-
76  __m256 half = _mm256_set1_ps (0.5f);
-
77  f8 += half;
-
78  __m256 one_255 = _mm256_set1_ps (1.f / 255.f);
-
79  return f8 * one_255;
-
80  }
-
81 #endif
-
82 };
-
83 
-
84 
-
85 struct Codec4bit {
+
46 namespace {
+
47 
+
48 typedef Index::idx_t idx_t;
+
49 typedef ScalarQuantizer::QuantizerType QuantizerType;
+
50 typedef ScalarQuantizer::RangeStat RangeStat;
+
51 using DistanceComputer = ScalarQuantizer::DistanceComputer;
+
52 
+
53 
+
54 /*******************************************************************
+
55  * Codec: converts between values in [0, 1] and an index in a code
+
56  * array. The "i" parameter is the vector component index (not byte
+
57  * index).
+
58  */
+
59 
+
60 struct Codec8bit {
+
61 
+
62  static void encode_component (float x, uint8_t *code, int i) {
+
63  code[i] = (int)(255 * x);
+
64  }
+
65 
+
66  static float decode_component (const uint8_t *code, int i) {
+
67  return (code[i] + 0.5f) / 255.0f;
+
68  }
+
69 
+
70 #ifdef USE_AVX
+
71  static __m256 decode_8_components (const uint8_t *code, int i) {
+
72  uint64_t c8 = *(uint64_t*)(code + i);
+
73  __m128i c4lo = _mm_cvtepu8_epi32 (_mm_set1_epi32(c8));
+
74  __m128i c4hi = _mm_cvtepu8_epi32 (_mm_set1_epi32(c8 >> 32));
+
75  // __m256i i8 = _mm256_set_m128i(c4lo, c4hi);
+
76  __m256i i8 = _mm256_castsi128_si256 (c4lo);
+
77  i8 = _mm256_insertf128_si256 (i8, c4hi, 1);
+
78  __m256 f8 = _mm256_cvtepi32_ps (i8);
+
79  __m256 half = _mm256_set1_ps (0.5f);
+
80  f8 += half;
+
81  __m256 one_255 = _mm256_set1_ps (1.f / 255.f);
+
82  return f8 * one_255;
+
83  }
+
84 #endif
+
85 };
86 
-
87  static void encode_component (float x, uint8_t *code, int i) {
-
88  code [i / 2] |= (int)(x * 15.0) << ((i & 1) << 2);
-
89  }
-
90 
-
91  static float decode_component (const uint8_t *code, int i) {
-
92  return (((code[i / 2] >> ((i & 1) << 2)) & 0xf) + 0.5f) / 15.0f;
-
93  }
-
94 
-
95 
-
96 #ifdef USE_AVX
-
97  static __m256 decode_8_components (const uint8_t *code, int i) {
-
98  uint32_t c4 = *(uint32_t*)(code + (i >> 1));
-
99  uint32_t mask = 0x0f0f0f0f;
-
100  uint32_t c4ev = c4 & mask;
-
101  uint32_t c4od = (c4 >> 4) & mask;
-
102 
-
103  // the 8 lower bytes of c8 contain the values
-
104  __m128i c8 = _mm_unpacklo_epi8 (_mm_set1_epi32(c4ev),
-
105  _mm_set1_epi32(c4od));
-
106  __m128i c4lo = _mm_cvtepu8_epi32 (c8);
-
107  __m128i c4hi = _mm_cvtepu8_epi32 (_mm_srli_si128(c8, 4));
-
108  __m256i i8 = _mm256_castsi128_si256 (c4lo);
-
109  i8 = _mm256_insertf128_si256 (i8, c4hi, 1);
-
110  __m256 f8 = _mm256_cvtepi32_ps (i8);
-
111  __m256 half = _mm256_set1_ps (0.5f);
-
112  f8 += half;
-
113  __m256 one_255 = _mm256_set1_ps (1.f / 15.f);
-
114  return f8 * one_255;
-
115  }
-
116 #endif
-
117 };
-
118 
-
119 
-
120 /*******************************************************************
-
121  * Similarity: gets vector components and computes a similarity wrt. a
-
122  * query vector stored in the object
-
123  */
-
124 
-
125 struct SimilarityL2 {
-
126  const float *y, *yi;
-
127 
-
128  explicit SimilarityL2 (const float * y): y(y) {}
+
87 
+
88 struct Codec4bit {
+
89 
+
90  static void encode_component (float x, uint8_t *code, int i) {
+
91  code [i / 2] |= (int)(x * 15.0) << ((i & 1) << 2);
+
92  }
+
93 
+
94  static float decode_component (const uint8_t *code, int i) {
+
95  return (((code[i / 2] >> ((i & 1) << 2)) & 0xf) + 0.5f) / 15.0f;
+
96  }
+
97 
+
98 
+
99 #ifdef USE_AVX
+
100  static __m256 decode_8_components (const uint8_t *code, int i) {
+
101  uint32_t c4 = *(uint32_t*)(code + (i >> 1));
+
102  uint32_t mask = 0x0f0f0f0f;
+
103  uint32_t c4ev = c4 & mask;
+
104  uint32_t c4od = (c4 >> 4) & mask;
+
105 
+
106  // the 8 lower bytes of c8 contain the values
+
107  __m128i c8 = _mm_unpacklo_epi8 (_mm_set1_epi32(c4ev),
+
108  _mm_set1_epi32(c4od));
+
109  __m128i c4lo = _mm_cvtepu8_epi32 (c8);
+
110  __m128i c4hi = _mm_cvtepu8_epi32 (_mm_srli_si128(c8, 4));
+
111  __m256i i8 = _mm256_castsi128_si256 (c4lo);
+
112  i8 = _mm256_insertf128_si256 (i8, c4hi, 1);
+
113  __m256 f8 = _mm256_cvtepi32_ps (i8);
+
114  __m256 half = _mm256_set1_ps (0.5f);
+
115  f8 += half;
+
116  __m256 one_255 = _mm256_set1_ps (1.f / 15.f);
+
117  return f8 * one_255;
+
118  }
+
119 #endif
+
120 };
+
121 
+
122 
+
123 
+
124 /*******************************************************************
+
125  * Quantizer: normalizes scalar vector components, then passes them
+
126  * through a codec
+
127  */
+
128 
129 
-
130  /******* scalar accumulator *******/
-
131 
-
132  float accu;
-
133 
-
134  void begin () {
-
135  accu = 0;
-
136  yi = y;
-
137  }
+
130 
+
131 struct Quantizer {
+
132  virtual void encode_vector(const float *x, uint8_t *code) const = 0;
+
133  virtual void decode_vector(const uint8_t *code, float *x) const = 0;
+
134 
+
135 
+
136  virtual ~Quantizer() {}
+
137 };
138 
-
139  void add_component (float x) {
-
140  float tmp = *yi++ - x;
-
141  accu += tmp * tmp;
-
142  }
-
143 
-
144  float result () {
-
145  return accu;
-
146  }
+
139 
+
140 
+
141 
+
142 
+
143 template<class Codec>
+
144 struct QuantizerUniform: Quantizer {
+
145  const size_t d;
+
146  const float vmin, vdiff;
147 
-
148 #ifdef USE_AVX
-
149  /******* AVX accumulator *******/
-
150 
-
151  __m256 accu8;
+
148  QuantizerUniform(size_t d, const std::vector<float> &trained):
+
149  d(d), vmin(trained[0]), vdiff(trained[1])
+
150  {
+
151  }
152 
-
153  void begin_8 () {
-
154  accu8 = _mm256_setzero_ps();
-
155  yi = y;
-
156  }
-
157 
-
158  void add_8_components (__m256 x) {
-
159  __m256 yiv = _mm256_loadu_ps (yi);
-
160  yi += 8;
-
161  __m256 tmp = yiv - x;
-
162  accu8 += tmp * tmp;
-
163  }
-
164 
-
165  float result_8 () {
-
166  __m256 sum = _mm256_hadd_ps(accu8, accu8);
-
167  __m256 sum2 = _mm256_hadd_ps(sum, sum);
-
168  // now add the 0th and 4th component
-
169  return
-
170  _mm_cvtss_f32 (_mm256_castps256_ps128(sum2)) +
-
171  _mm_cvtss_f32 (_mm256_extractf128_ps(sum2, 1));
-
172  }
-
173 #endif
-
174 };
-
175 
-
176 struct SimilarityIP {
-
177  const float *y, *yi;
-
178  const float accu0;
+
153  void encode_vector(const float* x, uint8_t* code) const override {
+
154  for (size_t i = 0; i < d; i++) {
+
155  float xi = (x[i] - vmin) / vdiff;
+
156  if (xi < 0)
+
157  xi = 0;
+
158  if (xi > 1.0)
+
159  xi = 1.0;
+
160  Codec::encode_component(xi, code, i);
+
161  }
+
162  }
+
163 
+
164  void decode_vector(const uint8_t* code, float* x) const override {
+
165  for (size_t i = 0; i < d; i++) {
+
166  float xi = Codec::decode_component(code, i);
+
167  x[i] = vmin + xi * vdiff;
+
168  }
+
169  }
+
170 
+
171  float reconstruct_component (const uint8_t * code, int i) const
+
172  {
+
173  float xi = Codec::decode_component (code, i);
+
174  return vmin + xi * vdiff;
+
175  }
+
176 
+
177 };
+
178 
179 
-
180  /******* scalar accumulator *******/
-
181 
-
182  float accu;
-
183 
-
184  SimilarityIP (const float * y, float accu0):
-
185  y (y), accu0 (accu0) {}
-
186 
-
187  void begin () {
-
188  accu = accu0;
-
189  yi = y;
-
190  }
-
191 
-
192  void add_component (float x) {
-
193  accu += *yi++ * x;
-
194  }
-
195 
-
196  float result () {
-
197  return accu;
-
198  }
+
180 
+
181 #ifdef USE_AVX
+
182 
+
183 template<class Codec>
+
184 struct QuantizerUniform8: QuantizerUniform<Codec> {
+
185 
+
186  QuantizerUniform8 (size_t d, const std::vector<float> &trained):
+
187  QuantizerUniform<Codec> (d, trained) {}
+
188 
+
189  __m256 reconstruct_8_components (const uint8_t * code, int i) const
+
190  {
+
191  __m256 xi = Codec::decode_8_components (code, i);
+
192  return _mm256_set1_ps(this->vmin) + xi * _mm256_set1_ps (this->vdiff);
+
193  }
+
194 
+
195 };
+
196 
+
197 #endif
+
198 
199 
-
200 #ifdef USE_AVX
-
201  /******* AVX accumulator *******/
-
202 
-
203  __m256 accu8;
-
204 
-
205  void begin_8 () {
-
206  accu8 = _mm256_setzero_ps();
-
207  yi = y;
-
208  }
-
209 
-
210  void add_8_components (__m256 x) {
-
211  __m256 yiv = _mm256_loadu_ps (yi);
-
212  yi += 8;
-
213  accu8 += yiv * x;
-
214  }
-
215 
-
216  float result_8 () {
-
217  __m256 sum = _mm256_hadd_ps(accu8, accu8);
-
218  __m256 sum2 = _mm256_hadd_ps(sum, sum);
-
219  // now add the 0th and 4th component
-
220  return
-
221  accu0 +
-
222  _mm_cvtss_f32 (_mm256_castps256_ps128(sum2)) +
-
223  _mm_cvtss_f32 (_mm256_extractf128_ps(sum2, 1));
-
224  }
-
225 #endif
-
226 };
-
227 
-
228 
-
229 /*******************************************************************
-
230  * templatized distance functions
-
231  */
+
200 
+
201 template<class Codec>
+
202 struct QuantizerNonUniform: Quantizer {
+
203  const size_t d;
+
204  const float *vmin, *vdiff;
+
205 
+
206  QuantizerNonUniform(size_t d, const std::vector<float> &trained):
+
207  d(d), vmin(trained.data()), vdiff(trained.data() + d) {}
+
208 
+
209  void encode_vector(const float* x, uint8_t* code) const override {
+
210  for (size_t i = 0; i < d; i++) {
+
211  float xi = (x[i] - vmin[i]) / vdiff[i];
+
212  if (xi < 0)
+
213  xi = 0;
+
214  if (xi > 1.0)
+
215  xi = 1.0;
+
216  Codec::encode_component(xi, code, i);
+
217  }
+
218  }
+
219 
+
220  void decode_vector(const uint8_t* code, float* x) const override {
+
221  for (size_t i = 0; i < d; i++) {
+
222  float xi = Codec::decode_component(code, i);
+
223  x[i] = vmin[i] + xi * vdiff[i];
+
224  }
+
225  }
+
226 
+
227  float reconstruct_component (const uint8_t * code, int i) const
+
228  {
+
229  float xi = Codec::decode_component (code, i);
+
230  return vmin[i] + xi * vdiff[i];
+
231  }
232 
-
233 
-
234 template<class Quantizer, class Similarity>
-
235 float compute_distance(const Quantizer & quant, Similarity & sim,
-
236  const uint8_t *code)
-
237 {
-
238  sim.begin();
-
239  for (size_t i = 0; i < quant.d; i++) {
-
240  float xi = quant.reconstruct_component (code, i);
-
241  sim.add_component (xi);
-
242  }
-
243  return sim.result();
-
244 }
-
245 
-
246 #ifdef USE_AVX
-
247 template<class Quantizer, class Similarity>
-
248 float compute_distance_8(const Quantizer & quant, Similarity & sim,
-
249  const uint8_t *code)
-
250 {
-
251  sim.begin_8();
-
252  for (size_t i = 0; i < quant.d; i += 8) {
-
253  __m256 xi = quant.reconstruct_8_components (code, i);
-
254  sim.add_8_components (xi);
-
255  }
-
256  return sim.result_8();
-
257 }
-
258 #endif
-
259 
-
260 
-
261 /*******************************************************************
-
262  * Quantizer range training
-
263  */
-
264 
-
265 static float sqr (float x) {
-
266  return x * x;
-
267 }
-
268 
-
269 
-
270 void train_Uniform(RangeStat rs, float rs_arg,
-
271  idx_t n, int k, const float *x,
-
272  std::vector<float> & trained)
-
273 {
-
274  trained.resize (2);
-
275  float & vmin = trained[0];
-
276  float & vmax = trained[1];
-
277 
-
278  if (rs == ScalarQuantizer::RS_minmax) {
-
279  vmin = HUGE_VAL; vmax = -HUGE_VAL;
-
280  for (size_t i = 0; i < n; i++) {
-
281  if (x[i] < vmin) vmin = x[i];
-
282  if (x[i] > vmax) vmax = x[i];
+
233 };
+
234 
+
235 
+
236 #ifdef USE_AVX
+
237 
+
238 template<class Codec>
+
239 struct QuantizerNonUniform8: QuantizerNonUniform<Codec> {
+
240 
+
241  QuantizerNonUniform8 (size_t d, const std::vector<float> &trained):
+
242  QuantizerNonUniform<Codec> (d, trained) {}
+
243 
+
244  __m256 reconstruct_8_components (const uint8_t * code, int i) const
+
245  {
+
246  __m256 xi = Codec::decode_8_components (code, i);
+
247  return _mm256_loadu_ps (this->vmin + i) + xi * _mm256_loadu_ps (this->vdiff + i);
+
248  }
+
249 
+
250 
+
251 };
+
252 
+
253 #endif
+
254 
+
255 Quantizer *select_quantizer (
+
256  QuantizerType qtype,
+
257  size_t d, const std::vector<float> & trained)
+
258 {
+
259 #ifdef USE_AVX
+
260  if (d % 8 == 0) {
+
261  switch(qtype) {
+
262  case ScalarQuantizer::QT_8bit:
+
263  return new QuantizerNonUniform8<Codec8bit>(d, trained);
+
264  case ScalarQuantizer::QT_4bit:
+
265  return new QuantizerNonUniform8<Codec4bit>(d, trained);
+
266  case ScalarQuantizer::QT_8bit_uniform:
+
267  return new QuantizerUniform8<Codec8bit>(d, trained);
+
268  case ScalarQuantizer::QT_4bit_uniform:
+
269  return new QuantizerUniform8<Codec4bit>(d, trained);
+
270  }
+
271  } else
+
272 #endif
+
273  {
+
274  switch(qtype) {
+
275  case ScalarQuantizer::QT_8bit:
+
276  return new QuantizerNonUniform<Codec8bit>(d, trained);
+
277  case ScalarQuantizer::QT_4bit:
+
278  return new QuantizerNonUniform<Codec4bit>(d, trained);
+
279  case ScalarQuantizer::QT_8bit_uniform:
+
280  return new QuantizerUniform<Codec8bit>(d, trained);
+
281  case ScalarQuantizer::QT_4bit_uniform:
+
282  return new QuantizerUniform<Codec4bit>(d, trained);
283  }
-
284  float vexp = (vmax - vmin) * rs_arg;
-
285  vmin -= vexp;
-
286  vmax += vexp;
-
287  } else if (rs == ScalarQuantizer::RS_meanstd) {
-
288  double sum = 0, sum2 = 0;
-
289  for (size_t i = 0; i < n; i++) {
-
290  sum += x[i];
-
291  sum2 += x[i] * x[i];
-
292  }
-
293  float mean = sum / n;
-
294  float var = sum2 / n - mean * mean;
-
295  float std = var <= 0 ? 1.0 : sqrt(var);
+
284  }
+
285  FAISS_THROW_MSG ("unknown qtype");
+
286  return nullptr;
+
287 }
+
288 
+
289 
+
290 Quantizer *select_quantizer (const ScalarQuantizer &sq)
+
291 {
+
292  return select_quantizer (sq.qtype, sq.d, sq.trained);
+
293 }
+
294 
+
295 
296 
-
297  vmin = mean - std * rs_arg ;
-
298  vmax = mean + std * rs_arg ;
-
299  } else if (rs == ScalarQuantizer::RS_quantiles) {
-
300  std::vector<float> x_copy(n);
-
301  memcpy(x_copy.data(), x, n * sizeof(*x));
-
302  // TODO just do a qucikselect
-
303  std::sort(x_copy.begin(), x_copy.end());
-
304  int o = int(rs_arg * n);
-
305  if (o < 0) o = 0;
-
306  if (o > n - o) o = n / 2;
-
307  vmin = x_copy[o];
-
308  vmax = x_copy[n - 1 - o];
-
309 
-
310  } else if (rs == ScalarQuantizer::RS_optim) {
-
311  float a, b;
-
312  float sx = 0;
-
313  {
-
314  vmin = HUGE_VAL, vmax = -HUGE_VAL;
-
315  for (size_t i = 0; i < n; i++) {
-
316  if (x[i] < vmin) vmin = x[i];
-
317  if (x[i] > vmax) vmax = x[i];
-
318  sx += x[i];
-
319  }
-
320  b = vmin;
-
321  a = (vmax - vmin) / (k - 1);
-
322  }
-
323  int verbose = false;
-
324  int niter = 2000;
-
325  float last_err = -1;
-
326  int iter_last_err = 0;
-
327  for (int it = 0; it < niter; it++) {
-
328  float sn = 0, sn2 = 0, sxn = 0, err1 = 0;
-
329 
-
330  for (idx_t i = 0; i < n; i++) {
-
331  float xi = x[i];
-
332  float ni = floor ((xi - b) / a + 0.5);
-
333  if (ni < 0) ni = 0;
-
334  if (ni >= k) ni = k - 1;
-
335  err1 += sqr (xi - (ni * a + b));
-
336  sn += ni;
-
337  sn2 += ni * ni;
-
338  sxn += ni * xi;
-
339  }
-
340 
-
341  if (err1 == last_err) {
-
342  iter_last_err ++;
-
343  if (iter_last_err == 16) break;
-
344  } else {
-
345  last_err = err1;
-
346  iter_last_err = 0;
-
347  }
-
348 
-
349  float det = sqr (sn) - sn2 * n;
-
350 
-
351  b = (sn * sxn - sn2 * sx) / det;
-
352  a = (sn * sx - n * sxn) / det;
-
353  if (verbose) {
-
354  printf ("it %d, err1=%g \r", it, err1);
-
355  fflush(stdout);
+
297 
+
298 /*******************************************************************
+
299  * Quantizer range training
+
300  */
+
301 
+
302 static float sqr (float x) {
+
303  return x * x;
+
304 }
+
305 
+
306 
+
307 void train_Uniform(RangeStat rs, float rs_arg,
+
308  idx_t n, int k, const float *x,
+
309  std::vector<float> & trained)
+
310 {
+
311  trained.resize (2);
+
312  float & vmin = trained[0];
+
313  float & vmax = trained[1];
+
314 
+
315  if (rs == ScalarQuantizer::RS_minmax) {
+
316  vmin = HUGE_VAL; vmax = -HUGE_VAL;
+
317  for (size_t i = 0; i < n; i++) {
+
318  if (x[i] < vmin) vmin = x[i];
+
319  if (x[i] > vmax) vmax = x[i];
+
320  }
+
321  float vexp = (vmax - vmin) * rs_arg;
+
322  vmin -= vexp;
+
323  vmax += vexp;
+
324  } else if (rs == ScalarQuantizer::RS_meanstd) {
+
325  double sum = 0, sum2 = 0;
+
326  for (size_t i = 0; i < n; i++) {
+
327  sum += x[i];
+
328  sum2 += x[i] * x[i];
+
329  }
+
330  float mean = sum / n;
+
331  float var = sum2 / n - mean * mean;
+
332  float std = var <= 0 ? 1.0 : sqrt(var);
+
333 
+
334  vmin = mean - std * rs_arg ;
+
335  vmax = mean + std * rs_arg ;
+
336  } else if (rs == ScalarQuantizer::RS_quantiles) {
+
337  std::vector<float> x_copy(n);
+
338  memcpy(x_copy.data(), x, n * sizeof(*x));
+
339  // TODO just do a qucikselect
+
340  std::sort(x_copy.begin(), x_copy.end());
+
341  int o = int(rs_arg * n);
+
342  if (o < 0) o = 0;
+
343  if (o > n - o) o = n / 2;
+
344  vmin = x_copy[o];
+
345  vmax = x_copy[n - 1 - o];
+
346 
+
347  } else if (rs == ScalarQuantizer::RS_optim) {
+
348  float a, b;
+
349  float sx = 0;
+
350  {
+
351  vmin = HUGE_VAL, vmax = -HUGE_VAL;
+
352  for (size_t i = 0; i < n; i++) {
+
353  if (x[i] < vmin) vmin = x[i];
+
354  if (x[i] > vmax) vmax = x[i];
+
355  sx += x[i];
356  }
-
357  }
-
358  if (verbose) printf("\n");
-
359 
-
360  vmin = b;
-
361  vmax = b + a * (k - 1);
-
362 
-
363  } else {
-
364  FAISS_THROW_MSG ("Invalid qtype");
-
365  }
-
366  vmax -= vmin;
-
367 }
-
368 
-
369 void train_NonUniform(RangeStat rs, float rs_arg,
-
370  idx_t n, int d, int k, const float *x,
-
371  std::vector<float> & trained)
-
372 {
-
373  trained.resize (2 * d);
-
374  float * vmin = trained.data();
-
375  float * vmax = trained.data() + d;
-
376  if (rs == ScalarQuantizer::RS_minmax) {
-
377  memcpy (vmin, x, sizeof(*x) * d);
-
378  memcpy (vmax, x, sizeof(*x) * d);
-
379  for (size_t i = 1; i < n; i++) {
-
380  const float *xi = x + i * d;
-
381  for (size_t j = 0; j < d; j++) {
-
382  if (xi[j] < vmin[j]) vmin[j] = xi[j];
-
383  if (xi[j] > vmax[j]) vmax[j] = xi[j];
+
357  b = vmin;
+
358  a = (vmax - vmin) / (k - 1);
+
359  }
+
360  int verbose = false;
+
361  int niter = 2000;
+
362  float last_err = -1;
+
363  int iter_last_err = 0;
+
364  for (int it = 0; it < niter; it++) {
+
365  float sn = 0, sn2 = 0, sxn = 0, err1 = 0;
+
366 
+
367  for (idx_t i = 0; i < n; i++) {
+
368  float xi = x[i];
+
369  float ni = floor ((xi - b) / a + 0.5);
+
370  if (ni < 0) ni = 0;
+
371  if (ni >= k) ni = k - 1;
+
372  err1 += sqr (xi - (ni * a + b));
+
373  sn += ni;
+
374  sn2 += ni * ni;
+
375  sxn += ni * xi;
+
376  }
+
377 
+
378  if (err1 == last_err) {
+
379  iter_last_err ++;
+
380  if (iter_last_err == 16) break;
+
381  } else {
+
382  last_err = err1;
+
383  iter_last_err = 0;
384  }
-
385  }
-
386  float *vdiff = vmax;
-
387  for (size_t j = 0; j < d; j++) {
-
388  float vexp = (vmax[j] - vmin[j]) * rs_arg;
-
389  vmin[j] -= vexp;
-
390  vmax[j] += vexp;
-
391  vdiff [j] = vmax[j] - vmin[j];
-
392  }
-
393  } else {
-
394  // transpose
-
395  std::vector<float> xt(n * d);
-
396  for (size_t i = 1; i < n; i++) {
-
397  const float *xi = x + i * d;
-
398  for (size_t j = 0; j < d; j++) {
-
399  xt[j * n + i] = xi[j];
-
400  }
-
401  }
-
402  std::vector<float> trained_d(2);
-
403 #pragma omp parallel for
-
404  for (size_t j = 0; j < d; j++) {
-
405  train_Uniform(rs, rs_arg,
-
406  n, k, xt.data() + j * n,
-
407  trained_d);
-
408  vmin[j] = trained_d[0];
-
409  vmax[j] = trained_d[1];
-
410  }
-
411  }
-
412 }
-
413 
-
414 
-
415 /*******************************************************************
-
416  * Quantizer: normalizes scalar vector components, then passes them
-
417  * through a codec
-
418  */
-
419 
-
420 
-
421 
-
422 struct Quantizer {
-
423  virtual void encode_vector(const float *x, uint8_t *code) const = 0;
-
424  virtual void decode_vector(const uint8_t *code, float *x) const = 0;
-
425 
-
426  virtual float compute_distance_L2 (SimilarityL2 &sim,
-
427  const uint8_t * codes) const = 0;
-
428  virtual float compute_distance_IP (SimilarityIP &sim,
-
429  const uint8_t * codes) const = 0;
-
430 
-
431  virtual ~Quantizer() {}
-
432 };
-
433 
-
434 
-
435 
-
436 
-
437 template<class Codec>
-
438 struct QuantizerUniform: Quantizer {
-
439  const size_t d;
-
440  const float vmin, vdiff;
-
441 
-
442  QuantizerUniform(size_t d, const std::vector<float> &trained):
-
443  d(d), vmin(trained[0]), vdiff(trained[1]) {
-
444  }
-
445 
-
446  void encode_vector(const float* x, uint8_t* code) const override {
-
447  for (size_t i = 0; i < d; i++) {
-
448  float xi = (x[i] - vmin) / vdiff;
-
449  if (xi < 0)
-
450  xi = 0;
-
451  if (xi > 1.0)
-
452  xi = 1.0;
-
453  Codec::encode_component(xi, code, i);
-
454  }
-
455  }
-
456 
-
457  void decode_vector(const uint8_t* code, float* x) const override {
-
458  for (size_t i = 0; i < d; i++) {
-
459  float xi = Codec::decode_component(code, i);
-
460  x[i] = vmin + xi * vdiff;
-
461  }
-
462  }
+
385 
+
386  float det = sqr (sn) - sn2 * n;
+
387 
+
388  b = (sn * sxn - sn2 * sx) / det;
+
389  a = (sn * sx - n * sxn) / det;
+
390  if (verbose) {
+
391  printf ("it %d, err1=%g \r", it, err1);
+
392  fflush(stdout);
+
393  }
+
394  }
+
395  if (verbose) printf("\n");
+
396 
+
397  vmin = b;
+
398  vmax = b + a * (k - 1);
+
399 
+
400  } else {
+
401  FAISS_THROW_MSG ("Invalid qtype");
+
402  }
+
403  vmax -= vmin;
+
404 }
+
405 
+
406 void train_NonUniform(RangeStat rs, float rs_arg,
+
407  idx_t n, int d, int k, const float *x,
+
408  std::vector<float> & trained)
+
409 {
+
410  trained.resize (2 * d);
+
411  float * vmin = trained.data();
+
412  float * vmax = trained.data() + d;
+
413  if (rs == ScalarQuantizer::RS_minmax) {
+
414  memcpy (vmin, x, sizeof(*x) * d);
+
415  memcpy (vmax, x, sizeof(*x) * d);
+
416  for (size_t i = 1; i < n; i++) {
+
417  const float *xi = x + i * d;
+
418  for (size_t j = 0; j < d; j++) {
+
419  if (xi[j] < vmin[j]) vmin[j] = xi[j];
+
420  if (xi[j] > vmax[j]) vmax[j] = xi[j];
+
421  }
+
422  }
+
423  float *vdiff = vmax;
+
424  for (size_t j = 0; j < d; j++) {
+
425  float vexp = (vmax[j] - vmin[j]) * rs_arg;
+
426  vmin[j] -= vexp;
+
427  vmax[j] += vexp;
+
428  vdiff [j] = vmax[j] - vmin[j];
+
429  }
+
430  } else {
+
431  // transpose
+
432  std::vector<float> xt(n * d);
+
433  for (size_t i = 1; i < n; i++) {
+
434  const float *xi = x + i * d;
+
435  for (size_t j = 0; j < d; j++) {
+
436  xt[j * n + i] = xi[j];
+
437  }
+
438  }
+
439  std::vector<float> trained_d(2);
+
440 #pragma omp parallel for
+
441  for (size_t j = 0; j < d; j++) {
+
442  train_Uniform(rs, rs_arg,
+
443  n, k, xt.data() + j * n,
+
444  trained_d);
+
445  vmin[j] = trained_d[0];
+
446  vmax[j] = trained_d[1];
+
447  }
+
448  }
+
449 }
+
450 
+
451 
+
452 
+
453 /*******************************************************************
+
454  * Similarity: gets vector components and computes a similarity wrt. a
+
455  * query vector stored in the object. The data fields just encapsulate
+
456  * an accumulator.
+
457  */
+
458 
+
459 struct SimilarityL2 {
+
460  const float *y, *yi;
+
461 
+
462  explicit SimilarityL2 (const float * y): y(y) {}
463 
-
464  float reconstruct_component (const uint8_t * code, int i) const
-
465  {
-
466  float xi = Codec::decode_component (code, i);
-
467  return vmin + xi * vdiff;
-
468  }
-
469 
-
470 #ifdef USE_AVX
-
471  __m256 reconstruct_8_components (const uint8_t * code, int i) const
-
472  {
-
473  __m256 xi = Codec::decode_8_components (code, i);
-
474  return _mm256_set1_ps(vmin) + xi * _mm256_set1_ps (vdiff);
-
475  }
-
476 #endif
-
477 
-
478  float compute_distance_L2(SimilarityL2& sim, const uint8_t* codes)
-
479  const override {
-
480  return compute_distance(*this, sim, codes);
+
464  /******* scalar accumulator *******/
+
465 
+
466  float accu;
+
467 
+
468  void begin () {
+
469  accu = 0;
+
470  yi = y;
+
471  }
+
472 
+
473  void add_component (float x) {
+
474  float tmp = *yi++ - x;
+
475  accu += tmp * tmp;
+
476  }
+
477 
+
478  void add_component_2 (float x1, float x2) {
+
479  float tmp = x1 - x2;
+
480  accu += tmp * tmp;
481  }
482 
-
483  float compute_distance_IP(SimilarityIP& sim, const uint8_t* codes)
-
484  const override {
-
485  return compute_distance(*this, sim, codes);
-
486  }
-
487 };
-
488 
-
489 #ifdef USE_AVX
-
490 template<class Codec>
-
491 struct QuantizerUniform8: QuantizerUniform<Codec> {
-
492 
-
493  QuantizerUniform8 (size_t d, const std::vector<float> &trained):
-
494  QuantizerUniform<Codec> (d, trained) {}
-
495 
-
496  float compute_distance_L2(SimilarityL2& sim, const uint8_t* codes)
-
497  const override {
-
498  return compute_distance_8(*this, sim, codes);
-
499  }
-
500 
-
501  float compute_distance_IP(SimilarityIP& sim, const uint8_t* codes)
-
502  const override {
-
503  return compute_distance_8(*this, sim, codes);
-
504  }
-
505 };
-
506 #endif
-
507 
-
508 
-
509 
-
510 
-
511 
-
512 template<class Codec>
-
513 struct QuantizerNonUniform: Quantizer {
-
514  const size_t d;
-
515  const float *vmin, *vdiff;
-
516 
-
517  QuantizerNonUniform(size_t d, const std::vector<float> &trained):
-
518  d(d), vmin(trained.data()), vdiff(trained.data() + d) {}
+
483  float result () {
+
484  return accu;
+
485  }
+
486 
+
487 #ifdef USE_AVX
+
488  __m256 accu8;
+
489 
+
490  void begin_8 () {
+
491  accu8 = _mm256_setzero_ps();
+
492  yi = y;
+
493  }
+
494 
+
495  void add_8_components (__m256 x) {
+
496  __m256 yiv = _mm256_loadu_ps (yi);
+
497  yi += 8;
+
498  __m256 tmp = yiv - x;
+
499  accu8 += tmp * tmp;
+
500  }
+
501 
+
502  void add_8_components_2 (__m256 x, __m256 y) {
+
503  __m256 tmp = y - x;
+
504  accu8 += tmp * tmp;
+
505  }
+
506 
+
507  float result_8 () {
+
508  __m256 sum = _mm256_hadd_ps(accu8, accu8);
+
509  __m256 sum2 = _mm256_hadd_ps(sum, sum);
+
510  // now add the 0th and 4th component
+
511  return
+
512  _mm_cvtss_f32 (_mm256_castps256_ps128(sum2)) +
+
513  _mm_cvtss_f32 (_mm256_extractf128_ps(sum2, 1));
+
514  }
+
515 #endif
+
516 
+
517 };
+
518 
519 
-
520  void encode_vector(const float* x, uint8_t* code) const override {
-
521  for (size_t i = 0; i < d; i++) {
-
522  float xi = (x[i] - vmin[i]) / vdiff[i];
-
523  if (xi < 0)
-
524  xi = 0;
-
525  if (xi > 1.0)
-
526  xi = 1.0;
-
527  Codec::encode_component(xi, code, i);
-
528  }
-
529  }
-
530 
-
531  void decode_vector(const uint8_t* code, float* x) const override {
-
532  for (size_t i = 0; i < d; i++) {
-
533  float xi = Codec::decode_component(code, i);
-
534  x[i] = vmin[i] + xi * vdiff[i];
-
535  }
+
520 struct SimilarityIP {
+
521  const float *y, *yi;
+
522  /******* scalar accumulator *******/
+
523 
+
524  float accu;
+
525 
+
526  explicit SimilarityIP (const float * y):
+
527  y (y) {}
+
528 
+
529  void begin () {
+
530  accu = 0;
+
531  yi = y;
+
532  }
+
533 
+
534  void add_component (float x) {
+
535  accu += *yi++ * x;
536  }
537 
-
538  float reconstruct_component (const uint8_t * code, int i) const
-
539  {
-
540  float xi = Codec::decode_component (code, i);
-
541  return vmin[i] + xi * vdiff[i];
-
542  }
-
543 
-
544 #ifdef USE_AVX
-
545  __m256 reconstruct_8_components (const uint8_t * code, int i) const
-
546  {
-
547  __m256 xi = Codec::decode_8_components (code, i);
-
548  return _mm256_loadu_ps(vmin + i) + xi * _mm256_loadu_ps (vdiff + i);
-
549  }
-
550 #endif
-
551 
-
552  float compute_distance_L2(SimilarityL2& sim, const uint8_t* codes)
-
553  const override {
-
554  return compute_distance(*this, sim, codes);
-
555  }
-
556 
-
557  float compute_distance_IP(SimilarityIP& sim, const uint8_t* codes)
-
558  const override {
-
559  return compute_distance(*this, sim, codes);
-
560  }
-
561 };
-
562 
-
563 #ifdef USE_AVX
-
564 template<class Codec>
-
565 struct QuantizerNonUniform8: QuantizerNonUniform<Codec> {
-
566 
-
567  QuantizerNonUniform8 (size_t d, const std::vector<float> &trained):
-
568  QuantizerNonUniform<Codec> (d, trained) {}
-
569 
-
570  float compute_distance_L2(SimilarityL2& sim, const uint8_t* codes)
-
571  const override {
-
572  return compute_distance_8(*this, sim, codes);
-
573  }
-
574 
-
575  float compute_distance_IP(SimilarityIP& sim, const uint8_t* codes)
-
576  const override {
-
577  return compute_distance_8(*this, sim, codes);
-
578  }
-
579 };
-
580 #endif
-
581 
+
538  void add_component_2 (float x1, float x2) {
+
539  accu += x1 * x2;
+
540  }
+
541 
+
542  float result () {
+
543  return accu;
+
544  }
+
545 
+
546 #ifdef USE_AVX
+
547 
+
548  __m256 accu8;
+
549 
+
550  void begin_8 () {
+
551  accu8 = _mm256_setzero_ps();
+
552  yi = y;
+
553  }
+
554 
+
555  void add_8_components (__m256 x) {
+
556  __m256 yiv = _mm256_loadu_ps (yi);
+
557  yi += 8;
+
558  accu8 += yiv * x;
+
559  }
+
560 
+
561  void add_8_components_2 (__m256 x1, __m256 x2) {
+
562  accu8 += x1 * x2;
+
563  }
+
564 
+
565  float result_8 () {
+
566  __m256 sum = _mm256_hadd_ps(accu8, accu8);
+
567  __m256 sum2 = _mm256_hadd_ps(sum, sum);
+
568  // now add the 0th and 4th component
+
569  return
+
570  _mm_cvtss_f32 (_mm256_castps256_ps128(sum2)) +
+
571  _mm_cvtss_f32 (_mm256_extractf128_ps(sum2, 1));
+
572  }
+
573 #endif
+
574 };
+
575 
+
576 
+
577 /*******************************************************************
+
578  * DistanceComputer: combines a similarity and a quantizer to do
+
579  * code-to-vector or code-to-code comparisons
+
580  */
+
581 
582 
-
583 
-
584 
+
583 template<class Quantizer, class Similarity>
+
584 struct DCTemplate : ScalarQuantizer::DistanceComputer {
585 
-
586 Quantizer *select_quantizer (
-
587  QuantizerType qtype,
-
588  size_t d, const std::vector<float> & trained)
-
589 {
-
590 #ifdef USE_AVX
-
591  if (d % 8 == 0) {
-
592  switch(qtype) {
-
593  case ScalarQuantizer::QT_8bit:
-
594  return new QuantizerNonUniform8<Codec8bit>(d, trained);
-
595  case ScalarQuantizer::QT_4bit:
-
596  return new QuantizerNonUniform8<Codec4bit>(d, trained);
-
597  case ScalarQuantizer::QT_8bit_uniform:
-
598  return new QuantizerUniform8<Codec8bit>(d, trained);
-
599  case ScalarQuantizer::QT_4bit_uniform:
-
600  return new QuantizerUniform8<Codec4bit>(d, trained);
-
601  }
-
602  } else
-
603 #endif
-
604  {
-
605  switch(qtype) {
-
606  case ScalarQuantizer::QT_8bit:
-
607  return new QuantizerNonUniform<Codec8bit>(d, trained);
-
608  case ScalarQuantizer::QT_4bit:
-
609  return new QuantizerNonUniform<Codec4bit>(d, trained);
-
610  case ScalarQuantizer::QT_8bit_uniform:
-
611  return new QuantizerUniform<Codec8bit>(d, trained);
-
612  case ScalarQuantizer::QT_4bit_uniform:
-
613  return new QuantizerUniform<Codec4bit>(d, trained);
-
614  }
+
586  Quantizer quant;
+
587 
+
588  DCTemplate(size_t d, const std::vector<float> &trained):
+
589  quant(d, trained)
+
590  {}
+
591 
+
592  float compute_distance (const float *x,
+
593  const uint8_t *code) override
+
594  {
+
595  Similarity sim(x);
+
596  sim.begin();
+
597  for (size_t i = 0; i < quant.d; i ++) {
+
598  float xi = quant.reconstruct_component (code, i);
+
599  sim.add_component (xi);
+
600  }
+
601  return sim.result();
+
602  }
+
603 
+
604  float compute_code_distance (const uint8_t *code1,
+
605  const uint8_t *code2) override
+
606  {
+
607  Similarity sim(nullptr);
+
608  sim.begin ();
+
609  for (size_t i = 0; i < quant.d; i ++) {
+
610  float x1 = quant.reconstruct_component (code1, i);
+
611  float x2 = quant.reconstruct_component (code2, i);
+
612  sim.add_component_2 (x1, x2);
+
613  }
+
614  return sim.result ();
615  }
-
616  FAISS_THROW_MSG ("unknown qtype");
-
617  return nullptr;
-
618 }
-
619 
-
620 Quantizer *select_quantizer (const ScalarQuantizer &sq)
-
621 {
-
622  return select_quantizer (sq.qtype, sq.d, sq.trained);
-
623 }
-
624 
+
616 
+
617 };
+
618 
+
619 #ifdef USE_AVX
+
620 
+
621 template<class Quantizer, class Similarity>
+
622 struct DCTemplate_8 : ScalarQuantizer::DistanceComputer {
+
623 
+
624  Quantizer quant;
625 
-
626 } // anonymous namespace
-
627 
-
628 
+
626  DCTemplate_8(size_t d, const std::vector<float> &trained):
+
627  quant(d, trained)
+
628  {}
629 
-
630 /*******************************************************************
-
631  * ScalarQuantizer implementation
-
632  ********************************************************************/
-
633 
-
634 ScalarQuantizer::ScalarQuantizer
-
635  (size_t d, QuantizerType qtype):
-
636  qtype (qtype), rangestat(RS_minmax), rangestat_arg(0), d (d)
-
637 {
-
638  switch (qtype) {
-
639  case QT_8bit: case QT_8bit_uniform:
-
640  code_size = d;
-
641  break;
-
642  case QT_4bit: case QT_4bit_uniform:
-
643  code_size = (d + 1) / 2;
-
644  break;
-
645  }
-
646 
-
647 }
-
648 
-
649 ScalarQuantizer::ScalarQuantizer ():
-
650  qtype(QT_8bit),
-
651  rangestat(RS_minmax), rangestat_arg(0), d (0), code_size(0)
-
652 {}
-
653 
-
654 void ScalarQuantizer::train (size_t n, const float *x)
-
655 {
-
656  int bit_per_dim =
-
657  qtype == QT_4bit_uniform ? 4 :
-
658  qtype == QT_4bit ? 4 :
-
659  qtype == QT_8bit_uniform ? 8 :
-
660  qtype == QT_8bit ? 8 : -1;
+
630  float compute_distance (const float *x,
+
631  const uint8_t *code) override
+
632  {
+
633  Similarity sim(x);
+
634  sim.begin_8();
+
635  for (size_t i = 0; i < quant.d; i += 8) {
+
636  __m256 xi = quant.reconstruct_8_components (code, i);
+
637  sim.add_8_components (xi);
+
638  }
+
639  return sim.result_8();
+
640  }
+
641 
+
642  float compute_code_distance (const uint8_t *code1,
+
643  const uint8_t *code2) override
+
644  {
+
645  Similarity sim(nullptr);
+
646  sim.begin_8 ();
+
647  for (size_t i = 0; i < quant.d; i += 8) {
+
648  __m256 x1 = quant.reconstruct_8_components (code1, i);
+
649  __m256 x2 = quant.reconstruct_8_components (code2, i);
+
650  sim.add_8_components_2 (x1, x2);
+
651  }
+
652  return sim.result_8 ();
+
653  }
+
654 
+
655 };
+
656 
+
657 
+
658 #endif
+
659 
+
660 
661 
-
662  switch (qtype) {
-
663  case QT_4bit_uniform: case QT_8bit_uniform:
-
664  train_Uniform (rangestat, rangestat_arg,
-
665  n * d, 1 << bit_per_dim, x, trained);
-
666  break;
-
667  case QT_4bit: case QT_8bit:
-
668  train_NonUniform (rangestat, rangestat_arg,
-
669  n, d, 1 << bit_per_dim, x, trained);
-
670  break;
-
671  }
-
672 }
-
673 
-
674 void ScalarQuantizer::compute_codes (const float * x,
-
675  uint8_t * codes,
-
676  size_t n) const
-
677 {
-
678  Quantizer *squant = select_quantizer (*this);
-
679  ScopeDeleter1<Quantizer> del(squant);
-
680 #pragma omp parallel for
-
681  for (size_t i = 0; i < n; i++)
-
682  squant->encode_vector (x + i * d, codes + i * code_size);
-
683 }
-
684 
-
685 void ScalarQuantizer::decode (const uint8_t *codes, float *x, size_t n) const
-
686 {
-
687  Quantizer *squant = select_quantizer (*this);
-
688  ScopeDeleter1<Quantizer> del(squant);
-
689 #pragma omp parallel for
-
690  for (size_t i = 0; i < n; i++)
-
691  squant->decode_vector (codes + i * code_size, x + i * d);
-
692 }
-
693 
-
694 /*******************************************************************
-
695  * IndexScalarQuantizer implementation
-
696  ********************************************************************/
-
697 
-
698 IndexScalarQuantizer::IndexScalarQuantizer
-
699  (int d, ScalarQuantizer::QuantizerType qtype,
-
700  MetricType metric):
-
701  Index(d, metric),
-
702  sq (d, qtype)
-
703 {
-
704  is_trained = false;
-
705  code_size = sq.code_size;
-
706 }
+
662 template<class Sim>
+
663 DistanceComputer *select_distance_computer (
+
664  QuantizerType qtype,
+
665  size_t d, const std::vector<float> & trained)
+
666 {
+
667 #ifdef USE_AVX
+
668  if (d % 8 == 0) {
+
669  switch(qtype) {
+
670  case ScalarQuantizer::QT_8bit:
+
671  return new DCTemplate_8<QuantizerNonUniform8
+
672  <Codec8bit>, Sim>(d, trained);
+
673  case ScalarQuantizer::QT_4bit:
+
674  return new DCTemplate_8<QuantizerNonUniform8
+
675  <Codec4bit>, Sim>(d, trained);
+
676  case ScalarQuantizer::QT_8bit_uniform:
+
677  return new DCTemplate_8<QuantizerUniform8
+
678  <Codec8bit>, Sim>(d, trained);
+
679  case ScalarQuantizer::QT_4bit_uniform:
+
680  return new DCTemplate_8<QuantizerUniform8
+
681  <Codec4bit>, Sim>(d, trained);
+
682  }
+
683  } else
+
684 #endif
+
685  {
+
686  switch(qtype) {
+
687  case ScalarQuantizer::QT_8bit:
+
688  return new DCTemplate<QuantizerNonUniform
+
689  <Codec8bit>, Sim>(d, trained);
+
690  case ScalarQuantizer::QT_4bit:
+
691  return new DCTemplate<QuantizerNonUniform
+
692  <Codec4bit>, Sim>(d, trained);
+
693  case ScalarQuantizer::QT_8bit_uniform:
+
694  return new DCTemplate<QuantizerUniform
+
695  <Codec8bit>, Sim>(d, trained);
+
696  case ScalarQuantizer::QT_4bit_uniform:
+
697  return new DCTemplate<QuantizerUniform
+
698  <Codec4bit>, Sim>(d, trained);
+
699  }
+
700  }
+
701  FAISS_THROW_MSG ("unknown qtype");
+
702  return nullptr;
+
703 }
+
704 
+
705 
+
706 
707 
708 
-
709 IndexScalarQuantizer::IndexScalarQuantizer ():
-
710  IndexScalarQuantizer(0, ScalarQuantizer::QT_8bit)
-
711 {}
+
709 } // anonymous namespace
+
710 
+
711 
712 
-
713 void IndexScalarQuantizer::train(idx_t n, const float* x)
-
714 {
-
715  sq.train(n, x);
-
716  is_trained = true;
-
717 }
-
718 
-
719 void IndexScalarQuantizer::add(idx_t n, const float* x)
+
713 /*******************************************************************
+
714  * ScalarQuantizer implementation
+
715  ********************************************************************/
+
716 
+
717 ScalarQuantizer::ScalarQuantizer
+
718  (size_t d, QuantizerType qtype):
+
719  qtype (qtype), rangestat(RS_minmax), rangestat_arg(0), d (d)
720 {
-
721  FAISS_THROW_IF_NOT (is_trained);
-
722  codes.resize ((n + ntotal) * code_size);
-
723  sq.compute_codes (x, &codes[ntotal * code_size], n);
-
724  ntotal += n;
-
725 }
-
726 
-
727 void IndexScalarQuantizer::search(
-
728  idx_t n,
-
729  const float* x,
-
730  idx_t k,
-
731  float* distances,
-
732  idx_t* labels) const
-
733 {
-
734  Quantizer *squant = select_quantizer (sq);
-
735  ScopeDeleter1<Quantizer> del(squant);
-
736  FAISS_THROW_IF_NOT (is_trained);
-
737 
-
738  if (metric_type == METRIC_INNER_PRODUCT) {
-
739 #pragma omp parallel for
-
740  for (size_t i = 0; i < n; i++) {
-
741  idx_t *idxi = labels + i * k;
-
742  float *simi = distances + i * k;
-
743  minheap_heapify (k, simi, idxi);
+
721  switch (qtype) {
+
722  case QT_8bit: case QT_8bit_uniform:
+
723  code_size = d;
+
724  break;
+
725  case QT_4bit: case QT_4bit_uniform:
+
726  code_size = (d + 1) / 2;
+
727  break;
+
728  }
+
729 
+
730 }
+
731 
+
732 ScalarQuantizer::ScalarQuantizer ():
+
733  qtype(QT_8bit),
+
734  rangestat(RS_minmax), rangestat_arg(0), d (0), code_size(0)
+
735 {}
+
736 
+
737 void ScalarQuantizer::train (size_t n, const float *x)
+
738 {
+
739  int bit_per_dim =
+
740  qtype == QT_4bit_uniform ? 4 :
+
741  qtype == QT_4bit ? 4 :
+
742  qtype == QT_8bit_uniform ? 8 :
+
743  qtype == QT_8bit ? 8 : -1;
744 
-
745  SimilarityIP sim(x + i * d, 0);
-
746  const uint8_t *ci = codes.data ();
-
747 
-
748  for (size_t j = 0; j < ntotal; j++) {
-
749  float accu = squant->compute_distance_IP(sim, ci);
-
750 
-
751  if (accu > simi [0]) {
-
752  minheap_pop (k, simi, idxi);
-
753  minheap_push (k, simi, idxi, accu, j);
-
754  }
-
755  ci += code_size;
-
756  }
-
757  minheap_reorder (k, simi, idxi);
-
758  }
-
759  } else {
-
760 #pragma omp parallel for
-
761  for (size_t i = 0; i < n; i++) {
-
762  idx_t *idxi = labels + i * k;
-
763  float *simi = distances + i * k;
-
764  maxheap_heapify (k, simi, idxi);
-
765 
-
766  SimilarityL2 sim(x + i * d);
-
767  const uint8_t *ci = codes.data ();
-
768 
-
769  for (size_t j = 0; j < ntotal; j++) {
-
770  float accu = squant->compute_distance_L2(sim, ci);
-
771 
-
772  if (accu < simi [0]) {
-
773  maxheap_pop (k, simi, idxi);
-
774  maxheap_push (k, simi, idxi, accu, j);
-
775  }
-
776  ci += code_size;
-
777  }
-
778  maxheap_reorder (k, simi, idxi);
-
779  }
-
780  }
-
781 
-
782 }
-
783 
-
784 void IndexScalarQuantizer::reset()
-
785 {
-
786  codes.clear();
-
787  ntotal = 0;
-
788 }
+
745  switch (qtype) {
+
746  case QT_4bit_uniform: case QT_8bit_uniform:
+
747  train_Uniform (rangestat, rangestat_arg,
+
748  n * d, 1 << bit_per_dim, x, trained);
+
749  break;
+
750  case QT_4bit: case QT_8bit:
+
751  train_NonUniform (rangestat, rangestat_arg,
+
752  n, d, 1 << bit_per_dim, x, trained);
+
753  break;
+
754  }
+
755 }
+
756 
+
757 void ScalarQuantizer::compute_codes (const float * x,
+
758  uint8_t * codes,
+
759  size_t n) const
+
760 {
+
761  Quantizer *squant = select_quantizer (*this);
+
762  ScopeDeleter1<Quantizer> del(squant);
+
763 #pragma omp parallel for
+
764  for (size_t i = 0; i < n; i++)
+
765  squant->encode_vector (x + i * d, codes + i * code_size);
+
766 }
+
767 
+
768 void ScalarQuantizer::decode (const uint8_t *codes, float *x, size_t n) const
+
769 {
+
770  Quantizer *squant = select_quantizer (*this);
+
771  ScopeDeleter1<Quantizer> del(squant);
+
772 #pragma omp parallel for
+
773  for (size_t i = 0; i < n; i++)
+
774  squant->decode_vector (codes + i * code_size, x + i * d);
+
775 }
+
776 
+
777 
+
778 ScalarQuantizer::DistanceComputer *ScalarQuantizer::get_distance_computer (
+
779  MetricType metric)
+
780  const
+
781 {
+
782  if (metric == METRIC_L2) {
+
783  return select_distance_computer<SimilarityL2>(qtype, d, trained);
+
784  } else {
+
785  return select_distance_computer<SimilarityIP>(qtype, d, trained);
+
786  }
+
787 }
+
788 
789 
-
790 void IndexScalarQuantizer::reconstruct_n(
-
791  idx_t i0, idx_t ni, float* recons) const
-
792 {
-
793  Quantizer *squant = select_quantizer (sq);
-
794  ScopeDeleter1<Quantizer> del (squant);
-
795  for (size_t i = 0; i < ni; i++) {
-
796  squant->decode_vector(&codes[(i + i0) * code_size], recons + i * d);
-
797  }
-
798 }
-
799 
-
800 void IndexScalarQuantizer::reconstruct(idx_t key, float* recons) const
-
801 {
-
802  reconstruct_n(key, 1, recons);
-
803 }
+
790 /*******************************************************************
+
791  * IndexScalarQuantizer implementation
+
792  ********************************************************************/
+
793 
+
794 IndexScalarQuantizer::IndexScalarQuantizer
+
795  (int d, ScalarQuantizer::QuantizerType qtype,
+
796  MetricType metric):
+
797  Index(d, metric),
+
798  sq (d, qtype)
+
799 {
+
800  is_trained = false;
+
801  code_size = sq.code_size;
+
802 }
+
803 
804 
-
805 
-
806 /*******************************************************************
-
807  * IndexIVFScalarQuantizer implementation
-
808  ********************************************************************/
-
809 
-
810 IndexIVFScalarQuantizer::IndexIVFScalarQuantizer
-
811  (Index *quantizer, size_t d, size_t nlist,
-
812  QuantizerType qtype, MetricType metric):
-
813  IndexIVF (quantizer, d, nlist, metric),
-
814  sq (d, qtype)
-
815 {
-
816  code_size = sq.code_size;
-
817  is_trained = false;
-
818 }
-
819 
-
820 IndexIVFScalarQuantizer::IndexIVFScalarQuantizer ():
-
821  IndexIVF ()
-
822 {}
+
805 IndexScalarQuantizer::IndexScalarQuantizer ():
+
806  IndexScalarQuantizer(0, ScalarQuantizer::QT_8bit)
+
807 {}
+
808 
+
809 void IndexScalarQuantizer::train(idx_t n, const float* x)
+
810 {
+
811  sq.train(n, x);
+
812  is_trained = true;
+
813 }
+
814 
+
815 void IndexScalarQuantizer::add(idx_t n, const float* x)
+
816 {
+
817  FAISS_THROW_IF_NOT (is_trained);
+
818  codes.resize ((n + ntotal) * code_size);
+
819  sq.compute_codes (x, &codes[ntotal * code_size], n);
+
820  ntotal += n;
+
821 }
+
822 
823 
-
824 void IndexIVFScalarQuantizer::train_residual (idx_t n, const float *x)
-
825 {
-
826  long * idx = new long [n];
-
827  ScopeDeleter<long> del (idx);
-
828  quantizer->assign (n, x, idx);
-
829  float *residuals = new float [n * d];
-
830  ScopeDeleter<float> del2 (residuals);
-
831 
-
832 #pragma omp parallel for
-
833  for (idx_t i = 0; i < n; i++) {
-
834  quantizer->compute_residual (x + i * d, residuals + i * d, idx[i]);
-
835  }
-
836 
-
837  sq.train (n, residuals);
+
824 
+
825 namespace {
+
826 
+
827 template<class C>
+
828 void search_flat_scalar_quantizer(
+
829  const IndexScalarQuantizer & index,
+
830  idx_t n,
+
831  const float* x,
+
832  idx_t k,
+
833  float* distances,
+
834  idx_t* labels)
+
835 {
+
836  size_t code_size = index.code_size;
+
837  size_t d = index.d;
838 
-
839 }
-
840 
-
841 
-
842 void IndexIVFScalarQuantizer::add_with_ids
-
843  (idx_t n, const float * x, const long *xids)
-
844 {
-
845  FAISS_THROW_IF_NOT (is_trained);
-
846  long * idx = new long [n];
-
847  ScopeDeleter<long> del (idx);
-
848  quantizer->assign (n, x, idx);
-
849  size_t nadd = 0;
-
850  Quantizer *squant = select_quantizer (sq);
-
851  ScopeDeleter1<Quantizer> del2 (squant);
-
852 
-
853 #pragma omp parallel reduction(+: nadd)
-
854  {
-
855  std::vector<float> residual (d);
-
856  int nt = omp_get_num_threads();
-
857  int rank = omp_get_thread_num();
-
858 
-
859  // each thread takes care of a subset of lists
-
860  for (size_t i = 0; i < n; i++) {
-
861 
-
862  long list_no = idx [i];
-
863  if (list_no >= 0 && list_no % nt == rank) {
-
864  long id = xids ? xids[i] : ntotal + i;
+
839 #pragma omp parallel
+
840  {
+
841  DistanceComputer *dc =
+
842  index.sq.get_distance_computer(index.metric_type);
+
843  ScopeDeleter1<DistanceComputer> del(dc);
+
844 
+
845 #pragma omp for
+
846  for (size_t i = 0; i < n; i++) {
+
847  idx_t *idxi = labels + i * k;
+
848  float *simi = distances + i * k;
+
849  heap_heapify<C> (k, simi, idxi);
+
850 
+
851  const float *xi = x + i * d;
+
852  const uint8_t *ci = index.codes.data ();
+
853 
+
854  for (size_t j = 0; j < index.ntotal; j++) {
+
855  float accu = dc->compute_distance(xi, ci);
+
856  if (C::cmp (simi [0], accu)) {
+
857  heap_pop<C> (k, simi, idxi);
+
858  heap_push<C> (k, simi, idxi, accu, j);
+
859  }
+
860  ci += code_size;
+
861  }
+
862  heap_reorder<C> (k, simi, idxi);
+
863  }
+
864  }
865 
-
866  assert (list_no < nlist);
+
866 };
867 
-
868  ids[list_no].push_back (id);
-
869  nadd++;
-
870  quantizer->compute_residual (
-
871  x + i * d, residual.data(), list_no);
-
872 
-
873  size_t cur_size = codes[list_no].size();
-
874  codes[list_no].resize (cur_size + code_size);
-
875 
-
876  squant->encode_vector (residual.data(),
-
877  codes[list_no].data() + cur_size);
-
878  }
-
879  }
-
880  }
-
881  ntotal += nadd;
-
882 }
-
883 
-
884 namespace {
-
885 
-
886 void search_with_probes_ip (const IndexIVFScalarQuantizer & index,
-
887  const float *x,
-
888  const idx_t *cent_ids, const float *cent_dis,
-
889  const Quantizer & quant,
-
890  int k, float *simi, idx_t *idxi,
-
891  bool store_pairs)
-
892 {
-
893  int nprobe = index.nprobe;
-
894  size_t code_size = index.code_size;
-
895  size_t d = index.d;
-
896  std::vector<float> decoded(d);
-
897  minheap_heapify (k, simi, idxi);
-
898  for (int i = 0; i < nprobe; i++) {
-
899  idx_t list_no = cent_ids[i];
-
900  if (list_no < 0) break;
-
901  float accu0 = cent_dis[i];
-
902 
-
903  const std::vector<idx_t> & ids = index.ids[list_no];
-
904  const uint8_t* codes = index.codes[list_no].data();
+
868 }
+
869 
+
870 void IndexScalarQuantizer::search(
+
871  idx_t n,
+
872  const float* x,
+
873  idx_t k,
+
874  float* distances,
+
875  idx_t* labels) const
+
876 {
+
877  FAISS_THROW_IF_NOT (is_trained);
+
878  if (metric_type == METRIC_L2) {
+
879  search_flat_scalar_quantizer<CMax<float, idx_t> > (*this, n, x, k, distances, labels);
+
880  } else {
+
881  search_flat_scalar_quantizer<CMin<float, idx_t> > (*this, n, x, k, distances, labels);
+
882  }
+
883 }
+
884 
+
885 void IndexScalarQuantizer::reset()
+
886 {
+
887  codes.clear();
+
888  ntotal = 0;
+
889 }
+
890 
+
891 void IndexScalarQuantizer::reconstruct_n(
+
892  idx_t i0, idx_t ni, float* recons) const
+
893 {
+
894  Quantizer *squant = select_quantizer (sq);
+
895  ScopeDeleter1<Quantizer> del (squant);
+
896  for (size_t i = 0; i < ni; i++) {
+
897  squant->decode_vector(&codes[(i + i0) * code_size], recons + i * d);
+
898  }
+
899 }
+
900 
+
901 void IndexScalarQuantizer::reconstruct(idx_t key, float* recons) const
+
902 {
+
903  reconstruct_n(key, 1, recons);
+
904 }
905 
-
906  SimilarityIP sim(x, accu0);
-
907 
-
908  for (size_t j = 0; j < ids.size(); j++) {
-
909 
-
910  float accu = quant.compute_distance_IP(sim, codes);
-
911 
-
912  if (accu > simi [0]) {
-
913  minheap_pop (k, simi, idxi);
-
914  long id = store_pairs ? (list_no << 32 | j) : ids[j];
-
915  minheap_push (k, simi, idxi, accu, id);
-
916  }
-
917  codes += code_size;
-
918  }
-
919 
-
920  }
-
921  minheap_reorder (k, simi, idxi);
-
922 }
-
923 
-
924 void search_with_probes_L2 (const IndexIVFScalarQuantizer & index,
-
925  const float *x_in,
-
926  const idx_t *cent_ids,
-
927  const Index *quantizer,
-
928  const Quantizer & quant,
-
929  int k, float *simi, idx_t *idxi,
-
930  bool store_pairs)
-
931 {
-
932  int nprobe = index.nprobe;
-
933  size_t code_size = index.code_size;
-
934  size_t d = index.d;
-
935  std::vector<float> decoded(d), x(d);
-
936  maxheap_heapify (k, simi, idxi);
-
937  for (int i = 0; i < nprobe; i++) {
-
938  idx_t list_no = cent_ids[i];
-
939  if (list_no < 0) break;
-
940 
-
941  const std::vector<idx_t> & ids = index.ids[list_no];
-
942  const uint8_t* codes = index.codes[list_no].data();
-
943 
-
944  // shift of x_in wrt centroid
-
945  quantizer->compute_residual (x_in, x.data(), list_no);
-
946 
-
947  SimilarityL2 sim(x.data());
-
948 
-
949  for (size_t j = 0; j < ids.size(); j++) {
-
950 
-
951  float dis = quant.compute_distance_L2 (sim, codes);
-
952 
-
953  if (dis < simi [0]) {
-
954  maxheap_pop (k, simi, idxi);
-
955  long id = store_pairs ? (list_no << 32 | j) : ids[j];
-
956  maxheap_push (k, simi, idxi, dis, id);
-
957  }
-
958  codes += code_size;
-
959  }
-
960  }
-
961  maxheap_reorder (k, simi, idxi);
-
962 }
-
963 
-
964 } // anonymous namespace
-
965 
-
966 void IndexIVFScalarQuantizer::search_preassigned (
-
967  idx_t n, const float *x, idx_t k,
-
968  const idx_t *idx,
-
969  const float *dis,
-
970  float *distances, idx_t *labels,
-
971  bool store_pairs) const
-
972 {
-
973  FAISS_THROW_IF_NOT (is_trained);
-
974 
-
975  Quantizer *squant = select_quantizer (sq);
-
976  ScopeDeleter1<Quantizer> del(squant);
-
977 
-
978  if (metric_type == METRIC_INNER_PRODUCT) {
-
979 #pragma omp parallel for
-
980  for (size_t i = 0; i < n; i++) {
-
981  search_with_probes_ip (*this, x + i * d,
-
982  idx + i * nprobe, dis + i * nprobe, *squant,
-
983  k, distances + i * k, labels + i * k,
-
984  store_pairs);
-
985  }
-
986  } else {
-
987 #pragma omp parallel for
-
988  for (size_t i = 0; i < n; i++) {
-
989  search_with_probes_L2 (*this, x + i * d,
-
990  idx + i * nprobe, quantizer, *squant,
-
991  k, distances + i * k, labels + i * k,
-
992  store_pairs);
-
993  }
-
994  }
-
995 
-
996 }
-
997 
-
998 
-
999 }
- +
906 
+
907 /*******************************************************************
+
908  * IndexIVFScalarQuantizer implementation
+
909  ********************************************************************/
+
910 
+
911 IndexIVFScalarQuantizer::IndexIVFScalarQuantizer
+
912  (Index *quantizer, size_t d, size_t nlist,
+
913  QuantizerType qtype, MetricType metric):
+
914  IndexIVF (quantizer, d, nlist, metric),
+
915  sq (d, qtype)
+
916 {
+
917  code_size = sq.code_size;
+
918  is_trained = false;
+
919 }
+
920 
+
921 IndexIVFScalarQuantizer::IndexIVFScalarQuantizer ():
+
922  IndexIVF ()
+
923 {}
+
924 
+
925 void IndexIVFScalarQuantizer::train_residual (idx_t n, const float *x)
+
926 {
+
927  long * idx = new long [n];
+
928  ScopeDeleter<long> del (idx);
+
929  quantizer->assign (n, x, idx);
+
930  float *residuals = new float [n * d];
+
931  ScopeDeleter<float> del2 (residuals);
+
932 
+
933 #pragma omp parallel for
+
934  for (idx_t i = 0; i < n; i++) {
+
935  quantizer->compute_residual (x + i * d, residuals + i * d, idx[i]);
+
936  }
+
937 
+
938  sq.train (n, residuals);
+
939 
+
940 }
+
941 
+
942 
+
943 void IndexIVFScalarQuantizer::add_with_ids
+
944  (idx_t n, const float * x, const long *xids)
+
945 {
+
946  FAISS_THROW_IF_NOT (is_trained);
+
947  long * idx = new long [n];
+
948  ScopeDeleter<long> del (idx);
+
949  quantizer->assign (n, x, idx);
+
950  size_t nadd = 0;
+
951  Quantizer *squant = select_quantizer (sq);
+
952  ScopeDeleter1<Quantizer> del2 (squant);
+
953 
+
954 #pragma omp parallel reduction(+: nadd)
+
955  {
+
956  std::vector<float> residual (d);
+
957  int nt = omp_get_num_threads();
+
958  int rank = omp_get_thread_num();
+
959 
+
960  // each thread takes care of a subset of lists
+
961  for (size_t i = 0; i < n; i++) {
+
962 
+
963  long list_no = idx [i];
+
964  if (list_no >= 0 && list_no % nt == rank) {
+
965  long id = xids ? xids[i] : ntotal + i;
+
966 
+
967  assert (list_no < nlist);
+
968 
+
969  ids[list_no].push_back (id);
+
970  nadd++;
+
971  quantizer->compute_residual (
+
972  x + i * d, residual.data(), list_no);
+
973 
+
974  size_t cur_size = codes[list_no].size();
+
975  codes[list_no].resize (cur_size + code_size);
+
976 
+
977  squant->encode_vector (residual.data(),
+
978  codes[list_no].data() + cur_size);
+
979  }
+
980  }
+
981  }
+
982  ntotal += nadd;
+
983 }
+
984 
+
985 namespace {
+
986 
+
987 
+
988 void search_with_probes_ip (const IndexIVFScalarQuantizer & index,
+
989  const float *x,
+
990  const idx_t *cent_ids, const float *cent_dis,
+
991  DistanceComputer & dc,
+
992  int k, float *simi, idx_t *idxi,
+
993  bool store_pairs)
+
994 {
+
995  int nprobe = index.nprobe;
+
996  size_t code_size = index.code_size;
+
997  size_t d = index.d;
+
998  std::vector<float> decoded(d);
+
999  minheap_heapify (k, simi, idxi);
+
1000  size_t nscan = 0;
+
1001  for (int i = 0; i < nprobe; i++) {
+
1002  idx_t list_no = cent_ids[i];
+
1003  if (list_no < 0) break;
+
1004  float accu0 = cent_dis[i];
+
1005 
+
1006  const std::vector<idx_t> & ids = index.ids[list_no];
+
1007  const uint8_t* codes = index.codes[list_no].data();
+
1008 
+
1009  SimilarityIP sim(x);
+
1010 
+
1011  for (size_t j = 0; j < ids.size(); j++) {
+
1012 
+
1013  float accu = accu0 + dc.compute_distance(x, codes);
+
1014 
+
1015  if (accu > simi [0]) {
+
1016  minheap_pop (k, simi, idxi);
+
1017  long id = store_pairs ? (list_no << 32 | j) : ids[j];
+
1018  minheap_push (k, simi, idxi, accu, id);
+
1019  }
+
1020  codes += code_size;
+
1021  }
+
1022  nscan += ids.size();
+
1023  if (index.max_codes && nscan > index.max_codes)
+
1024  break;
+
1025  }
+
1026  minheap_reorder (k, simi, idxi);
+
1027 }
+
1028 
+
1029 void search_with_probes_L2 (const IndexIVFScalarQuantizer & index,
+
1030  const float *x_in,
+
1031  const idx_t *cent_ids,
+
1032  const Index *quantizer,
+
1033  DistanceComputer & dc,
+
1034  int k, float *simi, idx_t *idxi,
+
1035  bool store_pairs)
+
1036 {
+
1037  int nprobe = index.nprobe;
+
1038  size_t code_size = index.code_size;
+
1039  size_t d = index.d;
+
1040  std::vector<float> x(d);
+
1041  maxheap_heapify (k, simi, idxi);
+
1042  size_t nscan = 0;
+
1043  for (int i = 0; i < nprobe; i++) {
+
1044  idx_t list_no = cent_ids[i];
+
1045  if (list_no < 0) break;
+
1046 
+
1047  const std::vector<idx_t> & ids = index.ids[list_no];
+
1048  const uint8_t* codes = index.codes[list_no].data();
+
1049 
+
1050  // shift of x_in wrt centroid
+
1051  quantizer->compute_residual (x_in, x.data(), list_no);
+
1052 
+
1053  for (size_t j = 0; j < ids.size(); j++) {
+
1054 
+
1055  float dis = dc.compute_distance (x.data(), codes);
+
1056 
+
1057  if (dis < simi [0]) {
+
1058  maxheap_pop (k, simi, idxi);
+
1059  long id = store_pairs ? (list_no << 32 | j) : ids[j];
+
1060  maxheap_push (k, simi, idxi, dis, id);
+
1061  }
+
1062  codes += code_size;
+
1063  }
+
1064  nscan += ids.size();
+
1065  if (index.max_codes && nscan > index.max_codes)
+
1066  break;
+
1067  }
+
1068  maxheap_reorder (k, simi, idxi);
+
1069 }
+
1070 
+
1071 } // anonymous namespace
+
1072 
+
1073 void IndexIVFScalarQuantizer::search_preassigned (
+
1074  idx_t n, const float *x, idx_t k,
+
1075  const idx_t *idx,
+
1076  const float *dis,
+
1077  float *distances, idx_t *labels,
+
1078  bool store_pairs) const
+
1079 {
+
1080  FAISS_THROW_IF_NOT (is_trained);
+
1081 
+
1082 
+
1083  if (metric_type == METRIC_INNER_PRODUCT) {
+
1084 #pragma omp parallel
+
1085  {
+
1086  DistanceComputer *dc = sq.get_distance_computer (metric_type);
+
1087  ScopeDeleter1<DistanceComputer> del(dc);
+
1088 #pragma omp for
+
1089  for (size_t i = 0; i < n; i++) {
+
1090  search_with_probes_ip (*this, x + i * d,
+
1091  idx + i * nprobe, dis + i * nprobe, *dc,
+
1092  k, distances + i * k, labels + i * k,
+
1093  store_pairs);
+
1094  }
+
1095  }
+
1096  } else {
+
1097 #pragma omp parallel
+
1098  {
+
1099  DistanceComputer *dc = sq.get_distance_computer (metric_type);
+
1100  ScopeDeleter1<DistanceComputer> del(dc);
+
1101 #pragma omp for
+
1102  for (size_t i = 0; i < n; i++) {
+
1103  search_with_probes_L2 (*this, x + i * d,
+
1104  idx + i * nprobe, quantizer, *dc,
+
1105  k, distances + i * k, labels + i * k,
+
1106  store_pairs);
+
1107  }
+
1108  }
+
1109  }
+
1110 
+
1111 }
+
1112 
+
1113 void IndexIVFScalarQuantizer::reconstruct_from_offset (long list_no,
+
1114  long offset,
+
1115  float* recons) const
+
1116 {
+
1117  std::vector<float> centroid(d);
+
1118  quantizer->reconstruct (list_no, centroid.data());
+
1119 
+
1120  const uint8_t* code = &(codes[list_no][offset * code_size]);
+
1121  sq.decode (code, recons, 1);
+
1122  for (int i = 0; i < d; ++i) {
+
1123  recons[i] += centroid[i];
+
1124  }
+
1125 }
+
1126 
+
1127 }
+
faiss::ScalarQuantizer::code_size
size_t code_size
bytes per vector
Definition: IndexScalarQuantizer.h:60
-
faiss::IndexScalarQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexScalarQuantizer.cpp:727
-
faiss::IndexIVFScalarQuantizer::train_residual
void train_residual(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:824
+
faiss::IndexScalarQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexScalarQuantizer.cpp:870
+
faiss::IndexIVFScalarQuantizer::train_residual
void train_residual(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:925
faiss::ScalarQuantizer::RS_optim
alternate optimization of reconstruction error
Definition: IndexScalarQuantizer.h:50
- +
faiss::ScalarQuantizer::QT_8bit_uniform
same, shared range for all dimensions
Definition: IndexScalarQuantizer.h:35
-
faiss::IndexScalarQuantizer::add
void add(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:719
-
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:47
+
faiss::IndexIVFScalarQuantizer::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexScalarQuantizer.cpp:1113
+
faiss::IndexScalarQuantizer::add
void add(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:815
+
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:78
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
-
faiss::IndexScalarQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexScalarQuantizer.cpp:784
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
+
faiss::IndexScalarQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexScalarQuantizer.cpp:885
-
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:62
-
faiss::IndexIVFScalarQuantizer::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexScalarQuantizer.cpp:843
+
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:81
+
faiss::IndexIVFScalarQuantizer::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexScalarQuantizer.cpp:944
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
-
faiss::IndexScalarQuantizer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexScalarQuantizer.h:87
-
faiss::IndexScalarQuantizer::sq
ScalarQuantizer sq
Used to encode the vectors.
Definition: IndexScalarQuantizer.h:84
+
faiss::IndexScalarQuantizer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexScalarQuantizer.h:104
+ +
faiss::IndexScalarQuantizer::sq
ScalarQuantizer sq
Used to encode the vectors.
Definition: IndexScalarQuantizer.h:101
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
- + -
faiss::IndexIVFScalarQuantizer::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexScalarQuantizer.cpp:966
+
faiss::IndexIVFScalarQuantizer::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexScalarQuantizer.cpp:1073
faiss::ScalarQuantizer::RS_meanstd
[mean - std * rs, mean + std * rs]
Definition: IndexScalarQuantizer.h:48
-
faiss::ScalarQuantizer::decode
void decode(const uint8_t *code, float *x, size_t n) const
decode a vector from a given code (or n vectors if third argument)
Definition: IndexScalarQuantizer.cpp:685
-
faiss::ScalarQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: IndexScalarQuantizer.cpp:674
+
faiss::ScalarQuantizer::decode
void decode(const uint8_t *code, float *x, size_t n) const
decode a vector from a given code (or n vectors if third argument)
Definition: IndexScalarQuantizer.cpp:768
+
faiss::ScalarQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: IndexScalarQuantizer.cpp:757
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
-
faiss::IndexScalarQuantizer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexScalarQuantizer.cpp:790
-
faiss::IndexScalarQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexScalarQuantizer.cpp:800
+
faiss::IndexScalarQuantizer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexScalarQuantizer.cpp:891
+
faiss::IndexScalarQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexScalarQuantizer.cpp:901
faiss::ScalarQuantizer::RS_minmax
[min - rs*(max-min), max + rs*(max-min)]
Definition: IndexScalarQuantizer.h:47
+
faiss::ScalarQuantizer::trained
std::vector< float > trained
trained values (including the range)
Definition: IndexScalarQuantizer.h:63
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:56
-
faiss::IndexScalarQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:713
-
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:64
+
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:86
+
faiss::IndexIVF::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVF.h:79
+
faiss::IndexScalarQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:809
+
faiss::ScalarQuantizer::d
size_t d
dimension of input vectors
Definition: IndexScalarQuantizer.h:57
+
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:83
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
diff --git a/docs/html/IndexScalarQuantizer_8h_source.html b/docs/html/IndexScalarQuantizer_8h_source.html index e07841614..d97c5a907 100644 --- a/docs/html/IndexScalarQuantizer_8h_source.html +++ b/docs/html/IndexScalarQuantizer_8h_source.html @@ -159,120 +159,144 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
74  size_t n) const ;
75 
76  /// decode a vector from a given code (or n vectors if third argument)
-
77  void decode (const uint8_t *code, float *x, size_t n) const;
+
77  void decode (const uint8_t *code, float *x, size_t n) const;
78 
-
79 };
-
80 
-
81 
-
82 struct IndexScalarQuantizer: Index {
-
83  /// Used to encode the vectors
-
84  ScalarQuantizer sq;
-
85 
-
86  /// Codes. Size ntotal * pq.code_size
-
87  std::vector<uint8_t> codes;
-
88 
-
89  size_t code_size;
-
90 
-
91  /** Constructor.
-
92  *
-
93  * @param d dimensionality of the input vectors
-
94  * @param M number of subquantizers
-
95  * @param nbits number of bit per subvector index
-
96  */
-
97  IndexScalarQuantizer (int d,
-
98  ScalarQuantizer::QuantizerType qtype,
-
99  MetricType metric = METRIC_L2);
-
100 
-
101  IndexScalarQuantizer ();
-
102 
-
103  void train(idx_t n, const float* x) override;
-
104 
-
105  void add(idx_t n, const float* x) override;
-
106 
-
107  void search(
-
108  idx_t n,
-
109  const float* x,
-
110  idx_t k,
-
111  float* distances,
-
112  idx_t* labels) const override;
-
113 
-
114  void reset() override;
-
115 
-
116  void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;
+
79  // fast, non thread-safe way of computing vector-to-code and
+
80  // code-to-code distances.
+
81  struct DistanceComputer {
+
82 
+
83  /// vector-to-code distance computation
+
84  virtual float compute_distance (const float *x,
+
85  const uint8_t *code) = 0;
+
86 
+
87  /// code-to-code distance computation
+
88  virtual float compute_code_distance (const uint8_t *code1,
+
89  const uint8_t *code2) = 0;
+
90  virtual ~DistanceComputer () {}
+
91  };
+
92 
+
93  DistanceComputer *get_distance_computer (MetricType metric = METRIC_L2)
+
94  const;
+
95 
+
96 };
+
97 
+
98 
+
99 struct IndexScalarQuantizer: Index {
+
100  /// Used to encode the vectors
+
101  ScalarQuantizer sq;
+
102 
+
103  /// Codes. Size ntotal * pq.code_size
+
104  std::vector<uint8_t> codes;
+
105 
+
106  size_t code_size;
+
107 
+
108  /** Constructor.
+
109  *
+
110  * @param d dimensionality of the input vectors
+
111  * @param M number of subquantizers
+
112  * @param nbits number of bit per subvector index
+
113  */
+
114  IndexScalarQuantizer (int d,
+
115  ScalarQuantizer::QuantizerType qtype,
+
116  MetricType metric = METRIC_L2);
117 
-
118  void reconstruct(idx_t key, float* recons) const override;
+
118  IndexScalarQuantizer ();
119 
-
120 };
+
120  void train(idx_t n, const float* x) override;
121 
-
122 
-
123  /** An IVF implementation where the components of the residuals are
-
124  * encoded with a scalar uniform quantizer. All distance computations
-
125  * are asymmetric, so the encoded vectors are decoded and approximate
-
126  * distances are computed.
-
127  */
-
128 
-
129 struct IndexIVFScalarQuantizer:IndexIVF {
-
130  ScalarQuantizer sq;
-
131 
-
132  IndexIVFScalarQuantizer(Index *quantizer, size_t d, size_t nlist,
-
133  ScalarQuantizer::QuantizerType qtype,
-
134  MetricType metric = METRIC_L2);
-
135 
-
136  IndexIVFScalarQuantizer();
-
137 
-
138  void train_residual(idx_t n, const float* x) override;
-
139 
-
140  void add_with_ids(idx_t n, const float* x, const long* xids) override;
-
141 
-
142  void search_preassigned (idx_t n, const float *x, idx_t k,
-
143  const idx_t *assign,
-
144  const float *centroid_dis,
-
145  float *distances, idx_t *labels,
-
146  bool store_pairs) const override;
-
147 
-
148 };
-
149 
-
150 
-
151 }
+
122  void add(idx_t n, const float* x) override;
+
123 
+
124  void search(
+
125  idx_t n,
+
126  const float* x,
+
127  idx_t k,
+
128  float* distances,
+
129  idx_t* labels) const override;
+
130 
+
131  void reset() override;
+
132 
+
133  void reconstruct_n(idx_t i0, idx_t ni, float* recons) const override;
+
134 
+
135  void reconstruct(idx_t key, float* recons) const override;
+
136 
+
137 };
+
138 
+
139 
+
140  /** An IVF implementation where the components of the residuals are
+
141  * encoded with a scalar uniform quantizer. All distance computations
+
142  * are asymmetric, so the encoded vectors are decoded and approximate
+
143  * distances are computed.
+
144  */
+
145 
+
146 struct IndexIVFScalarQuantizer: IndexIVF {
+
147  ScalarQuantizer sq;
+
148 
+
149  IndexIVFScalarQuantizer(Index *quantizer, size_t d, size_t nlist,
+
150  ScalarQuantizer::QuantizerType qtype,
+
151  MetricType metric = METRIC_L2);
152 
-
153 
-
154 #endif
- +
153  IndexIVFScalarQuantizer();
+
154 
+
155  void train_residual(idx_t n, const float* x) override;
+
156 
+
157  void add_with_ids(idx_t n, const float* x, const long* xids) override;
+
158 
+
159  void search_preassigned (idx_t n, const float *x, idx_t k,
+
160  const idx_t *assign,
+
161  const float *centroid_dis,
+
162  float *distances, idx_t *labels,
+
163  bool store_pairs) const override;
+
164 
+
165  void reconstruct_from_offset (long list_no, long offset,
+
166  float* recons) const override;
+
167 
+
168 };
+
169 
+
170 
+
171 }
+
172 
+
173 
+
174 #endif
+
faiss::ScalarQuantizer::code_size
size_t code_size
bytes per vector
Definition: IndexScalarQuantizer.h:60
-
faiss::IndexScalarQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexScalarQuantizer.cpp:727
-
faiss::IndexIVFScalarQuantizer::train_residual
void train_residual(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:824
+
faiss::IndexScalarQuantizer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexScalarQuantizer.cpp:870
+
faiss::IndexIVFScalarQuantizer::train_residual
void train_residual(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:925
faiss::ScalarQuantizer::RS_optim
alternate optimization of reconstruction error
Definition: IndexScalarQuantizer.h:50
- +
faiss::ScalarQuantizer::QT_8bit_uniform
same, shared range for all dimensions
Definition: IndexScalarQuantizer.h:35
-
faiss::IndexScalarQuantizer::add
void add(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:719
+
faiss::IndexIVFScalarQuantizer::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexScalarQuantizer.cpp:1113
+
faiss::IndexScalarQuantizer::add
void add(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:815
+
faiss::ScalarQuantizer::DistanceComputer::compute_code_distance
virtual float compute_code_distance(const uint8_t *code1, const uint8_t *code2)=0
code-to-code distance computation
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
-
faiss::IndexScalarQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexScalarQuantizer.cpp:784
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
+
faiss::IndexScalarQuantizer::reset
void reset() override
removes all elements from the database.
Definition: IndexScalarQuantizer.cpp:885
-
faiss::IndexIVFScalarQuantizer::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexScalarQuantizer.cpp:843
+
faiss::IndexIVFScalarQuantizer::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexScalarQuantizer.cpp:944
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
-
faiss::IndexScalarQuantizer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexScalarQuantizer.h:87
-
faiss::IndexScalarQuantizer::sq
ScalarQuantizer sq
Used to encode the vectors.
Definition: IndexScalarQuantizer.h:84
+
faiss::IndexScalarQuantizer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * pq.code_size.
Definition: IndexScalarQuantizer.h:104
+ +
faiss::IndexScalarQuantizer::sq
ScalarQuantizer sq
Used to encode the vectors.
Definition: IndexScalarQuantizer.h:101
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
- + -
faiss::IndexIVFScalarQuantizer::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexScalarQuantizer.cpp:966
+
faiss::IndexIVFScalarQuantizer::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexScalarQuantizer.cpp:1073
faiss::ScalarQuantizer::RS_meanstd
[mean - std * rs, mean + std * rs]
Definition: IndexScalarQuantizer.h:48
-
faiss::ScalarQuantizer::decode
void decode(const uint8_t *code, float *x, size_t n) const
decode a vector from a given code (or n vectors if third argument)
Definition: IndexScalarQuantizer.cpp:685
-
faiss::ScalarQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: IndexScalarQuantizer.cpp:674
-
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
+
faiss::ScalarQuantizer::decode
void decode(const uint8_t *code, float *x, size_t n) const
decode a vector from a given code (or n vectors if third argument)
Definition: IndexScalarQuantizer.cpp:768
+
faiss::ScalarQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: IndexScalarQuantizer.cpp:757
-
faiss::IndexScalarQuantizer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexScalarQuantizer.cpp:790
-
faiss::IndexScalarQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexScalarQuantizer.cpp:800
+
faiss::IndexScalarQuantizer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexScalarQuantizer.cpp:891
+
faiss::IndexScalarQuantizer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexScalarQuantizer.cpp:901
faiss::ScalarQuantizer::RS_minmax
[min - rs*(max-min), max + rs*(max-min)]
Definition: IndexScalarQuantizer.h:47
faiss::ScalarQuantizer::trained
std::vector< float > trained
trained values (including the range)
Definition: IndexScalarQuantizer.h:63
-
faiss::IndexScalarQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:713
+
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
+
faiss::IndexScalarQuantizer::train
void train(idx_t n, const float *x) override
Definition: IndexScalarQuantizer.cpp:809
faiss::ScalarQuantizer::d
size_t d
dimension of input vectors
Definition: IndexScalarQuantizer.h:57
+
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
+
faiss::ScalarQuantizer::DistanceComputer::compute_distance
virtual float compute_distance(const float *x, const uint8_t *code)=0
vector-to-code distance computation
diff --git a/docs/html/Index_8cpp_source.html b/docs/html/Index_8cpp_source.html index fc7556460..317dcdedc 100644 --- a/docs/html/Index_8cpp_source.html +++ b/docs/html/Index_8cpp_source.html @@ -96,76 +96,108 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
11 #include "IndexFlat.h"
12 #include "FaissAssert.h"
13 
-
14 namespace faiss {
+
14 #include <cstring>
15 
-
16 
-
17 void Index::range_search (idx_t , const float *, float,
-
18  RangeSearchResult *) const
-
19 {
-
20  FAISS_THROW_MSG ("range search not implemented");
-
21 }
+
16 namespace faiss {
+
17 
+
18 Index::~Index ()
+
19 {
+
20 }
+
21 
22 
-
23 void Index::assign (idx_t n, const float * x, idx_t * labels, idx_t k)
-
24 {
-
25  float * distances = new float[n * k];
-
26  ScopeDeleter<float> del(distances);
-
27  search (n, x, k, distances, labels);
-
28 }
-
29 
-
30 void Index::add_with_ids(
-
31  idx_t /*n*/,
-
32  const float* /*x*/,
-
33  const long* /*xids*/) {
-
34  FAISS_THROW_MSG ("add_with_ids not implemented for this type of index");
-
35 }
-
36 
-
37 long Index::remove_ids(const IDSelector& /*sel*/) {
-
38  FAISS_THROW_MSG ("remove_ids not implemented for this type of index");
-
39  return -1;
-
40 }
-
41 
-
42 
-
43 void Index::reconstruct (idx_t, float * ) const {
-
44  FAISS_THROW_MSG ("reconstruct not implemented for this type of index");
-
45 }
-
46 
+
23 void Index::train(idx_t /*n*/, const float* /*x*/) {
+
24  // does nothing by default
+
25 }
+
26 
+
27 
+
28 void Index::range_search (idx_t , const float *, float,
+
29  RangeSearchResult *) const
+
30 {
+
31  FAISS_THROW_MSG ("range search not implemented");
+
32 }
+
33 
+
34 void Index::assign (idx_t n, const float * x, idx_t * labels, idx_t k)
+
35 {
+
36  float * distances = new float[n * k];
+
37  ScopeDeleter<float> del(distances);
+
38  search (n, x, k, distances, labels);
+
39 }
+
40 
+
41 void Index::add_with_ids(
+
42  idx_t /*n*/,
+
43  const float* /*x*/,
+
44  const long* /*xids*/) {
+
45  FAISS_THROW_MSG ("add_with_ids not implemented for this type of index");
+
46 }
47 
-
48 void Index::reconstruct_n (idx_t i0, idx_t ni, float *recons) const {
-
49  for (idx_t i = 0; i < ni; i++) {
-
50  reconstruct (i0 + i, recons + i * d);
-
51  }
-
52 }
+
48 long Index::remove_ids(const IDSelector& /*sel*/) {
+
49  FAISS_THROW_MSG ("remove_ids not implemented for this type of index");
+
50  return -1;
+
51 }
+
52 
53 
-
54 
-
55 
-
56 void Index::compute_residual (const float * x,
-
57  float * residual, idx_t key) const {
-
58  reconstruct (key, residual);
-
59  for (size_t i = 0; i < d; i++)
-
60  residual[i] = x[i] - residual[i];
-
61 }
-
62 
-
63 
-
64 void Index::display () const {
-
65  printf ("Index: %s -> %ld elements\n", typeid (*this).name(), ntotal);
-
66 }
-
67 
-
68 }
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
+
54 void Index::reconstruct (idx_t, float * ) const {
+
55  FAISS_THROW_MSG ("reconstruct not implemented for this type of index");
+
56 }
+
57 
+
58 
+
59 void Index::reconstruct_n (idx_t i0, idx_t ni, float *recons) const {
+
60  for (idx_t i = 0; i < ni; i++) {
+
61  reconstruct (i0 + i, recons + i * d);
+
62  }
+
63 }
+
64 
+
65 
+
66 void Index::search_and_reconstruct (idx_t n, const float *x, idx_t k,
+
67  float *distances, idx_t *labels,
+
68  float *recons) const {
+
69  search (n, x, k, distances, labels);
+
70  for (idx_t i = 0; i < n; ++i) {
+
71  for (idx_t j = 0; j < k; ++j) {
+
72  idx_t ij = i * k + j;
+
73  idx_t key = labels[ij];
+
74  float* reconstructed = recons + ij * d;
+
75  if (key < 0) {
+
76  // Fill with NaNs
+
77  memset(reconstructed, -1, sizeof(*reconstructed) * d);
+
78  } else {
+
79  reconstruct (key, reconstructed);
+
80  }
+
81  }
+
82  }
+
83 }
+
84 
+
85 
+
86 void Index::compute_residual (const float * x,
+
87  float * residual, idx_t key) const {
+
88  reconstruct (key, residual);
+
89  for (size_t i = 0; i < d; i++)
+
90  residual[i] = x[i] - residual[i];
+
91 }
+
92 
+
93 
+
94 void Index::display () const {
+
95  printf ("Index: %s -> %ld elements\n", typeid (*this).name(), ntotal);
+
96 }
+
97 
+
98 }
+
faiss::Index::search_and_reconstruct
virtual void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const
Definition: Index.cpp:66
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
-
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:30
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
+
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:41
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::Index::reconstruct_n
virtual void reconstruct_n(idx_t i0, idx_t ni, float *recons) const
Definition: Index.cpp:48
+
faiss::Index::reconstruct_n
virtual void reconstruct_n(idx_t i0, idx_t ni, float *recons) const
Definition: Index.cpp:59
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
-
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:37
+
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:48
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
-
faiss::Index::display
void display() const
Definition: Index.cpp:64
-
faiss::Index::range_search
virtual void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const
Definition: Index.cpp:17
-
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:56
-
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:43
+
faiss::Index::display
void display() const
Definition: Index.cpp:94
+
faiss::Index::range_search
virtual void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const
Definition: Index.cpp:28
+
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:86
+
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:54
diff --git a/docs/html/Index_8h_source.html b/docs/html/Index_8h_source.html index 261a36aba..d79a35ecd 100644 --- a/docs/html/Index_8h_source.html +++ b/docs/html/Index_8h_source.html @@ -156,14 +156,14 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
71  /// type of metric this index uses for search
72  MetricType metric_type;
73 
-
74  explicit Index (idx_t d = 0, MetricType metric = METRIC_INNER_PRODUCT):
+
74  explicit Index (idx_t d = 0, MetricType metric = METRIC_L2):
75  d(d),
76  ntotal(0),
77  verbose(false),
78  is_trained(true),
79  metric_type (metric) {}
80 
-
81  virtual ~Index () { }
+
81  virtual ~Index ();
82 
83 
84  /** Perform training on a representative set of vectors
@@ -171,130 +171,140 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
86  * @param n nb of training vectors
87  * @param x training vecors, size n * d
88  */
-
89  virtual void train(idx_t /*n*/, const float* /*x*/) {
-
90  // does nothing by default
-
91  }
-
92 
-
93  /** Add n vectors of dimension d to the index.
-
94  *
-
95  * Vectors are implicitly assigned labels ntotal .. ntotal + n - 1
-
96  * This function slices the input vectors in chuncks smaller than
-
97  * blocksize_add and calls add_core.
-
98  * @param x input matrix, size n * d
-
99  */
-
100  virtual void add (idx_t n, const float *x) = 0;
-
101 
-
102  /** Same as add, but stores xids instead of sequential ids.
-
103  *
-
104  * The default implementation fails with an assertion, as it is
-
105  * not supported by all indexes.
-
106  *
-
107  * @param xids if non-null, ids to store for the vectors (size n)
-
108  */
-
109  virtual void add_with_ids (idx_t n, const float * x, const long *xids);
-
110 
-
111  /** query n vectors of dimension d to the index.
-
112  *
-
113  * return at most k vectors. If there are not enough results for a
-
114  * query, the result array is padded with -1s.
-
115  *
-
116  * @param x input vectors to search, size n * d
-
117  * @param labels output labels of the NNs, size n*k
-
118  * @param distances output pairwise distances, size n*k
-
119  */
-
120  virtual void search (idx_t n, const float *x, idx_t k,
-
121  float *distances, idx_t *labels) const = 0;
-
122 
-
123  /** query n vectors of dimension d to the index.
-
124  *
-
125  * return all vectors with distance < radius. Note that many
-
126  * indexes do not implement the range_search (only the k-NN search
-
127  * is mandatory).
-
128  *
-
129  * @param x input vectors to search, size n * d
-
130  * @param radius search radius
-
131  * @param result result table
-
132  */
-
133  virtual void range_search (idx_t n, const float *x, float radius,
-
134  RangeSearchResult *result) const;
-
135 
-
136  /** return the indexes of the k vectors closest to the query x.
-
137  *
-
138  * This function is identical as search but only return labels of neighbors.
-
139  * @param x input vectors to search, size n * d
-
140  * @param labels output labels of the NNs, size n*k
-
141  */
-
142  void assign (idx_t n, const float * x, idx_t * labels, idx_t k = 1);
-
143 
-
144  /// removes all elements from the database.
-
145  virtual void reset() = 0;
-
146 
-
147  /** removes IDs from the index. Not supported by all indexes
-
148  */
-
149  virtual long remove_ids (const IDSelector & sel);
-
150 
-
151  /** Reconstruct a stored vector (or an approximation if lossy coding)
-
152  *
-
153  * this function may not be defined for some indexes
-
154  * @param key id of the vector to reconstruct
-
155  * @param recons reconstucted vector (size d)
-
156  */
-
157  virtual void reconstruct (idx_t key, float * recons) const;
-
158 
-
159 
-
160  /** Reconstruct vectors i0 to i0 + ni - 1
-
161  *
-
162  * this function may not be defined for some indexes
-
163  * @param recons reconstucted vector (size ni * d)
-
164  */
-
165  virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const;
-
166 
-
167 
-
168  /** Computes a residual vector after indexing encoding.
-
169  *
-
170  * The residual vector is the difference between a vector and the
-
171  * reconstruction that can be decoded from its representation in
-
172  * the index. The residual can be used for multiple-stage indexing
-
173  * methods, like IndexIVF's methods.
-
174  *
-
175  * @param x input vector, size d
-
176  * @param residual output residual vector, size d
-
177  * @param key encoded index, as returned by search and assign
-
178  */
-
179  void compute_residual (const float * x, float * residual, idx_t key) const;
-
180 
-
181  /** Display the actual class name and some more info */
-
182  void display () const;
-
183 
-
184 
-
185 
-
186 };
-
187 
-
188 }
-
189 
-
190 
-
191 #endif
+
89  virtual void train(idx_t n, const float* x);
+
90 
+
91  /** Add n vectors of dimension d to the index.
+
92  *
+
93  * Vectors are implicitly assigned labels ntotal .. ntotal + n - 1
+
94  * This function slices the input vectors in chuncks smaller than
+
95  * blocksize_add and calls add_core.
+
96  * @param x input matrix, size n * d
+
97  */
+
98  virtual void add (idx_t n, const float *x) = 0;
+
99 
+
100  /** Same as add, but stores xids instead of sequential ids.
+
101  *
+
102  * The default implementation fails with an assertion, as it is
+
103  * not supported by all indexes.
+
104  *
+
105  * @param xids if non-null, ids to store for the vectors (size n)
+
106  */
+
107  virtual void add_with_ids (idx_t n, const float * x, const long *xids);
+
108 
+
109  /** query n vectors of dimension d to the index.
+
110  *
+
111  * return at most k vectors. If there are not enough results for a
+
112  * query, the result array is padded with -1s.
+
113  *
+
114  * @param x input vectors to search, size n * d
+
115  * @param labels output labels of the NNs, size n*k
+
116  * @param distances output pairwise distances, size n*k
+
117  */
+
118  virtual void search (idx_t n, const float *x, idx_t k,
+
119  float *distances, idx_t *labels) const = 0;
+
120 
+
121  /** query n vectors of dimension d to the index.
+
122  *
+
123  * return all vectors with distance < radius. Note that many
+
124  * indexes do not implement the range_search (only the k-NN search
+
125  * is mandatory).
+
126  *
+
127  * @param x input vectors to search, size n * d
+
128  * @param radius search radius
+
129  * @param result result table
+
130  */
+
131  virtual void range_search (idx_t n, const float *x, float radius,
+
132  RangeSearchResult *result) const;
+
133 
+
134  /** return the indexes of the k vectors closest to the query x.
+
135  *
+
136  * This function is identical as search but only return labels of neighbors.
+
137  * @param x input vectors to search, size n * d
+
138  * @param labels output labels of the NNs, size n*k
+
139  */
+
140  void assign (idx_t n, const float * x, idx_t * labels, idx_t k = 1);
+
141 
+
142  /// removes all elements from the database.
+
143  virtual void reset() = 0;
+
144 
+
145  /** removes IDs from the index. Not supported by all indexes
+
146  */
+
147  virtual long remove_ids (const IDSelector & sel);
+
148 
+
149  /** Reconstruct a stored vector (or an approximation if lossy coding)
+
150  *
+
151  * this function may not be defined for some indexes
+
152  * @param key id of the vector to reconstruct
+
153  * @param recons reconstucted vector (size d)
+
154  */
+
155  virtual void reconstruct (idx_t key, float * recons) const;
+
156 
+
157 
+
158  /** Reconstruct vectors i0 to i0 + ni - 1
+
159  *
+
160  * this function may not be defined for some indexes
+
161  * @param recons reconstucted vector (size ni * d)
+
162  */
+
163  virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const;
+
164 
+
165  /** Similar to search, but also reconstructs the stored vectors (or an
+
166  * approximation in the case of lossy coding) for the search results.
+
167  *
+
168  * If there are not enough results for a query, the resulting arrays
+
169  * is padded with -1s.
+
170  *
+
171  * @param recons reconstructed vectors size (n, k, d)
+
172  **/
+
173  virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k,
+
174  float *distances, idx_t *labels,
+
175  float *recons) const;
+
176 
+
177  /** Computes a residual vector after indexing encoding.
+
178  *
+
179  * The residual vector is the difference between a vector and the
+
180  * reconstruction that can be decoded from its representation in
+
181  * the index. The residual can be used for multiple-stage indexing
+
182  * methods, like IndexIVF's methods.
+
183  *
+
184  * @param x input vector, size d
+
185  * @param residual output residual vector, size d
+
186  * @param key encoded index, as returned by search and assign
+
187  */
+
188  void compute_residual (const float * x, float * residual, idx_t key) const;
+
189 
+
190  /** Display the actual class name and some more info */
+
191  void display () const;
+
192 
+
193 
+
194 
+
195 };
+
196 
+
197 }
+
198 
+
199 
+
200 #endif
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
-
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
+
faiss::Index::search_and_reconstruct
virtual void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const
Definition: Index.cpp:66
+
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
-
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:30
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
+
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:41
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::Index::reconstruct_n
virtual void reconstruct_n(idx_t i0, idx_t ni, float *recons) const
Definition: Index.cpp:48
+
faiss::Index::reconstruct_n
virtual void reconstruct_n(idx_t i0, idx_t ni, float *recons) const
Definition: Index.cpp:59
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
-
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:37
+
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:48
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
-
faiss::Index::display
void display() const
Definition: Index.cpp:64
+
faiss::Index::display
void display() const
Definition: Index.cpp:94
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
-
faiss::Index::range_search
virtual void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const
Definition: Index.cpp:17
+
faiss::Index::range_search
virtual void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const
Definition: Index.cpp:28
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:56
-
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:43
+
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:86
+
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:54
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
diff --git a/docs/html/Limits_8cuh_source.html b/docs/html/Limits_8cuh_source.html index ab8539b3b..49b25f26f 100644 --- a/docs/html/Limits_8cuh_source.html +++ b/docs/html/Limits_8cuh_source.html @@ -128,48 +128,54 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
39 #ifdef FAISS_USE_FLOAT16
40 
41 inline __device__ __host__ half kGetHalf(unsigned short v) {
-
42  half h;
-
43  h.x = v;
-
44  return h;
-
45 }
-
46 
-
47 template <>
-
48 struct Limits<half> {
-
49  static __device__ __host__ inline half getMin() {
-
50  return kGetHalf(0xfbffU);
-
51  }
-
52  static __device__ __host__ inline half getMax() {
-
53  return kGetHalf(0x7bffU);
-
54  }
-
55 };
-
56 
-
57 #endif // FAISS_USE_FLOAT16
-
58 
-
59 constexpr int kIntMax = std::numeric_limits<int>::max();
-
60 constexpr int kIntMin = std::numeric_limits<int>::lowest();
-
61 
-
62 template <>
-
63 struct Limits<int> {
-
64  static __device__ __host__ inline int getMin() {
-
65  return kIntMin;
-
66  }
-
67  static __device__ __host__ inline int getMax() {
-
68  return kIntMax;
-
69  }
-
70 };
-
71 
-
72 template<typename K, typename V>
-
73 struct Limits<Pair<K, V>> {
-
74  static __device__ __host__ inline Pair<K, V> getMin() {
-
75  return Pair<K, V>(Limits<K>::getMin(), Limits<V>::getMin());
-
76  }
+
42 #if CUDA_VERSION >= 9000
+
43  __half_raw h;
+
44  h.x = v;
+
45  return __half(h);
+
46 #else
+
47  half h;
+
48  h.x = v;
+
49  return h;
+
50 #endif
+
51 }
+
52 
+
53 template <>
+
54 struct Limits<half> {
+
55  static __device__ __host__ inline half getMin() {
+
56  return kGetHalf(0xfbffU);
+
57  }
+
58  static __device__ __host__ inline half getMax() {
+
59  return kGetHalf(0x7bffU);
+
60  }
+
61 };
+
62 
+
63 #endif // FAISS_USE_FLOAT16
+
64 
+
65 constexpr int kIntMax = std::numeric_limits<int>::max();
+
66 constexpr int kIntMin = std::numeric_limits<int>::lowest();
+
67 
+
68 template <>
+
69 struct Limits<int> {
+
70  static __device__ __host__ inline int getMin() {
+
71  return kIntMin;
+
72  }
+
73  static __device__ __host__ inline int getMax() {
+
74  return kIntMax;
+
75  }
+
76 };
77 
-
78  static __device__ __host__ inline Pair<K, V> getMax() {
-
79  return Pair<K, V>(Limits<K>::getMax(), Limits<V>::getMax());
-
80  }
-
81 };
-
82 
-
83 } } // namespace
+
78 template<typename K, typename V>
+
79 struct Limits<Pair<K, V>> {
+
80  static __device__ __host__ inline Pair<K, V> getMin() {
+
81  return Pair<K, V>(Limits<K>::getMin(), Limits<V>::getMin());
+
82  }
+
83 
+
84  static __device__ __host__ inline Pair<K, V> getMax() {
+
85  return Pair<K, V>(Limits<K>::getMax(), Limits<V>::getMax());
+
86  }
+
87 };
+
88 
+
89 } } // namespace
faiss::gpu::Pair
A simple pair type for CUDA device usage.
Definition: Pair.cuh:21
diff --git a/docs/html/LoadStoreOperators_8cuh_source.html b/docs/html/LoadStoreOperators_8cuh_source.html index c33e47792..fb8622ee7 100644 --- a/docs/html/LoadStoreOperators_8cuh_source.html +++ b/docs/html/LoadStoreOperators_8cuh_source.html @@ -101,61 +101,89 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
12 
13 #include "Float16.cuh"
14 
-
15 //
-
16 // Templated wrappers to express load/store for different scalar and vector
-
17 // types, so kernels can have the same written form but can operate
-
18 // over half and float, and on vector types transparently
-
19 //
+
15 #ifndef __HALF2_TO_UI
+
16 // cuda_fp16.hpp doesn't export this
+
17 #define __HALF2_TO_UI(var) *(reinterpret_cast<unsigned int *>(&(var)))
+
18 #endif
+
19 
20 
-
21 namespace faiss { namespace gpu {
-
22 
-
23 template <typename T>
-
24 struct LoadStore {
-
25  static inline __device__ T load(void* p) {
-
26  return *((T*) p);
-
27  }
+
21 //
+
22 // Templated wrappers to express load/store for different scalar and vector
+
23 // types, so kernels can have the same written form but can operate
+
24 // over half and float, and on vector types transparently
+
25 //
+
26 
+
27 namespace faiss { namespace gpu {
28 
-
29  static inline __device__ void store(void* p, const T& v) {
-
30  *((T*) p) = v;
-
31  }
-
32 };
-
33 
-
34 #ifdef FAISS_USE_FLOAT16
-
35 
-
36 template <>
-
37 struct LoadStore<Half4> {
-
38  static inline __device__ Half4 load(void* p) {
-
39  Half4 out;
-
40  asm("ld.global.v2.u32 {%0, %1}, [%2];" :
-
41  "=r"(out.a.x), "=r"(out.b.x) : "l"(p));
-
42  return out;
-
43  }
-
44 
-
45  static inline __device__ void store(void* p, const Half4& v) {
-
46  asm("st.v2.u32 [%0], {%1, %2};" : : "l"(p), "r"(v.a.x), "r"(v.b.x));
-
47  }
-
48 };
-
49 
-
50 template <>
-
51 struct LoadStore<Half8> {
-
52  static inline __device__ Half8 load(void* p) {
-
53  Half8 out;
-
54  asm("ld.global.v4.u32 {%0, %1, %2, %3}, [%4];" :
-
55  "=r"(out.a.a.x), "=r"(out.a.b.x),
-
56  "=r"(out.b.a.x), "=r"(out.b.b.x) : "l"(p));
-
57  return out;
-
58  }
-
59 
-
60  static inline __device__ void store(void* p, const Half8& v) {
-
61  asm("st.v4.u32 [%0], {%1, %2, %3, %4};"
-
62  : : "l"(p), "r"(v.a.a.x), "r"(v.a.b.x), "r"(v.b.a.x), "r"(v.b.b.x));
-
63  }
+
29 template <typename T>
+
30 struct LoadStore {
+
31  static inline __device__ T load(void* p) {
+
32  return *((T*) p);
+
33  }
+
34 
+
35  static inline __device__ void store(void* p, const T& v) {
+
36  *((T*) p) = v;
+
37  }
+
38 };
+
39 
+
40 #ifdef FAISS_USE_FLOAT16
+
41 
+
42 template <>
+
43 struct LoadStore<Half4> {
+
44  static inline __device__ Half4 load(void* p) {
+
45  Half4 out;
+
46 #if CUDA_VERSION >= 9000
+
47  asm("ld.global.v2.u32 {%0, %1}, [%2];" :
+
48  "=r"(__HALF2_TO_UI(out.a)), "=r"(__HALF2_TO_UI(out.b)) : "l"(p));
+
49 #else
+
50  asm("ld.global.v2.u32 {%0, %1}, [%2];" :
+
51  "=r"(out.a.x), "=r"(out.b.x) : "l"(p));
+
52 #endif
+
53  return out;
+
54  }
+
55 
+
56  static inline __device__ void store(void* p, Half4& v) {
+
57 #if CUDA_VERSION >= 9000
+
58  asm("st.v2.u32 [%0], {%1, %2};" : : "l"(p),
+
59  "r"(__HALF2_TO_UI(v.a)), "r"(__HALF2_TO_UI(v.b)));
+
60 #else
+
61  asm("st.v2.u32 [%0], {%1, %2};" : : "l"(p), "r"(v.a.x), "r"(v.b.x));
+
62 #endif
+
63  }
64 };
65 
-
66 #endif // FAISS_USE_FLOAT16
-
67 
-
68 } } // namespace
- +
66 template <>
+
67 struct LoadStore<Half8> {
+
68  static inline __device__ Half8 load(void* p) {
+
69  Half8 out;
+
70 #if CUDA_VERSION >= 9000
+
71  asm("ld.global.v4.u32 {%0, %1, %2, %3}, [%4];" :
+
72  "=r"(__HALF2_TO_UI(out.a.a)), "=r"(__HALF2_TO_UI(out.a.b)),
+
73  "=r"(__HALF2_TO_UI(out.b.a)), "=r"(__HALF2_TO_UI(out.b.b)) : "l"(p));
+
74 #else
+
75  asm("ld.global.v4.u32 {%0, %1, %2, %3}, [%4];" :
+
76  "=r"(out.a.a.x), "=r"(out.a.b.x),
+
77  "=r"(out.b.a.x), "=r"(out.b.b.x) : "l"(p));
+
78 #endif
+
79  return out;
+
80  }
+
81 
+
82  static inline __device__ void store(void* p, Half8& v) {
+
83 #if CUDA_VERSION >= 9000
+
84  asm("st.v4.u32 [%0], {%1, %2, %3, %4};"
+
85  : : "l"(p), "r"(__HALF2_TO_UI(v.a.a)), "r"(__HALF2_TO_UI(v.a.b)),
+
86  "r"(__HALF2_TO_UI(v.b.a)), "r"(__HALF2_TO_UI(v.b.b)));
+
87 #else
+
88  asm("st.v4.u32 [%0], {%1, %2, %3, %4};"
+
89  : : "l"(p), "r"(v.a.a.x), "r"(v.a.b.x), "r"(v.b.a.x), "r"(v.b.b.x));
+
90 #endif
+
91  }
+
92 };
+
93 
+
94 #endif // FAISS_USE_FLOAT16
+
95 
+
96 } } // namespace
+
diff --git a/docs/html/MathOperators_8cuh_source.html b/docs/html/MathOperators_8cuh_source.html index 8c7665846..c287fda70 100644 --- a/docs/html/MathOperators_8cuh_source.html +++ b/docs/html/MathOperators_8cuh_source.html @@ -374,286 +374,290 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
285  }
286 
287  static inline __device__ half zero() {
-
288  half h;
-
289  h.x = 0;
-
290  return h;
-
291  }
-
292 };
-
293 
-
294 template <>
-
295 struct Math<half2> {
-
296  typedef half ScalarType;
+
288 #if CUDA_VERSION >= 9000
+
289  return 0;
+
290 #else
+
291  half h;
+
292  h.x = 0;
+
293  return h;
+
294 #endif
+
295  }
+
296 };
297 
-
298  static inline __device__ half2 add(half2 a, half2 b) {
-
299 #ifdef FAISS_USE_FULL_FLOAT16
-
300  return __hadd2(a, b);
-
301 #else
-
302  float2 af = __half22float2(a);
-
303  float2 bf = __half22float2(b);
-
304 
-
305  af.x += bf.x;
-
306  af.y += bf.y;
-
307 
-
308  return __float22half2_rn(af);
-
309 #endif
-
310  }
+
298 template <>
+
299 struct Math<half2> {
+
300  typedef half ScalarType;
+
301 
+
302  static inline __device__ half2 add(half2 a, half2 b) {
+
303 #ifdef FAISS_USE_FULL_FLOAT16
+
304  return __hadd2(a, b);
+
305 #else
+
306  float2 af = __half22float2(a);
+
307  float2 bf = __half22float2(b);
+
308 
+
309  af.x += bf.x;
+
310  af.y += bf.y;
311 
-
312  static inline __device__ half2 sub(half2 a, half2 b) {
-
313 #ifdef FAISS_USE_FULL_FLOAT16
-
314  return __hsub2(a, b);
-
315 #else
-
316  float2 af = __half22float2(a);
-
317  float2 bf = __half22float2(b);
-
318 
-
319  af.x -= bf.x;
-
320  af.y -= bf.y;
-
321 
-
322  return __float22half2_rn(af);
-
323 #endif
-
324  }
+
312  return __float22half2_rn(af);
+
313 #endif
+
314  }
+
315 
+
316  static inline __device__ half2 sub(half2 a, half2 b) {
+
317 #ifdef FAISS_USE_FULL_FLOAT16
+
318  return __hsub2(a, b);
+
319 #else
+
320  float2 af = __half22float2(a);
+
321  float2 bf = __half22float2(b);
+
322 
+
323  af.x -= bf.x;
+
324  af.y -= bf.y;
325 
-
326  static inline __device__ half2 add(half2 a, half b) {
-
327 #ifdef FAISS_USE_FULL_FLOAT16
-
328  half2 b2 = __half2half2(b);
-
329  return __hadd2(a, b2);
-
330 #else
-
331  float2 af = __half22float2(a);
-
332  float bf = __half2float(b);
-
333 
-
334  af.x += bf;
-
335  af.y += bf;
-
336 
-
337  return __float22half2_rn(af);
-
338 #endif
-
339  }
+
326  return __float22half2_rn(af);
+
327 #endif
+
328  }
+
329 
+
330  static inline __device__ half2 add(half2 a, half b) {
+
331 #ifdef FAISS_USE_FULL_FLOAT16
+
332  half2 b2 = __half2half2(b);
+
333  return __hadd2(a, b2);
+
334 #else
+
335  float2 af = __half22float2(a);
+
336  float bf = __half2float(b);
+
337 
+
338  af.x += bf;
+
339  af.y += bf;
340 
-
341  static inline __device__ half2 sub(half2 a, half b) {
-
342 #ifdef FAISS_USE_FULL_FLOAT16
-
343  half2 b2 = __half2half2(b);
-
344  return __hsub2(a, b2);
-
345 #else
-
346  float2 af = __half22float2(a);
-
347  float bf = __half2float(b);
-
348 
-
349  af.x -= bf;
-
350  af.y -= bf;
-
351 
-
352  return __float22half2_rn(af);
-
353 #endif
-
354  }
+
341  return __float22half2_rn(af);
+
342 #endif
+
343  }
+
344 
+
345  static inline __device__ half2 sub(half2 a, half b) {
+
346 #ifdef FAISS_USE_FULL_FLOAT16
+
347  half2 b2 = __half2half2(b);
+
348  return __hsub2(a, b2);
+
349 #else
+
350  float2 af = __half22float2(a);
+
351  float bf = __half2float(b);
+
352 
+
353  af.x -= bf;
+
354  af.y -= bf;
355 
-
356  static inline __device__ half2 mul(half2 a, half2 b) {
-
357 #ifdef FAISS_USE_FULL_FLOAT16
-
358  return __hmul2(a, b);
-
359 #else
-
360  float2 af = __half22float2(a);
-
361  float2 bf = __half22float2(b);
-
362 
-
363  af.x *= bf.x;
-
364  af.y *= bf.y;
-
365 
-
366  return __float22half2_rn(af);
-
367 #endif
-
368  }
+
356  return __float22half2_rn(af);
+
357 #endif
+
358  }
+
359 
+
360  static inline __device__ half2 mul(half2 a, half2 b) {
+
361 #ifdef FAISS_USE_FULL_FLOAT16
+
362  return __hmul2(a, b);
+
363 #else
+
364  float2 af = __half22float2(a);
+
365  float2 bf = __half22float2(b);
+
366 
+
367  af.x *= bf.x;
+
368  af.y *= bf.y;
369 
-
370  static inline __device__ half2 mul(half2 a, half b) {
-
371 #ifdef FAISS_USE_FULL_FLOAT16
-
372  half2 b2 = __half2half2(b);
-
373  return __hmul2(a, b2);
-
374 #else
-
375  float2 af = __half22float2(a);
-
376  float bf = __half2float(b);
-
377 
-
378  af.x *= bf;
-
379  af.y *= bf;
-
380 
-
381  return __float22half2_rn(af);
-
382 #endif
-
383  }
+
370  return __float22half2_rn(af);
+
371 #endif
+
372  }
+
373 
+
374  static inline __device__ half2 mul(half2 a, half b) {
+
375 #ifdef FAISS_USE_FULL_FLOAT16
+
376  half2 b2 = __half2half2(b);
+
377  return __hmul2(a, b2);
+
378 #else
+
379  float2 af = __half22float2(a);
+
380  float bf = __half2float(b);
+
381 
+
382  af.x *= bf;
+
383  af.y *= bf;
384 
-
385  static inline __device__ half2 neg(half2 v) {
-
386 #ifdef FAISS_USE_FULL_FLOAT16
-
387  return __hneg2(v);
-
388 #else
-
389  float2 vf = __half22float2(v);
-
390  vf.x = -vf.x;
-
391  vf.y = -vf.y;
-
392 
-
393  return __float22half2_rn(vf);
-
394 #endif
-
395  }
+
385  return __float22half2_rn(af);
+
386 #endif
+
387  }
+
388 
+
389  static inline __device__ half2 neg(half2 v) {
+
390 #ifdef FAISS_USE_FULL_FLOAT16
+
391  return __hneg2(v);
+
392 #else
+
393  float2 vf = __half22float2(v);
+
394  vf.x = -vf.x;
+
395  vf.y = -vf.y;
396 
-
397  static inline __device__ half reduceAdd(half2 v) {
-
398 #ifdef FAISS_USE_FULL_FLOAT16
-
399  half hv = __high2half(v);
-
400  half lv = __low2half(v);
-
401 
-
402  return __hadd(hv, lv);
-
403 #else
-
404  float2 vf = __half22float2(v);
-
405  vf.x += vf.y;
-
406 
-
407  return __float2half(vf.x);
-
408 #endif
-
409  }
+
397  return __float22half2_rn(vf);
+
398 #endif
+
399  }
+
400 
+
401  static inline __device__ half reduceAdd(half2 v) {
+
402 #ifdef FAISS_USE_FULL_FLOAT16
+
403  half hv = __high2half(v);
+
404  half lv = __low2half(v);
+
405 
+
406  return __hadd(hv, lv);
+
407 #else
+
408  float2 vf = __half22float2(v);
+
409  vf.x += vf.y;
410 
-
411  // not implemented for vector types
-
412  // static inline __device__ bool lt(half2 a, half2 b);
-
413  // static inline __device__ bool gt(half2 a, half2 b);
-
414  // static inline __device__ bool eq(half2 a, half2 b);
-
415 
-
416  static inline __device__ half2 zero() {
-
417  return __half2half2(Math<half>::zero());
-
418  }
-
419 };
-
420 
-
421 template <>
-
422 struct Math<Half4> {
-
423  typedef half ScalarType;
+
411  return __float2half(vf.x);
+
412 #endif
+
413  }
+
414 
+
415  // not implemented for vector types
+
416  // static inline __device__ bool lt(half2 a, half2 b);
+
417  // static inline __device__ bool gt(half2 a, half2 b);
+
418  // static inline __device__ bool eq(half2 a, half2 b);
+
419 
+
420  static inline __device__ half2 zero() {
+
421  return __half2half2(Math<half>::zero());
+
422  }
+
423 };
424 
-
425  static inline __device__ Half4 add(Half4 a, Half4 b) {
-
426  Half4 h;
-
427  h.a = Math<half2>::add(a.a, b.a);
-
428  h.b = Math<half2>::add(a.b, b.b);
-
429  return h;
-
430  }
-
431 
-
432  static inline __device__ Half4 sub(Half4 a, Half4 b) {
-
433  Half4 h;
-
434  h.a = Math<half2>::sub(a.a, b.a);
-
435  h.b = Math<half2>::sub(a.b, b.b);
-
436  return h;
-
437  }
-
438 
-
439  static inline __device__ Half4 add(Half4 a, half b) {
-
440  Half4 h;
-
441  h.a = Math<half2>::add(a.a, b);
-
442  h.b = Math<half2>::add(a.b, b);
-
443  return h;
-
444  }
-
445 
-
446  static inline __device__ Half4 sub(Half4 a, half b) {
-
447  Half4 h;
-
448  h.a = Math<half2>::sub(a.a, b);
-
449  h.b = Math<half2>::sub(a.b, b);
-
450  return h;
-
451  }
-
452 
-
453  static inline __device__ Half4 mul(Half4 a, Half4 b) {
-
454  Half4 h;
-
455  h.a = Math<half2>::mul(a.a, b.a);
-
456  h.b = Math<half2>::mul(a.b, b.b);
-
457  return h;
-
458  }
-
459 
-
460  static inline __device__ Half4 mul(Half4 a, half b) {
-
461  Half4 h;
-
462  h.a = Math<half2>::mul(a.a, b);
-
463  h.b = Math<half2>::mul(a.b, b);
-
464  return h;
-
465  }
-
466 
-
467  static inline __device__ Half4 neg(Half4 v) {
-
468  Half4 h;
-
469  h.a = Math<half2>::neg(v.a);
-
470  h.b = Math<half2>::neg(v.b);
-
471  return h;
-
472  }
-
473 
-
474  static inline __device__ half reduceAdd(Half4 v) {
-
475  half hx = Math<half2>::reduceAdd(v.a);
-
476  half hy = Math<half2>::reduceAdd(v.b);
-
477  return Math<half>::add(hx, hy);
-
478  }
-
479 
-
480  // not implemented for vector types
-
481  // static inline __device__ bool lt(Half4 a, Half4 b);
-
482  // static inline __device__ bool gt(Half4 a, Half4 b);
-
483  // static inline __device__ bool eq(Half4 a, Half4 b);
-
484 
-
485  static inline __device__ Half4 zero() {
-
486  Half4 h;
-
487  h.a = Math<half2>::zero();
-
488  h.b = Math<half2>::zero();
-
489  return h;
-
490  }
-
491 };
-
492 
-
493 template <>
-
494 struct Math<Half8> {
-
495  typedef half ScalarType;
+
425 template <>
+
426 struct Math<Half4> {
+
427  typedef half ScalarType;
+
428 
+
429  static inline __device__ Half4 add(Half4 a, Half4 b) {
+
430  Half4 h;
+
431  h.a = Math<half2>::add(a.a, b.a);
+
432  h.b = Math<half2>::add(a.b, b.b);
+
433  return h;
+
434  }
+
435 
+
436  static inline __device__ Half4 sub(Half4 a, Half4 b) {
+
437  Half4 h;
+
438  h.a = Math<half2>::sub(a.a, b.a);
+
439  h.b = Math<half2>::sub(a.b, b.b);
+
440  return h;
+
441  }
+
442 
+
443  static inline __device__ Half4 add(Half4 a, half b) {
+
444  Half4 h;
+
445  h.a = Math<half2>::add(a.a, b);
+
446  h.b = Math<half2>::add(a.b, b);
+
447  return h;
+
448  }
+
449 
+
450  static inline __device__ Half4 sub(Half4 a, half b) {
+
451  Half4 h;
+
452  h.a = Math<half2>::sub(a.a, b);
+
453  h.b = Math<half2>::sub(a.b, b);
+
454  return h;
+
455  }
+
456 
+
457  static inline __device__ Half4 mul(Half4 a, Half4 b) {
+
458  Half4 h;
+
459  h.a = Math<half2>::mul(a.a, b.a);
+
460  h.b = Math<half2>::mul(a.b, b.b);
+
461  return h;
+
462  }
+
463 
+
464  static inline __device__ Half4 mul(Half4 a, half b) {
+
465  Half4 h;
+
466  h.a = Math<half2>::mul(a.a, b);
+
467  h.b = Math<half2>::mul(a.b, b);
+
468  return h;
+
469  }
+
470 
+
471  static inline __device__ Half4 neg(Half4 v) {
+
472  Half4 h;
+
473  h.a = Math<half2>::neg(v.a);
+
474  h.b = Math<half2>::neg(v.b);
+
475  return h;
+
476  }
+
477 
+
478  static inline __device__ half reduceAdd(Half4 v) {
+
479  half hx = Math<half2>::reduceAdd(v.a);
+
480  half hy = Math<half2>::reduceAdd(v.b);
+
481  return Math<half>::add(hx, hy);
+
482  }
+
483 
+
484  // not implemented for vector types
+
485  // static inline __device__ bool lt(Half4 a, Half4 b);
+
486  // static inline __device__ bool gt(Half4 a, Half4 b);
+
487  // static inline __device__ bool eq(Half4 a, Half4 b);
+
488 
+
489  static inline __device__ Half4 zero() {
+
490  Half4 h;
+
491  h.a = Math<half2>::zero();
+
492  h.b = Math<half2>::zero();
+
493  return h;
+
494  }
+
495 };
496 
-
497  static inline __device__ Half8 add(Half8 a, Half8 b) {
-
498  Half8 h;
-
499  h.a = Math<Half4>::add(a.a, b.a);
-
500  h.b = Math<Half4>::add(a.b, b.b);
-
501  return h;
-
502  }
-
503 
-
504  static inline __device__ Half8 sub(Half8 a, Half8 b) {
-
505  Half8 h;
-
506  h.a = Math<Half4>::sub(a.a, b.a);
-
507  h.b = Math<Half4>::sub(a.b, b.b);
-
508  return h;
-
509  }
-
510 
-
511  static inline __device__ Half8 add(Half8 a, half b) {
-
512  Half8 h;
-
513  h.a = Math<Half4>::add(a.a, b);
-
514  h.b = Math<Half4>::add(a.b, b);
-
515  return h;
-
516  }
-
517 
-
518  static inline __device__ Half8 sub(Half8 a, half b) {
-
519  Half8 h;
-
520  h.a = Math<Half4>::sub(a.a, b);
-
521  h.b = Math<Half4>::sub(a.b, b);
-
522  return h;
-
523  }
-
524 
-
525  static inline __device__ Half8 mul(Half8 a, Half8 b) {
-
526  Half8 h;
-
527  h.a = Math<Half4>::mul(a.a, b.a);
-
528  h.b = Math<Half4>::mul(a.b, b.b);
-
529  return h;
-
530  }
-
531 
-
532  static inline __device__ Half8 mul(Half8 a, half b) {
-
533  Half8 h;
-
534  h.a = Math<Half4>::mul(a.a, b);
-
535  h.b = Math<Half4>::mul(a.b, b);
-
536  return h;
-
537  }
-
538 
-
539  static inline __device__ Half8 neg(Half8 v) {
-
540  Half8 h;
-
541  h.a = Math<Half4>::neg(v.a);
-
542  h.b = Math<Half4>::neg(v.b);
-
543  return h;
-
544  }
-
545 
-
546  static inline __device__ half reduceAdd(Half8 v) {
-
547  half hx = Math<Half4>::reduceAdd(v.a);
-
548  half hy = Math<Half4>::reduceAdd(v.b);
-
549  return Math<half>::add(hx, hy);
-
550  }
-
551 
-
552  // not implemented for vector types
-
553  // static inline __device__ bool lt(Half8 a, Half8 b);
-
554  // static inline __device__ bool gt(Half8 a, Half8 b);
-
555  // static inline __device__ bool eq(Half8 a, Half8 b);
-
556 
-
557  static inline __device__ Half8 zero() {
-
558  Half8 h;
-
559  h.a = Math<Half4>::zero();
-
560  h.b = Math<Half4>::zero();
-
561  return h;
-
562  }
-
563 };
-
564 
-
565 #endif // FAISS_USE_FLOAT16
-
566 
-
567 } } // namespace
+
497 template <>
+
498 struct Math<Half8> {
+
499  typedef half ScalarType;
+
500 
+
501  static inline __device__ Half8 add(Half8 a, Half8 b) {
+
502  Half8 h;
+
503  h.a = Math<Half4>::add(a.a, b.a);
+
504  h.b = Math<Half4>::add(a.b, b.b);
+
505  return h;
+
506  }
+
507 
+
508  static inline __device__ Half8 sub(Half8 a, Half8 b) {
+
509  Half8 h;
+
510  h.a = Math<Half4>::sub(a.a, b.a);
+
511  h.b = Math<Half4>::sub(a.b, b.b);
+
512  return h;
+
513  }
+
514 
+
515  static inline __device__ Half8 add(Half8 a, half b) {
+
516  Half8 h;
+
517  h.a = Math<Half4>::add(a.a, b);
+
518  h.b = Math<Half4>::add(a.b, b);
+
519  return h;
+
520  }
+
521 
+
522  static inline __device__ Half8 sub(Half8 a, half b) {
+
523  Half8 h;
+
524  h.a = Math<Half4>::sub(a.a, b);
+
525  h.b = Math<Half4>::sub(a.b, b);
+
526  return h;
+
527  }
+
528 
+
529  static inline __device__ Half8 mul(Half8 a, Half8 b) {
+
530  Half8 h;
+
531  h.a = Math<Half4>::mul(a.a, b.a);
+
532  h.b = Math<Half4>::mul(a.b, b.b);
+
533  return h;
+
534  }
+
535 
+
536  static inline __device__ Half8 mul(Half8 a, half b) {
+
537  Half8 h;
+
538  h.a = Math<Half4>::mul(a.a, b);
+
539  h.b = Math<Half4>::mul(a.b, b);
+
540  return h;
+
541  }
+
542 
+
543  static inline __device__ Half8 neg(Half8 v) {
+
544  Half8 h;
+
545  h.a = Math<Half4>::neg(v.a);
+
546  h.b = Math<Half4>::neg(v.b);
+
547  return h;
+
548  }
+
549 
+
550  static inline __device__ half reduceAdd(Half8 v) {
+
551  half hx = Math<Half4>::reduceAdd(v.a);
+
552  half hy = Math<Half4>::reduceAdd(v.b);
+
553  return Math<half>::add(hx, hy);
+
554  }
+
555 
+
556  // not implemented for vector types
+
557  // static inline __device__ bool lt(Half8 a, Half8 b);
+
558  // static inline __device__ bool gt(Half8 a, Half8 b);
+
559  // static inline __device__ bool eq(Half8 a, Half8 b);
+
560 
+
561  static inline __device__ Half8 zero() {
+
562  Half8 h;
+
563  h.a = Math<Half4>::zero();
+
564  h.b = Math<Half4>::zero();
+
565  return h;
+
566  }
+
567 };
+
568 
+
569 #endif // FAISS_USE_FLOAT16
+
570 
+
571 } } // namespace
faiss::gpu::Math::reduceAdd
static __device__ T reduceAdd(T v)
For a vector type, this is a horizontal add, returning sum(v_i)
Definition: MathOperators.cuh:44
faiss::gpu::Math< float2 >::reduceAdd
static __device__ float reduceAdd(float2 v)
For a vector type, this is a horizontal add, returning sum(v_i)
Definition: MathOperators.cuh:118
diff --git a/docs/html/MetaIndexes_8cpp_source.html b/docs/html/MetaIndexes_8cpp_source.html index 74ab78673..f68df44d0 100644 --- a/docs/html/MetaIndexes_8cpp_source.html +++ b/docs/html/MetaIndexes_8cpp_source.html @@ -131,7 +131,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
46 
47 void IndexIDMap::train (idx_t n, const float *x)
48 {
-
49  index->train (n, x);
+
49  index->train (n, x);
50  is_trained = index->is_trained;
51 }
52 
@@ -833,12 +833,12 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::RangeSearchResult::nq
size_t nq
nb of queries
Definition: AuxIndexStructures.h:30
faiss::IndexShards::IndexShards
IndexShards(idx_t d, bool threaded=false, bool successive_ids=true)
Definition: MetaIndexes.cpp:367
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
faiss::IndexIDMap::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:69
faiss::IndexShards::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:437
faiss::IndexShards::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:498
faiss::IndexIDMap2::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:142
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
faiss::IndexSplitVectors::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:727
faiss::IndexIDMap::add
void add(idx_t n, const float *x) override
this will fail. Use add_with_ids
Definition: MetaIndexes.cpp:40
@@ -854,7 +854,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::IndexIDMap2::construct_rev_map
void construct_rev_map()
make the rev_map from scratch
Definition: MetaIndexes.cpp:151
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
faiss::IndexSplitVectors::add
void add(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:622
-
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:37
+
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:48
faiss::IndexIDMap2::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:160
faiss::IndexShards::threaded
bool threaded
should the sub-indexes be deleted along with this?
Definition: MetaIndexes.h:90
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
@@ -868,9 +868,9 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::IndexIDMap::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:47
faiss::RangeSearchResult::lims
size_t * lims
size (nq + 1)
Definition: AuxIndexStructures.h:31
-
faiss::Index::range_search
virtual void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const
Definition: Index.cpp:17
+
faiss::Index::range_search
virtual void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const
Definition: Index.cpp:28
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:43
+
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:54
faiss::IndexIDMap::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:60
faiss::RangeSearchResult::labels
idx_t * labels
result for query i is labels[lims[i]:lims[i+1]]
Definition: AuxIndexStructures.h:35
faiss::IndexSplitVectors::IndexSplitVectors
IndexSplitVectors(idx_t d, bool threaded=false)
sum of dimensions seen so far
Definition: MetaIndexes.cpp:592
diff --git a/docs/html/NoTypeTensor_8cuh_source.html b/docs/html/NoTypeTensor_8cuh_source.html index d87e80c72..cbef0123b 100644 --- a/docs/html/NoTypeTensor_8cuh_source.html +++ b/docs/html/NoTypeTensor_8cuh_source.html @@ -213,10 +213,10 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
124 
125 } } // namespace
-
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
-
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:232
+
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:230
diff --git a/docs/html/PQScanMultiPassNoPrecomputed_8cu_source.html b/docs/html/PQScanMultiPassNoPrecomputed_8cu_source.html index 69af10182..05dc9d114 100644 --- a/docs/html/PQScanMultiPassNoPrecomputed_8cu_source.html +++ b/docs/html/PQScanMultiPassNoPrecomputed_8cu_source.html @@ -595,113 +595,112 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
506  }
507 #else
508  FAISS_ASSERT(!useFloat16Lookup);
-
509  int codeSize = sizeof(float);
-
510 #endif
-
511 
-
512  int totalCodeDistancesSize =
-
513  queryTileSize * nprobe * numSubQuantizers * numSubQuantizerCodes *
-
514  codeDistanceTypeSize;
-
515 
-
516  DeviceTensor<char, 1, true> codeDistances1Mem(
-
517  mem, {totalCodeDistancesSize}, stream);
-
518  NoTypeTensor<4, true> codeDistances1(
-
519  codeDistances1Mem.data(),
-
520  codeDistanceTypeSize,
-
521  {queryTileSize, nprobe, numSubQuantizers, numSubQuantizerCodes});
-
522 
-
523  DeviceTensor<char, 1, true> codeDistances2Mem(
-
524  mem, {totalCodeDistancesSize}, stream);
-
525  NoTypeTensor<4, true> codeDistances2(
-
526  codeDistances2Mem.data(),
-
527  codeDistanceTypeSize,
-
528  {queryTileSize, nprobe, numSubQuantizers, numSubQuantizerCodes});
-
529 
-
530  NoTypeTensor<4, true>* codeDistances[2] =
-
531  {&codeDistances1, &codeDistances2};
-
532 
-
533  DeviceTensor<float, 1, true> allDistances1(
-
534  mem, {queryTileSize * nprobe * maxListLength}, stream);
-
535  DeviceTensor<float, 1, true> allDistances2(
-
536  mem, {queryTileSize * nprobe * maxListLength}, stream);
-
537  DeviceTensor<float, 1, true>* allDistances[2] =
-
538  {&allDistances1, &allDistances2};
-
539 
-
540  DeviceTensor<float, 3, true> heapDistances1(
-
541  mem, {queryTileSize, pass2Chunks, k}, stream);
-
542  DeviceTensor<float, 3, true> heapDistances2(
-
543  mem, {queryTileSize, pass2Chunks, k}, stream);
-
544  DeviceTensor<float, 3, true>* heapDistances[2] =
-
545  {&heapDistances1, &heapDistances2};
-
546 
-
547  DeviceTensor<int, 3, true> heapIndices1(
-
548  mem, {queryTileSize, pass2Chunks, k}, stream);
-
549  DeviceTensor<int, 3, true> heapIndices2(
-
550  mem, {queryTileSize, pass2Chunks, k}, stream);
-
551  DeviceTensor<int, 3, true>* heapIndices[2] =
-
552  {&heapIndices1, &heapIndices2};
-
553 
-
554  auto streams = res->getAlternateStreamsCurrentDevice();
-
555  streamWait(streams, {stream});
-
556 
-
557  int curStream = 0;
-
558 
-
559  for (int query = 0; query < queries.getSize(0); query += queryTileSize) {
-
560  int numQueriesInTile =
-
561  std::min(queryTileSize, queries.getSize(0) - query);
-
562 
-
563  auto prefixSumOffsetsView =
-
564  prefixSumOffsets[curStream]->narrowOutermost(0, numQueriesInTile);
-
565 
-
566  auto codeDistancesView =
-
567  codeDistances[curStream]->narrowOutermost(0, numQueriesInTile);
-
568  auto coarseIndicesView =
-
569  topQueryToCentroid.narrowOutermost(query, numQueriesInTile);
-
570  auto queryView =
-
571  queries.narrowOutermost(query, numQueriesInTile);
-
572 
-
573  auto heapDistancesView =
-
574  heapDistances[curStream]->narrowOutermost(0, numQueriesInTile);
-
575  auto heapIndicesView =
-
576  heapIndices[curStream]->narrowOutermost(0, numQueriesInTile);
-
577 
-
578  auto outDistanceView =
-
579  outDistances.narrowOutermost(query, numQueriesInTile);
-
580  auto outIndicesView =
-
581  outIndices.narrowOutermost(query, numQueriesInTile);
-
582 
-
583  runMultiPassTile(queryView,
-
584  centroids,
-
585  pqCentroidsInnermostCode,
-
586  codeDistancesView,
-
587  coarseIndicesView,
-
588  useFloat16Lookup,
-
589  bytesPerCode,
-
590  numSubQuantizers,
-
591  numSubQuantizerCodes,
-
592  listCodes,
-
593  listIndices,
-
594  indicesOptions,
-
595  listLengths,
-
596  *thrustMem[curStream],
-
597  prefixSumOffsetsView,
-
598  *allDistances[curStream],
-
599  heapDistancesView,
-
600  heapIndicesView,
-
601  k,
-
602  outDistanceView,
-
603  outIndicesView,
-
604  streams[curStream]);
-
605 
-
606  curStream = (curStream + 1) % 2;
-
607  }
-
608 
-
609  streamWait({stream}, streams);
-
610 }
-
611 
-
612 } } // namespace
- -
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
509 #endif
+
510 
+
511  int totalCodeDistancesSize =
+
512  queryTileSize * nprobe * numSubQuantizers * numSubQuantizerCodes *
+
513  codeDistanceTypeSize;
+
514 
+
515  DeviceTensor<char, 1, true> codeDistances1Mem(
+
516  mem, {totalCodeDistancesSize}, stream);
+
517  NoTypeTensor<4, true> codeDistances1(
+
518  codeDistances1Mem.data(),
+
519  codeDistanceTypeSize,
+
520  {queryTileSize, nprobe, numSubQuantizers, numSubQuantizerCodes});
+
521 
+
522  DeviceTensor<char, 1, true> codeDistances2Mem(
+
523  mem, {totalCodeDistancesSize}, stream);
+
524  NoTypeTensor<4, true> codeDistances2(
+
525  codeDistances2Mem.data(),
+
526  codeDistanceTypeSize,
+
527  {queryTileSize, nprobe, numSubQuantizers, numSubQuantizerCodes});
+
528 
+
529  NoTypeTensor<4, true>* codeDistances[2] =
+
530  {&codeDistances1, &codeDistances2};
+
531 
+
532  DeviceTensor<float, 1, true> allDistances1(
+
533  mem, {queryTileSize * nprobe * maxListLength}, stream);
+
534  DeviceTensor<float, 1, true> allDistances2(
+
535  mem, {queryTileSize * nprobe * maxListLength}, stream);
+
536  DeviceTensor<float, 1, true>* allDistances[2] =
+
537  {&allDistances1, &allDistances2};
+
538 
+
539  DeviceTensor<float, 3, true> heapDistances1(
+
540  mem, {queryTileSize, pass2Chunks, k}, stream);
+
541  DeviceTensor<float, 3, true> heapDistances2(
+
542  mem, {queryTileSize, pass2Chunks, k}, stream);
+
543  DeviceTensor<float, 3, true>* heapDistances[2] =
+
544  {&heapDistances1, &heapDistances2};
+
545 
+
546  DeviceTensor<int, 3, true> heapIndices1(
+
547  mem, {queryTileSize, pass2Chunks, k}, stream);
+
548  DeviceTensor<int, 3, true> heapIndices2(
+
549  mem, {queryTileSize, pass2Chunks, k}, stream);
+
550  DeviceTensor<int, 3, true>* heapIndices[2] =
+
551  {&heapIndices1, &heapIndices2};
+
552 
+
553  auto streams = res->getAlternateStreamsCurrentDevice();
+
554  streamWait(streams, {stream});
+
555 
+
556  int curStream = 0;
+
557 
+
558  for (int query = 0; query < queries.getSize(0); query += queryTileSize) {
+
559  int numQueriesInTile =
+
560  std::min(queryTileSize, queries.getSize(0) - query);
+
561 
+
562  auto prefixSumOffsetsView =
+
563  prefixSumOffsets[curStream]->narrowOutermost(0, numQueriesInTile);
+
564 
+
565  auto codeDistancesView =
+
566  codeDistances[curStream]->narrowOutermost(0, numQueriesInTile);
+
567  auto coarseIndicesView =
+
568  topQueryToCentroid.narrowOutermost(query, numQueriesInTile);
+
569  auto queryView =
+
570  queries.narrowOutermost(query, numQueriesInTile);
+
571 
+
572  auto heapDistancesView =
+
573  heapDistances[curStream]->narrowOutermost(0, numQueriesInTile);
+
574  auto heapIndicesView =
+
575  heapIndices[curStream]->narrowOutermost(0, numQueriesInTile);
+
576 
+
577  auto outDistanceView =
+
578  outDistances.narrowOutermost(query, numQueriesInTile);
+
579  auto outIndicesView =
+
580  outIndices.narrowOutermost(query, numQueriesInTile);
+
581 
+
582  runMultiPassTile(queryView,
+
583  centroids,
+
584  pqCentroidsInnermostCode,
+
585  codeDistancesView,
+
586  coarseIndicesView,
+
587  useFloat16Lookup,
+
588  bytesPerCode,
+
589  numSubQuantizers,
+
590  numSubQuantizerCodes,
+
591  listCodes,
+
592  listIndices,
+
593  indicesOptions,
+
594  listLengths,
+
595  *thrustMem[curStream],
+
596  prefixSumOffsetsView,
+
597  *allDistances[curStream],
+
598  heapDistancesView,
+
599  heapIndicesView,
+
600  k,
+
601  outDistanceView,
+
602  outIndicesView,
+
603  streams[curStream]);
+
604 
+
605  curStream = (curStream + 1) % 2;
+
606  }
+
607 
+
608  streamWait({stream}, streams);
+
609 }
+
610 
+
611 } } // namespace
+ +
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
diff --git a/docs/html/PolysemousTraining_8cpp_source.html b/docs/html/PolysemousTraining_8cpp_source.html index e3e58d433..918a15ccb 100644 --- a/docs/html/PolysemousTraining_8cpp_source.html +++ b/docs/html/PolysemousTraining_8cpp_source.html @@ -1052,7 +1052,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::RandomGenerator::rand_int
int rand_int()
random 31-bit positive integer
faiss::ProductQuantizer::ksub
size_t ksub
number of centroids for each subquantizer
Definition: ProductQuantizer.h:35
faiss::PolysemousTraining::optimize_ranking
void optimize_ranking(ProductQuantizer &pq, size_t n, const float *x) const
called by optimize_pq_for_hamming
Definition: PolysemousTraining.cpp:826
-
faiss::pairwise_L2sqr
void pairwise_L2sqr(long d, long nq, const float *xq, long nb, const float *xb, float *dis, long ldq, long ldb, long ldd)
Definition: utils.cpp:1342
+
faiss::pairwise_L2sqr
void pairwise_L2sqr(long d, long nq, const float *xq, long nb, const float *xb, float *dis, long ldq, long ldb, long ldd)
Definition: utils.cpp:1344
faiss::Score3Computer::compute_cost
double compute_cost(const int *perm) const override
Definition: PolysemousTraining.cpp:631
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
diff --git a/docs/html/ProductQuantizer_8cpp_source.html b/docs/html/ProductQuantizer_8cpp_source.html index d39d85ae6..13eda8bf7 100644 --- a/docs/html/ProductQuantizer_8cpp_source.html +++ b/docs/html/ProductQuantizer_8cpp_source.html @@ -440,7 +440,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
355 void ProductQuantizer::decode (const uint8_t *code, float *x, size_t n) const
356 {
357  for (size_t i = 0; i < n; i++) {
-
358  this->decode (code + M * i, x + d * i);
+
358  this->decode (code + code_size * i, x + d * i);
359  }
360 }
361 
@@ -770,7 +770,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
faiss::ProductQuantizer::ksub
size_t ksub
number of centroids for each subquantizer
Definition: ProductQuantizer.h:35
faiss::ProductQuantizer::search_ip
void search_ip(const float *x, size_t nx, const uint8_t *codes, const size_t ncodes, float_minheap_array_t *res, bool init_finalize_heap=true) const
Definition: ProductQuantizer.cpp:565
-
faiss::pairwise_L2sqr
void pairwise_L2sqr(long d, long nq, const float *xq, long nb, const float *xb, float *dis, long ldq, long ldb, long ldd)
Definition: utils.cpp:1342
+
faiss::pairwise_L2sqr
void pairwise_L2sqr(long d, long nq, const float *xq, long nb, const float *xb, float *dis, long ldq, long ldb, long ldd)
Definition: utils.cpp:1344
faiss::ProductQuantizer::compute_code
void compute_code(const float *x, uint8_t *code) const
Quantize one vector with the product quantizer.
Definition: ProductQuantizer.cpp:311
faiss::ProductQuantizer::Train_hot_start
the centroids are already initialized
Definition: ProductQuantizer.h:42
faiss::ProductQuantizer::cp
ClusteringParameters cp
parameters used during clustering
Definition: ProductQuantizer.h:49
diff --git a/docs/html/Select_8cuh_source.html b/docs/html/Select_8cuh_source.html index 511167d86..69a8c595f 100644 --- a/docs/html/Select_8cuh_source.html +++ b/docs/html/Select_8cuh_source.html @@ -231,433 +231,447 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
142  __device__ inline void checkThreadQ() {
143  bool needSort = (numVals == NumThreadQ);
144 
-
145  if (!__any(needSort)) {
-
146  return;
-
147  }
-
148 
-
149  // This has a trailing warpFence
-
150  mergeWarpQ();
-
151 
-
152  // Any top-k elements have been merged into the warp queue; we're
-
153  // free to reset the thread queues
-
154  numVals = 0;
+
145 #if CUDA_VERSION >= 9000
+
146  needSort = __any_sync(0xffffffff, needSort);
+
147 #else
+
148  needSort = __any(needSort);
+
149 #endif
+
150 
+
151  if (!needSort) {
+
152  // no lanes have triggered a sort
+
153  return;
+
154  }
155 
-
156 #pragma unroll
-
157  for (int i = 0; i < NumThreadQ; ++i) {
-
158  threadK[i] = initK;
-
159  threadV[i] = initV;
-
160  }
-
161 
-
162  // We have to beat at least this element
-
163  warpKTop = warpK[kMinus1];
-
164 
-
165  warpFence();
-
166  }
-
167 
-
168  /// This function handles sorting and merging together the
-
169  /// per-thread queues with the warp-wide queue, creating a sorted
-
170  /// list across both
-
171  __device__ inline void mergeWarpQ() {
-
172  int laneId = getLaneId();
-
173 
-
174  // Sort all of the per-thread queues
-
175  warpSortAnyRegisters<K, V, NumThreadQ, !Dir, Comp>(threadK, threadV);
-
176 
-
177  constexpr int kNumWarpQRegisters = NumWarpQ / kWarpSize;
-
178  K warpKRegisters[kNumWarpQRegisters];
-
179  V warpVRegisters[kNumWarpQRegisters];
+
156  // This has a trailing warpFence
+
157  mergeWarpQ();
+
158 
+
159  // Any top-k elements have been merged into the warp queue; we're
+
160  // free to reset the thread queues
+
161  numVals = 0;
+
162 
+
163 #pragma unroll
+
164  for (int i = 0; i < NumThreadQ; ++i) {
+
165  threadK[i] = initK;
+
166  threadV[i] = initV;
+
167  }
+
168 
+
169  // We have to beat at least this element
+
170  warpKTop = warpK[kMinus1];
+
171 
+
172  warpFence();
+
173  }
+
174 
+
175  /// This function handles sorting and merging together the
+
176  /// per-thread queues with the warp-wide queue, creating a sorted
+
177  /// list across both
+
178  __device__ inline void mergeWarpQ() {
+
179  int laneId = getLaneId();
180 
-
181 #pragma unroll
-
182  for (int i = 0; i < kNumWarpQRegisters; ++i) {
-
183  warpKRegisters[i] = warpK[i * kWarpSize + laneId];
-
184  warpVRegisters[i] = warpV[i * kWarpSize + laneId];
-
185  }
-
186 
-
187  warpFence();
-
188 
-
189  // The warp queue is already sorted, and now that we've sorted the
-
190  // per-thread queue, merge both sorted lists together, producing
-
191  // one sorted list
-
192  warpMergeAnyRegisters<K, V,
-
193  kNumWarpQRegisters, NumThreadQ, !Dir, Comp, false>(
-
194  warpKRegisters, warpVRegisters, threadK, threadV);
+
181  // Sort all of the per-thread queues
+
182  warpSortAnyRegisters<K, V, NumThreadQ, !Dir, Comp>(threadK, threadV);
+
183 
+
184  constexpr int kNumWarpQRegisters = NumWarpQ / kWarpSize;
+
185  K warpKRegisters[kNumWarpQRegisters];
+
186  V warpVRegisters[kNumWarpQRegisters];
+
187 
+
188 #pragma unroll
+
189  for (int i = 0; i < kNumWarpQRegisters; ++i) {
+
190  warpKRegisters[i] = warpK[i * kWarpSize + laneId];
+
191  warpVRegisters[i] = warpV[i * kWarpSize + laneId];
+
192  }
+
193 
+
194  warpFence();
195 
-
196  // Write back out the warp queue
-
197 #pragma unroll
-
198  for (int i = 0; i < kNumWarpQRegisters; ++i) {
-
199  warpK[i * kWarpSize + laneId] = warpKRegisters[i];
-
200  warpV[i * kWarpSize + laneId] = warpVRegisters[i];
-
201  }
+
196  // The warp queue is already sorted, and now that we've sorted the
+
197  // per-thread queue, merge both sorted lists together, producing
+
198  // one sorted list
+
199  warpMergeAnyRegisters<K, V,
+
200  kNumWarpQRegisters, NumThreadQ, !Dir, Comp, false>(
+
201  warpKRegisters, warpVRegisters, threadK, threadV);
202 
-
203  warpFence();
-
204  }
-
205 
-
206  /// WARNING: all threads in a warp must participate in this.
-
207  /// Otherwise, you must call the constituent parts separately.
-
208  __device__ inline void add(K k, V v) {
-
209  addThreadQ(k, v);
-
210  checkThreadQ();
+
203  // Write back out the warp queue
+
204 #pragma unroll
+
205  for (int i = 0; i < kNumWarpQRegisters; ++i) {
+
206  warpK[i * kWarpSize + laneId] = warpKRegisters[i];
+
207  warpV[i * kWarpSize + laneId] = warpVRegisters[i];
+
208  }
+
209 
+
210  warpFence();
211  }
-
212 
-
213  __device__ inline void reduce() {
-
214  // Have all warps dump and merge their queues; this will produce
-
215  // the final per-warp results
-
216  mergeWarpQ();
-
217 
-
218  // block-wide dep; thus far, all warps have been completely
-
219  // independent
-
220  __syncthreads();
-
221 
-
222  // All warp queues are contiguous in smem.
-
223  // Now, we have kNumWarps lists of NumWarpQ elements.
-
224  // This is a power of 2.
-
225  FinalBlockMerge<kNumWarps, ThreadsPerBlock, K, V, NumWarpQ, Dir, Comp>::
-
226  merge(sharedK, sharedV);
-
227 
-
228  // The block-wide merge has a trailing syncthreads
-
229  }
-
230 
-
231  // Default element key
-
232  const K initK;
-
233 
-
234  // Default element value
-
235  const V initV;
-
236 
-
237  // Number of valid elements in our thread queue
-
238  int numVals;
-
239 
-
240  // The k-th highest (Dir) or lowest (!Dir) element
-
241  K warpKTop;
-
242 
-
243  // Thread queue values
-
244  K threadK[NumThreadQ];
-
245  V threadV[NumThreadQ];
+
212 
+
213  /// WARNING: all threads in a warp must participate in this.
+
214  /// Otherwise, you must call the constituent parts separately.
+
215  __device__ inline void add(K k, V v) {
+
216  addThreadQ(k, v);
+
217  checkThreadQ();
+
218  }
+
219 
+
220  __device__ inline void reduce() {
+
221  // Have all warps dump and merge their queues; this will produce
+
222  // the final per-warp results
+
223  mergeWarpQ();
+
224 
+
225  // block-wide dep; thus far, all warps have been completely
+
226  // independent
+
227  __syncthreads();
+
228 
+
229  // All warp queues are contiguous in smem.
+
230  // Now, we have kNumWarps lists of NumWarpQ elements.
+
231  // This is a power of 2.
+
232  FinalBlockMerge<kNumWarps, ThreadsPerBlock, K, V, NumWarpQ, Dir, Comp>::
+
233  merge(sharedK, sharedV);
+
234 
+
235  // The block-wide merge has a trailing syncthreads
+
236  }
+
237 
+
238  // Default element key
+
239  const K initK;
+
240 
+
241  // Default element value
+
242  const V initV;
+
243 
+
244  // Number of valid elements in our thread queue
+
245  int numVals;
246 
-
247  // Queues for all warps
-
248  K* sharedK;
-
249  V* sharedV;
-
250 
-
251  // Our warp's queue (points into sharedK/sharedV)
-
252  // warpK[0] is highest (Dir) or lowest (!Dir)
-
253  K* warpK;
-
254  V* warpV;
-
255 
-
256  // This is a cached k-1 value
-
257  int kMinus1;
-
258 };
-
259 
-
260 /// Specialization for k == 1 (NumWarpQ == 1)
-
261 template <typename K,
-
262  typename V,
-
263  bool Dir,
-
264  typename Comp,
-
265  int NumThreadQ,
-
266  int ThreadsPerBlock>
-
267 struct BlockSelect<K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock> {
-
268  static constexpr int kNumWarps = ThreadsPerBlock / kWarpSize;
-
269 
-
270  __device__ inline BlockSelect(K initK, V initV, K* smemK, V* smemV, int k) :
-
271  sharedK(smemK),
-
272  sharedV(smemV),
-
273  threadK(initK),
-
274  threadV(initV) {
-
275  }
+
247  // The k-th highest (Dir) or lowest (!Dir) element
+
248  K warpKTop;
+
249 
+
250  // Thread queue values
+
251  K threadK[NumThreadQ];
+
252  V threadV[NumThreadQ];
+
253 
+
254  // Queues for all warps
+
255  K* sharedK;
+
256  V* sharedV;
+
257 
+
258  // Our warp's queue (points into sharedK/sharedV)
+
259  // warpK[0] is highest (Dir) or lowest (!Dir)
+
260  K* warpK;
+
261  V* warpV;
+
262 
+
263  // This is a cached k-1 value
+
264  int kMinus1;
+
265 };
+
266 
+
267 /// Specialization for k == 1 (NumWarpQ == 1)
+
268 template <typename K,
+
269  typename V,
+
270  bool Dir,
+
271  typename Comp,
+
272  int NumThreadQ,
+
273  int ThreadsPerBlock>
+
274 struct BlockSelect<K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock> {
+
275  static constexpr int kNumWarps = ThreadsPerBlock / kWarpSize;
276 
-
277  __device__ inline void addThreadQ(K k, V v) {
-
278  bool swap = Dir ? Comp::gt(k, threadK) : Comp::lt(k, threadK);
-
279  threadK = swap ? k : threadK;
-
280  threadV = swap ? v : threadV;
-
281  }
-
282 
-
283  __device__ inline void checkThreadQ() {
-
284  // We don't need to do anything here, since the warp doesn't
-
285  // cooperate until the end
-
286  }
-
287 
-
288  __device__ inline void add(K k, V v) {
-
289  addThreadQ(k, v);
-
290  }
-
291 
-
292  __device__ inline void reduce() {
-
293  // Reduce within the warp
-
294  Pair<K, V> pair(threadK, threadV);
-
295 
-
296  if (Dir) {
-
297  pair =
-
298  warpReduceAll<Pair<K, V>, Max<Pair<K, V>>>(pair, Max<Pair<K, V>>());
-
299  } else {
-
300  pair =
-
301  warpReduceAll<Pair<K, V>, Min<Pair<K, V>>>(pair, Min<Pair<K, V>>());
-
302  }
-
303 
-
304  // Each warp writes out a single value
-
305  int laneId = getLaneId();
-
306  int warpId = threadIdx.x / kWarpSize;
-
307 
-
308  if (laneId == 0) {
-
309  sharedK[warpId] = pair.k;
-
310  sharedV[warpId] = pair.v;
-
311  }
-
312 
-
313  __syncthreads();
+
277  __device__ inline BlockSelect(K initK, V initV, K* smemK, V* smemV, int k) :
+
278  sharedK(smemK),
+
279  sharedV(smemV),
+
280  threadK(initK),
+
281  threadV(initV) {
+
282  }
+
283 
+
284  __device__ inline void addThreadQ(K k, V v) {
+
285  bool swap = Dir ? Comp::gt(k, threadK) : Comp::lt(k, threadK);
+
286  threadK = swap ? k : threadK;
+
287  threadV = swap ? v : threadV;
+
288  }
+
289 
+
290  __device__ inline void checkThreadQ() {
+
291  // We don't need to do anything here, since the warp doesn't
+
292  // cooperate until the end
+
293  }
+
294 
+
295  __device__ inline void add(K k, V v) {
+
296  addThreadQ(k, v);
+
297  }
+
298 
+
299  __device__ inline void reduce() {
+
300  // Reduce within the warp
+
301  Pair<K, V> pair(threadK, threadV);
+
302 
+
303  if (Dir) {
+
304  pair =
+
305  warpReduceAll<Pair<K, V>, Max<Pair<K, V>>>(pair, Max<Pair<K, V>>());
+
306  } else {
+
307  pair =
+
308  warpReduceAll<Pair<K, V>, Min<Pair<K, V>>>(pair, Min<Pair<K, V>>());
+
309  }
+
310 
+
311  // Each warp writes out a single value
+
312  int laneId = getLaneId();
+
313  int warpId = threadIdx.x / kWarpSize;
314 
-
315  // We typically use this for small blocks (<= 128), just having the first
-
316  // thread in the block perform the reduction across warps is
-
317  // faster
-
318  if (threadIdx.x == 0) {
-
319  threadK = sharedK[0];
-
320  threadV = sharedV[0];
+
315  if (laneId == 0) {
+
316  sharedK[warpId] = pair.k;
+
317  sharedV[warpId] = pair.v;
+
318  }
+
319 
+
320  __syncthreads();
321 
-
322 #pragma unroll
-
323  for (int i = 1; i < kNumWarps; ++i) {
-
324  K k = sharedK[i];
-
325  V v = sharedV[i];
-
326 
-
327  bool swap = Dir ? Comp::gt(k, threadK) : Comp::lt(k, threadK);
-
328  threadK = swap ? k : threadK;
-
329  threadV = swap ? v : threadV;
-
330  }
-
331 
-
332  // Hopefully a thread's smem reads/writes are ordered wrt
-
333  // itself, so no barrier needed :)
-
334  sharedK[0] = threadK;
-
335  sharedV[0] = threadV;
-
336  }
-
337 
-
338  // In case other threads wish to read this value
-
339  __syncthreads();
-
340  }
-
341 
-
342  // threadK is lowest (Dir) or highest (!Dir)
-
343  K threadK;
-
344  V threadV;
-
345 
-
346  // Where we reduce in smem
-
347  K* sharedK;
-
348  V* sharedV;
-
349 };
-
350 
-
351 //
-
352 // per-warp WarpSelect
-
353 //
-
354 
-
355 // `Dir` true, produce largest values.
-
356 // `Dir` false, produce smallest values.
-
357 template <typename K,
-
358  typename V,
-
359  bool Dir,
-
360  typename Comp,
-
361  int NumWarpQ,
-
362  int NumThreadQ,
-
363  int ThreadsPerBlock>
-
364 struct WarpSelect {
-
365  static constexpr int kNumWarpQRegisters = NumWarpQ / kWarpSize;
-
366 
-
367  __device__ inline WarpSelect(K initKVal, V initVVal, int k) :
-
368  initK(initKVal),
-
369  initV(initVVal),
-
370  numVals(0),
-
371  warpKTop(initKVal),
-
372  kLane((k - 1) % kWarpSize) {
-
373  static_assert(utils::isPowerOf2(ThreadsPerBlock),
-
374  "threads must be a power-of-2");
-
375  static_assert(utils::isPowerOf2(NumWarpQ),
-
376  "warp queue must be power-of-2");
-
377 
-
378  // Fill the per-thread queue keys with the default value
-
379 #pragma unroll
-
380  for (int i = 0; i < NumThreadQ; ++i) {
-
381  threadK[i] = initK;
-
382  threadV[i] = initV;
-
383  }
+
322  // We typically use this for small blocks (<= 128), just having the first
+
323  // thread in the block perform the reduction across warps is
+
324  // faster
+
325  if (threadIdx.x == 0) {
+
326  threadK = sharedK[0];
+
327  threadV = sharedV[0];
+
328 
+
329 #pragma unroll
+
330  for (int i = 1; i < kNumWarps; ++i) {
+
331  K k = sharedK[i];
+
332  V v = sharedV[i];
+
333 
+
334  bool swap = Dir ? Comp::gt(k, threadK) : Comp::lt(k, threadK);
+
335  threadK = swap ? k : threadK;
+
336  threadV = swap ? v : threadV;
+
337  }
+
338 
+
339  // Hopefully a thread's smem reads/writes are ordered wrt
+
340  // itself, so no barrier needed :)
+
341  sharedK[0] = threadK;
+
342  sharedV[0] = threadV;
+
343  }
+
344 
+
345  // In case other threads wish to read this value
+
346  __syncthreads();
+
347  }
+
348 
+
349  // threadK is lowest (Dir) or highest (!Dir)
+
350  K threadK;
+
351  V threadV;
+
352 
+
353  // Where we reduce in smem
+
354  K* sharedK;
+
355  V* sharedV;
+
356 };
+
357 
+
358 //
+
359 // per-warp WarpSelect
+
360 //
+
361 
+
362 // `Dir` true, produce largest values.
+
363 // `Dir` false, produce smallest values.
+
364 template <typename K,
+
365  typename V,
+
366  bool Dir,
+
367  typename Comp,
+
368  int NumWarpQ,
+
369  int NumThreadQ,
+
370  int ThreadsPerBlock>
+
371 struct WarpSelect {
+
372  static constexpr int kNumWarpQRegisters = NumWarpQ / kWarpSize;
+
373 
+
374  __device__ inline WarpSelect(K initKVal, V initVVal, int k) :
+
375  initK(initKVal),
+
376  initV(initVVal),
+
377  numVals(0),
+
378  warpKTop(initKVal),
+
379  kLane((k - 1) % kWarpSize) {
+
380  static_assert(utils::isPowerOf2(ThreadsPerBlock),
+
381  "threads must be a power-of-2");
+
382  static_assert(utils::isPowerOf2(NumWarpQ),
+
383  "warp queue must be power-of-2");
384 
-
385  // Fill the warp queue with the default value
+
385  // Fill the per-thread queue keys with the default value
386 #pragma unroll
-
387  for (int i = 0; i < kNumWarpQRegisters; ++i) {
-
388  warpK[i] = initK;
-
389  warpV[i] = initV;
+
387  for (int i = 0; i < NumThreadQ; ++i) {
+
388  threadK[i] = initK;
+
389  threadV[i] = initV;
390  }
-
391  }
-
392 
-
393  __device__ inline void addThreadQ(K k, V v) {
-
394  if (Dir ? Comp::gt(k, warpKTop) : Comp::lt(k, warpKTop)) {
-
395  // Rotate right
-
396 #pragma unroll
-
397  for (int i = NumThreadQ - 1; i > 0; --i) {
-
398  threadK[i] = threadK[i - 1];
-
399  threadV[i] = threadV[i - 1];
-
400  }
-
401 
-
402  threadK[0] = k;
-
403  threadV[0] = v;
-
404  ++numVals;
-
405  }
-
406  }
-
407 
-
408  __device__ inline void checkThreadQ() {
-
409  bool needSort = (numVals == NumThreadQ);
-
410 
-
411  if (!__any(needSort)) {
-
412  return;
-
413  }
+
391 
+
392  // Fill the warp queue with the default value
+
393 #pragma unroll
+
394  for (int i = 0; i < kNumWarpQRegisters; ++i) {
+
395  warpK[i] = initK;
+
396  warpV[i] = initV;
+
397  }
+
398  }
+
399 
+
400  __device__ inline void addThreadQ(K k, V v) {
+
401  if (Dir ? Comp::gt(k, warpKTop) : Comp::lt(k, warpKTop)) {
+
402  // Rotate right
+
403 #pragma unroll
+
404  for (int i = NumThreadQ - 1; i > 0; --i) {
+
405  threadK[i] = threadK[i - 1];
+
406  threadV[i] = threadV[i - 1];
+
407  }
+
408 
+
409  threadK[0] = k;
+
410  threadV[0] = v;
+
411  ++numVals;
+
412  }
+
413  }
414 
-
415  mergeWarpQ();
-
416 
-
417  // Any top-k elements have been merged into the warp queue; we're
-
418  // free to reset the thread queues
-
419  numVals = 0;
-
420 
-
421 #pragma unroll
-
422  for (int i = 0; i < NumThreadQ; ++i) {
-
423  threadK[i] = initK;
-
424  threadV[i] = initV;
-
425  }
-
426 
-
427  // We have to beat at least this element
-
428  warpKTop = shfl(warpK[kNumWarpQRegisters - 1], kLane);
-
429  }
-
430 
-
431  /// This function handles sorting and merging together the
-
432  /// per-thread queues with the warp-wide queue, creating a sorted
-
433  /// list across both
-
434  __device__ inline void mergeWarpQ() {
-
435  // Sort all of the per-thread queues
-
436  warpSortAnyRegisters<K, V, NumThreadQ, !Dir, Comp>(threadK, threadV);
-
437 
-
438  // The warp queue is already sorted, and now that we've sorted the
-
439  // per-thread queue, merge both sorted lists together, producing
-
440  // one sorted list
-
441  warpMergeAnyRegisters<K, V,
-
442  kNumWarpQRegisters, NumThreadQ, !Dir, Comp, false>(
-
443  warpK, warpV, threadK, threadV);
-
444  }
-
445 
-
446  /// WARNING: all threads in a warp must participate in this.
-
447  /// Otherwise, you must call the constituent parts separately.
-
448  __device__ inline void add(K k, V v) {
-
449  addThreadQ(k, v);
-
450  checkThreadQ();
-
451  }
-
452 
-
453  __device__ inline void reduce() {
-
454  // Have all warps dump and merge their queues; this will produce
-
455  // the final per-warp results
-
456  mergeWarpQ();
-
457  }
-
458 
-
459  /// Dump final k selected values for this warp out
-
460  __device__ inline void writeOut(K* outK, V* outV, int k) {
-
461  int laneId = getLaneId();
-
462 
-
463 #pragma unroll
-
464  for (int i = 0; i < kNumWarpQRegisters; ++i) {
-
465  int idx = i * kWarpSize + laneId;
+
415  __device__ inline void checkThreadQ() {
+
416  bool needSort = (numVals == NumThreadQ);
+
417 
+
418 #if CUDA_VERSION >= 9000
+
419  needSort = __any_sync(0xffffffff, needSort);
+
420 #else
+
421  needSort = __any(needSort);
+
422 #endif
+
423 
+
424  if (!needSort) {
+
425  // no lanes have triggered a sort
+
426  return;
+
427  }
+
428 
+
429  mergeWarpQ();
+
430 
+
431  // Any top-k elements have been merged into the warp queue; we're
+
432  // free to reset the thread queues
+
433  numVals = 0;
+
434 
+
435 #pragma unroll
+
436  for (int i = 0; i < NumThreadQ; ++i) {
+
437  threadK[i] = initK;
+
438  threadV[i] = initV;
+
439  }
+
440 
+
441  // We have to beat at least this element
+
442  warpKTop = shfl(warpK[kNumWarpQRegisters - 1], kLane);
+
443  }
+
444 
+
445  /// This function handles sorting and merging together the
+
446  /// per-thread queues with the warp-wide queue, creating a sorted
+
447  /// list across both
+
448  __device__ inline void mergeWarpQ() {
+
449  // Sort all of the per-thread queues
+
450  warpSortAnyRegisters<K, V, NumThreadQ, !Dir, Comp>(threadK, threadV);
+
451 
+
452  // The warp queue is already sorted, and now that we've sorted the
+
453  // per-thread queue, merge both sorted lists together, producing
+
454  // one sorted list
+
455  warpMergeAnyRegisters<K, V,
+
456  kNumWarpQRegisters, NumThreadQ, !Dir, Comp, false>(
+
457  warpK, warpV, threadK, threadV);
+
458  }
+
459 
+
460  /// WARNING: all threads in a warp must participate in this.
+
461  /// Otherwise, you must call the constituent parts separately.
+
462  __device__ inline void add(K k, V v) {
+
463  addThreadQ(k, v);
+
464  checkThreadQ();
+
465  }
466 
-
467  if (idx < k) {
-
468  outK[idx] = warpK[i];
-
469  outV[idx] = warpV[i];
-
470  }
-
471  }
-
472  }
-
473 
-
474  // Default element key
-
475  const K initK;
+
467  __device__ inline void reduce() {
+
468  // Have all warps dump and merge their queues; this will produce
+
469  // the final per-warp results
+
470  mergeWarpQ();
+
471  }
+
472 
+
473  /// Dump final k selected values for this warp out
+
474  __device__ inline void writeOut(K* outK, V* outV, int k) {
+
475  int laneId = getLaneId();
476 
-
477  // Default element value
-
478  const V initV;
-
479 
-
480  // Number of valid elements in our thread queue
-
481  int numVals;
-
482 
-
483  // The k-th highest (Dir) or lowest (!Dir) element
-
484  K warpKTop;
-
485 
-
486  // Thread queue values
-
487  K threadK[NumThreadQ];
-
488  V threadV[NumThreadQ];
-
489 
-
490  // warpK[0] is highest (Dir) or lowest (!Dir)
-
491  K warpK[kNumWarpQRegisters];
-
492  V warpV[kNumWarpQRegisters];
+
477 #pragma unroll
+
478  for (int i = 0; i < kNumWarpQRegisters; ++i) {
+
479  int idx = i * kWarpSize + laneId;
+
480 
+
481  if (idx < k) {
+
482  outK[idx] = warpK[i];
+
483  outV[idx] = warpV[i];
+
484  }
+
485  }
+
486  }
+
487 
+
488  // Default element key
+
489  const K initK;
+
490 
+
491  // Default element value
+
492  const V initV;
493 
-
494  // This is what lane we should load an approximation (>=k) to the
-
495  // kth element from the last register in the warp queue (i.e.,
-
496  // warpK[kNumWarpQRegisters - 1]).
-
497  int kLane;
-
498 };
-
499 
-
500 /// Specialization for k == 1 (NumWarpQ == 1)
-
501 template <typename K,
-
502  typename V,
-
503  bool Dir,
-
504  typename Comp,
-
505  int NumThreadQ,
-
506  int ThreadsPerBlock>
-
507 struct WarpSelect<K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock> {
-
508  static constexpr int kNumWarps = ThreadsPerBlock / kWarpSize;
-
509 
-
510  __device__ inline WarpSelect(K initK, V initV, int k) :
-
511  threadK(initK),
-
512  threadV(initV) {
-
513  }
-
514 
-
515  __device__ inline void addThreadQ(K k, V v) {
-
516  bool swap = Dir ? Comp::gt(k, threadK) : Comp::lt(k, threadK);
-
517  threadK = swap ? k : threadK;
-
518  threadV = swap ? v : threadV;
-
519  }
-
520 
-
521  __device__ inline void checkThreadQ() {
-
522  // We don't need to do anything here, since the warp doesn't
-
523  // cooperate until the end
-
524  }
-
525 
-
526  __device__ inline void add(K k, V v) {
-
527  addThreadQ(k, v);
-
528  }
-
529 
-
530  __device__ inline void reduce() {
-
531  // Reduce within the warp
-
532  Pair<K, V> pair(threadK, threadV);
-
533 
-
534  if (Dir) {
-
535  pair =
-
536  warpReduceAll<Pair<K, V>, Max<Pair<K, V>>>(pair, Max<Pair<K, V>>());
-
537  } else {
-
538  pair =
-
539  warpReduceAll<Pair<K, V>, Min<Pair<K, V>>>(pair, Min<Pair<K, V>>());
-
540  }
-
541 
-
542  threadK = pair.k;
-
543  threadV = pair.v;
-
544  }
-
545 
-
546  /// Dump final k selected values for this warp out
-
547  __device__ inline void writeOut(K* outK, V* outV, int k) {
-
548  if (getLaneId() == 0) {
-
549  *outK = threadK;
-
550  *outV = threadV;
-
551  }
-
552  }
-
553 
-
554  // threadK is lowest (Dir) or highest (!Dir)
-
555  K threadK;
-
556  V threadV;
-
557 };
-
558 
-
559 } } // namespace
+
494  // Number of valid elements in our thread queue
+
495  int numVals;
+
496 
+
497  // The k-th highest (Dir) or lowest (!Dir) element
+
498  K warpKTop;
+
499 
+
500  // Thread queue values
+
501  K threadK[NumThreadQ];
+
502  V threadV[NumThreadQ];
+
503 
+
504  // warpK[0] is highest (Dir) or lowest (!Dir)
+
505  K warpK[kNumWarpQRegisters];
+
506  V warpV[kNumWarpQRegisters];
+
507 
+
508  // This is what lane we should load an approximation (>=k) to the
+
509  // kth element from the last register in the warp queue (i.e.,
+
510  // warpK[kNumWarpQRegisters - 1]).
+
511  int kLane;
+
512 };
+
513 
+
514 /// Specialization for k == 1 (NumWarpQ == 1)
+
515 template <typename K,
+
516  typename V,
+
517  bool Dir,
+
518  typename Comp,
+
519  int NumThreadQ,
+
520  int ThreadsPerBlock>
+
521 struct WarpSelect<K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock> {
+
522  static constexpr int kNumWarps = ThreadsPerBlock / kWarpSize;
+
523 
+
524  __device__ inline WarpSelect(K initK, V initV, int k) :
+
525  threadK(initK),
+
526  threadV(initV) {
+
527  }
+
528 
+
529  __device__ inline void addThreadQ(K k, V v) {
+
530  bool swap = Dir ? Comp::gt(k, threadK) : Comp::lt(k, threadK);
+
531  threadK = swap ? k : threadK;
+
532  threadV = swap ? v : threadV;
+
533  }
+
534 
+
535  __device__ inline void checkThreadQ() {
+
536  // We don't need to do anything here, since the warp doesn't
+
537  // cooperate until the end
+
538  }
+
539 
+
540  __device__ inline void add(K k, V v) {
+
541  addThreadQ(k, v);
+
542  }
+
543 
+
544  __device__ inline void reduce() {
+
545  // Reduce within the warp
+
546  Pair<K, V> pair(threadK, threadV);
+
547 
+
548  if (Dir) {
+
549  pair =
+
550  warpReduceAll<Pair<K, V>, Max<Pair<K, V>>>(pair, Max<Pair<K, V>>());
+
551  } else {
+
552  pair =
+
553  warpReduceAll<Pair<K, V>, Min<Pair<K, V>>>(pair, Min<Pair<K, V>>());
+
554  }
+
555 
+
556  threadK = pair.k;
+
557  threadV = pair.v;
+
558  }
+
559 
+
560  /// Dump final k selected values for this warp out
+
561  __device__ inline void writeOut(K* outK, V* outV, int k) {
+
562  if (getLaneId() == 0) {
+
563  *outK = threadK;
+
564  *outV = threadV;
+
565  }
+
566  }
+
567 
+
568  // threadK is lowest (Dir) or highest (!Dir)
+
569  K threadK;
+
570  V threadV;
+
571 };
+
572 
+
573 } } // namespace
faiss::gpu::Pair
A simple pair type for CUDA device usage.
Definition: Pair.cuh:21
-
faiss::gpu::WarpSelect< K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock >::writeOut
__device__ void writeOut(K *outK, V *outV, int k)
Dump final k selected values for this warp out.
Definition: Select.cuh:547
+
faiss::gpu::WarpSelect< K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock >::writeOut
__device__ void writeOut(K *outK, V *outV, int k)
Dump final k selected values for this warp out.
Definition: Select.cuh:561
- -
faiss::gpu::WarpSelect::writeOut
__device__ void writeOut(K *outK, V *outV, int k)
Dump final k selected values for this warp out.
Definition: Select.cuh:460
-
faiss::gpu::WarpSelect::add
__device__ void add(K k, V v)
Definition: Select.cuh:448
+ +
faiss::gpu::WarpSelect::writeOut
__device__ void writeOut(K *outK, V *outV, int k)
Dump final k selected values for this warp out.
Definition: Select.cuh:474
+
faiss::gpu::WarpSelect::add
__device__ void add(K k, V v)
Definition: Select.cuh:462
-
faiss::gpu::BlockSelect::add
__device__ void add(K k, V v)
Definition: Select.cuh:208
-
faiss::gpu::WarpSelect::mergeWarpQ
__device__ void mergeWarpQ()
Definition: Select.cuh:434
+
faiss::gpu::BlockSelect::add
__device__ void add(K k, V v)
Definition: Select.cuh:215
+
faiss::gpu::WarpSelect::mergeWarpQ
__device__ void mergeWarpQ()
Definition: Select.cuh:448
-
faiss::gpu::BlockSelect::mergeWarpQ
__device__ void mergeWarpQ()
Definition: Select.cuh:171
+
faiss::gpu::BlockSelect::mergeWarpQ
__device__ void mergeWarpQ()
Definition: Select.cuh:178
diff --git a/docs/html/Tensor-inl_8cuh_source.html b/docs/html/Tensor-inl_8cuh_source.html index bca25a325..cfc0c854b 100644 --- a/docs/html/Tensor-inl_8cuh_source.html +++ b/docs/html/Tensor-inl_8cuh_source.html @@ -119,687 +119,719 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
30 template <typename T, int Dim, bool InnerContig,
31  typename IndexT, template <typename U> class PtrTraits>
32 __host__ __device__
-
33 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&
-
34 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator=(
-
35  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&& t) {
-
36  data_ = t.data_; t.data_ = nullptr;
-
37  for (int i = 0; i < Dim; ++i) {
-
38  stride_[i] = t.stride_[i]; t.stride_[i] = 0;
-
39  size_[i] = t.size_[i]; t.size_[i] = 0;
-
40  }
-
41 
-
42  return *this;
-
43 }
-
44 
-
45 template <typename T, int Dim, bool InnerContig,
-
46  typename IndexT, template <typename U> class PtrTraits>
-
47 __host__ __device__
-
48 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::
-
49 Tensor(DataPtrType data, const IndexT sizes[Dim])
-
50  : data_(data) {
-
51  static_assert(Dim > 0, "must have > 0 dimensions");
-
52 
+
33 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::Tensor(
+
34  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t) {
+
35  this->operator=(t);
+
36 }
+
37 
+
38 template <typename T, int Dim, bool InnerContig,
+
39  typename IndexT, template <typename U> class PtrTraits>
+
40 __host__ __device__
+
41 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::Tensor(
+
42  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&& t) {
+
43  this->operator=(std::move(t));
+
44 }
+
45 
+
46 template <typename T, int Dim, bool InnerContig,
+
47  typename IndexT, template <typename U> class PtrTraits>
+
48 __host__ __device__
+
49 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&
+
50 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator=(
+
51  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t) {
+
52  data_ = t.data_;
53  for (int i = 0; i < Dim; ++i) {
-
54  size_[i] = sizes[i];
-
55  }
-
56 
-
57  stride_[Dim - 1] = (IndexT) 1;
-
58  for (int i = Dim - 2; i >= 0; --i) {
-
59  stride_[i] = stride_[i + 1] * sizes[i + 1];
-
60  }
-
61 }
-
62 
-
63 template <typename T, int Dim, bool InnerContig,
-
64  typename IndexT, template <typename U> class PtrTraits>
-
65 __host__ __device__
-
66 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::
-
67 Tensor(DataPtrType data, std::initializer_list<IndexT> sizes)
-
68  : data_(data) {
-
69  GPU_FAISS_ASSERT(sizes.size() == Dim);
-
70  static_assert(Dim > 0, "must have > 0 dimensions");
-
71 
-
72  int i = 0;
-
73  for (auto s : sizes) {
-
74  size_[i++] = s;
-
75  }
-
76 
-
77  stride_[Dim - 1] = (IndexT) 1;
-
78  for (int j = Dim - 2; j >= 0; --j) {
-
79  stride_[j] = stride_[j + 1] * size_[j + 1];
-
80  }
-
81 }
-
82 
+
54  size_[i] = t.size_[i];
+
55  stride_[i] = t.stride_[i];
+
56  }
+
57 
+
58  return *this;
+
59 }
+
60 
+
61 template <typename T, int Dim, bool InnerContig,
+
62  typename IndexT, template <typename U> class PtrTraits>
+
63 __host__ __device__
+
64 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&
+
65 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator=(
+
66  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&& t) {
+
67  data_ = t.data_; t.data_ = nullptr;
+
68  for (int i = 0; i < Dim; ++i) {
+
69  stride_[i] = t.stride_[i]; t.stride_[i] = 0;
+
70  size_[i] = t.size_[i]; t.size_[i] = 0;
+
71  }
+
72 
+
73  return *this;
+
74 }
+
75 
+
76 template <typename T, int Dim, bool InnerContig,
+
77  typename IndexT, template <typename U> class PtrTraits>
+
78 __host__ __device__
+
79 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::
+
80 Tensor(DataPtrType data, const IndexT sizes[Dim])
+
81  : data_(data) {
+
82  static_assert(Dim > 0, "must have > 0 dimensions");
83 
-
84 template <typename T, int Dim, bool InnerContig,
-
85  typename IndexT, template <typename U> class PtrTraits>
-
86 __host__ __device__
-
87 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::Tensor(
-
88  DataPtrType data, const IndexT sizes[Dim], const IndexT strides[Dim])
-
89  : data_(data) {
-
90  static_assert(Dim > 0, "must have > 0 dimensions");
-
91 
-
92  for (int i = 0; i < Dim; ++i) {
-
93  size_[i] = sizes[i];
-
94  stride_[i] = strides[i];
-
95  }
-
96 }
-
97 
-
98 template <typename T, int Dim, bool InnerContig,
-
99  typename IndexT, template <typename U> class PtrTraits>
-
100 __host__ void
-
101 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::copyFrom(
-
102  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
-
103  cudaStream_t stream) {
-
104  // The tensor must be fully contiguous
-
105  GPU_FAISS_ASSERT(this->isContiguous());
-
106 
-
107  // Size must be the same (since dimensions are checked and
-
108  // continuity is assumed, we need only check total number of
-
109  // elements
-
110  GPU_FAISS_ASSERT(this->numElements() == t.numElements());
-
111 
-
112  if (t.numElements() > 0) {
-
113  GPU_FAISS_ASSERT(this->data_);
-
114  GPU_FAISS_ASSERT(t.data());
-
115 
-
116  int ourDev = getDeviceForAddress(this->data_);
-
117  int tDev = getDeviceForAddress(t.data());
-
118 
-
119  if (tDev == -1) {
-
120  CUDA_VERIFY(cudaMemcpyAsync(this->data_,
-
121  t.data(),
-
122  this->getSizeInBytes(),
-
123  ourDev == -1 ? cudaMemcpyHostToHost :
-
124  cudaMemcpyHostToDevice,
-
125  stream));
-
126  } else {
-
127  CUDA_VERIFY(cudaMemcpyAsync(this->data_,
-
128  t.data(),
-
129  this->getSizeInBytes(),
-
130  ourDev == -1 ? cudaMemcpyDeviceToHost :
-
131  cudaMemcpyDeviceToDevice,
-
132  stream));
-
133  }
-
134  }
-
135 }
-
136 
-
137 template <typename T, int Dim, bool InnerContig,
-
138  typename IndexT, template <typename U> class PtrTraits>
-
139 __host__ void
-
140 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::copyTo(
-
141  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
-
142  cudaStream_t stream) {
-
143  // The tensor must be fully contiguous
-
144  GPU_FAISS_ASSERT(this->isContiguous());
-
145 
-
146  // Size must be the same (since dimensions are checked and
-
147  // continuity is assumed, we need only check total number of
-
148  // elements
-
149  GPU_FAISS_ASSERT(this->numElements() == t.numElements());
-
150 
-
151  if (t.numElements() > 0) {
-
152  GPU_FAISS_ASSERT(this->data_);
-
153  GPU_FAISS_ASSERT(t.data());
-
154 
-
155  int ourDev = getDeviceForAddress(this->data_);
-
156  int tDev = getDeviceForAddress(t.data());
-
157 
-
158  if (tDev == -1) {
-
159  CUDA_VERIFY(cudaMemcpyAsync(t.data(),
-
160  this->data_,
-
161  this->getSizeInBytes(),
-
162  ourDev == -1 ? cudaMemcpyHostToHost :
-
163  cudaMemcpyDeviceToHost,
-
164  stream));
-
165  } else {
-
166  CUDA_VERIFY(cudaMemcpyAsync(t.data(),
-
167  this->data_,
-
168  this->getSizeInBytes(),
-
169  ourDev == -1 ? cudaMemcpyHostToDevice :
-
170  cudaMemcpyDeviceToDevice,
-
171  stream));
-
172  }
-
173  }
-
174 }
-
175 
-
176 template <typename T, int Dim, bool InnerContig,
-
177  typename IndexT, template <typename U> class PtrTraits>
-
178 template <typename OtherT, int OtherDim>
-
179 __host__ __device__ bool
-
180 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isSame(
-
181  const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const {
-
182  if (Dim != OtherDim) {
-
183  return false;
-
184  }
+
84  for (int i = 0; i < Dim; ++i) {
+
85  size_[i] = sizes[i];
+
86  }
+
87 
+
88  stride_[Dim - 1] = (IndexT) 1;
+
89  for (int i = Dim - 2; i >= 0; --i) {
+
90  stride_[i] = stride_[i + 1] * sizes[i + 1];
+
91  }
+
92 }
+
93 
+
94 template <typename T, int Dim, bool InnerContig,
+
95  typename IndexT, template <typename U> class PtrTraits>
+
96 __host__ __device__
+
97 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::
+
98 Tensor(DataPtrType data, std::initializer_list<IndexT> sizes)
+
99  : data_(data) {
+
100  GPU_FAISS_ASSERT(sizes.size() == Dim);
+
101  static_assert(Dim > 0, "must have > 0 dimensions");
+
102 
+
103  int i = 0;
+
104  for (auto s : sizes) {
+
105  size_[i++] = s;
+
106  }
+
107 
+
108  stride_[Dim - 1] = (IndexT) 1;
+
109  for (int j = Dim - 2; j >= 0; --j) {
+
110  stride_[j] = stride_[j + 1] * size_[j + 1];
+
111  }
+
112 }
+
113 
+
114 
+
115 template <typename T, int Dim, bool InnerContig,
+
116  typename IndexT, template <typename U> class PtrTraits>
+
117 __host__ __device__
+
118 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::Tensor(
+
119  DataPtrType data, const IndexT sizes[Dim], const IndexT strides[Dim])
+
120  : data_(data) {
+
121  static_assert(Dim > 0, "must have > 0 dimensions");
+
122 
+
123  for (int i = 0; i < Dim; ++i) {
+
124  size_[i] = sizes[i];
+
125  stride_[i] = strides[i];
+
126  }
+
127 }
+
128 
+
129 template <typename T, int Dim, bool InnerContig,
+
130  typename IndexT, template <typename U> class PtrTraits>
+
131 __host__ void
+
132 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::copyFrom(
+
133  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
+
134  cudaStream_t stream) {
+
135  // The tensor must be fully contiguous
+
136  GPU_FAISS_ASSERT(this->isContiguous());
+
137 
+
138  // Size must be the same (since dimensions are checked and
+
139  // continuity is assumed, we need only check total number of
+
140  // elements
+
141  GPU_FAISS_ASSERT(this->numElements() == t.numElements());
+
142 
+
143  if (t.numElements() > 0) {
+
144  GPU_FAISS_ASSERT(this->data_);
+
145  GPU_FAISS_ASSERT(t.data());
+
146 
+
147  int ourDev = getDeviceForAddress(this->data_);
+
148  int tDev = getDeviceForAddress(t.data());
+
149 
+
150  if (tDev == -1) {
+
151  CUDA_VERIFY(cudaMemcpyAsync(this->data_,
+
152  t.data(),
+
153  this->getSizeInBytes(),
+
154  ourDev == -1 ? cudaMemcpyHostToHost :
+
155  cudaMemcpyHostToDevice,
+
156  stream));
+
157  } else {
+
158  CUDA_VERIFY(cudaMemcpyAsync(this->data_,
+
159  t.data(),
+
160  this->getSizeInBytes(),
+
161  ourDev == -1 ? cudaMemcpyDeviceToHost :
+
162  cudaMemcpyDeviceToDevice,
+
163  stream));
+
164  }
+
165  }
+
166 }
+
167 
+
168 template <typename T, int Dim, bool InnerContig,
+
169  typename IndexT, template <typename U> class PtrTraits>
+
170 __host__ void
+
171 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::copyTo(
+
172  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
+
173  cudaStream_t stream) {
+
174  // The tensor must be fully contiguous
+
175  GPU_FAISS_ASSERT(this->isContiguous());
+
176 
+
177  // Size must be the same (since dimensions are checked and
+
178  // continuity is assumed, we need only check total number of
+
179  // elements
+
180  GPU_FAISS_ASSERT(this->numElements() == t.numElements());
+
181 
+
182  if (t.numElements() > 0) {
+
183  GPU_FAISS_ASSERT(this->data_);
+
184  GPU_FAISS_ASSERT(t.data());
185 
-
186  for (int i = 0; i < Dim; ++i) {
-
187  if (this->getSize(i) != rhs.getSize(i)) {
-
188  return false;
-
189  }
-
190 
-
191  if (this->getStride(i) != rhs.getStride(i)) {
-
192  return false;
-
193  }
-
194  }
-
195 
-
196  return true;
-
197 }
-
198 
-
199 template <typename T, int Dim, bool InnerContig,
-
200  typename IndexT, template <typename U> class PtrTraits>
-
201 template <typename OtherT, int OtherDim>
-
202 __host__ __device__ bool
-
203 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isSameSize(
-
204  const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const {
-
205  if (Dim != OtherDim) {
-
206  return false;
-
207  }
-
208 
-
209  for (int i = 0; i < Dim; ++i) {
-
210  if (this->getSize(i) != rhs.getSize(i)) {
-
211  return false;
-
212  }
-
213  }
-
214 
-
215  return true;
-
216 }
-
217 
-
218 template <typename T, int Dim, bool InnerContig,
-
219  typename IndexT, template <typename U> class PtrTraits>
-
220 template <typename U>
-
221 __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
-
222 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::cast() {
-
223  static_assert(sizeof(U) == sizeof(T), "cast must be to same size object");
-
224 
-
225  return Tensor<U, Dim, InnerContig, IndexT, PtrTraits>(
-
226  reinterpret_cast<U*>(data_), size_, stride_);
-
227 }
-
228 
-
229 template <typename T, int Dim, bool InnerContig,
-
230  typename IndexT, template <typename U> class PtrTraits>
-
231 template <typename U>
-
232 __host__ __device__ const Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
-
233 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::cast() const {
-
234  static_assert(sizeof(U) == sizeof(T), "cast must be to same size object");
-
235 
-
236  return Tensor<U, Dim, InnerContig, IndexT, PtrTraits>(
-
237  reinterpret_cast<U*>(data_), size_, stride_);
-
238 }
+
186  int ourDev = getDeviceForAddress(this->data_);
+
187  int tDev = getDeviceForAddress(t.data());
+
188 
+
189  if (tDev == -1) {
+
190  CUDA_VERIFY(cudaMemcpyAsync(t.data(),
+
191  this->data_,
+
192  this->getSizeInBytes(),
+
193  ourDev == -1 ? cudaMemcpyHostToHost :
+
194  cudaMemcpyDeviceToHost,
+
195  stream));
+
196  } else {
+
197  CUDA_VERIFY(cudaMemcpyAsync(t.data(),
+
198  this->data_,
+
199  this->getSizeInBytes(),
+
200  ourDev == -1 ? cudaMemcpyHostToDevice :
+
201  cudaMemcpyDeviceToDevice,
+
202  stream));
+
203  }
+
204  }
+
205 }
+
206 
+
207 template <typename T, int Dim, bool InnerContig,
+
208  typename IndexT, template <typename U> class PtrTraits>
+
209 template <typename OtherT, int OtherDim>
+
210 __host__ __device__ bool
+
211 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isSame(
+
212  const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const {
+
213  if (Dim != OtherDim) {
+
214  return false;
+
215  }
+
216 
+
217  for (int i = 0; i < Dim; ++i) {
+
218  if (this->getSize(i) != rhs.getSize(i)) {
+
219  return false;
+
220  }
+
221 
+
222  if (this->getStride(i) != rhs.getStride(i)) {
+
223  return false;
+
224  }
+
225  }
+
226 
+
227  return true;
+
228 }
+
229 
+
230 template <typename T, int Dim, bool InnerContig,
+
231  typename IndexT, template <typename U> class PtrTraits>
+
232 template <typename OtherT, int OtherDim>
+
233 __host__ __device__ bool
+
234 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isSameSize(
+
235  const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const {
+
236  if (Dim != OtherDim) {
+
237  return false;
+
238  }
239 
-
240 template <typename T, int Dim, bool InnerContig,
-
241  typename IndexT, template <typename U> class PtrTraits>
-
242 template <typename U>
-
243 __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
-
244 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::castResize() {
-
245  static_assert(sizeof(U) >= sizeof(T), "only handles greater sizes");
-
246  constexpr int kMultiple = sizeof(U) / sizeof(T);
-
247 
-
248  GPU_FAISS_ASSERT(canCastResize<U>());
-
249 
-
250  IndexT newSize[Dim];
-
251  IndexT newStride[Dim];
-
252 
-
253  for (int i = 0; i < Dim - 1; ++i) {
-
254  newSize[i] = size_[i];
-
255  newStride[i] = stride_[i] / kMultiple;
-
256  }
-
257 
-
258  newStride[Dim - 1] = 1; // this is the same as the old stride
-
259  newSize[Dim - 1] = size_[Dim - 1] / kMultiple;
-
260 
-
261  return Tensor<U, Dim, InnerContig, IndexT, PtrTraits>(
-
262  reinterpret_cast<U*>(data_), newSize, newStride);
-
263 }
-
264 
-
265 template <typename T, int Dim, bool InnerContig,
-
266  typename IndexT, template <typename U> class PtrTraits>
-
267 template <typename U>
-
268 __host__ __device__ const Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
-
269 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::castResize() const {
-
270  return const_cast<Tensor<T, Dim, InnerContig, IndexT, PtrTraits>*>(this)->
-
271  castResize<U>();
-
272 }
-
273 
-
274 template <typename T, int Dim, bool InnerContig,
-
275  typename IndexT, template <typename U> class PtrTraits>
-
276 template <typename U>
-
277 __host__ __device__ bool
-
278 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::canCastResize() const {
-
279  static_assert(sizeof(U) >= sizeof(T), "only handles greater sizes");
-
280  constexpr int kMultiple = sizeof(U) / sizeof(T);
-
281 
-
282  // Check all outer strides
-
283  for (int i = 0; i < Dim - 1; ++i) {
-
284  if (stride_[i] % kMultiple != 0) {
-
285  return false;
-
286  }
+
240  for (int i = 0; i < Dim; ++i) {
+
241  if (this->getSize(i) != rhs.getSize(i)) {
+
242  return false;
+
243  }
+
244  }
+
245 
+
246  return true;
+
247 }
+
248 
+
249 template <typename T, int Dim, bool InnerContig,
+
250  typename IndexT, template <typename U> class PtrTraits>
+
251 template <typename U>
+
252 __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
+
253 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::cast() {
+
254  static_assert(sizeof(U) == sizeof(T), "cast must be to same size object");
+
255 
+
256  return Tensor<U, Dim, InnerContig, IndexT, PtrTraits>(
+
257  reinterpret_cast<U*>(data_), size_, stride_);
+
258 }
+
259 
+
260 template <typename T, int Dim, bool InnerContig,
+
261  typename IndexT, template <typename U> class PtrTraits>
+
262 template <typename U>
+
263 __host__ __device__ const Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
+
264 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::cast() const {
+
265  static_assert(sizeof(U) == sizeof(T), "cast must be to same size object");
+
266 
+
267  return Tensor<U, Dim, InnerContig, IndexT, PtrTraits>(
+
268  reinterpret_cast<U*>(data_), size_, stride_);
+
269 }
+
270 
+
271 template <typename T, int Dim, bool InnerContig,
+
272  typename IndexT, template <typename U> class PtrTraits>
+
273 template <typename U>
+
274 __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
+
275 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::castResize() {
+
276  static_assert(sizeof(U) >= sizeof(T), "only handles greater sizes");
+
277  constexpr int kMultiple = sizeof(U) / sizeof(T);
+
278 
+
279  GPU_FAISS_ASSERT(canCastResize<U>());
+
280 
+
281  IndexT newSize[Dim];
+
282  IndexT newStride[Dim];
+
283 
+
284  for (int i = 0; i < Dim - 1; ++i) {
+
285  newSize[i] = size_[i];
+
286  newStride[i] = stride_[i] / kMultiple;
287  }
288 
-
289  // Check inner size
-
290  if (size_[Dim - 1] % kMultiple != 0) {
-
291  return false;
-
292  }
-
293 
-
294  if (stride_[Dim - 1] != 1) {
-
295  return false;
-
296  }
-
297 
-
298  return true;
-
299 }
-
300 
-
301 template <typename T, int Dim, bool InnerContig,
-
302  typename IndexT, template <typename U> class PtrTraits>
-
303 template <typename NewIndexT>
-
304 __host__ Tensor<T, Dim, InnerContig, NewIndexT, PtrTraits>
-
305 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::castIndexType() const {
-
306  if (sizeof(NewIndexT) < sizeof(IndexT)) {
-
307  GPU_FAISS_ASSERT(this->canUseIndexType<NewIndexT>());
-
308  }
-
309 
-
310  NewIndexT newSize[Dim];
-
311  NewIndexT newStride[Dim];
-
312  for (int i = 0; i < Dim; ++i) {
-
313  newSize[i] = (NewIndexT) size_[i];
-
314  newStride[i] = (NewIndexT) stride_[i];
-
315  }
-
316 
-
317  return Tensor<T, Dim, InnerContig, NewIndexT, PtrTraits>(
-
318  data_, newSize, newStride);
-
319 }
-
320 
-
321 template <typename T, int Dim, bool InnerContig,
-
322  typename IndexT, template <typename U> class PtrTraits>
-
323 template <typename NewIndexT>
-
324 __host__ bool
-
325 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::canUseIndexType() const {
-
326  static_assert(sizeof(size_t) >= sizeof(IndexT),
-
327  "index size too large");
-
328  static_assert(sizeof(size_t) >= sizeof(NewIndexT),
-
329  "new index size too large");
-
330 
-
331  // Find maximum offset that can be calculated
-
332  // FIXME: maybe also consider offset in bytes? multiply by sizeof(T)?
-
333  size_t maxOffset = 0;
-
334 
-
335  for (int i = 0; i < Dim; ++i) {
-
336  size_t curMaxOffset = (size_t) size_[i] * (size_t) stride_[i];
-
337  if (curMaxOffset > maxOffset) {
-
338  maxOffset = curMaxOffset;
-
339  }
-
340  }
-
341 
-
342  if (maxOffset > (size_t) std::numeric_limits<NewIndexT>::max()) {
-
343  return false;
-
344  }
-
345 
-
346  return true;
-
347 }
-
348 
-
349 template <typename T, int Dim, bool InnerContig,
-
350  typename IndexT, template <typename U> class PtrTraits>
-
351 __host__ __device__ size_t
-
352 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::numElements() const {
-
353  size_t size = (size_t) getSize(0);
-
354 
-
355  for (int i = 1; i < Dim; ++i) {
-
356  size *= (size_t) getSize(i);
-
357  }
-
358 
-
359  return size;
-
360 }
+
289  newStride[Dim - 1] = 1; // this is the same as the old stride
+
290  newSize[Dim - 1] = size_[Dim - 1] / kMultiple;
+
291 
+
292  return Tensor<U, Dim, InnerContig, IndexT, PtrTraits>(
+
293  reinterpret_cast<U*>(data_), newSize, newStride);
+
294 }
+
295 
+
296 template <typename T, int Dim, bool InnerContig,
+
297  typename IndexT, template <typename U> class PtrTraits>
+
298 template <typename U>
+
299 __host__ __device__ const Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
+
300 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::castResize() const {
+
301  return const_cast<Tensor<T, Dim, InnerContig, IndexT, PtrTraits>*>(this)->
+
302  castResize<U>();
+
303 }
+
304 
+
305 template <typename T, int Dim, bool InnerContig,
+
306  typename IndexT, template <typename U> class PtrTraits>
+
307 template <typename U>
+
308 __host__ __device__ bool
+
309 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::canCastResize() const {
+
310  static_assert(sizeof(U) >= sizeof(T), "only handles greater sizes");
+
311  constexpr int kMultiple = sizeof(U) / sizeof(T);
+
312 
+
313  // Check all outer strides
+
314  for (int i = 0; i < Dim - 1; ++i) {
+
315  if (stride_[i] % kMultiple != 0) {
+
316  return false;
+
317  }
+
318  }
+
319 
+
320  // Check inner size
+
321  if (size_[Dim - 1] % kMultiple != 0) {
+
322  return false;
+
323  }
+
324 
+
325  if (stride_[Dim - 1] != 1) {
+
326  return false;
+
327  }
+
328 
+
329  return true;
+
330 }
+
331 
+
332 template <typename T, int Dim, bool InnerContig,
+
333  typename IndexT, template <typename U> class PtrTraits>
+
334 template <typename NewIndexT>
+
335 __host__ Tensor<T, Dim, InnerContig, NewIndexT, PtrTraits>
+
336 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::castIndexType() const {
+
337  if (sizeof(NewIndexT) < sizeof(IndexT)) {
+
338  GPU_FAISS_ASSERT(this->canUseIndexType<NewIndexT>());
+
339  }
+
340 
+
341  NewIndexT newSize[Dim];
+
342  NewIndexT newStride[Dim];
+
343  for (int i = 0; i < Dim; ++i) {
+
344  newSize[i] = (NewIndexT) size_[i];
+
345  newStride[i] = (NewIndexT) stride_[i];
+
346  }
+
347 
+
348  return Tensor<T, Dim, InnerContig, NewIndexT, PtrTraits>(
+
349  data_, newSize, newStride);
+
350 }
+
351 
+
352 template <typename T, int Dim, bool InnerContig,
+
353  typename IndexT, template <typename U> class PtrTraits>
+
354 template <typename NewIndexT>
+
355 __host__ bool
+
356 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::canUseIndexType() const {
+
357  static_assert(sizeof(size_t) >= sizeof(IndexT),
+
358  "index size too large");
+
359  static_assert(sizeof(size_t) >= sizeof(NewIndexT),
+
360  "new index size too large");
361 
-
362 template <typename T, int Dim, bool InnerContig,
-
363  typename IndexT, template <typename U> class PtrTraits>
-
364 __host__ __device__ bool
-
365 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isContiguous() const {
-
366  long prevSize = 1;
-
367 
-
368  for (int i = Dim - 1; i >= 0; --i) {
-
369  if (getSize(i) != (IndexT) 1) {
-
370  if (getStride(i) == prevSize) {
-
371  prevSize *= getSize(i);
-
372  } else {
-
373  return false;
-
374  }
-
375  }
-
376  }
-
377 
-
378  return true;
-
379 }
-
380 
-
381 template <typename T, int Dim, bool InnerContig,
-
382  typename IndexT, template <typename U> class PtrTraits>
-
383 __host__ __device__ bool
-
384 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isConsistentlySized(int i) const {
-
385  if (i == 0 && getStride(i) > 0 && getSize(i) > 0) {
-
386  return true;
-
387  } else if ((i > 0) && (i < Dim) && (getStride(i) > 0) &&
-
388  ((getStride(i - 1) / getStride(i)) >= getSize(i))) {
-
389  return true;
-
390  }
-
391 
-
392  return false;
-
393 }
-
394 
-
395 template <typename T, int Dim, bool InnerContig,
-
396  typename IndexT, template <typename U> class PtrTraits>
-
397 __host__ __device__ bool
-
398 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isConsistentlySized() const {
-
399  for (int i = 0; i < Dim; ++i) {
-
400  if (!isConsistentlySized(i)) {
-
401  return false;
-
402  }
-
403  }
-
404 
-
405  return true;
-
406 }
-
407 
-
408 template <typename T, int Dim, bool InnerContig,
-
409  typename IndexT, template <typename U> class PtrTraits>
-
410 __host__ __device__ bool
-
411 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isContiguousDim(int i) const {
-
412  return (i == Dim - 1) || // just in case
-
413  ((i < Dim - 1) &&
-
414  ((getStride(i) / getStride(i + 1)) == getSize(i + 1)));
-
415 }
-
416 
-
417 template <typename T, int Dim, bool InnerContig,
-
418  typename IndexT, template <typename U> class PtrTraits>
-
419 __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
-
420 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::transpose(int dim1,
-
421  int dim2) const {
-
422  GPU_FAISS_ASSERT(dim1 >= 0 && dim1 < Dim);
-
423  GPU_FAISS_ASSERT(dim1 >= 0 && dim2 < Dim);
-
424 
-
425  // If a tensor is innermost contiguous, one cannot transpose the innermost
-
426  // dimension
-
427  if (InnerContig) {
-
428  GPU_FAISS_ASSERT(dim1 != Dim - 1 && dim2 != Dim - 1);
-
429  }
-
430 
-
431  IndexT newSize[Dim];
-
432  IndexT newStride[Dim];
-
433 
-
434  for (int i = 0; i < Dim; ++i) {
-
435  newSize[i] = size_[i];
-
436  newStride[i] = stride_[i];
-
437  }
+
362  // Find maximum offset that can be calculated
+
363  // FIXME: maybe also consider offset in bytes? multiply by sizeof(T)?
+
364  size_t maxOffset = 0;
+
365 
+
366  for (int i = 0; i < Dim; ++i) {
+
367  size_t curMaxOffset = (size_t) size_[i] * (size_t) stride_[i];
+
368  if (curMaxOffset > maxOffset) {
+
369  maxOffset = curMaxOffset;
+
370  }
+
371  }
+
372 
+
373  if (maxOffset > (size_t) std::numeric_limits<NewIndexT>::max()) {
+
374  return false;
+
375  }
+
376 
+
377  return true;
+
378 }
+
379 
+
380 template <typename T, int Dim, bool InnerContig,
+
381  typename IndexT, template <typename U> class PtrTraits>
+
382 __host__ __device__ size_t
+
383 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::numElements() const {
+
384  size_t size = (size_t) getSize(0);
+
385 
+
386  for (int i = 1; i < Dim; ++i) {
+
387  size *= (size_t) getSize(i);
+
388  }
+
389 
+
390  return size;
+
391 }
+
392 
+
393 template <typename T, int Dim, bool InnerContig,
+
394  typename IndexT, template <typename U> class PtrTraits>
+
395 __host__ __device__ bool
+
396 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isContiguous() const {
+
397  long prevSize = 1;
+
398 
+
399  for (int i = Dim - 1; i >= 0; --i) {
+
400  if (getSize(i) != (IndexT) 1) {
+
401  if (getStride(i) == prevSize) {
+
402  prevSize *= getSize(i);
+
403  } else {
+
404  return false;
+
405  }
+
406  }
+
407  }
+
408 
+
409  return true;
+
410 }
+
411 
+
412 template <typename T, int Dim, bool InnerContig,
+
413  typename IndexT, template <typename U> class PtrTraits>
+
414 __host__ __device__ bool
+
415 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isConsistentlySized(int i) const {
+
416  if (i == 0 && getStride(i) > 0 && getSize(i) > 0) {
+
417  return true;
+
418  } else if ((i > 0) && (i < Dim) && (getStride(i) > 0) &&
+
419  ((getStride(i - 1) / getStride(i)) >= getSize(i))) {
+
420  return true;
+
421  }
+
422 
+
423  return false;
+
424 }
+
425 
+
426 template <typename T, int Dim, bool InnerContig,
+
427  typename IndexT, template <typename U> class PtrTraits>
+
428 __host__ __device__ bool
+
429 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isConsistentlySized() const {
+
430  for (int i = 0; i < Dim; ++i) {
+
431  if (!isConsistentlySized(i)) {
+
432  return false;
+
433  }
+
434  }
+
435 
+
436  return true;
+
437 }
438 
-
439  IndexT tmp = newSize[dim1];
-
440  newSize[dim1] = newSize[dim2];
-
441  newSize[dim2] = tmp;
-
442 
-
443  tmp = newStride[dim1];
-
444  newStride[dim1] = newStride[dim2];
-
445  newStride[dim2] = tmp;
-
446 
-
447  return Tensor<T, Dim, InnerContig, IndexT, PtrTraits>(data_, newSize, newStride);
-
448 }
-
449 
-
450 template <typename T, int Dim, bool InnerContig,
-
451  typename IndexT, template <typename U> class PtrTraits>
-
452 template <int NewDim>
-
453 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
454 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::upcastOuter() {
-
455  // Can only create tensors of greater dimension
-
456  static_assert(NewDim > Dim, "Can only upcast to greater dim");
-
457 
-
458  IndexT newSize[NewDim];
-
459  IndexT newStride[NewDim];
-
460 
-
461  int shift = NewDim - Dim;
-
462 
-
463  for (int i = 0; i < NewDim; ++i) {
-
464  if (i < shift) {
-
465  // These are the extended dimensions
-
466  newSize[i] = (IndexT) 1;
-
467  newStride[i] = size_[0] * stride_[0];
-
468  } else {
-
469  // Shift the remaining dimensions
-
470  newSize[i] = size_[i - shift];
-
471  newStride[i] = stride_[i - shift];
-
472  }
-
473  }
-
474 
-
475  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
-
476  data_, newSize, newStride);
-
477 }
-
478 
-
479 template <typename T, int Dim, bool InnerContig,
-
480  typename IndexT, template <typename U> class PtrTraits>
-
481 template <int NewDim>
-
482 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
483 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::upcastInner() {
-
484  // Can only create tensors of greater dimension
-
485  static_assert(NewDim > Dim, "Can only upcast to greater dim");
-
486 
-
487  IndexT newSize[NewDim];
-
488  IndexT newStride[NewDim];
-
489 
-
490  for (int i = 0; i < NewDim; ++i) {
-
491  if (i < Dim) {
-
492  // Existing dimensions get copied over
-
493  newSize[i] = size_[i];
-
494  newStride[i] = stride_[i];
-
495  } else {
-
496  // Extended dimensions
+
439 template <typename T, int Dim, bool InnerContig,
+
440  typename IndexT, template <typename U> class PtrTraits>
+
441 __host__ __device__ bool
+
442 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::isContiguousDim(int i) const {
+
443  return (i == Dim - 1) || // just in case
+
444  ((i < Dim - 1) &&
+
445  ((getStride(i) / getStride(i + 1)) == getSize(i + 1)));
+
446 }
+
447 
+
448 template <typename T, int Dim, bool InnerContig,
+
449  typename IndexT, template <typename U> class PtrTraits>
+
450 __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
+
451 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::transpose(int dim1,
+
452  int dim2) const {
+
453  GPU_FAISS_ASSERT(dim1 >= 0 && dim1 < Dim);
+
454  GPU_FAISS_ASSERT(dim1 >= 0 && dim2 < Dim);
+
455 
+
456  // If a tensor is innermost contiguous, one cannot transpose the innermost
+
457  // dimension
+
458  if (InnerContig) {
+
459  GPU_FAISS_ASSERT(dim1 != Dim - 1 && dim2 != Dim - 1);
+
460  }
+
461 
+
462  IndexT newSize[Dim];
+
463  IndexT newStride[Dim];
+
464 
+
465  for (int i = 0; i < Dim; ++i) {
+
466  newSize[i] = size_[i];
+
467  newStride[i] = stride_[i];
+
468  }
+
469 
+
470  IndexT tmp = newSize[dim1];
+
471  newSize[dim1] = newSize[dim2];
+
472  newSize[dim2] = tmp;
+
473 
+
474  tmp = newStride[dim1];
+
475  newStride[dim1] = newStride[dim2];
+
476  newStride[dim2] = tmp;
+
477 
+
478  return Tensor<T, Dim, InnerContig, IndexT, PtrTraits>(data_, newSize, newStride);
+
479 }
+
480 
+
481 template <typename T, int Dim, bool InnerContig,
+
482  typename IndexT, template <typename U> class PtrTraits>
+
483 template <int NewDim>
+
484 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
485 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::upcastOuter() {
+
486  // Can only create tensors of greater dimension
+
487  static_assert(NewDim > Dim, "Can only upcast to greater dim");
+
488 
+
489  IndexT newSize[NewDim];
+
490  IndexT newStride[NewDim];
+
491 
+
492  int shift = NewDim - Dim;
+
493 
+
494  for (int i = 0; i < NewDim; ++i) {
+
495  if (i < shift) {
+
496  // These are the extended dimensions
497  newSize[i] = (IndexT) 1;
-
498  newStride[i] = (IndexT) 1;
-
499  }
-
500  }
-
501 
-
502  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
-
503  data_, newSize, newStride);
-
504 }
+
498  newStride[i] = size_[0] * stride_[0];
+
499  } else {
+
500  // Shift the remaining dimensions
+
501  newSize[i] = size_[i - shift];
+
502  newStride[i] = stride_[i - shift];
+
503  }
+
504  }
505 
-
506 template <typename T, int Dim, bool InnerContig,
-
507  typename IndexT, template <typename U> class PtrTraits>
-
508 template <int NewDim>
-
509 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
510 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::downcastOuter() {
-
511  // Can only create tensors of lesser dimension
-
512  static_assert(NewDim < Dim, "Can only downcast to lesser dim");
-
513 
-
514  // We can't downcast non-contiguous tensors, since it leaves
-
515  // garbage data in the tensor. The tensor needs to be contiguous
-
516  // in all of the dimensions we are collapsing (no padding in
-
517  // them).
-
518  for (int i = 0; i < Dim - NewDim; ++i) {
-
519  bool cont = isContiguousDim(i);
-
520  GPU_FAISS_ASSERT(cont);
-
521  }
-
522 
-
523  IndexT newSize[NewDim];
-
524  IndexT newStride[NewDim];
-
525 
-
526  int ignoredDims = Dim - NewDim;
-
527  IndexT collapsedSize = 1;
-
528 
-
529  for (int i = 0; i < Dim; ++i) {
-
530  if (i < ignoredDims) {
-
531  // Collapse these dimensions
-
532  collapsedSize *= getSize(i);
-
533  } else {
-
534  // Non-collapsed dimensions
-
535  if (i == ignoredDims) {
-
536  // This is the first non-collapsed dimension
-
537  newSize[i - ignoredDims] = collapsedSize * getSize(i);
-
538  } else {
-
539  // Subsequent non-collapsed dimensions
-
540  newSize[i - ignoredDims] = getSize(i);
-
541  }
-
542 
-
543  newStride[i - ignoredDims] = getStride(i);
-
544  }
-
545  }
-
546 
-
547  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
-
548  data_, newSize, newStride);
-
549 }
-
550 
-
551 template <typename T, int Dim, bool InnerContig,
-
552  typename IndexT, template <typename U> class PtrTraits>
-
553 template <int NewDim>
-
554 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
555 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::downcastInner() {
-
556  // Can only create tensors of lesser dimension
-
557  static_assert(NewDim < Dim, "Can only downcast to lesser dim");
-
558 
-
559  // We can't downcast non-contiguous tensors, since it leaves
-
560  // garbage data in the tensor. The tensor needs to be contiguous
-
561  // in all of the dimensions we are collapsing (no padding in
-
562  // them).
-
563  for (int i = NewDim; i < Dim; ++i) {
-
564  GPU_FAISS_ASSERT(isContiguousDim(i));
-
565  }
-
566 
-
567  IndexT newSize[NewDim];
-
568  IndexT newStride[NewDim];
-
569 
-
570  IndexT collapsedSize = 1;
-
571 
-
572  for (int i = Dim - 1; i >= 0; --i) {
-
573  if (i >= NewDim) {
-
574  // Collapse these dimensions
-
575  collapsedSize *= getSize(i);
-
576  } else {
-
577  // Non-collapsed dimensions
-
578  if (i == NewDim - 1) {
-
579  // This is the first non-collapsed dimension
-
580  newSize[i] = collapsedSize * getSize(i);
-
581  newStride[i] = getStride(Dim - 1);
-
582  } else {
-
583  // Subsequent non-collapsed dimensions
-
584  newSize[i] = getSize(i);
-
585  newStride[i] = getStride(i);
-
586  }
-
587  }
-
588  }
+
506  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
+
507  data_, newSize, newStride);
+
508 }
+
509 
+
510 template <typename T, int Dim, bool InnerContig,
+
511  typename IndexT, template <typename U> class PtrTraits>
+
512 template <int NewDim>
+
513 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
514 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::upcastInner() {
+
515  // Can only create tensors of greater dimension
+
516  static_assert(NewDim > Dim, "Can only upcast to greater dim");
+
517 
+
518  IndexT newSize[NewDim];
+
519  IndexT newStride[NewDim];
+
520 
+
521  for (int i = 0; i < NewDim; ++i) {
+
522  if (i < Dim) {
+
523  // Existing dimensions get copied over
+
524  newSize[i] = size_[i];
+
525  newStride[i] = stride_[i];
+
526  } else {
+
527  // Extended dimensions
+
528  newSize[i] = (IndexT) 1;
+
529  newStride[i] = (IndexT) 1;
+
530  }
+
531  }
+
532 
+
533  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
+
534  data_, newSize, newStride);
+
535 }
+
536 
+
537 template <typename T, int Dim, bool InnerContig,
+
538  typename IndexT, template <typename U> class PtrTraits>
+
539 template <int NewDim>
+
540 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
541 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::downcastOuter() {
+
542  // Can only create tensors of lesser dimension
+
543  static_assert(NewDim < Dim, "Can only downcast to lesser dim");
+
544 
+
545  // We can't downcast non-contiguous tensors, since it leaves
+
546  // garbage data in the tensor. The tensor needs to be contiguous
+
547  // in all of the dimensions we are collapsing (no padding in
+
548  // them).
+
549  for (int i = 0; i < Dim - NewDim; ++i) {
+
550  bool cont = isContiguousDim(i);
+
551  GPU_FAISS_ASSERT(cont);
+
552  }
+
553 
+
554  IndexT newSize[NewDim];
+
555  IndexT newStride[NewDim];
+
556 
+
557  int ignoredDims = Dim - NewDim;
+
558  IndexT collapsedSize = 1;
+
559 
+
560  for (int i = 0; i < Dim; ++i) {
+
561  if (i < ignoredDims) {
+
562  // Collapse these dimensions
+
563  collapsedSize *= getSize(i);
+
564  } else {
+
565  // Non-collapsed dimensions
+
566  if (i == ignoredDims) {
+
567  // This is the first non-collapsed dimension
+
568  newSize[i - ignoredDims] = collapsedSize * getSize(i);
+
569  } else {
+
570  // Subsequent non-collapsed dimensions
+
571  newSize[i - ignoredDims] = getSize(i);
+
572  }
+
573 
+
574  newStride[i - ignoredDims] = getStride(i);
+
575  }
+
576  }
+
577 
+
578  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
+
579  data_, newSize, newStride);
+
580 }
+
581 
+
582 template <typename T, int Dim, bool InnerContig,
+
583  typename IndexT, template <typename U> class PtrTraits>
+
584 template <int NewDim>
+
585 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
586 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::downcastInner() {
+
587  // Can only create tensors of lesser dimension
+
588  static_assert(NewDim < Dim, "Can only downcast to lesser dim");
589 
-
590  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
-
591  data_, newSize, newStride);
-
592 }
-
593 
-
594 template <typename T, int Dim, bool InnerContig,
-
595  typename IndexT, template <typename U> class PtrTraits>
-
596 template <int SubDim>
-
597 __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
-
598 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::view(DataPtrType at) {
-
599  static_assert(SubDim >= 1 && SubDim < Dim,
-
600  "can only create view of lesser dim");
-
601 
-
602  IndexT viewSizes[SubDim];
-
603  IndexT viewStrides[SubDim];
-
604 
-
605  for (int i = 0; i < SubDim; ++i) {
-
606  viewSizes[i] = size_[Dim - SubDim + i];
-
607  viewStrides[i] = stride_[Dim - SubDim + i];
-
608  }
-
609 
-
610  return Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>(
-
611  at, viewSizes, viewStrides);
-
612 }
-
613 
-
614 template <typename T, int Dim, bool InnerContig,
-
615  typename IndexT, template <typename U> class PtrTraits>
-
616 template <int SubDim>
-
617 __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
-
618 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::view() {
-
619  return view<SubDim>(data_);
-
620 }
-
621 
-
622 template <typename T, int Dim, bool InnerContig,
-
623  typename IndexT, template <typename U> class PtrTraits>
-
624 __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
-
625 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::narrowOutermost(IndexT start,
-
626  IndexT size) {
-
627  return this->narrow(0, start, size);
-
628 }
-
629 
-
630 template <typename T, int Dim, bool InnerContig,
-
631  typename IndexT, template <typename U> class PtrTraits>
-
632 __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
-
633 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::narrow(int dim,
-
634  IndexT start,
-
635  IndexT size) {
-
636  DataPtrType newData = data_;
-
637 
-
638  GPU_FAISS_ASSERT(start >= 0 &&
-
639  start < size_[dim] &&
-
640  (start + size) <= size_[dim]);
-
641 
-
642  if (start > 0) {
-
643  newData += (size_t) start * stride_[dim];
-
644  }
-
645 
-
646  IndexT newSize[Dim];
-
647  for (int i = 0; i < Dim; ++i) {
-
648  if (i == dim) {
-
649  GPU_FAISS_ASSERT(start + size <= size_[dim]);
-
650  newSize[i] = size;
-
651  } else {
-
652  newSize[i] = size_[i];
-
653  }
-
654  }
-
655 
-
656  // If we were innermost contiguous before, we are still innermost contiguous
-
657  return Tensor<T, Dim, InnerContig, IndexT, PtrTraits>(newData, newSize, stride_);
-
658 }
-
659 
-
660 template <typename T, int Dim, bool InnerContig,
-
661  typename IndexT, template <typename U> class PtrTraits>
-
662 template <int NewDim>
-
663 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
664 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::view(
-
665  std::initializer_list<IndexT> sizes) {
-
666  GPU_FAISS_ASSERT(this->isContiguous());
-
667 
-
668  GPU_FAISS_ASSERT(sizes.size() == NewDim);
-
669 
-
670  // The total size of the new view must be the same as the total size
-
671  // of the old view
-
672  size_t curSize = numElements();
-
673  size_t newSize = 1;
-
674 
-
675  for (auto s : sizes) {
-
676  newSize *= s;
-
677  }
-
678 
-
679  GPU_FAISS_ASSERT(curSize == newSize);
-
680  return Tensor<T, NewDim, true, IndexT, PtrTraits>(data(), sizes);
-
681 }
-
682 
-
683 } } // namespace
-
faiss::gpu::Tensor::upcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastOuter()
Definition: Tensor-inl.cuh:454
-
faiss::gpu::Tensor::castIndexType
__host__ Tensor< T, Dim, InnerContig, NewIndexT, PtrTraits > castIndexType() const
Definition: Tensor-inl.cuh:305
-
faiss::gpu::Tensor::isContiguousDim
__host__ __device__ bool isContiguousDim(int i) const
Returns true if the given dimension index has no padding.
Definition: Tensor-inl.cuh:411
-
faiss::gpu::Tensor::cast
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > cast()
Definition: Tensor-inl.cuh:222
-
faiss::gpu::Tensor::numElements
__host__ __device__ size_t numElements() const
Definition: Tensor-inl.cuh:352
-
faiss::gpu::Tensor::downcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastOuter()
Definition: Tensor-inl.cuh:510
-
faiss::gpu::Tensor::canCastResize
__host__ __device__ bool canCastResize() const
Returns true if we can castResize() this tensor to the new type.
Definition: Tensor-inl.cuh:278
+
590  // We can't downcast non-contiguous tensors, since it leaves
+
591  // garbage data in the tensor. The tensor needs to be contiguous
+
592  // in all of the dimensions we are collapsing (no padding in
+
593  // them).
+
594  for (int i = NewDim; i < Dim; ++i) {
+
595  GPU_FAISS_ASSERT(isContiguousDim(i));
+
596  }
+
597 
+
598  IndexT newSize[NewDim];
+
599  IndexT newStride[NewDim];
+
600 
+
601  IndexT collapsedSize = 1;
+
602 
+
603  for (int i = Dim - 1; i >= 0; --i) {
+
604  if (i >= NewDim) {
+
605  // Collapse these dimensions
+
606  collapsedSize *= getSize(i);
+
607  } else {
+
608  // Non-collapsed dimensions
+
609  if (i == NewDim - 1) {
+
610  // This is the first non-collapsed dimension
+
611  newSize[i] = collapsedSize * getSize(i);
+
612  newStride[i] = getStride(Dim - 1);
+
613  } else {
+
614  // Subsequent non-collapsed dimensions
+
615  newSize[i] = getSize(i);
+
616  newStride[i] = getStride(i);
+
617  }
+
618  }
+
619  }
+
620 
+
621  return Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>(
+
622  data_, newSize, newStride);
+
623 }
+
624 
+
625 template <typename T, int Dim, bool InnerContig,
+
626  typename IndexT, template <typename U> class PtrTraits>
+
627 template <int SubDim>
+
628 __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
+
629 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::view(DataPtrType at) {
+
630  static_assert(SubDim >= 1 && SubDim < Dim,
+
631  "can only create view of lesser dim");
+
632 
+
633  IndexT viewSizes[SubDim];
+
634  IndexT viewStrides[SubDim];
+
635 
+
636  for (int i = 0; i < SubDim; ++i) {
+
637  viewSizes[i] = size_[Dim - SubDim + i];
+
638  viewStrides[i] = stride_[Dim - SubDim + i];
+
639  }
+
640 
+
641  return Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>(
+
642  at, viewSizes, viewStrides);
+
643 }
+
644 
+
645 template <typename T, int Dim, bool InnerContig,
+
646  typename IndexT, template <typename U> class PtrTraits>
+
647 template <int SubDim>
+
648 __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
+
649 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::view() {
+
650  return view<SubDim>(data_);
+
651 }
+
652 
+
653 template <typename T, int Dim, bool InnerContig,
+
654  typename IndexT, template <typename U> class PtrTraits>
+
655 __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
+
656 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::narrowOutermost(IndexT start,
+
657  IndexT size) {
+
658  return this->narrow(0, start, size);
+
659 }
+
660 
+
661 template <typename T, int Dim, bool InnerContig,
+
662  typename IndexT, template <typename U> class PtrTraits>
+
663 __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
+
664 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::narrow(int dim,
+
665  IndexT start,
+
666  IndexT size) {
+
667  DataPtrType newData = data_;
+
668 
+
669  GPU_FAISS_ASSERT(start >= 0 &&
+
670  start < size_[dim] &&
+
671  (start + size) <= size_[dim]);
+
672 
+
673  if (start > 0) {
+
674  newData += (size_t) start * stride_[dim];
+
675  }
+
676 
+
677  IndexT newSize[Dim];
+
678  for (int i = 0; i < Dim; ++i) {
+
679  if (i == dim) {
+
680  GPU_FAISS_ASSERT(start + size <= size_[dim]);
+
681  newSize[i] = size;
+
682  } else {
+
683  newSize[i] = size_[i];
+
684  }
+
685  }
+
686 
+
687  // If we were innermost contiguous before, we are still innermost contiguous
+
688  return Tensor<T, Dim, InnerContig, IndexT, PtrTraits>(newData, newSize, stride_);
+
689 }
+
690 
+
691 template <typename T, int Dim, bool InnerContig,
+
692  typename IndexT, template <typename U> class PtrTraits>
+
693 template <int NewDim>
+
694 __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
695 Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::view(
+
696  std::initializer_list<IndexT> sizes) {
+
697  GPU_FAISS_ASSERT(this->isContiguous());
+
698 
+
699  GPU_FAISS_ASSERT(sizes.size() == NewDim);
+
700 
+
701  // The total size of the new view must be the same as the total size
+
702  // of the old view
+
703  size_t curSize = numElements();
+
704  size_t newSize = 1;
+
705 
+
706  for (auto s : sizes) {
+
707  newSize *= s;
+
708  }
+
709 
+
710  GPU_FAISS_ASSERT(curSize == newSize);
+
711  return Tensor<T, NewDim, true, IndexT, PtrTraits>(data(), sizes);
+
712 }
+
713 
+
714 } } // namespace
+
faiss::gpu::Tensor::upcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastOuter()
Definition: Tensor-inl.cuh:485
+
faiss::gpu::Tensor::castIndexType
__host__ Tensor< T, Dim, InnerContig, NewIndexT, PtrTraits > castIndexType() const
Definition: Tensor-inl.cuh:336
+
faiss::gpu::Tensor::isContiguousDim
__host__ __device__ bool isContiguousDim(int i) const
Returns true if the given dimension index has no padding.
Definition: Tensor-inl.cuh:442
+
faiss::gpu::Tensor::cast
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > cast()
Definition: Tensor-inl.cuh:253
+
faiss::gpu::Tensor::numElements
__host__ __device__ size_t numElements() const
Definition: Tensor-inl.cuh:383
+
faiss::gpu::Tensor::downcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastOuter()
Definition: Tensor-inl.cuh:541
+
faiss::gpu::Tensor::canCastResize
__host__ __device__ bool canCastResize() const
Returns true if we can castResize() this tensor to the new type.
Definition: Tensor-inl.cuh:309
+
faiss::gpu::Tensor::data_
DataPtrType data_
Raw pointer to where the tensor data begins.
Definition: Tensor.cuh:344
faiss::gpu::Tensor::Tensor
__host__ __device__ Tensor()
Default constructor.
Definition: Tensor-inl.cuh:20
-
faiss::gpu::Tensor::upcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastInner()
Definition: Tensor-inl.cuh:483
-
faiss::gpu::Tensor::narrowOutermost
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrowOutermost(IndexT start, IndexT size)
Definition: Tensor-inl.cuh:625
-
faiss::gpu::Tensor::stride_
IndexT stride_[Dim]
Array of strides (in sizeof(T) terms) per each dimension.
Definition: Tensor.cuh:349
-
faiss::gpu::Tensor::isContiguous
__host__ __device__ bool isContiguous() const
Definition: Tensor-inl.cuh:365
-
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:247
-
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:101
-
faiss::gpu::Tensor::size_
IndexT size_[Dim]
Size per each dimension.
Definition: Tensor.cuh:352
-
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:252
-
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
-
faiss::gpu::Tensor::isSameSize
__host__ __device__ bool isSameSize(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Returns true if the two tensors are of the same dimensionality and size.
Definition: Tensor-inl.cuh:203
-
faiss::gpu::Tensor::downcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastInner()
Definition: Tensor-inl.cuh:555
-
faiss::gpu::Tensor::narrow
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrow(int dim, IndexT start, IndexT size)
Definition: Tensor-inl.cuh:633
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
-
faiss::gpu::Tensor::copyTo
__host__ void copyTo(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies ourselves into a tensor; sizes must match.
Definition: Tensor-inl.cuh:140
+
faiss::gpu::Tensor::upcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastInner()
Definition: Tensor-inl.cuh:514
+
faiss::gpu::Tensor::narrowOutermost
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrowOutermost(IndexT start, IndexT size)
Definition: Tensor-inl.cuh:656
+
faiss::gpu::Tensor::stride_
IndexT stride_[Dim]
Array of strides (in sizeof(T) terms) per each dimension.
Definition: Tensor.cuh:347
+
faiss::gpu::Tensor::isContiguous
__host__ __device__ bool isContiguous() const
Definition: Tensor-inl.cuh:396
+
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:245
+
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:132
+
faiss::gpu::Tensor::size_
IndexT size_[Dim]
Size per each dimension.
Definition: Tensor.cuh:350
+
faiss::gpu::Tensor::operator=
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)
Assignment.
Definition: Tensor-inl.cuh:50
+
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:250
+
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
+
faiss::gpu::Tensor::isSameSize
__host__ __device__ bool isSameSize(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Returns true if the two tensors are of the same dimensionality and size.
Definition: Tensor-inl.cuh:234
+
faiss::gpu::Tensor::downcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastInner()
Definition: Tensor-inl.cuh:586
+
faiss::gpu::Tensor::narrow
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrow(int dim, IndexT start, IndexT size)
Definition: Tensor-inl.cuh:664
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
+
faiss::gpu::Tensor::copyTo
__host__ void copyTo(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies ourselves into a tensor; sizes must match.
Definition: Tensor-inl.cuh:171
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
-
faiss::gpu::Tensor::canUseIndexType
__host__ bool canUseIndexType() const
Definition: Tensor-inl.cuh:325
-
faiss::gpu::Tensor::transpose
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > transpose(int dim1, int dim2) const
Definition: Tensor-inl.cuh:420
-
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:232
-
faiss::gpu::Tensor::operator=
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default
Assignment.
-
faiss::gpu::Tensor::castResize
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > castResize()
Definition: Tensor-inl.cuh:244
-
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view()
Definition: Tensor-inl.cuh:618
-
faiss::gpu::Tensor::isSame
__host__ __device__ bool isSame(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Definition: Tensor-inl.cuh:180
+
faiss::gpu::Tensor::canUseIndexType
__host__ bool canUseIndexType() const
Definition: Tensor-inl.cuh:356
+
faiss::gpu::Tensor::transpose
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > transpose(int dim1, int dim2) const
Definition: Tensor-inl.cuh:451
+
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:230
+
faiss::gpu::Tensor::castResize
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > castResize()
Definition: Tensor-inl.cuh:275
+
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view()
Definition: Tensor-inl.cuh:649
+
faiss::gpu::Tensor::isSame
__host__ __device__ bool isSame(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Definition: Tensor-inl.cuh:211
diff --git a/docs/html/Tensor_8cuh_source.html b/docs/html/Tensor_8cuh_source.html index 381d13868..cae3ce96d 100644 --- a/docs/html/Tensor_8cuh_source.html +++ b/docs/html/Tensor_8cuh_source.html @@ -176,637 +176,635 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
87  __host__ __device__ Tensor();
88 
89  /// Copy constructor
-
90  __host__ __device__ Tensor(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t)
-
91  = default;
-
92 
-
93  /// Move constructor
-
94  __host__ __device__ Tensor(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&& t)
-
95  = default;
-
96 
-
97  /// Assignment
-
98  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&
-
99  operator=(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t) = default;
-
100 
-
101  /// Move assignment
-
102  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&
-
103  operator=(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&& t);
-
104 
-
105  /// Constructor that calculates strides with no padding
-
106  __host__ __device__ Tensor(DataPtrType data,
-
107  const IndexT sizes[Dim]);
-
108  __host__ __device__ Tensor(DataPtrType data,
-
109  std::initializer_list<IndexT> sizes);
-
110 
-
111  /// Constructor that takes arbitrary size/stride arrays.
-
112  /// Errors if you attempt to pass non-contiguous strides to a
-
113  /// contiguous tensor.
-
114  __host__ __device__ Tensor(DataPtrType data,
-
115  const IndexT sizes[Dim],
-
116  const IndexT strides[Dim]);
-
117 
-
118  /// Copies a tensor into ourselves; sizes must match
-
119  __host__ void copyFrom(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
-
120  cudaStream_t stream);
-
121 
-
122  /// Copies ourselves into a tensor; sizes must match
-
123  __host__ void copyTo(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
-
124  cudaStream_t stream);
-
125 
-
126  /// Returns true if the two tensors are of the same dimensionality,
-
127  /// size and stride.
-
128  template <typename OtherT, int OtherDim>
-
129  __host__ __device__ bool
-
130  isSame(const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const;
-
131 
-
132  /// Returns true if the two tensors are of the same dimensionality and size
-
133  template <typename OtherT, int OtherDim>
-
134  __host__ __device__ bool
-
135  isSameSize(const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const;
-
136 
-
137  /// Cast to a tensor of a different type of the same size and
-
138  /// stride. U and our type T must be of the same size
-
139  template <typename U>
-
140  __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits> cast();
-
141 
-
142  /// Const version of `cast`
-
143  template <typename U>
-
144  __host__ __device__
-
145  const Tensor<U, Dim, InnerContig, IndexT, PtrTraits> cast() const;
-
146 
-
147  /// Cast to a tensor of a different type which is potentially a
-
148  /// different size than our type T. Tensor must be aligned and the
-
149  /// innermost dimension must be a size that is a multiple of
-
150  /// sizeof(U) / sizeof(T), and the stride of the innermost dimension
-
151  /// must be contiguous. The stride of all outer dimensions must be a
-
152  /// multiple of sizeof(U) / sizeof(T) as well.
-
153  template <typename U>
-
154  __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits> castResize();
-
155 
-
156  /// Const version of `castResize`
-
157  template <typename U>
-
158  __host__ __device__ const Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
-
159  castResize() const;
-
160 
-
161  /// Returns true if we can castResize() this tensor to the new type
-
162  template <typename U>
-
163  __host__ __device__ bool canCastResize() const;
-
164 
-
165  /// Attempts to cast this tensor to a tensor of a different IndexT.
-
166  /// Fails if size or stride entries are not representable in the new
-
167  /// IndexT.
-
168  template <typename NewIndexT>
-
169  __host__ Tensor<T, Dim, InnerContig, NewIndexT, PtrTraits>
-
170  castIndexType() const;
-
171 
-
172  /// Returns true if we can use this indexing type to access all elements
-
173  /// index type
-
174  template <typename NewIndexT>
-
175  __host__ bool canUseIndexType() const;
-
176 
-
177  /// Returns a raw pointer to the start of our data.
-
178  __host__ __device__ inline DataPtrType data() {
-
179  return data_;
-
180  }
-
181 
-
182  /// Returns a raw pointer to the end of our data, assuming
-
183  /// continuity
-
184  __host__ __device__ inline DataPtrType end() {
-
185  return data() + numElements();
-
186  }
-
187 
-
188  /// Returns a raw pointer to the start of our data (const).
-
189  __host__ __device__ inline
-
190  const DataPtrType data() const {
-
191  return data_;
-
192  }
-
193 
-
194  /// Returns a raw pointer to the end of our data, assuming
-
195  /// continuity (const)
-
196  __host__ __device__ inline DataPtrType end() const {
-
197  return data() + numElements();
-
198  }
-
199 
-
200  /// Cast to a different datatype
-
201  template <typename U>
-
202  __host__ __device__ inline
-
203  typename PtrTraits<U>::PtrType dataAs() {
-
204  return reinterpret_cast<typename PtrTraits<U>::PtrType>(data_);
-
205  }
-
206 
-
207  /// Cast to a different datatype
-
208  template <typename U>
-
209  __host__ __device__ inline
-
210  const typename PtrTraits<const U>::PtrType dataAs() const {
-
211  return reinterpret_cast<typename PtrTraits<const U>::PtrType>(data_);
-
212  }
-
213 
-
214  /// Returns a read/write view of a portion of our tensor.
-
215  __host__ __device__ inline
-
216  detail::SubTensor<TensorType, Dim - 1, PtrTraits>
-
217  operator[](IndexT);
-
218 
-
219  /// Returns a read/write view of a portion of our tensor (const).
-
220  __host__ __device__ inline
-
221  const detail::SubTensor<TensorType, Dim - 1, PtrTraits>
-
222  operator[](IndexT) const;
-
223 
-
224  /// Returns the size of a given dimension, `[0, Dim - 1]`. No bounds
-
225  /// checking.
-
226  __host__ __device__ inline IndexT getSize(int i) const {
-
227  return size_[i];
-
228  }
-
229 
-
230  /// Returns the stride of a given dimension, `[0, Dim - 1]`. No bounds
-
231  /// checking.
-
232  __host__ __device__ inline IndexT getStride(int i) const {
-
233  return stride_[i];
-
234  }
-
235 
-
236  /// Returns the total number of elements contained within our data
-
237  /// (product of `getSize(i)`)
-
238  __host__ __device__ size_t numElements() const;
-
239 
-
240  /// If we are contiguous, returns the total size in bytes of our
-
241  /// data
-
242  __host__ __device__ size_t getSizeInBytes() const {
-
243  return numElements() * sizeof(T);
-
244  }
-
245 
-
246  /// Returns the size array.
-
247  __host__ __device__ inline const IndexT* sizes() const {
-
248  return size_;
-
249  }
-
250 
-
251  /// Returns the stride array.
-
252  __host__ __device__ inline const IndexT* strides() const {
-
253  return stride_;
-
254  }
-
255 
-
256  /// Returns true if there is no padding within the tensor and no
-
257  /// re-ordering of the dimensions.
+
90  __host__ __device__ Tensor(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t);
+
91 
+
92  /// Move constructor
+
93  __host__ __device__ Tensor(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&& t);
+
94 
+
95  /// Assignment
+
96  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&
+
97  operator=(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t);
+
98 
+
99  /// Move assignment
+
100  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&
+
101  operator=(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>&& t);
+
102 
+
103  /// Constructor that calculates strides with no padding
+
104  __host__ __device__ Tensor(DataPtrType data,
+
105  const IndexT sizes[Dim]);
+
106  __host__ __device__ Tensor(DataPtrType data,
+
107  std::initializer_list<IndexT> sizes);
+
108 
+
109  /// Constructor that takes arbitrary size/stride arrays.
+
110  /// Errors if you attempt to pass non-contiguous strides to a
+
111  /// contiguous tensor.
+
112  __host__ __device__ Tensor(DataPtrType data,
+
113  const IndexT sizes[Dim],
+
114  const IndexT strides[Dim]);
+
115 
+
116  /// Copies a tensor into ourselves; sizes must match
+
117  __host__ void copyFrom(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
+
118  cudaStream_t stream);
+
119 
+
120  /// Copies ourselves into a tensor; sizes must match
+
121  __host__ void copyTo(Tensor<T, Dim, InnerContig, IndexT, PtrTraits>& t,
+
122  cudaStream_t stream);
+
123 
+
124  /// Returns true if the two tensors are of the same dimensionality,
+
125  /// size and stride.
+
126  template <typename OtherT, int OtherDim>
+
127  __host__ __device__ bool
+
128  isSame(const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const;
+
129 
+
130  /// Returns true if the two tensors are of the same dimensionality and size
+
131  template <typename OtherT, int OtherDim>
+
132  __host__ __device__ bool
+
133  isSameSize(const Tensor<OtherT, OtherDim, InnerContig, IndexT, PtrTraits>& rhs) const;
+
134 
+
135  /// Cast to a tensor of a different type of the same size and
+
136  /// stride. U and our type T must be of the same size
+
137  template <typename U>
+
138  __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits> cast();
+
139 
+
140  /// Const version of `cast`
+
141  template <typename U>
+
142  __host__ __device__
+
143  const Tensor<U, Dim, InnerContig, IndexT, PtrTraits> cast() const;
+
144 
+
145  /// Cast to a tensor of a different type which is potentially a
+
146  /// different size than our type T. Tensor must be aligned and the
+
147  /// innermost dimension must be a size that is a multiple of
+
148  /// sizeof(U) / sizeof(T), and the stride of the innermost dimension
+
149  /// must be contiguous. The stride of all outer dimensions must be a
+
150  /// multiple of sizeof(U) / sizeof(T) as well.
+
151  template <typename U>
+
152  __host__ __device__ Tensor<U, Dim, InnerContig, IndexT, PtrTraits> castResize();
+
153 
+
154  /// Const version of `castResize`
+
155  template <typename U>
+
156  __host__ __device__ const Tensor<U, Dim, InnerContig, IndexT, PtrTraits>
+
157  castResize() const;
+
158 
+
159  /// Returns true if we can castResize() this tensor to the new type
+
160  template <typename U>
+
161  __host__ __device__ bool canCastResize() const;
+
162 
+
163  /// Attempts to cast this tensor to a tensor of a different IndexT.
+
164  /// Fails if size or stride entries are not representable in the new
+
165  /// IndexT.
+
166  template <typename NewIndexT>
+
167  __host__ Tensor<T, Dim, InnerContig, NewIndexT, PtrTraits>
+
168  castIndexType() const;
+
169 
+
170  /// Returns true if we can use this indexing type to access all elements
+
171  /// index type
+
172  template <typename NewIndexT>
+
173  __host__ bool canUseIndexType() const;
+
174 
+
175  /// Returns a raw pointer to the start of our data.
+
176  __host__ __device__ inline DataPtrType data() {
+
177  return data_;
+
178  }
+
179 
+
180  /// Returns a raw pointer to the end of our data, assuming
+
181  /// continuity
+
182  __host__ __device__ inline DataPtrType end() {
+
183  return data() + numElements();
+
184  }
+
185 
+
186  /// Returns a raw pointer to the start of our data (const).
+
187  __host__ __device__ inline
+
188  const DataPtrType data() const {
+
189  return data_;
+
190  }
+
191 
+
192  /// Returns a raw pointer to the end of our data, assuming
+
193  /// continuity (const)
+
194  __host__ __device__ inline DataPtrType end() const {
+
195  return data() + numElements();
+
196  }
+
197 
+
198  /// Cast to a different datatype
+
199  template <typename U>
+
200  __host__ __device__ inline
+
201  typename PtrTraits<U>::PtrType dataAs() {
+
202  return reinterpret_cast<typename PtrTraits<U>::PtrType>(data_);
+
203  }
+
204 
+
205  /// Cast to a different datatype
+
206  template <typename U>
+
207  __host__ __device__ inline
+
208  const typename PtrTraits<const U>::PtrType dataAs() const {
+
209  return reinterpret_cast<typename PtrTraits<const U>::PtrType>(data_);
+
210  }
+
211 
+
212  /// Returns a read/write view of a portion of our tensor.
+
213  __host__ __device__ inline
+
214  detail::SubTensor<TensorType, Dim - 1, PtrTraits>
+
215  operator[](IndexT);
+
216 
+
217  /// Returns a read/write view of a portion of our tensor (const).
+
218  __host__ __device__ inline
+
219  const detail::SubTensor<TensorType, Dim - 1, PtrTraits>
+
220  operator[](IndexT) const;
+
221 
+
222  /// Returns the size of a given dimension, `[0, Dim - 1]`. No bounds
+
223  /// checking.
+
224  __host__ __device__ inline IndexT getSize(int i) const {
+
225  return size_[i];
+
226  }
+
227 
+
228  /// Returns the stride of a given dimension, `[0, Dim - 1]`. No bounds
+
229  /// checking.
+
230  __host__ __device__ inline IndexT getStride(int i) const {
+
231  return stride_[i];
+
232  }
+
233 
+
234  /// Returns the total number of elements contained within our data
+
235  /// (product of `getSize(i)`)
+
236  __host__ __device__ size_t numElements() const;
+
237 
+
238  /// If we are contiguous, returns the total size in bytes of our
+
239  /// data
+
240  __host__ __device__ size_t getSizeInBytes() const {
+
241  return numElements() * sizeof(T);
+
242  }
+
243 
+
244  /// Returns the size array.
+
245  __host__ __device__ inline const IndexT* sizes() const {
+
246  return size_;
+
247  }
+
248 
+
249  /// Returns the stride array.
+
250  __host__ __device__ inline const IndexT* strides() const {
+
251  return stride_;
+
252  }
+
253 
+
254  /// Returns true if there is no padding within the tensor and no
+
255  /// re-ordering of the dimensions.
+
256  /// ~~~
+
257  /// (stride(i) == size(i + 1) * stride(i + 1)) && stride(dim - 1) == 0
258  /// ~~~
-
259  /// (stride(i) == size(i + 1) * stride(i + 1)) && stride(dim - 1) == 0
-
260  /// ~~~
-
261  __host__ __device__ bool isContiguous() const;
-
262 
-
263  /// Returns whether a given dimension has only increasing stride
-
264  /// from the previous dimension. A tensor that was permuted by
-
265  /// exchanging size and stride only will fail this check.
-
266  /// If `i == 0` just check `size > 0`. Returns `false` if `stride` is `<= 0`.
-
267  __host__ __device__ bool isConsistentlySized(int i) const;
-
268 
-
269  // Returns whether at each dimension `stride <= size`.
-
270  // If this is not the case then iterating once over the size space will
-
271  // touch the same memory locations multiple times.
-
272  __host__ __device__ bool isConsistentlySized() const;
-
273 
-
274  /// Returns true if the given dimension index has no padding
-
275  __host__ __device__ bool isContiguousDim(int i) const;
-
276 
-
277  /// Returns a tensor of the same dimension after transposing the two
-
278  /// dimensions given. Does not actually move elements; transposition
-
279  /// is made by permuting the size/stride arrays.
-
280  /// If the dimensions are not valid, asserts.
-
281  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
-
282  transpose(int dim1, int dim2) const;
-
283 
-
284  /// Upcast a tensor of dimension `D` to some tensor of dimension
-
285  /// D' > D by padding the leading dimensions by 1
-
286  /// e.g., upcasting a 2-d tensor `[2][3]` to a 4-d tensor `[1][1][2][3]`
-
287  template <int NewDim>
-
288  __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
289  upcastOuter();
-
290 
-
291  /// Upcast a tensor of dimension `D` to some tensor of dimension
-
292  /// D' > D by padding the lowest/most varying dimensions by 1
-
293  /// e.g., upcasting a 2-d tensor `[2][3]` to a 4-d tensor `[2][3][1][1]`
-
294  template <int NewDim>
-
295  __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
296  upcastInner();
-
297 
-
298  /// Downcast a tensor of dimension `D` to some tensor of dimension
-
299  /// D' < D by collapsing the leading dimensions. asserts if there is
-
300  /// padding on the leading dimensions.
-
301  template <int NewDim>
-
302  __host__ __device__
-
303  Tensor<T, NewDim, InnerContig, IndexT, PtrTraits> downcastOuter();
-
304 
-
305  /// Downcast a tensor of dimension `D` to some tensor of dimension
-
306  /// D' < D by collapsing the leading dimensions. asserts if there is
-
307  /// padding on the leading dimensions.
-
308  template <int NewDim>
-
309  __host__ __device__
-
310  Tensor<T, NewDim, InnerContig, IndexT, PtrTraits> downcastInner();
-
311 
-
312  /// Returns a tensor that is a view of the `SubDim`-dimensional slice
-
313  /// of this tensor, starting at `at`.
-
314  template <int SubDim>
-
315  __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
-
316  view(DataPtrType at);
-
317 
-
318  /// Returns a tensor that is a view of the `SubDim`-dimensional slice
-
319  /// of this tensor, starting where our data begins
-
320  template <int SubDim>
-
321  __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
-
322  view();
-
323 
-
324  /// Returns a tensor of the same dimension that is a view of the
-
325  /// original tensor with the specified dimension restricted to the
-
326  /// elements in the range [start, start + size)
-
327  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
-
328  narrowOutermost(IndexT start, IndexT size);
-
329 
-
330  /// Returns a tensor of the same dimension that is a view of the
-
331  /// original tensor with the specified dimension restricted to the
-
332  /// elements in the range [start, start + size).
-
333  /// Can occur in an arbitrary dimension
-
334  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
-
335  narrow(int dim, IndexT start, IndexT size);
-
336 
-
337  /// Returns a view of the given tensor expressed as a tensor of a
-
338  /// different number of dimensions.
-
339  /// Only works if we are contiguous.
-
340  template <int NewDim>
-
341  __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
-
342  view(std::initializer_list<IndexT> sizes);
-
343 
-
344  protected:
-
345  /// Raw pointer to where the tensor data begins
-
346  DataPtrType data_;
-
347 
-
348  /// Array of strides (in sizeof(T) terms) per each dimension
-
349  IndexT stride_[Dim];
-
350 
-
351  /// Size per each dimension
-
352  IndexT size_[Dim];
-
353 };
-
354 
-
355 // Utilities for checking a collection of tensors
-
356 namespace detail {
-
357 
-
358 template <typename IndexType>
-
359 bool canUseIndexType() {
-
360  return true;
-
361 }
-
362 
-
363 template <typename IndexType, typename T, typename... U>
-
364 bool canUseIndexType(const T& arg, const U&... args) {
-
365  return arg.canUseIndexType<IndexType>() &&
-
366  canUseIndexType(args...);
-
367 }
+
259  __host__ __device__ bool isContiguous() const;
+
260 
+
261  /// Returns whether a given dimension has only increasing stride
+
262  /// from the previous dimension. A tensor that was permuted by
+
263  /// exchanging size and stride only will fail this check.
+
264  /// If `i == 0` just check `size > 0`. Returns `false` if `stride` is `<= 0`.
+
265  __host__ __device__ bool isConsistentlySized(int i) const;
+
266 
+
267  // Returns whether at each dimension `stride <= size`.
+
268  // If this is not the case then iterating once over the size space will
+
269  // touch the same memory locations multiple times.
+
270  __host__ __device__ bool isConsistentlySized() const;
+
271 
+
272  /// Returns true if the given dimension index has no padding
+
273  __host__ __device__ bool isContiguousDim(int i) const;
+
274 
+
275  /// Returns a tensor of the same dimension after transposing the two
+
276  /// dimensions given. Does not actually move elements; transposition
+
277  /// is made by permuting the size/stride arrays.
+
278  /// If the dimensions are not valid, asserts.
+
279  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
+
280  transpose(int dim1, int dim2) const;
+
281 
+
282  /// Upcast a tensor of dimension `D` to some tensor of dimension
+
283  /// D' > D by padding the leading dimensions by 1
+
284  /// e.g., upcasting a 2-d tensor `[2][3]` to a 4-d tensor `[1][1][2][3]`
+
285  template <int NewDim>
+
286  __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
287  upcastOuter();
+
288 
+
289  /// Upcast a tensor of dimension `D` to some tensor of dimension
+
290  /// D' > D by padding the lowest/most varying dimensions by 1
+
291  /// e.g., upcasting a 2-d tensor `[2][3]` to a 4-d tensor `[2][3][1][1]`
+
292  template <int NewDim>
+
293  __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
294  upcastInner();
+
295 
+
296  /// Downcast a tensor of dimension `D` to some tensor of dimension
+
297  /// D' < D by collapsing the leading dimensions. asserts if there is
+
298  /// padding on the leading dimensions.
+
299  template <int NewDim>
+
300  __host__ __device__
+
301  Tensor<T, NewDim, InnerContig, IndexT, PtrTraits> downcastOuter();
+
302 
+
303  /// Downcast a tensor of dimension `D` to some tensor of dimension
+
304  /// D' < D by collapsing the leading dimensions. asserts if there is
+
305  /// padding on the leading dimensions.
+
306  template <int NewDim>
+
307  __host__ __device__
+
308  Tensor<T, NewDim, InnerContig, IndexT, PtrTraits> downcastInner();
+
309 
+
310  /// Returns a tensor that is a view of the `SubDim`-dimensional slice
+
311  /// of this tensor, starting at `at`.
+
312  template <int SubDim>
+
313  __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
+
314  view(DataPtrType at);
+
315 
+
316  /// Returns a tensor that is a view of the `SubDim`-dimensional slice
+
317  /// of this tensor, starting where our data begins
+
318  template <int SubDim>
+
319  __host__ __device__ Tensor<T, SubDim, InnerContig, IndexT, PtrTraits>
+
320  view();
+
321 
+
322  /// Returns a tensor of the same dimension that is a view of the
+
323  /// original tensor with the specified dimension restricted to the
+
324  /// elements in the range [start, start + size)
+
325  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
+
326  narrowOutermost(IndexT start, IndexT size);
+
327 
+
328  /// Returns a tensor of the same dimension that is a view of the
+
329  /// original tensor with the specified dimension restricted to the
+
330  /// elements in the range [start, start + size).
+
331  /// Can occur in an arbitrary dimension
+
332  __host__ __device__ Tensor<T, Dim, InnerContig, IndexT, PtrTraits>
+
333  narrow(int dim, IndexT start, IndexT size);
+
334 
+
335  /// Returns a view of the given tensor expressed as a tensor of a
+
336  /// different number of dimensions.
+
337  /// Only works if we are contiguous.
+
338  template <int NewDim>
+
339  __host__ __device__ Tensor<T, NewDim, InnerContig, IndexT, PtrTraits>
+
340  view(std::initializer_list<IndexT> sizes);
+
341 
+
342  protected:
+
343  /// Raw pointer to where the tensor data begins
+
344  DataPtrType data_;
+
345 
+
346  /// Array of strides (in sizeof(T) terms) per each dimension
+
347  IndexT stride_[Dim];
+
348 
+
349  /// Size per each dimension
+
350  IndexT size_[Dim];
+
351 };
+
352 
+
353 // Utilities for checking a collection of tensors
+
354 namespace detail {
+
355 
+
356 template <typename IndexType>
+
357 bool canUseIndexType() {
+
358  return true;
+
359 }
+
360 
+
361 template <typename IndexType, typename T, typename... U>
+
362 bool canUseIndexType(const T& arg, const U&... args) {
+
363  return arg.canUseIndexType<IndexType>() &&
+
364  canUseIndexType(args...);
+
365 }
+
366 
+
367 } // namespace detail
368 
-
369 } // namespace detail
-
370 
-
371 template <typename IndexType, typename... T>
-
372 bool canUseIndexType(const T&... args) {
-
373  return detail::canUseIndexType(args...);
-
374 }
-
375 
-
376 namespace detail {
-
377 
-
378 /// Specialization for a view of a single value (0-dimensional)
-
379 template <typename TensorType, template <typename U> class PtrTraits>
-
380 class SubTensor<TensorType, 0, PtrTraits> {
-
381  public:
-
382  __host__ __device__ SubTensor<TensorType, 0, PtrTraits>
-
383  operator=(typename TensorType::DataType val) {
-
384  *data_ = val;
-
385  return *this;
-
386  }
-
387 
-
388  // operator T&
-
389  __host__ __device__ operator typename TensorType::DataType&() {
-
390  return *data_;
-
391  }
-
392 
-
393  // const operator T& returning const T&
-
394  __host__ __device__ operator const typename TensorType::DataType&() const {
-
395  return *data_;
-
396  }
-
397 
-
398  // operator& returning T*
-
399  __host__ __device__ typename TensorType::DataType* operator&() {
-
400  return data_;
-
401  }
-
402 
-
403  // const operator& returning const T*
-
404  __host__ __device__ const typename TensorType::DataType* operator&() const {
-
405  return data_;
-
406  }
-
407 
-
408  /// Returns a raw accessor to our slice.
-
409  __host__ __device__ inline typename TensorType::DataPtrType data() {
-
410  return data_;
-
411  }
-
412 
-
413  /// Returns a raw accessor to our slice (const).
-
414  __host__ __device__ inline
-
415  const typename TensorType::DataPtrType data() const {
-
416  return data_;
-
417  }
-
418 
-
419  /// Cast to a different datatype.
-
420  template <typename T>
-
421  __host__ __device__ T& as() {
-
422  return *dataAs<T>();
-
423  }
-
424 
-
425  /// Cast to a different datatype (const).
-
426  template <typename T>
-
427  __host__ __device__ const T& as() const {
-
428  return *dataAs<T>();
-
429  }
-
430 
-
431  /// Cast to a different datatype
-
432  template <typename T>
-
433  __host__ __device__ inline
-
434  typename PtrTraits<T>::PtrType dataAs() {
-
435  return reinterpret_cast<typename PtrTraits<T>::PtrType>(data_);
-
436  }
-
437 
-
438  /// Cast to a different datatype (const)
-
439  template <typename T>
-
440  __host__ __device__ inline
-
441  typename PtrTraits<const T>::PtrType dataAs() const {
-
442  return reinterpret_cast<typename PtrTraits<const T>::PtrType>(data_);
-
443  }
-
444 
-
445  /// Use the texture cache for reads
-
446  __device__ inline typename TensorType::DataType ldg() const {
-
447 #if __CUDA_ARCH__ >= 350
-
448  return __ldg(data_);
-
449 #else
-
450  return *data_;
-
451 #endif
-
452  }
-
453 
-
454  /// Use the texture cache for reads; cast as a particular type
-
455  template <typename T>
-
456  __device__ inline T ldgAs() const {
-
457 #if __CUDA_ARCH__ >= 350
-
458  return __ldg(dataAs<T>());
-
459 #else
-
460  return as<T>();
-
461 #endif
-
462  }
-
463 
-
464  protected:
-
465  /// One dimension greater can create us
-
466  friend class SubTensor<TensorType, 1, PtrTraits>;
-
467 
-
468  /// Our parent tensor can create us
-
469  friend class Tensor<typename TensorType::DataType,
-
470  1,
-
471  TensorType::IsInnerContig,
-
472  typename TensorType::IndexType,
-
473  PtrTraits>;
-
474 
-
475  __host__ __device__ inline SubTensor(
-
476  TensorType& t,
-
477  typename TensorType::DataPtrType data)
-
478  : tensor_(t),
-
479  data_(data) {
-
480  }
-
481 
-
482  /// The tensor we're referencing
-
483  TensorType& tensor_;
-
484 
-
485  /// Where our value is located
-
486  typename TensorType::DataPtrType const data_;
-
487 };
-
488 
-
489 /// A `SubDim`-rank slice of a parent Tensor
-
490 template <typename TensorType,
-
491  int SubDim,
-
492  template <typename U> class PtrTraits>
-
493 class SubTensor {
-
494  public:
-
495  /// Returns a view of the data located at our offset (the dimension
-
496  /// `SubDim` - 1 tensor).
-
497  __host__ __device__ inline
-
498  SubTensor<TensorType, SubDim - 1, PtrTraits>
-
499  operator[](typename TensorType::IndexType index) {
-
500  if (TensorType::IsInnerContig && SubDim == 1) {
-
501  // Innermost dimension is stride 1 for contiguous arrays
-
502  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
-
503  tensor_, data_ + index);
-
504  } else {
-
505  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
-
506  tensor_,
-
507  data_ + index * tensor_.getStride(TensorType::NumDim - SubDim));
-
508  }
-
509  }
-
510 
-
511  /// Returns a view of the data located at our offset (the dimension
-
512  /// `SubDim` - 1 tensor) (const).
-
513  __host__ __device__ inline
-
514  const SubTensor<TensorType, SubDim - 1, PtrTraits>
-
515  operator[](typename TensorType::IndexType index) const {
-
516  if (TensorType::IsInnerContig && SubDim == 1) {
-
517  // Innermost dimension is stride 1 for contiguous arrays
-
518  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
-
519  tensor_, data_ + index);
-
520  } else {
-
521  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
-
522  tensor_,
-
523  data_ + index * tensor_.getStride(TensorType::NumDim - SubDim));
-
524  }
-
525  }
-
526 
-
527  // operator& returning T*
-
528  __host__ __device__ typename TensorType::DataType* operator&() {
-
529  return data_;
-
530  }
-
531 
-
532  // const operator& returning const T*
-
533  __host__ __device__ const typename TensorType::DataType* operator&() const {
-
534  return data_;
-
535  }
-
536 
-
537  /// Returns a raw accessor to our slice.
-
538  __host__ __device__ inline typename TensorType::DataPtrType data() {
-
539  return data_;
-
540  }
-
541 
-
542  /// Returns a raw accessor to our slice (const).
-
543  __host__ __device__ inline
-
544  const typename TensorType::DataPtrType data() const {
-
545  return data_;
-
546  }
-
547 
-
548  /// Cast to a different datatype.
-
549  template <typename T>
-
550  __host__ __device__ T& as() {
-
551  return *dataAs<T>();
-
552  }
-
553 
-
554  /// Cast to a different datatype (const).
-
555  template <typename T>
-
556  __host__ __device__ const T& as() const {
-
557  return *dataAs<T>();
-
558  }
-
559 
-
560  /// Cast to a different datatype
-
561  template <typename T>
-
562  __host__ __device__ inline
-
563  typename PtrTraits<T>::PtrType dataAs() {
-
564  return reinterpret_cast<typename PtrTraits<T>::PtrType>(data_);
-
565  }
-
566 
-
567  /// Cast to a different datatype (const)
-
568  template <typename T>
-
569  __host__ __device__ inline
-
570  typename PtrTraits<const T>::PtrType dataAs() const {
-
571  return reinterpret_cast<typename PtrTraits<const T>::PtrType>(data_);
-
572  }
-
573 
-
574  /// Use the texture cache for reads
-
575  __device__ inline typename TensorType::DataType ldg() const {
-
576 #if __CUDA_ARCH__ >= 350
-
577  return __ldg(data_);
-
578 #else
-
579  return *data_;
-
580 #endif
-
581  }
-
582 
-
583  /// Use the texture cache for reads; cast as a particular type
-
584  template <typename T>
-
585  __device__ inline T ldgAs() const {
-
586 #if __CUDA_ARCH__ >= 350
-
587  return __ldg(dataAs<T>());
-
588 #else
-
589  return as<T>();
-
590 #endif
-
591  }
-
592 
-
593  /// Returns a tensor that is a view of the SubDim-dimensional slice
-
594  /// of this tensor, starting where our data begins
-
595  Tensor<typename TensorType::DataType,
-
596  SubDim,
-
597  TensorType::IsInnerContig,
-
598  typename TensorType::IndexType,
-
599  PtrTraits> view() {
-
600  return tensor_.template view<SubDim>(data_);
-
601  }
-
602 
-
603  protected:
-
604  /// One dimension greater can create us
-
605  friend class SubTensor<TensorType, SubDim + 1, PtrTraits>;
-
606 
-
607  /// Our parent tensor can create us
-
608  friend class
-
609  Tensor<typename TensorType::DataType,
-
610  TensorType::NumDim,
-
611  TensorType::IsInnerContig,
-
612  typename TensorType::IndexType,
-
613  PtrTraits>;
-
614 
-
615  __host__ __device__ inline SubTensor(
-
616  TensorType& t,
-
617  typename TensorType::DataPtrType data)
-
618  : tensor_(t),
-
619  data_(data) {
-
620  }
-
621 
-
622  /// The tensor we're referencing
-
623  TensorType& tensor_;
-
624 
-
625  /// The start of our sub-region
-
626  typename TensorType::DataPtrType const data_;
-
627 };
+
369 template <typename IndexType, typename... T>
+
370 bool canUseIndexType(const T&... args) {
+
371  return detail::canUseIndexType(args...);
+
372 }
+
373 
+
374 namespace detail {
+
375 
+
376 /// Specialization for a view of a single value (0-dimensional)
+
377 template <typename TensorType, template <typename U> class PtrTraits>
+
378 class SubTensor<TensorType, 0, PtrTraits> {
+
379  public:
+
380  __host__ __device__ SubTensor<TensorType, 0, PtrTraits>
+
381  operator=(typename TensorType::DataType val) {
+
382  *data_ = val;
+
383  return *this;
+
384  }
+
385 
+
386  // operator T&
+
387  __host__ __device__ operator typename TensorType::DataType&() {
+
388  return *data_;
+
389  }
+
390 
+
391  // const operator T& returning const T&
+
392  __host__ __device__ operator const typename TensorType::DataType&() const {
+
393  return *data_;
+
394  }
+
395 
+
396  // operator& returning T*
+
397  __host__ __device__ typename TensorType::DataType* operator&() {
+
398  return data_;
+
399  }
+
400 
+
401  // const operator& returning const T*
+
402  __host__ __device__ const typename TensorType::DataType* operator&() const {
+
403  return data_;
+
404  }
+
405 
+
406  /// Returns a raw accessor to our slice.
+
407  __host__ __device__ inline typename TensorType::DataPtrType data() {
+
408  return data_;
+
409  }
+
410 
+
411  /// Returns a raw accessor to our slice (const).
+
412  __host__ __device__ inline
+
413  const typename TensorType::DataPtrType data() const {
+
414  return data_;
+
415  }
+
416 
+
417  /// Cast to a different datatype.
+
418  template <typename T>
+
419  __host__ __device__ T& as() {
+
420  return *dataAs<T>();
+
421  }
+
422 
+
423  /// Cast to a different datatype (const).
+
424  template <typename T>
+
425  __host__ __device__ const T& as() const {
+
426  return *dataAs<T>();
+
427  }
+
428 
+
429  /// Cast to a different datatype
+
430  template <typename T>
+
431  __host__ __device__ inline
+
432  typename PtrTraits<T>::PtrType dataAs() {
+
433  return reinterpret_cast<typename PtrTraits<T>::PtrType>(data_);
+
434  }
+
435 
+
436  /// Cast to a different datatype (const)
+
437  template <typename T>
+
438  __host__ __device__ inline
+
439  typename PtrTraits<const T>::PtrType dataAs() const {
+
440  return reinterpret_cast<typename PtrTraits<const T>::PtrType>(data_);
+
441  }
+
442 
+
443  /// Use the texture cache for reads
+
444  __device__ inline typename TensorType::DataType ldg() const {
+
445 #if __CUDA_ARCH__ >= 350
+
446  return __ldg(data_);
+
447 #else
+
448  return *data_;
+
449 #endif
+
450  }
+
451 
+
452  /// Use the texture cache for reads; cast as a particular type
+
453  template <typename T>
+
454  __device__ inline T ldgAs() const {
+
455 #if __CUDA_ARCH__ >= 350
+
456  return __ldg(dataAs<T>());
+
457 #else
+
458  return as<T>();
+
459 #endif
+
460  }
+
461 
+
462  protected:
+
463  /// One dimension greater can create us
+
464  friend class SubTensor<TensorType, 1, PtrTraits>;
+
465 
+
466  /// Our parent tensor can create us
+
467  friend class Tensor<typename TensorType::DataType,
+
468  1,
+
469  TensorType::IsInnerContig,
+
470  typename TensorType::IndexType,
+
471  PtrTraits>;
+
472 
+
473  __host__ __device__ inline SubTensor(
+
474  TensorType& t,
+
475  typename TensorType::DataPtrType data)
+
476  : tensor_(t),
+
477  data_(data) {
+
478  }
+
479 
+
480  /// The tensor we're referencing
+
481  TensorType& tensor_;
+
482 
+
483  /// Where our value is located
+
484  typename TensorType::DataPtrType const data_;
+
485 };
+
486 
+
487 /// A `SubDim`-rank slice of a parent Tensor
+
488 template <typename TensorType,
+
489  int SubDim,
+
490  template <typename U> class PtrTraits>
+
491 class SubTensor {
+
492  public:
+
493  /// Returns a view of the data located at our offset (the dimension
+
494  /// `SubDim` - 1 tensor).
+
495  __host__ __device__ inline
+
496  SubTensor<TensorType, SubDim - 1, PtrTraits>
+
497  operator[](typename TensorType::IndexType index) {
+
498  if (TensorType::IsInnerContig && SubDim == 1) {
+
499  // Innermost dimension is stride 1 for contiguous arrays
+
500  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
+
501  tensor_, data_ + index);
+
502  } else {
+
503  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
+
504  tensor_,
+
505  data_ + index * tensor_.getStride(TensorType::NumDim - SubDim));
+
506  }
+
507  }
+
508 
+
509  /// Returns a view of the data located at our offset (the dimension
+
510  /// `SubDim` - 1 tensor) (const).
+
511  __host__ __device__ inline
+
512  const SubTensor<TensorType, SubDim - 1, PtrTraits>
+
513  operator[](typename TensorType::IndexType index) const {
+
514  if (TensorType::IsInnerContig && SubDim == 1) {
+
515  // Innermost dimension is stride 1 for contiguous arrays
+
516  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
+
517  tensor_, data_ + index);
+
518  } else {
+
519  return SubTensor<TensorType, SubDim - 1, PtrTraits>(
+
520  tensor_,
+
521  data_ + index * tensor_.getStride(TensorType::NumDim - SubDim));
+
522  }
+
523  }
+
524 
+
525  // operator& returning T*
+
526  __host__ __device__ typename TensorType::DataType* operator&() {
+
527  return data_;
+
528  }
+
529 
+
530  // const operator& returning const T*
+
531  __host__ __device__ const typename TensorType::DataType* operator&() const {
+
532  return data_;
+
533  }
+
534 
+
535  /// Returns a raw accessor to our slice.
+
536  __host__ __device__ inline typename TensorType::DataPtrType data() {
+
537  return data_;
+
538  }
+
539 
+
540  /// Returns a raw accessor to our slice (const).
+
541  __host__ __device__ inline
+
542  const typename TensorType::DataPtrType data() const {
+
543  return data_;
+
544  }
+
545 
+
546  /// Cast to a different datatype.
+
547  template <typename T>
+
548  __host__ __device__ T& as() {
+
549  return *dataAs<T>();
+
550  }
+
551 
+
552  /// Cast to a different datatype (const).
+
553  template <typename T>
+
554  __host__ __device__ const T& as() const {
+
555  return *dataAs<T>();
+
556  }
+
557 
+
558  /// Cast to a different datatype
+
559  template <typename T>
+
560  __host__ __device__ inline
+
561  typename PtrTraits<T>::PtrType dataAs() {
+
562  return reinterpret_cast<typename PtrTraits<T>::PtrType>(data_);
+
563  }
+
564 
+
565  /// Cast to a different datatype (const)
+
566  template <typename T>
+
567  __host__ __device__ inline
+
568  typename PtrTraits<const T>::PtrType dataAs() const {
+
569  return reinterpret_cast<typename PtrTraits<const T>::PtrType>(data_);
+
570  }
+
571 
+
572  /// Use the texture cache for reads
+
573  __device__ inline typename TensorType::DataType ldg() const {
+
574 #if __CUDA_ARCH__ >= 350
+
575  return __ldg(data_);
+
576 #else
+
577  return *data_;
+
578 #endif
+
579  }
+
580 
+
581  /// Use the texture cache for reads; cast as a particular type
+
582  template <typename T>
+
583  __device__ inline T ldgAs() const {
+
584 #if __CUDA_ARCH__ >= 350
+
585  return __ldg(dataAs<T>());
+
586 #else
+
587  return as<T>();
+
588 #endif
+
589  }
+
590 
+
591  /// Returns a tensor that is a view of the SubDim-dimensional slice
+
592  /// of this tensor, starting where our data begins
+
593  Tensor<typename TensorType::DataType,
+
594  SubDim,
+
595  TensorType::IsInnerContig,
+
596  typename TensorType::IndexType,
+
597  PtrTraits> view() {
+
598  return tensor_.template view<SubDim>(data_);
+
599  }
+
600 
+
601  protected:
+
602  /// One dimension greater can create us
+
603  friend class SubTensor<TensorType, SubDim + 1, PtrTraits>;
+
604 
+
605  /// Our parent tensor can create us
+
606  friend class
+
607  Tensor<typename TensorType::DataType,
+
608  TensorType::NumDim,
+
609  TensorType::IsInnerContig,
+
610  typename TensorType::IndexType,
+
611  PtrTraits>;
+
612 
+
613  __host__ __device__ inline SubTensor(
+
614  TensorType& t,
+
615  typename TensorType::DataPtrType data)
+
616  : tensor_(t),
+
617  data_(data) {
+
618  }
+
619 
+
620  /// The tensor we're referencing
+
621  TensorType& tensor_;
+
622 
+
623  /// The start of our sub-region
+
624  typename TensorType::DataPtrType const data_;
+
625 };
+
626 
+
627 } // namespace detail
628 
-
629 } // namespace detail
-
630 
-
631 template <typename T, int Dim, bool InnerContig,
-
632  typename IndexT, template <typename U> class PtrTraits>
-
633 __host__ __device__ inline
-
634 detail::SubTensor<Tensor<T, Dim, InnerContig, IndexT, PtrTraits>,
-
635  Dim - 1, PtrTraits>
-
636  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator[](IndexT index) {
-
637  return detail::SubTensor<TensorType, Dim - 1, PtrTraits>(
-
638  detail::SubTensor<TensorType, Dim, PtrTraits>(
-
639  *this, data_)[index]);
-
640 }
-
641 
-
642 template <typename T, int Dim, bool InnerContig,
-
643  typename IndexT, template <typename U> class PtrTraits>
-
644 __host__ __device__ inline
-
645 const detail::SubTensor<Tensor<T, Dim, InnerContig, IndexT, PtrTraits>,
-
646  Dim - 1, PtrTraits>
-
647  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator[](IndexT index) const {
-
648  return detail::SubTensor<TensorType, Dim - 1, PtrTraits>(
-
649  detail::SubTensor<TensorType, Dim, PtrTraits>(
-
650  const_cast<TensorType&>(*this), data_)[index]);
-
651 }
+
629 template <typename T, int Dim, bool InnerContig,
+
630  typename IndexT, template <typename U> class PtrTraits>
+
631 __host__ __device__ inline
+
632 detail::SubTensor<Tensor<T, Dim, InnerContig, IndexT, PtrTraits>,
+
633  Dim - 1, PtrTraits>
+
634  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator[](IndexT index) {
+
635  return detail::SubTensor<TensorType, Dim - 1, PtrTraits>(
+
636  detail::SubTensor<TensorType, Dim, PtrTraits>(
+
637  *this, data_)[index]);
+
638 }
+
639 
+
640 template <typename T, int Dim, bool InnerContig,
+
641  typename IndexT, template <typename U> class PtrTraits>
+
642 __host__ __device__ inline
+
643 const detail::SubTensor<Tensor<T, Dim, InnerContig, IndexT, PtrTraits>,
+
644  Dim - 1, PtrTraits>
+
645  Tensor<T, Dim, InnerContig, IndexT, PtrTraits>::operator[](IndexT index) const {
+
646  return detail::SubTensor<TensorType, Dim - 1, PtrTraits>(
+
647  detail::SubTensor<TensorType, Dim, PtrTraits>(
+
648  const_cast<TensorType&>(*this), data_)[index]);
+
649 }
+
650 
+
651 } } // namespace
652 
-
653 } } // namespace
-
654 
-
655 #include "Tensor-inl.cuh"
-
faiss::gpu::Tensor::upcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastOuter()
Definition: Tensor-inl.cuh:454
-
faiss::gpu::Tensor::operator[]
__host__ __device__ detail::SubTensor< TensorType, Dim-1, PtrTraits > operator[](IndexT)
Returns a read/write view of a portion of our tensor.
Definition: Tensor.cuh:636
-
faiss::gpu::Tensor::castIndexType
__host__ Tensor< T, Dim, InnerContig, NewIndexT, PtrTraits > castIndexType() const
Definition: Tensor-inl.cuh:305
-
faiss::gpu::detail::SubTensor::view
Tensor< typename TensorType::DataType, SubDim, TensorType::IsInnerContig, typename TensorType::IndexType, PtrTraits > view()
Definition: Tensor.cuh:599
-
faiss::gpu::Tensor::isContiguousDim
__host__ __device__ bool isContiguousDim(int i) const
Returns true if the given dimension index has no padding.
Definition: Tensor-inl.cuh:411
-
faiss::gpu::Tensor::cast
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > cast()
Definition: Tensor-inl.cuh:222
-
faiss::gpu::Tensor::numElements
__host__ __device__ size_t numElements() const
Definition: Tensor-inl.cuh:352
-
faiss::gpu::Tensor::downcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastOuter()
Definition: Tensor-inl.cuh:510
-
faiss::gpu::detail::SubTensor::dataAs
__host__ __device__ PtrTraits< const T >::PtrType dataAs() const
Cast to a different datatype (const)
Definition: Tensor.cuh:570
-
faiss::gpu::Tensor::dataAs
__host__ __device__ const PtrTraits< const U >::PtrType dataAs() const
Cast to a different datatype.
Definition: Tensor.cuh:210
-
faiss::gpu::Tensor::dataAs
__host__ __device__ PtrTraits< U >::PtrType dataAs()
Cast to a different datatype.
Definition: Tensor.cuh:203
-
faiss::gpu::detail::SubTensor::ldgAs
__device__ T ldgAs() const
Use the texture cache for reads; cast as a particular type.
Definition: Tensor.cuh:585
-
faiss::gpu::Tensor::canCastResize
__host__ __device__ bool canCastResize() const
Returns true if we can castResize() this tensor to the new type.
Definition: Tensor-inl.cuh:278
-
faiss::gpu::Tensor::data_
DataPtrType data_
Raw pointer to where the tensor data begins.
Definition: Tensor.cuh:346
-
faiss::gpu::detail::SubTensor::dataAs
__host__ __device__ PtrTraits< T >::PtrType dataAs()
Cast to a different datatype.
Definition: Tensor.cuh:563
+
653 #include "Tensor-inl.cuh"
+
faiss::gpu::Tensor::upcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastOuter()
Definition: Tensor-inl.cuh:485
+
faiss::gpu::Tensor::operator[]
__host__ __device__ detail::SubTensor< TensorType, Dim-1, PtrTraits > operator[](IndexT)
Returns a read/write view of a portion of our tensor.
Definition: Tensor.cuh:634
+
faiss::gpu::Tensor::castIndexType
__host__ Tensor< T, Dim, InnerContig, NewIndexT, PtrTraits > castIndexType() const
Definition: Tensor-inl.cuh:336
+
faiss::gpu::detail::SubTensor::view
Tensor< typename TensorType::DataType, SubDim, TensorType::IsInnerContig, typename TensorType::IndexType, PtrTraits > view()
Definition: Tensor.cuh:597
+
faiss::gpu::Tensor::isContiguousDim
__host__ __device__ bool isContiguousDim(int i) const
Returns true if the given dimension index has no padding.
Definition: Tensor-inl.cuh:442
+
faiss::gpu::Tensor::cast
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > cast()
Definition: Tensor-inl.cuh:253
+
faiss::gpu::Tensor::numElements
__host__ __device__ size_t numElements() const
Definition: Tensor-inl.cuh:383
+
faiss::gpu::Tensor::downcastOuter
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastOuter()
Definition: Tensor-inl.cuh:541
+
faiss::gpu::detail::SubTensor::dataAs
__host__ __device__ PtrTraits< const T >::PtrType dataAs() const
Cast to a different datatype (const)
Definition: Tensor.cuh:568
+
faiss::gpu::Tensor::dataAs
__host__ __device__ const PtrTraits< const U >::PtrType dataAs() const
Cast to a different datatype.
Definition: Tensor.cuh:208
+
faiss::gpu::Tensor::dataAs
__host__ __device__ PtrTraits< U >::PtrType dataAs()
Cast to a different datatype.
Definition: Tensor.cuh:201
+
faiss::gpu::detail::SubTensor::ldgAs
__device__ T ldgAs() const
Use the texture cache for reads; cast as a particular type.
Definition: Tensor.cuh:583
+
faiss::gpu::Tensor::canCastResize
__host__ __device__ bool canCastResize() const
Returns true if we can castResize() this tensor to the new type.
Definition: Tensor-inl.cuh:309
+
faiss::gpu::Tensor::data_
DataPtrType data_
Raw pointer to where the tensor data begins.
Definition: Tensor.cuh:344
+
faiss::gpu::detail::SubTensor::dataAs
__host__ __device__ PtrTraits< T >::PtrType dataAs()
Cast to a different datatype.
Definition: Tensor.cuh:561
faiss::gpu::Tensor::Tensor
__host__ __device__ Tensor()
Default constructor.
Definition: Tensor-inl.cuh:20
-
faiss::gpu::Tensor::end
__host__ __device__ DataPtrType end() const
Definition: Tensor.cuh:196
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::dataAs
__host__ __device__ PtrTraits< const T >::PtrType dataAs() const
Cast to a different datatype (const)
Definition: Tensor.cuh:441
-
faiss::gpu::Tensor::upcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastInner()
Definition: Tensor-inl.cuh:483
-
faiss::gpu::detail::SubTensor::data
__host__ __device__ const TensorType::DataPtrType data() const
Returns a raw accessor to our slice (const).
Definition: Tensor.cuh:544
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::ldg
__device__ TensorType::DataType ldg() const
Use the texture cache for reads.
Definition: Tensor.cuh:446
-
faiss::gpu::Tensor::narrowOutermost
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrowOutermost(IndexT start, IndexT size)
Definition: Tensor-inl.cuh:625
-
faiss::gpu::Tensor::stride_
IndexT stride_[Dim]
Array of strides (in sizeof(T) terms) per each dimension.
Definition: Tensor.cuh:349
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::as
__host__ __device__ T & as()
Cast to a different datatype.
Definition: Tensor.cuh:421
-
faiss::gpu::detail::SubTensor::as
__host__ __device__ T & as()
Cast to a different datatype.
Definition: Tensor.cuh:550
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::tensor_
TensorType & tensor_
The tensor we&#39;re referencing.
Definition: Tensor.cuh:483
-
faiss::gpu::Tensor::isContiguous
__host__ __device__ bool isContiguous() const
Definition: Tensor-inl.cuh:365
-
faiss::gpu::Tensor::data
__host__ __device__ const DataPtrType data() const
Returns a raw pointer to the start of our data (const).
Definition: Tensor.cuh:190
-
faiss::gpu::detail::SubTensor::ldg
__device__ TensorType::DataType ldg() const
Use the texture cache for reads.
Definition: Tensor.cuh:575
-
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:247
-
faiss::gpu::detail::SubTensor::data_
TensorType::DataPtrType const data_
The start of our sub-region.
Definition: Tensor.cuh:625
-
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:101
-
faiss::gpu::Tensor::size_
IndexT size_[Dim]
Size per each dimension.
Definition: Tensor.cuh:352
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::ldgAs
__device__ T ldgAs() const
Use the texture cache for reads; cast as a particular type.
Definition: Tensor.cuh:456
-
faiss::gpu::detail::SubTensor::operator[]
__host__ __device__ const SubTensor< TensorType, SubDim-1, PtrTraits > operator[](typename TensorType::IndexType index) const
Definition: Tensor.cuh:515
-
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:252
-
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
+
faiss::gpu::Tensor::end
__host__ __device__ DataPtrType end() const
Definition: Tensor.cuh:194
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::dataAs
__host__ __device__ PtrTraits< const T >::PtrType dataAs() const
Cast to a different datatype (const)
Definition: Tensor.cuh:439
+
faiss::gpu::Tensor::upcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > upcastInner()
Definition: Tensor-inl.cuh:514
+
faiss::gpu::detail::SubTensor::data
__host__ __device__ const TensorType::DataPtrType data() const
Returns a raw accessor to our slice (const).
Definition: Tensor.cuh:542
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::ldg
__device__ TensorType::DataType ldg() const
Use the texture cache for reads.
Definition: Tensor.cuh:444
+
faiss::gpu::Tensor::narrowOutermost
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrowOutermost(IndexT start, IndexT size)
Definition: Tensor-inl.cuh:656
+
faiss::gpu::Tensor::stride_
IndexT stride_[Dim]
Array of strides (in sizeof(T) terms) per each dimension.
Definition: Tensor.cuh:347
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::as
__host__ __device__ T & as()
Cast to a different datatype.
Definition: Tensor.cuh:419
+
faiss::gpu::detail::SubTensor::as
__host__ __device__ T & as()
Cast to a different datatype.
Definition: Tensor.cuh:548
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::tensor_
TensorType & tensor_
The tensor we&#39;re referencing.
Definition: Tensor.cuh:481
+
faiss::gpu::Tensor::isContiguous
__host__ __device__ bool isContiguous() const
Definition: Tensor-inl.cuh:396
+
faiss::gpu::Tensor::data
__host__ __device__ const DataPtrType data() const
Returns a raw pointer to the start of our data (const).
Definition: Tensor.cuh:188
+
faiss::gpu::detail::SubTensor::ldg
__device__ TensorType::DataType ldg() const
Use the texture cache for reads.
Definition: Tensor.cuh:573
+
faiss::gpu::Tensor::sizes
__host__ __device__ const IndexT * sizes() const
Returns the size array.
Definition: Tensor.cuh:245
+
faiss::gpu::detail::SubTensor::data_
TensorType::DataPtrType const data_
The start of our sub-region.
Definition: Tensor.cuh:623
+
faiss::gpu::Tensor::copyFrom
__host__ void copyFrom(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies a tensor into ourselves; sizes must match.
Definition: Tensor-inl.cuh:132
+
faiss::gpu::Tensor::size_
IndexT size_[Dim]
Size per each dimension.
Definition: Tensor.cuh:350
+
faiss::gpu::Tensor::operator=
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)
Assignment.
Definition: Tensor-inl.cuh:50
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::ldgAs
__device__ T ldgAs() const
Use the texture cache for reads; cast as a particular type.
Definition: Tensor.cuh:454
+
faiss::gpu::detail::SubTensor::operator[]
__host__ __device__ const SubTensor< TensorType, SubDim-1, PtrTraits > operator[](typename TensorType::IndexType index) const
Definition: Tensor.cuh:513
+
faiss::gpu::Tensor::strides
__host__ __device__ const IndexT * strides() const
Returns the stride array.
Definition: Tensor.cuh:250
+
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
-
faiss::gpu::Tensor::isSameSize
__host__ __device__ bool isSameSize(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Returns true if the two tensors are of the same dimensionality and size.
Definition: Tensor-inl.cuh:203
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::data_
TensorType::DataPtrType const data_
Where our value is located.
Definition: Tensor.cuh:486
-
faiss::gpu::Tensor::downcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastInner()
Definition: Tensor-inl.cuh:555
-
faiss::gpu::Tensor::narrow
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrow(int dim, IndexT start, IndexT size)
Definition: Tensor-inl.cuh:633
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
-
faiss::gpu::Tensor::copyTo
__host__ void copyTo(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies ourselves into a tensor; sizes must match.
Definition: Tensor-inl.cuh:140
+
faiss::gpu::Tensor::isSameSize
__host__ __device__ bool isSameSize(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Returns true if the two tensors are of the same dimensionality and size.
Definition: Tensor-inl.cuh:234
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::data_
TensorType::DataPtrType const data_
Where our value is located.
Definition: Tensor.cuh:484
+
faiss::gpu::Tensor::downcastInner
__host__ __device__ Tensor< T, NewDim, InnerContig, IndexT, PtrTraits > downcastInner()
Definition: Tensor-inl.cuh:586
+
faiss::gpu::Tensor::narrow
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > narrow(int dim, IndexT start, IndexT size)
Definition: Tensor-inl.cuh:664
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
+
faiss::gpu::Tensor::copyTo
__host__ void copyTo(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t, cudaStream_t stream)
Copies ourselves into a tensor; sizes must match.
Definition: Tensor-inl.cuh:171
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
-
faiss::gpu::Tensor::canUseIndexType
__host__ bool canUseIndexType() const
Definition: Tensor-inl.cuh:325
-
faiss::gpu::Tensor::transpose
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > transpose(int dim1, int dim2) const
Definition: Tensor-inl.cuh:420
-
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:232
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >
Specialization for a view of a single value (0-dimensional)
Definition: Tensor.cuh:380
-
faiss::gpu::Tensor::end
__host__ __device__ DataPtrType end()
Definition: Tensor.cuh:184
+
faiss::gpu::Tensor::canUseIndexType
__host__ bool canUseIndexType() const
Definition: Tensor-inl.cuh:356
+
faiss::gpu::Tensor::transpose
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > transpose(int dim1, int dim2) const
Definition: Tensor-inl.cuh:451
+
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:230
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >
Specialization for a view of a single value (0-dimensional)
Definition: Tensor.cuh:378
+
faiss::gpu::Tensor::end
__host__ __device__ DataPtrType end()
Definition: Tensor.cuh:182
-
faiss::gpu::detail::SubTensor::tensor_
TensorType & tensor_
The tensor we&#39;re referencing.
Definition: Tensor.cuh:622
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::data
__host__ __device__ const TensorType::DataPtrType data() const
Returns a raw accessor to our slice (const).
Definition: Tensor.cuh:415
-
faiss::gpu::Tensor::operator=
__host__ __device__ Tensor< T, Dim, InnerContig, IndexT, PtrTraits > & operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default
Assignment.
-
faiss::gpu::detail::SubTensor::operator[]
__host__ __device__ SubTensor< TensorType, SubDim-1, PtrTraits > operator[](typename TensorType::IndexType index)
Definition: Tensor.cuh:499
-
faiss::gpu::detail::SubTensor::as
__host__ __device__ const T & as() const
Cast to a different datatype (const).
Definition: Tensor.cuh:556
+
faiss::gpu::detail::SubTensor::tensor_
TensorType & tensor_
The tensor we&#39;re referencing.
Definition: Tensor.cuh:620
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::data
__host__ __device__ const TensorType::DataPtrType data() const
Returns a raw accessor to our slice (const).
Definition: Tensor.cuh:413
+
faiss::gpu::detail::SubTensor::operator[]
__host__ __device__ SubTensor< TensorType, SubDim-1, PtrTraits > operator[](typename TensorType::IndexType index)
Definition: Tensor.cuh:497
+
faiss::gpu::detail::SubTensor::as
__host__ __device__ const T & as() const
Cast to a different datatype (const).
Definition: Tensor.cuh:554
faiss::gpu::detail::SubTensor
A SubDim-rank slice of a parent Tensor.
Definition: Tensor.cuh:37
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::dataAs
__host__ __device__ PtrTraits< T >::PtrType dataAs()
Cast to a different datatype.
Definition: Tensor.cuh:434
-
faiss::gpu::detail::SubTensor::data
__host__ __device__ TensorType::DataPtrType data()
Returns a raw accessor to our slice.
Definition: Tensor.cuh:538
-
faiss::gpu::Tensor::castResize
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > castResize()
Definition: Tensor-inl.cuh:244
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::data
__host__ __device__ TensorType::DataPtrType data()
Returns a raw accessor to our slice.
Definition: Tensor.cuh:409
-
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::as
__host__ __device__ const T & as() const
Cast to a different datatype (const).
Definition: Tensor.cuh:427
-
faiss::gpu::Tensor::getSizeInBytes
__host__ __device__ size_t getSizeInBytes() const
Definition: Tensor.cuh:242
-
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view()
Definition: Tensor-inl.cuh:618
-
faiss::gpu::Tensor::isSame
__host__ __device__ bool isSame(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Definition: Tensor-inl.cuh:180
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::dataAs
__host__ __device__ PtrTraits< T >::PtrType dataAs()
Cast to a different datatype.
Definition: Tensor.cuh:432
+
faiss::gpu::detail::SubTensor::data
__host__ __device__ TensorType::DataPtrType data()
Returns a raw accessor to our slice.
Definition: Tensor.cuh:536
+
faiss::gpu::Tensor::castResize
__host__ __device__ Tensor< U, Dim, InnerContig, IndexT, PtrTraits > castResize()
Definition: Tensor-inl.cuh:275
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::data
__host__ __device__ TensorType::DataPtrType data()
Returns a raw accessor to our slice.
Definition: Tensor.cuh:407
+
faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >::as
__host__ __device__ const T & as() const
Cast to a different datatype (const).
Definition: Tensor.cuh:425
+
faiss::gpu::Tensor::getSizeInBytes
__host__ __device__ size_t getSizeInBytes() const
Definition: Tensor.cuh:240
+
faiss::gpu::Tensor::view
__host__ __device__ Tensor< T, SubDim, InnerContig, IndexT, PtrTraits > view()
Definition: Tensor-inl.cuh:649
+
faiss::gpu::Tensor::isSame
__host__ __device__ bool isSame(const Tensor< OtherT, OtherDim, InnerContig, IndexT, PtrTraits > &rhs) const
Definition: Tensor-inl.cuh:211
diff --git a/docs/html/TestGpuIndexIVFFlat_8cpp_source.html b/docs/html/TestGpuIndexIVFFlat_8cpp_source.html index df1eb178b..a69abfaee 100644 --- a/docs/html/TestGpuIndexIVFFlat_8cpp_source.html +++ b/docs/html/TestGpuIndexIVFFlat_8cpp_source.html @@ -705,7 +705,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); - +
faiss::gpu::GpuIndexFlatConfig::useFloat16
bool useFloat16
Whether or not data is stored as float16.
Definition: GpuIndexFlat.h:35
faiss::gpu::GpuIndexConfig::device
int device
GPU device on which the index is resident.
Definition: GpuIndex.h:27
diff --git a/docs/html/Transpose_8cuh_source.html b/docs/html/Transpose_8cuh_source.html index 9fb287c0b..d40a7488e 100644 --- a/docs/html/Transpose_8cuh_source.html +++ b/docs/html/Transpose_8cuh_source.html @@ -245,10 +245,10 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
156 
157 } } // namespace
-
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:226
-
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:178
+
faiss::gpu::Tensor::getSize
__host__ __device__ IndexT getSize(int i) const
Definition: Tensor.cuh:224
+
faiss::gpu::Tensor::data
__host__ __device__ DataPtrType data()
Returns a raw pointer to the start of our data.
Definition: Tensor.cuh:176
faiss::gpu::Tensor
Our tensor type.
Definition: Tensor.cuh:30
-
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:232
+
faiss::gpu::Tensor::getStride
__host__ __device__ IndexT getStride(int i) const
Definition: Tensor.cuh:230
diff --git a/docs/html/VectorTransform_8cpp_source.html b/docs/html/VectorTransform_8cpp_source.html index 464fcfb89..67496906a 100644 --- a/docs/html/VectorTransform_8cpp_source.html +++ b/docs/html/VectorTransform_8cpp_source.html @@ -179,973 +179,1055 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
94 
95 /*********************************************
96  * LinearTransform
-
97  *********************************************/
-
98 /// both d_in > d_out and d_out < d_in are supported
-
99 LinearTransform::LinearTransform (int d_in, int d_out,
-
100  bool have_bias):
-
101  VectorTransform (d_in, d_out), have_bias (have_bias),
-
102  verbose (false)
-
103 {}
-
104 
-
105 void LinearTransform::apply_noalloc (Index::idx_t n, const float * x,
-
106  float * xt) const
-
107 {
-
108  FAISS_THROW_IF_NOT_MSG(is_trained, "Transformation not trained yet");
-
109 
-
110  float c_factor;
-
111  if (have_bias) {
-
112  FAISS_THROW_IF_NOT_MSG (b.size() == d_out, "Bias not initialized");
-
113  float * xi = xt;
-
114  for (int i = 0; i < n; i++)
-
115  for(int j = 0; j < d_out; j++)
-
116  *xi++ = b[j];
-
117  c_factor = 1.0;
-
118  } else {
-
119  c_factor = 0.0;
-
120  }
-
121 
-
122  FAISS_THROW_IF_NOT_MSG (A.size() == d_out * d_in,
-
123  "Transformation matrix not initialized");
-
124 
-
125  float one = 1;
-
126  FINTEGER nbiti = d_out, ni = n, di = d_in;
-
127  sgemm_ ("Transposed", "Not transposed",
-
128  &nbiti, &ni, &di,
-
129  &one, A.data(), &di, x, &di, &c_factor, xt, &nbiti);
-
130 
-
131 }
-
132 
+
97  *********************************************/
+
98 
+
99 /// both d_in > d_out and d_out < d_in are supported
+
100 LinearTransform::LinearTransform (int d_in, int d_out,
+
101  bool have_bias):
+
102  VectorTransform (d_in, d_out), have_bias (have_bias),
+
103  is_orthonormal (false), verbose (false)
+
104 {}
+
105 
+
106 void LinearTransform::apply_noalloc (Index::idx_t n, const float * x,
+
107  float * xt) const
+
108 {
+
109  FAISS_THROW_IF_NOT_MSG(is_trained, "Transformation not trained yet");
+
110 
+
111  float c_factor;
+
112  if (have_bias) {
+
113  FAISS_THROW_IF_NOT_MSG (b.size() == d_out, "Bias not initialized");
+
114  float * xi = xt;
+
115  for (int i = 0; i < n; i++)
+
116  for(int j = 0; j < d_out; j++)
+
117  *xi++ = b[j];
+
118  c_factor = 1.0;
+
119  } else {
+
120  c_factor = 0.0;
+
121  }
+
122 
+
123  FAISS_THROW_IF_NOT_MSG (A.size() == d_out * d_in,
+
124  "Transformation matrix not initialized");
+
125 
+
126  float one = 1;
+
127  FINTEGER nbiti = d_out, ni = n, di = d_in;
+
128  sgemm_ ("Transposed", "Not transposed",
+
129  &nbiti, &ni, &di,
+
130  &one, A.data(), &di, x, &di, &c_factor, xt, &nbiti);
+
131 
+
132 }
133 
-
134 void LinearTransform::transform_transpose (idx_t n, const float * y,
-
135  float *x) const
-
136 {
-
137  if (have_bias) { // allocate buffer to store bias-corrected data
-
138  float *y_new = new float [n * d_out];
-
139  const float *yr = y;
-
140  float *yw = y_new;
-
141  for (idx_t i = 0; i < n; i++) {
-
142  for (int j = 0; j < d_out; j++) {
-
143  *yw++ = *yr++ - b [j];
-
144  }
-
145  }
-
146  y = y_new;
-
147  }
-
148 
-
149  {
-
150  FINTEGER dii = d_in, doi = d_out, ni = n;
-
151  float one = 1.0, zero = 0.0;
-
152  sgemm_ ("Not", "Not", &dii, &ni, &doi,
-
153  &one, A.data (), &dii, y, &doi, &zero, x, &dii);
-
154  }
-
155 
-
156  if (have_bias) delete [] y;
-
157 }
-
158 
+
134 
+
135 void LinearTransform::transform_transpose (idx_t n, const float * y,
+
136  float *x) const
+
137 {
+
138  if (have_bias) { // allocate buffer to store bias-corrected data
+
139  float *y_new = new float [n * d_out];
+
140  const float *yr = y;
+
141  float *yw = y_new;
+
142  for (idx_t i = 0; i < n; i++) {
+
143  for (int j = 0; j < d_out; j++) {
+
144  *yw++ = *yr++ - b [j];
+
145  }
+
146  }
+
147  y = y_new;
+
148  }
+
149 
+
150  {
+
151  FINTEGER dii = d_in, doi = d_out, ni = n;
+
152  float one = 1.0, zero = 0.0;
+
153  sgemm_ ("Not", "Not", &dii, &ni, &doi,
+
154  &one, A.data (), &dii, y, &doi, &zero, x, &dii);
+
155  }
+
156 
+
157  if (have_bias) delete [] y;
+
158 }
159 
-
160 /*********************************************
-
161  * RandomRotationMatrix
-
162  *********************************************/
-
163 
-
164 void RandomRotationMatrix::init (int seed)
-
165 {
-
166 
-
167  if(d_out <= d_in) {
-
168  A.resize (d_out * d_in);
-
169  float *q = A.data();
-
170  float_randn(q, d_out * d_in, seed);
-
171  matrix_qr(d_in, d_out, q);
-
172  } else {
-
173  A.resize (d_out * d_out);
-
174  float *q = A.data();
-
175  float_randn(q, d_out * d_out, seed);
-
176  matrix_qr(d_out, d_out, q);
-
177  // remove columns
-
178  int i, j;
-
179  for (i = 0; i < d_out; i++) {
-
180  for(j = 0; j < d_in; j++) {
-
181  q[i * d_in + j] = q[i * d_out + j];
-
182  }
-
183  }
-
184  A.resize(d_in * d_out);
-
185  }
-
186 
-
187 }
-
188 
-
189 void RandomRotationMatrix::reverse_transform (idx_t n, const float * xt,
-
190  float *x) const
-
191 {
-
192  transform_transpose (n, xt, x);
-
193 }
-
194 
-
195 /*********************************************
-
196  * PCAMatrix
-
197  *********************************************/
+
160 void LinearTransform::set_is_orthonormal ()
+
161 {
+
162  if (d_out > d_in) {
+
163  // not clear what we should do in this case
+
164  is_orthonormal = false;
+
165  return;
+
166  }
+
167  if (d_out == 0) { // borderline case, unnormalized matrix
+
168  is_orthonormal = true;
+
169  return;
+
170  }
+
171 
+
172  double eps = 4e-5;
+
173  FAISS_ASSERT(A.size() >= d_out * d_in);
+
174  {
+
175  std::vector<float> ATA(d_out * d_out);
+
176  FINTEGER dii = d_in, doi = d_out;
+
177  float one = 1.0, zero = 0.0;
+
178 
+
179  sgemm_ ("Transposed", "Not", &doi, &doi, &dii,
+
180  &one, A.data (), &dii,
+
181  A.data(), &dii,
+
182  &zero, ATA.data(), &doi);
+
183 
+
184  is_orthonormal = true;
+
185  for (long i = 0; i < d_out; i++) {
+
186  for (long j = 0; j < d_out; j++) {
+
187  float v = ATA[i + j * d_out];
+
188  if (i == j) v-= 1;
+
189  if (fabs(v) > eps) {
+
190  is_orthonormal = false;
+
191  }
+
192  }
+
193  }
+
194  }
+
195 
+
196 }
+
197 
198 
-
199 PCAMatrix::PCAMatrix (int d_in, int d_out,
-
200  float eigen_power, bool random_rotation):
-
201  LinearTransform(d_in, d_out, true),
-
202  eigen_power(eigen_power), random_rotation(random_rotation)
-
203 {
-
204  is_trained = false;
-
205  max_points_per_d = 1000;
-
206  balanced_bins = 0;
+
199 void LinearTransform::reverse_transform (idx_t n, const float * xt,
+
200  float *x) const
+
201 {
+
202  if (is_orthonormal) {
+
203  transform_transpose (n, xt, x);
+
204  } else {
+
205  FAISS_THROW_MSG ("reverse transform not implemented for non-orthonormal matrices");
+
206  }
207 }
208 
209 
-
210 namespace {
-
211 
-
212 /// Compute the eigenvalue decomposition of symmetric matrix cov,
-
213 /// dimensions d_in-by-d_in. Output eigenvectors in cov.
-
214 
-
215 void eig(size_t d_in, double *cov, double *eigenvalues, int verbose)
+
210 
+
211 /*********************************************
+
212  * RandomRotationMatrix
+
213  *********************************************/
+
214 
+
215 void RandomRotationMatrix::init (int seed)
216 {
-
217  { // compute eigenvalues and vectors
-
218  FINTEGER info = 0, lwork = -1, di = d_in;
-
219  double workq;
-
220 
-
221  dsyev_ ("Vectors as well", "Upper",
-
222  &di, cov, &di, eigenvalues, &workq, &lwork, &info);
-
223  lwork = FINTEGER(workq);
-
224  double *work = new double[lwork];
-
225 
-
226  dsyev_ ("Vectors as well", "Upper",
-
227  &di, cov, &di, eigenvalues, work, &lwork, &info);
-
228 
-
229  delete [] work;
-
230 
-
231  if (info != 0) {
-
232  fprintf (stderr, "WARN ssyev info returns %d, "
-
233  "a very bad PCA matrix is learnt\n",
-
234  int(info));
-
235  // do not throw exception, as the matrix could still be useful
-
236  }
-
237 
-
238 
-
239  if(verbose && d_in <= 10) {
-
240  printf("info=%ld new eigvals=[", long(info));
-
241  for(int j = 0; j < d_in; j++) printf("%g ", eigenvalues[j]);
-
242  printf("]\n");
+
217 
+
218  if(d_out <= d_in) {
+
219  A.resize (d_out * d_in);
+
220  float *q = A.data();
+
221  float_randn(q, d_out * d_in, seed);
+
222  matrix_qr(d_in, d_out, q);
+
223  } else {
+
224  A.resize (d_out * d_out);
+
225  float *q = A.data();
+
226  float_randn(q, d_out * d_out, seed);
+
227  matrix_qr(d_out, d_out, q);
+
228  // remove columns
+
229  int i, j;
+
230  for (i = 0; i < d_out; i++) {
+
231  for(j = 0; j < d_in; j++) {
+
232  q[i * d_in + j] = q[i * d_out + j];
+
233  }
+
234  }
+
235  A.resize(d_in * d_out);
+
236  }
+
237  is_orthonormal = true;
+
238 }
+
239 
+
240 /*********************************************
+
241  * PCAMatrix
+
242  *********************************************/
243 
-
244  double *ci = cov;
-
245  printf("eigenvecs=\n");
-
246  for(int i = 0; i < d_in; i++) {
-
247  for(int j = 0; j < d_in; j++)
-
248  printf("%10.4g ", *ci++);
-
249  printf("\n");
-
250  }
-
251  }
-
252 
-
253  }
+
244 PCAMatrix::PCAMatrix (int d_in, int d_out,
+
245  float eigen_power, bool random_rotation):
+
246  LinearTransform(d_in, d_out, true),
+
247  eigen_power(eigen_power), random_rotation(random_rotation)
+
248 {
+
249  is_trained = false;
+
250  max_points_per_d = 1000;
+
251  balanced_bins = 0;
+
252 }
+
253 
254 
-
255  // revert order of eigenvectors & values
-
256 
-
257  for(int i = 0; i < d_in / 2; i++) {
-
258 
-
259  std::swap(eigenvalues[i], eigenvalues[d_in - 1 - i]);
-
260  double *v1 = cov + i * d_in;
-
261  double *v2 = cov + (d_in - 1 - i) * d_in;
-
262  for(int j = 0; j < d_in; j++)
-
263  std::swap(v1[j], v2[j]);
-
264  }
+
255 namespace {
+
256 
+
257 /// Compute the eigenvalue decomposition of symmetric matrix cov,
+
258 /// dimensions d_in-by-d_in. Output eigenvectors in cov.
+
259 
+
260 void eig(size_t d_in, double *cov, double *eigenvalues, int verbose)
+
261 {
+
262  { // compute eigenvalues and vectors
+
263  FINTEGER info = 0, lwork = -1, di = d_in;
+
264  double workq;
265 
-
266 }
-
267 
-
268 
-
269 }
+
266  dsyev_ ("Vectors as well", "Upper",
+
267  &di, cov, &di, eigenvalues, &workq, &lwork, &info);
+
268  lwork = FINTEGER(workq);
+
269  double *work = new double[lwork];
270 
-
271 void PCAMatrix::train (Index::idx_t n, const float *x)
-
272 {
-
273  const float * x_in = x;
-
274 
-
275  x = fvecs_maybe_subsample (d_in, (size_t*)&n,
-
276  max_points_per_d * d_in, x, verbose);
-
277 
-
278  ScopeDeleter<float> del_x (x != x_in ? x : nullptr);
-
279 
-
280  // compute mean
-
281  mean.clear(); mean.resize(d_in, 0.0);
-
282  if (have_bias) { // we may want to skip the bias
-
283  const float *xi = x;
-
284  for (int i = 0; i < n; i++) {
-
285  for(int j = 0; j < d_in; j++)
-
286  mean[j] += *xi++;
-
287  }
-
288  for(int j = 0; j < d_in; j++)
-
289  mean[j] /= n;
-
290  }
-
291  if(verbose) {
-
292  printf("mean=[");
-
293  for(int j = 0; j < d_in; j++) printf("%g ", mean[j]);
-
294  printf("]\n");
-
295  }
-
296 
-
297  if(n >= d_in) {
-
298  // compute covariance matrix, store it in PCA matrix
-
299  PCAMat.resize(d_in * d_in);
-
300  float * cov = PCAMat.data();
-
301  { // initialize with mean * mean^T term
-
302  float *ci = cov;
-
303  for(int i = 0; i < d_in; i++) {
-
304  for(int j = 0; j < d_in; j++)
-
305  *ci++ = - n * mean[i] * mean[j];
-
306  }
-
307  }
-
308  {
-
309  FINTEGER di = d_in, ni = n;
-
310  float one = 1.0;
-
311  ssyrk_ ("Up", "Non transposed",
-
312  &di, &ni, &one, (float*)x, &di, &one, cov, &di);
+
271  dsyev_ ("Vectors as well", "Upper",
+
272  &di, cov, &di, eigenvalues, work, &lwork, &info);
+
273 
+
274  delete [] work;
+
275 
+
276  if (info != 0) {
+
277  fprintf (stderr, "WARN ssyev info returns %d, "
+
278  "a very bad PCA matrix is learnt\n",
+
279  int(info));
+
280  // do not throw exception, as the matrix could still be useful
+
281  }
+
282 
+
283 
+
284  if(verbose && d_in <= 10) {
+
285  printf("info=%ld new eigvals=[", long(info));
+
286  for(int j = 0; j < d_in; j++) printf("%g ", eigenvalues[j]);
+
287  printf("]\n");
+
288 
+
289  double *ci = cov;
+
290  printf("eigenvecs=\n");
+
291  for(int i = 0; i < d_in; i++) {
+
292  for(int j = 0; j < d_in; j++)
+
293  printf("%10.4g ", *ci++);
+
294  printf("\n");
+
295  }
+
296  }
+
297 
+
298  }
+
299 
+
300  // revert order of eigenvectors & values
+
301 
+
302  for(int i = 0; i < d_in / 2; i++) {
+
303 
+
304  std::swap(eigenvalues[i], eigenvalues[d_in - 1 - i]);
+
305  double *v1 = cov + i * d_in;
+
306  double *v2 = cov + (d_in - 1 - i) * d_in;
+
307  for(int j = 0; j < d_in; j++)
+
308  std::swap(v1[j], v2[j]);
+
309  }
+
310 
+
311 }
+
312 
313 
-
314  }
-
315  if(verbose && d_in <= 10) {
-
316  float *ci = cov;
-
317  printf("cov=\n");
-
318  for(int i = 0; i < d_in; i++) {
-
319  for(int j = 0; j < d_in; j++)
-
320  printf("%10g ", *ci++);
-
321  printf("\n");
-
322  }
-
323  }
+
314 }
+
315 
+
316 void PCAMatrix::train (Index::idx_t n, const float *x)
+
317 {
+
318  const float * x_in = x;
+
319 
+
320  x = fvecs_maybe_subsample (d_in, (size_t*)&n,
+
321  max_points_per_d * d_in, x, verbose);
+
322 
+
323  ScopeDeleter<float> del_x (x != x_in ? x : nullptr);
324 
-
325  std::vector<double> covd (d_in * d_in);
-
326  for (size_t i = 0; i < d_in * d_in; i++) covd [i] = cov [i];
-
327 
-
328  std::vector<double> eigenvaluesd (d_in);
-
329 
-
330  eig (d_in, covd.data (), eigenvaluesd.data (), verbose);
-
331 
-
332  for (size_t i = 0; i < d_in * d_in; i++) PCAMat [i] = covd [i];
-
333  eigenvalues.resize (d_in);
-
334 
-
335  for (size_t i = 0; i < d_in; i++)
-
336  eigenvalues [i] = eigenvaluesd [i];
-
337 
-
338 
-
339  } else {
-
340 
-
341  std::vector<float> xc (n * d_in);
-
342 
-
343  for (size_t i = 0; i < n; i++)
-
344  for(size_t j = 0; j < d_in; j++)
-
345  xc [i * d_in + j] = x [i * d_in + j] - mean[j];
-
346 
-
347  // compute Gram matrix
-
348  std::vector<float> gram (n * n);
-
349  {
-
350  FINTEGER di = d_in, ni = n;
-
351  float one = 1.0, zero = 0.0;
-
352  ssyrk_ ("Up", "Transposed",
-
353  &ni, &di, &one, xc.data(), &di, &zero, gram.data(), &ni);
-
354  }
-
355 
-
356  if(verbose && d_in <= 10) {
-
357  float *ci = gram.data();
-
358  printf("gram=\n");
-
359  for(int i = 0; i < n; i++) {
-
360  for(int j = 0; j < n; j++)
-
361  printf("%10g ", *ci++);
-
362  printf("\n");
-
363  }
-
364  }
-
365 
-
366  std::vector<double> gramd (n * n);
-
367  for (size_t i = 0; i < n * n; i++)
-
368  gramd [i] = gram [i];
+
325  // compute mean
+
326  mean.clear(); mean.resize(d_in, 0.0);
+
327  if (have_bias) { // we may want to skip the bias
+
328  const float *xi = x;
+
329  for (int i = 0; i < n; i++) {
+
330  for(int j = 0; j < d_in; j++)
+
331  mean[j] += *xi++;
+
332  }
+
333  for(int j = 0; j < d_in; j++)
+
334  mean[j] /= n;
+
335  }
+
336  if(verbose) {
+
337  printf("mean=[");
+
338  for(int j = 0; j < d_in; j++) printf("%g ", mean[j]);
+
339  printf("]\n");
+
340  }
+
341 
+
342  if(n >= d_in) {
+
343  // compute covariance matrix, store it in PCA matrix
+
344  PCAMat.resize(d_in * d_in);
+
345  float * cov = PCAMat.data();
+
346  { // initialize with mean * mean^T term
+
347  float *ci = cov;
+
348  for(int i = 0; i < d_in; i++) {
+
349  for(int j = 0; j < d_in; j++)
+
350  *ci++ = - n * mean[i] * mean[j];
+
351  }
+
352  }
+
353  {
+
354  FINTEGER di = d_in, ni = n;
+
355  float one = 1.0;
+
356  ssyrk_ ("Up", "Non transposed",
+
357  &di, &ni, &one, (float*)x, &di, &one, cov, &di);
+
358 
+
359  }
+
360  if(verbose && d_in <= 10) {
+
361  float *ci = cov;
+
362  printf("cov=\n");
+
363  for(int i = 0; i < d_in; i++) {
+
364  for(int j = 0; j < d_in; j++)
+
365  printf("%10g ", *ci++);
+
366  printf("\n");
+
367  }
+
368  }
369 
-
370  std::vector<double> eigenvaluesd (n);
-
371 
-
372  // eig will fill in only the n first eigenvals
-
373 
-
374  eig (n, gramd.data (), eigenvaluesd.data (), verbose);
-
375 
-
376  PCAMat.resize(d_in * n);
-
377 
-
378  for (size_t i = 0; i < n * n; i++)
-
379  gram [i] = gramd [i];
-
380 
-
381  eigenvalues.resize (d_in);
-
382  // fill in only the n first ones
-
383  for (size_t i = 0; i < n; i++)
-
384  eigenvalues [i] = eigenvaluesd [i];
+
370  std::vector<double> covd (d_in * d_in);
+
371  for (size_t i = 0; i < d_in * d_in; i++) covd [i] = cov [i];
+
372 
+
373  std::vector<double> eigenvaluesd (d_in);
+
374 
+
375  eig (d_in, covd.data (), eigenvaluesd.data (), verbose);
+
376 
+
377  for (size_t i = 0; i < d_in * d_in; i++) PCAMat [i] = covd [i];
+
378  eigenvalues.resize (d_in);
+
379 
+
380  for (size_t i = 0; i < d_in; i++)
+
381  eigenvalues [i] = eigenvaluesd [i];
+
382 
+
383 
+
384  } else {
385 
-
386  { // compute PCAMat = x' * v
-
387  FINTEGER di = d_in, ni = n;
-
388  float one = 1.0;
-
389 
-
390  sgemm_ ("Non", "Non Trans",
-
391  &di, &ni, &ni,
-
392  &one, xc.data(), &di, gram.data(), &ni,
-
393  &one, PCAMat.data(), &di);
-
394  }
-
395 
-
396  if(verbose && d_in <= 10) {
-
397  float *ci = PCAMat.data();
-
398  printf("PCAMat=\n");
-
399  for(int i = 0; i < n; i++) {
-
400  for(int j = 0; j < d_in; j++)
-
401  printf("%10g ", *ci++);
-
402  printf("\n");
-
403  }
-
404  }
-
405  fvec_renorm_L2 (d_in, n, PCAMat.data());
-
406 
-
407  }
-
408 
-
409  prepare_Ab();
-
410  is_trained = true;
-
411 }
-
412 
-
413 void PCAMatrix::copy_from (const PCAMatrix & other)
-
414 {
-
415  FAISS_THROW_IF_NOT (other.is_trained);
-
416  mean = other.mean;
-
417  eigenvalues = other.eigenvalues;
-
418  PCAMat = other.PCAMat;
-
419  prepare_Ab ();
-
420  is_trained = true;
-
421 }
+
386  std::vector<float> xc (n * d_in);
+
387 
+
388  for (size_t i = 0; i < n; i++)
+
389  for(size_t j = 0; j < d_in; j++)
+
390  xc [i * d_in + j] = x [i * d_in + j] - mean[j];
+
391 
+
392  // compute Gram matrix
+
393  std::vector<float> gram (n * n);
+
394  {
+
395  FINTEGER di = d_in, ni = n;
+
396  float one = 1.0, zero = 0.0;
+
397  ssyrk_ ("Up", "Transposed",
+
398  &ni, &di, &one, xc.data(), &di, &zero, gram.data(), &ni);
+
399  }
+
400 
+
401  if(verbose && d_in <= 10) {
+
402  float *ci = gram.data();
+
403  printf("gram=\n");
+
404  for(int i = 0; i < n; i++) {
+
405  for(int j = 0; j < n; j++)
+
406  printf("%10g ", *ci++);
+
407  printf("\n");
+
408  }
+
409  }
+
410 
+
411  std::vector<double> gramd (n * n);
+
412  for (size_t i = 0; i < n * n; i++)
+
413  gramd [i] = gram [i];
+
414 
+
415  std::vector<double> eigenvaluesd (n);
+
416 
+
417  // eig will fill in only the n first eigenvals
+
418 
+
419  eig (n, gramd.data (), eigenvaluesd.data (), verbose);
+
420 
+
421  PCAMat.resize(d_in * n);
422 
-
423 void PCAMatrix::prepare_Ab ()
-
424 {
+
423  for (size_t i = 0; i < n * n; i++)
+
424  gram [i] = gramd [i];
425 
-
426  if (!random_rotation) {
-
427  FAISS_THROW_IF_NOT_MSG (
-
428  d_out * d_in <= PCAMat.size(),
-
429  "PCA matrix was trained on too few examples "
-
430  "to output this number of dimensions");
-
431  A = PCAMat;
-
432  A.resize(d_out * d_in); // strip off useless dimensions
-
433 
-
434  // first scale the components
-
435  if (eigen_power != 0) {
-
436  float *ai = A.data();
-
437  for (int i = 0; i < d_out; i++) {
-
438  float factor = pow(eigenvalues[i], eigen_power);
-
439  for(int j = 0; j < d_in; j++)
-
440  *ai++ *= factor;
-
441  }
-
442  }
-
443 
-
444  if (balanced_bins != 0) {
-
445  FAISS_THROW_IF_NOT (d_out % balanced_bins == 0);
-
446  int dsub = d_out / balanced_bins;
-
447  std::vector <float> Ain;
-
448  std::swap(A, Ain);
-
449  A.resize(d_out * d_in);
-
450 
-
451  std::vector <float> accu(balanced_bins);
-
452  std::vector <int> counter(balanced_bins);
+
426  eigenvalues.resize (d_in);
+
427  // fill in only the n first ones
+
428  for (size_t i = 0; i < n; i++)
+
429  eigenvalues [i] = eigenvaluesd [i];
+
430 
+
431  { // compute PCAMat = x' * v
+
432  FINTEGER di = d_in, ni = n;
+
433  float one = 1.0;
+
434 
+
435  sgemm_ ("Non", "Non Trans",
+
436  &di, &ni, &ni,
+
437  &one, xc.data(), &di, gram.data(), &ni,
+
438  &one, PCAMat.data(), &di);
+
439  }
+
440 
+
441  if(verbose && d_in <= 10) {
+
442  float *ci = PCAMat.data();
+
443  printf("PCAMat=\n");
+
444  for(int i = 0; i < n; i++) {
+
445  for(int j = 0; j < d_in; j++)
+
446  printf("%10g ", *ci++);
+
447  printf("\n");
+
448  }
+
449  }
+
450  fvec_renorm_L2 (d_in, n, PCAMat.data());
+
451 
+
452  }
453 
-
454  // greedy assignment
-
455  for (int i = 0; i < d_out; i++) {
-
456  // find best bin
-
457  int best_j = -1;
-
458  float min_w = 1e30;
-
459  for (int j = 0; j < balanced_bins; j++) {
-
460  if (counter[j] < dsub && accu[j] < min_w) {
-
461  min_w = accu[j];
-
462  best_j = j;
-
463  }
-
464  }
-
465  int row_dst = best_j * dsub + counter[best_j];
-
466  accu[best_j] += eigenvalues[i];
-
467  counter[best_j] ++;
-
468  memcpy (&A[row_dst * d_in], &Ain[i * d_in],
-
469  d_in * sizeof (A[0]));
-
470  }
-
471 
-
472  if (verbose) {
-
473  printf(" bin accu=[");
-
474  for (int i = 0; i < balanced_bins; i++)
-
475  printf("%g ", accu[i]);
-
476  printf("]\n");
-
477  }
-
478  }
-
479 
-
480 
-
481  } else {
-
482  FAISS_THROW_IF_NOT_MSG (balanced_bins == 0,
-
483  "both balancing bins and applying a random rotation "
-
484  "does not make sense");
-
485  RandomRotationMatrix rr(d_out, d_out);
-
486 
-
487  rr.init(5);
+
454  prepare_Ab();
+
455  is_trained = true;
+
456 }
+
457 
+
458 void PCAMatrix::copy_from (const PCAMatrix & other)
+
459 {
+
460  FAISS_THROW_IF_NOT (other.is_trained);
+
461  mean = other.mean;
+
462  eigenvalues = other.eigenvalues;
+
463  PCAMat = other.PCAMat;
+
464  prepare_Ab ();
+
465  is_trained = true;
+
466 }
+
467 
+
468 void PCAMatrix::prepare_Ab ()
+
469 {
+
470  FAISS_THROW_IF_NOT_FMT (
+
471  d_out * d_in <= PCAMat.size(),
+
472  "PCA matrix cannot output %d dimensions from %d ",
+
473  d_out, d_in);
+
474 
+
475  if (!random_rotation) {
+
476  A = PCAMat;
+
477  A.resize(d_out * d_in); // strip off useless dimensions
+
478 
+
479  // first scale the components
+
480  if (eigen_power != 0) {
+
481  float *ai = A.data();
+
482  for (int i = 0; i < d_out; i++) {
+
483  float factor = pow(eigenvalues[i], eigen_power);
+
484  for(int j = 0; j < d_in; j++)
+
485  *ai++ *= factor;
+
486  }
+
487  }
488 
-
489  // apply scaling on the rotation matrix (right multiplication)
-
490  if (eigen_power != 0) {
-
491  for (int i = 0; i < d_out; i++) {
-
492  float factor = pow(eigenvalues[i], eigen_power);
-
493  for(int j = 0; j < d_out; j++)
-
494  rr.A[j * d_out + i] *= factor;
-
495  }
-
496  }
-
497 
-
498  A.resize(d_in * d_out);
-
499  {
-
500  FINTEGER dii = d_in, doo = d_out;
-
501  float one = 1.0, zero = 0.0;
-
502 
-
503  sgemm_ ("Not", "Not", &dii, &doo, &doo,
-
504  &one, PCAMat.data(), &dii, rr.A.data(), &doo, &zero,
-
505  A.data(), &dii);
-
506 
-
507  }
-
508 
-
509  }
-
510 
-
511  b.clear(); b.resize(d_out);
-
512 
-
513  for (int i = 0; i < d_out; i++) {
-
514  float accu = 0;
-
515  for (int j = 0; j < d_in; j++)
-
516  accu -= mean[j] * A[j + i * d_in];
-
517  b[i] = accu;
-
518  }
-
519 
-
520 }
-
521 
-
522 void PCAMatrix::reverse_transform (idx_t n, const float * xt,
-
523  float *x) const
-
524 {
-
525  FAISS_THROW_IF_NOT_MSG (eigen_power == 0,
-
526  "reverse only implemented for orthogonal transforms");
-
527  transform_transpose (n, xt, x);
-
528 }
-
529 
-
530 /*********************************************
-
531  * OPQMatrix
-
532  *********************************************/
+
489  if (balanced_bins != 0) {
+
490  FAISS_THROW_IF_NOT (d_out % balanced_bins == 0);
+
491  int dsub = d_out / balanced_bins;
+
492  std::vector <float> Ain;
+
493  std::swap(A, Ain);
+
494  A.resize(d_out * d_in);
+
495 
+
496  std::vector <float> accu(balanced_bins);
+
497  std::vector <int> counter(balanced_bins);
+
498 
+
499  // greedy assignment
+
500  for (int i = 0; i < d_out; i++) {
+
501  // find best bin
+
502  int best_j = -1;
+
503  float min_w = 1e30;
+
504  for (int j = 0; j < balanced_bins; j++) {
+
505  if (counter[j] < dsub && accu[j] < min_w) {
+
506  min_w = accu[j];
+
507  best_j = j;
+
508  }
+
509  }
+
510  int row_dst = best_j * dsub + counter[best_j];
+
511  accu[best_j] += eigenvalues[i];
+
512  counter[best_j] ++;
+
513  memcpy (&A[row_dst * d_in], &Ain[i * d_in],
+
514  d_in * sizeof (A[0]));
+
515  }
+
516 
+
517  if (verbose) {
+
518  printf(" bin accu=[");
+
519  for (int i = 0; i < balanced_bins; i++)
+
520  printf("%g ", accu[i]);
+
521  printf("]\n");
+
522  }
+
523  }
+
524 
+
525 
+
526  } else {
+
527  FAISS_THROW_IF_NOT_MSG (balanced_bins == 0,
+
528  "both balancing bins and applying a random rotation "
+
529  "does not make sense");
+
530  RandomRotationMatrix rr(d_out, d_out);
+
531 
+
532  rr.init(5);
533 
-
534 
-
535 OPQMatrix::OPQMatrix (int d, int M, int d2):
-
536  LinearTransform (d, d2 == -1 ? d : d2, false), M(M),
-
537  niter (50),
-
538  niter_pq (4), niter_pq_0 (40),
-
539  verbose(false)
-
540 {
-
541  is_trained = false;
-
542  // OPQ is quite expensive to train, so set this right.
-
543  max_train_points = 256 * 256;
-
544 }
-
545 
-
546 
+
534  // apply scaling on the rotation matrix (right multiplication)
+
535  if (eigen_power != 0) {
+
536  for (int i = 0; i < d_out; i++) {
+
537  float factor = pow(eigenvalues[i], eigen_power);
+
538  for(int j = 0; j < d_out; j++)
+
539  rr.A[j * d_out + i] *= factor;
+
540  }
+
541  }
+
542 
+
543  A.resize(d_in * d_out);
+
544  {
+
545  FINTEGER dii = d_in, doo = d_out;
+
546  float one = 1.0, zero = 0.0;
547 
-
548 void OPQMatrix::train (Index::idx_t n, const float *x)
-
549 {
-
550 
-
551  const float * x_in = x;
-
552 
-
553  x = fvecs_maybe_subsample (d_in, (size_t*)&n,
-
554  max_train_points, x, verbose);
+
548  sgemm_ ("Not", "Not", &dii, &doo, &doo,
+
549  &one, PCAMat.data(), &dii, rr.A.data(), &doo, &zero,
+
550  A.data(), &dii);
+
551 
+
552  }
+
553 
+
554  }
555 
-
556  ScopeDeleter<float> del_x (x != x_in ? x : nullptr);
+
556  b.clear(); b.resize(d_out);
557 
-
558  // To support d_out > d_in, we pad input vectors with 0s to d_out
-
559  size_t d = d_out <= d_in ? d_in : d_out;
-
560  size_t d2 = d_out;
-
561 
-
562 #if 0
-
563  // what this test shows: the only way of getting bit-exact
-
564  // reproducible results with sgeqrf and sgesvd seems to be forcing
-
565  // single-threading.
-
566  { // test repro
-
567  std::vector<float> r (d * d);
-
568  float * rotation = r.data();
-
569  float_randn (rotation, d * d, 1234);
-
570  printf("CS0: %016lx\n",
-
571  ivec_checksum (128*128, (int*)rotation));
-
572  matrix_qr (d, d, rotation);
-
573  printf("CS1: %016lx\n",
-
574  ivec_checksum (128*128, (int*)rotation));
-
575  return;
-
576  }
-
577 #endif
-
578 
-
579  if (verbose) {
-
580  printf ("OPQMatrix::train: training an OPQ rotation matrix "
-
581  "for M=%d from %ld vectors in %dD -> %dD\n",
-
582  M, n, d_in, d_out);
-
583  }
+
558  for (int i = 0; i < d_out; i++) {
+
559  float accu = 0;
+
560  for (int j = 0; j < d_in; j++)
+
561  accu -= mean[j] * A[j + i * d_in];
+
562  b[i] = accu;
+
563  }
+
564 
+
565  is_orthonormal = eigen_power == 0;
+
566 
+
567 }
+
568 
+
569 /*********************************************
+
570  * OPQMatrix
+
571  *********************************************/
+
572 
+
573 
+
574 OPQMatrix::OPQMatrix (int d, int M, int d2):
+
575  LinearTransform (d, d2 == -1 ? d : d2, false), M(M),
+
576  niter (50),
+
577  niter_pq (4), niter_pq_0 (40),
+
578  verbose(false)
+
579 {
+
580  is_trained = false;
+
581  // OPQ is quite expensive to train, so set this right.
+
582  max_train_points = 256 * 256;
+
583 }
584 
-
585  std::vector<float> xtrain (n * d);
-
586  // center x
-
587  {
-
588  std::vector<float> sum (d);
-
589  const float *xi = x;
-
590  for (size_t i = 0; i < n; i++) {
-
591  for (int j = 0; j < d_in; j++)
-
592  sum [j] += *xi++;
-
593  }
-
594  for (int i = 0; i < d; i++) sum[i] /= n;
-
595  float *yi = xtrain.data();
-
596  xi = x;
-
597  for (size_t i = 0; i < n; i++) {
-
598  for (int j = 0; j < d_in; j++)
-
599  *yi++ = *xi++ - sum[j];
-
600  yi += d - d_in;
-
601  }
-
602  }
-
603  float *rotation;
-
604 
-
605  if (A.size () == 0) {
-
606  A.resize (d * d);
-
607  rotation = A.data();
-
608  if (verbose)
-
609  printf(" OPQMatrix::train: making random %ld*%ld rotation\n",
-
610  d, d);
-
611  float_randn (rotation, d * d, 1234);
-
612  matrix_qr (d, d, rotation);
-
613  // we use only the d * d2 upper part of the matrix
-
614  A.resize (d * d2);
-
615  } else {
-
616  FAISS_THROW_IF_NOT (A.size() == d * d2);
-
617  rotation = A.data();
-
618  }
-
619 
-
620 
-
621  std::vector<float>
-
622  xproj (d2 * n), pq_recons (d2 * n), xxr (d * n),
-
623  tmp(d * d * 4);
-
624 
-
625  std::vector<uint8_t> codes (M * n);
-
626  ProductQuantizer pq_regular (d2, M, 8);
-
627  double t0 = getmillisecs();
-
628  for (int iter = 0; iter < niter; iter++) {
-
629 
-
630  { // torch.mm(xtrain, rotation:t())
-
631  FINTEGER di = d, d2i = d2, ni = n;
-
632  float zero = 0, one = 1;
-
633  sgemm_ ("Transposed", "Not transposed",
-
634  &d2i, &ni, &di,
-
635  &one, rotation, &di,
-
636  xtrain.data(), &di,
-
637  &zero, xproj.data(), &d2i);
-
638  }
-
639 
-
640  pq_regular.cp.max_points_per_centroid = 1000;
-
641  pq_regular.cp.niter = iter == 0 ? niter_pq_0 : niter_pq;
-
642  pq_regular.cp.verbose = verbose;
-
643  pq_regular.train (n, xproj.data());
-
644 
-
645  pq_regular.compute_codes (xproj.data(), codes.data(), n);
-
646  pq_regular.decode (codes.data(), pq_recons.data(), n);
-
647 
-
648  float pq_err = fvec_L2sqr (pq_recons.data(), xproj.data(), n * d2) / n;
-
649 
-
650  if (verbose)
-
651  printf (" Iteration %d (%d PQ iterations):"
-
652  "%.3f s, obj=%g\n", iter, pq_regular.cp.niter,
-
653  (getmillisecs () - t0) / 1000.0, pq_err);
-
654 
-
655  {
-
656  float *u = tmp.data(), *vt = &tmp [d * d];
-
657  float *sing_val = &tmp [2 * d * d];
-
658  FINTEGER di = d, d2i = d2, ni = n;
-
659  float one = 1, zero = 0;
-
660 
-
661  // torch.mm(xtrain:t(), pq_recons)
-
662  sgemm_ ("Not", "Transposed",
-
663  &d2i, &di, &ni,
-
664  &one, pq_recons.data(), &d2i,
-
665  xtrain.data(), &di,
-
666  &zero, xxr.data(), &d2i);
-
667 
+
585 
+
586 
+
587 void OPQMatrix::train (Index::idx_t n, const float *x)
+
588 {
+
589 
+
590  const float * x_in = x;
+
591 
+
592  x = fvecs_maybe_subsample (d_in, (size_t*)&n,
+
593  max_train_points, x, verbose);
+
594 
+
595  ScopeDeleter<float> del_x (x != x_in ? x : nullptr);
+
596 
+
597  // To support d_out > d_in, we pad input vectors with 0s to d_out
+
598  size_t d = d_out <= d_in ? d_in : d_out;
+
599  size_t d2 = d_out;
+
600 
+
601 #if 0
+
602  // what this test shows: the only way of getting bit-exact
+
603  // reproducible results with sgeqrf and sgesvd seems to be forcing
+
604  // single-threading.
+
605  { // test repro
+
606  std::vector<float> r (d * d);
+
607  float * rotation = r.data();
+
608  float_randn (rotation, d * d, 1234);
+
609  printf("CS0: %016lx\n",
+
610  ivec_checksum (128*128, (int*)rotation));
+
611  matrix_qr (d, d, rotation);
+
612  printf("CS1: %016lx\n",
+
613  ivec_checksum (128*128, (int*)rotation));
+
614  return;
+
615  }
+
616 #endif
+
617 
+
618  if (verbose) {
+
619  printf ("OPQMatrix::train: training an OPQ rotation matrix "
+
620  "for M=%d from %ld vectors in %dD -> %dD\n",
+
621  M, n, d_in, d_out);
+
622  }
+
623 
+
624  std::vector<float> xtrain (n * d);
+
625  // center x
+
626  {
+
627  std::vector<float> sum (d);
+
628  const float *xi = x;
+
629  for (size_t i = 0; i < n; i++) {
+
630  for (int j = 0; j < d_in; j++)
+
631  sum [j] += *xi++;
+
632  }
+
633  for (int i = 0; i < d; i++) sum[i] /= n;
+
634  float *yi = xtrain.data();
+
635  xi = x;
+
636  for (size_t i = 0; i < n; i++) {
+
637  for (int j = 0; j < d_in; j++)
+
638  *yi++ = *xi++ - sum[j];
+
639  yi += d - d_in;
+
640  }
+
641  }
+
642  float *rotation;
+
643 
+
644  if (A.size () == 0) {
+
645  A.resize (d * d);
+
646  rotation = A.data();
+
647  if (verbose)
+
648  printf(" OPQMatrix::train: making random %ld*%ld rotation\n",
+
649  d, d);
+
650  float_randn (rotation, d * d, 1234);
+
651  matrix_qr (d, d, rotation);
+
652  // we use only the d * d2 upper part of the matrix
+
653  A.resize (d * d2);
+
654  } else {
+
655  FAISS_THROW_IF_NOT (A.size() == d * d2);
+
656  rotation = A.data();
+
657  }
+
658 
+
659 
+
660  std::vector<float>
+
661  xproj (d2 * n), pq_recons (d2 * n), xxr (d * n),
+
662  tmp(d * d * 4);
+
663 
+
664  std::vector<uint8_t> codes (M * n);
+
665  ProductQuantizer pq_regular (d2, M, 8);
+
666  double t0 = getmillisecs();
+
667  for (int iter = 0; iter < niter; iter++) {
668 
-
669  FINTEGER lwork = -1, info = -1;
-
670  float worksz;
-
671  // workspace query
-
672  sgesvd_ ("All", "All",
-
673  &d2i, &di, xxr.data(), &d2i,
-
674  sing_val,
-
675  vt, &d2i, u, &di,
-
676  &worksz, &lwork, &info);
-
677 
-
678  lwork = int(worksz);
-
679  std::vector<float> work (lwork);
-
680  // u and vt swapped
-
681  sgesvd_ ("All", "All",
-
682  &d2i, &di, xxr.data(), &d2i,
-
683  sing_val,
-
684  vt, &d2i, u, &di,
-
685  work.data(), &lwork, &info);
+
669  { // torch.mm(xtrain, rotation:t())
+
670  FINTEGER di = d, d2i = d2, ni = n;
+
671  float zero = 0, one = 1;
+
672  sgemm_ ("Transposed", "Not transposed",
+
673  &d2i, &ni, &di,
+
674  &one, rotation, &di,
+
675  xtrain.data(), &di,
+
676  &zero, xproj.data(), &d2i);
+
677  }
+
678 
+
679  pq_regular.cp.max_points_per_centroid = 1000;
+
680  pq_regular.cp.niter = iter == 0 ? niter_pq_0 : niter_pq;
+
681  pq_regular.cp.verbose = verbose;
+
682  pq_regular.train (n, xproj.data());
+
683 
+
684  pq_regular.compute_codes (xproj.data(), codes.data(), n);
+
685  pq_regular.decode (codes.data(), pq_recons.data(), n);
686 
-
687  sgemm_ ("Transposed", "Transposed",
-
688  &di, &d2i, &d2i,
-
689  &one, u, &di, vt, &d2i,
-
690  &zero, rotation, &di);
-
691 
-
692  }
-
693  pq_regular.train_type = ProductQuantizer::Train_hot_start;
-
694  }
-
695 
-
696  // revert A matrix
-
697  if (d > d_in) {
-
698  for (long i = 0; i < d_out; i++)
-
699  memmove (&A[i * d_in], &A[i * d], sizeof(A[0]) * d_in);
-
700  A.resize (d_in * d_out);
-
701  }
-
702 
-
703  is_trained = true;
-
704 }
-
705 
+
687  float pq_err = fvec_L2sqr (pq_recons.data(), xproj.data(), n * d2) / n;
+
688 
+
689  if (verbose)
+
690  printf (" Iteration %d (%d PQ iterations):"
+
691  "%.3f s, obj=%g\n", iter, pq_regular.cp.niter,
+
692  (getmillisecs () - t0) / 1000.0, pq_err);
+
693 
+
694  {
+
695  float *u = tmp.data(), *vt = &tmp [d * d];
+
696  float *sing_val = &tmp [2 * d * d];
+
697  FINTEGER di = d, d2i = d2, ni = n;
+
698  float one = 1, zero = 0;
+
699 
+
700  // torch.mm(xtrain:t(), pq_recons)
+
701  sgemm_ ("Not", "Transposed",
+
702  &d2i, &di, &ni,
+
703  &one, pq_recons.data(), &d2i,
+
704  xtrain.data(), &di,
+
705  &zero, xxr.data(), &d2i);
706 
707 
-
708 
-
709 void OPQMatrix::reverse_transform (idx_t n, const float * xt,
-
710  float *x) const
-
711 {
-
712  transform_transpose (n, xt, x);
-
713 }
-
714 
-
715 
-
716 /*********************************************
-
717  * NormalizationTransform
-
718  *********************************************/
-
719 
-
720 NormalizationTransform::NormalizationTransform (int d, float norm):
-
721  VectorTransform (d, d), norm (norm)
-
722 {
-
723 }
-
724 
-
725 NormalizationTransform::NormalizationTransform ():
-
726  VectorTransform (-1, -1), norm (-1)
-
727 {
-
728 }
-
729 
-
730 void NormalizationTransform::apply_noalloc
-
731  (idx_t n, const float* x, float* xt) const
-
732 {
-
733  if (norm == 2.0) {
-
734  memcpy (xt, x, sizeof (x[0]) * n * d_in);
-
735  fvec_renorm_L2 (d_in, n, xt);
-
736  } else {
-
737  FAISS_THROW_MSG ("not implemented");
-
738  }
-
739 }
-
740 
-
741 /*********************************************
-
742  * IndexPreTransform
-
743  *********************************************/
-
744 
-
745 IndexPreTransform::IndexPreTransform ():
-
746  index(nullptr), own_fields (false)
-
747 {
-
748 }
-
749 
+
708  FINTEGER lwork = -1, info = -1;
+
709  float worksz;
+
710  // workspace query
+
711  sgesvd_ ("All", "All",
+
712  &d2i, &di, xxr.data(), &d2i,
+
713  sing_val,
+
714  vt, &d2i, u, &di,
+
715  &worksz, &lwork, &info);
+
716 
+
717  lwork = int(worksz);
+
718  std::vector<float> work (lwork);
+
719  // u and vt swapped
+
720  sgesvd_ ("All", "All",
+
721  &d2i, &di, xxr.data(), &d2i,
+
722  sing_val,
+
723  vt, &d2i, u, &di,
+
724  work.data(), &lwork, &info);
+
725 
+
726  sgemm_ ("Transposed", "Transposed",
+
727  &di, &d2i, &d2i,
+
728  &one, u, &di, vt, &d2i,
+
729  &zero, rotation, &di);
+
730 
+
731  }
+
732  pq_regular.train_type = ProductQuantizer::Train_hot_start;
+
733  }
+
734 
+
735  // revert A matrix
+
736  if (d > d_in) {
+
737  for (long i = 0; i < d_out; i++)
+
738  memmove (&A[i * d_in], &A[i * d], sizeof(A[0]) * d_in);
+
739  A.resize (d_in * d_out);
+
740  }
+
741 
+
742  is_trained = true;
+
743  is_orthonormal = true;
+
744 }
+
745 
+
746 
+
747 /*********************************************
+
748  * NormalizationTransform
+
749  *********************************************/
750 
-
751 IndexPreTransform::IndexPreTransform (
-
752  Index * index):
-
753  Index (index->d, index->metric_type),
-
754  index (index), own_fields (false)
-
755 {
-
756  is_trained = index->is_trained;
-
757 }
-
758 
-
759 
-
760 IndexPreTransform::IndexPreTransform (
-
761  VectorTransform * ltrans,
-
762  Index * index):
-
763  Index (index->d, index->metric_type),
-
764  index (index), own_fields (false)
-
765 {
-
766  is_trained = index->is_trained;
-
767  prepend_transform (ltrans);
-
768 }
-
769 
-
770 void IndexPreTransform::prepend_transform (VectorTransform *ltrans)
-
771 {
-
772  FAISS_THROW_IF_NOT (ltrans->d_out == d);
-
773  is_trained = is_trained && ltrans->is_trained;
-
774  chain.insert (chain.begin(), ltrans);
-
775  d = ltrans->d_in;
+
751 NormalizationTransform::NormalizationTransform (int d, float norm):
+
752  VectorTransform (d, d), norm (norm)
+
753 {
+
754 }
+
755 
+
756 NormalizationTransform::NormalizationTransform ():
+
757  VectorTransform (-1, -1), norm (-1)
+
758 {
+
759 }
+
760 
+
761 void NormalizationTransform::apply_noalloc
+
762  (idx_t n, const float* x, float* xt) const
+
763 {
+
764  if (norm == 2.0) {
+
765  memcpy (xt, x, sizeof (x[0]) * n * d_in);
+
766  fvec_renorm_L2 (d_in, n, xt);
+
767  } else {
+
768  FAISS_THROW_MSG ("not implemented");
+
769  }
+
770 }
+
771 
+
772 void NormalizationTransform::reverse_transform (idx_t n, const float* xt,
+
773  float* x) const
+
774 {
+
775  memcpy (x, xt, sizeof (xt[0]) * n * d_in);
776 }
777 
-
778 
-
779 IndexPreTransform::~IndexPreTransform ()
-
780 {
-
781  if (own_fields) {
-
782  for (int i = 0; i < chain.size(); i++)
-
783  delete chain[i];
-
784  delete index;
-
785  }
-
786 }
+
778 /*********************************************
+
779  * IndexPreTransform
+
780  *********************************************/
+
781 
+
782 IndexPreTransform::IndexPreTransform ():
+
783  index(nullptr), own_fields (false)
+
784 {
+
785 }
+
786 
787 
-
788 
-
789 
-
790 
-
791 void IndexPreTransform::train (idx_t n, const float *x)
+
788 IndexPreTransform::IndexPreTransform (
+
789  Index * index):
+
790  Index (index->d, index->metric_type),
+
791  index (index), own_fields (false)
792 {
-
793  int last_untrained = 0;
-
794  if (!index->is_trained) {
-
795  last_untrained = chain.size();
-
796  } else {
-
797  for (int i = chain.size() - 1; i >= 0; i--) {
-
798  if (!chain[i]->is_trained) {
-
799  last_untrained = i;
-
800  break;
-
801  }
-
802  }
-
803  }
-
804  const float *prev_x = x;
-
805  ScopeDeleter<float> del;
+
793  is_trained = index->is_trained;
+
794 }
+
795 
+
796 
+
797 IndexPreTransform::IndexPreTransform (
+
798  VectorTransform * ltrans,
+
799  Index * index):
+
800  Index (index->d, index->metric_type),
+
801  index (index), own_fields (false)
+
802 {
+
803  is_trained = index->is_trained;
+
804  prepend_transform (ltrans);
+
805 }
806 
-
807  if (verbose) {
-
808  printf("IndexPreTransform::train: training chain 0 to %d\n",
-
809  last_untrained);
-
810  }
-
811 
-
812  for (int i = 0; i <= last_untrained; i++) {
-
813  if (i < chain.size()) {
-
814  VectorTransform *ltrans = chain [i];
-
815  if (!ltrans->is_trained) {
-
816  if (verbose) {
-
817  printf(" Training chain component %d/%zd\n",
-
818  i, chain.size());
-
819  if (OPQMatrix *opqm = dynamic_cast<OPQMatrix*>(ltrans)) {
-
820  opqm->verbose = true;
-
821  }
-
822  }
-
823  ltrans->train (n, prev_x);
-
824  }
-
825  } else {
-
826  if (verbose) {
-
827  printf(" Training sub-index\n");
-
828  }
-
829  index->train (n, prev_x);
-
830  }
-
831  if (i == last_untrained) break;
-
832  if (verbose) {
-
833  printf(" Applying transform %d/%zd\n",
-
834  i, chain.size());
-
835  }
-
836 
-
837  float * xt = chain[i]->apply (n, prev_x);
-
838  if (prev_x != x) delete [] prev_x;
-
839  prev_x = xt;
-
840  del.set(xt);
-
841  }
-
842 
-
843  is_trained = true;
-
844 }
-
845 
-
846 
-
847 const float *IndexPreTransform::apply_chain (idx_t n, const float *x) const
-
848 {
-
849  const float *prev_x = x;
-
850  ScopeDeleter<float> del;
-
851 
-
852  for (int i = 0; i < chain.size(); i++) {
-
853  float * xt = chain[i]->apply (n, prev_x);
-
854  ScopeDeleter<float> del2 (xt);
-
855  del2.swap (del);
-
856  prev_x = xt;
-
857  }
-
858  del.release ();
-
859  return prev_x;
-
860 }
-
861 
-
862 void IndexPreTransform::add (idx_t n, const float *x)
-
863 {
-
864  FAISS_THROW_IF_NOT (is_trained);
-
865  const float *xt = apply_chain (n, x);
-
866  ScopeDeleter<float> del(xt == x ? nullptr : xt);
-
867  index->add (n, xt);
-
868  ntotal = index->ntotal;
-
869 }
-
870 
-
871 void IndexPreTransform::add_with_ids (idx_t n, const float * x,
-
872  const long *xids)
-
873 {
-
874  FAISS_THROW_IF_NOT (is_trained);
-
875  const float *xt = apply_chain (n, x);
-
876  ScopeDeleter<float> del(xt == x ? nullptr : xt);
-
877  index->add_with_ids (n, xt, xids);
-
878  ntotal = index->ntotal;
-
879 }
-
880 
+
807 void IndexPreTransform::prepend_transform (VectorTransform *ltrans)
+
808 {
+
809  FAISS_THROW_IF_NOT (ltrans->d_out == d);
+
810  is_trained = is_trained && ltrans->is_trained;
+
811  chain.insert (chain.begin(), ltrans);
+
812  d = ltrans->d_in;
+
813 }
+
814 
+
815 
+
816 IndexPreTransform::~IndexPreTransform ()
+
817 {
+
818  if (own_fields) {
+
819  for (int i = 0; i < chain.size(); i++)
+
820  delete chain[i];
+
821  delete index;
+
822  }
+
823 }
+
824 
+
825 
+
826 
+
827 
+
828 void IndexPreTransform::train (idx_t n, const float *x)
+
829 {
+
830  int last_untrained = 0;
+
831  if (!index->is_trained) {
+
832  last_untrained = chain.size();
+
833  } else {
+
834  for (int i = chain.size() - 1; i >= 0; i--) {
+
835  if (!chain[i]->is_trained) {
+
836  last_untrained = i;
+
837  break;
+
838  }
+
839  }
+
840  }
+
841  const float *prev_x = x;
+
842  ScopeDeleter<float> del;
+
843 
+
844  if (verbose) {
+
845  printf("IndexPreTransform::train: training chain 0 to %d\n",
+
846  last_untrained);
+
847  }
+
848 
+
849  for (int i = 0; i <= last_untrained; i++) {
+
850 
+
851  if (i < chain.size()) {
+
852  VectorTransform *ltrans = chain [i];
+
853  if (!ltrans->is_trained) {
+
854  if (verbose) {
+
855  printf(" Training chain component %d/%zd\n",
+
856  i, chain.size());
+
857  if (OPQMatrix *opqm = dynamic_cast<OPQMatrix*>(ltrans)) {
+
858  opqm->verbose = true;
+
859  }
+
860  }
+
861  ltrans->train (n, prev_x);
+
862  }
+
863  } else {
+
864  if (verbose) {
+
865  printf(" Training sub-index\n");
+
866  }
+
867  index->train (n, prev_x);
+
868  }
+
869  if (i == last_untrained) break;
+
870  if (verbose) {
+
871  printf(" Applying transform %d/%zd\n",
+
872  i, chain.size());
+
873  }
+
874 
+
875  float * xt = chain[i]->apply (n, prev_x);
+
876 
+
877  if (prev_x != x) delete [] prev_x;
+
878  prev_x = xt;
+
879  del.set(xt);
+
880  }
881 
-
882 
-
883 
-
884 void IndexPreTransform::search (idx_t n, const float *x, idx_t k,
-
885  float *distances, idx_t *labels) const
-
886 {
-
887  FAISS_THROW_IF_NOT (is_trained);
-
888  const float *xt = apply_chain (n, x);
-
889  ScopeDeleter<float> del(xt == x ? nullptr : xt);
-
890  index->search (n, xt, k, distances, labels);
-
891 }
-
892 
-
893 
-
894 void IndexPreTransform::reset () {
-
895  index->reset();
-
896  ntotal = 0;
-
897 }
-
898 
-
899 long IndexPreTransform::remove_ids (const IDSelector & sel) {
-
900  long nremove = index->remove_ids (sel);
-
901  ntotal = index->ntotal;
-
902  return nremove;
-
903 }
-
904 
+
882  is_trained = true;
+
883 }
+
884 
+
885 
+
886 const float *IndexPreTransform::apply_chain (idx_t n, const float *x) const
+
887 {
+
888  const float *prev_x = x;
+
889  ScopeDeleter<float> del;
+
890 
+
891  for (int i = 0; i < chain.size(); i++) {
+
892  float * xt = chain[i]->apply (n, prev_x);
+
893  ScopeDeleter<float> del2 (xt);
+
894  del2.swap (del);
+
895  prev_x = xt;
+
896  }
+
897  del.release ();
+
898  return prev_x;
+
899 }
+
900 
+
901 void IndexPreTransform::reverse_chain (idx_t n, const float* xt, float* x) const
+
902 {
+
903  const float* next_x = xt;
+
904  ScopeDeleter<float> del;
905 
-
906 void IndexPreTransform::reconstruct_n (idx_t i0, idx_t ni, float *recons) const
-
907 {
-
908  float *x = chain.empty() ? recons : new float [ni * index->d];
-
909  ScopeDeleter<float> del (recons == x ? nullptr : x);
-
910  // initial reconstruction
-
911  index->reconstruct_n (i0, ni, x);
-
912 
-
913  // revert transformations from last to first
-
914  for (int i = chain.size() - 1; i >= 0; i--) {
-
915  float *x_pre = i == 0 ? recons : new float [chain[i]->d_in * ni];
-
916  ScopeDeleter<float> del2 (x_pre == recons ? nullptr : x_pre);
-
917  chain [i]->reverse_transform (ni, x, x_pre);
-
918  del2.swap (del); // delete [] x;
-
919  x = x_pre;
-
920  }
-
921 }
-
922 
+
906  for (int i = chain.size() - 1; i >= 0; i--) {
+
907  float* prev_x = (i == 0) ? x : new float [n * chain[i]->d_in];
+
908  ScopeDeleter<float> del2 ((prev_x == x) ? nullptr : prev_x);
+
909  chain [i]->reverse_transform (n, next_x, prev_x);
+
910  del2.swap (del);
+
911  next_x = prev_x;
+
912  }
+
913 }
+
914 
+
915 void IndexPreTransform::add (idx_t n, const float *x)
+
916 {
+
917  FAISS_THROW_IF_NOT (is_trained);
+
918  const float *xt = apply_chain (n, x);
+
919  ScopeDeleter<float> del(xt == x ? nullptr : xt);
+
920  index->add (n, xt);
+
921  ntotal = index->ntotal;
+
922 }
923 
-
924 
-
925 /*********************************************
-
926  * RemapDimensionsTransform
-
927  *********************************************/
-
928 
-
929 
-
930 RemapDimensionsTransform::RemapDimensionsTransform (
-
931  int d_in, int d_out, const int *map_in):
-
932  VectorTransform (d_in, d_out)
-
933 {
-
934  map.resize (d_out);
-
935  for (int i = 0; i < d_out; i++) {
-
936  map[i] = map_in[i];
-
937  FAISS_THROW_IF_NOT (map[i] == -1 || (map[i] >= 0 && map[i] < d_in));
-
938  }
-
939 }
-
940 
-
941 RemapDimensionsTransform::RemapDimensionsTransform (
-
942  int d_in, int d_out, bool uniform): VectorTransform (d_in, d_out)
-
943 {
-
944  map.resize (d_out, -1);
+
924 void IndexPreTransform::add_with_ids (idx_t n, const float * x,
+
925  const long *xids)
+
926 {
+
927  FAISS_THROW_IF_NOT (is_trained);
+
928  const float *xt = apply_chain (n, x);
+
929  ScopeDeleter<float> del(xt == x ? nullptr : xt);
+
930  index->add_with_ids (n, xt, xids);
+
931  ntotal = index->ntotal;
+
932 }
+
933 
+
934 
+
935 
+
936 
+
937 void IndexPreTransform::search (idx_t n, const float *x, idx_t k,
+
938  float *distances, idx_t *labels) const
+
939 {
+
940  FAISS_THROW_IF_NOT (is_trained);
+
941  const float *xt = apply_chain (n, x);
+
942  ScopeDeleter<float> del(xt == x ? nullptr : xt);
+
943  index->search (n, xt, k, distances, labels);
+
944 }
945 
-
946  if (uniform) {
-
947  if (d_in < d_out) {
-
948  for (int i = 0; i < d_in; i++) {
-
949  map [i * d_out / d_in] = i;
-
950  }
-
951  } else {
-
952  for (int i = 0; i < d_out; i++) {
-
953  map [i] = i * d_in / d_out;
-
954  }
-
955  }
-
956  } else {
-
957  for (int i = 0; i < d_in && i < d_out; i++)
-
958  map [i] = i;
-
959  }
-
960 }
-
961 
-
962 
-
963 void RemapDimensionsTransform::apply_noalloc (idx_t n, const float * x,
-
964  float *xt) const
-
965 {
-
966  for (idx_t i = 0; i < n; i++) {
-
967  for (int j = 0; j < d_out; j++) {
-
968  xt[j] = map[j] < 0 ? 0 : x[map[j]];
-
969  }
-
970  x += d_in;
-
971  xt += d_out;
-
972  }
-
973 }
-
974 
-
975 void RemapDimensionsTransform::reverse_transform (idx_t n, const float * xt,
-
976  float *x) const
-
977 {
-
978  memset (x, 0, sizeof (*x) * n * d_in);
-
979  for (idx_t i = 0; i < n; i++) {
-
980  for (int j = 0; j < d_out; j++) {
-
981  if (map[j] >= 0) x[map[j]] = xt[j];
-
982  }
-
983  x += d_in;
-
984  xt += d_out;
-
985  }
-
986 }
-
faiss::LinearTransform::transform_transpose
void transform_transpose(idx_t n, const float *y, float *x) const
Definition: VectorTransform.cpp:134
-
faiss::IndexPreTransform::index
Index * index
! chain of tranforms
Definition: VectorTransform.h:242
-
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:107
+
946 
+
947 void IndexPreTransform::reset () {
+
948  index->reset();
+
949  ntotal = 0;
+
950 }
+
951 
+
952 long IndexPreTransform::remove_ids (const IDSelector & sel) {
+
953  long nremove = index->remove_ids (sel);
+
954  ntotal = index->ntotal;
+
955  return nremove;
+
956 }
+
957 
+
958 
+
959 void IndexPreTransform::reconstruct (idx_t key, float * recons) const
+
960 {
+
961  float *x = chain.empty() ? recons : new float [index->d];
+
962  ScopeDeleter<float> del (recons == x ? nullptr : x);
+
963  // Initial reconstruction
+
964  index->reconstruct (key, x);
+
965 
+
966  // Revert transformations from last to first
+
967  reverse_chain (1, x, recons);
+
968 }
+
969 
+
970 
+
971 void IndexPreTransform::reconstruct_n (idx_t i0, idx_t ni, float *recons) const
+
972 {
+
973  float *x = chain.empty() ? recons : new float [ni * index->d];
+
974  ScopeDeleter<float> del (recons == x ? nullptr : x);
+
975  // Initial reconstruction
+
976  index->reconstruct_n (i0, ni, x);
+
977 
+
978  // Revert transformations from last to first
+
979  reverse_chain (ni, x, recons);
+
980 }
+
981 
+
982 
+
983 void IndexPreTransform::search_and_reconstruct (
+
984  idx_t n, const float *x, idx_t k,
+
985  float *distances, idx_t *labels, float* recons) const
+
986 {
+
987  FAISS_THROW_IF_NOT (is_trained);
+
988 
+
989  const float* xt = apply_chain (n, x);
+
990  ScopeDeleter<float> del ((xt == x) ? nullptr : xt);
+
991 
+
992  float* recons_temp = chain.empty() ? recons : new float [n * k * index->d];
+
993  ScopeDeleter<float> del2 ((recons_temp == recons) ? nullptr : recons_temp);
+
994  index->search_and_reconstruct (n, xt, k, distances, labels, recons_temp);
+
995 
+
996  // Revert transformations from last to first
+
997  reverse_chain (n * k, recons_temp, recons);
+
998 }
+
999 
+
1000 
+
1001 /*********************************************
+
1002  * RemapDimensionsTransform
+
1003  *********************************************/
+
1004 
+
1005 
+
1006 RemapDimensionsTransform::RemapDimensionsTransform (
+
1007  int d_in, int d_out, const int *map_in):
+
1008  VectorTransform (d_in, d_out)
+
1009 {
+
1010  map.resize (d_out);
+
1011  for (int i = 0; i < d_out; i++) {
+
1012  map[i] = map_in[i];
+
1013  FAISS_THROW_IF_NOT (map[i] == -1 || (map[i] >= 0 && map[i] < d_in));
+
1014  }
+
1015 }
+
1016 
+
1017 RemapDimensionsTransform::RemapDimensionsTransform (
+
1018  int d_in, int d_out, bool uniform): VectorTransform (d_in, d_out)
+
1019 {
+
1020  map.resize (d_out, -1);
+
1021 
+
1022  if (uniform) {
+
1023  if (d_in < d_out) {
+
1024  for (int i = 0; i < d_in; i++) {
+
1025  map [i * d_out / d_in] = i;
+
1026  }
+
1027  } else {
+
1028  for (int i = 0; i < d_out; i++) {
+
1029  map [i] = i * d_in / d_out;
+
1030  }
+
1031  }
+
1032  } else {
+
1033  for (int i = 0; i < d_in && i < d_out; i++)
+
1034  map [i] = i;
+
1035  }
+
1036 }
+
1037 
+
1038 
+
1039 void RemapDimensionsTransform::apply_noalloc (idx_t n, const float * x,
+
1040  float *xt) const
+
1041 {
+
1042  for (idx_t i = 0; i < n; i++) {
+
1043  for (int j = 0; j < d_out; j++) {
+
1044  xt[j] = map[j] < 0 ? 0 : x[map[j]];
+
1045  }
+
1046  x += d_in;
+
1047  xt += d_out;
+
1048  }
+
1049 }
+
1050 
+
1051 void RemapDimensionsTransform::reverse_transform (idx_t n, const float * xt,
+
1052  float *x) const
+
1053 {
+
1054  memset (x, 0, sizeof (*x) * n * d_in);
+
1055  for (idx_t i = 0; i < n; i++) {
+
1056  for (int j = 0; j < d_out; j++) {
+
1057  if (map[j] >= 0) x[map[j]] = xt[j];
+
1058  }
+
1059  x += d_in;
+
1060  xt += d_out;
+
1061  }
+
1062 }
+
faiss::LinearTransform::transform_transpose
void transform_transpose(idx_t n, const float *y, float *x) const
Definition: VectorTransform.cpp:135
+
faiss::IndexPreTransform::index
Index * index
! chain of tranforms
Definition: VectorTransform.h:249
+
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:117
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:25
-
faiss::OPQMatrix::niter
int niter
Number of outer training iterations.
Definition: VectorTransform.h:183
+
faiss::OPQMatrix::niter
int niter
Number of outer training iterations.
Definition: VectorTransform.h:189
faiss::ProductQuantizer::decode
void decode(const uint8_t *code, float *x) const
decode a vector from a given code (or n vectors if third argument)
Definition: ProductQuantizer.cpp:338
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils.cpp:574
-
faiss::RandomRotationMatrix::init
void init(int seed)
must be called before the transform is used
Definition: VectorTransform.cpp:164
-
faiss::IndexPreTransform::reset
void reset() override
removes all elements from the database.
Definition: VectorTransform.cpp:894
+
faiss::RandomRotationMatrix::init
void init(int seed)
must be called before the transform is used
Definition: VectorTransform.cpp:215
+
faiss::IndexPreTransform::reset
void reset() override
removes all elements from the database.
Definition: VectorTransform.cpp:947
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
-
faiss::OPQMatrix::niter_pq
int niter_pq
Number of training iterations for the PQ.
Definition: VectorTransform.h:184
-
faiss::LinearTransform::A
std::vector< float > A
! whether to use the bias term
Definition: VectorTransform.h:82
-
faiss::fvecs_maybe_subsample
const float * fvecs_maybe_subsample(size_t d, size_t *n, size_t nmax, const float *x, bool verbose, long seed)
Definition: utils.cpp:1973
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
- +
faiss::OPQMatrix::niter_pq
int niter_pq
Number of training iterations for the PQ.
Definition: VectorTransform.h:190
+
faiss::LinearTransform::A
std::vector< float > A
Transformation matrix, size d_out * d_in.
Definition: VectorTransform.h:85
+
faiss::Index::search_and_reconstruct
virtual void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const
Definition: Index.cpp:66
+
faiss::fvecs_maybe_subsample
const float * fvecs_maybe_subsample(size_t d, size_t *n, size_t nmax, const float *x, bool verbose, long seed)
Definition: utils.cpp:1975
+ -
faiss::LinearTransform::LinearTransform
LinearTransform(int d_in=0, int d_out=0, bool have_bias=false)
both d_in &gt; d_out and d_out &lt; d_in are supported
Definition: VectorTransform.cpp:99
- -
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:30
- +
faiss::LinearTransform::LinearTransform
LinearTransform(int d_in=0, int d_out=0, bool have_bias=false)
both d_in &gt; d_out and d_out &lt; d_in are supported
Definition: VectorTransform.cpp:100
+ +
faiss::LinearTransform::set_is_orthonormal
void set_is_orthonormal()
compute A^T * A to set the is_orthonormal flag
Definition: VectorTransform.cpp:160
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
+
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:41
+ -
faiss::PCAMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:271
-
faiss::PCAMatrix::mean
std::vector< float > mean
Mean, size d_in.
Definition: VectorTransform.h:144
- -
faiss::IndexPreTransform::apply_chain
const float * apply_chain(idx_t n, const float *x) const
Definition: VectorTransform.cpp:847
-
faiss::PCAMatrix::PCAMat
std::vector< float > PCAMat
PCA matrix, size d_in * d_in.
Definition: VectorTransform.h:150
-
faiss::IndexPreTransform::train
void train(idx_t n, const float *x) override
Definition: VectorTransform.cpp:791
+
faiss::PCAMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:316
+
faiss::PCAMatrix::mean
std::vector< float > mean
Mean, size d_in.
Definition: VectorTransform.h:152
+ +
faiss::IndexPreTransform::apply_chain
const float * apply_chain(idx_t n, const float *x) const
Definition: VectorTransform.cpp:886
+
faiss::PCAMatrix::PCAMat
std::vector< float > PCAMat
PCA matrix, size d_in * d_in.
Definition: VectorTransform.h:158
+
faiss::IndexPreTransform::train
void train(idx_t n, const float *x) override
Definition: VectorTransform.cpp:828
faiss::ProductQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: ProductQuantizer.cpp:385
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
-
faiss::LinearTransform::b
std::vector< float > b
bias vector, size d_out
Definition: VectorTransform.h:85
-
faiss::Index::reconstruct_n
virtual void reconstruct_n(idx_t i0, idx_t ni, float *recons) const
Definition: Index.cpp:48
+
faiss::LinearTransform::b
std::vector< float > b
bias vector, size d_out
Definition: VectorTransform.h:88
+
faiss::LinearTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
works only if is_orthonormal
Definition: VectorTransform.cpp:199
+
faiss::Index::reconstruct_n
virtual void reconstruct_n(idx_t i0, idx_t ni, float *recons) const
Definition: Index.cpp:59
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
+
faiss::NormalizationTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Identity transform since norm is not revertible.
Definition: VectorTransform.cpp:772
-
faiss::OPQMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:548
-
faiss::PCAMatrix::balanced_bins
int balanced_bins
try to distribute output eigenvectors in this many bins
Definition: VectorTransform.h:141
+
faiss::OPQMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:587
+
faiss::PCAMatrix::balanced_bins
int balanced_bins
try to distribute output eigenvectors in this many bins
Definition: VectorTransform.h:149
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
-
faiss::IndexPreTransform::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: VectorTransform.cpp:906
+
faiss::IndexPreTransform::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: VectorTransform.cpp:971
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
-
faiss::NormalizationTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:731
+
faiss::NormalizationTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:762
faiss::ProductQuantizer::Train_hot_start
the centroids are already initialized
Definition: ProductQuantizer.h:42
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
-
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:37
+
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:48
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
-
faiss::matrix_qr
void matrix_qr(int m, int n, float *a)
Definition: utils.cpp:1320
-
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:244
-
faiss::OPQMatrix::niter_pq_0
int niter_pq_0
same, for the first outer iteration
Definition: VectorTransform.h:185
-
faiss::OPQMatrix::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:709
+
faiss::matrix_qr
void matrix_qr(int m, int n, float *a)
Definition: utils.cpp:1322
+
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:251
+
faiss::OPQMatrix::niter_pq_0
int niter_pq_0
same, for the first outer iteration
Definition: VectorTransform.h:191
faiss::ProductQuantizer::cp
ClusteringParameters cp
parameters used during clustering
Definition: ProductQuantizer.h:49
-
faiss::ivec_checksum
size_t ivec_checksum(size_t n, const int *a)
compute a checksum on a table.
Definition: utils.cpp:1670
+
faiss::ivec_checksum
size_t ivec_checksum(size_t n, const int *a)
compute a checksum on a table.
Definition: utils.cpp:1672
faiss::VectorTransform::train
virtual void train(idx_t n, const float *x)
Definition: VectorTransform.cpp:80
-
faiss::RandomRotationMatrix::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:189
faiss::VectorTransform::reverse_transform
virtual void reverse_transform(idx_t n, const float *xt, float *x) const
Definition: VectorTransform.cpp:85
-
faiss::RemapDimensionsTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
reverse transform correct only when the mapping is a permuation
Definition: VectorTransform.cpp:975
-
faiss::PCAMatrix::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:522
-
faiss::OPQMatrix::max_train_points
size_t max_train_points
if there are too many training points, resample
Definition: VectorTransform.h:188
-
faiss::PCAMatrix::copy_from
void copy_from(const PCAMatrix &other)
copy pre-trained PCA matrix
Definition: VectorTransform.cpp:413
+
faiss::IndexPreTransform::search_and_reconstruct
void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const override
Definition: VectorTransform.cpp:983
+
faiss::RemapDimensionsTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
reverse transform correct only when the mapping is a permuation
Definition: VectorTransform.cpp:1051
+
faiss::OPQMatrix::max_train_points
size_t max_train_points
if there are too many training points, resample
Definition: VectorTransform.h:194
+
faiss::PCAMatrix::copy_from
void copy_from(const PCAMatrix &other)
copy pre-trained PCA matrix
Definition: VectorTransform.cpp:458
faiss::VectorTransform::d_out
int d_out
! input dimension
Definition: VectorTransform.h:33
-
faiss::OPQMatrix::OPQMatrix
OPQMatrix(int d=0, int M=1, int d2=-1)
if d2 != -1, output vectors of this dimension
Definition: VectorTransform.cpp:535
+
faiss::OPQMatrix::OPQMatrix
OPQMatrix(int d=0, int M=1, int d2=-1)
if d2 != -1, output vectors of this dimension
Definition: VectorTransform.cpp:574
-
faiss::PCAMatrix::prepare_Ab
void prepare_Ab()
called after mean, PCAMat and eigenvalues are computed
Definition: VectorTransform.cpp:423
-
faiss::IndexPreTransform::add
void add(idx_t n, const float *x) override
Definition: VectorTransform.cpp:862
-
faiss::RemapDimensionsTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:963
+
faiss::PCAMatrix::prepare_Ab
void prepare_Ab()
called after mean, PCAMat and eigenvalues are computed
Definition: VectorTransform.cpp:468
+
faiss::IndexPreTransform::add
void add(idx_t n, const float *x) override
Definition: VectorTransform.cpp:915
+
faiss::RemapDimensionsTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:1039
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
-
faiss::PCAMatrix::eigenvalues
std::vector< float > eigenvalues
eigenvalues of covariance matrix (= squared singular values)
Definition: VectorTransform.h:147
-
faiss::IndexPreTransform::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: VectorTransform.cpp:884
-
faiss::IndexPreTransform::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: VectorTransform.cpp:871
-
faiss::PCAMatrix::random_rotation
bool random_rotation
random rotation after PCA
Definition: VectorTransform.h:135
-
faiss::PCAMatrix::max_points_per_d
size_t max_points_per_d
ratio between # training vectors and dimension
Definition: VectorTransform.h:138
+
faiss::IndexPreTransform::reverse_chain
void reverse_chain(idx_t n, const float *xt, float *x) const
Definition: VectorTransform.cpp:901
+
faiss::LinearTransform::is_orthonormal
bool is_orthonormal
! whether to use the bias term
Definition: VectorTransform.h:82
+
faiss::PCAMatrix::eigenvalues
std::vector< float > eigenvalues
eigenvalues of covariance matrix (= squared singular values)
Definition: VectorTransform.h:155
+
faiss::IndexPreTransform::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: VectorTransform.cpp:937
+
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:54
+
faiss::IndexPreTransform::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: VectorTransform.cpp:924
+
faiss::PCAMatrix::random_rotation
bool random_rotation
random rotation after PCA
Definition: VectorTransform.h:143
+
faiss::PCAMatrix::max_points_per_d
size_t max_points_per_d
ratio between # training vectors and dimension
Definition: VectorTransform.h:146
faiss::ClusteringParameters::max_points_per_centroid
int max_points_per_centroid
to limit size of dataset
Definition: Clustering.h:34
faiss::VectorTransform::apply
float * apply(idx_t n, const float *x) const
Definition: VectorTransform.cpp:72
-
faiss::IndexPreTransform::remove_ids
long remove_ids(const IDSelector &sel) override
Definition: VectorTransform.cpp:899
+
faiss::IndexPreTransform::remove_ids
long remove_ids(const IDSelector &sel) override
Definition: VectorTransform.cpp:952
faiss::VectorTransform::apply_noalloc
virtual void apply_noalloc(idx_t n, const float *x, float *xt) const =0
same as apply, but result is pre-allocated
-
faiss::OPQMatrix::M
int M
nb of subquantizers
Definition: VectorTransform.h:182
-
faiss::LinearTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:105
+
faiss::IndexPreTransform::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: VectorTransform.cpp:959
+
faiss::OPQMatrix::M
int M
nb of subquantizers
Definition: VectorTransform.h:188
+
faiss::LinearTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:106
diff --git a/docs/html/VectorTransform_8h_source.html b/docs/html/VectorTransform_8h_source.html index 42653fe7f..d887b9d44 100644 --- a/docs/html/VectorTransform_8h_source.html +++ b/docs/html/VectorTransform_8h_source.html @@ -122,7 +122,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
37  {}
38 
39 
-
40  /// set if the LinearTransform does not require training, or if
+
40  /// set if the VectorTransform does not require training, or if
41  /// training is done already
42  bool is_trained;
43 
@@ -163,275 +163,296 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
78 
79  bool have_bias; ///! whether to use the bias term
80 
-
81  /// Transformation matrix, size d_out * d_in
-
82  std::vector<float> A;
+
81  /// check if matrix A is orthonormal (enables reverse_transform)
+
82  bool is_orthonormal;
83 
-
84  /// bias vector, size d_out
-
85  std::vector<float> b;
+
84  /// Transformation matrix, size d_out * d_in
+
85  std::vector<float> A;
86 
-
87  /// both d_in > d_out and d_out < d_in are supported
-
88  explicit LinearTransform (int d_in = 0, int d_out = 0,
-
89  bool have_bias = false);
-
90 
-
91  /// same as apply, but result is pre-allocated
-
92  void apply_noalloc(idx_t n, const float* x, float* xt) const override;
+
87  /// bias vector, size d_out
+
88  std::vector<float> b;
+
89 
+
90  /// both d_in > d_out and d_out < d_in are supported
+
91  explicit LinearTransform (int d_in = 0, int d_out = 0,
+
92  bool have_bias = false);
93 
-
94  /// compute x = A^T * (x - b)
-
95  /// is reverse transform if A has orthonormal lines
-
96  void transform_transpose (idx_t n, const float * y,
-
97  float *x) const;
-
98 
-
99  bool verbose;
-
100 
-
101  ~LinearTransform() override {}
-
102 };
-
103 
-
104 
+
94  /// same as apply, but result is pre-allocated
+
95  void apply_noalloc(idx_t n, const float* x, float* xt) const override;
+
96 
+
97  /// compute x = A^T * (x - b)
+
98  /// is reverse transform if A has orthonormal lines
+
99  void transform_transpose (idx_t n, const float * y,
+
100  float *x) const;
+
101 
+
102  /// works only if is_orthonormal
+
103  void reverse_transform (idx_t n, const float * xt,
+
104  float *x) const override;
105 
-
106 /// Randomly rotate a set of vectors
-
107 struct RandomRotationMatrix: LinearTransform {
-
108 
-
109  /// both d_in > d_out and d_out < d_in are supported
-
110  RandomRotationMatrix (int d_in, int d_out):
-
111  LinearTransform(d_in, d_out, false) {}
-
112 
-
113  /// must be called before the transform is used
-
114  void init(int seed);
-
115 
-
116  void reverse_transform(idx_t n, const float* xt, float* x) const override;
-
117 
-
118  RandomRotationMatrix () {}
-
119 };
-
120 
-
121 
-
122 /** Applies a principal component analysis on a set of vectors,
-
123  * with optionally whitening and random rotation. */
-
124 struct PCAMatrix: LinearTransform {
-
125 
-
126  /** after transformation the components are multiplied by
-
127  * eigenvalues^eigen_power
-
128  *
-
129  * =0: no whitening
-
130  * =-2: full whitening
-
131  */
-
132  float eigen_power;
+
106  /// compute A^T * A to set the is_orthonormal flag
+
107  void set_is_orthonormal ();
+
108 
+
109  bool verbose;
+
110 
+
111  ~LinearTransform() override {}
+
112 };
+
113 
+
114 
+
115 
+
116 /// Randomly rotate a set of vectors
+
117 struct RandomRotationMatrix: LinearTransform {
+
118 
+
119  /// both d_in > d_out and d_out < d_in are supported
+
120  RandomRotationMatrix (int d_in, int d_out):
+
121  LinearTransform(d_in, d_out, false) {}
+
122 
+
123  /// must be called before the transform is used
+
124  void init(int seed);
+
125 
+
126  RandomRotationMatrix () {}
+
127 };
+
128 
+
129 
+
130 /** Applies a principal component analysis on a set of vectors,
+
131  * with optionally whitening and random rotation. */
+
132 struct PCAMatrix: LinearTransform {
133 
-
134  /// random rotation after PCA
-
135  bool random_rotation;
-
136 
-
137  /// ratio between # training vectors and dimension
-
138  size_t max_points_per_d;
-
139 
-
140  /// try to distribute output eigenvectors in this many bins
-
141  int balanced_bins;
-
142 
-
143  /// Mean, size d_in
-
144  std::vector<float> mean;
-
145 
-
146  /// eigenvalues of covariance matrix (= squared singular values)
-
147  std::vector<float> eigenvalues;
-
148 
-
149  /// PCA matrix, size d_in * d_in
-
150  std::vector<float> PCAMat;
-
151 
-
152  // the final matrix is computed after random rotation and/or whitening
-
153  explicit PCAMatrix (int d_in = 0, int d_out = 0,
-
154  float eigen_power = 0, bool random_rotation = false);
-
155 
-
156  /// train on n vectors. If n < d_in then the eigenvector matrix
-
157  /// will be completed with 0s
-
158  void train(Index::idx_t n, const float* x) override;
+
134  /** after transformation the components are multiplied by
+
135  * eigenvalues^eigen_power
+
136  *
+
137  * =0: no whitening
+
138  * =-2: full whitening
+
139  */
+
140  float eigen_power;
+
141 
+
142  /// random rotation after PCA
+
143  bool random_rotation;
+
144 
+
145  /// ratio between # training vectors and dimension
+
146  size_t max_points_per_d;
+
147 
+
148  /// try to distribute output eigenvectors in this many bins
+
149  int balanced_bins;
+
150 
+
151  /// Mean, size d_in
+
152  std::vector<float> mean;
+
153 
+
154  /// eigenvalues of covariance matrix (= squared singular values)
+
155  std::vector<float> eigenvalues;
+
156 
+
157  /// PCA matrix, size d_in * d_in
+
158  std::vector<float> PCAMat;
159 
-
160  void reverse_transform(idx_t n, const float* xt, float* x) const override;
-
161 
-
162  /// copy pre-trained PCA matrix
-
163  void copy_from (const PCAMatrix & other);
-
164 
-
165  /// called after mean, PCAMat and eigenvalues are computed
-
166  void prepare_Ab();
-
167 
-
168 };
-
169 
-
170 
-
171 
-
172 /** Applies a rotation to align the dimensions with a PQ to minimize
-
173  * the reconstruction error. Can be used before an IndexPQ or an
-
174  * IndexIVFPQ. The method is the non-parametric version described in:
-
175  *
-
176  * "Optimized Product Quantization for Approximate Nearest Neighbor Search"
-
177  * Tiezheng Ge, Kaiming He, Qifa Ke, Jian Sun, CVPR'13
-
178  *
-
179  */
-
180 struct OPQMatrix: LinearTransform {
-
181 
-
182  int M; ///< nb of subquantizers
-
183  int niter; ///< Number of outer training iterations
-
184  int niter_pq; ///< Number of training iterations for the PQ
-
185  int niter_pq_0; ///< same, for the first outer iteration
-
186 
-
187  /// if there are too many training points, resample
-
188  size_t max_train_points;
-
189  bool verbose;
-
190 
-
191  /// if d2 != -1, output vectors of this dimension
-
192  explicit OPQMatrix (int d = 0, int M = 1, int d2 = -1);
-
193 
-
194  void train(Index::idx_t n, const float* x) override;
-
195 
-
196  void reverse_transform(idx_t n, const float* xt, float* x) const override;
-
197 };
-
198 
-
199 
-
200 /** remap dimensions for intput vectors, possibly inserting 0s
-
201  * strictly speaking this is also a linear transform but we don't want
-
202  * to compute it with matrix multiplies */
-
203 struct RemapDimensionsTransform: VectorTransform {
-
204 
-
205  /// map from output dimension to input, size d_out
-
206  /// -1 -> set output to 0
-
207  std::vector<int> map;
-
208 
-
209  RemapDimensionsTransform (int d_in, int d_out, const int *map);
-
210 
-
211  /// remap input to output, skipping or inserting dimensions as needed
-
212  /// if uniform: distribute dimensions uniformly
-
213  /// otherwise just take the d_out first ones.
-
214  RemapDimensionsTransform (int d_in, int d_out, bool uniform = true);
-
215 
-
216  void apply_noalloc(idx_t n, const float* x, float* xt) const override;
-
217 
-
218  /// reverse transform correct only when the mapping is a permuation
-
219  void reverse_transform(idx_t n, const float* xt, float* x) const override;
-
220 
-
221  RemapDimensionsTransform () {}
-
222 };
-
223 
-
224 
-
225 /** per-vector normalization */
-
226 struct NormalizationTransform: VectorTransform {
-
227  float norm;
-
228 
-
229  explicit NormalizationTransform (int d, float norm = 2.0);
-
230  NormalizationTransform ();
-
231 
-
232  void apply_noalloc(idx_t n, const float* x, float* xt) const override;
-
233 };
-
234 
+
160  // the final matrix is computed after random rotation and/or whitening
+
161  explicit PCAMatrix (int d_in = 0, int d_out = 0,
+
162  float eigen_power = 0, bool random_rotation = false);
+
163 
+
164  /// train on n vectors. If n < d_in then the eigenvector matrix
+
165  /// will be completed with 0s
+
166  void train(Index::idx_t n, const float* x) override;
+
167 
+
168  /// copy pre-trained PCA matrix
+
169  void copy_from (const PCAMatrix & other);
+
170 
+
171  /// called after mean, PCAMat and eigenvalues are computed
+
172  void prepare_Ab();
+
173 
+
174 };
+
175 
+
176 
+
177 
+
178 /** Applies a rotation to align the dimensions with a PQ to minimize
+
179  * the reconstruction error. Can be used before an IndexPQ or an
+
180  * IndexIVFPQ. The method is the non-parametric version described in:
+
181  *
+
182  * "Optimized Product Quantization for Approximate Nearest Neighbor Search"
+
183  * Tiezheng Ge, Kaiming He, Qifa Ke, Jian Sun, CVPR'13
+
184  *
+
185  */
+
186 struct OPQMatrix: LinearTransform {
+
187 
+
188  int M; ///< nb of subquantizers
+
189  int niter; ///< Number of outer training iterations
+
190  int niter_pq; ///< Number of training iterations for the PQ
+
191  int niter_pq_0; ///< same, for the first outer iteration
+
192 
+
193  /// if there are too many training points, resample
+
194  size_t max_train_points;
+
195  bool verbose;
+
196 
+
197  /// if d2 != -1, output vectors of this dimension
+
198  explicit OPQMatrix (int d = 0, int M = 1, int d2 = -1);
+
199 
+
200  void train(Index::idx_t n, const float* x) override;
+
201 };
+
202 
+
203 
+
204 /** remap dimensions for intput vectors, possibly inserting 0s
+
205  * strictly speaking this is also a linear transform but we don't want
+
206  * to compute it with matrix multiplies */
+
207 struct RemapDimensionsTransform: VectorTransform {
+
208 
+
209  /// map from output dimension to input, size d_out
+
210  /// -1 -> set output to 0
+
211  std::vector<int> map;
+
212 
+
213  RemapDimensionsTransform (int d_in, int d_out, const int *map);
+
214 
+
215  /// remap input to output, skipping or inserting dimensions as needed
+
216  /// if uniform: distribute dimensions uniformly
+
217  /// otherwise just take the d_out first ones.
+
218  RemapDimensionsTransform (int d_in, int d_out, bool uniform = true);
+
219 
+
220  void apply_noalloc(idx_t n, const float* x, float* xt) const override;
+
221 
+
222  /// reverse transform correct only when the mapping is a permuation
+
223  void reverse_transform(idx_t n, const float* xt, float* x) const override;
+
224 
+
225  RemapDimensionsTransform () {}
+
226 };
+
227 
+
228 
+
229 /** per-vector normalization */
+
230 struct NormalizationTransform: VectorTransform {
+
231  float norm;
+
232 
+
233  explicit NormalizationTransform (int d, float norm = 2.0);
+
234  NormalizationTransform ();
235 
-
236 
-
237 /** Index that applies a LinearTransform transform on vectors before
-
238  * handing them over to a sub-index */
-
239 struct IndexPreTransform: Index {
-
240 
-
241  std::vector<VectorTransform *> chain; ///! chain of tranforms
-
242  Index * index; ///! the sub-index
+
236  void apply_noalloc(idx_t n, const float* x, float* xt) const override;
+
237 
+
238  /// Identity transform since norm is not revertible
+
239  void reverse_transform(idx_t n, const float* xt, float* x) const override;
+
240 };
+
241 
+
242 
243 
-
244  bool own_fields; ///! whether pointers are deleted in destructor
-
245 
-
246  explicit IndexPreTransform (Index *index);
+
244 /** Index that applies a LinearTransform transform on vectors before
+
245  * handing them over to a sub-index */
+
246 struct IndexPreTransform: Index {
247 
-
248  IndexPreTransform ();
-
249 
-
250  /// ltrans is the last transform before the index
-
251  IndexPreTransform (VectorTransform * ltrans, Index * index);
-
252 
-
253  void prepend_transform (VectorTransform * ltrans);
+
248  std::vector<VectorTransform *> chain; ///! chain of tranforms
+
249  Index * index; ///! the sub-index
+
250 
+
251  bool own_fields; ///! whether pointers are deleted in destructor
+
252 
+
253  explicit IndexPreTransform (Index *index);
254 
-
255  void train(idx_t n, const float* x) override;
-
256 
-
257  void add(idx_t n, const float* x) override;
-
258 
-
259  void add_with_ids(idx_t n, const float* x, const long* xids) override;
-
260 
-
261  void reset() override;
-
262 
-
263  /** removes IDs from the index. Not supported by all indexes.
-
264  */
-
265  long remove_ids(const IDSelector& sel) override;
-
266 
-
267  void search(
-
268  idx_t n,
-
269  const float* x,
-
270  idx_t k,
-
271  float* distances,
-
272  idx_t* labels) const override;
+
255  IndexPreTransform ();
+
256 
+
257  /// ltrans is the last transform before the index
+
258  IndexPreTransform (VectorTransform * ltrans, Index * index);
+
259 
+
260  void prepend_transform (VectorTransform * ltrans);
+
261 
+
262  void train(idx_t n, const float* x) override;
+
263 
+
264  void add(idx_t n, const float* x) override;
+
265 
+
266  void add_with_ids(idx_t n, const float* x, const long* xids) override;
+
267 
+
268  void reset() override;
+
269 
+
270  /** removes IDs from the index. Not supported by all indexes.
+
271  */
+
272  long remove_ids(const IDSelector& sel) override;
273 
-
274  void reconstruct_n (idx_t i0, idx_t ni, float *recons)
-
275  const override;
-
276 
-
277  /// apply the transforms in the chain. The returned float * may be
-
278  /// equal to x, otherwise it should be deallocated.
-
279  const float * apply_chain (idx_t n, const float *x) const;
+
274  void search(
+
275  idx_t n,
+
276  const float* x,
+
277  idx_t k,
+
278  float* distances,
+
279  idx_t* labels) const override;
280 
-
281  ~IndexPreTransform() override;
-
282 };
-
283 
-
284 
+
281  void reconstruct (idx_t key, float * recons) const override;
+
282 
+
283  void reconstruct_n (idx_t i0, idx_t ni, float *recons)
+
284  const override;
285 
-
286 } // namespace faiss
-
287 
-
288 
-
289 
-
290 #endif
-
faiss::LinearTransform::transform_transpose
void transform_transpose(idx_t n, const float *y, float *x) const
Definition: VectorTransform.cpp:134
-
faiss::IndexPreTransform::index
Index * index
! chain of tranforms
Definition: VectorTransform.h:242
-
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:107
-
faiss::OPQMatrix::niter
int niter
Number of outer training iterations.
Definition: VectorTransform.h:183
-
faiss::RandomRotationMatrix::RandomRotationMatrix
RandomRotationMatrix(int d_in, int d_out)
both d_in &gt; d_out and d_out &lt; d_in are supported
Definition: VectorTransform.h:110
-
faiss::RandomRotationMatrix::init
void init(int seed)
must be called before the transform is used
Definition: VectorTransform.cpp:164
-
faiss::IndexPreTransform::reset
void reset() override
removes all elements from the database.
Definition: VectorTransform.cpp:894
-
faiss::OPQMatrix::niter_pq
int niter_pq
Number of training iterations for the PQ.
Definition: VectorTransform.h:184
-
faiss::LinearTransform::A
std::vector< float > A
! whether to use the bias term
Definition: VectorTransform.h:82
- +
286  void search_and_reconstruct (idx_t n, const float *x, idx_t k,
+
287  float *distances, idx_t *labels,
+
288  float *recons) const override;
+
289 
+
290  /// apply the transforms in the chain. The returned float * may be
+
291  /// equal to x, otherwise it should be deallocated.
+
292  const float * apply_chain (idx_t n, const float *x) const;
+
293 
+
294  /// Reverse the transforms in the chain. May not be implemented for
+
295  /// all transforms in the chain or may return approximate results.
+
296  void reverse_chain (idx_t n, const float* xt, float* x) const;
+
297 
+
298  ~IndexPreTransform() override;
+
299 };
+
300 
+
301 
+
302 
+
303 } // namespace faiss
+
304 
+
305 
+
306 
+
307 #endif
+
faiss::LinearTransform::transform_transpose
void transform_transpose(idx_t n, const float *y, float *x) const
Definition: VectorTransform.cpp:135
+
faiss::IndexPreTransform::index
Index * index
! chain of tranforms
Definition: VectorTransform.h:249
+
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:117
+
faiss::OPQMatrix::niter
int niter
Number of outer training iterations.
Definition: VectorTransform.h:189
+
faiss::RandomRotationMatrix::RandomRotationMatrix
RandomRotationMatrix(int d_in, int d_out)
both d_in &gt; d_out and d_out &lt; d_in are supported
Definition: VectorTransform.h:120
+
faiss::RandomRotationMatrix::init
void init(int seed)
must be called before the transform is used
Definition: VectorTransform.cpp:215
+
faiss::IndexPreTransform::reset
void reset() override
removes all elements from the database.
Definition: VectorTransform.cpp:947
+
faiss::OPQMatrix::niter_pq
int niter_pq
Number of training iterations for the PQ.
Definition: VectorTransform.h:190
+
faiss::LinearTransform::A
std::vector< float > A
Transformation matrix, size d_out * d_in.
Definition: VectorTransform.h:85
+ -
faiss::LinearTransform::LinearTransform
LinearTransform(int d_in=0, int d_out=0, bool have_bias=false)
both d_in &gt; d_out and d_out &lt; d_in are supported
Definition: VectorTransform.cpp:99
- +
faiss::LinearTransform::LinearTransform
LinearTransform(int d_in=0, int d_out=0, bool have_bias=false)
both d_in &gt; d_out and d_out &lt; d_in are supported
Definition: VectorTransform.cpp:100
+
faiss::VectorTransform::VectorTransform
VectorTransform(int d_in=0, int d_out=0)
! output dimension
Definition: VectorTransform.h:35
- -
faiss::PCAMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:271
- -
faiss::PCAMatrix::mean
std::vector< float > mean
Mean, size d_in.
Definition: VectorTransform.h:144
- -
faiss::IndexPreTransform::apply_chain
const float * apply_chain(idx_t n, const float *x) const
Definition: VectorTransform.cpp:847
-
faiss::PCAMatrix::PCAMat
std::vector< float > PCAMat
PCA matrix, size d_in * d_in.
Definition: VectorTransform.h:150
-
faiss::IndexPreTransform::train
void train(idx_t n, const float *x) override
Definition: VectorTransform.cpp:791
-
faiss::LinearTransform::b
std::vector< float > b
bias vector, size d_out
Definition: VectorTransform.h:85
+
faiss::LinearTransform::set_is_orthonormal
void set_is_orthonormal()
compute A^T * A to set the is_orthonormal flag
Definition: VectorTransform.cpp:160
+ +
faiss::PCAMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:316
+ +
faiss::PCAMatrix::mean
std::vector< float > mean
Mean, size d_in.
Definition: VectorTransform.h:152
+ +
faiss::IndexPreTransform::apply_chain
const float * apply_chain(idx_t n, const float *x) const
Definition: VectorTransform.cpp:886
+
faiss::PCAMatrix::PCAMat
std::vector< float > PCAMat
PCA matrix, size d_in * d_in.
Definition: VectorTransform.h:158
+
faiss::IndexPreTransform::train
void train(idx_t n, const float *x) override
Definition: VectorTransform.cpp:828
+
faiss::LinearTransform::b
std::vector< float > b
bias vector, size d_out
Definition: VectorTransform.h:88
+
faiss::LinearTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
works only if is_orthonormal
Definition: VectorTransform.cpp:199
+
faiss::NormalizationTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Identity transform since norm is not revertible.
Definition: VectorTransform.cpp:772
-
faiss::OPQMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:548
-
faiss::PCAMatrix::balanced_bins
int balanced_bins
try to distribute output eigenvectors in this many bins
Definition: VectorTransform.h:141
+
faiss::OPQMatrix::train
void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:587
+
faiss::PCAMatrix::balanced_bins
int balanced_bins
try to distribute output eigenvectors in this many bins
Definition: VectorTransform.h:149
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
-
faiss::IndexPreTransform::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: VectorTransform.cpp:906
-
faiss::NormalizationTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:731
+
faiss::IndexPreTransform::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: VectorTransform.cpp:971
+
faiss::NormalizationTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:762
-
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:244
-
faiss::OPQMatrix::niter_pq_0
int niter_pq_0
same, for the first outer iteration
Definition: VectorTransform.h:185
-
faiss::OPQMatrix::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:709
+
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:251
+
faiss::OPQMatrix::niter_pq_0
int niter_pq_0
same, for the first outer iteration
Definition: VectorTransform.h:191
faiss::VectorTransform::train
virtual void train(idx_t n, const float *x)
Definition: VectorTransform.cpp:80
-
faiss::RandomRotationMatrix::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:189
faiss::VectorTransform::reverse_transform
virtual void reverse_transform(idx_t n, const float *xt, float *x) const
Definition: VectorTransform.cpp:85
-
faiss::RemapDimensionsTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
reverse transform correct only when the mapping is a permuation
Definition: VectorTransform.cpp:975
- -
faiss::PCAMatrix::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:522
-
faiss::OPQMatrix::max_train_points
size_t max_train_points
if there are too many training points, resample
Definition: VectorTransform.h:188
-
faiss::PCAMatrix::copy_from
void copy_from(const PCAMatrix &other)
copy pre-trained PCA matrix
Definition: VectorTransform.cpp:413
+
faiss::IndexPreTransform::search_and_reconstruct
void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const override
Definition: VectorTransform.cpp:983
+
faiss::RemapDimensionsTransform::reverse_transform
void reverse_transform(idx_t n, const float *xt, float *x) const override
reverse transform correct only when the mapping is a permuation
Definition: VectorTransform.cpp:1051
+ +
faiss::OPQMatrix::max_train_points
size_t max_train_points
if there are too many training points, resample
Definition: VectorTransform.h:194
+
faiss::PCAMatrix::copy_from
void copy_from(const PCAMatrix &other)
copy pre-trained PCA matrix
Definition: VectorTransform.cpp:458
faiss::VectorTransform::d_out
int d_out
! input dimension
Definition: VectorTransform.h:33
-
faiss::OPQMatrix::OPQMatrix
OPQMatrix(int d=0, int M=1, int d2=-1)
if d2 != -1, output vectors of this dimension
Definition: VectorTransform.cpp:535
+
faiss::OPQMatrix::OPQMatrix
OPQMatrix(int d=0, int M=1, int d2=-1)
if d2 != -1, output vectors of this dimension
Definition: VectorTransform.cpp:574
-
faiss::PCAMatrix::prepare_Ab
void prepare_Ab()
called after mean, PCAMat and eigenvalues are computed
Definition: VectorTransform.cpp:423
-
faiss::IndexPreTransform::add
void add(idx_t n, const float *x) override
Definition: VectorTransform.cpp:862
-
faiss::RemapDimensionsTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:963
-
faiss::PCAMatrix::eigenvalues
std::vector< float > eigenvalues
eigenvalues of covariance matrix (= squared singular values)
Definition: VectorTransform.h:147
-
faiss::IndexPreTransform::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: VectorTransform.cpp:884
- -
faiss::IndexPreTransform::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: VectorTransform.cpp:871
-
faiss::PCAMatrix::random_rotation
bool random_rotation
random rotation after PCA
Definition: VectorTransform.h:135
-
faiss::PCAMatrix::max_points_per_d
size_t max_points_per_d
ratio between # training vectors and dimension
Definition: VectorTransform.h:138
+
faiss::PCAMatrix::prepare_Ab
void prepare_Ab()
called after mean, PCAMat and eigenvalues are computed
Definition: VectorTransform.cpp:468
+
faiss::IndexPreTransform::add
void add(idx_t n, const float *x) override
Definition: VectorTransform.cpp:915
+
faiss::RemapDimensionsTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:1039
+
faiss::IndexPreTransform::reverse_chain
void reverse_chain(idx_t n, const float *xt, float *x) const
Definition: VectorTransform.cpp:901
+
faiss::LinearTransform::is_orthonormal
bool is_orthonormal
! whether to use the bias term
Definition: VectorTransform.h:82
+
faiss::PCAMatrix::eigenvalues
std::vector< float > eigenvalues
eigenvalues of covariance matrix (= squared singular values)
Definition: VectorTransform.h:155
+
faiss::IndexPreTransform::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: VectorTransform.cpp:937
+ +
faiss::IndexPreTransform::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: VectorTransform.cpp:924
+
faiss::PCAMatrix::random_rotation
bool random_rotation
random rotation after PCA
Definition: VectorTransform.h:143
+
faiss::PCAMatrix::max_points_per_d
size_t max_points_per_d
ratio between # training vectors and dimension
Definition: VectorTransform.h:146
faiss::VectorTransform::apply
float * apply(idx_t n, const float *x) const
Definition: VectorTransform.cpp:72
-
faiss::IndexPreTransform::remove_ids
long remove_ids(const IDSelector &sel) override
Definition: VectorTransform.cpp:899
+
faiss::IndexPreTransform::remove_ids
long remove_ids(const IDSelector &sel) override
Definition: VectorTransform.cpp:952
faiss::VectorTransform::apply_noalloc
virtual void apply_noalloc(idx_t n, const float *x, float *xt) const =0
same as apply, but result is pre-allocated
-
faiss::OPQMatrix::M
int M
nb of subquantizers
Definition: VectorTransform.h:182
-
faiss::LinearTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:105
+
faiss::IndexPreTransform::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: VectorTransform.cpp:959
+
faiss::OPQMatrix::M
int M
nb of subquantizers
Definition: VectorTransform.h:188
+
faiss::LinearTransform::apply_noalloc
void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:106
diff --git a/docs/html/WarpShuffles_8cuh_source.html b/docs/html/WarpShuffles_8cuh_source.html index ab346584b..02e2588b7 100644 --- a/docs/html/WarpShuffles_8cuh_source.html +++ b/docs/html/WarpShuffles_8cuh_source.html @@ -108,86 +108,107 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
19 template <typename T>
20 inline __device__ T shfl(const T val,
21  int srcLane, int width = kWarpSize) {
-
22  return __shfl(val, srcLane, width);
-
23 }
-
24 
-
25 // CUDA SDK does not provide specializations for T*
-
26 template <typename T>
-
27 inline __device__ T* shfl(T* const val,
-
28  int srcLane, int width = kWarpSize) {
-
29  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
-
30  long long v = (long long) val;
-
31  return (T*) __shfl(v, srcLane, width);
-
32 }
-
33 
-
34 template <typename T>
-
35 inline __device__ T shfl_up(const T val,
-
36  unsigned int delta, int width = kWarpSize) {
-
37  return __shfl_up(val, delta, width);
-
38 }
-
39 
-
40 // CUDA SDK does not provide specializations for T*
-
41 template <typename T>
-
42 inline __device__ T* shfl_up(T* const val,
-
43  unsigned int delta, int width = kWarpSize) {
-
44  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
-
45  long long v = (long long) val;
-
46  return (T*) __shfl_up(v, delta, width);
-
47 }
+
22 #if CUDA_VERSION >= 9000
+
23  return __shfl_sync(0xffffffff, val, srcLane, width);
+
24 #else
+
25  return __shfl(val, srcLane, width);
+
26 #endif
+
27 }
+
28 
+
29 // CUDA SDK does not provide specializations for T*
+
30 template <typename T>
+
31 inline __device__ T* shfl(T* const val,
+
32  int srcLane, int width = kWarpSize) {
+
33  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
+
34  long long v = (long long) val;
+
35 
+
36  return (T*) shfl(v, srcLane, width);
+
37 }
+
38 
+
39 template <typename T>
+
40 inline __device__ T shfl_up(const T val,
+
41  unsigned int delta, int width = kWarpSize) {
+
42 #if CUDA_VERSION >= 9000
+
43  return __shfl_up_sync(0xffffffff, val, delta, width);
+
44 #else
+
45  return __shfl_up(val, delta, width);
+
46 #endif
+
47 }
48 
-
49 template <typename T>
-
50 inline __device__ T shfl_down(const T val,
-
51  unsigned int delta, int width = kWarpSize) {
-
52  return __shfl_down(val, delta, width);
-
53 }
-
54 
-
55 // CUDA SDK does not provide specializations for T*
-
56 template <typename T>
-
57 inline __device__ T* shfl_down(T* const val,
-
58  unsigned int delta, int width = kWarpSize) {
-
59  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
-
60  long long v = (long long) val;
-
61  return (T*) __shfl_down(v, delta, width);
-
62 }
-
63 
-
64 template <typename T>
-
65 inline __device__ T shfl_xor(const T val,
-
66  int laneMask, int width = kWarpSize) {
-
67  return __shfl_xor(val, laneMask, width);
-
68 }
-
69 
-
70 // CUDA SDK does not provide specializations for T*
-
71 template <typename T>
-
72 inline __device__ T* shfl_xor(T* const val,
-
73  int laneMask, int width = kWarpSize) {
-
74  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
-
75  long long v = (long long) val;
-
76  return (T*) __shfl_xor(v, laneMask, width);
-
77 }
-
78 
-
79 #ifdef FAISS_USE_FLOAT16
-
80 inline __device__ half shfl(half v,
-
81  int srcLane, int width = kWarpSize) {
-
82  unsigned int vu = v.x;
-
83  vu = __shfl(vu, srcLane, width);
-
84 
-
85  half h;
-
86  h.x = (unsigned short) vu;
-
87  return h;
-
88 }
-
89 
-
90 inline __device__ half shfl_xor(half v,
-
91  int laneMask, int width = kWarpSize) {
-
92  unsigned int vu = v.x;
-
93  vu = __shfl_xor(vu, laneMask, width);
-
94 
-
95  half h;
-
96  h.x = (unsigned short) vu;
-
97  return h;
-
98 }
-
99 #endif
-
100 
-
101 } } // namespace
+
49 // CUDA SDK does not provide specializations for T*
+
50 template <typename T>
+
51 inline __device__ T* shfl_up(T* const val,
+
52  unsigned int delta, int width = kWarpSize) {
+
53  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
+
54  long long v = (long long) val;
+
55 
+
56  return (T*) shfl_up(v, delta, width);
+
57 }
+
58 
+
59 template <typename T>
+
60 inline __device__ T shfl_down(const T val,
+
61  unsigned int delta, int width = kWarpSize) {
+
62 #if CUDA_VERSION >= 9000
+
63  return __shfl_down_sync(0xffffffff, val, delta, width);
+
64 #else
+
65  return __shfl_down(val, delta, width);
+
66 #endif
+
67 }
+
68 
+
69 // CUDA SDK does not provide specializations for T*
+
70 template <typename T>
+
71 inline __device__ T* shfl_down(T* const val,
+
72  unsigned int delta, int width = kWarpSize) {
+
73  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
+
74  long long v = (long long) val;
+
75  return (T*) shfl_down(v, delta, width);
+
76 }
+
77 
+
78 template <typename T>
+
79 inline __device__ T shfl_xor(const T val,
+
80  int laneMask, int width = kWarpSize) {
+
81 #if CUDA_VERSION >= 9000
+
82  return __shfl_xor_sync(0xffffffff, val, laneMask, width);
+
83 #else
+
84  return __shfl_xor(val, laneMask, width);
+
85 #endif
+
86 }
+
87 
+
88 // CUDA SDK does not provide specializations for T*
+
89 template <typename T>
+
90 inline __device__ T* shfl_xor(T* const val,
+
91  int laneMask, int width = kWarpSize) {
+
92  static_assert(sizeof(T*) == sizeof(long long), "pointer size");
+
93  long long v = (long long) val;
+
94  return (T*) shfl_xor(v, laneMask, width);
+
95 }
+
96 
+
97 #ifdef FAISS_USE_FLOAT16
+
98 // CUDA 9.0 has half shuffle
+
99 #if CUDA_VERSION < 9000
+
100 inline __device__ half shfl(half v,
+
101  int srcLane, int width = kWarpSize) {
+
102  unsigned int vu = v.x;
+
103  vu = __shfl(vu, srcLane, width);
+
104 
+
105  half h;
+
106  h.x = (unsigned short) vu;
+
107  return h;
+
108 }
+
109 
+
110 inline __device__ half shfl_xor(half v,
+
111  int laneMask, int width = kWarpSize) {
+
112  unsigned int vu = v.x;
+
113  vu = __shfl_xor(vu, laneMask, width);
+
114 
+
115  half h;
+
116  h.x = (unsigned short) vu;
+
117  return h;
+
118 }
+
119 #endif // CUDA_VERSION
+
120 #endif // FAISS_USE_FLOAT16
+
121 
+
122 } } // namespace
diff --git a/docs/html/WriteIndex_8cpp_source.html b/docs/html/WriteIndex_8cpp_source.html index 3948f6f9c..6e1a2f535 100644 --- a/docs/html/WriteIndex_8cpp_source.html +++ b/docs/html/WriteIndex_8cpp_source.html @@ -193,7 +193,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
104 }
- + diff --git a/docs/html/annotated.html b/docs/html/annotated.html index 27a2cd324..b62d77be4 100644 --- a/docs/html/annotated.html +++ b/docs/html/annotated.html @@ -88,215 +88,235 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
Here are the classes, structs, unions and interfaces with brief descriptions:
[detail level 1234]
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
oNfaiss
|oNgpu
|oCAutoTuneCriterion
|oCOneRecallAtRCriterion
|oCIntersectionCriterion
|oCOperatingPoint
|oCOperatingPoints
|oCParameterRangePossible values of a parameter, sorted from least to most expensive/accurate
|oCParameterSpace
|oCRangeSearchResult
|oCIDSelector
|oCIDSelectorRange
|oCIDSelectorBatch
|oCBufferList
|oCRangeSearchPartialResultEntries in the buffers are split per query
oNfaiss
|oCClusteringParameters
|oCClustering
|oCFaissExceptionBase class for Faiss exceptions
|oCScopeDeleter
|oCScopeDeleter1
|oCHammingComputer4
|oCHammingComputer8
|oCHammingComputer16
|oCHammingComputer20
|oCHammingComputer32
|oCHammingComputer64
|oCHammingComputerDefault
|oCHammingComputerM8
|oCHammingComputerM4
|oCHammingComputer
|oCGenHammingComputer8
|oCGenHammingComputer16
|oCGenHammingComputer32
|oCGenHammingComputerM8
|oCCMax
|oCCMin
|oCHeapArray
|oCIndex
|oCScopeFileCloser
|oCCloner
|oCIndexFlat
|oCIndexFlatIP
|oCIndexFlatL2
|oCIndexFlatL2BaseShift
|oCIndexRefineFlat
|oCIndexFlat1DOptimized version for 1D "vectors"
|oCIndexIVF
|oCIndexIVFFlatStats
|oCIndexIVFFlat
|oCCodeCmp
|oCIndexIVFPQ
|oCIndexIVFPQStats
|oCIndexIVFPQR
|oCIndexIVFPQCompact
|oCIndexLSH
|oCArgSort
|oCSortedArray
|oCSemiSortedArray
|oCMinSumK
|oCIndexPQ
|oCIndexPQStats
|oCMultiIndexQuantizer
|oCScalarQuantizer
|oCIndexScalarQuantizer
|oCIndexIVFScalarQuantizer
|oCIndexIDMap
|oCIndexIDMap2
|oCIndexShards
|oCIndexSplitVectors
|oCScore3Computer
|oCIndirectSort
|oCRankingScore2
|oCSimulatedAnnealingParametersParameters used for the simulated annealing method
|oCPermutationObjectiveAbstract class for the loss function
|oCReproduceDistancesObjective
|oCSimulatedAnnealingOptimizerSimulated annealing optimization algorithm for permutations
|oCPolysemousTrainingOptimizes the order of indices in a ProductQuantizer
|oCProductQuantizer
|oCNopDistanceCorrection
|oCBaseShiftDistanceCorrection
|oCRandomGeneratorRandom generator that can be used in multithreaded contexts
|oCVectorTransform
|oCLinearTransform
|oCRandomRotationMatrixRandomly rotate a set of vectors
|oCPCAMatrix
|oCOPQMatrix
|oCRemapDimensionsTransform
|oCNormalizationTransform
|\CIndexPreTransform
oCOptions
\CTestFlatOptions
diff --git a/docs/html/classes.html b/docs/html/classes.html index 472a1b7f7..fe736fbbf 100644 --- a/docs/html/classes.html +++ b/docs/html/classes.html @@ -89,77 +89,80 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
A | B | C | D | F | G | H | I | K | L | M | N | O | P | Q | R | S | T | V | W
- - - + + + + - - - - + + + + + + + + - - - - - - - - - - - - + + + + + + + + - - - - - + + + + + + + + + + - - - - - - - - - - + + + + + + + + + - - - + + + - - - - - - + + + + + + - - - - - - + +
  A  
-		FinalBlockMerge (faiss::gpu)   IndexFlat1D (faiss)   LoadCode32< 2 > (faiss::gpu)   RandomRotationMatrix (faiss)   
FinalBlockMerge< 1, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexFlatIP (faiss)   LoadCode32< 20 > (faiss::gpu)   StackDeviceMemory::Range (faiss::gpu)   
ArgSort (faiss)   FinalBlockMerge< 2, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexFlatL2 (faiss)   LoadCode32< 24 > (faiss::gpu)   RangeSearchPartialResult (faiss)   
AutoTuneCriterion (faiss)   FinalBlockMerge< 4, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexFlatL2BaseShift (faiss)   LoadCode32< 28 > (faiss::gpu)   RangeSearchResult (faiss)   
  F  
+		IDSelectorRange (faiss)   Limits< Pair< K, V > > (faiss::gpu)   
  R  
+
Index (faiss)   LinearTransform (faiss)   
ArgSort (faiss)   FaissException (faiss)   Index2Layer (faiss)   LoadCode32 (faiss::gpu)   RandomGenerator (faiss)   
AutoTuneCriterion (faiss)   FinalBlockMerge (faiss::gpu)   IndexFlat (faiss)   LoadCode32< 1 > (faiss::gpu)   RandomRotationMatrix (faiss)   
  B  
-		FinalBlockMerge< 8, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexIDMap (faiss)   LoadCode32< 3 > (faiss::gpu)   RankingScore2 (faiss)   
FlatIndex (faiss::gpu)   IndexIDMap2 (faiss)   LoadCode32< 32 > (faiss::gpu)   RemapDimensionsTransform (faiss)   
BaseShiftDistanceCorrection (faiss)   
  G  
-		IndexIVF (faiss)   LoadCode32< 4 > (faiss::gpu)   ReproduceDistancesObjective (faiss)   
BitonicMergeStep (faiss::gpu)   IndexIVFFlat (faiss)   LoadCode32< 40 > (faiss::gpu)   RestrictPtrTraits (faiss::gpu::traits)   
BitonicMergeStep< K, V, 1, Dir, Comp, Low, true > (faiss::gpu)   GenHammingComputer16 (faiss)   IndexIVFFlatStats (faiss)   LoadCode32< 48 > (faiss::gpu)   
  S  
+		FinalBlockMerge< 1, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexFlat1D (faiss)   LoadCode32< 12 > (faiss::gpu)   StackDeviceMemory::Range (faiss::gpu)   
FinalBlockMerge< 2, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexFlatIP (faiss)   LoadCode32< 16 > (faiss::gpu)   RangeSearchPartialResult (faiss)   
BaseShiftDistanceCorrection (faiss)   FinalBlockMerge< 4, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexFlatL2 (faiss)   LoadCode32< 2 > (faiss::gpu)   RangeSearchResult (faiss)   
BitonicMergeStep (faiss::gpu)   FinalBlockMerge< 8, NumThreads, K, V, NumWarpQ, Dir, Comp > (faiss::gpu)   IndexFlatL2BaseShift (faiss)   LoadCode32< 20 > (faiss::gpu)   RankingScore2 (faiss)   
BitonicMergeStep< K, V, 1, Dir, Comp, Low, true > (faiss::gpu)   FlatIndex (faiss::gpu)   IndexHNSW (faiss)   LoadCode32< 24 > (faiss::gpu)   ReconstructFromNeighbors (faiss)   
BitonicMergeStep< K, V, N, Dir, Comp, false, false > (faiss::gpu)   FlatL2Dis (faiss)   IndexHNSW2Level (faiss)   LoadCode32< 28 > (faiss::gpu)   RemapDimensionsTransform (faiss)   
BitonicMergeStep< K, V, N, Dir, Comp, Low, true > (faiss::gpu)   
  G  
+		IndexHNSWFlat (faiss)   LoadCode32< 3 > (faiss::gpu)   ReproduceDistancesObjective (faiss)   
BitonicMergeStep< K, V, N, Dir, Comp, true, false > (faiss::gpu)   IndexHNSWPQ (faiss)   LoadCode32< 32 > (faiss::gpu)   RestrictPtrTraits (faiss::gpu::traits)   
BitonicSortStep (faiss::gpu)   GenericDistanceComputer (faiss)   IndexHNSWSQ (faiss)   LoadCode32< 4 > (faiss::gpu)   
  S  
BitonicMergeStep< K, V, N, Dir, Comp, false, false > (faiss::gpu)   GenHammingComputer32 (faiss)   IndexIVFPQ (faiss)   LoadCode32< 56 > (faiss::gpu)   
BitonicMergeStep< K, V, N, Dir, Comp, Low, true > (faiss::gpu)   GenHammingComputer8 (faiss)   IndexIVFPQCompact (faiss)   LoadCode32< 64 > (faiss::gpu)   ScalarQuantizer (faiss)   
BitonicMergeStep< K, V, N, Dir, Comp, true, false > (faiss::gpu)   GenHammingComputerM8 (faiss)   IndexIVFPQR (faiss)   LoadCode32< 8 > (faiss::gpu)   ScopeDeleter (faiss)   
BitonicSortStep (faiss::gpu)   GpuClonerOptions (faiss::gpu)   IndexIVFPQStats (faiss)   LoadCode32< 96 > (faiss::gpu)   ScopeDeleter1 (faiss)   
BitonicSortStep< K, V, 1, Dir, Comp > (faiss::gpu)   GpuIndex (faiss::gpu)   IndexIVFScalarQuantizer (faiss)   LoadCodeDistances (faiss::gpu)   ScopeFileCloser (faiss)   
BlockMerge (faiss::gpu)   GpuIndexConfig (faiss::gpu)   IndexLSH (faiss)   LoadStore (faiss::gpu)   Score3Computer (faiss)   
BlockMerge< NumThreads, K, V, N, L, Dir, Comp, false, FullMerge > (faiss::gpu)   GpuIndexFlat (faiss::gpu)   IndexPQ (faiss)   
  M  
-		SegmentedReduce (faiss::gpu)   
BlockMerge< NumThreads, K, V, N, L, Dir, Comp, true, FullMerge > (faiss::gpu)   GpuIndexFlatConfig (faiss::gpu)   IndexPQStats (faiss)   SemiSortedArray (faiss)   
BlockSelect (faiss::gpu)   GpuIndexFlatIP (faiss::gpu)   IndexPreTransform (faiss)   Math (faiss::gpu)   SimulatedAnnealingOptimizer (faiss)   
BlockSelect< K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock > (faiss::gpu)   GpuIndexFlatL2 (faiss::gpu)   IndexProxy (faiss::gpu)   Math< float2 > (faiss::gpu)   SimulatedAnnealingParameters (faiss)   
BufferList::Buffer (faiss)   GpuIndexIVF (faiss::gpu)   IndexRefineFlat (faiss)   Math< float4 > (faiss::gpu)   SortedArray (faiss)   
BufferList (faiss)   GpuIndexIVFConfig (faiss::gpu)   IndexScalarQuantizer (faiss)   Max (faiss::gpu)   StackDeviceMemory::Stack (faiss::gpu)   
BitonicSortStep< K, V, 1, Dir, Comp > (faiss::gpu)   GenHammingComputer16 (faiss)   IndexIDMap (faiss)   LoadCode32< 40 > (faiss::gpu)   
BlockMerge (faiss::gpu)   GenHammingComputer32 (faiss)   IndexIDMap2 (faiss)   LoadCode32< 48 > (faiss::gpu)   ScalarQuantizer (faiss)   
BlockMerge< NumThreads, K, V, N, L, Dir, Comp, false, FullMerge > (faiss::gpu)   GenHammingComputer8 (faiss)   IndexIVF (faiss)   LoadCode32< 56 > (faiss::gpu)   ScopeDeleter (faiss)   
BlockMerge< NumThreads, K, V, N, L, Dir, Comp, true, FullMerge > (faiss::gpu)   GenHammingComputerM8 (faiss)   IndexIVFFlat (faiss)   LoadCode32< 64 > (faiss::gpu)   ScopeDeleter1 (faiss)   
BlockSelect (faiss::gpu)   GpuClonerOptions (faiss::gpu)   IndexIVFPQ (faiss)   LoadCode32< 8 > (faiss::gpu)   ScopeFileCloser (faiss)   
BlockSelect< K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock > (faiss::gpu)   GpuIndex (faiss::gpu)   IndexIVFPQCompact (faiss)   LoadCode32< 96 > (faiss::gpu)   Score3Computer (faiss)   
BufferList::Buffer (faiss)   GpuIndexConfig (faiss::gpu)   IndexIVFPQR (faiss)   LoadCodeDistances (faiss::gpu)   SegmentedReduce (faiss::gpu)   
BufferList (faiss)   GpuIndexFlat (faiss::gpu)   IndexIVFPQStats (faiss)   LoadStore (faiss::gpu)   SemiSortedArray (faiss)   
  C  
-		GpuIndexIVFFlat (faiss::gpu)   IndexShards (faiss)   Min (faiss::gpu)   StackDeviceMemory (faiss::gpu)   
GpuIndexIVFFlatConfig (faiss::gpu)   IndexSplitVectors (faiss)   MinSumK (faiss)   StandardGpuResources (faiss::gpu)   
Cloner (faiss)   GpuIndexIVFPQ (faiss::gpu)   IndexWrapper (faiss::gpu)   MultiIndexQuantizer (faiss)   SubTensor (faiss::gpu::detail)   
Clustering (faiss)   GpuIndexIVFPQConfig (faiss::gpu)   IndirectSort (faiss)   
  N  
-		SubTensor< TensorType, 0, PtrTraits > (faiss::gpu::detail)   
ClusteringParameters (faiss)   GpuMultipleClonerOptions (faiss::gpu)   IntersectionCriterion (faiss)   Sum (faiss::gpu)   
CMax (faiss)   GpuParameterSpace (faiss::gpu)   IntToLong (faiss::gpu)   NopDistanceCorrection (faiss)   
  T  
+		GpuIndexFlatConfig (faiss::gpu)   IndexIVFScalarQuantizer (faiss)   
  M  
+		SimulatedAnnealingOptimizer (faiss)   
GpuIndexFlatIP (faiss::gpu)   IndexIVFStats (faiss)   SimulatedAnnealingParameters (faiss)   
Cloner (faiss)   GpuIndexFlatL2 (faiss::gpu)   IndexLSH (faiss)   Math (faiss::gpu)   SortedArray (faiss)   
Clustering (faiss)   GpuIndexIVF (faiss::gpu)   IndexPQ (faiss)   Math< float2 > (faiss::gpu)   SQDis (faiss)   
ClusteringParameters (faiss)   GpuIndexIVFConfig (faiss::gpu)   IndexPQStats (faiss)   Math< float4 > (faiss::gpu)   StackDeviceMemory::Stack (faiss::gpu)   
CMax (faiss)   GpuIndexIVFFlat (faiss::gpu)   IndexPreTransform (faiss)   Max (faiss::gpu)   StackDeviceMemory (faiss::gpu)   
CMin (faiss)   GpuIndexIVFFlatConfig (faiss::gpu)   IndexProxy (faiss::gpu)   Min (faiss::gpu)   StandardGpuResources (faiss::gpu)   
CodeCmp (faiss)   GpuIndexIVFPQ (faiss::gpu)   IndexRefineFlat (faiss)   MinSumK (faiss)   SubTensor (faiss::gpu::detail)   
Comparator (faiss::gpu)   GpuIndexIVFPQConfig (faiss::gpu)   IndexScalarQuantizer (faiss)   MultiIndexQuantizer (faiss)   SubTensor< TensorType, 0, PtrTraits > (faiss::gpu::detail)   
Converter (faiss::gpu)   GpuMultipleClonerOptions (faiss::gpu)   IndexShards (faiss)   
  N  
+		Sum (faiss::gpu)   
Converter< float > (faiss::gpu)   GpuParameterSpace (faiss::gpu)   IndexSplitVectors (faiss)   
  T  
CMin (faiss)   GpuResources (faiss::gpu)   IVFBase (faiss::gpu)   NormalizationTransform (faiss)   
CodeCmp (faiss)   GpuResourcesThrustAllocator (faiss::gpu)   IVFFlat (faiss::gpu)   NoTypeTensor (faiss::gpu)   Tensor (faiss::gpu)   
Comparator (faiss::gpu)   
  H  
-		IVFFlatScan (faiss::gpu)   
  O  
-		TensorInfo (faiss::gpu)   
Converter (faiss::gpu)   IVFFlatScan< 0, L2, T > (faiss::gpu)   TensorInfoOffset (faiss::gpu)   
Converter< float > (faiss::gpu)   HammingComputer (faiss)   IVFFlatScan< 128, L2, float > (faiss::gpu)   OneRecallAtRCriterion (faiss)   TensorInfoOffset< T, IndexT,-1 > (faiss::gpu)   
ConvertTo (faiss::gpu)   HammingComputer16 (faiss)   IVFFlatScan< 256, L2, float > (faiss::gpu)   OperatingPoint (faiss)   TestFlatOptions   
ConvertTo< float > (faiss::gpu)   HammingComputer20 (faiss)   IVFFlatScan< 64, L2, float > (faiss::gpu)   OperatingPoints (faiss)   ToCPUCloner (faiss::gpu)   
ConvertTo< float2 > (faiss::gpu)   HammingComputer32 (faiss)   IVFFlatScan<-1, L2, T > (faiss::gpu)   OPQMatrix (faiss)   ToGpuCloner (faiss::gpu)   
ConvertTo< float4 > (faiss::gpu)   HammingComputer4 (faiss)   IVFPQ (faiss::gpu)   Options   ToGpuClonerMultiple (faiss::gpu)   
CpuTimer (faiss::gpu)   HammingComputer64 (faiss)   
  K  
-
  P  
-
  V  
+
ConvertTo (faiss::gpu)   GpuResources (faiss::gpu)   IndexWrapper (faiss::gpu)   NopDistanceCorrection (faiss)   
ConvertTo< float > (faiss::gpu)   GpuResourcesThrustAllocator (faiss::gpu)   IndirectSort (faiss)   NormalizationTransform (faiss)   Tensor (faiss::gpu)   
ConvertTo< float2 > (faiss::gpu)   
  H  
+		IntersectionCriterion (faiss)   NoTypeTensor (faiss::gpu)   TensorInfo (faiss::gpu)   
ConvertTo< float4 > (faiss::gpu)   IntToLong (faiss::gpu)   
  O  
+		TensorInfoOffset (faiss::gpu)   
CpuTimer (faiss::gpu)   HammingComputer (faiss)   IVFBase (faiss::gpu)   TensorInfoOffset< T, IndexT,-1 > (faiss::gpu)   
CublasGemm (faiss::gpu)   HammingComputer16 (faiss)   IVFFlat (faiss::gpu)   OneRecallAtRCriterion (faiss)   TestFlatOptions   
CublasGemm< float > (faiss::gpu)   HammingComputer20 (faiss)   IVFFlatScan (faiss::gpu)   OperatingPoint (faiss)   ToCPUCloner (faiss::gpu)   
CublasHandleScope (faiss::gpu)   HammingComputer32 (faiss)   IVFFlatScan< 0, L2, T > (faiss::gpu)   OperatingPoints (faiss)   ToGpuCloner (faiss::gpu)   
CudaEvent (faiss::gpu)   HammingComputer4 (faiss)   IVFFlatScan< 128, L2, float > (faiss::gpu)   OPQMatrix (faiss)   ToGpuClonerMultiple (faiss::gpu)   
  D  
+		HammingComputer64 (faiss)   IVFFlatScan< 256, L2, float > (faiss::gpu)   Options   
  V  
CublasGemm (faiss::gpu)   HammingComputer8 (faiss)   
CublasGemm< float > (faiss::gpu)   HammingComputerDefault (faiss)   KernelTimer (faiss::gpu)   Pair (faiss::gpu)   VectorTransform (faiss)   
CublasHandleScope (faiss::gpu)   HammingComputerM4 (faiss)   
  L  
+
HammingComputer8 (faiss)   IVFFlatScan< 64, L2, float > (faiss::gpu)   
  P  
+
DefaultPtrTraits (faiss::gpu::traits)   HammingComputerDefault (faiss)   IVFFlatScan<-1, L2, T > (faiss::gpu)   VectorTransform (faiss)   
DeviceMemory (faiss::gpu)   HammingComputerM4 (faiss)   IVFPQ (faiss::gpu)   Pair (faiss::gpu)   VisitedTable (faiss)   
DeviceMemoryReservation (faiss::gpu)   HammingComputerM8 (faiss)   
  K  
ParameterRange (faiss)   
  W  
CudaEvent (faiss::gpu)   HammingComputerM8 (faiss)   ParameterSpace (faiss)   
  D  
-		HeapArray (faiss)   Limits (faiss::gpu)   PCAMatrix (faiss)   WarpSelect (faiss::gpu)   
HostTensor (faiss::gpu)   Limits< float > (faiss::gpu)   PermutationObjective (faiss)   WarpSelect< K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock > (faiss::gpu)   
DefaultPtrTraits (faiss::gpu::traits)   
  I  
-		Limits< int > (faiss::gpu)   PolysemousTraining (faiss)   WorkerThread (faiss::gpu)   
DeviceMemory (faiss::gpu)   Limits< Pair< K, V > > (faiss::gpu)   ProductQuantizer (faiss)   
DeviceMemoryReservation (faiss::gpu)   IDSelector (faiss)   LinearTransform (faiss)   
  Q  
+
DeviceScope (faiss::gpu)   HeapArray (faiss)   ParameterSpace (faiss)   
DeviceTensor (faiss::gpu)   HNSW (faiss)   KernelTimer (faiss::gpu)   PCAMatrix (faiss)   WarpSelect (faiss::gpu)   
DeviceVector (faiss::gpu)   HNSWStats (faiss)   
  L  
+		PermutationObjective (faiss)   WarpSelect< K, V, Dir, Comp, 1, NumThreadQ, ThreadsPerBlock > (faiss::gpu)   
Distance2Level (faiss)   HostTensor (faiss::gpu)   PolysemousTraining (faiss)   WorkerThread (faiss::gpu)   
Distance2xXPQ4 (faiss)   
  I  
+		Level1Quantizer (faiss)   PQDis (faiss)   
ScalarQuantizer::DistanceComputer (faiss)   Limits (faiss::gpu)   ProductQuantizer (faiss)   
HNSW::DistanceComputer (faiss)   IDSelector (faiss)   Limits< float > (faiss::gpu)   
  Q  
DeviceScope (faiss::gpu)   IDSelectorBatch (faiss)   LoadCode32 (faiss::gpu)   
DeviceTensor (faiss::gpu)   IDSelectorRange (faiss)   LoadCode32< 1 > (faiss::gpu)   RangeSearchPartialResult::QueryResult (faiss)   
DeviceVector (faiss::gpu)   Index (faiss)   LoadCode32< 12 > (faiss::gpu)   
  R  
-
  F  
-		IndexFlat (faiss)   LoadCode32< 16 > (faiss::gpu)   
RandomGenerator (faiss)   
FaissException (faiss)   
DistanceXPQ4 (faiss)   IDSelectorBatch (faiss)   Limits< int > (faiss::gpu)   
RangeSearchPartialResult::QueryResult (faiss)   
A | B | C | D | F | G | H | I | K | L | M | N | O | P | Q | R | S | T | V | W
diff --git a/docs/html/classfaiss_1_1gpu_1_1DeviceTensor-members.html b/docs/html/classfaiss_1_1gpu_1_1DeviceTensor-members.html index d4861d27d..33b1b1ca0 100644 --- a/docs/html/classfaiss_1_1gpu_1_1DeviceTensor-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1DeviceTensor-members.html @@ -140,7 +140,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); NumDim enum value (defined in faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > numElements() const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > operator=(DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)faiss::gpu::DeviceTensor< T, Dim, InnerContig, IndexT, PtrTraits > - faiss::gpu::Tensor::operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + faiss::gpu::Tensor::operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > faiss::gpu::Tensor::operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > operator[](IndexT)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline operator[](IndexT) const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline @@ -149,8 +149,8 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); stride_faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >protected strides() const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline Tensor()faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, const IndexT sizes[Dim])faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, std::initializer_list< IndexT > sizes) (defined in faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, const IndexT sizes[Dim], const IndexT strides[Dim])faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > diff --git a/docs/html/classfaiss_1_1gpu_1_1DeviceTensor.html b/docs/html/classfaiss_1_1gpu_1_1DeviceTensor.html index 6568bd4e3..1a07bfb87 100644 --- a/docs/html/classfaiss_1_1gpu_1_1DeviceTensor.html +++ b/docs/html/classfaiss_1_1gpu_1_1DeviceTensor.html @@ -185,20 +185,20 @@ InnerContig, IndexT, PtrTraits > & Tensor ()  Default constructor.
  - -__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default - Copy constructor.
-  - -__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)=default - Move constructor.
-  - + +__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) + Copy constructor.
+  + +__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t) + Move constructor.
+  + __host__ __device__ Tensor< T,
Dim, InnerContig, IndexT,
-PtrTraits > & operator= (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default - Assignment.
-  +PtrTraits > & operator= (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) + Assignment.
+  __host__ __device__ Tensor< T,
Dim, InnerContig, IndexT,
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndex-members.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndex-members.html index cd5797477..40c6223f4 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndex-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndex-members.html @@ -106,7 +106,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); getResources() (defined in faiss::gpu::GpuIndex)faiss::gpu::GpuIndexinline GpuIndex(GpuResources *resources, int dims, faiss::MetricType metric, GpuIndexConfig config) (defined in faiss::gpu::GpuIndex)faiss::gpu::GpuIndex idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit is_trainedfaiss::Index memorySpace_faiss::gpu::GpuIndexprotected metric_typefaiss::Index @@ -118,10 +118,11 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset()=0faiss::Indexpure virtual resources_faiss::gpu::GpuIndexprotected search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexvirtual - searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const =0faiss::gpu::GpuIndexprotectedpure virtual - train(idx_t, const float *)faiss::Indexinlinevirtual - verbosefaiss::Index - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const =0faiss::gpu::GpuIndexprotectedpure virtual + train(idx_t n, const float *x)faiss::Indexvirtual + verbosefaiss::Index + ~Index() (defined in faiss::Index)faiss::Indexvirtual
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndex.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndex.html index 9bc8c5580..ba2872602 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndex.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndex.html @@ -129,11 +129,11 @@ int getDevice () const void search (faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const override   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  -virtual void train (idx_t, const float *) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  +virtual void train (idx_t n, const float *x) +  virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const   void assign (idx_t n, const float *x, idx_t *labels, idx_t k=1) @@ -148,6 +148,8 @@ virtual void   virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat-members.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat-members.html index 969c73777..56e7829bd 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat-members.html @@ -115,7 +115,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); GpuIndexFlat(GpuResources *resources, const faiss::IndexFlat *index, GpuIndexFlatConfig config=GpuIndexFlatConfig())faiss::gpu::GpuIndexFlat GpuIndexFlat(GpuResources *resources, int dims, faiss::MetricType metric, GpuIndexFlatConfig config=GpuIndexFlatConfig())faiss::gpu::GpuIndexFlat idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit is_trainedfaiss::Index memorySpace_faiss::gpu::GpuIndexprotected metric_typefaiss::Index @@ -128,14 +128,15 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset() overridefaiss::gpu::GpuIndexFlatvirtual resources_faiss::gpu::GpuIndexprotected search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatvirtual - searchFromCpuPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const faiss::gpu::GpuIndexFlatprotected - searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatprotectedvirtual - searchNonPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlatprotected - setMinPagingSize(size_t size)faiss::gpu::GpuIndexFlat - train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexFlatvirtual - verbosefaiss::Index - ~GpuIndexFlat() override (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlat - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + searchFromCpuPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const faiss::gpu::GpuIndexFlatprotected + searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatprotectedvirtual + searchNonPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlatprotected + setMinPagingSize(size_t size)faiss::gpu::GpuIndexFlat + train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexFlatvirtual + verbosefaiss::Index + ~GpuIndexFlat() override (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlat + ~Index() (defined in faiss::Index)faiss::Indexvirtual
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat.html index cdcb2b105..71b7b0b75 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlat.html @@ -169,15 +169,17 @@ int getDevice () const void add_with_ids (Index::idx_t n, const float *x, const Index::idx_t *ids) override   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const   void assign (idx_t n, const float *x, idx_t *labels, idx_t k=1)   virtual long remove_ids (const IDSelector &sel)   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP-members.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP-members.html index db371d02b..7052d32ea 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP-members.html @@ -119,7 +119,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); GpuIndexFlatIP(GpuResources *resources, faiss::IndexFlatIP *index, GpuIndexFlatConfig config=GpuIndexFlatConfig())faiss::gpu::GpuIndexFlatIP GpuIndexFlatIP(GpuResources *resources, int dims, GpuIndexFlatConfig config=GpuIndexFlatConfig())faiss::gpu::GpuIndexFlatIP idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit is_trainedfaiss::Index memorySpace_faiss::gpu::GpuIndexprotected metric_typefaiss::Index @@ -132,14 +132,15 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset() overridefaiss::gpu::GpuIndexFlatvirtual resources_faiss::gpu::GpuIndexprotected search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatvirtual - searchFromCpuPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const faiss::gpu::GpuIndexFlatprotected - searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatprotectedvirtual - searchNonPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlatprotected - setMinPagingSize(size_t size)faiss::gpu::GpuIndexFlat - train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexFlatvirtual - verbosefaiss::Index - ~GpuIndexFlat() override (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlat - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + searchFromCpuPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const faiss::gpu::GpuIndexFlatprotected + searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatprotectedvirtual + searchNonPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlatprotected + setMinPagingSize(size_t size)faiss::gpu::GpuIndexFlat + train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexFlatvirtual + verbosefaiss::Index + ~GpuIndexFlat() override (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlat + ~Index() (defined in faiss::Index)faiss::Indexvirtual
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP.html index 085f67e13..6be0064bc 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatIP.html @@ -177,15 +177,17 @@ int getDevice () const void add_with_ids (Index::idx_t n, const float *x, const Index::idx_t *ids) override   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const   void assign (idx_t n, const float *x, idx_t *labels, idx_t k=1)   virtual long remove_ids (const IDSelector &sel)   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2-members.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2-members.html index b7fa62d24..418d123ba 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2-members.html @@ -119,7 +119,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); GpuIndexFlatL2(GpuResources *resources, faiss::IndexFlatL2 *index, GpuIndexFlatConfig config=GpuIndexFlatConfig())faiss::gpu::GpuIndexFlatL2 GpuIndexFlatL2(GpuResources *resources, int dims, GpuIndexFlatConfig config=GpuIndexFlatConfig())faiss::gpu::GpuIndexFlatL2 idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit is_trainedfaiss::Index memorySpace_faiss::gpu::GpuIndexprotected metric_typefaiss::Index @@ -132,14 +132,15 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset() overridefaiss::gpu::GpuIndexFlatvirtual resources_faiss::gpu::GpuIndexprotected search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatvirtual - searchFromCpuPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const faiss::gpu::GpuIndexFlatprotected - searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatprotectedvirtual - searchNonPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlatprotected - setMinPagingSize(size_t size)faiss::gpu::GpuIndexFlat - train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexFlatvirtual - verbosefaiss::Index - ~GpuIndexFlat() override (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlat - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + searchFromCpuPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const faiss::gpu::GpuIndexFlatprotected + searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexFlatprotectedvirtual + searchNonPaged_(int n, const float *x, int k, float *outDistancesData, int *outIndicesData) const (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlatprotected + setMinPagingSize(size_t size)faiss::gpu::GpuIndexFlat + train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexFlatvirtual + verbosefaiss::Index + ~GpuIndexFlat() override (defined in faiss::gpu::GpuIndexFlat)faiss::gpu::GpuIndexFlat + ~Index() (defined in faiss::Index)faiss::Indexvirtual
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2.html index 366ac958b..fd09ac7fd 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexFlatL2.html @@ -177,15 +177,17 @@ int getDevice () const void add_with_ids (Index::idx_t n, const float *x, const Index::idx_t *ids) override   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const   void assign (idx_t n, const float *x, idx_t *labels, idx_t k=1)   virtual long remove_ids (const IDSelector &sel)   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF-members.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF-members.html index 0463f26c0..23ba22049 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF-members.html @@ -113,7 +113,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); GpuIndex(GpuResources *resources, int dims, faiss::MetricType metric, GpuIndexConfig config) (defined in faiss::gpu::GpuIndex)faiss::gpu::GpuIndex GpuIndexIVF(GpuResources *resources, int dims, faiss::MetricType metric, int nlist, GpuIndexIVFConfig config=GpuIndexIVFConfig()) (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVF idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit is_trainedfaiss::Index ivfConfig_ (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected memorySpace_faiss::gpu::GpuIndexprotected @@ -129,13 +129,14 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset()=0faiss::Indexpure virtual resources_faiss::gpu::GpuIndexprotected search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexvirtual - searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const =0faiss::gpu::GpuIndexprotectedpure virtual - setNumProbes(int nprobe)faiss::gpu::GpuIndexIVF - train(idx_t, const float *)faiss::Indexinlinevirtual - trainQuantizer_(faiss::Index::idx_t n, const float *x) (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected - verbosefaiss::Index - ~GpuIndexIVF() override (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVF - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const =0faiss::gpu::GpuIndexprotectedpure virtual + setNumProbes(int nprobe)faiss::gpu::GpuIndexIVF + train(idx_t n, const float *x)faiss::Indexvirtual + trainQuantizer_(faiss::Index::idx_t n, const float *x) (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected + verbosefaiss::Index + ~GpuIndexIVF() override (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVF + ~Index() (defined in faiss::Index)faiss::Indexvirtual
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF.html index 0ad15ed47..7082a7b51 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVF.html @@ -155,11 +155,11 @@ int getDevice () const void search (faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const override   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  -virtual void train (idx_t, const float *) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  +virtual void train (idx_t n, const float *x) +  virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const   void assign (idx_t n, const float *x, idx_t *labels, idx_t k=1) @@ -174,6 +174,8 @@ virtual void   virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const @@ -300,7 +302,7 @@ typedef long Reimplemented from faiss::gpu::GpuIndex.

-

Definition at line 215 of file GpuIndexIVF.cu.

+

Definition at line 216 of file GpuIndexIVF.cu.

diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat-members.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat-members.html index 15390ed2b..0cda936fc 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat-members.html @@ -117,7 +117,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); GpuIndexIVFFlat(GpuResources *resources, const faiss::IndexIVFFlat *index, GpuIndexIVFFlatConfig config=GpuIndexIVFFlatConfig())faiss::gpu::GpuIndexIVFFlat GpuIndexIVFFlat(GpuResources *resources, int dims, int nlist, faiss::MetricType metric, GpuIndexIVFFlatConfig config=GpuIndexIVFFlatConfig())faiss::gpu::GpuIndexIVFFlat idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit is_trainedfaiss::Index ivfConfig_ (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected memorySpace_faiss::gpu::GpuIndexprotected @@ -135,14 +135,15 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset() overridefaiss::gpu::GpuIndexIVFFlatvirtual resources_faiss::gpu::GpuIndexprotected search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexvirtual - searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexIVFFlatprotectedvirtual - setNumProbes(int nprobe)faiss::gpu::GpuIndexIVF - train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexIVFFlatvirtual - trainQuantizer_(faiss::Index::idx_t n, const float *x) (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected - verbosefaiss::Index - ~GpuIndexIVF() override (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVF - ~GpuIndexIVFFlat() override (defined in faiss::gpu::GpuIndexIVFFlat)faiss::gpu::GpuIndexIVFFlat - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexIVFFlatprotectedvirtual + setNumProbes(int nprobe)faiss::gpu::GpuIndexIVF + train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexIVFFlatvirtual + trainQuantizer_(faiss::Index::idx_t n, const float *x) (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected + verbosefaiss::Index + ~GpuIndexIVF() override (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVF + ~GpuIndexIVFFlat() override (defined in faiss::gpu::GpuIndexIVFFlat)faiss::gpu::GpuIndexIVFFlat + ~Index() (defined in faiss::Index)faiss::Indexvirtual
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat.html index 3855d778c..ab6634f12 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFFlat.html @@ -175,9 +175,9 @@ int getDevice () const void search (faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const override   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const   void assign (idx_t n, const float *x, idx_t *labels, idx_t k=1) @@ -188,6 +188,8 @@ int getDevice () const   virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const @@ -427,13 +429,13 @@ const MemorySpace void faiss::gpu::GpuIndexIVFFlat::train ( Index::idx_t  - , + n, const float *  -   + x  @@ -456,7 +458,7 @@ const MemorySpace faiss::Index.

+

Reimplemented from faiss::Index.

Definition at line 177 of file GpuIndexIVFFlat.cu.

diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ-members.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ-members.html index 3c95d73b6..e60dcab57 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ-members.html @@ -124,7 +124,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); GpuIndexIVFPQ(GpuResources *resources, const faiss::IndexIVFPQ *index, GpuIndexIVFPQConfig config=GpuIndexIVFPQConfig())faiss::gpu::GpuIndexIVFPQ GpuIndexIVFPQ(GpuResources *resources, int dims, int nlist, int subQuantizers, int bitsPerCode, faiss::MetricType metric, GpuIndexIVFPQConfig config=GpuIndexIVFPQConfig())faiss::gpu::GpuIndexIVFPQ idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit is_trainedfaiss::Index ivfConfig_ (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected memorySpace_faiss::gpu::GpuIndexprotected @@ -142,15 +142,16 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset() overridefaiss::gpu::GpuIndexIVFPQvirtual resources_faiss::gpu::GpuIndexprotected search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexvirtual - searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexIVFPQprotectedvirtual - setNumProbes(int nprobe)faiss::gpu::GpuIndexIVF - setPrecomputedCodes(bool enable)faiss::gpu::GpuIndexIVFPQ - train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexIVFPQvirtual - trainQuantizer_(faiss::Index::idx_t n, const float *x) (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected - verbosefaiss::Index - ~GpuIndexIVF() override (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVF - ~GpuIndexIVFPQ() override (defined in faiss::gpu::GpuIndexIVFPQ)faiss::gpu::GpuIndexIVFPQ - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + searchImpl_(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::GpuIndexIVFPQprotectedvirtual + setNumProbes(int nprobe)faiss::gpu::GpuIndexIVF + setPrecomputedCodes(bool enable)faiss::gpu::GpuIndexIVFPQ + train(Index::idx_t n, const float *x) overridefaiss::gpu::GpuIndexIVFPQvirtual + trainQuantizer_(faiss::Index::idx_t n, const float *x) (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVFprotected + verbosefaiss::Index + ~GpuIndexIVF() override (defined in faiss::gpu::GpuIndexIVF)faiss::gpu::GpuIndexIVF + ~GpuIndexIVFPQ() override (defined in faiss::gpu::GpuIndexIVFPQ)faiss::gpu::GpuIndexIVFPQ + ~Index() (defined in faiss::Index)faiss::Indexvirtual
diff --git a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ.html b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ.html index 1266bbb69..1198a03e3 100644 --- a/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ.html +++ b/docs/html/classfaiss_1_1gpu_1_1GpuIndexIVFPQ.html @@ -204,9 +204,9 @@ int getDevice () const void search (faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const override   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const   void assign (idx_t n, const float *x, idx_t *labels, idx_t k=1) @@ -217,6 +217,8 @@ int getDevice () const   virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const @@ -494,13 +496,13 @@ const MemorySpace void faiss::gpu::GpuIndexIVFPQ::train ( Index::idx_t  - , + n, const float *  -   + x  @@ -523,7 +525,7 @@ const MemorySpace faiss::Index.

+

Reimplemented from faiss::Index.

Definition at line 300 of file GpuIndexIVFPQ.cu.

diff --git a/docs/html/classfaiss_1_1gpu_1_1HostTensor-members.html b/docs/html/classfaiss_1_1gpu_1_1HostTensor-members.html index 38f9004fc..2a669888a 100644 --- a/docs/html/classfaiss_1_1gpu_1_1HostTensor-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1HostTensor-members.html @@ -137,7 +137,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); narrowOutermost(IndexT start, IndexT size)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > NumDim enum value (defined in faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > numElements() const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > operator[](IndexT)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline operator[](IndexT) const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline @@ -146,8 +146,8 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); stride_faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >protected strides() const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline Tensor()faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, const IndexT sizes[Dim])faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, std::initializer_list< IndexT > sizes) (defined in faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, const IndexT sizes[Dim], const IndexT strides[Dim])faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > diff --git a/docs/html/classfaiss_1_1gpu_1_1HostTensor.html b/docs/html/classfaiss_1_1gpu_1_1HostTensor.html index f37628732..60a9ef19e 100644 --- a/docs/html/classfaiss_1_1gpu_1_1HostTensor.html +++ b/docs/html/classfaiss_1_1gpu_1_1HostTensor.html @@ -175,20 +175,20 @@ __host__ bool Tensor ()  Default constructor.
  - -__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default - Copy constructor.
-  - -__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)=default - Move constructor.
-  - + +__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) + Copy constructor.
+  + +__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t) + Move constructor.
+  + __host__ __device__ Tensor< T,
Dim, InnerContig, IndexT,
-PtrTraits > & operator= (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default - Assignment.
-  +PtrTraits > & operator= (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) + Assignment.
+  __host__ __device__ Tensor< T,
Dim, InnerContig, IndexT,
diff --git a/docs/html/classfaiss_1_1gpu_1_1IVFFlat.html b/docs/html/classfaiss_1_1gpu_1_1IVFFlat.html index 6e33a878e..4f68f9109 100644 --- a/docs/html/classfaiss_1_1gpu_1_1IVFFlat.html +++ b/docs/html/classfaiss_1_1gpu_1_1IVFFlat.html @@ -310,7 +310,7 @@ std::vector< std::unique_ptr

Add vectors to a specific list; the input data can be on the host or on our current device

-

Definition at line 58 of file IVFFlat.cu.

+

Definition at line 54 of file IVFFlat.cu.

@@ -339,7 +339,7 @@ std::vector< std::unique_ptr

Adds the given vectors to this index. The input data must be on our current device. Returns the number of vectors successfully added. Vectors may not be able to be added because they contain NaNs.

-

Definition at line 130 of file IVFFlat.cu.

+

Definition at line 129 of file IVFFlat.cu.

@@ -386,7 +386,7 @@ std::vector< std::unique_ptr

Find the approximate k nearest neigbors for queries against our database

-

Definition at line 288 of file IVFFlat.cu.

+

Definition at line 287 of file IVFFlat.cu.

diff --git a/docs/html/classfaiss_1_1gpu_1_1IndexProxy-members.html b/docs/html/classfaiss_1_1gpu_1_1IndexProxy-members.html index 5b61c5867..383f676cc 100644 --- a/docs/html/classfaiss_1_1gpu_1_1IndexProxy-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1IndexProxy-members.html @@ -104,7 +104,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); dfaiss::Index display() const faiss::Index idx_t typedeffaiss::Index - Index(idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) (defined in faiss::Index)faiss::Indexinlineexplicit + Index(idx_t d=0, MetricType metric=METRIC_L2) (defined in faiss::Index)faiss::Indexinlineexplicit IndexProxy() (defined in faiss::gpu::IndexProxy)faiss::gpu::IndexProxy is_trainedfaiss::Index metric_typefaiss::Index @@ -118,10 +118,11 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); reset() overridefaiss::gpu::IndexProxyvirtual runOnIndex(std::function< void(faiss::Index *)> f)faiss::gpu::IndexProxy search(faiss::Index::idx_t n, const float *x, faiss::Index::idx_t k, float *distances, faiss::Index::idx_t *labels) const overridefaiss::gpu::IndexProxyvirtual - train(Index::idx_t n, const float *x) overridefaiss::gpu::IndexProxyvirtual - verbosefaiss::Index - ~Index() (defined in faiss::Index)faiss::Indexinlinevirtual - ~IndexProxy() override (defined in faiss::gpu::IndexProxy)faiss::gpu::IndexProxy + search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const faiss::Indexvirtual + train(Index::idx_t n, const float *x) overridefaiss::gpu::IndexProxyvirtual + verbosefaiss::Index + ~Index() (defined in faiss::Index)faiss::Indexvirtual + ~IndexProxy() override (defined in faiss::gpu::IndexProxy)faiss::gpu::IndexProxy
diff --git a/docs/html/classfaiss_1_1gpu_1_1IndexProxy.html b/docs/html/classfaiss_1_1gpu_1_1IndexProxy.html index 3e9ebcb81..1e1ff34c2 100644 --- a/docs/html/classfaiss_1_1gpu_1_1IndexProxy.html +++ b/docs/html/classfaiss_1_1gpu_1_1IndexProxy.html @@ -136,9 +136,9 @@ int count () const faiss::Index * at (int i) const   - Public Member Functions inherited from faiss::Index - - Index (idx_t d=0, MetricType metric=METRIC_INNER_PRODUCT) -  + + Index (idx_t d=0, MetricType metric=METRIC_L2) +  virtual void add_with_ids (idx_t n, const float *x, const long *xids)   virtual void range_search (idx_t n, const float *x, float radius, RangeSearchResult *result) const @@ -149,6 +149,8 @@ const faiss::Index *   virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const   +virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const +  void compute_residual (const float *x, float *residual, idx_t key) const   void display () const @@ -408,7 +410,7 @@ typedef long 

faiss::Index API All indices receive the same call

-

Reimplemented from faiss::Index.

+

Reimplemented from faiss::Index.

Definition at line 106 of file IndexProxy.cpp.

diff --git a/docs/html/classfaiss_1_1gpu_1_1Tensor-members.html b/docs/html/classfaiss_1_1gpu_1_1Tensor-members.html index c8a4c75dc..1eba55895 100644 --- a/docs/html/classfaiss_1_1gpu_1_1Tensor-members.html +++ b/docs/html/classfaiss_1_1gpu_1_1Tensor-members.html @@ -128,7 +128,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); narrowOutermost(IndexT start, IndexT size)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > NumDim enum value (defined in faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > numElements() const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > operator=(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > operator[](IndexT)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline operator[](IndexT) const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline @@ -137,8 +137,8 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); stride_faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >protected strides() const faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >inline Tensor()faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > - Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)=defaultfaiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > + Tensor(Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, const IndexT sizes[Dim])faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, std::initializer_list< IndexT > sizes) (defined in faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits >)faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > Tensor(DataPtrType data, const IndexT sizes[Dim], const IndexT strides[Dim])faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > diff --git a/docs/html/classfaiss_1_1gpu_1_1Tensor.html b/docs/html/classfaiss_1_1gpu_1_1Tensor.html index f1afd52e7..13736544c 100644 --- a/docs/html/classfaiss_1_1gpu_1_1Tensor.html +++ b/docs/html/classfaiss_1_1gpu_1_1Tensor.html @@ -137,20 +137,20 @@ Public Member Functions __host__ __device__ Tensor ()  Default constructor.
  - -__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default - Copy constructor.
-  - -__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t)=default - Move constructor.
-  - + +__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) + Copy constructor.
+  + +__host__ __device__ Tensor (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &&t) + Move constructor.
+  + __host__ __device__ Tensor< T,
Dim, InnerContig, IndexT,
-PtrTraits > & operator= (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t)=default - Assignment.
-  +PtrTraits > & operator= (Tensor< T, Dim, InnerContig, IndexT, PtrTraits > &t) + Assignment.
+  __host__ __device__ Tensor< T,
Dim, InnerContig, IndexT,
@@ -402,7 +402,7 @@ template<typename T, int Dim, bool InnerContig, typename IndexT, template<

Constructor that takes arbitrary size/stride arrays. Errors if you attempt to pass non-contiguous strides to a contiguous tensor.

-

Definition at line 87 of file Tensor-inl.cuh.

+

Definition at line 118 of file Tensor-inl.cuh.

@@ -425,7 +425,7 @@ template<typename NewIndexT >

Returns true if we can use this indexing type to access all elements index type

-

Definition at line 325 of file Tensor-inl.cuh.

+

Definition at line 356 of file Tensor-inl.cuh.

@@ -447,7 +447,7 @@ template<typename U >

Cast to a tensor of a different type of the same size and stride. U and our type T must be of the same size

-

Definition at line 222 of file Tensor-inl.cuh.

+

Definition at line 253 of file Tensor-inl.cuh.

@@ -469,7 +469,7 @@ template<typename NewIndexT >

Attempts to cast this tensor to a tensor of a different IndexT. Fails if size or stride entries are not representable in the new IndexT.

-

Definition at line 305 of file Tensor-inl.cuh.

+

Definition at line 336 of file Tensor-inl.cuh.

@@ -491,7 +491,7 @@ template<typename U >

Cast to a tensor of a different type which is potentially a different size than our type T. Tensor must be aligned and the innermost dimension must be a size that is a multiple of sizeof(U) / sizeof(T), and the stride of the innermost dimension must be contiguous. The stride of all outer dimensions must be a multiple of sizeof(U) / sizeof(T) as well.

-

Definition at line 244 of file Tensor-inl.cuh.

+

Definition at line 275 of file Tensor-inl.cuh.

@@ -513,7 +513,7 @@ template<int NewDim>

Downcast a tensor of dimension D to some tensor of dimension D' < D by collapsing the leading dimensions. asserts if there is padding on the leading dimensions.

-

Definition at line 555 of file Tensor-inl.cuh.

+

Definition at line 586 of file Tensor-inl.cuh.

@@ -535,7 +535,7 @@ template<int NewDim>

Downcast a tensor of dimension D to some tensor of dimension D' < D by collapsing the leading dimensions. asserts if there is padding on the leading dimensions.

-

Definition at line 510 of file Tensor-inl.cuh.

+

Definition at line 541 of file Tensor-inl.cuh.

@@ -563,7 +563,7 @@ template<typename T, int Dim, bool InnerContig, typename IndexT, template<

Returns a raw pointer to the end of our data, assuming continuity

-

Definition at line 184 of file Tensor.cuh.

+

Definition at line 182 of file Tensor.cuh.

@@ -591,7 +591,7 @@ template<typename T, int Dim, bool InnerContig, typename IndexT, template<

Returns a raw pointer to the end of our data, assuming continuity (const)

-

Definition at line 196 of file Tensor.cuh.

+

Definition at line 194 of file Tensor.cuh.

@@ -620,7 +620,7 @@ template<typename T, int Dim, bool InnerContig, typename IndexT, template<

Returns the size of a given dimension, [0, Dim - 1]. No bounds checking.

-

Definition at line 226 of file Tensor.cuh.

+

Definition at line 224 of file Tensor.cuh.

@@ -648,7 +648,7 @@ template<typename T, int Dim, bool InnerContig, typename IndexT, template<

If we are contiguous, returns the total size in bytes of our data

-

Definition at line 242 of file Tensor.cuh.

+

Definition at line 240 of file Tensor.cuh.

@@ -677,7 +677,7 @@ template<typename T, int Dim, bool InnerContig, typename IndexT, template<

Returns the stride of a given dimension, [0, Dim - 1]. No bounds checking.

-

Definition at line 232 of file Tensor.cuh.

+

Definition at line 230 of file Tensor.cuh.

@@ -698,7 +698,7 @@ template<typename T , int Dim, bool InnerContig, typename IndexT , template&l

Returns whether a given dimension has only increasing stride from the previous dimension. A tensor that was permuted by exchanging size and stride only will fail this check. If i == 0 just check size > 0. Returns false if stride is <= 0.

-

Definition at line 384 of file Tensor-inl.cuh.

+

Definition at line 415 of file Tensor-inl.cuh.

@@ -719,7 +719,7 @@ template<typename T , int Dim, bool InnerContig, typename IndexT , template&l

Returns true if there is no padding within the tensor and no re-ordering of the dimensions.

(stride(i) == size(i + 1) * stride(i + 1)) && stride(dim - 1) == 0
-

Definition at line 365 of file Tensor-inl.cuh.

+

Definition at line 396 of file Tensor-inl.cuh.

@@ -742,7 +742,7 @@ template<typename OtherT , int OtherDim>

Returns true if the two tensors are of the same dimensionality, size and stride.

-

Definition at line 180 of file Tensor-inl.cuh.

+

Definition at line 211 of file Tensor-inl.cuh.

@@ -779,7 +779,7 @@ template<typename T , int Dim, bool InnerContig, typename IndexT, template<

Returns a tensor of the same dimension that is a view of the original tensor with the specified dimension restricted to the elements in the range [start, start + size). Can occur in an arbitrary dimension

-

Definition at line 633 of file Tensor-inl.cuh.

+

Definition at line 664 of file Tensor-inl.cuh.

@@ -810,7 +810,7 @@ template<typename T , int Dim, bool InnerContig, typename IndexT, template<

Returns a tensor of the same dimension that is a view of the original tensor with the specified dimension restricted to the elements in the range [start, start + size)

-

Definition at line 625 of file Tensor-inl.cuh.

+

Definition at line 656 of file Tensor-inl.cuh.

@@ -830,7 +830,7 @@ template<typename T , int Dim, bool InnerContig, typename IndexT , template&l

Returns the total number of elements contained within our data (product of getSize(i))

-

Definition at line 352 of file Tensor-inl.cuh.

+

Definition at line 383 of file Tensor-inl.cuh.

@@ -861,7 +861,7 @@ template<typename T , int Dim, bool InnerContig, typename IndexT , template&l

Returns a tensor of the same dimension after transposing the two dimensions given. Does not actually move elements; transposition is made by permuting the size/stride arrays. If the dimensions are not valid, asserts.

-

Definition at line 420 of file Tensor-inl.cuh.

+

Definition at line 451 of file Tensor-inl.cuh.

@@ -883,7 +883,7 @@ template<int NewDim>

Upcast a tensor of dimension D to some tensor of dimension D' > D by padding the lowest/most varying dimensions by 1 e.g., upcasting a 2-d tensor [2][3] to a 4-d tensor [2][3][1][1]

-

Definition at line 483 of file Tensor-inl.cuh.

+

Definition at line 514 of file Tensor-inl.cuh.

@@ -905,7 +905,7 @@ template<int NewDim>

Upcast a tensor of dimension D to some tensor of dimension D' > D by padding the leading dimensions by 1 e.g., upcasting a 2-d tensor [2][3] to a 4-d tensor [1][1][2][3]

-

Definition at line 454 of file Tensor-inl.cuh.

+

Definition at line 485 of file Tensor-inl.cuh.

@@ -928,7 +928,7 @@ template<int SubDim>

Returns a tensor that is a view of the SubDim-dimensional slice of this tensor, starting at at.

-

Definition at line 598 of file Tensor-inl.cuh.

+

Definition at line 629 of file Tensor-inl.cuh.

@@ -950,7 +950,7 @@ template<int SubDim>

Returns a tensor that is a view of the SubDim-dimensional slice of this tensor, starting where our data begins

-

Definition at line 618 of file Tensor-inl.cuh.

+

Definition at line 649 of file Tensor-inl.cuh.

@@ -973,7 +973,7 @@ template<int NewDim>

Returns a view of the given tensor expressed as a tensor of a different number of dimensions. Only works if we are contiguous.

-

Definition at line 664 of file Tensor-inl.cuh.

+

Definition at line 695 of file Tensor-inl.cuh.

diff --git a/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor.html b/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor.html index fb6f520a9..f42d67d2b 100644 --- a/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor.html +++ b/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor.html @@ -227,7 +227,7 @@ template<typename TensorType , int SubDim, template< typename U > class

Returns a view of the data located at our offset (the dimension SubDim - 1 tensor).

-

Definition at line 499 of file Tensor.cuh.

+

Definition at line 497 of file Tensor.cuh.

@@ -256,7 +256,7 @@ template<typename TensorType , int SubDim, template< typename U > class

Returns a view of the data located at our offset (the dimension SubDim - 1 tensor) (const).

-

Definition at line 515 of file Tensor.cuh.

+

Definition at line 513 of file Tensor.cuh.

@@ -284,7 +284,7 @@ template<typename TensorType , int SubDim, template< typename U > class

Returns a tensor that is a view of the SubDim-dimensional slice of this tensor, starting where our data begins

-

Definition at line 599 of file Tensor.cuh.

+

Definition at line 597 of file Tensor.cuh.

diff --git a/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor_3_01TensorType_00_010_00_01PtrTraits_01_4.html b/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor_3_01TensorType_00_010_00_01PtrTraits_01_4.html index 2cfe7ad0c..798731ad2 100644 --- a/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor_3_01TensorType_00_010_00_01PtrTraits_01_4.html +++ b/docs/html/classfaiss_1_1gpu_1_1detail_1_1SubTensor_3_01TensorType_00_010_00_01PtrTraits_01_4.html @@ -196,7 +196,7 @@ class faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits >

Specialization for a view of a single value (0-dimensional)

-

Definition at line 380 of file Tensor.cuh.

+

Definition at line 378 of file Tensor.cuh.

The documentation for this class was generated from the following file:
  • /data/users/matthijs/github_faiss/faiss/gpu/utils/Tensor.cuh
diff --git a/docs/html/demo__imi__flat_8cpp_source.html b/docs/html/demo__imi__flat_8cpp_source.html index b10a42332..6076c4f2a 100644 --- a/docs/html/demo__imi__flat_8cpp_source.html +++ b/docs/html/demo__imi__flat_8cpp_source.html @@ -240,7 +240,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
151  }
152  return 0;
153 }
- +
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
diff --git a/docs/html/demo__sift1M_8cpp_source.html b/docs/html/demo__sift1M_8cpp_source.html index a1dc0de16..8ac80fd19 100644 --- a/docs/html/demo__sift1M_8cpp_source.html +++ b/docs/html/demo__sift1M_8cpp_source.html @@ -206,7 +206,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
117 
118  printf ("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nt);
119 
-
120  index->train(nt, xt);
+
120  index->train(nt, xt);
121  delete [] xt;
122  }
123 
@@ -341,19 +341,19 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
252  delete index;
253  return 0;
254 }
-
faiss::ParameterSpace::explore
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:535
+
faiss::ParameterSpace::explore
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:554
faiss::ParameterSpace::parameter_ranges
std::vector< ParameterRange > parameter_ranges
all tunable parameters
Definition: AutoTune.h:134
-
faiss::Index::train
virtual void train(idx_t, const float *)
Definition: Index.h:89
+
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
-
faiss::ParameterSpace::initialize
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:331
-
faiss::ParameterSpace::n_combinations
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:268
-
faiss::ParameterSpace::set_index_parameters
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:390
+
faiss::ParameterSpace::initialize
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:337
+
faiss::ParameterSpace::n_combinations
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:269
+
faiss::ParameterSpace::set_index_parameters
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:410
-
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:668
+
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:687
diff --git a/docs/html/files.html b/docs/html/files.html index 4f22bee90..26d148f67 100644 --- a/docs/html/files.html +++ b/docs/html/files.html @@ -290,26 +290,28 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); o*index_io.h o*IndexFlat.cpp o*IndexFlat.h -o*IndexIVF.cpp -o*IndexIVF.h -o*IndexIVFPQ.cpp -o*IndexIVFPQ.h -o*IndexLSH.cpp -o*IndexLSH.h -o*IndexPQ.cpp -o*IndexPQ.h -o*IndexScalarQuantizer.cpp -o*IndexScalarQuantizer.h -o*MetaIndexes.cpp -o*MetaIndexes.h -o*PolysemousTraining.cpp -o*PolysemousTraining.h -o*ProductQuantizer.cpp -o*ProductQuantizer.h -o*utils.cpp -o*utils.h -o*VectorTransform.cpp -\*VectorTransform.h +o*IndexHNSW.cpp +o*IndexHNSW.h +o*IndexIVF.cpp +o*IndexIVF.h +o*IndexIVFPQ.cpp +o*IndexIVFPQ.h +o*IndexLSH.cpp +o*IndexLSH.h +o*IndexPQ.cpp +o*IndexPQ.h +o*IndexScalarQuantizer.cpp +o*IndexScalarQuantizer.h +o*MetaIndexes.cpp +o*MetaIndexes.h +o*PolysemousTraining.cpp +o*PolysemousTraining.h +o*ProductQuantizer.cpp +o*ProductQuantizer.h +o*utils.cpp +o*utils.h +o*VectorTransform.cpp +\*VectorTransform.h diff --git a/docs/html/functions.html b/docs/html/functions.html index 4f5fa0599..1dcc236bf 100644 --- a/docs/html/functions.html +++ b/docs/html/functions.html @@ -134,28 +134,33 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
  • add() : faiss::gpu::GpuIndex , faiss::gpu::IndexProxy +, faiss::IndexHNSW , faiss::IndexIVF -, faiss::IndexIVFPQCompact , faiss::gpu::BlockSelect< K, V, Dir, Comp, NumWarpQ, NumThreadQ, ThreadsPerBlock > -, faiss::IndexLSH -, faiss::IndexPQ +, faiss::IndexIVFPQCompact +, faiss::Index2Layer , faiss::gpu::GpuIndexFlat , faiss::gpu::WarpSelect< K, V, Dir, Comp, NumWarpQ, NumThreadQ, ThreadsPerBlock > +, faiss::IndexLSH +, faiss::IndexPQ +, faiss::gpu::WorkerThread , faiss::MultiIndexQuantizer , faiss::IndexScalarQuantizer -, faiss::gpu::WorkerThread -, faiss::IndexIDMap -, faiss::IndexShards , faiss::OperatingPoints , faiss::gpu::GpuIndexIVF , faiss::Index +, faiss::IndexIDMap +, faiss::IndexShards +, faiss::IndexFlat , faiss::IndexSplitVectors , faiss::IndexPreTransform -, faiss::IndexFlat , faiss::gpu::FlatIndex , faiss::IndexRefineFlat , faiss::IndexFlat1D
    • +
  • add_codes() +: faiss::ReconstructFromNeighbors +
  • add_core() : faiss::IndexIVFFlat , faiss::IndexIVFPQR @@ -167,16 +172,19 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); : faiss::ParameterSpace
  • add_with_ids() -: faiss::IndexIVFFlat -, faiss::IndexPreTransform +: faiss::gpu::GpuIndex +, faiss::Index +, faiss::IndexIVFFlat , faiss::IndexIVFPQ , faiss::IndexIVFPQR , faiss::IndexIVFScalarQuantizer , faiss::IndexIDMap , faiss::IndexIDMap2 , faiss::IndexShards -, faiss::gpu::GpuIndex -, faiss::Index +, faiss::IndexPreTransform +
    • +
  • add_with_locks() +: faiss::HNSW
  • addCodeVectorsFromCpu() : faiss::gpu::IVFFlat @@ -203,6 +211,9 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
  • addn_with_ids() : faiss::HeapArray< C >
    • +
  • advance() +: faiss::VisitedTable +
  • all_pts : faiss::OperatingPoints
    • @@ -228,21 +239,25 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); : faiss::IndexPreTransform
  • apply_noalloc() -: faiss::NormalizationTransform -, faiss::RemapDimensionsTransform -, faiss::VectorTransform +: faiss::VectorTransform , faiss::LinearTransform +, faiss::NormalizationTransform +, faiss::RemapDimensionsTransform
  • apply_preprocess() : faiss::IndexLSH
  • as() -: faiss::gpu::detail::SubTensor< TensorType, SubDim, PtrTraits > +: faiss::gpu::detail::SubTensor< TensorType, SubDim, PtrTraits > , faiss::gpu::detail::SubTensor< TensorType, 0, PtrTraits > +, faiss::gpu::detail::SubTensor< TensorType, SubDim, PtrTraits >
  • assign() : faiss::Index
    • +
  • assign_probas +: faiss::HNSW +
    • diff --git a/docs/html/functions_0x63.html b/docs/html/functions_0x63.html index cbce3a2b8..f19c194b2 100644 --- a/docs/html/functions_0x63.html +++ b/docs/html/functions_0x63.html @@ -144,6 +144,9 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); : faiss::Clustering , faiss::ProductQuantizer +
  • check_relative_distance +: faiss::HNSW +
  • classifyAndAddVectors() : faiss::gpu::IVFFlat , faiss::gpu::IVFPQ @@ -155,7 +158,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); : faiss::Clustering
  • clustering_index -: faiss::IndexIVF +: faiss::Level1Quantizer
  • ClusteringParameters() : faiss::ClusteringParameters @@ -165,11 +168,19 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
  • code_size : faiss::IndexIVF +, faiss::Index2Layer , faiss::ScalarQuantizer , faiss::ProductQuantizer
    • +
  • code_size_1 +: faiss::Index2Layer +
    • +
  • code_size_2 +: faiss::Index2Layer +
  • codes -: faiss::IndexLSH +: faiss::Index2Layer +, faiss::IndexLSH , faiss::IndexPQ , faiss::IndexScalarQuantizer
    • @@ -191,6 +202,9 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
  • compute_code() : faiss::ProductQuantizer
    • +
  • compute_code_distance() +: faiss::ScalarQuantizer::DistanceComputer +
  • compute_code_from_distance_table() : faiss::ProductQuantizer
    • @@ -201,6 +215,9 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
  • compute_cost() : faiss::Score3Computer< Ttab, Taccu >
    • +
  • compute_distance() +: faiss::ScalarQuantizer::DistanceComputer +
  • compute_distance_subset() : faiss::IndexFlat
    • @@ -235,27 +252,27 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search'); : faiss::IndexIVF
  • copyFrom() -: faiss::gpu::GpuIndexIVF -, faiss::gpu::GpuIndexFlatL2 -, faiss::gpu::GpuIndexFlat -, faiss::gpu::GpuIndexIVFFlat -, faiss::gpu::GpuIndexIVFPQ +: faiss::gpu::GpuIndexFlat , faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > +, faiss::gpu::GpuIndexIVFFlat +, faiss::gpu::GpuIndexFlatL2 +, faiss::gpu::GpuIndexIVF +, faiss::gpu::GpuIndexIVFPQ , faiss::gpu::GpuIndexFlatIP
  • copyTo() -: faiss::gpu::GpuIndexIVFPQ -, faiss::gpu::GpuIndexIVF +: faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > +, faiss::gpu::GpuIndexIVFPQ , faiss::gpu::GpuIndexFlat -, faiss::gpu::GpuIndexFlatL2 -, faiss::gpu::Tensor< T, Dim, InnerContig, IndexT, PtrTraits > -, faiss::gpu::GpuIndexIVFFlat , faiss::gpu::GpuIndexFlatIP +, faiss::gpu::GpuIndexIVF +, faiss::gpu::GpuIndexFlatL2 +, faiss::gpu::GpuIndexIVFFlat
  • cp -: faiss::gpu::GpuIndexIVF -, faiss::ProductQuantizer -, faiss::IndexIVF +: faiss::ProductQuantizer +, faiss::Level1Quantizer +, faiss::gpu::GpuIndexIVF
  • CpuTimer() : faiss::gpu::CpuTimer @@ -266,6 +283,12 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
  • CudaEvent() : faiss::gpu::CudaEvent
    • +
  • cum_nb_neighbors() +: faiss::HNSW +
    • +
  • cum_nneighbor_per_level +: faiss::HNSW +
    • diff --git a/docs/html/functions_0x65.html b/docs/html/functions_0x65.html index d4764f17e..f2bcd05db 100644 --- a/docs/html/functions_0x65.html +++ b/docs/html/functions_0x65.html @@ -125,6 +125,12 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    Here is a list of all documented class members with links to the class documentation for each member:

    - e -