facebookresearch/faiss
1
0
Fork 0
mirror of https://github.com/facebookresearch/faiss.git synced 2026-10-12 02:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert D80734790 (#4563)

Browse Source 
Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/4563

This diff reverts D80734790
(The context such as a Sandcastle job, Task, SEV, etc. was not provided.)

Depends on D80734790

Reviewed By: limqiying

Differential Revision: D81147721

fbshipit-source-id: 33031f34660155a82f30cc2327a932e3f47ff141
This commit is contained in:
generatedunixname89002005232357
2025-08-27 12:51:59 -07:00
committed by Facebook GitHub Bot
parent ee6b7ddd9d
commit 0031d61da8
10 changed files with 65 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/torch_utils.py
+11 -29
View File
@@ -80,15 +80,6 @@ def swig_ptr_from_IndicesTensor(x):
return faiss.cast_integer_to_idx_t_ptr(
x.untyped_storage().data_ptr() + x.storage_offset() * 8)
def swig_ptr_from_Int64Tensor(x):
""" gets a Faiss SWIG pointer from a pytorch tensor (on CPU or GPU) """
assert x.is_contiguous()
assert x.dtype == torch.int64, 'dtype=%s' % x.dtype
return faiss.cast_integer_to_int64_ptr(
x.untyped_storage().data_ptr() + x.storage_offset() * 8)
##################################################################
# utilities
##################################################################
@@ -168,7 +159,7 @@ def handle_torch_Index(the_class):
# CPU torch
self.add_ex(n, x_ptr, numeric_type)
def torch_replacement_add_with_ids(self, x, ids, numeric_type = faiss.Float32, ids_type = faiss.Int64):
def torch_replacement_add_with_ids(self, x, ids, numeric_type = faiss.Float32):
if type(x) is np.ndarray:
# forward to faiss __init__.py base method
return self.add_with_ids_numpy(x, ids)
@@ -185,20 +176,17 @@ def handle_torch_Index(the_class):
assert type(ids) is torch.Tensor
assert ids.shape == (n, ), 'not same number of vectors as ids'
if ids_type == faiss.Int64:
ids_ptr = swig_ptr_from_Int64Tensor(ids)
else:
raise ValueError("ids type for add_with_ids must be faiss.Int64")
ids_ptr = swig_ptr_from_IndicesTensor(ids)
if x.is_cuda:
assert hasattr(self, 'getDevice'), 'GPU tensor on CPU index not allowed'
# On the GPU, use proper stream ordering
with using_stream(self.getResources()):
self.add_with_ids_ex(n, x_ptr, numeric_type, ids_ptr, ids_type)
self.add_with_ids_ex(n, x_ptr, numeric_type, ids_ptr)
else:
# CPU torch
self.add_with_ids_ex(n, x_ptr, numeric_type, ids_ptr, ids_type)
self.add_with_ids_ex(n, x_ptr, numeric_type, ids_ptr)
def torch_replacement_assign(self, x, k, labels=None):
if type(x) is np.ndarray:
@@ -272,18 +260,12 @@ def handle_torch_Index(the_class):
if I is None:
I = torch.empty(n, k, device=x.device, dtype=torch.int64)
I_ptr = swig_ptr_from_Int64Tensor(I)
I_type = faiss.Int64
else:
assert type(I) is torch.Tensor
assert I.shape == (n, k)
if I.dtype == torch.int64:
I_ptr = swig_ptr_from_Int64Tensor(I)
I_type = faiss.Int64
else:
raise ValueError("labels for search should be int64 type")
I_ptr = swig_ptr_from_IndicesTensor(I)
return x_ptr, D_ptr, I_ptr, D, I, I_type
return x_ptr, D_ptr, I_ptr, D, I
def torch_replacement_search(self, x, k, D=None, I=None, numeric_type=faiss.Float32):
if type(x) is np.ndarray:
@@ -294,17 +276,17 @@ def handle_torch_Index(the_class):
n, d = x.shape
assert d == self.d
x_ptr, D_ptr, I_ptr, D, I, I_type = search_methods_common(x, k, D, I)
x_ptr, D_ptr, I_ptr, D, I = search_methods_common(x, k, D, I)
if x.is_cuda:
assert hasattr(self, 'getDevice'), 'GPU tensor on CPU index not allowed'
# On the GPU, use proper stream ordering
with using_stream(self.getResources()):
self.search_ex(n, x_ptr, numeric_type, k, D_ptr, I_ptr, I_type)
self.search_ex(n, x_ptr, numeric_type, k, D_ptr, I_ptr)
else:
# CPU torch
self.search_ex(n, x_ptr, numeric_type, k, D_ptr, I_ptr, I_type)
self.search_ex(n, x_ptr, numeric_type, k, D_ptr, I_ptr)
return D, I
@@ -317,7 +299,7 @@ def handle_torch_Index(the_class):
n, d = x.shape
assert d == self.d
x_ptr, D_ptr, I_ptr, D, I, _ = search_methods_common(x, k, D, I)
x_ptr, D_ptr, I_ptr, D, I = search_methods_common(x, k, D, I)
if R is None:
R = torch.empty(n, k, d, device=x.device, dtype=torch.float32)
@@ -347,7 +329,7 @@ def handle_torch_Index(the_class):
n, d = x.shape
assert d == self.d
x_ptr, D_ptr, I_ptr, D, I, _ = search_methods_common(x, k, D, I)
x_ptr, D_ptr, I_ptr, D, I = search_methods_common(x, k, D, I)
assert Iq.shape == (n, self.nprobe)
Iq = Iq.contiguous()
faiss/Index.h
+9 -25
View File
@@ -13,7 +13,6 @@
#include <faiss/MetricType.h>
#include <faiss/impl/FaissAssert.h>
#include <cstdint>
#include <cstdio>
#include <sstream>
@@ -60,16 +59,12 @@ struct DistanceComputer;
enum NumericType {
Float32,
Float16,
Int64,
UInt8,
Int8,
NONE, // corresponding to nullptr
};
inline size_t get_numeric_type_size(NumericType numeric_type) {
switch (numeric_type) {
case NumericType::Int64:
return 8;
case NumericType::Float32:
return 4;
case NumericType::Float16:
@@ -77,11 +72,9 @@ inline size_t get_numeric_type_size(NumericType numeric_type) {
case NumericType::UInt8:
case NumericType::Int8:
return 1;
case NumericType::NONE:
return 0;
default:
FAISS_THROW_MSG(
"Unknown Numeric Type. Only supports Float32, Float16, Int64, UInt8, Int8");
"Unknown Numeric Type. Only supports Float32, Float16");
}
}
@@ -176,17 +169,11 @@ struct Index {
idx_t n,
const void* x,
NumericType numeric_type,
const void* xids,
NumericType xids_type) {
if (numeric_type == NumericType::Float32 &&
xids_type == NumericType::Int64) {
add_with_ids(
n,
static_cast<const float*>(x),
static_cast<const idx_t*>(xids));
const idx_t* xids) {
if (numeric_type == NumericType::Float32) {
add_with_ids(n, static_cast<const float*>(x), xids);
} else {
FAISS_THROW_MSG(
"Index::add_with_ids: unsupported numeric type or xids type");
FAISS_THROW_MSG("Index::add_with_ids: unsupported numeric type");
}
}
@@ -215,20 +202,17 @@ struct Index {
NumericType numeric_type,
idx_t k,
float* distances,
void* labels,
NumericType labels_type,
idx_t* labels,
const SearchParameters* params = nullptr) const {
if (numeric_type == NumericType::Float32 &&
labels_type == NumericType::Int64) {
if (numeric_type == NumericType::Float32) {
search(n,
static_cast<const float*>(x),
k,
distances,
static_cast<idx_t*>(labels),
labels,
params);
} else {
FAISS_THROW_MSG(
"Index::search: unsupported numeric type or label type");
FAISS_THROW_MSG("Index::search: unsupported numeric type");
}
}
faiss/IndexBinary.h
+8 -16
View File
@@ -86,17 +86,12 @@ struct IndexBinary {
idx_t n,
const void* x,
NumericType numeric_type,
const void* xids,
NumericType xids_type) {
if (numeric_type == NumericType::UInt8 &&
xids_type == NumericType::Int64) {
add_with_ids(
n,
static_cast<const uint8_t*>(x),
static_cast<const idx_t*>(xids));
const idx_t* xids) {
if (numeric_type == NumericType::UInt8) {
add_with_ids(n, static_cast<const uint8_t*>(x), xids);
} else {
FAISS_THROW_MSG(
"IndexBinary::add_with_ids: unsupported numeric type or xids type");
"IndexBinary::add_with_ids: unsupported numeric type");
}
};
@@ -122,20 +117,17 @@ struct IndexBinary {
NumericType numeric_type,
idx_t k,
int32_t* distances,
void* labels,
NumericType labels_type,
idx_t* labels,
const SearchParameters* params = nullptr) const {
if (numeric_type == NumericType::UInt8 &&
labels_type == NumericType::Int64) {
if (numeric_type == NumericType::UInt8) {
search(n,
static_cast<const uint8_t*>(x),
k,
distances,
static_cast<idx_t*>(labels),
labels,
params);
} else {
FAISS_THROW_MSG(
"IndexBinary::search: unsupported numeric type or label type");
FAISS_THROW_MSG("IndexBinary::search: unsupported numeric type");
}
};
faiss/IndexIDMap.cpp
+10 -26
View File
@@ -12,7 +12,6 @@
#include <cinttypes>
#include <cstdint>
#include <cstdio>
#include "Index.h"
#include "faiss/Index.h"
#include <faiss/impl/AuxIndexStructures.h>
@@ -109,14 +108,10 @@ void IndexIDMapTemplate<IndexT>::add_with_ids_ex(
idx_t n,
const void* x,
NumericType numeric_type,
const void* xids,
NumericType xids_type) {
FAISS_THROW_IF_NOT_MSG(
xids_type == NumericType::Int64,
"IndexIDMapTemplate::add_with_ids_ex only supports int64 as xids type");
const idx_t* xids) {
index->add_ex(n, x, numeric_type);
for (idx_t i = 0; i < n; i++) {
id_map.push_back(static_cast<const idx_t*>(xids)[i]);
id_map.push_back(xids[i]);
}
this->ntotal = index->ntotal;
}
@@ -130,8 +125,7 @@ void IndexIDMapTemplate<IndexT>::add_with_ids(
n,
static_cast<const void*>(x),
component_t_to_numeric<typename IndexT::component_t>(),
static_cast<const void*>(xids),
NumericType::Int64);
xids);
}
template <typename IndexT>
@@ -179,8 +173,7 @@ void IndexIDMapTemplate<IndexT>::search_ex(
NumericType numeric_type,
idx_t k,
typename IndexT::distance_t* distances,
void* labels,
NumericType labels_type,
idx_t* labels,
const SearchParameters* params) const {
IDSelectorTranslated this_idtrans(this->id_map, nullptr);
ScopedSelChange sel_change;
@@ -201,13 +194,8 @@ void IndexIDMapTemplate<IndexT>::search_ex(
sel_change.set(params_non_const, &this_idtrans);
}
}
index->search_ex(
n, x, numeric_type, k, distances, labels, labels_type, params);
FAISS_THROW_IF_NOT_MSG(
labels_type == NumericType::Int64,
"IndexIDMapTemplate::search_ex only supports int64 as labels type");
idx_t* li = static_cast<idx_t*>(labels);
index->search_ex(n, x, numeric_type, k, distances, labels, params);
idx_t* li = labels;
#pragma omp parallel for
for (idx_t i = 0; i < n * k; i++) {
li[i] = li[i] < 0 ? li[i] : id_map[li[i]];
@@ -228,8 +216,7 @@ void IndexIDMapTemplate<IndexT>::search(
component_t_to_numeric<typename IndexT::component_t>(),
k,
distances,
static_cast<void*>(labels),
NumericType::Int64,
labels,
params);
}
@@ -319,11 +306,9 @@ void IndexIDMap2Template<IndexT>::add_with_ids_ex(
idx_t n,
const void* x,
NumericType numeric_type,
const void* xids,
NumericType xids_type) {
const idx_t* xids) {
size_t prev_ntotal = this->ntotal;
IndexIDMapTemplate<IndexT>::add_with_ids_ex(
n, x, numeric_type, xids, xids_type);
IndexIDMapTemplate<IndexT>::add_with_ids_ex(n, x, numeric_type, xids);
for (size_t i = prev_ntotal; i < this->ntotal; i++) {
rev_map[this->id_map[i]] = i;
}
@@ -338,8 +323,7 @@ void IndexIDMap2Template<IndexT>::add_with_ids(
n,
static_cast<const void*>(x),
component_t_to_numeric<typename IndexT::component_t>(),
static_cast<const void*>(xids),
NumericType::Int64);
xids);
}
template <typename IndexT>
faiss/IndexIDMap.h
+3 -6
View File
@@ -35,8 +35,7 @@ struct IndexIDMapTemplate : IndexT {
idx_t n,
const void* x,
NumericType numeric_type,
const void* xids,
NumericType xids_type) override;
const idx_t* xids) override;
/// this will fail. Use add_with_ids
void add(idx_t n, const component_t* x) override;
@@ -55,8 +54,7 @@ struct IndexIDMapTemplate : IndexT {
NumericType numeric_type,
idx_t k,
distance_t* distances,
void* labels,
NumericType labels_type,
idx_t* labels,
const SearchParameters* params = nullptr) const override;
void train(idx_t n, const component_t* x) override;
@@ -110,8 +108,7 @@ struct IndexIDMap2Template : IndexIDMapTemplate<IndexT> {
idx_t n,
const void* x,
NumericType numeric_type,
const void* xids,
NumericType xids_type) override;
const idx_t* xids) override;
size_t remove_ids(const IDSelector& sel) override;
faiss/gpu/GpuIndex.cu
+8 -31
View File
@@ -111,7 +111,7 @@ size_t GpuIndex::getMinPagingSize() const {
}
void GpuIndex::add_ex(idx_t n, const void* x, NumericType numeric_type) {
add_with_ids_ex(n, x, numeric_type, nullptr, NumericType::NONE);
add_with_ids_ex(n, x, numeric_type, nullptr);
}
void GpuIndex::add(idx_t n, const float* x) {
@@ -123,8 +123,7 @@ void GpuIndex::add_with_ids_ex(
idx_t n,
const void* x,
NumericType numeric_type,
const void* ids,
NumericType ids_type) {
const idx_t* ids) {
DeviceScope scope(config_.device);
FAISS_THROW_IF_NOT_MSG(this->is_trained, "Index not trained");
@@ -135,10 +134,6 @@ void GpuIndex::add_with_ids_ex(
std::vector<idx_t> generatedIds;
FAISS_THROW_IF_NOT_MSG(
ids_type == NumericType::Int64 || !ids,
"GpuIndex::add_with_ids_ex only supports int64 as ids type");
// Generate IDs if we need them
if (!ids && addImplRequiresIDs_()) {
generatedIds = std::vector<idx_t>(n);
@@ -148,20 +143,11 @@ void GpuIndex::add_with_ids_ex(
}
}
addPaged_ex_(
n,
x,
numeric_type,
ids ? static_cast<const idx_t*>(ids) : generatedIds.data());
addPaged_ex_(n, x, numeric_type, ids ? ids : generatedIds.data());
}
void GpuIndex::add_with_ids(idx_t n, const float* x, const idx_t* ids) {
add_with_ids_ex(
n,
static_cast<const void*>(x),
NumericType::Float32,
static_cast<const void*>(ids),
NumericType::Int64);
add_with_ids_ex(n, static_cast<const void*>(x), NumericType::Float32, ids);
}
void GpuIndex::addPaged_ex_(
@@ -302,14 +288,10 @@ void GpuIndex::search_ex(
NumericType numeric_type,
idx_t k,
float* distances,
void* labels,
NumericType labels_type,
idx_t* labels,
const SearchParameters* params) const {
DeviceScope scope(config_.device);
FAISS_THROW_IF_NOT_MSG(this->is_trained, "Index not trained");
FAISS_THROW_IF_NOT_MSG(
labels_type == NumericType::Int64,
"GpuIndex::search_ex only supports int64 as labels type");
validateKSelect(k, should_use_cuvs(config_));
@@ -333,11 +315,7 @@ void GpuIndex::search_ex(
resources_.get(), config_.device, distances, stream, {n, k});
auto outLabels = toDeviceTemporary<idx_t, 2>(
resources_.get(),
config_.device,
static_cast<idx_t*>(labels),
stream,
{n, k});
resources_.get(), config_.device, labels, stream, {n, k});
bool usePaged = false;
@@ -377,7 +355,7 @@ void GpuIndex::search_ex(
// Copy back if necessary
fromDevice<float, 2>(outDistances, distances, stream);
fromDevice<idx_t, 2>(outLabels, static_cast<idx_t*>(labels), stream);
fromDevice<idx_t, 2>(outLabels, labels, stream);
}
void GpuIndex::search(
@@ -393,8 +371,7 @@ void GpuIndex::search(
NumericType::Float32,
k,
distances,
static_cast<void*>(labels),
NumericType::Int64,
labels,
params);
}
faiss/gpu/GpuIndex.h
+2 -4
View File
@@ -87,8 +87,7 @@ class GpuIndex : public faiss::Index {
idx_t n,
const void* x,
NumericType numeric_type,
const void* ids,
NumericType ids_type) override;
const idx_t* xids) override;
/// `x` and `labels` can be resident on the CPU or any GPU; copies are
/// performed as needed
@@ -110,8 +109,7 @@ class GpuIndex : public faiss::Index {
NumericType numeric_type,
idx_t k,
float* distances,
void* labels,
NumericType labels_type,
idx_t* labels,
const SearchParameters* params = nullptr) const override;
/// `x`, `distances` and `labels` and `recons` can be resident on the CPU or
faiss/gpu/test/TestGpuIndexCagra.cu
+3 -6
View File
@@ -278,8 +278,7 @@ void queryTestFP16(faiss::MetricType metric, double expected_recall) {
faiss::NumericType::Float16,
opt.k,
testDistance.data(),
testIndices.data(),
faiss::NumericType::Int64);
testIndices.data());
auto refDistanceDev = faiss::gpu::toDeviceTemporary(
gpuRes.get(),
@@ -836,8 +835,7 @@ void copyFromTestFP16(faiss::MetricType metric, double expected_recall) {
faiss::NumericType::Float16,
opt.k,
copyTestDistance.data(),
copyTestIndices.data(),
faiss::NumericType::Int64);
copyTestIndices.data());
faiss::gpu::DeviceTensor<float, 2, true> testDistance(
gpuRes.get(), devAlloc, {opt.numQuery, opt.k});
@@ -849,8 +847,7 @@ void copyFromTestFP16(faiss::MetricType metric, double expected_recall) {
faiss::NumericType::Float16,
opt.k,
testDistance.data(),
testIndices.data(),
faiss::NumericType::Int64);
testIndices.data());
// test quality of searches
auto raft_handle = gpuRes->getRaftHandleCurrentDevice();
faiss/python/class_wrappers.py
+11 -26
View File
@@ -49,26 +49,8 @@ def _numeric_to_str(numeric_type):
return 'float16'
elif numeric_type == faiss.Int8:
return 'int8'
elif numeric_type == faiss.Int64:
return 'int64'
else:
raise ValueError("numeric type must be either faiss.Float32, faiss.Float16, faiss.Int8, or faiss.Int64")
def _np_dtype_to_faiss_type(dtype):
if dtype == np.int64:
return faiss.Int64
elif dtype == np.float32:
return faiss.Float32
elif dtype == np.float16:
return faiss.Float16
elif dtype == np.int8:
return faiss.Int8
elif dtype == np.uint8:
return faiss.UInt8
else:
raise ValueError("given numpy dtype should be either np.int64, np.float32, np.float16, np.int8, or np.uint8 to convert to faiss numeric types")
raise ValueError("numeric type must be either faiss.Float32, faiss.Float16, or faiss.Int8")
def replace_method(the_class, name, replacement, ignore_missing=False):
@@ -260,7 +242,7 @@ def handle_Index(the_class):
else:
self.add_ex(n, swig_ptr(x), numeric_type)
def replacement_add_with_ids(self, x, ids, numeric_type = faiss.Float32, ids_type = faiss.Int64):
def replacement_add_with_ids(self, x, ids, numeric_type = faiss.Float32):
"""Adds vectors with arbitrary ids to the index (not all indexes support this).
The index must be trained before vectors can be added to it.
Vector `i` is stored in `x[i]` and has id `ids[i]`.
@@ -278,9 +260,11 @@ def handle_Index(the_class):
assert d == self.d
assert ids.shape == (n, ), 'not same nb of vectors as ids'
x = np.ascontiguousarray(x, dtype=_numeric_to_str(numeric_type))
ids = np.ascontiguousarray(ids, dtype=_numeric_to_str(ids_type))
self.add_with_ids_ex(n, swig_ptr(x), numeric_type, swig_ptr(ids), ids_type)
ids = np.ascontiguousarray(ids, dtype='int64')
if numeric_type == faiss.Float32:
self.add_with_ids_c(n, swig_ptr(x), swig_ptr(ids))
else:
self.add_with_ids_ex(n, swig_ptr(x), numeric_type, swig_ptr(ids))
def replacement_assign(self, x, k, labels=None):
@@ -374,12 +358,13 @@ def handle_Index(the_class):
if I is None:
I = np.empty((n, k), dtype=np.int64)
labels_type = faiss.Int64 # if I is not given, default to int64 types
else:
assert I.shape == (n, k)
labels_type = _np_dtype_to_faiss_type(I.dtype)
self.search_ex(n, swig_ptr(x), numeric_type, k, swig_ptr(D), swig_ptr(I), labels_type, params)
if numeric_type == faiss.Float32:
self.search_c(n, swig_ptr(x), k, swig_ptr(D), swig_ptr(I), params)
else:
self.search_ex(n, swig_ptr(x), numeric_type, k, swig_ptr(D), swig_ptr(I), params)
return D, I
def replacement_search_and_reconstruct(self, x, k, *, params=None, D=None, I=None, R=None):
faiss/python/swigfaiss.swig
-4
View File
@@ -1221,10 +1221,6 @@ faiss::idx_t* cast_integer_to_idx_t_ptr (int64_t x) {
return (faiss::idx_t*)x;
}
int64_t* cast_integer_to_int64_ptr (int64_t x) {
return (int64_t*)x;
}
int * cast_integer_to_int_ptr (int64_t x) {
return (int*)x;
}