mirror of
https://github.com/facebookresearch/faiss.git
synced 2026-10-12 02:13:36 +00:00
Portable SWIG Vectors (#1742)
Summary: After initial positive feedback to the idea in https://github.com/facebookresearch/faiss/issues/1741 from mdouze, here are the patches I currently have as a basis for discussion. Matthijs suggests to not bother with the deprecation warnings at all, which is fine for me as well, though I would normally still advocate to provide users with _some_ advance notice before removing parts of an interface. Fixes https://github.com/facebookresearch/faiss/issues/1741 PS. The deprecation warning is only shown once per session (per class) PPS. I have tested in https://github.com/conda-forge/faiss-split-feedstock/pull/32 that the respective classes remain available both through `import faiss` and `from faiss import *`. Pull Request resolved: https://github.com/facebookresearch/faiss/pull/1742 Reviewed By: mdouze Differential Revision: D26978886 Pulled By: beauby fbshipit-source-id: b52e2b5b5b0117af7cd95ef5df3128e9914633ad
This commit is contained in:
H. Vetinari
committed by
Facebook GitHub Bot
Facebook GitHub Bot
parent
06f1ef86ac
commit
9c58ae00f1
@@ -15,6 +15,8 @@ at the moment.
|
||||
### Changed
|
||||
- The order of xb an xq was different between `faiss.knn` and `faiss.knn_gpu`.
|
||||
Also the metric argument was called distance_type.
|
||||
- The typed vectors (LongVector, LongLongVector, etc.) of the SWIG interface have
|
||||
been deprecated. They have been replaced with Int32Vector, Int64Vector, etc.
|
||||
|
||||
### Fixed
|
||||
- Fixed a bug causing kNN search functions for IndexBinaryHash and
|
||||
|
||||
+78
-16
@@ -5,13 +5,14 @@
|
||||
|
||||
#@nolint
|
||||
|
||||
# not linting this file because it imports * form swigfaiss, which
|
||||
# not linting this file because it imports * from swigfaiss, which
|
||||
# causes a ton of useless warnings.
|
||||
|
||||
import numpy as np
|
||||
import sys
|
||||
import inspect
|
||||
import array
|
||||
import warnings
|
||||
|
||||
# We import * so that the symbol foo can be accessed as faiss.foo.
|
||||
from .loader import *
|
||||
@@ -707,6 +708,47 @@ for symbol in dir(this_module):
|
||||
if issubclass(the_class, IndexNSG):
|
||||
handle_NSG(the_class)
|
||||
|
||||
###########################################
|
||||
# Utility to add a deprecation warning to
|
||||
# classes from the SWIG interface
|
||||
###########################################
|
||||
|
||||
def _make_deprecated_swig_class(deprecated_name, base_name):
|
||||
"""
|
||||
Dynamically construct deprecated classes as wrappers around renamed ones
|
||||
|
||||
The deprecation warning added in their __new__-method will trigger upon
|
||||
construction of an instance of the class, but only once per session.
|
||||
|
||||
We do this here (in __init__.py) because the base classes are defined in
|
||||
the SWIG interface, making it cumbersome to add the deprecation there.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
deprecated_name : string
|
||||
Name of the class to be deprecated; _not_ present in SWIG interface.
|
||||
base_name : string
|
||||
Name of the class that is replacing deprecated_name; must already be
|
||||
imported into the current namespace.
|
||||
|
||||
Returns
|
||||
-------
|
||||
None
|
||||
However, the deprecated class gets added to the faiss namespace
|
||||
"""
|
||||
base_class = globals()[base_name]
|
||||
def new_meth(cls, *args, **kwargs):
|
||||
msg = f"The class faiss.{deprecated_name} is deprecated in favour of faiss.{base_name}!"
|
||||
warnings.warn(msg, DeprecationWarning, stacklevel=2)
|
||||
instance = super(base_class, cls).__new__(cls, *args, **kwargs)
|
||||
return instance
|
||||
|
||||
# three-argument version of "type" uses (name, tuple-of-bases, dict-of-attributes)
|
||||
klazz = type(deprecated_name, (base_class,), {"__new__": new_meth})
|
||||
|
||||
# this ends up adding the class to the "faiss" namespace, in a way that it
|
||||
# is available both through "import faiss" and "from faiss import *"
|
||||
globals()[deprecated_name] = klazz
|
||||
|
||||
###########################################
|
||||
# Add Python references to objects
|
||||
@@ -799,7 +841,7 @@ def index_cpu_to_gpu_multiple_py(resources, index, co=None, gpus=None):
|
||||
if gpus is None:
|
||||
gpus = range(len(resources))
|
||||
vres = GpuResourcesVector()
|
||||
vdev = IntVector()
|
||||
vdev = Int32Vector()
|
||||
for i, res in zip(gpus, resources):
|
||||
vdev.push_back(i)
|
||||
vres.push_back(res)
|
||||
@@ -940,23 +982,43 @@ def knn_gpu(res, xq, xb, k, D=None, I=None, metric=METRIC_L2):
|
||||
# numpy array / std::vector conversions
|
||||
###########################################
|
||||
|
||||
# mapping from vector names in swigfaiss.swig and the numpy dtype names
|
||||
vector_name_map = {
|
||||
'Float': 'float32',
|
||||
'Byte': 'uint8',
|
||||
'Char': 'int8',
|
||||
'Uint64': 'uint64',
|
||||
'LongLong': 'int64',
|
||||
'Int': 'int32',
|
||||
'Double': 'float64'
|
||||
sizeof_long = array.array('l').itemsize
|
||||
deprecated_name_map = {
|
||||
# deprecated: replacement
|
||||
'Float': 'Float32',
|
||||
'Double': 'Float64',
|
||||
'Char': 'Int8',
|
||||
'Int': 'Int32',
|
||||
'Long': 'Int32' if sizeof_long == 4 else 'Int64',
|
||||
'LongLong': 'Int64',
|
||||
'Byte': 'UInt8',
|
||||
# previously misspelled variant
|
||||
'Uint64': 'UInt64',
|
||||
}
|
||||
|
||||
sizeof_long = array.array('l').itemsize
|
||||
if sizeof_long == 4:
|
||||
vector_name_map["Long"] = 'int32'
|
||||
elif sizeof_long == 8:
|
||||
vector_name_map["Long"] = 'int64'
|
||||
for depr_prefix, base_prefix in deprecated_name_map.items():
|
||||
_make_deprecated_swig_class(depr_prefix + "Vector", base_prefix + "Vector")
|
||||
|
||||
# same for the three legacy *VectorVector classes
|
||||
if depr_prefix in ['Float', 'Long', 'Byte']:
|
||||
_make_deprecated_swig_class(depr_prefix + "VectorVector",
|
||||
base_prefix + "VectorVector")
|
||||
|
||||
# mapping from vector names in swigfaiss.swig and the numpy dtype names
|
||||
# TODO: once deprecated classes are removed, remove the dict and just use .lower() below
|
||||
vector_name_map = {
|
||||
'Float32': 'float32',
|
||||
'Float64': 'float64',
|
||||
'Int8': 'int8',
|
||||
'Int16': 'int16',
|
||||
'Int32': 'int32',
|
||||
'Int64': 'int64',
|
||||
'UInt8': 'uint8',
|
||||
'UInt16': 'uint16',
|
||||
'UInt32': 'uint32',
|
||||
'UInt64': 'uint64',
|
||||
**{k: v.lower() for k, v in deprecated_name_map.items()}
|
||||
}
|
||||
|
||||
|
||||
def vector_to_array(v):
|
||||
|
||||
+24
-26
@@ -207,23 +207,21 @@ namespace std {
|
||||
%include <std_shared_ptr.i>
|
||||
|
||||
// primitive array types
|
||||
%template(FloatVector) std::vector<float>;
|
||||
%template(DoubleVector) std::vector<double>;
|
||||
%template(ByteVector) std::vector<uint8_t>;
|
||||
%template(CharVector) std::vector<char>;
|
||||
// NOTE(hoss): Using unsigned long instead of uint64_t because OSX defines
|
||||
// uint64_t as unsigned long long, which SWIG is not aware of.
|
||||
%template(Uint64Vector) std::vector<unsigned long>;
|
||||
%template(LongVector) std::vector<long>;
|
||||
%template(LongLongVector) std::vector<long long>;
|
||||
%template(IntVector) std::vector<int>;
|
||||
%template(UInt32Vector) std::vector<uint32_t>;
|
||||
%template(Float32Vector) std::vector<float>;
|
||||
%template(Float64Vector) std::vector<double>;
|
||||
%template(Int8Vector) std::vector<int8_t>;
|
||||
%template(Int16Vector) std::vector<int16_t>;
|
||||
%template(Int32Vector) std::vector<int32_t>;
|
||||
%template(Int64Vector) std::vector<int64_t>;
|
||||
%template(UInt8Vector) std::vector<uint8_t>;
|
||||
%template(UInt16Vector) std::vector<uint16_t>;
|
||||
%template(UInt32Vector) std::vector<uint32_t>;
|
||||
%template(UInt64Vector) std::vector<uint64_t>;
|
||||
|
||||
%template(FloatVectorVector) std::vector<std::vector<float> >;
|
||||
%template(ByteVectorVector) std::vector<std::vector<unsigned char> >;
|
||||
%template(LongVectorVector) std::vector<std::vector<long> >;
|
||||
%template(Float32VectorVector) std::vector<std::vector<float> >;
|
||||
%template(UInt8VectorVector) std::vector<std::vector<uint8_t> >;
|
||||
%template(Int32VectorVector) std::vector<std::vector<int32_t> >;
|
||||
%template(Int64VectorVector) std::vector<std::vector<int64_t> >;
|
||||
%template(VectorTransformVector) std::vector<faiss::VectorTransform*>;
|
||||
%template(OperatingPointVector) std::vector<faiss::OperatingPoint>;
|
||||
%template(InvertedListsPtrVector) std::vector<faiss::InvertedLists*>;
|
||||
@@ -307,9 +305,9 @@ void gpu_sync_all_devices()
|
||||
|
||||
%}
|
||||
|
||||
%template() std::pair<int, unsigned long>;
|
||||
%template() std::map<std::string, std::pair<int, unsigned long> >;
|
||||
%template() std::map<int, std::map<std::string, std::pair<int, unsigned long> > >;
|
||||
%template() std::pair<int, uint64_t>;
|
||||
%template() std::map<std::string, std::pair<int, uint64_t> >;
|
||||
%template() std::map<int, std::map<std::string, std::pair<int, uint64_t> > >;
|
||||
|
||||
// causes weird wrapper bug
|
||||
%ignore *::allocMemoryHandle;
|
||||
@@ -324,12 +322,12 @@ typedef CUstream_st* cudaStream_t;
|
||||
%inline %{
|
||||
|
||||
// interop between pytorch exposed cudaStream_t and faiss
|
||||
cudaStream_t cast_integer_to_cudastream_t(long long x) {
|
||||
cudaStream_t cast_integer_to_cudastream_t(int64_t x) {
|
||||
return (cudaStream_t) x;
|
||||
}
|
||||
|
||||
long long cast_cudastream_t_to_integer(cudaStream_t x) {
|
||||
return (long) x;
|
||||
int64_t cast_cudastream_t_to_integer(cudaStream_t x) {
|
||||
return (int64_t) x;
|
||||
}
|
||||
|
||||
%}
|
||||
@@ -914,23 +912,23 @@ void *memcpy(void *dest, const void *src, size_t n);
|
||||
*******************************************************************/
|
||||
|
||||
%inline %{
|
||||
uint8_t * cast_integer_to_uint8_ptr (long long x) {
|
||||
uint8_t * cast_integer_to_uint8_ptr (int64_t x) {
|
||||
return (uint8_t*)x;
|
||||
}
|
||||
|
||||
float * cast_integer_to_float_ptr (long long x) {
|
||||
float * cast_integer_to_float_ptr (int64_t x) {
|
||||
return (float*)x;
|
||||
}
|
||||
|
||||
faiss::Index::idx_t* cast_integer_to_idx_t_ptr (long long x) {
|
||||
faiss::Index::idx_t* cast_integer_to_idx_t_ptr (int64_t x) {
|
||||
return (faiss::Index::idx_t*)x;
|
||||
}
|
||||
|
||||
int * cast_integer_to_int_ptr (long long x) {
|
||||
int * cast_integer_to_int_ptr (int64_t x) {
|
||||
return (int*)x;
|
||||
}
|
||||
|
||||
void * cast_integer_to_void_ptr (long long x) {
|
||||
void * cast_integer_to_void_ptr (int64_t x) {
|
||||
return (void*)x;
|
||||
}
|
||||
|
||||
@@ -957,7 +955,7 @@ struct MapLong2Long {
|
||||
}
|
||||
}
|
||||
|
||||
long search(int64_t key) {
|
||||
int64_t search(int64_t key) {
|
||||
if (map.count(key) == 0) {
|
||||
return -1;
|
||||
} else {
|
||||
|
||||
@@ -614,13 +614,7 @@ class TestSWIGWrap(unittest.TestCase):
|
||||
|
||||
def test_int64(self):
|
||||
# see https://github.com/facebookresearch/faiss/issues/1529
|
||||
sizeof_long = array.array("l").itemsize
|
||||
if sizeof_long == 4:
|
||||
v = faiss.LongLongVector()
|
||||
elif sizeof_long == 8:
|
||||
v = faiss.LongVector()
|
||||
else:
|
||||
raise AssertionError("weird long size")
|
||||
v = faiss.Int64Vector()
|
||||
|
||||
for i in range(10):
|
||||
v.push_back(i)
|
||||
|
||||
@@ -19,7 +19,7 @@ class TestOOMException(unittest.TestCase):
|
||||
# make sure that allocing more than 10G will fail
|
||||
resource.setrlimit(resource.RLIMIT_AS, (10 * 1024 * 1024, hard_as))
|
||||
try:
|
||||
x = faiss.IntVector()
|
||||
x = faiss.Int32Vector()
|
||||
try:
|
||||
x.resize(10**11) # 400 G of RAM
|
||||
except MemoryError:
|
||||
|
||||
Reference in New Issue
Block a user