facebookresearch/faiss
1
0
Fork 0
mirror of https://github.com/facebookresearch/faiss.git synced 2026-10-12 02:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix int64 for python tests in windows (#1381)

Browse Source 
Summary:
`long` is 32 bits on windows and so is the default int type for numpy (eg. the one used for `np.arange`).
This diff explicitly specifies 64-bit ints for all occurrences where it matters.

Pull Request resolved: https://github.com/facebookresearch/faiss/pull/1381

Reviewed By: wickedfoo

Differential Revision: D23371232

Pulled By: mdouze

fbshipit-source-id: 220262cd70ee70379f83de93561a4eae71c94b04
This commit is contained in:
Matthijs Douze
2020-08-27 12:40:55 -07:00
committed by Facebook GitHub Bot
parent 57506c6f4d
commit 6d73c2ff69
9 changed files with 32 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
faiss/VectorTransform.cpp
+1 -1
View File
@@ -271,7 +271,7 @@ void RandomRotationMatrix::init (int seed)
is_trained = true;
}
void RandomRotationMatrix::train (Index::idx_t /*n*/, const float */*x*/)
void RandomRotationMatrix::train (Index::idx_t /*n*/, const float * /*x*/)
{
// initialize with some arbitrary seed
init (12345);
faiss/impl/index_read.cpp
+7 -2
View File
@@ -199,7 +199,11 @@ InvertedLists *read_InvertedLists (IOReader *f, int io_flags) {
}
return ails;
#ifndef _MSC_VER
#ifdef _MSC_VER
} else {
FAISS_THROW_MSG("Unsupported inverted list format for Windows");
}
#else
} else if (h == fourcc ("ilar") && (io_flags & IO_FLAG_SKIP_IVF_DATA)) {
// code is always ilxx where xx is specific to the type of invlists we want
// so we get the 16 high bits from the io_flag and the 16 low bits as "il"
@@ -213,8 +217,9 @@ InvertedLists *read_InvertedLists (IOReader *f, int io_flags) {
f, io_flags, nlist, code_size, sizes);
} else {
return InvertedListsIOHook::lookup(h)->read(f, io_flags);
#endif // !_MSC_VER
}
#endif // !_MSC_VER
}
tests/test_build_blocks.py
+4 -4
View File
@@ -317,8 +317,8 @@ class TestException(unittest.TestCase):
class TestMapLong2Long(unittest.TestCase):
def test_maplong2long(self):
keys = np.array([13, 45, 67])
vals = np.array([3, 8, 2])
keys = np.array([13, 45, 67], dtype=np.int64)
vals = np.array([3, 8, 2], dtype=np.int64)
m = faiss.MapLong2Long()
m.add(keys, vals)
@@ -503,7 +503,7 @@ class TestScalarQuantizer(unittest.TestCase):
D, I = index.search(x[3:], 1)
# assert D[0, 0] == Dref[0, 0]
print(D[0, 0], ((x[3] - x[2]) ** 2).sum())
# print(D[0, 0], ((x[3] - x[2]) ** 2).sum())
assert D[0, 0] == ((x[3] - x[2]) ** 2).sum()
def test_6bit_equiv(self):
@@ -533,7 +533,7 @@ class TestScalarQuantizer(unittest.TestCase):
for i in range(20):
for j in range(10):
dis = ((y[i] - x2[I[i, j]]) ** 2).sum()
print(dis, D[i, j])
# print(dis, D[i, j])
assert abs(D[i, j] - dis) / dis < 1e-5
class TestRandom(unittest.TestCase):
tests/test_index.py
+1 -1
View File
@@ -581,7 +581,7 @@ class TestReconsHash(unittest.TestCase):
# with lookup
index.reset()
rs = np.random.RandomState(123)
ids = rs.choice(10000, size=200, replace=False)
ids = rs.choice(10000, size=200, replace=False).astype(np.int64)
index.add_with_ids(faiss.randn((100, d), 345), ids[:100])
index.set_direct_map_type(faiss.DirectMap.Hashtable)
index.add_with_ids(faiss.randn((100, d), 678), ids[100:])
tests/test_index_binary.py
+1 -1
View File
@@ -241,7 +241,7 @@ class TestBinaryIVF(unittest.TestCase):
# try w/ hashtable
index = faiss.IndexBinaryIVF(quantizer, d, 8)
rs = np.random.RandomState(123)
ids = rs.choice(10000, size=len(self.xb), replace=False)
ids = rs.choice(10000, size=len(self.xb), replace=False).astype(np.int64)
index.add_with_ids(self.xb, ids)
index.set_direct_map_type(faiss.DirectMap.Hashtable)
tests/test_index_composite.py
+10 -8
View File
@@ -74,9 +74,9 @@ class TestRemove(unittest.TestCase):
index = faiss.IndexFlat(5)
xb = np.zeros((10, 5), dtype='float32')
xb[:, 0] = np.arange(10) + 1000
xb[:, 0] = np.arange(10, dtype='int64') + 1000
index.add(xb)
index.remove_ids(np.arange(5) * 2)
index.remove_ids(np.arange(5, dtype='int64') * 2)
xb2 = faiss.vector_float_to_array(index.xb).reshape(5, 5)
assert np.all(xb2[:, 0] == xb[np.arange(5) * 2 + 1, 0])
@@ -85,9 +85,9 @@ class TestRemove(unittest.TestCase):
xb = np.zeros((10, 5), dtype='float32')
xb[:, 0] = np.arange(10) + 1000
index = faiss.IndexIDMap2(sub_index)
index.add_with_ids(xb, np.arange(10) + 100)
index.add_with_ids(xb, np.arange(10, dtype='int64') + 100)
assert index.reconstruct(104)[0] == 1004
index.remove_ids(np.array([103]))
index.remove_ids(np.array([103], dtype='int64'))
assert index.reconstruct(104)[0] == 1004
try:
index.reconstruct(103)
@@ -121,9 +121,9 @@ class TestRemove(unittest.TestCase):
xb = np.zeros((10, 5), dtype='uint8')
xb[:, 0] = np.arange(10) + 100
index = faiss.IndexBinaryIDMap2(sub_index)
index.add_with_ids(xb, np.arange(10) + 1000)
index.add_with_ids(xb, np.arange(10, dtype='int64') + 1000)
assert index.reconstruct(1004)[0] == 104
index.remove_ids(np.array([1003]))
index.remove_ids(np.array([1003], dtype='int64'))
assert index.reconstruct(1004)[0] == 104
try:
index.reconstruct(1003)
@@ -158,7 +158,7 @@ class TestRangeSearch(unittest.TestCase):
xb = np.zeros((10, 5), dtype='float32')
xb[:, 0] = np.arange(10) + 1000
index = faiss.IndexIDMap2(sub_index)
index.add_with_ids(xb, np.arange(10) + 100)
index.add_with_ids(xb, np.arange(10, dtype=np.int64) + 100)
dist = float(np.linalg.norm(xb[3] - xb[0])) * 0.99
res_subindex = sub_index.range_search(xb[[0], :], dist)
res_index = index.range_search(xb[[0], :], dist)
@@ -189,7 +189,8 @@ class TestUpdate(unittest.TestCase):
# revert order of the 200 first vectors
nu = 200
index.update_vectors(np.arange(nu), xb[nu - 1::-1].copy())
index.update_vectors(np.arange(nu).astype('int64'),
xb[nu - 1::-1].copy())
recons_after = np.vstack([index.reconstruct(i) for i in range(nb)])
@@ -432,6 +433,7 @@ class TestIVFFlatDedup(unittest.TestCase):
# test remove
toremove = np.hstack((np.arange(3, 1000, 5), np.arange(850, 950)))
toremove = toremove.astype(np.int64)
index_ref.remove_ids(toremove)
index_new.remove_ids(toremove)
tests/test_meta_index.py
+4 -4
View File
@@ -34,7 +34,7 @@ class IDRemap(unittest.TestCase):
_Dref, Iref = index.search(xq, k)
# try a remapping
ids = np.arange(nb)[::-1].copy()
ids = np.arange(nb)[::-1].copy().astype('int64')
sub_index = faiss.IndexPQ(d, 8, 8)
index2 = faiss.IndexIDMap(sub_index)
@@ -62,7 +62,7 @@ class IDRemap(unittest.TestCase):
_Dref, Iref = index.search(xq, k)
# try a remapping
ids = np.arange(nb)[::-1].copy()
ids = np.arange(nb)[::-1].copy().astype('int64')
index2 = faiss.IndexIVFPQ(coarse_quantizer, d,
ncentroids, 8, 8)
@@ -210,14 +210,14 @@ class Merge(unittest.TestCase):
index.add(xb)
else:
gen = np.random.RandomState(1234)
id_list = gen.permutation(nb * 7)[:nb]
id_list = gen.permutation(nb * 7)[:nb].astype('int64')
index.add_with_ids(xb, id_list)
print('ref search ntotal=%d' % index.ntotal)
Dref, Iref = index.search(xq, k)
toremove = np.zeros(nq * k, dtype=int)
toremove = np.zeros(nq * k, dtype='int64')
nr = 0
for i in range(nq):
for j in range(k):
tests/test_referenced_objects.py
+2 -2
View File
@@ -70,10 +70,10 @@ class TestReferenced(unittest.TestCase):
def test_IDMap(self):
sub_index = faiss.IndexFlatL2(d)
index = faiss.IndexIDMap(sub_index)
index.add_with_ids(xb, np.arange(len(xb)))
index.add_with_ids(xb, np.arange(len(xb), dtype='int64'))
del sub_index
gc.collect()
index.add_with_ids(xb, np.arange(len(xb)))
index.add_with_ids(xb, np.arange(len(xb), dtype='int64'))
def test_shards(self):
index = faiss.IndexShards(d)
tests/test_standalone_codec.py
+2 -2
View File
@@ -250,7 +250,7 @@ class LatticeTest(unittest.TestCase):
codec = faiss.ZnSphereCodecAlt(dim, r2)
rs = np.random.RandomState(123)
n = 100
codes = rs.randint(codec.nv, size=n).astype('uint64')
codes = rs.randint(codec.nv, size=n, dtype='uint64')
x = np.empty((n, dim), dtype='float32')
codec.decode_multi(n, swig_ptr(codes), swig_ptr(x))
codes2 = np.empty(n, dtype='uint64')
@@ -286,7 +286,7 @@ class TestBitstring(unittest.TestCase):
nbit = int(1 + 62 * rs.rand() ** 4)
if sz + nbit > nbyte * 8:
break
x = rs.randint(1 << nbit)
x = int(rs.randint(1 << nbit, dtype='int64'))
bw.write(x, nbit)
ctrl.append((nbit, x))
sz += nbit