Facebook sync 20191129 (#1048)

Looks good!

Index.h

@@ -18,7 +18,7 @@
 
 #define FAISS_VERSION_MAJOR 1
 #define FAISS_VERSION_MINOR 6
-#define FAISS_VERSION_PATCH 0
+#define FAISS_VERSION_PATCH 1
 
 /**
  * @namespace faiss

benchs/bench_all_ivf/datasets.py

@@ -64,6 +64,7 @@ def fvecs_write(fname, m):
     ivecs_write(fname, m.view('int32'))
 
 
+
 #################################################################
 # Dataset
 #################################################################

benchs/distributed_ondisk/README.md

@@ -1,3 +1,4 @@
+
 # Distributed on-disk index for 1T-scale datasets 
 
 This is code corresponding to the description in [Indexing 1T vectors](https://github.com/facebookresearch/faiss/wiki/Indexing-1T-vectors). 

benchs/distributed_ondisk/combined_index.py

@@ -3,6 +3,8 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+#!/usr/bin/env python3
+
 import os
 import faiss
 import numpy as np
@@ -29,7 +31,7 @@ class CombinedIndex:
                 indexes.append(index)
                 il = faiss.extract_index_ivf(index).invlists
             else:
-                assert False
+                raise AssertionError
             ilv.push_back(il)
         print()
 

benchs/distributed_ondisk/distributed_kmeans.py

@@ -2,6 +2,7 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
+
 #! /usr/bin/env python3
 
 """
@@ -356,7 +357,7 @@ def main():
         elif args.indata.endswith('.npy'):
             x = np.load(args.indata, mmap_mode='r')
         else:
-            assert False
+            raise AssertionError
 
         if args.i1 == -1:
             args.i1 = len(x)
@@ -386,7 +387,8 @@ def main():
             True
         )
     else:
-        assert False
+        raise AssertionError
+
 
     if args.server:
         print('starting server')

benchs/distributed_ondisk/distributed_query_demo.py

@@ -2,6 +2,7 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
+
 import os
 import faiss
 import numpy as np

benchs/distributed_ondisk/make_index_vslice.py

@@ -2,6 +2,7 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
+
 import os
 import time
 import numpy as np

benchs/distributed_ondisk/merge_to_ondisk.py

@@ -49,7 +49,7 @@ if __name__ == '__main__':
 
     index0 = None
 
-    for fname, index in pool.imap(load_index, args.inputs):
+    for _, index in pool.imap(load_index, args.inputs):
         if index is None:
             continue
         index_ivf = faiss.extract_index_ivf(index)

benchs/distributed_ondisk/rpc.py

@@ -2,6 +2,9 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
+
+#!/usr/bin/env python3
+
 """
 Simplistic RPC implementation.
 Exposes all functions of a Server object.
@@ -163,7 +166,7 @@ class Server:
         except EOFError:
             self.log("EOF during communication")
             traceback.print_exc(50,self.logf)
-        except:
+        except BaseException:
             # unexpected
             traceback.print_exc(50,sys.stderr)
             sys.exit(1)

benchs/distributed_ondisk/search_server.py

@@ -2,6 +2,7 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
+
 import os
 import time
 import rpc

clone_index.cpp

@@ -107,6 +107,12 @@ Index *Cloner::clone_Index (const Index *index)
                dynamic_cast<const IndexPreTransform*> (index)) {
         IndexPreTransform *res = new IndexPreTransform ();
         res->d = ipt->d;
+        res->ntotal = ipt->ntotal;
+        res->is_trained = ipt->is_trained;
+        res->metric_type = ipt->metric_type;
+        res->metric_arg = ipt->metric_arg;
+
+
         res->index = clone_Index (ipt->index);
         for (int i = 0; i < ipt->chain.size(); i++)
             res->chain.push_back (clone_VectorTransform (ipt->chain[i]));

python/faiss.py

@@ -8,8 +8,6 @@
 # not linting this file because it imports * form swigfaiss, which
 # causes a ton of useless warnings.
 
-from __future__ import print_function
-
 import numpy as np
 import sys
 import inspect
@@ -47,7 +45,7 @@ try:
 
 except ImportError:
     # we import * so that the symbol X can be accessed as faiss.X
-    logger.error("Loading faiss.")
+    logger.info("Loading faiss.")
     from .swigfaiss import *
 
 

python/setup.py

@@ -32,7 +32,7 @@ are implemented on the GPU. It is developed by Facebook AI Research.
 """
 setup(
     name='faiss',
-    version='1.6.0',
+    version='1.6.1',
     description='A library for efficient similarity search and clustering of dense vectors',
     long_description=long_description,
     url='https://github.com/facebookresearch/faiss',

tests/test_factory.py

@@ -5,10 +5,13 @@
 
 #! /usr/bin/env python2
 
+import numpy as np
 import unittest
 import faiss
 
 
+
+
 class TestFactory(unittest.TestCase):
 
     def test_factory_1(self):
@@ -50,3 +53,15 @@ class TestFactory(unittest.TestCase):
     def test_factory_4(self):
         index = faiss.index_factory(12, "IVF10,FlatDedup")
         assert index.instances is not None
+
+
+
+class TestCloneSize(unittest.TestCase):
+
+    def test_clone_size(self):
+        index = faiss.index_factory(20, 'PCA10,Flat')
+        xb = faiss.rand((100, 20))
+        index.train(xb)
+        index.add(xb)
+        index2 = faiss.clone_index(index)
+        assert index2.ntotal == 100