Avoid building OnDiskInvertedLists on Windows. (#1374)

Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/1374

Test Plan: Imported from OSS

Reviewed By: mdouze

Differential Revision: D23314729

Pulled By: beauby

fbshipit-source-id: 5ad7fa3ed830b17a5be66fb2995dd94e079d8507

faiss/CMakeLists.txt

@@ -34,7 +34,6 @@ add_library(faiss
   InvertedLists.cpp
   MatrixStats.cpp
   MetaIndexes.cpp
-  OnDiskInvertedLists.cpp
   VectorTransform.cpp
   clone_index.cpp
   index_factory.cpp
@@ -89,7 +88,6 @@ set(FAISS_HEADERS
   MatrixStats.h
   MetaIndexes.h
   MetricType.h
-  OnDiskInvertedLists.h
   VectorTransform.h
   clone_index.h
   index_factory.h
@@ -118,6 +116,11 @@ set(FAISS_HEADERS
   utils/utils.h
 )
 
+if(NOT WIN32)
+  target_sources(faiss PRIVATE OnDiskInvertedLists.cpp)
+  list(APPEND FAISS_HEADERS OnDiskInvertedLists.h)
+endif()
+
 if(FAISS_OPT_LEVEL STREQUAL "avx2")
   target_compile_options(faiss PRIVATE $<$<COMPILE_LANGUAGE:CXX>:-mavx2 -mpopcnt>)
   set_target_properties(faiss PROPERTIES OUTPUT_NAME "faiss_avx2")

faiss/impl/index_read.cpp

@@ -12,10 +12,13 @@
 #include <cstdio>
 #include <cstdlib>
 
-#include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#ifndef _MSC_VER
+#include <sys/mman.h>
+#endif // !_MSC_VER
+
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/io.h>
 #include <faiss/impl/io_macros.h>
@@ -36,14 +39,15 @@
 #include <faiss/IndexScalarQuantizer.h>
 #include <faiss/IndexHNSW.h>
 #include <faiss/IndexLattice.h>
-
-#include <faiss/OnDiskInvertedLists.h>
 #include <faiss/IndexBinaryFlat.h>
 #include <faiss/IndexBinaryFromFloat.h>
 #include <faiss/IndexBinaryHNSW.h>
 #include <faiss/IndexBinaryIVF.h>
 #include <faiss/IndexBinaryHash.h>
 
+#ifndef _MSC_VER
+#include <faiss/OnDiskInvertedLists.h>
+#endif // !_MSC_VER
 
 
 namespace faiss {
@@ -194,6 +198,8 @@ InvertedLists *read_InvertedLists (IOReader *f, int io_flags) {
             }
         }
         return ails;
+
+#ifndef _MSC_VER
     } else if (h == fourcc ("ilar") && (io_flags & IO_FLAG_SKIP_IVF_DATA)) {
         // code is always ilxx where xx is specific to the type of invlists we want
         // so we get the 16 high bits from the io_flag and the 16 low bits as "il"
@@ -207,6 +213,7 @@ InvertedLists *read_InvertedLists (IOReader *f, int io_flags) {
                 f, io_flags, nlist, code_size, sizes);
     } else {
         return InvertedListsIOHook::lookup(h)->read(f, io_flags);
+#endif // !_MSC_VER
     }
 }
 
@@ -785,6 +792,7 @@ IndexBinary *read_index_binary (const char *fname, int io_flags) {
     return idx;
 }
 
+#ifndef _MSC_VER
 
 /**********************************************************
  * InvertedListIOHook's
@@ -852,7 +860,7 @@ void InvertedListsIOHook::print_callbacks()
     }
 }
 
-
+#endif // !_MSC_VER
 
 
 

faiss/impl/index_write.cpp

@@ -12,10 +12,13 @@
 #include <cstdio>
 #include <cstdlib>
 
-#include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#ifndef _MSC_VER
+#include <sys/mman.h>
+#endif // !_MSC_VER
+
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/io.h>
 #include <faiss/impl/io_macros.h>
@@ -37,13 +40,15 @@
 #include <faiss/IndexHNSW.h>
 #include <faiss/IndexLattice.h>
 
-#include <faiss/OnDiskInvertedLists.h>
 #include <faiss/IndexBinaryFlat.h>
 #include <faiss/IndexBinaryFromFloat.h>
 #include <faiss/IndexBinaryHNSW.h>
 #include <faiss/IndexBinaryIVF.h>
 #include <faiss/IndexBinaryHash.h>
 
+#ifndef _MSC_VER
+#include <faiss/OnDiskInvertedLists.h>
+#endif // !_MSC_VER
 
 
 /*************************************************************
@@ -211,6 +216,7 @@ void write_InvertedLists (const InvertedLists *ils, IOWriter *f) {
                 WRITEANDCHECK (ails->ids[i].data(), n);
             }
         }
+#ifndef _MSC_VER
     } else {
 
         InvertedListsIOHook::lookup_classname(
@@ -222,6 +228,7 @@ void write_InvertedLists (const InvertedLists *ils, IOWriter *f) {
         uint32_t h = fourcc ("il00");
         WRITE1 (h);
         */
+#endif // !_MSC_VER
     }
 }
 

faiss/index_io.h

@@ -76,6 +76,7 @@ void write_InvertedLists (const InvertedLists *ils, IOWriter *f);
 InvertedLists *read_InvertedLists (IOReader *reader, int io_flags = 0);
 
 
+#ifndef _MSC_VER
 /** Callbacks to handle other types of InvertedList objects.
  *
  * The callbacks should be registered with add_callback before calling
@@ -119,6 +120,7 @@ struct InvertedListsIOHook {
 
 };
 
+#endif // !_MSC_VER
 
 
 } // namespace faiss

faiss/python/swigfaiss.swig

@@ -114,7 +114,10 @@ extern "C" {
 #include <faiss/utils/random.h>
 #include <faiss/utils/Heap.h>
 #include <faiss/impl/AuxIndexStructures.h>
+
+#ifndef _MSC_VER
 #include <faiss/OnDiskInvertedLists.h>
+#endif // !_MSC_VER
 
 #include <faiss/Clustering.h>
 
@@ -225,7 +228,10 @@ namespace std {
 %template(InvertedListsPtrVector) std::vector<faiss::InvertedLists*>;
 %template(RepeatVector) std::vector<faiss::Repeat>;
 %template(ClusteringIterationStatsVector) std::vector<faiss::ClusteringIterationStats>;
+
+#ifndef SWIGWIN
 %template(OnDiskOneListVector) std::vector<faiss::OnDiskOneList>;
+#endif // !SWIGWIN
 
 #ifdef GPU_WRAPPER
 %template(GpuResourcesVector) std::vector<faiss::gpu::GpuResourcesProvider*>;
@@ -366,9 +372,11 @@ void gpu_sync_all_devices()
 %include  <faiss/IndexHNSW.h>
 %include  <faiss/IndexIVFFlat.h>
 
+#ifndef SWIGWIN
 %warnfilter(401) faiss::OnDiskInvertedListsIOHook;
 %ignore OnDiskInvertedListsIOHook;
 %include  <faiss/OnDiskInvertedLists.h>
+#endif // !SWIGWIN
 
 %include  <faiss/impl/lattice_Zn.h>
 %include  <faiss/IndexLattice.h>
@@ -670,7 +678,9 @@ struct AsyncIndexSearchC {
 
 %typemap(out) faiss::InvertedLists * {
     DOWNCAST (ArrayInvertedLists)
+#ifndef SWIGWIN
     DOWNCAST (OnDiskInvertedLists)
+#endif // !SWIGWIN
     DOWNCAST (VStackInvertedLists)
     DOWNCAST (HStackInvertedLists)
     DOWNCAST (MaskedInvertedLists)
@@ -707,7 +717,6 @@ faiss::InvertedLists * downcast_InvertedLists (faiss::InvertedLists *il)
 %include  <faiss/impl/io.h>
 %include  <faiss/index_io.h>
 %include  <faiss/clone_index.h>
-
 %newobject index_factory;
 %newobject index_binary_factory;
 
@@ -1029,3 +1038,4 @@ struct MapLong2Long {
  %}
 
 // End of file...
+ 

tests/test_index_composite.py

@@ -12,6 +12,7 @@ import faiss
 import os
 import shutil
 import tempfile
+import platform
 
 from common import get_dataset_2
 
@@ -63,6 +64,8 @@ class TestRemove(unittest.TestCase):
     def test_remove_regular(self):
         self.do_merge_then_remove(False)
 
+    @unittest.skipIf(platform.system() == 'Windows',
+                     'OnDiskInvertedLists is unsupported on Windows.')
     def test_remove_ondisk(self):
         self.do_merge_then_remove(True)
 
@@ -306,6 +309,8 @@ class TestTransformChain(unittest.TestCase):
 
         assert np.all(I == I2)
 
+@unittest.skipIf(platform.system() == 'Windows', \
+                 'Mmap not supported on Windows.')
 class TestRareIO(unittest.TestCase):
 
     def compare_results(self, index1, index2, xq):
@@ -476,7 +481,8 @@ class TestSerialize(unittest.TestCase):
         Dnew, Inew = index3.search(xq, 5)
         assert np.all(Dnew == Dref) and np.all(Inew == Iref)
 
-
+@unittest.skipIf(platform.system() == 'Windows',
+                 'OnDiskInvertedLists is unsupported on Windows.')
 class TestRenameOndisk(unittest.TestCase):
 
     def test_rename(self):