diff --git a/acinclude/ax_check_cpu.m4 b/acinclude/ax_check_cpu.m4 index d0a04a770..fc61fc91e 100644 --- a/acinclude/ax_check_cpu.m4 +++ b/acinclude/ax_check_cpu.m4 @@ -3,24 +3,22 @@ AC_DEFUN([AX_CPU_ARCH], [ AC_MSG_CHECKING([for cpu arch]) -AC_CANONICAL_TARGET -case $target in - amd64-* | x86_64-*) - ARCH_CPUFLAGS="-mpopcnt" - ARCH_CXXFLAGS="-m64" + AC_CANONICAL_TARGET - AX_GCC_X86_CPU_SUPPORTS(avx2, - [ARCH_CPUFLAGS+=" -mavx2 -mf16c"], - [ARCH_CPUFLAGS+=" -msse4"]) - ;; - aarch64*-*) + case $target in + amd64-* | x86_64-*) + ARCH_CPUFLAGS="-mpopcnt -msse4" + ARCH_CXXFLAGS="-m64" + ;; + aarch64*-*) dnl This is an arch for Nvidia Xavier a proper detection would be nice. - ARCH_CPUFLAGS="-march=armv8.2-a" - ;; - *) ;; -esac -AC_MSG_RESULT([$target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS]) + ARCH_CPUFLAGS="-march=armv8.2-a" + ;; + *) ;; + esac + +AC_MSG_RESULT([$target CPUFLAGS+="$ARCH_CPUFLAGS" CXXFLAGS+="$ARCH_CXXFLAGS"]) AC_SUBST(ARCH_CPUFLAGS) AC_SUBST(ARCH_CXXFLAGS) diff --git a/acinclude/ax_gcc_x86_cpu_supports.m4 b/acinclude/ax_gcc_x86_cpu_supports.m4 deleted file mode 100644 index fd37b4fa1..000000000 --- a/acinclude/ax_gcc_x86_cpu_supports.m4 +++ /dev/null @@ -1,104 +0,0 @@ -# ============================================================================ -# https://www.gnu.org/software/autoconf-archive/ax_gcc_x86_cpu_supports.html -# ============================================================================ -# -# SYNOPSIS -# -# AX_GCC_X86_CPU_SUPPORTS(X86-INSTRUCTION-SET, -# [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND]) -# -# DESCRIPTION -# -# Checks if the host cpu supports X86-INSTRUCTION-SET. The instruction set -# that can be tested are "mmx, popcnt, sse, sse2, sse3, sse4.1, sse4.2, -# sse4a, avx, avx2, avx512f, fma, fma4, bmi, bmi2". If the instruction set -# is supported by the host cpu, the C preprocessor macro -# HAVE_XXX_INSTRUCTIONS is set to 1. The XXX is up-cased instruction case -# with dot replaced by underscore. For example, the test for "sse4.2" -# would export HAVE_SSE4_2_INSTRUCTIONS=1. This macro requires gcc -# extended builtin function "__builtin_cpu_init" and -# "__builtin_cpu_supports" to detect the cpu features. It will error out -# if the compiler doesn't has these builtins. -# -# If the test for the instruction set succeeded, the hook ACTION-IF-FOUND -# would run. Otherwise the hook ACTION-IF-NOT-FOUND would run if -# specified. -# -# See also AX_CHECK_X86_FEATURES, which checks all the possible -# instruction set and export the corresponding CFLAGS. -# -# LICENSE -# -# Copyright (c) 2016 Felix Chern -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. - -#serial 3 - -AC_DEFUN_ONCE([_AX_GCC_X86_CPU_INIT], - [AC_LANG_PUSH([C]) - AC_CACHE_CHECK([for gcc __builtin_cpu_init function], - [ax_cv_gcc_check_x86_cpu_init], - [AC_RUN_IFELSE( - [AC_LANG_PROGRAM([#include ], - [__builtin_cpu_init ();]) - ], - [ax_cv_gcc_check_x86_cpu_init=yes], - [ax_cv_gcc_check_x86_cpu_init=no])]) - AS_IF([test "X$ax_cv_gcc_check_x86_cpu_init" = "Xno"], - [AC_MSG_ERROR([Need GCC to support X86 CPU features tests])]) -]) - -AC_DEFUN([AX_GCC_X86_CPU_SUPPORTS], - [AC_REQUIRE([AC_PROG_CC]) - AC_REQUIRE([_AX_GCC_X86_CPU_INIT]) - AC_LANG_PUSH([C]) - AS_VAR_PUSHDEF([gcc_x86_feature], [AS_TR_SH([ax_cv_gcc_x86_cpu_supports_$1])]) - AC_CACHE_CHECK([for x86 $1 instruction support], - [gcc_x86_feature], - [AC_RUN_IFELSE( - [AC_LANG_PROGRAM( [#include ], - [ __builtin_cpu_init (); - if (__builtin_cpu_supports("$1")) - return 0; - return 1; - ])], - [gcc_x86_feature=yes], - [gcc_x86_feature=no] - )] - ) - AC_LANG_POP([C]) - AS_VAR_IF([gcc_x86_feature],[yes], - [AC_DEFINE( - AS_TR_CPP([HAVE_$1_INSTRUCTIONS]), - [1], - [Define if $1 instructions are supported]) - $2], - [$3] - ) - AS_VAR_POPDEF([gcc_x86_feature]) -]) diff --git a/conda/faiss-gpu/build.sh b/conda/faiss-gpu/build.sh index b0da4d8c5..25326c90d 100644 --- a/conda/faiss-gpu/build.sh +++ b/conda/faiss-gpu/build.sh @@ -1,5 +1,16 @@ +# Build avx2 version +CXXFLAGS="-mavx2 -mf16c" ./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH" +make -j $CPU_COUNT +make -C python _swigfaiss_avx2.so +make clean + +# Build vanilla version (no avx) ./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH" make -j $CPU_COUNT +make -C python _swigfaiss.so + +make -C python build + cd python -make + $PYTHON setup.py install --single-version-externally-managed --record=record.txt diff --git a/conda/faiss-gpu/conda_build_config.yaml b/conda/faiss-gpu/conda_build_config.yaml index 0c9ab8900..da98e5d41 100644 --- a/conda/faiss-gpu/conda_build_config.yaml +++ b/conda/faiss-gpu/conda_build_config.yaml @@ -5,7 +5,7 @@ cudatoolkit: - 9.0 - 9.2 - 10.0 -# - 10.1 # cudatoolkit 10.1 packages are not yet available on Anaconda. + - 10.1 pin_run_as_build: cudatoolkit: max_pin: x.x diff --git a/conda/faiss/build.sh b/conda/faiss/build.sh index 66ca439af..87ccb4cad 100644 --- a/conda/faiss/build.sh +++ b/conda/faiss/build.sh @@ -1,5 +1,16 @@ +# Build avx2 version +CXXFLAGS="-mavx2 -mf16c" ./configure --without-cuda +make -j $CPU_COUNT +make -C python _swigfaiss_avx2.so +make clean + +# Build vanilla version (no avx) ./configure --without-cuda make -j $CPU_COUNT +make -C python _swigfaiss.so + +make -C python build + cd python -make + $PYTHON setup.py install --single-version-externally-managed --record=record.txt diff --git a/configure b/configure index d5968a501..ed40daefd 100755 --- a/configure +++ b/configure @@ -2065,48 +2065,6 @@ $as_echo "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_cxx_check_func - -# ac_fn_c_try_run LINENO -# ---------------------- -# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes -# that executables *can* be run. -ac_fn_c_try_run () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' - { { case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_try") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; }; then : - ac_retval=0 -else - $as_echo "$as_me: program exited with status $ac_status" >&5 - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=$ac_status -fi - rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_run cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. @@ -6849,136 +6807,25 @@ test -n "$target_alias" && NONENONEs,x,x, && program_prefix=${target_alias}- -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for gcc __builtin_cpu_init function" >&5 -$as_echo_n "checking for gcc __builtin_cpu_init function... " >&6; } -if ${ax_cv_gcc_check_x86_cpu_init+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot run test program while cross compiling -See \`config.log' for more details" "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -__builtin_cpu_init (); - ; - return 0; -} - -_ACEOF -if ac_fn_c_try_run "$LINENO"; then : - ax_cv_gcc_check_x86_cpu_init=yes -else - ax_cv_gcc_check_x86_cpu_init=no -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_gcc_check_x86_cpu_init" >&5 -$as_echo "$ax_cv_gcc_check_x86_cpu_init" >&6; } - if test "X$ax_cv_gcc_check_x86_cpu_init" = "Xno"; then : - as_fn_error $? "Need GCC to support X86 CPU features tests" "$LINENO" 5 -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for cpu arch" >&5 $as_echo_n "checking for cpu arch... " >&6; } -case $target in - amd64-* | x86_64-*) - ARCH_CPUFLAGS="-mpopcnt" - ARCH_CXXFLAGS="-m64" + case $target in + amd64-* | x86_64-*) + ARCH_CPUFLAGS="-mpopcnt -msse4" + ARCH_CXXFLAGS="-m64" + ;; + aarch64*-*) + ARCH_CPUFLAGS="-march=armv8.2-a" + ;; + *) ;; + esac - - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for x86 avx2 instruction support" >&5 -$as_echo_n "checking for x86 avx2 instruction support... " >&6; } -if ${ax_cv_gcc_x86_cpu_supports_avx2+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot run test program while cross compiling -See \`config.log' for more details" "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ - __builtin_cpu_init (); - if (__builtin_cpu_supports("avx2")) - return 0; - return 1; - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_run "$LINENO"; then : - ax_cv_gcc_x86_cpu_supports_avx2=yes -else - ax_cv_gcc_x86_cpu_supports_avx2=no - -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_gcc_x86_cpu_supports_avx2" >&5 -$as_echo "$ax_cv_gcc_x86_cpu_supports_avx2" >&6; } - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - if test "x$ax_cv_gcc_x86_cpu_supports_avx2" = xyes; then : - -$as_echo "#define HAVE_AVX2_INSTRUCTIONS 1" >>confdefs.h - - ARCH_CPUFLAGS+=" -mavx2 -mf16c" -else - ARCH_CPUFLAGS+=" -msse4" - -fi - - - ;; - aarch64*-*) - ARCH_CPUFLAGS="-march=armv8.2-a" - ;; - *) ;; -esac -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS" >&5 -$as_echo "$target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $target CPUFLAGS+=\"$ARCH_CPUFLAGS\" CXXFLAGS+=\"$ARCH_CXXFLAGS\"" >&5 +$as_echo "$target CPUFLAGS+=\"$ARCH_CPUFLAGS\" CXXFLAGS+=\"$ARCH_CXXFLAGS\"" >&6; } diff --git a/makefile.inc.in b/makefile.inc.in index bc46e879b..19d8511cd 100644 --- a/makefile.inc.in +++ b/makefile.inc.in @@ -8,7 +8,7 @@ CXXCPP = @CXXCPP@ CPPFLAGS = -DFINTEGER=int @CPPFLAGS@ @OPENMP_CXXFLAGS@ @NVCC_CPPFLAGS@ CXXFLAGS = -fPIC @ARCH_CXXFLAGS@ -Wno-sign-compare @CXXFLAGS@ CPUFLAGS = @ARCH_CPUFLAGS@ -LDFLAGS = @OPENMP_CXXFLAGS@ @LDFLAGS@ @NVCC_LDFLAGS@ +LDFLAGS = @OPENMP_LDFLAGS@ @LDFLAGS@ @NVCC_LDFLAGS@ LIBS = @BLAS_LIBS@ @LAPACK_LIBS@ @LIBS@ @NVCC_LIBS@ PYTHONCFLAGS = @PYTHON_CFLAGS@ -I@NUMPY_INCLUDE@ SWIGFLAGS = -DSWIGWORDSIZE64 diff --git a/python/Makefile b/python/Makefile index 28fed702c..283656825 100644 --- a/python/Makefile +++ b/python/Makefile @@ -15,12 +15,15 @@ all: build swigfaiss.cpp: swigfaiss.swig ../libfaiss.a $(SWIG) -python -c++ -Doverride= -I../ $(SWIGFLAGS) -o $@ $< -swigfaiss.o: swigfaiss.cpp +swigfaiss_avx2.cpp: swigfaiss.swig ../libfaiss.a + $(SWIG) -python -c++ -Doverride= -module swigfaiss_avx2 -I../ $(SWIGFLAGS) -o $@ $< + +%.o: %.cpp $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) $(PYTHONCFLAGS) \ -I../ -c $< -o $@ # Extension is .so even on OSX. -_swigfaiss.so: swigfaiss.o ../libfaiss.a +_%.so: %.o ../libfaiss.a $(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS) build: _swigfaiss.so faiss.py @@ -30,8 +33,7 @@ install: build $(PYTHON) setup.py install clean: - rm -f swigfaiss.cpp swigfaiss.o swigfaiss.py - rm -f _swigfaiss.so + rm -f swigfaiss*.cpp swigfaiss*.o swigfaiss*.py _swigfaiss*.so rm -rf build/ .PHONY: all build clean install diff --git a/python/faiss.py b/python/faiss.py index fe0f2ee16..3adbbf4a8 100644 --- a/python/faiss.py +++ b/python/faiss.py @@ -8,14 +8,44 @@ # not linting this file because it imports * form swigfaiss, which # causes a ton of useless warnings. +from __future__ import print_function + import numpy as np import sys import inspect import pdb +import platform +import subprocess -# we import * so that the symbol X can be accessed as faiss.X -from .swigfaiss import * +def instruction_set(): + if platform.system() == "Darwin": + if subprocess.check_output(["/usr/sbin/sysctl", "hw.optional.avx2_0"])[-1] == '1': + return "AVX2" + else: + return "default" + elif platform.system() == "Linux": + import numpy.distutils.cpuinfo + if "avx2" in numpy.distutils.cpuinfo.cpu.info[0]['flags']: + return "AVX2" + else: + return "default" + + +try: + instr_set = instruction_set() + if instr_set == "AVX2": + print("Loading faiss with AVX2 support.", file=sys.stderr) + from .swigfaiss_avx2 import * + else: + print("Loading faiss.", file=sys.stderr) + from .swigfaiss import * + +except ImportError: + # we import * so that the symbol X can be accessed as faiss.X + print("Loading faiss.", file=sys.stderr) + from .swigfaiss import * + __version__ = "%d.%d.%d" % (FAISS_VERSION_MAJOR, FAISS_VERSION_MINOR, diff --git a/python/setup.py b/python/setup.py index cd92fb133..854d01a14 100644 --- a/python/setup.py +++ b/python/setup.py @@ -16,6 +16,11 @@ os.mkdir("faiss") shutil.copyfile("faiss.py", "faiss/__init__.py") shutil.copyfile("swigfaiss.py", "faiss/swigfaiss.py") shutil.copyfile("_swigfaiss.so", "faiss/_swigfaiss.so") +try: + shutil.copyfile("swigfaiss_avx2.py", "faiss/swigfaiss_avx2.py") + shutil.copyfile("_swigfaiss_avx2.so", "faiss/_swigfaiss_avx2.so") +except: + pass long_description=""" Faiss is a library for efficient similarity search and clustering of dense