Run time detection of avx2 for conda packages. (#957)

* Build conda CPU packages with run-time detection of AVX2.

* Add cudatoolkit 10.1 to conda list.

* Default to sse4 on x86_64.
This commit is contained in:
Lucas Hosseini
2019-09-20 20:17:01 +02:00
committed by GitHub
parent 36ddba9196
commit 3d0ece0cb7
10 changed files with 94 additions and 294 deletions
+13 -15
View File
@@ -3,24 +3,22 @@
AC_DEFUN([AX_CPU_ARCH], [
AC_MSG_CHECKING([for cpu arch])
AC_CANONICAL_TARGET
case $target in
amd64-* | x86_64-*)
ARCH_CPUFLAGS="-mpopcnt"
ARCH_CXXFLAGS="-m64"
AC_CANONICAL_TARGET
AX_GCC_X86_CPU_SUPPORTS(avx2,
[ARCH_CPUFLAGS+=" -mavx2 -mf16c"],
[ARCH_CPUFLAGS+=" -msse4"])
;;
aarch64*-*)
case $target in
amd64-* | x86_64-*)
ARCH_CPUFLAGS="-mpopcnt -msse4"
ARCH_CXXFLAGS="-m64"
;;
aarch64*-*)
dnl This is an arch for Nvidia Xavier a proper detection would be nice.
ARCH_CPUFLAGS="-march=armv8.2-a"
;;
*) ;;
esac
AC_MSG_RESULT([$target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS])
ARCH_CPUFLAGS="-march=armv8.2-a"
;;
*) ;;
esac
AC_MSG_RESULT([$target CPUFLAGS+="$ARCH_CPUFLAGS" CXXFLAGS+="$ARCH_CXXFLAGS"])
AC_SUBST(ARCH_CPUFLAGS)
AC_SUBST(ARCH_CXXFLAGS)
-104
View File
@@ -1,104 +0,0 @@
# ============================================================================
# https://www.gnu.org/software/autoconf-archive/ax_gcc_x86_cpu_supports.html
# ============================================================================
#
# SYNOPSIS
#
# AX_GCC_X86_CPU_SUPPORTS(X86-INSTRUCTION-SET,
# [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND])
#
# DESCRIPTION
#
# Checks if the host cpu supports X86-INSTRUCTION-SET. The instruction set
# that can be tested are "mmx, popcnt, sse, sse2, sse3, sse4.1, sse4.2,
# sse4a, avx, avx2, avx512f, fma, fma4, bmi, bmi2". If the instruction set
# is supported by the host cpu, the C preprocessor macro
# HAVE_XXX_INSTRUCTIONS is set to 1. The XXX is up-cased instruction case
# with dot replaced by underscore. For example, the test for "sse4.2"
# would export HAVE_SSE4_2_INSTRUCTIONS=1. This macro requires gcc
# extended builtin function "__builtin_cpu_init" and
# "__builtin_cpu_supports" to detect the cpu features. It will error out
# if the compiler doesn't has these builtins.
#
# If the test for the instruction set succeeded, the hook ACTION-IF-FOUND
# would run. Otherwise the hook ACTION-IF-NOT-FOUND would run if
# specified.
#
# See also AX_CHECK_X86_FEATURES, which checks all the possible
# instruction set and export the corresponding CFLAGS.
#
# LICENSE
#
# Copyright (c) 2016 Felix Chern <[email protected]>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.
#
# As a special exception, the respective Autoconf Macro's copyright owner
# gives unlimited permission to copy, distribute and modify the configure
# scripts that are the output of Autoconf when processing the Macro. You
# need not follow the terms of the GNU General Public License when using
# or distributing such scripts, even though portions of the text of the
# Macro appear in them. The GNU General Public License (GPL) does govern
# all other use of the material that constitutes the Autoconf Macro.
#
# This special exception to the GPL applies to versions of the Autoconf
# Macro released by the Autoconf Archive. When you make and distribute a
# modified version of the Autoconf Macro, you may extend this special
# exception to the GPL to apply to your modified version as well.
#serial 3
AC_DEFUN_ONCE([_AX_GCC_X86_CPU_INIT],
[AC_LANG_PUSH([C])
AC_CACHE_CHECK([for gcc __builtin_cpu_init function],
[ax_cv_gcc_check_x86_cpu_init],
[AC_RUN_IFELSE(
[AC_LANG_PROGRAM([#include <stdlib.h>],
[__builtin_cpu_init ();])
],
[ax_cv_gcc_check_x86_cpu_init=yes],
[ax_cv_gcc_check_x86_cpu_init=no])])
AS_IF([test "X$ax_cv_gcc_check_x86_cpu_init" = "Xno"],
[AC_MSG_ERROR([Need GCC to support X86 CPU features tests])])
])
AC_DEFUN([AX_GCC_X86_CPU_SUPPORTS],
[AC_REQUIRE([AC_PROG_CC])
AC_REQUIRE([_AX_GCC_X86_CPU_INIT])
AC_LANG_PUSH([C])
AS_VAR_PUSHDEF([gcc_x86_feature], [AS_TR_SH([ax_cv_gcc_x86_cpu_supports_$1])])
AC_CACHE_CHECK([for x86 $1 instruction support],
[gcc_x86_feature],
[AC_RUN_IFELSE(
[AC_LANG_PROGRAM( [#include <stdlib.h> ],
[ __builtin_cpu_init ();
if (__builtin_cpu_supports("$1"))
return 0;
return 1;
])],
[gcc_x86_feature=yes],
[gcc_x86_feature=no]
)]
)
AC_LANG_POP([C])
AS_VAR_IF([gcc_x86_feature],[yes],
[AC_DEFINE(
AS_TR_CPP([HAVE_$1_INSTRUCTIONS]),
[1],
[Define if $1 instructions are supported])
$2],
[$3]
)
AS_VAR_POPDEF([gcc_x86_feature])
])
+12 -1
View File
@@ -1,5 +1,16 @@
# Build avx2 version
CXXFLAGS="-mavx2 -mf16c" ./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH"
make -j $CPU_COUNT
make -C python _swigfaiss_avx2.so
make clean
# Build vanilla version (no avx)
./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH"
make -j $CPU_COUNT
make -C python _swigfaiss.so
make -C python build
cd python
make
$PYTHON setup.py install --single-version-externally-managed --record=record.txt
+1 -1
View File
@@ -5,7 +5,7 @@ cudatoolkit:
- 9.0
- 9.2
- 10.0
# - 10.1 # cudatoolkit 10.1 packages are not yet available on Anaconda.
- 10.1
pin_run_as_build:
cudatoolkit:
max_pin: x.x
+12 -1
View File
@@ -1,5 +1,16 @@
# Build avx2 version
CXXFLAGS="-mavx2 -mf16c" ./configure --without-cuda
make -j $CPU_COUNT
make -C python _swigfaiss_avx2.so
make clean
# Build vanilla version (no avx)
./configure --without-cuda
make -j $CPU_COUNT
make -C python _swigfaiss.so
make -C python build
cd python
make
$PYTHON setup.py install --single-version-externally-managed --record=record.txt
Vendored
+12 -165
View File
@@ -2065,48 +2065,6 @@ $as_echo "$ac_res" >&6; }
eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
} # ac_fn_cxx_check_func
# ac_fn_c_try_run LINENO
# ----------------------
# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes
# that executables *can* be run.
ac_fn_c_try_run ()
{
as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
if { { ac_try="$ac_link"
case "(($ac_try" in
*\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
*) ac_try_echo=$ac_try;;
esac
eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
$as_echo "$ac_try_echo"; } >&5
(eval "$ac_link") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; } && { ac_try='./conftest$ac_exeext'
{ { case "(($ac_try" in
*\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
*) ac_try_echo=$ac_try;;
esac
eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
$as_echo "$ac_try_echo"; } >&5
(eval "$ac_try") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; }; then :
ac_retval=0
else
$as_echo "$as_me: program exited with status $ac_status" >&5
$as_echo "$as_me: failed program was:" >&5
sed 's/^/| /' conftest.$ac_ext >&5
ac_retval=$ac_status
fi
rm -rf conftest.dSYM conftest_ipa8_conftest.oo
eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
as_fn_set_status $ac_retval
} # ac_fn_c_try_run
cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
@@ -6849,136 +6807,25 @@ test -n "$target_alias" &&
NONENONEs,x,x, &&
program_prefix=${target_alias}-
ac_ext=c
ac_cpp='$CPP $CPPFLAGS'
ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
ac_compiler_gnu=$ac_cv_c_compiler_gnu
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for gcc __builtin_cpu_init function" >&5
$as_echo_n "checking for gcc __builtin_cpu_init function... " >&6; }
if ${ax_cv_gcc_check_x86_cpu_init+:} false; then :
$as_echo_n "(cached) " >&6
else
if test "$cross_compiling" = yes; then :
{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
as_fn_error $? "cannot run test program while cross compiling
See \`config.log' for more details" "$LINENO" 5; }
else
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
#include <stdlib.h>
int
main ()
{
__builtin_cpu_init ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_run "$LINENO"; then :
ax_cv_gcc_check_x86_cpu_init=yes
else
ax_cv_gcc_check_x86_cpu_init=no
fi
rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
conftest.$ac_objext conftest.beam conftest.$ac_ext
fi
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_gcc_check_x86_cpu_init" >&5
$as_echo "$ax_cv_gcc_check_x86_cpu_init" >&6; }
if test "X$ax_cv_gcc_check_x86_cpu_init" = "Xno"; then :
as_fn_error $? "Need GCC to support X86 CPU features tests" "$LINENO" 5
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for cpu arch" >&5
$as_echo_n "checking for cpu arch... " >&6; }
case $target in
amd64-* | x86_64-*)
ARCH_CPUFLAGS="-mpopcnt"
ARCH_CXXFLAGS="-m64"
case $target in
amd64-* | x86_64-*)
ARCH_CPUFLAGS="-mpopcnt -msse4"
ARCH_CXXFLAGS="-m64"
;;
aarch64*-*)
ARCH_CPUFLAGS="-march=armv8.2-a"
;;
*) ;;
esac
ac_ext=c
ac_cpp='$CPP $CPPFLAGS'
ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
ac_compiler_gnu=$ac_cv_c_compiler_gnu
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for x86 avx2 instruction support" >&5
$as_echo_n "checking for x86 avx2 instruction support... " >&6; }
if ${ax_cv_gcc_x86_cpu_supports_avx2+:} false; then :
$as_echo_n "(cached) " >&6
else
if test "$cross_compiling" = yes; then :
{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
as_fn_error $? "cannot run test program while cross compiling
See \`config.log' for more details" "$LINENO" 5; }
else
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
#include <stdlib.h>
int
main ()
{
__builtin_cpu_init ();
if (__builtin_cpu_supports("avx2"))
return 0;
return 1;
;
return 0;
}
_ACEOF
if ac_fn_c_try_run "$LINENO"; then :
ax_cv_gcc_x86_cpu_supports_avx2=yes
else
ax_cv_gcc_x86_cpu_supports_avx2=no
fi
rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
conftest.$ac_objext conftest.beam conftest.$ac_ext
fi
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_gcc_x86_cpu_supports_avx2" >&5
$as_echo "$ax_cv_gcc_x86_cpu_supports_avx2" >&6; }
ac_ext=c
ac_cpp='$CPP $CPPFLAGS'
ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
ac_compiler_gnu=$ac_cv_c_compiler_gnu
if test "x$ax_cv_gcc_x86_cpu_supports_avx2" = xyes; then :
$as_echo "#define HAVE_AVX2_INSTRUCTIONS 1" >>confdefs.h
ARCH_CPUFLAGS+=" -mavx2 -mf16c"
else
ARCH_CPUFLAGS+=" -msse4"
fi
;;
aarch64*-*)
ARCH_CPUFLAGS="-march=armv8.2-a"
;;
*) ;;
esac
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS" >&5
$as_echo "$target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS" >&6; }
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $target CPUFLAGS+=\"$ARCH_CPUFLAGS\" CXXFLAGS+=\"$ARCH_CXXFLAGS\"" >&5
$as_echo "$target CPUFLAGS+=\"$ARCH_CPUFLAGS\" CXXFLAGS+=\"$ARCH_CXXFLAGS\"" >&6; }
+1 -1
View File
@@ -8,7 +8,7 @@ CXXCPP = @CXXCPP@
CPPFLAGS = -DFINTEGER=int @CPPFLAGS@ @OPENMP_CXXFLAGS@ @NVCC_CPPFLAGS@
CXXFLAGS = -fPIC @ARCH_CXXFLAGS@ -Wno-sign-compare @CXXFLAGS@
CPUFLAGS = @ARCH_CPUFLAGS@
LDFLAGS = @OPENMP_CXXFLAGS@ @LDFLAGS@ @NVCC_LDFLAGS@
LDFLAGS = @OPENMP_LDFLAGS@ @LDFLAGS@ @NVCC_LDFLAGS@
LIBS = @BLAS_LIBS@ @LAPACK_LIBS@ @LIBS@ @NVCC_LIBS@
PYTHONCFLAGS = @PYTHON_CFLAGS@ -I@NUMPY_INCLUDE@
SWIGFLAGS = -DSWIGWORDSIZE64
+6 -4
View File
@@ -15,12 +15,15 @@ all: build
swigfaiss.cpp: swigfaiss.swig ../libfaiss.a
$(SWIG) -python -c++ -Doverride= -I../ $(SWIGFLAGS) -o $@ $<
swigfaiss.o: swigfaiss.cpp
swigfaiss_avx2.cpp: swigfaiss.swig ../libfaiss.a
$(SWIG) -python -c++ -Doverride= -module swigfaiss_avx2 -I../ $(SWIGFLAGS) -o $@ $<
%.o: %.cpp
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPUFLAGS) $(PYTHONCFLAGS) \
-I../ -c $< -o $@
# Extension is .so even on OSX.
_swigfaiss.so: swigfaiss.o ../libfaiss.a
_%.so: %.o ../libfaiss.a
$(CXX) $(SHAREDFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
build: _swigfaiss.so faiss.py
@@ -30,8 +33,7 @@ install: build
$(PYTHON) setup.py install
clean:
rm -f swigfaiss.cpp swigfaiss.o swigfaiss.py
rm -f _swigfaiss.so
rm -f swigfaiss*.cpp swigfaiss*.o swigfaiss*.py _swigfaiss*.so
rm -rf build/
.PHONY: all build clean install
+32 -2
View File
@@ -8,14 +8,44 @@
# not linting this file because it imports * form swigfaiss, which
# causes a ton of useless warnings.
from __future__ import print_function
import numpy as np
import sys
import inspect
import pdb
import platform
import subprocess
# we import * so that the symbol X can be accessed as faiss.X
from .swigfaiss import *
def instruction_set():
if platform.system() == "Darwin":
if subprocess.check_output(["/usr/sbin/sysctl", "hw.optional.avx2_0"])[-1] == '1':
return "AVX2"
else:
return "default"
elif platform.system() == "Linux":
import numpy.distutils.cpuinfo
if "avx2" in numpy.distutils.cpuinfo.cpu.info[0]['flags']:
return "AVX2"
else:
return "default"
try:
instr_set = instruction_set()
if instr_set == "AVX2":
print("Loading faiss with AVX2 support.", file=sys.stderr)
from .swigfaiss_avx2 import *
else:
print("Loading faiss.", file=sys.stderr)
from .swigfaiss import *
except ImportError:
# we import * so that the symbol X can be accessed as faiss.X
print("Loading faiss.", file=sys.stderr)
from .swigfaiss import *
__version__ = "%d.%d.%d" % (FAISS_VERSION_MAJOR,
FAISS_VERSION_MINOR,
+5
View File
@@ -16,6 +16,11 @@ os.mkdir("faiss")
shutil.copyfile("faiss.py", "faiss/__init__.py")
shutil.copyfile("swigfaiss.py", "faiss/swigfaiss.py")
shutil.copyfile("_swigfaiss.so", "faiss/_swigfaiss.so")
try:
shutil.copyfile("swigfaiss_avx2.py", "faiss/swigfaiss_avx2.py")
shutil.copyfile("_swigfaiss_avx2.so", "faiss/_swigfaiss_avx2.so")
except:
pass
long_description="""
Faiss is a library for efficient similarity search and clustering of dense