diff --git a/.github/workflows/build-pip.yml b/.github/workflows/build-pip.yml new file mode 100644 index 000000000..855da5821 --- /dev/null +++ b/.github/workflows/build-pip.yml @@ -0,0 +1,82 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +name: Build pip wheels + +on: + workflow_call: + workflow_dispatch: + +jobs: + build-wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-14, macos-15-intel, windows-2022, 2-core-ubuntu-arm] + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Build wheels + uses: pypa/cibuildwheel@ee02a1537ce3071a004a6b08c41e72f0fdc42d9a # v3.4.0 + env: + CIBW_BUILD_VERBOSITY: 1 + + - name: Upload wheels + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse/*.whl + + publish: + name: Publish to PyPI + needs: [build-wheels] + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + environment: + name: pypi + url: https://pypi.org/p/faiss-cpu + permissions: + id-token: write + contents: read + steps: + - name: Download all artifacts + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + path: dist + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # release/v1 + + publish-testpypi: + name: Publish to TestPyPI + needs: [build-wheels] + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + environment: + name: testpypi + url: https://test.pypi.org/p/faiss-cpu + permissions: + id-token: write + contents: read + steps: + - name: Download all artifacts + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + path: dist + merge-multiple: true + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82792cbc7..05f66a49f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,3 +15,5 @@ jobs: secrets: ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + build-pip: + uses: ./.github/workflows/build-pip.yml diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES new file mode 100644 index 000000000..e06ba0bed --- /dev/null +++ b/THIRD_PARTY_NOTICES @@ -0,0 +1,410 @@ +Third-Party Notices for faiss pip wheels +======================================== + +The faiss pip wheels bundle the following third-party libraries as binary +dependencies. Their licenses are reproduced below. + + +1. OpenBLAS (Linux, Windows wheels) +------------------------------------------- + +Copyright (c) 2011-2014, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: BSD-3-Clause +Source: https://github.com/OpenMathLib/OpenBLAS + + +2. LLVM OpenMP Runtime (libomp) (macOS wheels) +----------------------------------------------- + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== + +The software contained in this directory tree is dual licensed under both the +University of Illinois "BSD-Like" license and the MIT license. As a user of +this code you may choose to use it under either license. As a contributor, +you agree to allow your code to be used under both. The full text of the +relevant licenses is included below. + +In addition, a license agreement from the copyright/patent holders of the +software contained in this directory tree is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 1997-2019 Intel Corporation + +All rights reserved. + +Developed by: + OpenMP Runtime Team + Intel Corporation + http://www.openmprtl.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of Intel Corporation OpenMP Runtime Team nor the + names of its contributors may be used to endorse or promote products + derived from this Software without specific prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 1997-2019 Intel Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +============================================================================== + +Intel Corporation + +Software Grant License Agreement ("Agreement") + +Except for the license granted herein to you, Intel Corporation ("Intel") reserves +all right, title, and interest in and to the Software (defined below). + +Definition + +"Software" means the code and documentation as well as any original work of +authorship, including any modifications or additions to an existing work, that +is intentionally submitted by Intel to llvm.org (http://llvm.org) ("LLVM") for +inclusion in, or documentation of, any of the products owned or managed by LLVM +(the "Work"). For the purposes of this definition, "submitted" means any form of +electronic, verbal, or written communication sent to LLVM or its +representatives, including but not limited to communication on electronic +mailing lists, source code control systems, and issue tracking systems that are +managed by, or on behalf of, LLVM for the purpose of discussing and improving +the Work, but excluding communication that is conspicuously marked otherwise. + +1. Grant of Copyright License. Subject to the terms and conditions of this + Agreement, Intel hereby grants to you and to recipients of the Software + distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable copyright license to reproduce, prepare derivative + works of, publicly display, publicly perform, sublicense, and distribute the + Software and such derivative works. + +2. Grant of Patent License. Subject to the terms and conditions of this + Agreement, Intel hereby grants you and to recipients of the Software + distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable (except as stated in this section) patent license + to make, have made, use, offer to sell, sell, import, and otherwise transfer + the Work, where such license applies only to those patent claims licensable + by Intel that are necessarily infringed by Intel's Software alone or by + combination of the Software with the Work to which such Software was + submitted. If any entity institutes patent litigation against Intel or any + other entity (including a cross-claim or counterclaim in a lawsuit) alleging + that Intel's Software, or the Work to which Intel has contributed constitutes + direct or contributory patent infringement, then any patent licenses granted + to that entity under this Agreement for the Software or Work shall terminate + as of the date such litigation is filed. + +Unless required by applicable law or agreed to in writing, the software is +provided on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied, including, without limitation, any warranties or +conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. + +============================================================================== +License: Apache-2.0 WITH LLVM-exception +Source: https://github.com/llvm/llvm-project diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt index 0a08f8783..ea7033182 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -652,14 +652,28 @@ if(MKL_FOUND) target_link_libraries(faiss_avx512_spr PRIVATE ${MKL_LIBRARIES}) target_link_libraries(faiss_sve PRIVATE ${MKL_LIBRARIES}) else() - find_package(BLAS REQUIRED) + # Prefer threaded OpenBLAS (OpenMP or pthreads) over the serial default. + # On RHEL/Fedora, the default libopenblas.so is the serial variant; + # libopenblaso is OpenMP-threaded and libopenblasp is pthreads-threaded. + find_library(BLAS_PREFER_THREADED NAMES openblaso openblasp) + if(BLAS_PREFER_THREADED) + message(STATUS "Using threaded OpenBLAS: ${BLAS_PREFER_THREADED}") + set(BLAS_LIBRARIES "${BLAS_PREFER_THREADED}") + # OpenBLAS bundles LAPACK routines, so no separate find_package(LAPACK) + # is needed. Calling it would trigger FindBLAS internally, which + # unconditionally resets BLAS_LIBRARIES to the serial libopenblas, + # causing auditwheel to bundle both serial and threaded variants. + set(LAPACK_LIBRARIES "${BLAS_PREFER_THREADED}") + else() + find_package(BLAS REQUIRED) + find_package(LAPACK REQUIRED) + endif() target_link_libraries(faiss PRIVATE ${BLAS_LIBRARIES}) target_link_libraries(faiss_avx2 PRIVATE ${BLAS_LIBRARIES}) target_link_libraries(faiss_avx512 PRIVATE ${BLAS_LIBRARIES}) target_link_libraries(faiss_avx512_spr PRIVATE ${BLAS_LIBRARIES}) target_link_libraries(faiss_sve PRIVATE ${BLAS_LIBRARIES}) - find_package(LAPACK REQUIRED) target_link_libraries(faiss PRIVATE ${LAPACK_LIBRARIES}) target_link_libraries(faiss_avx2 PRIVATE ${LAPACK_LIBRARIES}) target_link_libraries(faiss_avx512 PRIVATE ${LAPACK_LIBRARIES}) diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index ca521d756..765c9d2b9 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -184,15 +184,7 @@ set_property(TARGET faiss_example_external_module PROPERTY SWIG_COMPILE_OPTIONS if(AIX) set_target_properties(swigfaiss PROPERTIES SUFFIX .a) set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .a) -elseif(NOT WIN32) - # NOTE: Python does not recognize the dylib extension. - set_target_properties(swigfaiss PROPERTIES SUFFIX .so) - set_target_properties(swigfaiss_avx2 PROPERTIES SUFFIX .so) - set_target_properties(swigfaiss_avx512 PROPERTIES SUFFIX .so) - set_target_properties(swigfaiss_avx512_spr PROPERTIES SUFFIX .so) - set_target_properties(swigfaiss_sve PROPERTIES SUFFIX .so) - set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .so) -else() +elseif(WIN32) # we need bigobj for the swig wrapper target_compile_options(swigfaiss PRIVATE /bigobj) target_compile_options(swigfaiss_avx2 PRIVATE /bigobj) @@ -200,6 +192,15 @@ else() target_compile_options(swigfaiss_avx512_spr PRIVATE /bigobj) target_compile_options(swigfaiss_sve PRIVATE /bigobj) target_compile_options(faiss_example_external_module PRIVATE /bigobj) +elseif(NOT SKBUILD_SABI_VERSION) + # NOTE: Python does not recognize the dylib extension. + # When building abi3, the suffix is set to .abi3.so below. + set_target_properties(swigfaiss PROPERTIES SUFFIX .so) + set_target_properties(swigfaiss_avx2 PROPERTIES SUFFIX .so) + set_target_properties(swigfaiss_avx512 PROPERTIES SUFFIX .so) + set_target_properties(swigfaiss_avx512_spr PROPERTIES SUFFIX .so) + set_target_properties(swigfaiss_sve PROPERTIES SUFFIX .so) + set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .so) endif() if(FAISS_ENABLE_SVS) @@ -239,43 +240,80 @@ endif() find_package(OpenMP REQUIRED) +# When scikit-build-core requests abi3 (Stable ABI), it passes +# SKBUILD_SABI_VERSION. Use Development.SABIModule to link against the +# stable Python ABI instead of the version-specific one. +if(SKBUILD_SABI_VERSION) + find_package(Python REQUIRED + COMPONENTS Development.SABIModule NumPy + ) +else() + find_package(Python REQUIRED + COMPONENTS Development.Module NumPy + ) +endif() + +# Select the Python module target based on whether abi3 is requested. +if(SKBUILD_SABI_VERSION) + set(FAISS_PYTHON_MODULE_TARGET Python::SABIModule) +else() + set(FAISS_PYTHON_MODULE_TARGET Python::Module) +endif() + +# When building for abi3, define Py_LIMITED_API and set the .abi3.so suffix +# so the wheel is tagged as abi3-compatible. +if(SKBUILD_SABI_VERSION) + # 0x030A0000 = Python 3.10 + set(FAISS_PY_LIMITED_API "0x030A0000") + foreach(tgt swigfaiss swigfaiss_avx2 swigfaiss_avx512 + swigfaiss_avx512_spr swigfaiss_sve faiss_example_external_module) + if(TARGET ${tgt}) + target_compile_definitions(${tgt} + PRIVATE Py_LIMITED_API=${FAISS_PY_LIMITED_API}) + if(NOT WIN32) + set_target_properties(${tgt} PROPERTIES SUFFIX ".abi3.so") + endif() + endif() + endforeach() +endif() + target_link_libraries(swigfaiss PRIVATE faiss - Python::Module + ${FAISS_PYTHON_MODULE_TARGET} Python::NumPy OpenMP::OpenMP_CXX ) target_link_libraries(swigfaiss_avx2 PRIVATE faiss_avx2 - Python::Module + ${FAISS_PYTHON_MODULE_TARGET} Python::NumPy OpenMP::OpenMP_CXX ) target_link_libraries(swigfaiss_avx512 PRIVATE faiss_avx512 - Python::Module + ${FAISS_PYTHON_MODULE_TARGET} Python::NumPy OpenMP::OpenMP_CXX ) target_link_libraries(swigfaiss_avx512_spr PRIVATE faiss_avx512_spr - Python::Module + ${FAISS_PYTHON_MODULE_TARGET} Python::NumPy OpenMP::OpenMP_CXX ) target_link_libraries(swigfaiss_sve PRIVATE faiss_sve - Python::Module + ${FAISS_PYTHON_MODULE_TARGET} Python::NumPy OpenMP::OpenMP_CXX ) target_link_libraries(faiss_example_external_module PRIVATE - Python::Module + ${FAISS_PYTHON_MODULE_TARGET} Python::NumPy OpenMP::OpenMP_CXX swigfaiss @@ -291,16 +329,16 @@ target_include_directories(swigfaiss_avx512_spr PRIVATE ${PROJECT_SOURCE_DIR}/.. target_include_directories(swigfaiss_sve PRIVATE ${PROJECT_SOURCE_DIR}/../..) target_include_directories(faiss_example_external_module PRIVATE ${PROJECT_SOURCE_DIR}/../..) -find_package(Python REQUIRED - COMPONENTS Development.Module NumPy -) - add_library(faiss_python_callbacks STATIC EXCLUDE_FROM_ALL python_callbacks.cpp ) set_property(TARGET faiss_python_callbacks PROPERTY POSITION_INDEPENDENT_CODE ON ) +if(SKBUILD_SABI_VERSION) + target_compile_definitions(faiss_python_callbacks + PRIVATE Py_LIMITED_API=${FAISS_PY_LIMITED_API}) +endif() if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX") target_link_libraries(faiss_python_callbacks PRIVATE faiss) @@ -328,3 +366,95 @@ configure_file(array_conversions.py array_conversions.py COPYONLY) # file(GLOB files "${PROJECT_SOURCE_DIR}/../../contrib/*.py") file(COPY ${PROJECT_SOURCE_DIR}/../../contrib DESTINATION .) + +# Install targets for scikit-build-core / pip wheel packaging. + +# Set RPATH so the SWIG modules find libfaiss in the same directory. +# auditwheel/delocate follow RPATH to locate libraries for bundling. +foreach(tgt swigfaiss swigfaiss_avx2 swigfaiss_avx512 + swigfaiss_avx512_spr swigfaiss_sve) + if(TARGET ${tgt}) + if(APPLE) + set_target_properties(${tgt} PROPERTIES INSTALL_RPATH "@loader_path") + else() + set_target_properties(${tgt} PROPERTIES INSTALL_RPATH "$ORIGIN") + endif() + endif() +endforeach() + +# Install the SWIG extension modules (.so/.pyd). +# In DD mode, swigfaiss_avx2/avx512/avx512_spr/sve are also built and +# loader.py selects the best one at runtime based on CPU capabilities. +foreach(tgt swigfaiss swigfaiss_avx2 swigfaiss_avx512 + swigfaiss_avx512_spr swigfaiss_sve) + if(TARGET ${tgt}) + get_target_property(_excluded ${tgt} EXCLUDE_FROM_ALL) + if(NOT _excluded) + install(TARGETS ${tgt} + LIBRARY DESTINATION faiss + RUNTIME DESTINATION faiss + ) + endif() + endif() +endforeach() + +# Install the core faiss shared libraries into the Python package directory +# so that auditwheel/delocate can find and bundle them with the SWIG modules. +# Guard: when Python bindings are built standalone (e.g. conda), the faiss +# target is an installed import library, not a build target. +foreach(tgt faiss faiss_avx2 faiss_avx512 faiss_avx512_spr faiss_sve) + if(TARGET ${tgt}) + get_target_property(_imported ${tgt} IMPORTED) + get_target_property(_excluded ${tgt} EXCLUDE_FROM_ALL) + if(NOT _imported AND NOT _excluded) + install(TARGETS ${tgt} + LIBRARY DESTINATION faiss + RUNTIME DESTINATION faiss + ) + endif() + endif() +endforeach() + +# Install SWIG-generated Python wrappers. +# In DD mode, each variant has its own .py wrapper that loader.py imports. +foreach(mod swigfaiss swigfaiss_avx2 swigfaiss_avx512 + swigfaiss_avx512_spr swigfaiss_sve) + if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${mod}.py" OR TARGET ${mod}) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${mod}.py" + DESTINATION faiss + OPTIONAL + ) + endif() +endforeach() + +# Install hand-written Python source files +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/__init__.py" + "${CMAKE_CURRENT_SOURCE_DIR}/loader.py" + "${CMAKE_CURRENT_SOURCE_DIR}/class_wrappers.py" + "${CMAKE_CURRENT_SOURCE_DIR}/extra_wrappers.py" + "${CMAKE_CURRENT_SOURCE_DIR}/gpu_wrappers.py" + "${CMAKE_CURRENT_SOURCE_DIR}/array_conversions.py" + DESTINATION faiss +) + +# Install type stubs if present +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/__init__.pyi") + install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/__init__.pyi" DESTINATION faiss) +endif() +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/py.typed") + install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/py.typed" DESTINATION faiss) +endif() + +# Install contrib subpackage (excluding Meta-internal *_fb.py files) +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../../contrib/" + DESTINATION faiss/contrib + FILES_MATCHING PATTERN "*.py" + PATTERN "*_fb.py" EXCLUDE +) + +# Install contrib/torch subpackage data files +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../../contrib/torch/" + DESTINATION faiss/contrib/torch + FILES_MATCHING PATTERN "*.md" +) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..476c9aa36 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,168 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +[build-system] +requires = ["scikit-build-core>=0.10", "swig>=4.2,<5", "numpy>=2.0"] +build-backend = "scikit_build_core.build" + +[project] +name = "faiss-cpu" +dynamic = ["version"] +description = "A library for efficient similarity search and clustering of dense vectors." +readme = "README.md" +license = "MIT" +license-files = ["LICENSE", "THIRD_PARTY_NOTICES"] +requires-python = ">=3.10" +authors = [ + {name = "Meta AI Research"}, +] +keywords = ["search", "nearest-neighbors", "clustering", "vectors", "similarity"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = ["numpy>=1.25", "packaging"] + +[project.urls] +Homepage = "https://github.com/facebookresearch/faiss" +Documentation = "https://github.com/facebookresearch/faiss/wiki" +Repository = "https://github.com/facebookresearch/faiss" +Issues = "https://github.com/facebookresearch/faiss/issues" + +[tool.scikit-build] +cmake.build-type = "Release" + +# Pull version from CMakeLists.txt's project(VERSION ...) — single source of truth. +metadata.version.provider = "scikit_build_core.metadata.regex" +metadata.version.input = "CMakeLists.txt" +metadata.version.regex = '^\s+VERSION\s+(?P[0-9]+\.[0-9]+\.[0-9]+)' + +# Exclude C++ headers, cmake configs, static libs, and binaries from the wheel. +# Only the Python package (faiss/) and bundled shared libs remain. +wheel.exclude = ["include/**", "share/**", "lib/**", "lib64/**", "bin/**"] + +[tool.scikit-build.cmake.define] +FAISS_OPT_LEVEL = "dd" +FAISS_ENABLE_GPU = "OFF" +FAISS_ENABLE_PYTHON = "ON" +FAISS_ENABLE_MKL = "OFF" +FAISS_ENABLE_C_API = "OFF" +FAISS_ENABLE_EXTRAS = "OFF" +FAISS_ENABLE_SVS = "OFF" +FAISS_USE_LTO = "OFF" +BUILD_TESTING = "OFF" +BUILD_SHARED_LIBS = "ON" + +# Linux: enable LTO and strip dead code and symbols to reduce wheel size. +[[tool.scikit-build.overrides]] +if.platform-system = "linux" +inherit.cmake.define = "append" +cmake.define.FAISS_USE_LTO = "ON" +cmake.define.CMAKE_C_FLAGS = "-fdata-sections -ffunction-sections" +cmake.define.CMAKE_CXX_FLAGS = "-fdata-sections -ffunction-sections" +cmake.define.CMAKE_SHARED_LINKER_FLAGS = "-Wl,--gc-sections -Wl,--strip-all" +cmake.define.CMAKE_MODULE_LINKER_FLAGS = "-Wl,--gc-sections -Wl,--strip-all" + +# macOS: enable LTO and strip dead code to reduce wheel size. +[[tool.scikit-build.overrides]] +if.platform-system = "darwin" +inherit.cmake.define = "append" +cmake.define.FAISS_USE_LTO = "ON" +cmake.define.CMAKE_SHARED_LINKER_FLAGS = "-Wl,-dead_strip" +cmake.define.CMAKE_MODULE_LINKER_FLAGS = "-Wl,-dead_strip" + +# Stable ABI (abi3): one cp310 wheel covers all 3.10+. +# Excluded: Windows (abi3+SWIG+NumPy broken), free-threaded (no stable ABI). +[[tool.scikit-build.overrides]] +if.platform-system = "^(darwin|linux)" +if.abi-flags = "^$" # Free-threaded Python does not support stable ABI. +wheel.py-api = "cp310" + +# macOS arm64: AppleClang lacks OpenMP; point CMake at Homebrew's libomp. +# BLAS uses Apple Accelerate (found automatically). +[[tool.scikit-build.overrides]] +if.platform-system = "darwin" +if.platform-machine = "arm64" +inherit.cmake.define = "append" +cmake.define.OpenMP_CXX_FLAGS = "-Xpreprocessor -fopenmp -I/opt/homebrew/opt/libomp/include" +cmake.define.OpenMP_CXX_LIB_NAMES = "omp" +cmake.define.OpenMP_omp_LIBRARY = "/opt/homebrew/opt/libomp/lib/libomp.dylib" + +# macOS Intel: Homebrew installs to /usr/local instead of /opt/homebrew. +[[tool.scikit-build.overrides]] +if.platform-system = "darwin" +if.platform-machine = "x86_64" +inherit.cmake.define = "append" +cmake.define.OpenMP_CXX_FLAGS = "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include" +cmake.define.OpenMP_CXX_LIB_NAMES = "omp" +cmake.define.OpenMP_omp_LIBRARY = "/usr/local/opt/libomp/lib/libomp.dylib" + +# Windows: compile generic (no SIMD), use OpenBLAS. +[[tool.scikit-build.overrides]] +if.platform-system = "win32" +inherit.cmake.define = "append" +cmake.define.FAISS_OPT_LEVEL = "generic" + +[tool.cibuildwheel] +# Build all Python versions on Windows (no abi3); cp310 only on Linux/macOS (abi3). +# Skip 32-bit, musl, and free-threaded builds. +build = "cp310-* cp311-* cp312-* cp313-* cp314-*" +skip = "*-win32 *-manylinux_i686 *-musllinux_* cp3*t-*" +test-requires = ["numpy", "pytest", "scipy"] +test-command = "python -m pytest {project}/tests/test_wheel_smoke.py -v" + +# On Linux and macOS, only build the cp310 abi3 wheel. +[[tool.cibuildwheel.overrides]] +select = "*-manylinux* *-macosx*" +build = "cp310-*" + +# Run abi3audit on Linux and macOS to guarantee Stable ABI compliance. +# We use 'append' so that the default auditwheel/delocate commands still run first. +[[tool.cibuildwheel.overrides]] +select = "*-manylinux* *-macosx*" +inherit.repair-wheel-command = "append" +repair-wheel-command = "pipx run abi3audit --strict --report {wheel}" + +# Linux: Install OpenBLAS (threaded variant for multi-threaded BLAS). +[[tool.cibuildwheel.overrides]] +select = "*-manylinux*" +before-all = "dnf install -y openblas-devel openblas-openmp" + +[tool.cibuildwheel.macos] +before-all = "brew install libomp" + +# Deployment target must match brew dylibs' minos for delocate compatibility. +[[tool.cibuildwheel.overrides]] +select = "*-macosx_arm64" +environment = {MACOSX_DEPLOYMENT_TARGET = "14.0"} + +# macOS Intel: runner uses macOS 15, so brew dylibs have minos 15.0. +[[tool.cibuildwheel.overrides]] +select = "*-macosx_x86_64" +environment = {MACOSX_DEPLOYMENT_TARGET = "15.0"} + +[tool.cibuildwheel.windows] +# Install prebuilt OpenBLAS (statically-linked MinGW runtime, no external deps). +before-all = "powershell -Command \"Invoke-WebRequest -Uri 'https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.30/OpenBLAS-0.3.30-x64.zip' -OutFile openblas.zip; Expand-Archive openblas.zip -DestinationPath C:/openblas -Force\"" +repair-wheel-command = "pip install delvewheel && delvewheel repair -w {dest_dir} {wheel} --add-path C:/openblas/bin --ignore-in-wheel" + +[tool.cibuildwheel.windows.environment] +# BLAS_ROOT tells CMake's FindBLAS (via CMP0074) where to find OpenBLAS. +# PATH includes the bin/ dir so delvewheel and tests can find libopenblas.dll. +BLAS_ROOT = "C:/openblas" +LAPACK_ROOT = "C:/openblas" +PATH = "C:/openblas/bin;$PATH" diff --git a/tests/test_wheel_smoke.py b/tests/test_wheel_smoke.py new file mode 100644 index 000000000..5697fe236 --- /dev/null +++ b/tests/test_wheel_smoke.py @@ -0,0 +1,199 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +""" +Wheel smoke tests for faiss-cpu pip package. + +These tests validate that a pip-installed wheel is correctly packaged: + - Shared libraries load (SWIG, BLAS, OpenMP) + - Core search functionality works + - Key index types are registered + - Serialization roundtrips + - Python reference counting is safe + - Contrib submodules are included + +They are intentionally minimal and fast (~15s total). The full test suite +runs separately in the cmake-based CI. +""" + +import platform +import unittest +import numpy as np + +import faiss + + +class TestImportAndMetadata(unittest.TestCase): + """Catch: missing shared libs, broken SWIG, wrong Python ABI.""" + + def test_version_attribute(self): + assert hasattr(faiss, "__version__") + parts = faiss.__version__.split(".") + assert len(parts) == 3 + assert all(p.isdigit() for p in parts) + + def test_compile_options(self): + opts = faiss.get_compile_options() + assert isinstance(opts, str) + + +class TestOpenMP(unittest.TestCase): + """Catch: OpenMP not linked.""" + + def test_openmp_threads(self): + n = faiss.omp_get_max_threads() + assert n >= 1 + faiss.omp_set_num_threads(2) + assert faiss.omp_get_max_threads() == 2 + # Restore + faiss.omp_set_num_threads(n) + + +class TestFlatSearch(unittest.TestCase): + """Catch: BLAS not linked, SIMD codepath issues.""" + + def test_flat_l2(self): + d, n = 64, 1000 + np.random.seed(42) + xb = np.random.random((n, d)).astype("float32") + index = faiss.IndexFlatL2(d) + index.add(xb) + D, I = index.search(xb[:5], 10) + assert I.shape == (5, 10) + # Nearest neighbor of each vector should be itself + np.testing.assert_array_equal(I[:, 0], np.arange(5)) + # Self-distance should be ~0 + np.testing.assert_allclose(D[:, 0], 0, atol=1e-5) + + def test_flat_ip(self): + d, n = 64, 1000 + np.random.seed(42) + xb = np.random.random((n, d)).astype("float32") + faiss.normalize_L2(xb) + index = faiss.IndexFlatIP(d) + index.add(xb) + D, I = index.search(xb[:5], 10) + assert I.shape == (5, 10) + np.testing.assert_array_equal(I[:, 0], np.arange(5)) + np.testing.assert_allclose(D[:, 0], 1.0, atol=1e-5) + + +class TestIndexFactory(unittest.TestCase): + """Catch: missing index implementations, broken factory string parsing.""" + + def test_ivf_pq(self): + d, n = 64, 10000 + np.random.seed(42) + xb = np.random.random((n, d)).astype("float32") + index = faiss.index_factory(d, "IVF32,PQ8") + assert not index.is_trained + index.train(xb) + assert index.is_trained + index.add(xb) + assert index.ntotal == n + D, I = index.search(xb[:5], 10) + assert I.shape == (5, 10) + + def test_hnsw(self): + d, n = 64, 2000 + np.random.seed(42) + xb = np.random.random((n, d)).astype("float32") + index = faiss.IndexHNSWFlat(d, 16) + index.add(xb) + assert index.ntotal == n + D, I = index.search(xb[:5], 10) + assert I.shape == (5, 10) + # HNSW should find the vector itself as nearest neighbor + np.testing.assert_array_equal(I[:, 0], np.arange(5)) + + +class TestSerialization(unittest.TestCase): + """Catch: broken I/O, buffer handling.""" + + def test_serialize_deserialize_roundtrip(self): + d = 32 + np.random.seed(42) + xb = np.random.random((100, d)).astype("float32") + index = faiss.IndexFlatL2(d) + index.add(xb) + + # Serialize to bytes + data = faiss.serialize_index(index) + assert isinstance(data, np.ndarray) + assert data.dtype == np.uint8 + + # Deserialize + index2 = faiss.deserialize_index(data) + assert index2.ntotal == 100 + + # Results should match + D1, I1 = index.search(xb[:3], 5) + D2, I2 = index2.search(xb[:3], 5) + np.testing.assert_array_equal(I1, I2) + np.testing.assert_allclose(D1, D2) + + +class TestGCSafety(unittest.TestCase): + """Catch: Python reference counting bugs, crashes on GC.""" + + def test_ivf_quantizer_lifecycle(self): + d, n = 32, 5000 + np.random.seed(42) + xb = np.random.random((n, d)).astype("float32") + + quantizer = faiss.IndexFlatL2(d) + index = faiss.IndexIVFFlat(quantizer, d, 32) + index.train(xb) + index.add(xb) + + # Delete the quantizer reference — the index should still work + # because faiss.__init__.py adds it to referenced_objects + del quantizer + + D, I = index.search(xb[:3], 5) + assert I.shape == (3, 5) + + +class TestContribImports(unittest.TestCase): + """Catch: missing Python files in the wheel.""" + + def test_contrib_modules(self): + import faiss.contrib.datasets + import faiss.contrib.evaluation + import faiss.contrib.inspect_tools + import faiss.contrib.ivf_tools + import faiss.contrib.exhaustive_search + import faiss.contrib.factory_tools + import faiss.contrib.vecs_io # noqa: F401 + + def test_contrib_torch_subpackage(self): + try: + import torch # noqa: F401 + except ImportError: + raise unittest.SkipTest("torch not available") + import faiss.contrib.torch + import faiss.contrib.torch.clustering + import faiss.contrib.torch.quantization # noqa: F401 + + +class TestSIMD(unittest.TestCase): + """Catch: Dynamic Dispatch not selecting a SIMD path (x86_64 only).""" + + @unittest.skipIf( + platform.machine() not in ("x86_64", "AMD64"), + "SIMD level check only meaningful on x86_64", + ) + @unittest.skipIf( + platform.system() == "Windows", + "Windows wheels are built generic (no SIMD) — see pyproject.toml", + ) + def test_dd_selects_simd_path(self): + opts = faiss.get_compile_options() + if "DD" not in opts: + raise unittest.SkipTest("Not a Dynamic Dispatch build") + # DD builds should select at least AVX2 on modern x86_64 + assert "AVX2" in opts or "AVX512" in opts, ( + f"DD build did not select a SIMD path: {opts}" + )