facebookresearch/faiss
1
0
Fork 0
mirror of https://github.com/facebookresearch/faiss.git synced 2026-10-12 02:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
d63faeaf5a9b5133c1a500d6c0cf9d934c075cf8
faiss/pyproject-gpu-cuvs.toml
T
Alibek Zhakubayev 3bb48643ab Pin faiss-gpu-cuvs RAPIDS deps to the 26.06 build series (#5643) 
Summary:
Pull Request resolved: https://github.com/facebookresearch/faiss/pull/5643

`faiss-gpu-cuvs` 1.15.0 fails at import on a clean install:

```
ImportError: libfaiss.so: undefined symbol: _ZTIN3rmm9bad_allocE
```

## Root cause, measured

RAPIDS 26.08 wrapped RMM in a per-release ABI inline namespace. `nm -D` on the published `librmm-cu13` wheels:

| Version | `typeinfo for rmm::bad_alloc` | Exported instead |
| --- | --- | --- |
| 26.2.0 | present | |
| 26.4.0 | present | |
| 26.6.0 | present | |
| 26.8.0 | **absent** | `typeinfo for rmm::_RMM_26_8::bad_alloc` |

`pyproject-gpu-cuvs.toml` builds against `==26.06.*` in `before-all` but declares `>=26.06,<27` at runtime and in `test-requires`. A fresh install therefore pairs a 26.06-built `libfaiss.so` with a 26.8 `librmm.so` that renamed every `rmm::` symbol. Nothing vendored covers the gap: `repair-wheel-command` hands every RAPIDS `lib*.so*` under `/opt/cuvs` to `auditwheel --exclude`.

The 1.15.0 wheel was uploaded 2026-08-03, two days before RAPIDS 26.08 reached PyPI. It published green and began failing for every new installer afterwards, which matches `build-pip-gpu / Build & test faiss-gpu-cuvs` going red on 2026-08-05 after 68 consecutive passes.

An exact-series pin rather than a narrower range is required: the inline namespace encodes the release, so **any** range spanning more than one RAPIDS minor series is broken by construction.

## This diff

- Pin `libcuvs-cu13`, `libraft-cu13`, and `librmm-cu13` to `==26.06.*` in both `dependencies` and `test-requires`.
- Declare `libraft-cu13`. `faiss/python/__init__.py` imports `libraft` and calls `libraft.load_library()`, but only `before-all` installed it, so at runtime its version floated on `libcuvs-cu13` transitive metadata.
- Correct the cuSOLVER soname in the `auditwheel` exclude list from `libcusolver.so.11` to `libcusolver.so.12`, which is what CUDA 13.2 ships.
- Update the `_preload_gpu_libs` fix-it message, which told users to install the range this diff removes.
- Add `TestCuvsRapidsPinning` to `tests/test_wheel_smoke_gpu.py`.

## Regression guard

`tests/test_wheel_smoke_gpu.py` is already the `test-command` for the cuVS wheel, so the new class runs inside `build-pip-gpu.yml` with no workflow change and no new test file. It asserts three properties of the built wheel's own `Requires-Dist`:

1. every RAPIDS distribution `_preload_gpu_libs` dlopens is declared
2. each is pinned to `==26.06.*` rather than an open range
3. the installed versions sit in that series

`_RAPIDS_SERIES` duplicates the series from `pyproject-gpu-cuvs.toml` because the wheel does not ship that file. The duplication is the tripwire: a RAPIDS bump has to touch both places deliberately.

## Prior attempt

D117099095 made the version-pin change on 2026-08-23 and was abandoned on 2026-09-04 without landing. It never failed review. `faiss/public_tld/` is ShipIt-mirrored with `requireExportConfig`, so `github-export-checks` held it Unpublished pending a `facebookresearch/faiss` PR that a bot cannot open, and three pings over 12 days went unanswered. This diff needs that same export.

## Notes

The published 1.15.0 metadata declares only `libcuvs-cu13`, not `librmm-cu13`. That is not a fbsource/GitHub divergence: `librmm-cu13` was added to `dependencies` on 2026-08-14, after the 08-03 release, and has never shipped.

`rapids-logger` is installed by `before-all` but stays undeclared here. `librmm-cu13` owns that link-time dependency and carries it in its own metadata; an unversioned duplicate would not make the resolve safer.

Reviewed By: mnorris11

Differential Revision: D120162004

fbshipit-source-id: eb6dc316a4fba2a4228eab62d7b86ac8ea94a1b7
2026-09-15 14:42:20 -07:00

169 lines
8.5 KiB
TOML
Raw Blame History

# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
[build-system]
# cuVS requires rapids-cmake which needs cmake >= 3.30.4.
requires = ["scikit-build-core>=0.10", "swig>=4.2,<5", "numpy>=2.0", "cmake>=3.30.4", "ninja"]
build-backend = "scikit_build_core.build"
[project]
name = "faiss-gpu-cuvs"
dynamic = ["version"]
description = "A library for efficient similarity search and clustering of dense vectors (GPU + cuVS/CAGRA support)."
readme = "README.md"
license = "MIT"
license-files = ["LICENSE", "THIRD_PARTY_NOTICES"]
requires-python = ">=3.11"
authors = [
{name = "Meta AI Research"},
]
keywords = ["search", "nearest-neighbors", "clustering", "vectors", "similarity", "gpu", "cuda", "cuvs", "cagra"]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"Intended Audience :: Science/Research",
"Operating System :: POSIX :: Linux",
"Programming Language :: C++",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: Implementation :: CPython",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Topic :: Software Development :: Libraries :: Python Modules",
"Environment :: GPU :: NVIDIA CUDA :: 13",
]
dependencies = [
"numpy>=1.25",
"packaging",
"nvidia-cuda-runtime>=13.2,<14",
"nvidia-cublas>=13.2,<14",
"nvidia-curand>=10.3.7,<11",
"nvidia-nvjitlink>=13.2,<14",
# Exact-series pin, not a range: RAPIDS >=26.08 wraps RMM in a per-release
# inline namespace (rmm::_RMM_26_8::), so each minor series renames the
# symbols libfaiss.so is linked against. libraft is listed because
# _preload_gpu_libs dlopens it directly.
"libcuvs-cu13==26.06.*",
"libraft-cu13==26.06.*",
"librmm-cu13==26.06.*",
]
[project.urls]
Homepage = "https://github.com/facebookresearch/faiss"
Documentation = "https://github.com/facebookresearch/faiss/wiki"
Repository = "https://github.com/facebookresearch/faiss"
Issues = "https://github.com/facebookresearch/faiss/issues"
[tool.scikit-build]
cmake.build-type = "Release"
# Pull version from CMakeLists.txt's project(VERSION ...) — single source of truth.
metadata.version.provider = "scikit_build_core.metadata.regex"
metadata.version.input = "CMakeLists.txt"
metadata.version.regex = '^\s+VERSION\s+(?P<value>[0-9]+\.[0-9]+\.[0-9]+)'
# Exclude C++ headers, cmake configs, static libs, and binaries from the wheel.
# Only the Python package (faiss/) and bundled shared libs remain.
wheel.exclude = ["include/**", "share/**", "lib/**", "lib64/**", "bin/**"]
[tool.scikit-build.cmake.define]
FAISS_OPT_LEVEL = "dd"
FAISS_ENABLE_GPU = "ON"
FAISS_ENABLE_CUVS = "ON"
FAISS_ENABLE_PYTHON = "ON"
FAISS_ENABLE_MKL = "OFF"
FAISS_ENABLE_C_API = "OFF"
FAISS_ENABLE_EXTRAS = "OFF"
FAISS_ENABLE_SVS = "OFF"
FAISS_USE_LTO = "ON"
# Turing → Hopper SASS + Hopper PTX trailer for forward compat (e.g. Blackwell JIT).
# CUDA 13 dropped offline compilation for Maxwell/Pascal/Volta, so 70 (V100) is gone
# and 75 (Turing) is the floor. Drop 72 (Jetson Xavier) for a datacenter-targeted
# wheel; include 89 (L4/L40/RTX 40xx).
CMAKE_CUDA_ARCHITECTURES = "75-real;80-real;86-real;89-real;90"
CMAKE_CUDA_FLAGS_RELEASE = "-O3 -Xfatbin=-compress-all --threads 4 -Xcompiler=-fdata-sections,-ffunction-sections"
CMAKE_C_FLAGS = "-fdata-sections -ffunction-sections"
CMAKE_CXX_FLAGS = "-fdata-sections -ffunction-sections"
CMAKE_SHARED_LINKER_FLAGS = "-Wl,--gc-sections -Wl,--strip-all"
CMAKE_MODULE_LINKER_FLAGS = "-Wl,--gc-sections -Wl,--strip-all"
BUILD_TESTING = "OFF"
BUILD_SHARED_LIBS = "ON"
# cuVS + transitive deps (raft, rmm, rapids_logger, header-only cutlass/cuco/nvtx3)
# are found via CMAKE_PREFIX_PATH (environment block below) at the RAPIDS wheel roots,
# mirroring the conda build — no explicit *_DIR, agnostic to lib/ vs lib64/.
# Stable ABI (abi3): one cp311 wheel covers all 3.11+.
# Floor is 3.11 because the runtime dep libcuvs-cu13>=26.06 (RAPIDS) requires
# Python >=3.11 — RAPIDS dropped 3.10 in the 26.04 release.
# GPU is Linux-only; free-threaded Python (cp3*t) cannot use abi3.
[[tool.scikit-build.overrides]]
if.platform-system = "linux"
if.abi-flags = "^$" # Free-threaded Python does not support stable ABI.
wheel.py-api = "cp311"
[tool.cibuildwheel]
# Pass through the host GPU so the smoke test can actually exercise CUDA + cuVS.
# Without this, faiss.get_num_gpus() returns 0 inside the container and all
# GPU tests silently skip — masking real packaging / driver / SASS-arch problems.
container-engine = { name = "docker", create-args = ["--gpus", "all"] }
build = "cp311-manylinux_x86_64"
test-requires = [
"numpy>=1.25,<3.0",
"pytest",
"nvidia-cuda-runtime>=13.2,<14",
"nvidia-cublas>=13.2,<14",
"nvidia-curand>=10.3.7,<11",
"nvidia-nvjitlink>=13.2,<14",
"libcuvs-cu13==26.06.*",
"libraft-cu13==26.06.*",
"librmm-cu13==26.06.*",
]
test-command = "python -m pytest {project}/tests/test_wheel_smoke_gpu.py -v"
[tool.cibuildwheel.linux]
# manylinux_2_28 ships gcc-toolset-14; CUDA 13.2 supports GCC 14, so install
# gcc-toolset-14 explicitly and force the build to use it via CC/CXX/CUDAHOSTCXX below.
# epel-release is needed for swig. Install the FULL cuda-toolkit-13-2 (not just nvcc +
# narrow dev libs as faiss-gpu does) because cuVS pulls in RAFT which transitively
# needs cuSOLVER, cuSPARSE, nvrtc, etc. headers — easier to install all than to track
# the exact list. Wheel size is unaffected since auditwheel excludes all CUDA libs.
#
# cuVS dev files (headers + cmake) come from RAPIDS' manylinux PyPI wheels via
# `pip install --target /opt/cuvs` — ABI-clean, so no libstdc++/libgcc_s/libgomp strip
# or RMM symlink shim. cp311 interpreter is explicit (manylinux puts no `pip` on PATH).
manylinux-x86_64-image = "manylinux_2_28"
before-all = "dnf install -y gcc-toolset-14 epel-release && dnf install -y openblas-devel openblas-openmp swig && dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && dnf install -y --setopt=obsoletes=0 cuda-toolkit-13-2 && /opt/python/cp311-cp311/bin/python -m pip install --target /opt/cuvs --no-deps --only-binary ':all:' --extra-index-url https://pypi.nvidia.com 'libcuvs-cu13==26.06.*' 'libraft-cu13==26.06.*' 'librmm-cu13==26.06.*' 'rapids-logger'"
# auditwheel excludes (not vendored; supplied at runtime by nvidia-* / libcuvs-cu13
# wheels): host driver libcuda, CUDA runtime/math libs (cusparse/cusolver are explicit —
# under the cuda-toolkit RPM, not /opt/cuvs), and every RAPIDS .so found under /opt/cuvs.
repair-wheel-command = "bash -c 'EXCLUDES=\"--exclude libcuda.so.1 --exclude libcudart.so.13 --exclude libcublas.so.13 --exclude libcublasLt.so.13 --exclude libcurand.so.10 --exclude libnvJitLink.so.13 --exclude libcusparse.so.12 --exclude libcusolver.so.12\"; for f in $(find /opt/cuvs -name \"lib*.so*\" -printf \"%f\\n\" | sort -u); do EXCLUDES=\"$EXCLUDES --exclude $f\"; done; auditwheel repair -w {dest_dir} {wheel} $EXCLUDES'"
# Canary: ensure the host GPU is actually visible to the container before tests
# run. Without this, smoke tests would silently skip on broken --gpus passthrough.
before-test = "nvidia-smi || (echo 'GPU not visible to container; check container-engine create-args' && exit 1)"
# Run abi3audit on Linux to guarantee Stable ABI compliance.
# We use 'append' so that the auditwheel repair-wheel-command from
# [tool.cibuildwheel.linux] (with the CUDA + cuVS --exclude flags) still runs first.
[[tool.cibuildwheel.overrides]]
select = "*-manylinux*"
inherit.repair-wheel-command = "append"
repair-wheel-command = "pipx run abi3audit --strict --report {wheel}"
# Build-tool paths. CUDA from /usr/local/cuda-13.2 (NVIDIA RPM convention);
# GCC pinned to toolset-14 for CUDA 13.2 support.
# CUDAHOSTCXX must match CXX or nvcc host TUs get an ABI-mismatched libstdc++.
# CMAKE_PREFIX_PATH points at the RAPIDS wheel roots so find_package(cuvs) resolves.
# Set as env (not cmake.define) so CMake merges it with scikit-build-core's own path.
[tool.cibuildwheel.linux.environment]
CUDA_HOME = "/usr/local/cuda-13.2"
CUDACXX = "/usr/local/cuda-13.2/bin/nvcc"
CC = "/opt/rh/gcc-toolset-14/root/usr/bin/gcc"
CXX = "/opt/rh/gcc-toolset-14/root/usr/bin/g++"
CUDAHOSTCXX = "/opt/rh/gcc-toolset-14/root/usr/bin/g++"
CMAKE_PREFIX_PATH = "/opt/cuvs/libcuvs:/opt/cuvs/libraft:/opt/cuvs/librmm:/opt/cuvs/rapids_logger"
Reference in New Issue View Git Blame Copy Permalink