yindo/flash-attention-prebuild-wheels-rocm
1
0
Fork 0
mirror of https://github.com/BillyOutlast/flash-attention-prebuild-wheels-rocm.git synced 2026-06-30 23:57:53 -04:00
Code Issues Packages Projects Releases 2 Wiki Activity

Merge branch 'main' of github.com:mjun0812/flash-attention-prebuild-wheels

Browse Source
This commit is contained in:
Junya Morioka
2025-05-22 23:26:46 +09:00
parent 7077c99339 60b025bc4e
commit b4d8949d68
4 changed files with 247 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/FUNDING.yml
+15
View File
@@ -0,0 +1,15 @@
# These are supported funding model platforms
github: mjun0812
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
polar: # Replace with a single Polar username
buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
thanks_dev: # Replace with a single thanks.dev username
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
.github/workflows/build.yml
+138 -138
View File
@@ -168,152 +168,152 @@ jobs:
asset_content_type: application/*
build_wheels_self_hosted:
name: Build wheels and Upload
needs: create_releases
runs-on: self-hosted
container:
image: ubuntu:22.04
env:
DEBIAN_FRONTEND: noninteractive
TERM: xterm-256color
timeout-minutes: 1000
strategy:
fail-fast: false
matrix:
flash-attn-version: ["2.7.4"]
python-version: ["3.10", "3.11", "3.12"]
torch-version: ["2.7.0"]
# https://developer.nvidia.com/cuda-toolkit-archive
cuda-version: ["12.8.1"]
exclude:
# torch < 2.2 does not support Python 3.12
- python-version: "3.12"
torch-version: "2.0.1"
- python-version: "3.12"
torch-version: "2.1.2"
# torch 2.0.1 does not support CUDA 12.x
- torch-version: "2.0.1"
cuda-version: "12.1.1"
- torch-version: "2.0.1"
cuda-version: "12.4.1"
- torch-version: "2.0.1"
cuda-version: "12.6.3"
- torch-version: "2.0.1"
cuda-version: "12.8.1"
# torch 2.7.0 does not support CUDA 12.4
- torch-version: "2.7.0"
cuda-version: "12.4.1"
name: Build wheels and Upload
needs: create_releases
runs-on: self-hosted
container:
image: ubuntu:22.04
env:
DEBIAN_FRONTEND: noninteractive
TERM: xterm-256color
timeout-minutes: 1000
strategy:
fail-fast: false
matrix:
flash-attn-version: ["2.7.4"]
python-version: ["3.10", "3.11", "3.12"]
torch-version: ["2.7.0"]
# https://developer.nvidia.com/cuda-toolkit-archive
cuda-version: ["12.8.1"]
exclude:
# torch < 2.2 does not support Python 3.12
- python-version: "3.12"
torch-version: "2.0.1"
- python-version: "3.12"
torch-version: "2.1.2"
# torch 2.0.1 does not support CUDA 12.x
- torch-version: "2.0.1"
cuda-version: "12.1.1"
- torch-version: "2.0.1"
cuda-version: "12.4.1"
- torch-version: "2.0.1"
cuda-version: "12.6.3"
- torch-version: "2.0.1"
cuda-version: "12.8.1"
# torch 2.7.0 does not support CUDA 12.4
- torch-version: "2.7.0"
cuda-version: "12.4.1"
steps:
- name: Install tools
run: |
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration
steps:
- name: Install tools
run: |
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- uses: Jimver/cuda-toolkit@master
env:
DEBIAN_FRONTEND: noninteractive
with:
cuda: ${{ matrix.cuda-version }}
linux-local-args: '["--toolkit"]'
method: "network"
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- uses: Jimver/cuda-toolkit@master
env:
DEBIAN_FRONTEND: noninteractive
with:
cuda: ${{ matrix.cuda-version }}
linux-local-args: '["--toolkit"]'
method: "network"
- name: Set CUDA and PyTorch versions
run: |
echo "MATRIX_CUDA_VERSION=$(echo ${{ matrix.cuda-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV
echo "MATRIX_TORCH_VERSION=$(echo ${{ matrix.torch-version }} | awk -F \. {'print $1 "." $2'})" >> $GITHUB_ENV
echo "CACHE_KEY=cuda-ext-${{ matrix.flash-attn-version }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-cuda${{ matrix.cuda-version }}" >> $GITHUB_ENV
- name: Set CUDA and PyTorch versions
run: |
echo "MATRIX_CUDA_VERSION=$(echo ${{ matrix.cuda-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV
echo "MATRIX_TORCH_VERSION=$(echo ${{ matrix.torch-version }} | awk -F \. {'print $1 "." $2'})" >> $GITHUB_ENV
echo "CACHE_KEY=cuda-ext-${{ matrix.flash-attn-version }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-cuda${{ matrix.cuda-version }}" >> $GITHUB_ENV
- name: Install build dependencies
run: |
sudo apt install -y ninja-build clang
pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil
- name: Install build dependencies
run: |
sudo apt install -y ninja-build clang
pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil
- name: Install PyTorch ${{ matrix.torch-version }}+cu${{ matrix.cuda-version }}
run: |
export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; \
support_cuda_versions = { \
'2.0': [117, 118], \
'2.1': [118, 121], \
'2.2': [118, 121], \
'2.3': [118, 121], \
'2.4': [118, 121, 124], \
'2.5': [118, 121, 124], \
'2.6': [118, 124, 126], \
'2.7': [118, 126, 128], \
}; \
target_cuda_versions = support_cuda_versions[env['MATRIX_TORCH_VERSION']]; \
cuda_version = int(env['MATRIX_CUDA_VERSION']); \
closest_version = min(target_cuda_versions, key=lambda x: abs(x - cuda_version)); \
print(closest_version) \
")
pip install --no-cache-dir torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/cu${TORCH_CUDA_VERSION}
nvcc --version
python -V
python -c "import torch; print('PyTorch:', torch.__version__)"
python -c "import torch; print('CUDA:', torch.version.cuda)"
python -c "from torch.utils import cpp_extension; print(cpp_extension.CUDA_HOME)"
- name: Install PyTorch ${{ matrix.torch-version }}+cu${{ matrix.cuda-version }}
run: |
export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; \
support_cuda_versions = { \
'2.0': [117, 118], \
'2.1': [118, 121], \
'2.2': [118, 121], \
'2.3': [118, 121], \
'2.4': [118, 121, 124], \
'2.5': [118, 121, 124], \
'2.6': [118, 124, 126], \
'2.7': [118, 126, 128], \
}; \
target_cuda_versions = support_cuda_versions[env['MATRIX_TORCH_VERSION']]; \
cuda_version = int(env['MATRIX_CUDA_VERSION']); \
closest_version = min(target_cuda_versions, key=lambda x: abs(x - cuda_version)); \
print(closest_version) \
")
pip install --force-reinstall --no-cache-dir torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/cu${TORCH_CUDA_VERSION}
nvcc --version
python -V
python -c "import torch; print('PyTorch:', torch.__version__)"
python -c "import torch; print('CUDA:', torch.version.cuda)"
python -c "from torch.utils import cpp_extension; print(cpp_extension.CUDA_HOME)"
- name: Checkout flash-attn
run: |
git clone https://github.com/Dao-AILab/flash-attention.git -b "v${{ matrix.flash-attn-version }}"
cd flash-attention
- name: Checkout flash-attn
run: |
git clone https://github.com/Dao-AILab/flash-attention.git -b "v${{ matrix.flash-attn-version }}"
cd flash-attention
- name: Build wheels
timeout-minutes: 800
run: |
export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export MAX_JOBS=2
export NVCC_THREADS=4
export FLASH_ATTENTION_FORCE_BUILD="TRUE"
cd flash-attention
python setup.py bdist_wheel --dist-dir=dist
base_wheel_name=$(basename $(ls dist/*.whl | head -n 1))
wheel_name=$(echo $base_wheel_name | sed "s/${{ matrix.flash-attn-version }}/${{ matrix.flash-attn-version }}+cu${{ env.MATRIX_CUDA_VERSION }}torch${{ env.MATRIX_TORCH_VERSION }}/")
mv dist/$base_wheel_name dist/$wheel_name
echo "wheel_name=$wheel_name" >> $GITHUB_ENV
- name: Build wheels
timeout-minutes: 800
run: |
export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export MAX_JOBS=2
export NVCC_THREADS=4
export FLASH_ATTENTION_FORCE_BUILD="TRUE"
cd flash-attention
python setup.py bdist_wheel --dist-dir=dist
base_wheel_name=$(basename $(ls dist/*.whl | head -n 1))
wheel_name=$(echo $base_wheel_name | sed "s/${{ matrix.flash-attn-version }}/${{ matrix.flash-attn-version }}+cu${{ env.MATRIX_CUDA_VERSION }}torch${{ env.MATRIX_TORCH_VERSION }}/")
mv dist/$base_wheel_name dist/$wheel_name
echo "wheel_name=$wheel_name" >> $GITHUB_ENV
- name: Install Test
run: |
pip install flash-attention/dist/${{ env.wheel_name }}
python -c "import flash_attn; print(flash_attn.__version__)"
- name: Install Test
run: |
pip install flash-attention/dist/${{ env.wheel_name }}
python -c "import flash_attn; print(flash_attn.__version__)"
- name: Get the tag version
id: extract_branch
run: echo "branch=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: Get the tag version
id: extract_branch
run: echo "branch=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: Get Release with Tag
id: get_release
uses: joutvhu/get-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.extract_branch.outputs.branch }}
- name: Get Release with Tag
id: get_release
uses: joutvhu/get-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.extract_branch.outputs.branch }}
- name: Upload Release Asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.get_release.outputs.upload_url }}
asset_path: flash-attention/dist/${{ env.wheel_name }}
asset_name: ${{ env.wheel_name }}
asset_content_type: application/*
- name: Upload Release Asset
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.get_release.outputs.upload_url }}
asset_path: flash-attention/dist/${{ env.wheel_name }}
asset_name: ${{ env.wheel_name }}
asset_content_type: application/*
README.md
+94 -28
View File
@@ -1,13 +1,16 @@
# flash-attention pre-build wheels
This repository provides wheels for the pre-built [flash-attention](https://github.com/Dao-AILab/flash-attention).
This repository provides wheels for the pre-built [flash-attention](https://github.com/Dao-AILab/flash-attention).
Since building flash-attention takes a **very long time** and is resource-intensive,
Since building flash-attention takes a **very long time** and is resource-intensive,
I also build and provide combinations of CUDA and PyTorch that are not officially distributed.
The building Github Actions Workflow can be found [here](./.github/workflows/build.yml).
The built packages are available on the [release page](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases).
This repository uses a self-hosted runner for building the wheels. If you find this project helpful, please consider supporting or sponsoring to help maintain the infrastructure.
[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/mjun0812)
## Install
@@ -22,7 +25,7 @@ flash_attn-2.6.3+cu124torch2.5-cp312-cp312-linux_x86_64.whl
2. Find the corresponding version of a wheel from the below table and [releases](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases)
5. Direct Install or Download and Local Install
3. Direct Install or Download and Local Install
```bash
# Direct Install
@@ -35,20 +38,32 @@ pip install ./flash_attn-2.6.3+cu124torch2.5-cp312-cp312-linux_x86_64.whl
## Packages
### v0.1.0
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.1.0)
| Flash-Attention | Python | PyTorch | CUDA |
| -------------------------- | ---------------- | ------- | ------ |
| 2.4.3, 2.5.9, 2.6.3, 2.7.4 | 3.10, 3.11, 3.12 | 2.7.0 | 12.8.1 |
v2.7.4 and v2.7.4.post1 are the same version.
From this release, self-hosted runners are used for building the some wheels.
### v0.0.9
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.9)
| Flash-Attention | Python | PyTorch | CUDA |
|-----------------|--------|---------|------|
| 2.4.3, 2.5.9, 2.6.3 | 3.10, 3.11, 3.12 | 2.7.0 | 12.8.1 |
| Flash-Attention | Python | PyTorch | CUDA |
| ------------------- | ---------------- | ------- | ------ |
| 2.4.3, 2.5.9, 2.6.3 | 3.10, 3.11, 3.12 | 2.7.0 | 12.8.1 |
### v0.0.8
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.8)
| Flash-Attention | Python | PyTorch | CUDA |
|-----------------|--------|---------|------|
| Flash-Attention | Python | PyTorch | CUDA |
| -------------------------------- | ---------------- | -------------------------- | ---------------------- |
| 2.4.3, 2.5.9, 2.6.3, 2.7.4.post1 | 3.10, 3.11, 3.12 | 2.4.1, 2.5.1, 2.6.0, 2.7.0 | 11.8.0, 12.4.1, 12.6.3 |
### v0.0.7
@@ -59,61 +74,112 @@ Skip for experimental reasons.
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.6)
| Flash-Attention | Python | PyTorch | CUDA |
|-----------------|--------|---------|------|
| Flash-Attention | Python | PyTorch | CUDA |
| -------------------------------- | ---------------- | --------------------------------- | -------------- |
| 2.4.3, 2.5.9, 2.6.3, 2.7.4.post1 | 3.10, 3.11, 3.12 | 2.2.2, 2.3.1, 2.4.1, 2.5.1, 2.6.0 | 12.4.1, 12.6.3 |
### v0.0.5
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.5)
| Flash-Attention | Python | PyTorch | CUDA |
|-----------------|--------|---------|------|
| Flash-Attention | Python | PyTorch | CUDA |
| ------------------ | ---------------- | ----------------------------------------------- | -------------- |
| 2.6.3, 2.7.4.post1 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1, 2.6.0 | 12.4.1, 12.6.3 |
### v0.0.4
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.4)
| Flash-Attention | Python | PyTorch | CUDA |
|-----------------|--------|---------|------|
| 2.7.3 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 |
| Flash-Attention | Python | PyTorch | CUDA |
| --------------- | ---------------- | ---------------------------------------- | ---------------------- |
| 2.7.3 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 |
### v0.0.3
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.3)
| Flash-Attention | Python | PyTorch | CUDA |
|-----------------|--------|---------|------|
| 2.7.2.post1 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 |
| Flash-Attention | Python | PyTorch | CUDA |
| --------------- | ---------------- | ---------------------------------------- | ---------------------- |
| 2.7.2.post1 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 |
### v0.0.2
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.2)
| Flash-Attention | Python | PyTorch | CUDA |
|-----------------|--------|---------|------|
| Flash-Attention | Python | PyTorch | CUDA |
| -------------------------------- | ---------------- | ---------------------------------------- | ---------------------- |
| 2.4.3, 2.5.6, 2.6.3, 2.7.0.post2 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 |
### v0.0.1
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.1)
|flash-attention|Python|PyTorch|CUDA|
|-|-|-|-|
|1.0.9, 2.4.3, 2.5.6, 2.5.9, 2.6.3|3.10, 3.11, 3.12|2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0|11.8.0, 12.1.1, 12.4.1|
| flash-attention | Python | PyTorch | CUDA |
| --------------------------------- | ---------------- | ---------------------------------------- | ---------------------- |
| 1.0.9, 2.4.3, 2.5.6, 2.5.9, 2.6.3 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0 | 11.8.0, 12.1.1, 12.4.1 |
### v0.0.0
[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.0)
|flash-attention|Python|PyTorch|CUDA|
|-|-|-|-|
|2.4.3, 2.5.6, 2.5.9, 2.6.3|3.11, 3.12|2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0|11.8.0, 12.1.1, 12.4.1|
| flash-attention | Python | PyTorch | CUDA |
| -------------------------- | ---------- | ---------------------------------------- | ---------------------- |
| 2.4.3, 2.5.6, 2.5.9, 2.6.3 | 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0 | 11.8.0, 12.1.1, 12.4.1 |
## Self build
## Original
If you want to build the wheels yourself, you can fork this repository and run the build workflow.
1. Fork this repository
2. Edit workflow file `.github/workflows/build.yml` to set the version you want to build.
3. Add tag `v*.*.*` to trigger the build workflow.
### Self-Hosted Runner Build
In some version combinations, you cannot build wheels on GitHub-hosted runners due to job time limitations.
To build the wheels for these versions, you can use self-hosted runners.
```bash
git clone https://github.com/mjun0812/flash-attention-prebuild-wheels.git
cd self-hosted-runner
cp env.template env
```
Edit `env` file to set the environment variables.
```bash
# Edit env
PERSONAL_ACCESS_TOKEN=[Github Personal Access Token]
```
Edit compose.yml file if you use repository folked from this repository.
```yaml
services:
runner:
privileged: true
build:
context: .
dockerfile: Dockerfile
args:
REPOSITORY_URL: [Target Repository URL]
PERSONAL_ACCESS_TOKEN: $PERSONAL_ACCESS_TOKEN
GH_RUNNER_VERSION: 2.324.0
RUNNER_NAME: self-hosted-runner
RUNNER_GROUP: default
RUNNER_LABELS: self-hosted
TARGET_ARCH: x64
```
Then, build and run the docker container.
```bash
# Build and run
docker compose build
docker compose up -d
```
## Original Repository
[repo](https://github.com/Dao-AILab/flash-attention)
self-hosted-runner/env.template
-1
View File
@@ -1,2 +1 @@
PERSONAL_ACCESS_TOKEN=
DOCKER_GID=