This commit is contained in:
Junya Morioka
2025-06-10 14:41:15 +09:00
parent 3cbc2db6f1
commit b24d8d5c9a
+200 -200
View File
@@ -21,123 +21,123 @@ jobs:
| Flash-Attention | Python | PyTorch | CUDA | | Flash-Attention | Python | PyTorch | CUDA |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| 2.4.3, 2.5.9, 2.6.3, 2.7.4 | 3.10, 3.11, 3.12 | 2.7.1 | 12.8.1 | | 2.7.4 | 3.10, 3.11, 3.12 | 2.7.1 | 12.8.1 |"
## Windows x86_64 # ## Windows x86_64
| Flash-Attention | Python | PyTorch | CUDA | # | Flash-Attention | Python | PyTorch | CUDA |
| --- | --- | --- | --- | # | --- | --- | --- | --- |
| 2.5.9, 2.6.3, 2.7.4 | 3.10, 3.11, 3.12 | 2.4.1, 2.5.1, 2.6.0, 2.7.1 | 12.4.1, 12.8.1 |" # | 2.5.9, 2.6.3, 2.7.4 | 3.10, 3.11, 3.12 | 2.4.1, 2.5.1, 2.6.0, 2.7.1 | 12.4.1, 12.8.1 |"
# ######################################################### # #########################################################
# Build wheels with GitHub hosted runner # Build wheels with GitHub hosted runner
# ######################################################### # #########################################################
build_wheels: # build_wheels:
name: Build wheels and Upload # name: Build wheels and Upload
needs: create_releases # needs: create_releases
runs-on: ubuntu-22.04 # runs-on: ubuntu-22.04
env: # env:
DEBIAN_FRONTEND: noninteractive # DEBIAN_FRONTEND: noninteractive
TERM: xterm-256color # TERM: xterm-256color
timeout-minutes: 1000 # timeout-minutes: 1000
strategy: # strategy:
fail-fast: false # fail-fast: false
matrix: # matrix:
flash-attn-version: ["2.4.3", "2.5.9", "2.6.3"] # flash-attn-version: ["2.4.3", "2.5.9", "2.6.3"]
python-version: ["3.10", "3.11", "3.12"] # python-version: ["3.10", "3.11", "3.12"]
torch-version: ["2.7.1"] # torch-version: ["2.7.1"]
# # https://developer.nvidia.com/cuda-toolkit-archive # # # https://developer.nvidia.com/cuda-toolkit-archive
cuda-version: ["12.8.1"] # cuda-version: ["12.8.1"]
exclude: # exclude:
# torch < 2.2 does not support Python 3.12 # # torch < 2.2 does not support Python 3.12
- python-version: "3.12" # - python-version: "3.12"
torch-version: "2.0.1" # torch-version: "2.0.1"
- python-version: "3.12" # - python-version: "3.12"
torch-version: "2.1.2" # torch-version: "2.1.2"
# torch 2.0.1 does not support CUDA 12.x # # torch 2.0.1 does not support CUDA 12.x
- torch-version: "2.0.1" # - torch-version: "2.0.1"
cuda-version: "12.1.1" # cuda-version: "12.1.1"
- torch-version: "2.0.1" # - torch-version: "2.0.1"
cuda-version: "12.4.1" # cuda-version: "12.4.1"
- torch-version: "2.0.1" # - torch-version: "2.0.1"
cuda-version: "12.6.3" # cuda-version: "12.6.3"
- torch-version: "2.0.1" # - torch-version: "2.0.1"
cuda-version: "12.8.1" # cuda-version: "12.8.1"
# torch 2.7.0 does not support CUDA 12.4 # # torch 2.7.0 does not support CUDA 12.4
- torch-version: "2.7.0" # - torch-version: "2.7.0"
cuda-version: "12.4.1" # cuda-version: "12.4.1"
steps: # steps:
- uses: actions/checkout@v4 # - uses: actions/checkout@v4
- name: Maximize build space # - name: Maximize build space
run: | # run: |
df -h # df -h
echo "-----------------------------" # echo "-----------------------------"
sudo rm -rf /usr/share/dotnet # sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android # sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc # sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL # sudo rm -rf /opt/hostedtoolcache/CodeQL
df -h # df -h
- name: Set Swap Space # - name: Set Swap Space
uses: pierotofy/set-swap-space@master # uses: pierotofy/set-swap-space@master
with: # with:
swap-size-gb: 48 # swap-size-gb: 48
- uses: actions/setup-python@v5 # - uses: actions/setup-python@v5
with: # with:
python-version: ${{ matrix.python-version }} # python-version: ${{ matrix.python-version }}
- uses: Jimver/cuda-toolkit@master # - uses: Jimver/cuda-toolkit@master
with: # with:
cuda: ${{ matrix.cuda-version }} # cuda: ${{ matrix.cuda-version }}
sub-packages: '["nvcc", "toolkit"]' # sub-packages: '["nvcc", "toolkit"]'
method: "network" # method: "network"
use-github-cache: false # use-github-cache: false
use-local-cache: false # use-local-cache: false
- name: Install build dependencies # - name: Install build dependencies
run: | # run: |
sudo apt install -y ninja-build clang # sudo apt install -y ninja-build clang
pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil # pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil
- name: Set environment variables # - name: Set environment variables
run: | # run: |
export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH # export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH # export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export MAX_JOBS=2 # export MAX_JOBS=2
export NVCC_THREADS=2 # export NVCC_THREADS=2
- name: Build wheels # - name: Build wheels
timeout-minutes: 800 # timeout-minutes: 800
run: | # run: |
chmod +x build_linux.sh # chmod +x build_linux.sh
./build_linux.sh ${{ matrix.flash-attn-version }} ${{ matrix.python-version }} ${{ matrix.torch-version }} ${{ matrix.cuda-version }} # ./build_linux.sh ${{ matrix.flash-attn-version }} ${{ matrix.python-version }} ${{ matrix.torch-version }} ${{ matrix.cuda-version }}
wheel_name=$(basename $(ls flash-attention/dist/*.whl | head -n 1)) # wheel_name=$(basename $(ls flash-attention/dist/*.whl | head -n 1))
echo "wheel_name=$wheel_name" >> $GITHUB_ENV # echo "wheel_name=$wheel_name" >> $GITHUB_ENV
- name: Install Test # - name: Install Test
run: | # run: |
pip install flash-attention/dist/${{ env.wheel_name }} # pip install flash-attention/dist/${{ env.wheel_name }}
python -c "import flash_attn; print(flash_attn.__version__)" # python -c "import flash_attn; print(flash_attn.__version__)"
- name: Upload Release Asset # - name: Upload Release Asset
env: # env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: | # run: |
tag_name=${{ github.ref_name }} # tag_name=${{ github.ref_name }}
wheel_path="flash-attention/dist/${{ env.wheel_name }}" # wheel_path="flash-attention/dist/${{ env.wheel_name }}"
# Check if the file exists # # Check if the file exists
if [ ! -f "$wheel_path" ]; then # if [ ! -f "$wheel_path" ]; then
echo "Error: Wheel file not found at $wheel_path" # echo "Error: Wheel file not found at $wheel_path"
exit 1 # exit 1
fi # fi
# Upload the release asset using GitHub CLI # # Upload the release asset using GitHub CLI
gh release upload "$tag_name" "$wheel_path" --clobber # gh release upload "$tag_name" "$wheel_path" --clobber
echo "Successfully uploaded $wheel_name to release $tag_name" # echo "Successfully uploaded $wheel_name to release $tag_name"
# ######################################################### # #########################################################
# Build wheels with self-hosted runner # Build wheels with self-hosted runner
@@ -286,122 +286,122 @@ jobs:
run: | run: |
sudo rm -rf /opt/hostedtoolcache/Python/${{ matrix.python-version }}* sudo rm -rf /opt/hostedtoolcache/Python/${{ matrix.python-version }}*
build_windows_wheels: # build_windows_wheels:
name: Build wheels and Upload # name: Build wheels and Upload
runs-on: windows-latest # runs-on: windows-latest
timeout-minutes: 1000 # timeout-minutes: 1000
strategy: # strategy:
fail-fast: false # fail-fast: false
matrix: # matrix:
flash-attn-version: ["2.5.9", "2.6.3", "2.7.4"] # flash-attn-version: ["2.5.9", "2.6.3", "2.7.4"]
python-version: ["3.10", "3.11", "3.12"] # python-version: ["3.10", "3.11", "3.12"]
torch-version: ["2.4.1", "2.5.1", "2.6.0", "2.7.1"] # torch-version: ["2.4.1", "2.5.1", "2.6.0", "2.7.1"]
# https://developer.nvidia.com/cuda-toolkit-archive # # https://developer.nvidia.com/cuda-toolkit-archive
# CUDA 11.8 cannot build flash-attn. # # CUDA 11.8 cannot build flash-attn.
# https://github.com/Dao-AILab/flash-attention/issues/595 # # https://github.com/Dao-AILab/flash-attention/issues/595
cuda-version: ["12.4.1", "12.8.1"] # cuda-version: ["12.4.1", "12.8.1"]
steps: # steps:
- uses: actions/checkout@v4 # - uses: actions/checkout@v4
- name: Enable Git long paths # - name: Enable Git long paths
run: git config --system core.longpaths true # run: git config --system core.longpaths true
shell: pwsh # shell: pwsh
- name: Install VS2022 BuildTools # - name: Install VS2022 BuildTools
run: | # run: |
choco install -y visualstudio2022buildtools ` # choco install -y visualstudio2022buildtools `
--version=117.14.1 ` # --version=117.14.1 `
--params "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64" # --params "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64"
shell: pwsh # shell: pwsh
- uses: actions/setup-python@v5 # - uses: actions/setup-python@v5
with: # with:
python-version: ${{ matrix.python-version }} # python-version: ${{ matrix.python-version }}
- uses: Jimver/cuda-toolkit@v0.2.24 # - uses: Jimver/cuda-toolkit@v0.2.24
with: # with:
cuda: ${{ matrix.cuda-version }} # cuda: ${{ matrix.cuda-version }}
method: "network" # method: "network"
use-github-cache: false # use-github-cache: false
use-local-cache: false # use-local-cache: false
- name: Set CUDA and PyTorch versions # - name: Set CUDA and PyTorch versions
run: | # run: |
$cudaVersion = "${{ matrix.cuda-version }}" -replace '\.', '' # $cudaVersion = "${{ matrix.cuda-version }}" -replace '\.', ''
$cudaVersion = $cudaVersion.Substring(0, [Math]::Min(3, $cudaVersion.Length)) # $cudaVersion = $cudaVersion.Substring(0, [Math]::Min(3, $cudaVersion.Length))
echo "MATRIX_CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV # echo "MATRIX_CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV
$torchVersion = "${{ matrix.torch-version }}" -replace '^(\d+\.\d+).*', '$1' # $torchVersion = "${{ matrix.torch-version }}" -replace '^(\d+\.\d+).*', '$1'
echo "MATRIX_TORCH_VERSION=$torchVersion" >> $env:GITHUB_ENV # echo "MATRIX_TORCH_VERSION=$torchVersion" >> $env:GITHUB_ENV
shell: pwsh # shell: pwsh
- name: Install build dependencies # - name: Install build dependencies
run: | # run: |
pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil ninja # pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil ninja
shell: pwsh # shell: pwsh
- name: Install PyTorch ${{ matrix.torch-version }}+cu${{ matrix.cuda-version }} # - name: Install PyTorch ${{ matrix.torch-version }}+cu${{ matrix.cuda-version }}
run: | # run: |
$env:TORCH_CUDA_VERSION = python -c "from os import environ as env; support_cuda_versions = { '2.1': [121], '2.2': [121], '2.3': [121], '2.4': [121, 124], '2.5': [121, 124], '2.6': [124, 126], '2.7': [126, 128], '2.8': [128], }; target_cuda_versions = support_cuda_versions[env['MATRIX_TORCH_VERSION']]; cuda_version = int(env['MATRIX_CUDA_VERSION']); closest_version = min(target_cuda_versions, key=lambda x: abs(x - cuda_version)); print(closest_version)" # $env:TORCH_CUDA_VERSION = python -c "from os import environ as env; support_cuda_versions = { '2.1': [121], '2.2': [121], '2.3': [121], '2.4': [121, 124], '2.5': [121, 124], '2.6': [124, 126], '2.7': [126, 128], '2.8': [128], }; target_cuda_versions = support_cuda_versions[env['MATRIX_TORCH_VERSION']]; cuda_version = int(env['MATRIX_CUDA_VERSION']); closest_version = min(target_cuda_versions, key=lambda x: abs(x - cuda_version)); print(closest_version)"
if ("${{ matrix.torch-version }}" -like "*dev*") { # if ("${{ matrix.torch-version }}" -like "*dev*") {
pip install --pre torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/nightly/cu$env:TORCH_CUDA_VERSION # pip install --pre torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/nightly/cu$env:TORCH_CUDA_VERSION
} else { # } else {
pip install --no-cache-dir torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/cu$env:TORCH_CUDA_VERSION # pip install --no-cache-dir torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/cu$env:TORCH_CUDA_VERSION
} # }
nvcc --version # nvcc --version
python -V # python -V
python -c "import torch; print('PyTorch:', torch.__version__)" # python -c "import torch; print('PyTorch:', torch.__version__)"
python -c "import torch; print('CUDA:', torch.version.cuda)" # python -c "import torch; print('CUDA:', torch.version.cuda)"
python -c "from torch.utils import cpp_extension; print(cpp_extension.CUDA_HOME)" # python -c "from torch.utils import cpp_extension; print(cpp_extension.CUDA_HOME)"
shell: pwsh # shell: pwsh
- name: Checkout flash-attn # - name: Checkout flash-attn
run: | # run: |
git clone https://github.com/Dao-AILab/flash-attention.git -b "v${{ matrix.flash-attn-version }}" # git clone https://github.com/Dao-AILab/flash-attention.git -b "v${{ matrix.flash-attn-version }}"
shell: pwsh # shell: pwsh
- name: Build wheels # - name: Build wheels
timeout-minutes: 800 # timeout-minutes: 800
run: | # run: |
Import-Module 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Microsoft.VisualStudio.DevShell.dll' # Import-Module 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
Enter-VsDevShell -VsInstallPath 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools' -DevCmdArguments '-arch=x64 -host_arch=x64' # Enter-VsDevShell -VsInstallPath 'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools' -DevCmdArguments '-arch=x64 -host_arch=x64'
$env:DISTUTILS_USE_SDK = 1 # $env:DISTUTILS_USE_SDK = 1
$env:BUILD_TARGET = "cuda" # $env:BUILD_TARGET = "cuda"
$env:MAX_JOBS = "2" # $env:MAX_JOBS = "2"
$env:NVCC_THREADS = "2" # $env:NVCC_THREADS = "2"
$env:FLASH_ATTENTION_FORCE_BUILD = "TRUE" # $env:FLASH_ATTENTION_FORCE_BUILD = "TRUE"
cd flash-attention # cd flash-attention
python setup.py bdist_wheel --dist-dir=dist # python setup.py bdist_wheel --dist-dir=dist
$baseWheelName = Get-ChildItem -Path "dist\*.whl" | Select-Object -First 1 | ForEach-Object { $_.Name } # $baseWheelName = Get-ChildItem -Path "dist\*.whl" | Select-Object -First 1 | ForEach-Object { $_.Name }
$wheelName = $baseWheelName.Replace("${{ matrix.flash-attn-version }}", "${{ matrix.flash-attn-version }}+cu$env:MATRIX_CUDA_VERSION" + "torch$env:MATRIX_TORCH_VERSION") # $wheelName = $baseWheelName.Replace("${{ matrix.flash-attn-version }}", "${{ matrix.flash-attn-version }}+cu$env:MATRIX_CUDA_VERSION" + "torch$env:MATRIX_TORCH_VERSION")
Move-Item "dist\$baseWheelName" "dist\$wheelName" # Move-Item "dist\$baseWheelName" "dist\$wheelName"
echo "wheel_name=$wheelName" >> $env:GITHUB_ENV # echo "wheel_name=$wheelName" >> $env:GITHUB_ENV
shell: pwsh # shell: pwsh
- name: Install Test # - name: Install Test
run: | # run: |
pip install flash-attention/dist/$env:wheel_name # pip install flash-attention/dist/$env:wheel_name
python -c "import flash_attn; print(flash_attn.__version__)" # python -c "import flash_attn; print(flash_attn.__version__)"
shell: pwsh # shell: pwsh
- name: Upload Release Asset # - name: Upload Release Asset
env: # env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: | # run: |
$tag_name = "${env:GITHUB_REF}".Replace("refs/tags/", "") # $tag_name = "${env:GITHUB_REF}".Replace("refs/tags/", "")
$wheel_path = "flash-attention/dist/$env:wheel_name" # $wheel_path = "flash-attention/dist/$env:wheel_name"
# Check if the file exists # # Check if the file exists
if (-not (Test-Path $wheel_path)) { # if (-not (Test-Path $wheel_path)) {
Write-Host "Error: Wheel file not found at $wheel_path" # Write-Host "Error: Wheel file not found at $wheel_path"
exit 1 # exit 1
} # }
# Upload the release asset using GitHub CLI # # Upload the release asset using GitHub CLI
gh release upload "$tag_name" "$wheel_path" --clobber # gh release upload "$tag_name" "$wheel_path" --clobber
Write-Host "Successfully uploaded $env:wheel_name to release $tag_name" # Write-Host "Successfully uploaded $env:wheel_name to release $tag_name"
shell: pwsh # shell: pwsh