diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..617026d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,15 @@ +# These are supported funding model platforms + +github: mjun0812 +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +polar: # Replace with a single Polar username +buy_me_a_coffee: # Replace with a single Buy Me a Coffee username +thanks_dev: # Replace with a single thanks.dev username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87de9a1..6f86357 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -168,152 +168,152 @@ jobs: asset_content_type: application/* build_wheels_self_hosted: - name: Build wheels and Upload - needs: create_releases - runs-on: self-hosted - container: - image: ubuntu:22.04 - env: - DEBIAN_FRONTEND: noninteractive - TERM: xterm-256color - timeout-minutes: 1000 - strategy: - fail-fast: false - matrix: - flash-attn-version: ["2.7.4"] - python-version: ["3.10", "3.11", "3.12"] - torch-version: ["2.7.0"] - # https://developer.nvidia.com/cuda-toolkit-archive - cuda-version: ["12.8.1"] - exclude: - # torch < 2.2 does not support Python 3.12 - - python-version: "3.12" - torch-version: "2.0.1" - - python-version: "3.12" - torch-version: "2.1.2" - # torch 2.0.1 does not support CUDA 12.x - - torch-version: "2.0.1" - cuda-version: "12.1.1" - - torch-version: "2.0.1" - cuda-version: "12.4.1" - - torch-version: "2.0.1" - cuda-version: "12.6.3" - - torch-version: "2.0.1" - cuda-version: "12.8.1" - # torch 2.7.0 does not support CUDA 12.4 - - torch-version: "2.7.0" - cuda-version: "12.4.1" + name: Build wheels and Upload + needs: create_releases + runs-on: self-hosted + container: + image: ubuntu:22.04 + env: + DEBIAN_FRONTEND: noninteractive + TERM: xterm-256color + timeout-minutes: 1000 + strategy: + fail-fast: false + matrix: + flash-attn-version: ["2.7.4"] + python-version: ["3.10", "3.11", "3.12"] + torch-version: ["2.7.0"] + # https://developer.nvidia.com/cuda-toolkit-archive + cuda-version: ["12.8.1"] + exclude: + # torch < 2.2 does not support Python 3.12 + - python-version: "3.12" + torch-version: "2.0.1" + - python-version: "3.12" + torch-version: "2.1.2" + # torch 2.0.1 does not support CUDA 12.x + - torch-version: "2.0.1" + cuda-version: "12.1.1" + - torch-version: "2.0.1" + cuda-version: "12.4.1" + - torch-version: "2.0.1" + cuda-version: "12.6.3" + - torch-version: "2.0.1" + cuda-version: "12.8.1" + # torch 2.7.0 does not support CUDA 12.4 + - torch-version: "2.7.0" + cuda-version: "12.4.1" - steps: - - name: Install tools - run: | - apt-get update && apt-get install -y --no-install-recommends \ - curl \ - ca-certificates \ - sudo \ - software-properties-common \ - wget \ - unzip \ - zip \ - git \ - build-essential \ - gcc \ - g++ \ - clang \ - ninja-build \ - keyboard-configuration + steps: + - name: Install tools + run: | + apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + sudo \ + software-properties-common \ + wget \ + unzip \ + zip \ + git \ + build-essential \ + gcc \ + g++ \ + clang \ + ninja-build \ + keyboard-configuration - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - uses: Jimver/cuda-toolkit@master - env: - DEBIAN_FRONTEND: noninteractive - with: - cuda: ${{ matrix.cuda-version }} - linux-local-args: '["--toolkit"]' - method: "network" + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - uses: Jimver/cuda-toolkit@master + env: + DEBIAN_FRONTEND: noninteractive + with: + cuda: ${{ matrix.cuda-version }} + linux-local-args: '["--toolkit"]' + method: "network" - - name: Set CUDA and PyTorch versions - run: | - echo "MATRIX_CUDA_VERSION=$(echo ${{ matrix.cuda-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV - echo "MATRIX_TORCH_VERSION=$(echo ${{ matrix.torch-version }} | awk -F \. {'print $1 "." $2'})" >> $GITHUB_ENV - echo "CACHE_KEY=cuda-ext-${{ matrix.flash-attn-version }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-cuda${{ matrix.cuda-version }}" >> $GITHUB_ENV + - name: Set CUDA and PyTorch versions + run: | + echo "MATRIX_CUDA_VERSION=$(echo ${{ matrix.cuda-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV + echo "MATRIX_TORCH_VERSION=$(echo ${{ matrix.torch-version }} | awk -F \. {'print $1 "." $2'})" >> $GITHUB_ENV + echo "CACHE_KEY=cuda-ext-${{ matrix.flash-attn-version }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-cuda${{ matrix.cuda-version }}" >> $GITHUB_ENV - - name: Install build dependencies - run: | - sudo apt install -y ninja-build clang - pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil + - name: Install build dependencies + run: | + sudo apt install -y ninja-build clang + pip install -U pip setuptools==75.8.0 wheel setuptools packaging psutil - - name: Install PyTorch ${{ matrix.torch-version }}+cu${{ matrix.cuda-version }} - run: | - export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; \ - support_cuda_versions = { \ - '2.0': [117, 118], \ - '2.1': [118, 121], \ - '2.2': [118, 121], \ - '2.3': [118, 121], \ - '2.4': [118, 121, 124], \ - '2.5': [118, 121, 124], \ - '2.6': [118, 124, 126], \ - '2.7': [118, 126, 128], \ - }; \ - target_cuda_versions = support_cuda_versions[env['MATRIX_TORCH_VERSION']]; \ - cuda_version = int(env['MATRIX_CUDA_VERSION']); \ - closest_version = min(target_cuda_versions, key=lambda x: abs(x - cuda_version)); \ - print(closest_version) \ - ") - pip install --no-cache-dir torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/cu${TORCH_CUDA_VERSION} - nvcc --version - python -V - python -c "import torch; print('PyTorch:', torch.__version__)" - python -c "import torch; print('CUDA:', torch.version.cuda)" - python -c "from torch.utils import cpp_extension; print(cpp_extension.CUDA_HOME)" + - name: Install PyTorch ${{ matrix.torch-version }}+cu${{ matrix.cuda-version }} + run: | + export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; \ + support_cuda_versions = { \ + '2.0': [117, 118], \ + '2.1': [118, 121], \ + '2.2': [118, 121], \ + '2.3': [118, 121], \ + '2.4': [118, 121, 124], \ + '2.5': [118, 121, 124], \ + '2.6': [118, 124, 126], \ + '2.7': [118, 126, 128], \ + }; \ + target_cuda_versions = support_cuda_versions[env['MATRIX_TORCH_VERSION']]; \ + cuda_version = int(env['MATRIX_CUDA_VERSION']); \ + closest_version = min(target_cuda_versions, key=lambda x: abs(x - cuda_version)); \ + print(closest_version) \ + ") + pip install --force-reinstall --no-cache-dir torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/cu${TORCH_CUDA_VERSION} + nvcc --version + python -V + python -c "import torch; print('PyTorch:', torch.__version__)" + python -c "import torch; print('CUDA:', torch.version.cuda)" + python -c "from torch.utils import cpp_extension; print(cpp_extension.CUDA_HOME)" - - name: Checkout flash-attn - run: | - git clone https://github.com/Dao-AILab/flash-attention.git -b "v${{ matrix.flash-attn-version }}" - cd flash-attention + - name: Checkout flash-attn + run: | + git clone https://github.com/Dao-AILab/flash-attention.git -b "v${{ matrix.flash-attn-version }}" + cd flash-attention - - name: Build wheels - timeout-minutes: 800 - run: | - export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH - export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH - export MAX_JOBS=2 - export NVCC_THREADS=4 - export FLASH_ATTENTION_FORCE_BUILD="TRUE" - cd flash-attention - python setup.py bdist_wheel --dist-dir=dist - base_wheel_name=$(basename $(ls dist/*.whl | head -n 1)) - wheel_name=$(echo $base_wheel_name | sed "s/${{ matrix.flash-attn-version }}/${{ matrix.flash-attn-version }}+cu${{ env.MATRIX_CUDA_VERSION }}torch${{ env.MATRIX_TORCH_VERSION }}/") - mv dist/$base_wheel_name dist/$wheel_name - echo "wheel_name=$wheel_name" >> $GITHUB_ENV + - name: Build wheels + timeout-minutes: 800 + run: | + export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH + export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH + export MAX_JOBS=2 + export NVCC_THREADS=4 + export FLASH_ATTENTION_FORCE_BUILD="TRUE" + cd flash-attention + python setup.py bdist_wheel --dist-dir=dist + base_wheel_name=$(basename $(ls dist/*.whl | head -n 1)) + wheel_name=$(echo $base_wheel_name | sed "s/${{ matrix.flash-attn-version }}/${{ matrix.flash-attn-version }}+cu${{ env.MATRIX_CUDA_VERSION }}torch${{ env.MATRIX_TORCH_VERSION }}/") + mv dist/$base_wheel_name dist/$wheel_name + echo "wheel_name=$wheel_name" >> $GITHUB_ENV - - name: Install Test - run: | - pip install flash-attention/dist/${{ env.wheel_name }} - python -c "import flash_attn; print(flash_attn.__version__)" + - name: Install Test + run: | + pip install flash-attention/dist/${{ env.wheel_name }} + python -c "import flash_attn; print(flash_attn.__version__)" - - name: Get the tag version - id: extract_branch - run: echo "branch=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - name: Get the tag version + id: extract_branch + run: echo "branch=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - - name: Get Release with Tag - id: get_release - uses: joutvhu/get-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.extract_branch.outputs.branch }} + - name: Get Release with Tag + id: get_release + uses: joutvhu/get-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.extract_branch.outputs.branch }} - - name: Upload Release Asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: flash-attention/dist/${{ env.wheel_name }} - asset_name: ${{ env.wheel_name }} - asset_content_type: application/* + - name: Upload Release Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.get_release.outputs.upload_url }} + asset_path: flash-attention/dist/${{ env.wheel_name }} + asset_name: ${{ env.wheel_name }} + asset_content_type: application/* diff --git a/README.md b/README.md index 11f8871..2057764 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ # flash-attention pre-build wheels -This repository provides wheels for the pre-built [flash-attention](https://github.com/Dao-AILab/flash-attention). +This repository provides wheels for the pre-built [flash-attention](https://github.com/Dao-AILab/flash-attention). -Since building flash-attention takes a **very long time** and is resource-intensive, +Since building flash-attention takes a **very long time** and is resource-intensive, I also build and provide combinations of CUDA and PyTorch that are not officially distributed. The building Github Actions Workflow can be found [here](./.github/workflows/build.yml). The built packages are available on the [release page](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases). +This repository uses a self-hosted runner for building the wheels. If you find this project helpful, please consider supporting or sponsoring to help maintain the infrastructure. + +[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/mjun0812) ## Install @@ -22,7 +25,7 @@ flash_attn-2.6.3+cu124torch2.5-cp312-cp312-linux_x86_64.whl 2. Find the corresponding version of a wheel from the below table and [releases](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases) -5. Direct Install or Download and Local Install +3. Direct Install or Download and Local Install ```bash # Direct Install @@ -35,20 +38,32 @@ pip install ./flash_attn-2.6.3+cu124torch2.5-cp312-cp312-linux_x86_64.whl ## Packages +### v0.1.0 + +[Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.1.0) + +| Flash-Attention | Python | PyTorch | CUDA | +| -------------------------- | ---------------- | ------- | ------ | +| 2.4.3, 2.5.9, 2.6.3, 2.7.4 | 3.10, 3.11, 3.12 | 2.7.0 | 12.8.1 | + +v2.7.4 and v2.7.4.post1 are the same version. + +From this release, self-hosted runners are used for building the some wheels. + ### v0.0.9 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.9) -| Flash-Attention | Python | PyTorch | CUDA | -|-----------------|--------|---------|------| -| 2.4.3, 2.5.9, 2.6.3 | 3.10, 3.11, 3.12 | 2.7.0 | 12.8.1 | +| Flash-Attention | Python | PyTorch | CUDA | +| ------------------- | ---------------- | ------- | ------ | +| 2.4.3, 2.5.9, 2.6.3 | 3.10, 3.11, 3.12 | 2.7.0 | 12.8.1 | ### v0.0.8 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.8) -| Flash-Attention | Python | PyTorch | CUDA | -|-----------------|--------|---------|------| +| Flash-Attention | Python | PyTorch | CUDA | +| -------------------------------- | ---------------- | -------------------------- | ---------------------- | | 2.4.3, 2.5.9, 2.6.3, 2.7.4.post1 | 3.10, 3.11, 3.12 | 2.4.1, 2.5.1, 2.6.0, 2.7.0 | 11.8.0, 12.4.1, 12.6.3 | ### v0.0.7 @@ -59,61 +74,112 @@ Skip for experimental reasons. [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.6) -| Flash-Attention | Python | PyTorch | CUDA | -|-----------------|--------|---------|------| +| Flash-Attention | Python | PyTorch | CUDA | +| -------------------------------- | ---------------- | --------------------------------- | -------------- | | 2.4.3, 2.5.9, 2.6.3, 2.7.4.post1 | 3.10, 3.11, 3.12 | 2.2.2, 2.3.1, 2.4.1, 2.5.1, 2.6.0 | 12.4.1, 12.6.3 | ### v0.0.5 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.5) -| Flash-Attention | Python | PyTorch | CUDA | -|-----------------|--------|---------|------| +| Flash-Attention | Python | PyTorch | CUDA | +| ------------------ | ---------------- | ----------------------------------------------- | -------------- | | 2.6.3, 2.7.4.post1 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1, 2.6.0 | 12.4.1, 12.6.3 | ### v0.0.4 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.4) -| Flash-Attention | Python | PyTorch | CUDA | -|-----------------|--------|---------|------| -| 2.7.3 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 | +| Flash-Attention | Python | PyTorch | CUDA | +| --------------- | ---------------- | ---------------------------------------- | ---------------------- | +| 2.7.3 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 | ### v0.0.3 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.3) -| Flash-Attention | Python | PyTorch | CUDA | -|-----------------|--------|---------|------| -| 2.7.2.post1 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 | +| Flash-Attention | Python | PyTorch | CUDA | +| --------------- | ---------------- | ---------------------------------------- | ---------------------- | +| 2.7.2.post1 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 | ### v0.0.2 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.2) -| Flash-Attention | Python | PyTorch | CUDA | -|-----------------|--------|---------|------| +| Flash-Attention | Python | PyTorch | CUDA | +| -------------------------------- | ---------------- | ---------------------------------------- | ---------------------- | | 2.4.3, 2.5.6, 2.6.3, 2.7.0.post2 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.1 | 11.8.0, 12.1.1, 12.4.1 | ### v0.0.1 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.1) -|flash-attention|Python|PyTorch|CUDA| -|-|-|-|-| -|1.0.9, 2.4.3, 2.5.6, 2.5.9, 2.6.3|3.10, 3.11, 3.12|2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0|11.8.0, 12.1.1, 12.4.1| - +| flash-attention | Python | PyTorch | CUDA | +| --------------------------------- | ---------------- | ---------------------------------------- | ---------------------- | +| 1.0.9, 2.4.3, 2.5.6, 2.5.9, 2.6.3 | 3.10, 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0 | 11.8.0, 12.1.1, 12.4.1 | ### v0.0.0 [Release](https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/tag/v0.0.0) -|flash-attention|Python|PyTorch|CUDA| -|-|-|-|-| -|2.4.3, 2.5.6, 2.5.9, 2.6.3|3.11, 3.12|2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0|11.8.0, 12.1.1, 12.4.1| +| flash-attention | Python | PyTorch | CUDA | +| -------------------------- | ---------- | ---------------------------------------- | ---------------------- | +| 2.4.3, 2.5.6, 2.5.9, 2.6.3 | 3.11, 3.12 | 2.0.1, 2.1.2, 2.2.2, 2.3.1, 2.4.1, 2.5.0 | 11.8.0, 12.1.1, 12.4.1 | +## Self build -## Original +If you want to build the wheels yourself, you can fork this repository and run the build workflow. + +1. Fork this repository +2. Edit workflow file `.github/workflows/build.yml` to set the version you want to build. +3. Add tag `v*.*.*` to trigger the build workflow. + +### Self-Hosted Runner Build + +In some version combinations, you cannot build wheels on GitHub-hosted runners due to job time limitations. +To build the wheels for these versions, you can use self-hosted runners. + +```bash +git clone https://github.com/mjun0812/flash-attention-prebuild-wheels.git +cd self-hosted-runner +cp env.template env +``` + +Edit `env` file to set the environment variables. + +```bash +# Edit env +PERSONAL_ACCESS_TOKEN=[Github Personal Access Token] +``` + +Edit compose.yml file if you use repository folked from this repository. + +```yaml +services: + runner: + privileged: true + build: + context: . + dockerfile: Dockerfile + args: + REPOSITORY_URL: [Target Repository URL] + PERSONAL_ACCESS_TOKEN: $PERSONAL_ACCESS_TOKEN + GH_RUNNER_VERSION: 2.324.0 + RUNNER_NAME: self-hosted-runner + RUNNER_GROUP: default + RUNNER_LABELS: self-hosted + TARGET_ARCH: x64 +``` + +Then, build and run the docker container. + +```bash +# Build and run +docker compose build +docker compose up -d +``` + +## Original Repository [repo](https://github.com/Dao-AILab/flash-attention) diff --git a/self-hosted-runner/env.template b/self-hosted-runner/env.template index a6bfb2a..cf1aca7 100644 --- a/self-hosted-runner/env.template +++ b/self-hosted-runner/env.template @@ -1,2 +1 @@ PERSONAL_ACCESS_TOKEN= -DOCKER_GID= \ No newline at end of file