ci: refactor Python installation and enable manylinux wheel generation

- Replace actions/setup-python with uv-based Python installation
- Consolidate build dependencies into Python installation step
- Enable auditwheel repair and manylinux wheel generation across all Linux builds
- Add patchelf as build dependency for glibc compatibility
- Update Python version to 3.14 in build and test workflows

.github/workflows/_build_linux.yml

@@ -54,19 +54,25 @@ jobs:
           sudo rm -rf /opt/hostedtoolcache/CodeQL
           df -h
 
-      - uses: actions/setup-python@v5
+      - name: Install build dependencies
-        with:
+        run: |
-          python-version: ${{ inputs.python-version }}
+          sudo apt install -y ninja-build clang time patchelf
+
+      # Install Python using uv because setup-python needs newer version of glibc
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install Python
+        shell: bash
+        run: |
+          uv venv -p ${{ inputs.python-version }}
+          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil auditwheel
+          current_dir=$(pwd)
+          echo "$current_dir/.venv/bin" >> $GITHUB_PATH
 
       - uses: mjun0812/setup-cuda@v1
         with:
           version: ${{ inputs.cuda-version }}
 
-      - name: Install build dependencies
-        run: |
-          sudo apt install -y ninja-build clang time
-          pip install -U pip setuptools==75.8.0 wheel packaging psutil
-
       - name: Build wheels
         id: build_wheels
         run: |
@@ -99,30 +105,32 @@ jobs:
           # Upload the release asset using GitHub CLI
           gh release upload "$tag_name" "$wheel_path" --clobber
 
-      # - name: Apply auditwheel repair
+      - name: Apply auditwheel repair
-      #   continue-on-error: true
+        continue-on-error: true
-      #   run: |
+        run: |
-      #     pip install auditwheel
+          auditwheel show flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }}
-      #     auditwheel show flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }}
+          auditwheel repair \
-      #     auditwheel repair flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }} -w flash-attention/dist_manylinux
+            --exclude libc10* --exclude libtorch* --exclude libcu* --exclude libnv* --exclude 'libtorch*' \
-      #     wheel_name=$(basename $(ls flash-attention/dist_manylinux/*.whl | head -n 1))
+            flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }} -w flash-attention/dist_manylinux
-      #     echo "WHEEL_NAME_MANYLINUX=$wheel_name" >> $GITHUB_OUTPUT
+          wheel_name=$(basename $(ls flash-attention/dist_manylinux/*manylinux*.whl | head -n 1))
+          echo "WHEEL_NAME_MANYLINUX=$wheel_name" >> $GITHUB_OUTPUT
 
-      # - name: Apply auditwheel repair
+      - name: Test manylinux wheel
-      #   if: ${{ inputs.is-upload }}
+        continue-on-error: true
-      #   continue-on-error: true
+        run: |
-      #   env:
+          pip uninstall -y flash-attn > /dev/null 2>&1
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          pip install --no-cache-dir flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}
-      #   run: |
+          python -c "import flash_attn; print(flash_attn.__version__)"
-      #     pip uninstall -y flash-attn > /dev/null 2>&1
-      #     pip install --no-cache-dir flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}
-      #     python -c "import flash_attn; print(flash_attn.__version__)"
 
-      #     wheel_path_manylinux="flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}"
+      - name: Upload manylinux wheel
-      #     if [ ! -f "$wheel_path_manylinux" ]; then
+        continue-on-error: true
-      #       echo "Error: Wheel file not found at $wheel_path_manylinux"
+        env:
-      #       exit 1
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     fi
+        run: |
-
+          wheel_path_manylinux="flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}"
-      #     # Upload the release asset using GitHub CLI
+          if [ ! -f "$wheel_path_manylinux" ]; then
-      #     gh release upload "$tag_name" "$wheel_path_manylinux" --clobber
+            echo "Error: Wheel file not found at $wheel_path_manylinux"
+            exit 1
+          fi
+          # Upload the release asset using GitHub CLI
+          gh release upload "$tag_name" "$wheel_path_manylinux" --clobber

.github/workflows/_build_linux_arm_self_host.yml

@@ -94,20 +94,21 @@ jobs:
         run: |
           git config --global --add safe.directory $(pwd)
 
-      - uses: actions/setup-python@v5
+      # Install Python using uv because setup-python needs newer version of glibc
-        with:
+      - name: Install uv
-          python-version: ${{ inputs.python-version }}
+        uses: astral-sh/setup-uv@v7
+      - name: Install Python
+        shell: bash
+        run: |
+          uv venv -p ${{ inputs.python-version }}
+          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil auditwheel
+          current_dir=$(pwd)
+          echo "$current_dir/.venv/bin" >> $GITHUB_PATH
 
       - uses: mjun0812/setup-cuda@v1
         with:
           version: ${{ inputs.cuda-version }}
 
-      - name: Install build dependencies
-        shell: bash
-        run: |
-          sudo apt install -y ninja-build clang
-          pip install -U pip setuptools==75.8.0 wheel packaging psutil
-
       - name: Build wheels
         timeout-minutes: 2160
         id: build_wheels
@@ -143,6 +144,36 @@ jobs:
           # Upload the release asset using GitHub CLI
           gh release upload "$tag_name" "$wheel_path" --clobber
 
+      - name: Apply auditwheel repair
+        continue-on-error: true
+        run: |
+          auditwheel show flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }}
+          auditwheel repair \
+            --exclude libc10* --exclude libtorch* --exclude libcu* --exclude libnv* --exclude 'libtorch*' \
+            flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }} -w flash-attention/dist_manylinux
+          wheel_name=$(basename $(ls flash-attention/dist_manylinux/*manylinux*.whl | head -n 1))
+          echo "WHEEL_NAME_MANYLINUX=$wheel_name" >> $GITHUB_OUTPUT
+
+      - name: Test manylinux wheel
+        continue-on-error: true
+        run: |
+          pip uninstall -y flash-attn > /dev/null 2>&1
+          pip install --no-cache-dir flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}
+          python -c "import flash_attn; print(flash_attn.__version__)"
+
+      - name: Upload manylinux wheel
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          wheel_path_manylinux="flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}"
+          if [ ! -f "$wheel_path_manylinux" ]; then
+            echo "Error: Wheel file not found at $wheel_path_manylinux"
+            exit 1
+          fi
+          # Upload the release asset using GitHub CLI
+          gh release upload "$tag_name" "$wheel_path_manylinux" --clobber
+
       - name: Clean up
         shell: bash
         if: always()

.github/workflows/_build_linux_self_host.yml

@@ -99,19 +99,21 @@ jobs:
         run: |
           git config --global --add safe.directory $(pwd)
 
-      - uses: actions/setup-python@v5
+      # Install Python using uv because setup-python needs newer version of glibc
-        with:
+      - name: Install uv
-          python-version: ${{ inputs.python-version }}
+        uses: astral-sh/setup-uv@v7
+      - name: Install Python
+        shell: bash
+        run: |
+          uv venv -p ${{ inputs.python-version }}
+          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil auditwheel
+          current_dir=$(pwd)
+          echo "$current_dir/.venv/bin" >> $GITHUB_PATH
 
       - uses: mjun0812/setup-cuda@v1
         with:
           version: ${{ inputs.cuda-version }}
 
-      - name: Install build dependencies
-        shell: bash
-        run: |
-          pip install -U pip setuptools==75.8.0 wheel packaging psutil
-
       - name: Build wheels
         timeout-minutes: 2160
         id: build_wheels
@@ -224,7 +226,8 @@ jobs:
             clang \
             ninja-build \
             keyboard-configuration \
-            time
+            time \
+            patchelf
 
       - name: Install gh
         shell: bash
@@ -245,20 +248,21 @@ jobs:
         run: |
           git config --global --add safe.directory $(pwd)
 
-      - uses: actions/setup-python@v5
+      # Install Python using uv because setup-python needs newer version of glibc
-        with:
+      - name: Install uv
-          python-version: ${{ inputs.python-version }}
+        uses: astral-sh/setup-uv@v7
+      - name: Install Python
+        shell: bash
+        run: |
+          uv venv -p ${{ inputs.python-version }}
+          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil auditwheel
+          current_dir=$(pwd)
+          echo "$current_dir/.venv/bin" >> $GITHUB_PATH
 
       - uses: mjun0812/setup-cuda@v1
         with:
           version: ${{ inputs.cuda-version }}
 
-      - name: Install build dependencies
-        shell: bash
-        run: |
-          sudo apt install -y ninja-build clang
-          pip install -U pip setuptools==75.8.0 wheel packaging psutil
-
       - name: Build wheels
         timeout-minutes: 2160
         id: build_wheels
@@ -294,33 +298,35 @@ jobs:
           # Upload the release asset using GitHub CLI
           gh release upload "$tag_name" "$wheel_path" --clobber
 
-      # - name: Apply auditwheel repair
+      - name: Apply auditwheel repair
-      #   continue-on-error: true
+        continue-on-error: true
-      #   run: |
+        run: |
-      #     pip install auditwheel
+          auditwheel show flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }}
-      #     auditwheel show flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }}
+          auditwheel repair \
-      #     auditwheel repair flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }} -w flash-attention/dist_manylinux
+            --exclude libc10* --exclude libtorch* --exclude libcu* --exclude libnv* --exclude 'libtorch*' \
-      #     wheel_name=$(basename $(ls flash-attention/dist_manylinux/*.whl | head -n 1))
+            flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }} -w flash-attention/dist_manylinux
-      #     echo "WHEEL_NAME_MANYLINUX=$wheel_name" >> $GITHUB_OUTPUT
+          wheel_name=$(basename $(ls flash-attention/dist_manylinux/*manylinux*.whl | head -n 1))
+          echo "WHEEL_NAME_MANYLINUX=$wheel_name" >> $GITHUB_OUTPUT
 
-      # - name: Apply auditwheel repair
+      - name: Test manylinux wheel
-      #   if: ${{ inputs.is-upload }}
+        continue-on-error: true
-      #   continue-on-error: true
+        run: |
-      #   env:
+          pip uninstall -y flash-attn > /dev/null 2>&1
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          pip install --no-cache-dir flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}
-      #   run: |
+          python -c "import flash_attn; print(flash_attn.__version__)"
-      #     pip uninstall -y flash-attn > /dev/null 2>&1
-      #     pip install --no-cache-dir flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}
-      #     python -c "import flash_attn; print(flash_attn.__version__)"
 
-      #     wheel_path_manylinux="flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}"
+      - name: Upload manylinux wheel
-      #     if [ ! -f "$wheel_path_manylinux" ]; then
+        continue-on-error: true
-      #       echo "Error: Wheel file not found at $wheel_path_manylinux"
+        env:
-      #       exit 1
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     fi
+        run: |
-
+          wheel_path_manylinux="flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}"
-      #     # Upload the release asset using GitHub CLI
+          if [ ! -f "$wheel_path_manylinux" ]; then
-      #     gh release upload "$tag_name" "$wheel_path_manylinux" --clobber
+            echo "Error: Wheel file not found at $wheel_path_manylinux"
+            exit 1
+          fi
+          # Upload the release asset using GitHub CLI
+          gh release upload "$tag_name" "$wheel_path_manylinux" --clobber
 
       - name: Clean up
         shell: bash

.github/workflows/_build_linux_self_host_manylinux.yml

@@ -68,7 +68,8 @@ jobs:
             gcc-c++ \
             clang \
             ninja-build \
-            time
+            time \
+            patchelf
 
       - name: Install gh
         shell: bash
@@ -84,17 +85,14 @@ jobs:
         run: |
           git config --global --add safe.directory $(pwd)
 
-      # - uses: actions/setup-python@v5
+      # Install Python using uv because setup-python needs newer version of glibc
-      #   with:
+      - name: Install uv
-      #     python-version: ${{ inputs.python-version }}
+        uses: astral-sh/setup-uv@v7
-
       - name: Install Python
         shell: bash
         run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          source $HOME/.local/bin/env
           uv venv -p ${{ inputs.python-version }}
-          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil
+          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil auditwheel
           current_dir=$(pwd)
           echo "$current_dir/.venv/bin" >> $GITHUB_PATH
 
@@ -137,6 +135,36 @@ jobs:
           # Upload the release asset using GitHub CLI
           gh release upload "$tag_name" "$wheel_path" --clobber
 
+      - name: Apply auditwheel repair
+        continue-on-error: true
+        run: |
+          auditwheel show flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }}
+          auditwheel repair \
+            --exclude libc10* --exclude libtorch* --exclude libcu* --exclude libnv* --exclude 'libtorch*' \
+            flash-attention/dist/${{ steps.build_wheels.outputs.WHEEL_NAME }} -w flash-attention/dist_manylinux
+          wheel_name=$(basename $(ls flash-attention/dist_manylinux/*manylinux*.whl | head -n 1))
+          echo "WHEEL_NAME_MANYLINUX=$wheel_name" >> $GITHUB_OUTPUT
+
+      - name: Test manylinux wheel
+        continue-on-error: true
+        run: |
+          pip uninstall -y flash-attn > /dev/null 2>&1
+          pip install --no-cache-dir flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}
+          python -c "import flash_attn; print(flash_attn.__version__)"
+
+      - name: Upload manylinux wheel
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          wheel_path_manylinux="flash-attention/dist_manylinux/${{ steps.build_wheels.outputs.WHEEL_NAME_MANYLINUX }}"
+          if [ ! -f "$wheel_path_manylinux" ]; then
+            echo "Error: Wheel file not found at $wheel_path_manylinux"
+            exit 1
+          fi
+          # Upload the release asset using GitHub CLI
+          gh release upload "$tag_name" "$wheel_path_manylinux" --clobber
+
       - name: Clean up
         shell: bash
         if: always()

.github/workflows/_build_windows.yml

@@ -48,9 +48,16 @@ jobs:
         shell: pwsh
         run: git config --system core.longpaths true
 
-      - uses: actions/setup-python@v5
+      # Install Python using uv because setup-python needs newer version of glibc
-        with:
+      - name: Install uv
-          python-version: ${{ inputs.python-version }}
+        uses: astral-sh/setup-uv@v7
+      - name: Install Python
+        shell: bash
+        run: |
+          uv venv -p ${{ inputs.python-version }}
+          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil ninja auditwheel
+          current_dir=$(pwd)
+          echo "$current_dir/.venv/bin" >> $GITHUB_PATH
 
       - uses: mjun0812/setup-cuda@v1
         with:
@@ -73,11 +80,6 @@ jobs:
       - name: Add msbuild to PATH
         uses: microsoft/setup-msbuild@v2
 
-      - name: Install build dependencies
-        shell: pwsh
-        run: |
-          python -m pip install -U setuptools==75.8.0 wheel packaging psutil ninja
-
       - name: Build wheels
         shell: pwsh
         run: |

.github/workflows/_build_windows_code_build.yml

@@ -44,9 +44,16 @@ jobs:
         shell: pwsh
         run: git config --system core.longpaths true
 
-      - uses: actions/setup-python@v5
+      # Install Python using uv because setup-python needs newer version of glibc
-        with:
+      - name: Install uv
-          python-version: ${{ inputs.python-version }}
+        uses: astral-sh/setup-uv@v7
+      - name: Install Python
+        shell: pwsh
+        run: |
+          uv venv -p ${{ inputs.python-version }}
+          uv pip install -U pip setuptools==75.8.0 wheel packaging psutil numpy ninja auditwheel
+          current_dir=$(pwd)
+          echo "$current_dir/.venv/bin" >> $GITHUB_PATH
 
       - uses: mjun0812/setup-cuda@v1
         with:
@@ -69,11 +76,6 @@ jobs:
       - name: Add msbuild to PATH
         uses: microsoft/setup-msbuild@v2
 
-      - name: Install build dependencies
-        shell: pwsh
-        run: |
-          pip install -U pip setuptools==75.8.0 wheel packaging psutil numpy ninja
-
       - name: Build wheels
         shell: pwsh
         timeout-minutes: 2160

.github/workflows/build.yml

@@ -177,7 +177,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: 3.14
 
       - name: Install dependencies
         run: pip install pandas
@@ -211,7 +211,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: 3.14
 
       - name: Install dependencies
         run: pip install pandas

.github/workflows/test_dynamic.yml

@@ -76,4 +76,4 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: 3.14

.github/workflows/update-download-stats.yml

@@ -19,6 +19,8 @@ jobs:
 
       - name: Set up Python
         uses: actions/setup-python@v5
+        with:
+          python-version: 3.14
 
       - name: Install dependencies
         run: |