yindo/flash-attention-prebuild-wheels-rocm
1
0
Fork 0
mirror of https://github.com/BillyOutlast/flash-attention-prebuild-wheels-rocm.git synced 2026-07-01 01:37:53 -04:00
Code Issues Packages Projects Releases 2 Wiki Activity

refactor: consolidate self-hosted workflows with auto-detection

Browse Source 
- Merge _build_manylinux_self_host.yml into _build_linux_self_host.yml
- Add automatic package manager detection (apt-get/dnf) for both x86_64 and ARM64
- Add environment check steps to all self-hosted workflows
- Update build.yml to use unified workflow with container-image parameter
- Remove duplicate build_wheels_manylinux_self_hosted job
- Update test workflows to use consolidated workflow
- Remove manylinux_self_hosted matrix configuration
- Set manylinux container image for ARM64 test workflow
This commit is contained in:
Junya Morioka
2025-12-21 04:06:56 +09:00
parent df03c553f7
commit 2f3b1245f7
7 changed files with 198 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_build_linux_arm_self_host.yml
+69 -25
View File
@@ -55,38 +55,82 @@ jobs:
timeout-minutes: 2160
steps:
- name: Check environment
shell: bash
run: |
cat /etc/os-release || true
echo "-----------------------------"
cat /etc/lsb-release || true
echo "-----------------------------"
lscpu || true
echo "-----------------------------"
df -h || true
echo "-----------------------------"
free -h || true
- name: Install tools
shell: bash
run: |
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration \
time \
patchelf
# Auto-detect package manager and install dependencies
if command -v apt-get >/dev/null 2>&1; then
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration \
time \
patchelf
elif command -v dnf >/dev/null 2>&1; then
dnf install -y \
curl \
ca-certificates \
sudo \
wget \
unzip \
zip \
git \
gcc \
gcc-c++ \
clang \
ninja-build \
time \
patchelf
else
echo "Error: No supported package manager found (apt-get or dnf)"
exit 1
fi
- name: Install gh
shell: bash
run: |
sudo mkdir -p -m 755 /etc/apt/keyrings
out=$(mktemp)
wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg
cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
sudo apt update
sudo apt install gh -y
# Auto-detect package manager and install GitHub CLI
if command -v apt-get >/dev/null 2>&1; then
sudo mkdir -p -m 755 /etc/apt/keyrings
out=$(mktemp)
wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg
cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
sudo apt update
sudo apt install gh -y
elif command -v dnf >/dev/null 2>&1; then
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
dnf -y install gh --repo gh-cli
else
echo "Error: No supported package manager found (apt-get or dnf)"
exit 1
fi
- uses: actions/checkout@v4
.github/workflows/_build_linux_self_host.yml
+125 -50
View File
@@ -60,38 +60,82 @@ jobs:
timeout-minutes: 2160
steps:
- name: Check environment
shell: bash
run: |
cat /etc/os-release || true
echo "-----------------------------"
cat /etc/lsb-release || true
echo "-----------------------------"
lscpu || true
echo "-----------------------------"
df -h || true
echo "-----------------------------"
free -h || true
- name: Install tools
shell: bash
run: |
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration \
time \
patchelf
# Auto-detect package manager and install dependencies
if command -v apt-get >/dev/null 2>&1; then
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration \
time \
patchelf
elif command -v dnf >/dev/null 2>&1; then
dnf install -y \
curl \
ca-certificates \
sudo \
wget \
unzip \
zip \
git \
gcc \
gcc-c++ \
clang \
ninja-build \
time \
patchelf
else
echo "Error: No supported package manager found (apt-get or dnf)"
exit 1
fi
- name: Install gh
shell: bash
run: |
sudo mkdir -p -m 755 /etc/apt/keyrings
out=$(mktemp)
wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg
cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
sudo apt update
sudo apt install gh -y
# Auto-detect package manager and install GitHub CLI
if command -v apt-get >/dev/null 2>&1; then
sudo mkdir -p -m 755 /etc/apt/keyrings
out=$(mktemp)
wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg
cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
sudo apt update
sudo apt install gh -y
elif command -v dnf >/dev/null 2>&1; then
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
dnf -y install gh --repo gh-cli
else
echo "Error: No supported package manager found (apt-get or dnf)"
exit 1
fi
- uses: actions/checkout@v4
@@ -138,35 +182,66 @@ jobs:
- name: Install tools
shell: bash
run: |
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration \
time \
patchelf
# Auto-detect package manager and install dependencies
if command -v apt-get >/dev/null 2>&1; then
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
software-properties-common \
wget \
unzip \
zip \
git \
build-essential \
gcc \
g++ \
clang \
ninja-build \
keyboard-configuration \
time \
patchelf
elif command -v dnf >/dev/null 2>&1; then
dnf install -y \
curl \
ca-certificates \
sudo \
wget \
unzip \
zip \
git \
gcc \
gcc-c++ \
clang \
ninja-build \
time \
patchelf
else
echo "Error: No supported package manager found (apt-get or dnf)"
exit 1
fi
- name: Install gh
shell: bash
run: |
sudo mkdir -p -m 755 /etc/apt/keyrings
out=$(mktemp)
wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg
cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
sudo apt update
sudo apt install gh -y
# Auto-detect package manager and install GitHub CLI
if command -v apt-get >/dev/null 2>&1; then
sudo mkdir -p -m 755 /etc/apt/keyrings
out=$(mktemp)
wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg
cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
sudo apt update
sudo apt install gh -y
elif command -v dnf >/dev/null 2>&1; then
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
dnf -y install gh --repo gh-cli
else
echo "Error: No supported package manager found (apt-get or dnf)"
exit 1
fi
- uses: actions/checkout@v4
.github/workflows/_build_manylinux_self_host.yml
-101
View File
@@ -1,101 +0,0 @@
# #########################################################
# Build wheels with self-hosted runner
# #########################################################
name: "[manylinux x86_64, self-hosted] Build wheels and upload to GitHub Releases"
on:
workflow_call:
inputs:
flash-attn-version:
description: "Flash-Attention version"
required: true
type: string
python-version:
description: "Python version"
required: true
type: string
torch-version:
description: "PyTorch version"
required: true
type: string
cuda-version:
description: "CUDA version"
required: true
type: string
runner:
description: "Runner type"
required: false
type: string
default: '["self-hosted"]'
is-upload:
description: "Whether to upload the release asset"
required: false
type: boolean
default: true
container-image:
description: "Container image"
required: false
type: string
default: "quay.io/pypa/manylinux_2_28_x86_64"
jobs:
build_wheels_self_hosted:
name: Build wheels and Upload (manylinux x86_64, self-hosted runner)
runs-on: ${{ fromjson(inputs.runner) }}
container:
image: ${{ inputs.container-image }}
defaults:
run:
shell: bash
env:
TERM: xterm-256color
timeout-minutes: 2160
steps:
- name: Install tools
shell: bash
run: |
dnf install -y \
curl \
ca-certificates \
sudo \
wget \
unzip \
zip \
git \
gcc \
gcc-c++ \
clang \
ninja-build \
time \
patchelf
- name: Configure sudo
shell: bash
run: |
echo "root ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
- name: Install gh
shell: bash
run: |
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
dnf -y install gh --repo gh-cli
- uses: actions/checkout@v4
- name: Configure Git safe directory
shell: bash
run: |
git config --global --add safe.directory $(pwd)
- name: Build and upload wheels
uses: ./.github/actions/build-and-upload
with:
flash-attn-version: ${{ inputs.flash-attn-version }}
python-version: ${{ inputs.python-version }}
torch-version: ${{ inputs.torch-version }}
cuda-version: ${{ inputs.cuda-version }}
is-upload: ${{ inputs.is-upload }}
cleanup: "true"
.github/workflows/build.yml
+3 -24
View File
@@ -91,13 +91,14 @@ jobs:
torch-version: ${{ fromjson(needs.create_matrix.outputs.matrix).linux_self_hosted.torch-version }}
cuda-version: ${{ fromjson(needs.create_matrix.outputs.matrix).linux_self_hosted.cuda-version }}
exclude: ${{ fromjson(needs.create_matrix.outputs.matrix).exclude }}
uses: ./.github/workflows/_build_manylinux_self_host.yml
uses: ./.github/workflows/_build_linux_self_host.yml
with:
flash-attn-version: ${{ matrix.flash-attn-version }}
python-version: ${{ matrix.python-version }}
torch-version: ${{ matrix.torch-version }}
cuda-version: ${{ matrix.cuda-version }}
runner: '["self-hosted", "x64"]'
container-image: "quay.io/pypa/manylinux_2_28_x86_64"
secrets: inherit
build_wheels_linux_arm64_self_hosted:
@@ -119,27 +120,7 @@ jobs:
torch-version: ${{ matrix.torch-version }}
cuda-version: ${{ matrix.cuda-version }}
runner: '["self-hosted-arm", "arm64"]'
secrets: inherit
build_wheels_manylinux_self_hosted:
name: Build manylinux (self-hosted)
needs: [create_releases, create_matrix]
if: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted }}
strategy:
fail-fast: false
matrix:
flash-attn-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.flash-attn-version }}
python-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.python-version }}
torch-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.torch-version }}
cuda-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.cuda-version }}
exclude: ${{ fromjson(needs.create_matrix.outputs.matrix).exclude }}
uses: ./.github/workflows/_build_manylinux_self_host.yml
with:
flash-attn-version: ${{ matrix.flash-attn-version }}
python-version: ${{ matrix.python-version }}
torch-version: ${{ matrix.torch-version }}
cuda-version: ${{ matrix.cuda-version }}
runner: '["self-hosted", "x64"]'
container-image: "quay.io/pypa/manylinux_2_28_aarch64"
secrets: inherit
# #########################################################
@@ -192,7 +173,6 @@ jobs:
- build_wheels_linux_arm64
- build_wheels_linux_self_hosted
- build_wheels_linux_arm64_self_hosted
- build_wheels_manylinux_self_hosted
- build_wheels_windows
- build_wheels_windows_code_build
if: always()
@@ -222,7 +202,6 @@ jobs:
- build_wheels_linux_arm64
- build_wheels_linux_self_hosted
- build_wheels_linux_arm64_self_hosted
- build_wheels_manylinux_self_hosted
- build_wheels_windows
- build_wheels_windows_code_build
permissions:
.github/workflows/test-arm-self-hosted.yml
+1 -1
View File
@@ -26,4 +26,4 @@ jobs:
cuda-version: ${{ matrix.cuda-version }}
is-upload: false
runner: ${{ matrix.runner}}
container-image: "ubuntu:22.04"
container-image: "quay.io/pypa/manylinux_2_28_aarch64"
.github/workflows/test-self-hosted-manylinux.yml
-29
View File
@@ -1,29 +0,0 @@
name: Test self-hosted Linux build (Manylinux 2_28)
on:
workflow_dispatch:
jobs:
# #########################################################
# Build wheels with self-hosted runner
# #########################################################
build_wheels_self_hosted:
name: Build wheels and Upload (Linux x86_64, self-hosted runner, Manylinux 2_28)
uses: ./.github/workflows/_build_manylinux_self_host.yml
strategy:
fail-fast: false
matrix:
flash-attn-version: ["2.8.3"]
python-version: ["3.13"]
torch-version: ["2.9.1"]
# https://developer.nvidia.com/cuda-toolkit-archive
cuda-version: ["13.0.1"]
runner: ['["self-hosted", "x64"]']
use-container: [true]
with:
flash-attn-version: ${{ matrix.flash-attn-version }}
python-version: ${{ matrix.python-version }}
torch-version: ${{ matrix.torch-version }}
cuda-version: ${{ matrix.cuda-version }}
is-upload: false
runner: ${{ matrix.runner}}
create_matrix.py
-3
View File
@@ -200,9 +200,6 @@ def main():
"linux_self_hosted": False,
# "linux_self_hosted": LINUX_SELF_HOSTED_MATRIX,
#
# "manylinux_self_hosted": False,
"manylinux_self_hosted": LINUX_SELF_HOSTED_MATRIX,
#
"linux_arm64_self_hosted": False,
# "linux_arm64_self_hosted": LINUX_ARM64_SELF_HOSTED_MATRIX,
#