From 2f3b1245f79f2f79ce3e252797e2928e6f25c05a Mon Sep 17 00:00:00 2001 From: Junya Morioka Date: Sun, 21 Dec 2025 04:06:56 +0900 Subject: [PATCH] refactor: consolidate self-hosted workflows with auto-detection - Merge _build_manylinux_self_host.yml into _build_linux_self_host.yml - Add automatic package manager detection (apt-get/dnf) for both x86_64 and ARM64 - Add environment check steps to all self-hosted workflows - Update build.yml to use unified workflow with container-image parameter - Remove duplicate build_wheels_manylinux_self_hosted job - Update test workflows to use consolidated workflow - Remove manylinux_self_hosted matrix configuration - Set manylinux container image for ARM64 test workflow --- .../workflows/_build_linux_arm_self_host.yml | 94 +++++++--- .github/workflows/_build_linux_self_host.yml | 175 +++++++++++++----- .../workflows/_build_manylinux_self_host.yml | 101 ---------- .github/workflows/build.yml | 27 +-- .github/workflows/test-arm-self-hosted.yml | 2 +- .../workflows/test-self-hosted-manylinux.yml | 29 --- create_matrix.py | 3 - 7 files changed, 198 insertions(+), 233 deletions(-) delete mode 100644 .github/workflows/_build_manylinux_self_host.yml delete mode 100644 .github/workflows/test-self-hosted-manylinux.yml diff --git a/.github/workflows/_build_linux_arm_self_host.yml b/.github/workflows/_build_linux_arm_self_host.yml index 9a1ee36..2c0aa44 100644 --- a/.github/workflows/_build_linux_arm_self_host.yml +++ b/.github/workflows/_build_linux_arm_self_host.yml @@ -55,38 +55,82 @@ jobs: timeout-minutes: 2160 steps: + - name: Check environment + shell: bash + run: | + cat /etc/os-release || true + echo "-----------------------------" + cat /etc/lsb-release || true + echo "-----------------------------" + lscpu || true + echo "-----------------------------" + df -h || true + echo "-----------------------------" + free -h || true + - name: Install tools shell: bash run: | - apt-get update && apt-get install -y --no-install-recommends \ - curl \ - ca-certificates \ - sudo \ - software-properties-common \ - wget \ - unzip \ - zip \ - git \ - build-essential \ - gcc \ - g++ \ - clang \ - ninja-build \ - keyboard-configuration \ - time \ - patchelf + # Auto-detect package manager and install dependencies + if command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + sudo \ + software-properties-common \ + wget \ + unzip \ + zip \ + git \ + build-essential \ + gcc \ + g++ \ + clang \ + ninja-build \ + keyboard-configuration \ + time \ + patchelf + elif command -v dnf >/dev/null 2>&1; then + dnf install -y \ + curl \ + ca-certificates \ + sudo \ + wget \ + unzip \ + zip \ + git \ + gcc \ + gcc-c++ \ + clang \ + ninja-build \ + time \ + patchelf + else + echo "Error: No supported package manager found (apt-get or dnf)" + exit 1 + fi - name: Install gh shell: bash run: | - sudo mkdir -p -m 755 /etc/apt/keyrings - out=$(mktemp) - wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg - cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null - sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null - sudo apt update - sudo apt install gh -y + # Auto-detect package manager and install GitHub CLI + if command -v apt-get >/dev/null 2>&1; then + sudo mkdir -p -m 755 /etc/apt/keyrings + out=$(mktemp) + wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg + cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh -y + elif command -v dnf >/dev/null 2>&1; then + dnf install -y 'dnf-command(config-manager)' + dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo + dnf -y install gh --repo gh-cli + else + echo "Error: No supported package manager found (apt-get or dnf)" + exit 1 + fi - uses: actions/checkout@v4 diff --git a/.github/workflows/_build_linux_self_host.yml b/.github/workflows/_build_linux_self_host.yml index 7b8ccdc..d7b4765 100644 --- a/.github/workflows/_build_linux_self_host.yml +++ b/.github/workflows/_build_linux_self_host.yml @@ -60,38 +60,82 @@ jobs: timeout-minutes: 2160 steps: + - name: Check environment + shell: bash + run: | + cat /etc/os-release || true + echo "-----------------------------" + cat /etc/lsb-release || true + echo "-----------------------------" + lscpu || true + echo "-----------------------------" + df -h || true + echo "-----------------------------" + free -h || true + - name: Install tools shell: bash run: | - apt-get update && apt-get install -y --no-install-recommends \ - curl \ - ca-certificates \ - sudo \ - software-properties-common \ - wget \ - unzip \ - zip \ - git \ - build-essential \ - gcc \ - g++ \ - clang \ - ninja-build \ - keyboard-configuration \ - time \ - patchelf + # Auto-detect package manager and install dependencies + if command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + sudo \ + software-properties-common \ + wget \ + unzip \ + zip \ + git \ + build-essential \ + gcc \ + g++ \ + clang \ + ninja-build \ + keyboard-configuration \ + time \ + patchelf + elif command -v dnf >/dev/null 2>&1; then + dnf install -y \ + curl \ + ca-certificates \ + sudo \ + wget \ + unzip \ + zip \ + git \ + gcc \ + gcc-c++ \ + clang \ + ninja-build \ + time \ + patchelf + else + echo "Error: No supported package manager found (apt-get or dnf)" + exit 1 + fi - name: Install gh shell: bash run: | - sudo mkdir -p -m 755 /etc/apt/keyrings - out=$(mktemp) - wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg - cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null - sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null - sudo apt update - sudo apt install gh -y + # Auto-detect package manager and install GitHub CLI + if command -v apt-get >/dev/null 2>&1; then + sudo mkdir -p -m 755 /etc/apt/keyrings + out=$(mktemp) + wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg + cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh -y + elif command -v dnf >/dev/null 2>&1; then + dnf install -y 'dnf-command(config-manager)' + dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo + dnf -y install gh --repo gh-cli + else + echo "Error: No supported package manager found (apt-get or dnf)" + exit 1 + fi - uses: actions/checkout@v4 @@ -138,35 +182,66 @@ jobs: - name: Install tools shell: bash run: | - apt-get update && apt-get install -y --no-install-recommends \ - curl \ - ca-certificates \ - sudo \ - software-properties-common \ - wget \ - unzip \ - zip \ - git \ - build-essential \ - gcc \ - g++ \ - clang \ - ninja-build \ - keyboard-configuration \ - time \ - patchelf + # Auto-detect package manager and install dependencies + if command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + sudo \ + software-properties-common \ + wget \ + unzip \ + zip \ + git \ + build-essential \ + gcc \ + g++ \ + clang \ + ninja-build \ + keyboard-configuration \ + time \ + patchelf + elif command -v dnf >/dev/null 2>&1; then + dnf install -y \ + curl \ + ca-certificates \ + sudo \ + wget \ + unzip \ + zip \ + git \ + gcc \ + gcc-c++ \ + clang \ + ninja-build \ + time \ + patchelf + else + echo "Error: No supported package manager found (apt-get or dnf)" + exit 1 + fi - name: Install gh shell: bash run: | - sudo mkdir -p -m 755 /etc/apt/keyrings - out=$(mktemp) - wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg - cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null - sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null - sudo apt update - sudo apt install gh -y + # Auto-detect package manager and install GitHub CLI + if command -v apt-get >/dev/null 2>&1; then + sudo mkdir -p -m 755 /etc/apt/keyrings + out=$(mktemp) + wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg + cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh -y + elif command -v dnf >/dev/null 2>&1; then + dnf install -y 'dnf-command(config-manager)' + dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo + dnf -y install gh --repo gh-cli + else + echo "Error: No supported package manager found (apt-get or dnf)" + exit 1 + fi - uses: actions/checkout@v4 diff --git a/.github/workflows/_build_manylinux_self_host.yml b/.github/workflows/_build_manylinux_self_host.yml deleted file mode 100644 index 8b6f1e4..0000000 --- a/.github/workflows/_build_manylinux_self_host.yml +++ /dev/null @@ -1,101 +0,0 @@ -# ######################################################### -# Build wheels with self-hosted runner -# ######################################################### - -name: "[manylinux x86_64, self-hosted] Build wheels and upload to GitHub Releases" - -on: - workflow_call: - inputs: - flash-attn-version: - description: "Flash-Attention version" - required: true - type: string - python-version: - description: "Python version" - required: true - type: string - torch-version: - description: "PyTorch version" - required: true - type: string - cuda-version: - description: "CUDA version" - required: true - type: string - runner: - description: "Runner type" - required: false - type: string - default: '["self-hosted"]' - is-upload: - description: "Whether to upload the release asset" - required: false - type: boolean - default: true - container-image: - description: "Container image" - required: false - type: string - default: "quay.io/pypa/manylinux_2_28_x86_64" - -jobs: - build_wheels_self_hosted: - name: Build wheels and Upload (manylinux x86_64, self-hosted runner) - runs-on: ${{ fromjson(inputs.runner) }} - container: - image: ${{ inputs.container-image }} - defaults: - run: - shell: bash - env: - TERM: xterm-256color - timeout-minutes: 2160 - - steps: - - name: Install tools - shell: bash - run: | - dnf install -y \ - curl \ - ca-certificates \ - sudo \ - wget \ - unzip \ - zip \ - git \ - gcc \ - gcc-c++ \ - clang \ - ninja-build \ - time \ - patchelf - - - name: Configure sudo - shell: bash - run: | - echo "root ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - - - name: Install gh - shell: bash - run: | - dnf install -y 'dnf-command(config-manager)' - dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo - dnf -y install gh --repo gh-cli - - - uses: actions/checkout@v4 - - - name: Configure Git safe directory - shell: bash - run: | - git config --global --add safe.directory $(pwd) - - - name: Build and upload wheels - uses: ./.github/actions/build-and-upload - with: - flash-attn-version: ${{ inputs.flash-attn-version }} - python-version: ${{ inputs.python-version }} - torch-version: ${{ inputs.torch-version }} - cuda-version: ${{ inputs.cuda-version }} - is-upload: ${{ inputs.is-upload }} - cleanup: "true" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ae1f86..a8286bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,13 +91,14 @@ jobs: torch-version: ${{ fromjson(needs.create_matrix.outputs.matrix).linux_self_hosted.torch-version }} cuda-version: ${{ fromjson(needs.create_matrix.outputs.matrix).linux_self_hosted.cuda-version }} exclude: ${{ fromjson(needs.create_matrix.outputs.matrix).exclude }} - uses: ./.github/workflows/_build_manylinux_self_host.yml + uses: ./.github/workflows/_build_linux_self_host.yml with: flash-attn-version: ${{ matrix.flash-attn-version }} python-version: ${{ matrix.python-version }} torch-version: ${{ matrix.torch-version }} cuda-version: ${{ matrix.cuda-version }} runner: '["self-hosted", "x64"]' + container-image: "quay.io/pypa/manylinux_2_28_x86_64" secrets: inherit build_wheels_linux_arm64_self_hosted: @@ -119,27 +120,7 @@ jobs: torch-version: ${{ matrix.torch-version }} cuda-version: ${{ matrix.cuda-version }} runner: '["self-hosted-arm", "arm64"]' - secrets: inherit - - build_wheels_manylinux_self_hosted: - name: Build manylinux (self-hosted) - needs: [create_releases, create_matrix] - if: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted }} - strategy: - fail-fast: false - matrix: - flash-attn-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.flash-attn-version }} - python-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.python-version }} - torch-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.torch-version }} - cuda-version: ${{ fromjson(needs.create_matrix.outputs.matrix).manylinux_self_hosted.cuda-version }} - exclude: ${{ fromjson(needs.create_matrix.outputs.matrix).exclude }} - uses: ./.github/workflows/_build_manylinux_self_host.yml - with: - flash-attn-version: ${{ matrix.flash-attn-version }} - python-version: ${{ matrix.python-version }} - torch-version: ${{ matrix.torch-version }} - cuda-version: ${{ matrix.cuda-version }} - runner: '["self-hosted", "x64"]' + container-image: "quay.io/pypa/manylinux_2_28_aarch64" secrets: inherit # ######################################################### @@ -192,7 +173,6 @@ jobs: - build_wheels_linux_arm64 - build_wheels_linux_self_hosted - build_wheels_linux_arm64_self_hosted - - build_wheels_manylinux_self_hosted - build_wheels_windows - build_wheels_windows_code_build if: always() @@ -222,7 +202,6 @@ jobs: - build_wheels_linux_arm64 - build_wheels_linux_self_hosted - build_wheels_linux_arm64_self_hosted - - build_wheels_manylinux_self_hosted - build_wheels_windows - build_wheels_windows_code_build permissions: diff --git a/.github/workflows/test-arm-self-hosted.yml b/.github/workflows/test-arm-self-hosted.yml index 049d57a..2a3f2a0 100644 --- a/.github/workflows/test-arm-self-hosted.yml +++ b/.github/workflows/test-arm-self-hosted.yml @@ -26,4 +26,4 @@ jobs: cuda-version: ${{ matrix.cuda-version }} is-upload: false runner: ${{ matrix.runner}} - container-image: "ubuntu:22.04" + container-image: "quay.io/pypa/manylinux_2_28_aarch64" diff --git a/.github/workflows/test-self-hosted-manylinux.yml b/.github/workflows/test-self-hosted-manylinux.yml deleted file mode 100644 index 7145a40..0000000 --- a/.github/workflows/test-self-hosted-manylinux.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Test self-hosted Linux build (Manylinux 2_28) - -on: - workflow_dispatch: - -jobs: - # ######################################################### - # Build wheels with self-hosted runner - # ######################################################### - build_wheels_self_hosted: - name: Build wheels and Upload (Linux x86_64, self-hosted runner, Manylinux 2_28) - uses: ./.github/workflows/_build_manylinux_self_host.yml - strategy: - fail-fast: false - matrix: - flash-attn-version: ["2.8.3"] - python-version: ["3.13"] - torch-version: ["2.9.1"] - # https://developer.nvidia.com/cuda-toolkit-archive - cuda-version: ["13.0.1"] - runner: ['["self-hosted", "x64"]'] - use-container: [true] - with: - flash-attn-version: ${{ matrix.flash-attn-version }} - python-version: ${{ matrix.python-version }} - torch-version: ${{ matrix.torch-version }} - cuda-version: ${{ matrix.cuda-version }} - is-upload: false - runner: ${{ matrix.runner}} diff --git a/create_matrix.py b/create_matrix.py index 345bb53..f23167a 100644 --- a/create_matrix.py +++ b/create_matrix.py @@ -200,9 +200,6 @@ def main(): "linux_self_hosted": False, # "linux_self_hosted": LINUX_SELF_HOSTED_MATRIX, # - # "manylinux_self_hosted": False, - "manylinux_self_hosted": LINUX_SELF_HOSTED_MATRIX, - # "linux_arm64_self_hosted": False, # "linux_arm64_self_hosted": LINUX_ARM64_SELF_HOSTED_MATRIX, #