From fc8fd33b161cc3cfd20b06ab9151592ee26dc483 Mon Sep 17 00:00:00 2001 From: Manan Shah <52329525+Manan17@users.noreply.github.com> Date: Thu, 17 Jul 2025 22:03:24 -0700 Subject: [PATCH] Giving an option to update benchmark results for previous commits. (#791) ## Summary This pr does two things: 1) Making H100 a strict requirement 2) It gives an option to run benchmarks for any commit manually but with the current (latest ie from main branch) benchmark scripts. This will help us get accurate benchmarks results for previous commits after the benchmark scripts are updated (merge this after the new benchmark scripts are merged). ## Testing Done - Hardware Type: - [ ] run `make test` to ensure correctness - [ ] run `make checkstyle` to ensure code style - [ ] run `make test-convergence` to ensure convergence --------- Co-authored-by: Manan Shah Co-authored-by: Vaibhav Jindal --- .github/workflows/benchmark.yml | 93 ++++++++++++++++++++++++++++----- dev/modal/benchmarks.py | 2 +- 2 files changed, 80 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b04cf093..0cc9f8fb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -5,6 +5,14 @@ on: # Runs at 00:00 UTC every Friday - cron: '0 0 * * 5' workflow_dispatch: # Enables manual trigger + inputs: + commit_hash: + description: 'Commit hash to benchmark' + default: 'main' + overwrite: + description: 'Overwrite existing benchmark data if it exists' + type: boolean + default: false permissions: contents: write @@ -28,13 +36,64 @@ jobs: steps: - - name: Checkout code - uses: actions/checkout@v3 + # Step: Decide the commit hash to use + - name: Determine commit hash to checkout + id: choose_commit + run: | + if [ "${{ github.event.inputs.commit_hash }}" != "" ]; then + echo "Using manual input commit: ${{ github.event.inputs.commit_hash }}" + echo "hash=${{ github.event.inputs.commit_hash }}" >> $GITHUB_OUTPUT + else + echo "Using latest commit from main" + git fetch origin main + echo "hash=$(git rev-parse origin/main)" >> $GITHUB_OUTPUT + fi - # Get the latest commit hash from main branch - - name: Get commit hash - id: get_hash - run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + # Step: Checkout full history so we can check out any commit + - name: Checkout full repo history + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Important: so we can checkout arbitrary commit + # Step: Conditionally replace benchmark folder from main + - name: Replace benchmark folder from main (manual only, commit ≠ main) + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.commit_hash != 'main' }} + run: | + echo "Detected manual trigger with commit_hash = ${{ github.event.inputs.commit_hash }}" + + # Save current branch (detached HEAD at old commit) + ORIG_COMMIT=${{ github.event.inputs.commit_hash }} + + # Fetch and checkout main + git fetch origin main + git checkout origin/main -- benchmark/ + + # Save benchmark folder from main + cp -r benchmark /tmp/benchmark_main + + # Checkout back to target commit + git checkout $ORIG_COMMIT + + # Replace old benchmark with one from main + rm -rf benchmark + cp -r /tmp/benchmark_main benchmark + + # Step: Check if benchmark exists and exit if overwrite is false + - name: Check existing benchmark + run: | + COMMIT_HASH="${{ steps.choose_commit.outputs.hash }}" + BENCHMARK_URL="https://raw.githubusercontent.com/linkedin/Liger-Kernel/refs/heads/gh-pages/benchmarks/${COMMIT_HASH}/benchmark.csv" + + if curl --output /dev/null --silent --head --fail "$BENCHMARK_URL"; then + echo "Benchmark already exists for commit $COMMIT_HASH" + if [ "${{ github.event.inputs.overwrite }}" != "true" ]; then + echo "Overwrite is false - exiting" + exit 1 + else + echo "Overwrite is true - proceeding" + fi + else + echo "No existing benchmark found - proceeding" + fi - name: Set up Python uses: actions/setup-python@v3 @@ -46,7 +105,6 @@ jobs: run: | python -m pip install --upgrade pip pip install modal - pip install pandas # Delete previous benchmark results. - name: Remove previous benchmark data @@ -67,8 +125,16 @@ jobs: # Step 6: Copy benchmark CSV to gh-pages directory - name: Copy generated benchmark to gh-pages run: | - mkdir -p gh-pages/${OUTPUT_DIR}/${{ steps.get_hash.outputs.hash }} - cp ${GENERATED_CSV} gh-pages/${OUTPUT_DIR}/${{ steps.get_hash.outputs.hash }}/${OUTPUT_FILENAME} + COMMIT_DIR="gh-pages/${OUTPUT_DIR}/${{ steps.choose_commit.outputs.hash }}" + mkdir -p "$COMMIT_DIR" + + if [ -f "$COMMIT_DIR/${OUTPUT_FILENAME}" ]; then + echo "Removing existing benchmark.csv for this commit" + rm "$COMMIT_DIR/${OUTPUT_FILENAME}" + fi + + cp "${GENERATED_CSV}" "$COMMIT_DIR/${OUTPUT_FILENAME}" + # Step 7: Append commit hash to commits.txt if not already present - name: Update commits.txt run: | @@ -77,10 +143,9 @@ jobs: # Create file if it doesn't exist mkdir -p ${OUTPUT_DIR} touch ${OUTPUT_DIR}/commits.txt - # Append only if not already present - if ! grep -q "${{ steps.get_hash.outputs.hash }}" ${OUTPUT_DIR}/commits.txt; then - echo "${{ steps.get_hash.outputs.hash }}" >> ${OUTPUT_DIR}/commits.txt - fi + + echo "${{ steps.choose_commit.outputs.hash }}" >> ${OUTPUT_DIR}/commits.txt + echo "Added commit hash to commits.txt" # Step 7: Commit and push - name: Commit and push to gh-pages run: | @@ -88,6 +153,6 @@ jobs: git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com git add . - git commit -m "Add benchmark for commit ${{ steps.get_hash.outputs.hash }}" || echo "No changes to commit" + git commit -m "Add benchmark for commit ${{ steps.choose_commit.outputs.hash }}" || echo "No changes to commit" git push origin gh-pages diff --git a/dev/modal/benchmarks.py b/dev/modal/benchmarks.py index 1bb97c90..a54fa47d 100644 --- a/dev/modal/benchmarks.py +++ b/dev/modal/benchmarks.py @@ -14,7 +14,7 @@ app = modal.App("liger_benchmarks", image=image) repo = image.add_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH) -@app.function(gpu=["H100"], image=repo, timeout=60 * 90) +@app.function(gpu="H100!", image=repo, timeout=60 * 90) def liger_benchmarks(): import os import subprocess