Giving an option to update benchmark results for previous commits. (#791)

## Summary
<!--- This is a required section; please describe the main purpose of
this proposed code change. --->
This pr does two things:
1) Making H100 a strict requirement
2) It gives an option to run benchmarks for any commit manually but with
the current (latest ie from main branch) benchmark scripts. This will
help us get accurate benchmarks results for previous commits after the
benchmark scripts are updated (merge this after the new benchmark
scripts are merged).
<!---
## Details
This is an optional section; is there anything specific that reviewers
should be aware of?
--->

## Testing Done
<!--- This is a required section; please describe how this change was
tested. --->

<!-- 
Replace BLANK with your device type. For example, A100-80G-PCIe

Complete the following tasks before sending your PR, and replace `[ ]`
with
`[x]` to indicate you have done them. 
-->

- Hardware Type: <BLANK>
- [ ] run `make test` to ensure correctness
- [ ] run `make checkstyle` to ensure code style
- [ ] run `make test-convergence` to ensure convergence

---------

Co-authored-by: Manan Shah <[email protected]>
Co-authored-by: Vaibhav Jindal <[email protected]>

.github/workflows/benchmark.yml

@@ -5,6 +5,14 @@ on:
     # Runs at 00:00 UTC every Friday
     - cron: '0 0 * * 5'
   workflow_dispatch:  # Enables manual trigger
+    inputs:
+      commit_hash:
+        description: 'Commit hash to benchmark'
+        default: 'main'
+      overwrite:
+        description: 'Overwrite existing benchmark data if it exists'
+        type: boolean
+        default: false
 
 permissions:
   contents: write
@@ -28,13 +36,64 @@ jobs:
 
 
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
+      # Step: Decide the commit hash to use
+      - name: Determine commit hash to checkout
+        id: choose_commit
+        run: |
+          if [ "${{ github.event.inputs.commit_hash }}" != "" ]; then
+            echo "Using manual input commit: ${{ github.event.inputs.commit_hash }}"
+            echo "hash=${{ github.event.inputs.commit_hash }}" >> $GITHUB_OUTPUT
+          else
+            echo "Using latest commit from main"
+            git fetch origin main
+            echo "hash=$(git rev-parse origin/main)" >> $GITHUB_OUTPUT
+          fi
 
-      # Get the latest commit hash from main branch
-      - name: Get commit hash
-        id: get_hash
-        run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      # Step: Checkout full history so we can check out any commit
+      - name: Checkout full repo history
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # Important: so we can checkout arbitrary commit
+            # Step: Conditionally replace benchmark folder from main
+      - name: Replace benchmark folder from main (manual only, commit ≠ main)
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.commit_hash != 'main' }}
+        run: |
+          echo "Detected manual trigger with commit_hash = ${{ github.event.inputs.commit_hash }}"
+          
+          # Save current branch (detached HEAD at old commit)
+          ORIG_COMMIT=${{ github.event.inputs.commit_hash }}
+
+          # Fetch and checkout main
+          git fetch origin main
+          git checkout origin/main -- benchmark/
+
+          # Save benchmark folder from main
+          cp -r benchmark /tmp/benchmark_main
+
+          # Checkout back to target commit
+          git checkout $ORIG_COMMIT
+
+          # Replace old benchmark with one from main
+          rm -rf benchmark
+          cp -r /tmp/benchmark_main benchmark
+    
+      # Step: Check if benchmark exists and exit if overwrite is false
+      - name: Check existing benchmark
+        run: |
+          COMMIT_HASH="${{ steps.choose_commit.outputs.hash }}"
+          BENCHMARK_URL="https://raw.githubusercontent.com/linkedin/Liger-Kernel/refs/heads/gh-pages/benchmarks/${COMMIT_HASH}/benchmark.csv"
+          
+          if curl --output /dev/null --silent --head --fail "$BENCHMARK_URL"; then
+            echo "Benchmark already exists for commit $COMMIT_HASH"
+            if [ "${{ github.event.inputs.overwrite }}" != "true" ]; then
+              echo "Overwrite is false - exiting"
+              exit 1
+            else
+              echo "Overwrite is true - proceeding"
+            fi
+          else
+            echo "No existing benchmark found - proceeding"
+          fi
 
       - name: Set up Python
         uses: actions/setup-python@v3
@@ -46,7 +105,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install modal
-          pip install pandas
 
       # Delete previous benchmark results.
       - name: Remove previous benchmark data
@@ -67,8 +125,16 @@ jobs:
       # Step 6: Copy benchmark CSV to gh-pages directory
       - name: Copy generated benchmark to gh-pages
         run: |
-          mkdir -p gh-pages/${OUTPUT_DIR}/${{ steps.get_hash.outputs.hash }}
-          cp ${GENERATED_CSV} gh-pages/${OUTPUT_DIR}/${{ steps.get_hash.outputs.hash }}/${OUTPUT_FILENAME}
+          COMMIT_DIR="gh-pages/${OUTPUT_DIR}/${{ steps.choose_commit.outputs.hash }}"
+          mkdir -p "$COMMIT_DIR"
+
+          if [ -f "$COMMIT_DIR/${OUTPUT_FILENAME}" ]; then
+            echo "Removing existing benchmark.csv for this commit"
+            rm "$COMMIT_DIR/${OUTPUT_FILENAME}"
+          fi
+
+          cp "${GENERATED_CSV}" "$COMMIT_DIR/${OUTPUT_FILENAME}"
+
       # Step 7: Append commit hash to commits.txt if not already present
       - name: Update commits.txt
         run: |
@@ -77,10 +143,9 @@ jobs:
           # Create file if it doesn't exist
           mkdir -p ${OUTPUT_DIR}
           touch ${OUTPUT_DIR}/commits.txt
-          # Append only if not already present
-          if ! grep -q "${{ steps.get_hash.outputs.hash }}" ${OUTPUT_DIR}/commits.txt; then
-            echo "${{ steps.get_hash.outputs.hash }}" >> ${OUTPUT_DIR}/commits.txt
-          fi
+
+          echo "${{ steps.choose_commit.outputs.hash }}" >> ${OUTPUT_DIR}/commits.txt
+          echo "Added commit hash to commits.txt"
       # Step 7: Commit and push
       - name: Commit and push to gh-pages
         run: |
@@ -88,6 +153,6 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
           git add .
-          git commit -m "Add benchmark for commit ${{ steps.get_hash.outputs.hash }}" || echo "No changes to commit"
+          git commit -m "Add benchmark for commit ${{ steps.choose_commit.outputs.hash }}" || echo "No changes to commit"
           git push origin gh-pages
 

dev/modal/benchmarks.py

@@ -14,7 +14,7 @@ app = modal.App("liger_benchmarks", image=image)
 repo = image.add_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH)
 
 
-@app.function(gpu=["H100"], image=repo, timeout=60 * 90)
+@app.function(gpu="H100!", image=repo, timeout=60 * 90)
 def liger_benchmarks():
     import os
     import subprocess