Skip to content

chore: rollback call and chain to accept a single argument #527

chore: rollback call and chain to accept a single argument

chore: rollback call and chain to accept a single argument #527

# Run GPU regression check only when attempting to merge, shown as skipped status check beforehand
name: GPU merge group regression check
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
branches: [main]
merge_group:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
# Run comparative benchmark against main, open issue on regression
gpu-benchmark:
if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
name: GPU bench regression check
runs-on: [self-hosted, gpu-bench]
steps:
- uses: actions/checkout@v4
with:
repository: lurk-lab/ci-workflows
- uses: ./.github/actions/gpu-setup
with:
gpu-framework: 'cuda'
- uses: ./.github/actions/ci-env
- uses: actions/checkout@v4
# Install dependencies
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: taiki-e/install-action@v2
with:
tool: [email protected]
- name: Install criterion
run: |
cargo install cargo-criterion
cargo install criterion-table
- name: Set bench output format and base SHA
run: |
echo "LURK_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV
echo "BASE_COMMIT=${{ github.event.merge_group.base_sha }}" | tee -a $GITHUB_ENV
GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1)
echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV
echo "GPU_NAME=$GPU_NAME" | tee -a $GITHUB_ENV
# Load Lurk bench env vars used in `justfile`, namely `LURK_BENCH_NOISE_THRESHOLD`
- uses: cardinalby/export-env-action@v2
with:
envFile: 'benches/bench.env'
# Checkout gh-pages to check for cached bench result
- name: Checkout gh-pages
uses: actions/checkout@v4
with:
ref: gh-pages
path: gh-pages
- name: Check for cached bench result
id: cached-bench
run: |
if [ -f "fibonacci-${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json" ]
then
echo "cached=true" | tee -a $GITHUB_OUTPUT
cp fibonacci-${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json ../fibonacci-${{ env.BASE_COMMIT }}.json
else
echo "cached=false" | tee -a $GITHUB_OUTPUT
fi
working-directory: ${{ github.workspace }}/gh-pages
# Checkout base branch for comparative bench
- uses: actions/checkout@v4
if: steps.cached-bench.outputs.cached == 'false'
with:
ref: main
path: main
# Copy the script so the base can bench with the same parameters
- name: Run GPU bench on base branch
if: steps.cached-bench.outputs.cached == 'false'
run: |
# Copy justfile & env to main, overwriting existing config with that of PR branch
cp ../benches/justfile ../benches/bench.env .
# Run benchmark
just gpu-bench-ci fibonacci
# Copy bench output to PR branch
cp fibonacci-${{ env.BASE_COMMIT }}.json ..
working-directory: ${{ github.workspace }}/main
- name: Run GPU bench on PR branch
run: |
just gpu-bench-ci fibonacci
cp fibonacci-${{ github.sha }}.json ..
working-directory: ${{ github.workspace }}/benches
- name: copy the benchmark template and prepare it with data
run: |
cp .github/tables.toml .
# Get CPU model
CPU_MODEL=$(grep '^model name' /proc/cpuinfo | head -1 | awk -F ': ' '{ print $2 }')
# Get vCPU count
NUM_VCPUS=$(nproc --all)
# Get total RAM in GB
TOTAL_RAM=$(grep MemTotal /proc/meminfo | awk '{$2=$2/(1024^2); print int($2), "GB RAM";}')
WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
# Use conditionals to ensure that only non-empty variables are inserted
[[ ! -z "${{ env.GPU_NAME }}" ]] && sed -i "/^\"\"\"$/i ${{ env.GPU_NAME }}" tables.toml
[[ ! -z "$CPU_MODEL" ]] && sed -i "/^\"\"\"$/i $CPU_MODEL" tables.toml
[[ ! -z "$NUM_VCPUS" ]] && sed -i "/^\"\"\"$/i $NUM_VCPUS vCPUs" tables.toml
[[ ! -z "$TOTAL_RAM" ]] && sed -i "/^\"\"\"$/i $TOTAL_RAM" tables.toml
sed -i "/^\"\"\"$/i Workflow run: $WORKFLOW_URL" tables.toml
echo "WORKFLOW_URL=$WORKFLOW_URL" | tee -a $GITHUB_ENV
working-directory: ${{ github.workspace }}
# Create a `criterion-table` and write in commit comment
- name: Run `criterion-table`
run: cat fibonacci-${{ env.BASE_COMMIT }}.json fibonacci-${{ github.sha }}.json | criterion-table > BENCHMARKS.md
- name: Write bench on commit comment
uses: peter-evans/commit-comment@v3
with:
body-path: BENCHMARKS.md
# Check for a slowdown >= `$ARECIBO_BENCH_NOISE_THRESHOLD` (fallback is 10%/1.1x). If so, open an issue but don't block merge
# Since we are parsing for slowdowns, we simply add 1 to the noise threshold decimal to get the regression factor
- name: Check for perf regression
id: regression-check
run: |
REGRESSIONS=$(grep -o '[0-9.]*x slower' BENCHMARKS.md | cut -d 'x' -f1)
echo $REGRESSIONS
if [ ! -z "${{ env.LURK_BENCH_NOISE_THRESHOLD}}" ]; then
REGRESSION_FACTOR=$(echo "${{ env.LURK_BENCH_NOISE_THRESHOLD }}+1" | bc)
else
REGRESSION_FACTOR=1.1
fi
echo "NOISE_THRESHOLD=$(echo "($REGRESSION_FACTOR-1)*100" | bc)" | tee -a $GITHUB_ENV
for r in $REGRESSIONS
do
if (( $(echo "$r >= $REGRESSION_FACTOR" | bc -l) ))
then
echo "regression=true" | tee -a $GITHUB_OUTPUT
fi
done
# Not possible to use ${{ github.event.number }} with the `merge_group` trigger
- name: Get PR number from merge branch
if: steps.regression-check.outputs.regression == 'true'
run: |
echo "PR_NUMBER=$(echo ${{ github.event.merge_group.head_ref }} | sed -e 's/.*pr-\(.*\)-.*/\1/')" | tee -a $GITHUB_ENV
- uses: JasonEtco/create-an-issue@v2
if: steps.regression-check.outputs.regression == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ env.PR_NUMBER }}
GIT_SHA: ${{ github.sha }}
WORKFLOW_URL: ${{ env.WORKFLOW_URL }}
NOISE_THRESHOLD: ${{ env.NOISE_THRESHOLD }}
with:
filename: .github/PERF_REGRESSION.md
- name: Remove old main bench
if: steps.regression-check.outputs.regression != 'true'
run: |
rm fibonacci-${{ env.BASE_COMMIT }}.json
mv fibonacci-${{ github.sha }}.json fibonacci-${{ github.sha }}-${{ env.GPU_ID }}.json
working-directory: ${{ github.workspace }}
- name: Commit bench result to `gh-pages` branch if no regression
if: steps.regression-check.outputs.regression != 'true'
uses: stefanzweifel/git-auto-commit-action@v5
with:
branch: gh-pages
commit_message: '[automated] GPU Benchmark from PR #${{ env.PR_NUMBER }}'
file_pattern: 'fibonacci-${{ github.sha }}-${{ env.GPU_ID }}.json'