From 5e74d4b65a6782baf0d4006f5ee2ffcddc99fd56 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 29 Aug 2024 13:45:23 -0700 Subject: [PATCH] Simplify cache cleanup actions In our old approach, a workflow file contains a job that uploads the PR number as an artifact. While the PR is still open, the workflow_run triggered by it will download the artifact and use the information to clean up all except the last used cache associated with that original workflow. When a PR is merged or closed, there will be a post-pr workflow that uploads the PR number as an artifact and triggers a workflow_run that clean up all caches associated with the PR. The reason we did it this way was in the cache cleanup workflows, we did not find an easy way to get the number of the PR triggering them. This is not convenient because we have to add jobs uploading artifacts to workflow files. After some experiments, we have found a reliable way to find the PR number without using artifacts. The workflow_run's payload always contains the head SHA of the commit that triggers it, whether the PR comes from a fork or not. We can then use `gh pr list` to search for that head and obtain the PR number. --- .github/workflows/apps.yml | 15 --------------- .github/workflows/bittree.yml | 15 --------------- .github/workflows/clang.yml | 15 --------------- .github/workflows/cleanup-cache-postpr.yml | 7 +++++-- .github/workflows/cleanup-cache.yml | 8 ++++++-- .github/workflows/codeql.yml | 15 --------------- .github/workflows/cuda.yml | 15 --------------- .github/workflows/gcc.yml | 15 --------------- .github/workflows/hip.yml | 15 --------------- .github/workflows/hypre.yml | 15 --------------- .github/workflows/intel.yml | 15 --------------- .github/workflows/macos.yml | 15 --------------- .github/workflows/petsc.yml | 15 --------------- .github/workflows/post-pr.yml | 18 +++++++----------- .github/workflows/smoke.yml | 15 --------------- .github/workflows/sundials.yml | 15 --------------- .github/workflows/windows.yml | 14 -------------- 17 files changed, 18 insertions(+), 224 deletions(-) diff --git a/.github/workflows/apps.yml b/.github/workflows/apps.yml index 7de78c964a2..e909193601b 100644 --- a/.github/workflows/apps.yml +++ b/.github/workflows/apps.yml @@ -152,18 +152,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/bittree.yml b/.github/workflows/bittree.yml index 45320008728..299fb025d88 100644 --- a/.github/workflows/bittree.yml +++ b/.github/workflows/bittree.yml @@ -110,18 +110,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/clang.yml b/.github/workflows/clang.yml index d02342899b8..c84d19850a7 100644 --- a/.github/workflows/clang.yml +++ b/.github/workflows/clang.yml @@ -212,18 +212,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 9a2ffb0f61a..5e9a70cd5a4 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -23,8 +23,11 @@ jobs: REPO=${{ github.repository }} - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + # For debugging cat ${GITHUB_EVENT_PATH} to see the payload. + + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number') + echo "Post-PR cache cleanup for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge # Setting this to not fail the workflow while deleting cache keys. diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index d18acbaa788..75bd8453aa4 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -29,9 +29,12 @@ jobs: # Triggering workflow run name (e.g., LinuxClang) WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + # For debugging, cat ${GITHUB_EVENT_PATH} to see the payload. + if [[ $EVENT == "pull_request" ]]; then - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --search $pr_head_sha --json number --jq '.[0].number') + echo "Clean up cache for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge else BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} @@ -54,6 +57,7 @@ jobs: IFS=$'\n' for j in $cached_jobs do + # Delete all entries except the last used one old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) for k in $old_keys do diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c7340ee449b..c9c34bf34f3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -92,18 +92,3 @@ jobs: uses: github/codeql-action/analyze@v3 with: category: "/language:${{ matrix.language }}" - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 1ceed52ba63..50d45846f02 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -205,18 +205,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml index 5d8c9ad9c30..26ad6eaf529 100644 --- a/.github/workflows/gcc.yml +++ b/.github/workflows/gcc.yml @@ -664,18 +664,3 @@ jobs: mpirun -np 2 ./main3d.gnu.TPROF.MPI.ex ./inputs h5dump -d "level_0/data:offsets=0" -s "1" -c "1" ./plt00000.h5 h5dump -d "level_0/data:datatype=1" -s "1" -c "1" ./plt00000/particle0/particle0.h5 - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 3e11dda151a..a38c1f65479 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -181,18 +181,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/hypre.yml b/.github/workflows/hypre.yml index b147423273b..c6f2ee20edc 100644 --- a/.github/workflows/hypre.yml +++ b/.github/workflows/hypre.yml @@ -154,18 +154,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 4d1612d0ec4..36fc674ecd1 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -249,18 +249,3 @@ jobs: set -e cd build ctest --output-on-failure - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index d1e217380e9..76aba4cef6d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -91,18 +91,3 @@ jobs: du -hs ~/Library/Caches/ccache ccache -s - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/petsc.yml b/.github/workflows/petsc.yml index 7ca42f01415..efde21b89ad 100644 --- a/.github/workflows/petsc.yml +++ b/.github/workflows/petsc.yml @@ -56,18 +56,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml index 2768ef376cc..5f0b1534970 100644 --- a/.github/workflows/post-pr.yml +++ b/.github/workflows/post-pr.yml @@ -4,17 +4,13 @@ on: types: - closed +# This workflow does not have the permission to clean up cache for PRs +# originated from a fork. The purpose here is to trigger a workflow_run +# cleanup-cache-postpr.yml that has the right permission. + jobs: - cleanup: + noop: runs-on: ubuntu-latest steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 + - name: No OP + run: echo "This workflow is going to trigger CleanUpCachePostPR." diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 84e67d9aaee..650ef144e89 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -63,18 +63,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/sundials.yml b/.github/workflows/sundials.yml index a44b21b8d0e..e17234ba28a 100644 --- a/.github/workflows/sundials.yml +++ b/.github/workflows/sundials.yml @@ -133,18 +133,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e0e7bf69118..4dd9a3f391b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -136,17 +136,3 @@ jobs: cmake --build build --config Release --target test_install # If we add ccache back, don't forget to update cleanup-cache.yml - #save_pr_number: - # if: github.event_name == 'pull_request' - # runs-on: ubuntu-latest - # steps: - # - name: Save PR number - # env: - # PR_NUMBER: ${{ github.event.number }} - # run: | - # echo $PR_NUMBER > pr_number.txt - # - uses: actions/upload-artifact@v3 - # with: - # name: pr_number - # path: pr_number.txt - # retention-days: 1