Simplify cache cleanup actions (#4121)

In our old approach, a workflow file contains a job that uploads the PR number as an artifact. While the PR is still open, the workflow_run triggered by it will download the artifact and use the information to clean up all except the last used cache associated with that original workflow. When a PR is merged or closed, there will be a post-pr workflow that uploads the PR number as an artifact and triggers a workflow_run that clean up all caches associated with the PR. The reason we did it this way was in the cache cleanup workflows, we did not find an easy way to get the number of the PR triggering them. This is not convenient because we have to add jobs uploading artifacts to workflow files. After some experiments, we have found a reliable way to find the PR number without using artifacts. The workflow_run's payload always contains the head SHA of the commit that triggers it, whether the PR comes from a fork or not. We can then use `gh pr list` to search for that head and obtain the PR number.
AMReX-Codes · Aug 30, 2024 · d06504a · d06504a
1 parent 7c2ef81
commit d06504a
Show file tree

Hide file tree

Showing 17 changed files with 18 additions and 224 deletions.
diff --git a/.github/workflows/apps.yml b/.github/workflows/apps.yml
@@ -152,18 +152,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/bittree.yml b/.github/workflows/bittree.yml
@@ -110,18 +110,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/clang.yml b/.github/workflows/clang.yml
@@ -212,18 +212,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml
@@ -23,8 +23,11 @@ jobs:
 
           REPO=${{ github.repository }}
 
-          gh run download ${{ github.event.workflow_run.id }} -n pr_number
-          pr_number=`cat pr_number.txt`
+          # For debugging cat ${GITHUB_EVENT_PATH} to see the payload.
+
+          pr_head_sha=${{ github.event.workflow_run.head_sha }}
+          pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number')
+          echo "Post-PR cache cleanup for PR ${pr_number}"
           BRANCH=refs/pull/${pr_number}/merge
 
           # Setting this to not fail the workflow while deleting cache keys.

diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml
@@ -29,9 +29,12 @@ jobs:
           # Triggering workflow run name (e.g., LinuxClang)
           WORKFLOW_NAME="${{ github.event.workflow_run.name }}"
 
+          # For debugging, cat ${GITHUB_EVENT_PATH} to see the payload.
+
           if [[ $EVENT == "pull_request" ]]; then
-            gh run download ${{ github.event.workflow_run.id }} -n pr_number
-            pr_number=`cat pr_number.txt`
+            pr_head_sha=${{ github.event.workflow_run.head_sha }}
+            pr_number=$(gh pr list --search $pr_head_sha --json number --jq '.[0].number')
+            echo "Clean up cache for PR ${pr_number}"
             BRANCH=refs/pull/${pr_number}/merge
           else
             BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }}
@@ -54,6 +57,7 @@ jobs:
           IFS=$'\n'
           for j in $cached_jobs
           do
+            # Delete all entries except the last used one
             old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2)
             for k in $old_keys
             do

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -92,18 +92,3 @@ jobs:
         uses: github/codeql-action/analyze@v3
         with:
           category: "/language:${{ matrix.language }}"
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -205,18 +205,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml
@@ -664,18 +664,3 @@ jobs:
         mpirun -np 2 ./main3d.gnu.TPROF.MPI.ex ./inputs
         h5dump -d "level_0/data:offsets=0"  -s "1" -c "1" ./plt00000.h5
         h5dump -d "level_0/data:datatype=1" -s "1" -c "1" ./plt00000/particle0/particle0.h5
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml
@@ -181,18 +181,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/hypre.yml b/.github/workflows/hypre.yml
@@ -154,18 +154,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
@@ -249,18 +249,3 @@ jobs:
         set -e
         cd build
         ctest --output-on-failure
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -91,18 +91,3 @@ jobs:
 
         du -hs ~/Library/Caches/ccache
         ccache -s
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/petsc.yml b/.github/workflows/petsc.yml
@@ -56,18 +56,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml
@@ -4,17 +4,13 @@ on:
     types:
       - closed
 
+# This workflow does not have the permission to clean up cache for PRs
+# originated from a fork. The purpose here is to trigger a workflow_run
+# cleanup-cache-postpr.yml that has the right permission.
+
 jobs:
-  cleanup:
+  noop:
     runs-on: ubuntu-latest
     steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
+      - name: No OP
+        run: echo "This workflow is going to trigger CleanUpCachePostPR."
diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml
@@ -63,18 +63,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/sundials.yml b/.github/workflows/sundials.yml
@@ -133,18 +133,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -136,17 +136,3 @@ jobs:
         cmake --build build --config Release --target test_install
 
   # If we add ccache back, don't forget to update cleanup-cache.yml
-  #save_pr_number:
-  #  if: github.event_name == 'pull_request'
-  #  runs-on: ubuntu-latest
-  #  steps:
-  #    - name: Save PR number
-  #      env:
-  #        PR_NUMBER: ${{ github.event.number }}
-  #      run: |
-  #        echo $PR_NUMBER > pr_number.txt
-  #    - uses: actions/upload-artifact@v3
-  #      with:
-  #        name: pr_number
-  #        path: pr_number.txt
-  #        retention-days: 1