GitHub cache and ccache

Use ccache in CIs. Set up GitHub Cache for storing the ccache data.

.github/workflows/ci.yml

@@ -33,3 +33,18 @@ jobs:
     name: 🔄 Update Stub Files
     needs: [ubuntu, intel, hip, macos, windows]
     uses: ./.github/workflows/stubs.yml
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v3
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1

.github/workflows/cleanup-cache-postpr.yml

@@ -0,0 +1,37 @@
+name: CleanUpCachePostPR
+
+on:
+  workflow_run:
+    workflows: [PostPR]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCachePostPR:
+    name: Clean Up Ccahe Cache Post PR
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Clean up ccahe
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          gh run download ${{ github.event.workflow_run.id }} -n pr_number
+          pr_number=`cat pr_number.txt`
+          BRANCH=refs/pull/${pr_number}/merge
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1)
+          for k in "$keys"
+          do
+            gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
+          done

.github/workflows/cleanup-cache.yml

@@ -0,0 +1,60 @@
+name: CleanUpCache
+
+on:
+  workflow_run:
+    workflows: ["👑 CI"]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCache:
+    name: Clean Up Ccahe Cache for ${{ github.event.workflow_run.name }}
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Clean up ccahe
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          # push or pull_request or schedule or ...
+          EVENT=${{ github.event.workflow_run.event }}
+
+          # Triggering workflow run name (e.g., LinuxClang)
+          WORKFLOW_NAME=${{ github.event.workflow_run.name }}
+
+          if [[ $EVENT == "pull_request" ]]; then
+            gh run download ${{ github.event.workflow_run.id }} -n pr_number
+            pr_number=`cat pr_number.txt`
+            BRANCH=refs/pull/${pr_number}/merge
+          else
+            BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }}
+          fi
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          # In our cache keys, substring after `-git-` is git hash, substring
+          # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`).
+          # The goal is to keep the last used key of each job and delete all others.
+
+          # something like ccache-LinuxClang-
+          keyprefix=ccache-${WORKFLOW_NAME}-
+
+          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq)
+
+          # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d".
+          for j in "$cached_jobs"
+          do
+            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2)
+            for k in "$old_keys"
+            do
+              gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
+            done
+          done

.github/workflows/dependencies/dependencies_ccache.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+if [[ $# -eq 2 ]]; then
+  CVER=$1
+else
+  CVER=4.8
+fi
+
+wget https://github.com/ccache/ccache/releases/download/v${CVER}/ccache-${CVER}-linux-x86_64.tar.xz
+tar xvf ccache-${CVER}-linux-x86_64.tar.xz
+sudo cp -f ccache-${CVER}-linux-x86_64/ccache /usr/local/bin/

.github/workflows/dependencies/dependencies_mac.sh

@@ -11,3 +11,4 @@ brew update
 brew install gfortran || true
 brew install libomp || true
 brew install open-mpi || true
+brew install ccache || true

.github/workflows/hip.yml

@@ -18,10 +18,24 @@ jobs:
     - uses: actions/checkout@v3
     - name: install dependencies
       shell: bash
-      run: .github/workflows/dependencies/hip.sh
+      run: |
+        .github/workflows/dependencies/hip.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: build ImpactX
       shell: bash
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=300M
+        ccache -z
+
         source /etc/profile.d/rocm.sh
         hipcc --version
         which clang
@@ -43,3 +57,6 @@ jobs:
             -DAMReX_AMD_ARCH=gfx900      \
             -DAMReX_SPACEDIM="1;2;3"
         cmake --build build --target pip_install -j 2
+
+        ccache -s
+        du -hs ~/.cache/ccache

.github/workflows/intel.yml

@@ -15,6 +15,14 @@ jobs:
     - name: Dependencies
       run: |
         .github/workflows/dependencies/dpcpp.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has a number of sign-compare error
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has missing braces in array-array initalization
@@ -24,6 +32,12 @@ jobs:
         source /opt/intel/oneapi/setvars.sh
         set -e
 
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=200M
+        export CCACHE_DEPEND=1
+        ccache -z
+
         export CC=$(which icx)
         export CXX=$(which icpx)
         python3 -m pip install -U pip setuptools wheel
@@ -39,6 +53,9 @@ jobs:
               -DAMReX_SPACEDIM="3"
         cmake --build build --target pip_install -j 2
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
   tests-icpx:
     name: ICPX
     runs-on: ubuntu-20.04
@@ -47,6 +64,14 @@ jobs:
     - name: Dependencies
       run: |
         .github/workflows/dependencies/dpcpp.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has a number of sign-compare error
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has missing braces in array-array initalization
@@ -56,6 +81,11 @@ jobs:
         source /opt/intel/oneapi/setvars.sh
         set -e
 
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=200M
+        ccache -z
+
         export CC=$(which icx)
         export CXX=$(which icpx)
         python3 -m pip install -U pip setuptools wheel
@@ -69,6 +99,9 @@ jobs:
               -DAMReX_SPACEDIM="1;2;3"
         cmake --build build --target pip_install -j 2
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     - name: Run tests
       run: |
         set +e
@@ -95,13 +128,26 @@ jobs:
         echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
         sudo apt-get update
         sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-mpi-devel
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: build
       env: {CXXFLAGS: "-Werror"}
       run: |
         set +e
         source /opt/intel/oneapi/setvars.sh
         set -e
 
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=600M
+        ccache -z
+
         export CXX=$(which icpc)
         export CC=$(which icc)
         python3 -m pip install -U pip setuptools wheel
@@ -116,6 +162,9 @@ jobs:
               -DAMReX_SPACEDIM="1;2;3"
         cmake --build build --target pip_install -j 2
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     - name: Run tests
       run: |
         set +e

.github/workflows/macos.yml

@@ -18,8 +18,20 @@ jobs:
     - uses: actions/checkout@v3
     - name: Dependencies
       run: .github/workflows/dependencies/dependencies_mac.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: /Users/runner/Library/Caches/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=600M
+        ccache -z
+
         export CMAKE_BUILD_PARALLEL_LEVEL=3
 
         python3 -m pip install -U pip setuptools wheel pytest
@@ -28,6 +40,10 @@ jobs:
         python3 -c "import amrex.space1d as amr; print(amr.__version__)"
         python3 -c "import amrex.space2d as amr; print(amr.__version__)"
         python3 -c "import amrex.space3d as amr; print(amr.__version__)"
+
+        ccache -s
+        du -hs /Users/runner/Library/Caches/ccache
+
     - name: Unit tests
       run: |
         python3 -m pytest tests/

.github/workflows/post-pr.yml

@@ -0,0 +1,20 @@
+name: PostPR
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v3
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1

.github/workflows/stubs.yml

@@ -40,15 +40,33 @@ jobs:
         python-version: '3.9'
 
     - name: Dependencies
-      run: .github/workflows/dependencies/dependencies_gcc10.sh
+      run: |
+        .github/workflows/dependencies/dependencies_gcc10.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
 
     - name: Build & Install
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=200M
+        ccache -z
+
         python3 -m pip install -U pip setuptools wheel
         python3 -m pip install -U pip mpi4py pytest pybind11-stubgen pre-commit
         cmake -S . -B build -DAMReX_SPACEDIM="1;2;3" -DpyAMReX_IPO=OFF
         cmake --build build -j 2 --target pip_install
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     - name: Update Stubs
       run: |
         .github/update_stub.sh