GitHub cache and ccache (#188)

Use ccache in CIs. Set up GitHub Cache for storing the ccache data.

.github/workflows/ci.yml

@@ -33,3 +33,18 @@ jobs:
     name: 🔄 Update Stub Files
     needs: [ubuntu, intel, hip, macos, windows]
     uses: ./.github/workflows/stubs.yml
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v3
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1

.github/workflows/cleanup-cache-postpr.yml

@@ -0,0 +1,40 @@
+name: CleanUpCachePostPR
+
+on:
+  workflow_run:
+    workflows: [PostPR]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCachePostPR:
+    name: Clean Up Ccahe Cache Post PR
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Clean up ccahe
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          gh run download ${{ github.event.workflow_run.id }} -n pr_number
+          pr_number=`cat pr_number.txt`
+          BRANCH=refs/pull/${pr_number}/merge
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1)
+          # $keys might contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
+          for k in $keys
+          do
+            gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
+          done
+          unset IFS

.github/workflows/cleanup-cache.yml

@@ -0,0 +1,63 @@
+name: CleanUpCache
+
+on:
+  workflow_run:
+    workflows: [👑 CI]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCache:
+    name: Clean Up Ccahe Cache for ${{ github.event.workflow_run.name }}
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Clean up ccahe
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          # push or pull_request or schedule or ...
+          EVENT=${{ github.event.workflow_run.event }}
+
+          # Triggering workflow run name (e.g., LinuxClang)
+          WORKFLOW_NAME="${{ github.event.workflow_run.name }}"
+
+          if [[ $EVENT == "pull_request" ]]; then
+            gh run download ${{ github.event.workflow_run.id }} -n pr_number
+            pr_number=`cat pr_number.txt`
+            BRANCH=refs/pull/${pr_number}/merge
+          else
+            BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }}
+          fi
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          # In our cache keys, substring after `-git-` is git hash, substring
+          # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`).
+          # The goal is to keep the last used key of each job and delete all others.
+
+          # something like ccache-LinuxClang-
+          keyprefix="ccache-${WORKFLOW_NAME}-"
+
+          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq)
+
+          # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d".
+          # It might also contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
+          for j in $cached_jobs
+          do
+            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2)
+            for k in $old_keys
+            do
+              gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
+            done
+          done
+          unset IFS

.github/workflows/codeql.yml

@@ -44,10 +44,23 @@ jobs:
           python -m pip install --upgrade mpi4py
           python -m pip install --upgrade pytest
 
+          .github/workflows/dependencies/dependencies_ccache.sh
+          sudo ln -s /usr/local/bin/ccache /usr/local/bin/g++
+
+      - name: Set Up Cache
+        if: ${{ matrix.language == 'cpp' }}
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/ccache
+          key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+          restore-keys: |
+               ccache-${{ github.workflow }}-${{ github.job }}-git-
+
       - name: Configure (C++)
         if: ${{ matrix.language == 'cpp' }}
         run: |
-          $CMAKE -S . -B build -DAMReX_SPACEDIM="1;2;3"
+          $CMAKE -S . -B build -DAMReX_SPACEDIM="1;2;3" \
+                 -DCMAKE_CXX_COMPILER="/usr/local/bin/g++"
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2
@@ -62,6 +75,19 @@ jobs:
       - name: Build (C++)
         if: ${{ matrix.language == 'cpp' }}
         run: |
+          export CCACHE_COMPRESS=1
+          export CCACHE_COMPRESSLEVEL=10
+          export CCACHE_MAXSIZE=400M
+          ccache -z
+
+          $CMAKE --build build -j 2
+
+          ccache -s
+          du -hs ~/.cache/ccache
+
+          # Make sure CodeQL has something to do
+          touch build/_deps/fetchedamrex-src/Src/Base/AMReX.cpp
+          export CCACHE_DISABLE=1
           $CMAKE --build build -j 2
 
       - name: Perform CodeQL Analysis

.github/workflows/dependencies/dependencies_ccache.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+if [[ $# -eq 2 ]]; then
+  CVER=$1
+else
+  CVER=4.8
+fi
+
+wget https://github.com/ccache/ccache/releases/download/v${CVER}/ccache-${CVER}-linux-x86_64.tar.xz
+tar xvf ccache-${CVER}-linux-x86_64.tar.xz
+sudo mv -f ccache-${CVER}-linux-x86_64/ccache /usr/local/bin/
+sudo rm -rf ccache-${CVER}-linux-x86_64
+sudo rm -f ccache-${CVER}-linux-x86_64.tar.xz

.github/workflows/dependencies/dependencies_mac.sh

@@ -11,3 +11,4 @@ brew update
 brew install gfortran || true
 brew install libomp || true
 brew install open-mpi || true
+brew install ccache || true

.github/workflows/hip.yml

@@ -18,10 +18,24 @@ jobs:
     - uses: actions/checkout@v3
     - name: install dependencies
       shell: bash
-      run: .github/workflows/dependencies/hip.sh
+      run: |
+        .github/workflows/dependencies/hip.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: build ImpactX
       shell: bash
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=300M
+        ccache -z
+
         source /etc/profile.d/rocm.sh
         hipcc --version
         which clang
@@ -43,3 +57,6 @@ jobs:
             -DAMReX_AMD_ARCH=gfx900      \
             -DAMReX_SPACEDIM="1;2;3"
         cmake --build build --target pip_install -j 2
+
+        ccache -s
+        du -hs ~/.cache/ccache

.github/workflows/intel.yml

@@ -15,6 +15,14 @@ jobs:
     - name: Dependencies
       run: |
         .github/workflows/dependencies/dpcpp.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has a number of sign-compare error
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has missing braces in array-array initalization
@@ -24,6 +32,12 @@ jobs:
         source /opt/intel/oneapi/setvars.sh
         set -e
 
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=200M
+        export CCACHE_DEPEND=1
+        ccache -z
+
         export CC=$(which icx)
         export CXX=$(which icpx)
         python3 -m pip install -U pip setuptools wheel
@@ -39,6 +53,9 @@ jobs:
               -DAMReX_SPACEDIM="3"
         cmake --build build --target pip_install -j 2
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
   tests-icpx:
     name: ICPX
     runs-on: ubuntu-20.04
@@ -47,6 +64,14 @@ jobs:
     - name: Dependencies
       run: |
         .github/workflows/dependencies/dpcpp.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has a number of sign-compare error
       # mkl/rng/device/detail/mrg32k3a_impl.hpp has missing braces in array-array initalization
@@ -56,6 +81,11 @@ jobs:
         source /opt/intel/oneapi/setvars.sh
         set -e
 
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=200M
+        ccache -z
+
         export CC=$(which icx)
         export CXX=$(which icpx)
         python3 -m pip install -U pip setuptools wheel
@@ -69,6 +99,9 @@ jobs:
               -DAMReX_SPACEDIM="1;2;3"
         cmake --build build --target pip_install -j 2
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     - name: Run tests
       run: |
         set +e
@@ -95,13 +128,26 @@ jobs:
         echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
         sudo apt-get update
         sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-mpi-devel
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: build
       env: {CXXFLAGS: "-Werror"}
       run: |
         set +e
         source /opt/intel/oneapi/setvars.sh
         set -e
 
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=600M
+        ccache -z
+
         export CXX=$(which icpc)
         export CC=$(which icc)
         python3 -m pip install -U pip setuptools wheel
@@ -116,6 +162,9 @@ jobs:
               -DAMReX_SPACEDIM="1;2;3"
         cmake --build build --target pip_install -j 2
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     - name: Run tests
       run: |
         set +e

.github/workflows/macos.yml

@@ -18,8 +18,20 @@ jobs:
     - uses: actions/checkout@v3
     - name: Dependencies
       run: .github/workflows/dependencies/dependencies_mac.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: /Users/runner/Library/Caches/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=600M
+        ccache -z
+
         export CMAKE_BUILD_PARALLEL_LEVEL=3
 
         python3 -m pip install -U pip setuptools wheel pytest
@@ -28,6 +40,10 @@ jobs:
         python3 -c "import amrex.space1d as amr; print(amr.__version__)"
         python3 -c "import amrex.space2d as amr; print(amr.__version__)"
         python3 -c "import amrex.space3d as amr; print(amr.__version__)"
+
+        ccache -s
+        du -hs /Users/runner/Library/Caches/ccache
+
     - name: Unit tests
       run: |
         python3 -m pytest tests/

.github/workflows/post-pr.yml

@@ -0,0 +1,20 @@
+name: PostPR
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v3
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1