-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #213 from WeiqunZhang/ccache
Update CI
- Loading branch information
Showing
13 changed files
with
347 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| name: CleanUpCachePostPR | ||
|
|
||
| on: | ||
| workflow_run: | ||
| workflows: [PostPR] | ||
| types: | ||
| - completed | ||
|
|
||
| jobs: | ||
| CleanUpCcacheCachePostPR: | ||
| name: Clean Up Ccache Cache Post PR | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| actions: write | ||
| contents: read | ||
| env: | ||
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Clean up ccache | ||
| run: | | ||
| gh extension install actions/gh-actions-cache | ||
| REPO=${{ github.repository }} | ||
| gh run download ${{ github.event.workflow_run.id }} -n pr_number | ||
| pr_number=`cat pr_number.txt` | ||
| BRANCH=refs/pull/${pr_number}/merge | ||
| # Setting this to not fail the workflow while deleting cache keys. | ||
| set +e | ||
| keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) | ||
| # $keys might contain spaces. Thus we set IFS to \n. | ||
| IFS=$'\n' | ||
| for k in $keys | ||
| do | ||
| gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm | ||
| done | ||
| unset IFS |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| name: CleanUpCache | ||
|
|
||
| on: | ||
| workflow_run: | ||
| workflows: [REMORA CI, REMORA CUDA CI, REMORA CI (hip), REMORA CI (sycl), Linux GCC, MacOS] | ||
| types: | ||
| - completed | ||
|
|
||
| jobs: | ||
| CleanUpCcacheCache: | ||
| name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| actions: write | ||
| contents: read | ||
| env: | ||
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Clean up ccache | ||
| run: | | ||
| gh extension install actions/gh-actions-cache | ||
| REPO=${{ github.repository }} | ||
| # push or pull_request or schedule or ... | ||
| EVENT=${{ github.event.workflow_run.event }} | ||
| # Triggering workflow run name (e.g., LinuxClang) | ||
| WORKFLOW_NAME="${{ github.event.workflow_run.name }}" | ||
| if [[ $EVENT == "pull_request" ]]; then | ||
| gh run download ${{ github.event.workflow_run.id }} -n pr_number | ||
| pr_number=`cat pr_number.txt` | ||
| BRANCH=refs/pull/${pr_number}/merge | ||
| else | ||
| BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} | ||
| fi | ||
| # Setting this to not fail the workflow while deleting cache keys. | ||
| set +e | ||
| # In our cache keys, substring after `-git-` is git hash, substring | ||
| # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`). | ||
| # The goal is to keep the last used key of each job and delete all others. | ||
| # something like ccache-LinuxClang- | ||
| keyprefix="ccache-${WORKFLOW_NAME}-" | ||
| cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) | ||
| # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". | ||
| # It might also contain spaces. Thus we set IFS to \n. | ||
| IFS=$'\n' | ||
| for j in $cached_jobs | ||
| do | ||
| old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) | ||
| for k in $old_keys | ||
| do | ||
| gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm | ||
| done | ||
| done | ||
| unset IFS |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,21 +23,55 @@ jobs: | |
| uses: styfle/[email protected] | ||
| with: | ||
| access_token: ${{github.token}} | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| submodules: true | ||
| - name: Prepare CUDA environment | ||
| run: Submodules/AMReX/.github/workflows/dependencies/dependencies_nvcc.sh ${{matrix.cuda_ver}} | ||
| - name: Install CCache | ||
| run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh | ||
| - name: Set Up Cache | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: ~/.cache/ccache | ||
| key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} | ||
| restore-keys: | | ||
| ccache-${{ github.workflow }}-${{ github.job }}-git- | ||
| - name: Configure and build | ||
| run: | | ||
| export CCACHE_COMPRESS=1 | ||
| export CCACHE_COMPRESSLEVEL=10 | ||
| export CCACHE_MAXSIZE=600M | ||
| ccache -z | ||
| export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} | ||
| export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} | ||
| which nvcc || echo "nvcc not in PATH!" | ||
| cmake -Bbuild-${{matrix.cuda_pkg}} \ | ||
| -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ | ||
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | ||
| -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \ | ||
| -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ | ||
| -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON \ | ||
| -DREMORA_DIM:STRING=3 \ | ||
| -DREMORA_ENABLE_MPI:BOOL=OFF \ | ||
| -DREMORA_ENABLE_CUDA:BOOL=ON . | ||
| cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc) | ||
| ccache -s | ||
| du -hs ~/.cache/ccache | ||
| save_pr_number: | ||
| if: github.event_name == 'pull_request' | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Save PR number | ||
| env: | ||
| PR_NUMBER: ${{ github.event.number }} | ||
| run: | | ||
| echo $PR_NUMBER > pr_number.txt | ||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: pr_number | ||
| path: pr_number.txt | ||
| retention-days: 1 | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.