Merge branch 'development' of github.com:hjjvandam/ExaEpi into plot_t…

…ime_series

.github/workflows/dependencies/dependencies_ccache.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+if [[ $# -eq 2 ]]; then
+  CVER=$1
+else
+  CVER=4.8
+fi
+
+wget https://github.com/ccache/ccache/releases/download/v${CVER}/ccache-${CVER}-linux-x86_64.tar.xz
+tar xvf ccache-${CVER}-linux-x86_64.tar.xz
+sudo cp -f ccache-${CVER}-linux-x86_64/ccache /usr/local/bin/

.github/workflows/dependencies/dependencies_hip.sh

@@ -12,11 +12,15 @@
 
 set -eu -o pipefail
 
+# `man apt.conf`:
+#   Number of retries to perform. If this is non-zero APT will retry
+#   failed files the given number of times.
+echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
 
 # Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu
 curl -O https://repo.radeon.com/rocm/rocm.gpg.key
 sudo apt-key add rocm.gpg.key
-echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \
+echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-debian}/ ubuntu main" \
   | sudo tee /etc/apt/sources.list.d/rocm.list
 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
   | sudo tee -a /etc/profile.d/rocm.sh
@@ -41,16 +45,14 @@ sudo apt-get install -y --no-install-recommends \
     rocrand-dev     \
     rocprim-dev
 
+# hiprand-dev is a new package that does not exist in old versions
+sudo apt-get install -y --no-install-recommends hiprand-dev || true
+
 # activate
 #
 source /etc/profile.d/rocm.sh
 hipcc --version
+hipconfig --full
 which clang
 which clang++
 which flang
-
-# cmake-easyinstall
-#
-sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY
-sudo chmod a+x /usr/local/bin/cmake-easyinstall
-export CEI_SUDO="sudo"

.github/workflows/hip.yml

@@ -7,32 +7,45 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-hip:
-    name: HIP ROCm C++17
+  tests-hip:
+    name: HIP ROCm C++17 [tests]
     runs-on: ubuntu-20.04
-    # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
-    # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
-    #    amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments]
-    #        __launch_bounds__(amrex_launch_bounds_max_threads)
-    #        ^
-    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__'
-    #        select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
-    #                                                                          ^
-    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here
-    #    #define select_impl_(_1, _2, impl_, ...) impl_
-    # Have to have -Wno-unused-command-line-argument to avoid
-    #    clang-15: error: argument unused during compilation: '-Xoffload-linker --whole-archive' [-Werror,-Wunused-command-line-argument]
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments -Wno-unused-command-line-argument"}
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
-      run: .github/workflows/dependencies/dependencies_hip.sh
+      run: |
+        .github/workflows/dependencies/dependencies_hip.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
+      # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
+      # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
+      #    amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments]
+      #        __launch_bounds__(amrex_launch_bounds_max_threads)
+      #        ^
+      #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__'
+      #        select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
+      #                                                                          ^
+      #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here
+      #    #define select_impl_(_1, _2, impl_, ...) impl_
+      env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments"}
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=100M
+        ccache -z
+
         source /etc/profile.d/rocm.sh
         hipcc --version
         which clang
         which clang++
+        which flang
 
         # "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
         #   https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
@@ -50,25 +63,39 @@ jobs:
         cmake --build build -j 2
 
   tests-hip-wrapper:
-    name: HIP ROCm [email protected] C++17
+    name: HIP ROCm [email protected] C++17 [tests-hipcc]
     runs-on: ubuntu-20.04
-    # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
-    # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
-    #    amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments]
-    #        __launch_bounds__(amrex_launch_bounds_max_threads)
-    #        ^
-    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__'
-    #        select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
-    #                                                                          ^
-    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here
-    #    #define select_impl_(_1, _2, impl_, ...) impl_
-    env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments"}
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
-      run: .github/workflows/dependencies/dependencies_hip.sh
+      run: |
+        .github/workflows/dependencies/dependencies_hip.sh
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
     - name: Build & Install
+      # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
+      # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
+      #    amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments]
+      #        __launch_bounds__(amrex_launch_bounds_max_threads)
+      #        ^
+      #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__'
+      #        select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
+      #                                                                          ^
+      #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here
+      #    #define select_impl_(_1, _2, impl_, ...) impl_
+      env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments"}
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=40M
+        ccache -z
+
         source /etc/profile.d/rocm.sh
         hipcc --version
 

.gitignore

@@ -0,0 +1,6 @@
+docs/source/_static
+docs/build
+docs/doxyhtml
+docs/doxyxml
+docs/exaepi-doxygen-web.tag.xml
+docs/amrex-doxygen-web.tag.xml

docs/Doxyfile

@@ -44,7 +44,7 @@ PROJECT_NUMBER         =
 # for a project that appears at the top of each page and should give viewer a
 # quick idea about the purpose of the project. Keep the description short.
 
-PROJECT_BRIEF          =
+PROJECT_BRIEF          = "Exascale Agent-Based Code for Epidemic Simulations"
 
 # With the PROJECT_LOGO tag one can specify a logo or an icon that is included
 # in the documentation. The maximum height of the logo should not exceed 55
@@ -963,13 +963,13 @@ USE_MDFILE_AS_MAINPAGE =
 # also VERBATIM_HEADERS is set to NO.
 # The default value is: NO.
 
-SOURCE_BROWSER         = NO
+SOURCE_BROWSER         = YES
 
 # Setting the INLINE_SOURCES tag to YES will include the body of functions,
 # classes and enums directly into the documentation.
 # The default value is: NO.
 
-INLINE_SOURCES         = NO
+INLINE_SOURCES         = YES
 
 # Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
 # special comment blocks from generated source code fragments. Normal C, C++ and

docs/source/install/cmake.rst

@@ -57,4 +57,4 @@ or by adding arguments with ``-D<OPTION>=<VALUE>`` to the first CMake call, e.g.
 
 .. code-block:: bash
 
-   cmake -S . -B build -DAMReX_COMPUTE=CUDA
+   cmake -S . -B build -DAMReX_GPU_BACKEND=CUDA

docs/source/usage/how_to_run.rst

@@ -28,14 +28,6 @@ the runtime parameters than can be set in the inputs file.
 
 In addition to the ExaEpi inputs, there are also a number of runtime options that can be configured for AMReX itself. Please see XXX for more information on these options.
 
-* ``agents.size`` (`tuple of 2 integers`: e.g. ``(1, 1)``)
-    This option is deprecated and will removed in a future version of ExaEpi. It controls
-    the number of cells in the domain when running in `demo` mode. During actual usage,
-    this number will be overridden and is irrelevant.
-* ``agent.max_grid_size`` (`integer`)
-    This option sets the maximum grid size used for MPI domain decomposition. If set to
-    ``16``, for example, the domain will be broken up into grids of `16^2` communities, and
-    these grids will be assigned to different MPI ranks / GPUs.
 * ``agent.ic_type`` (`string`: either ``"census"`` or ``"demo"``)
     If ``"census"``, initial conditions will be read from the provided census data file.
     If ``"demo"``, agents will be initialized according to a power law distribution.
@@ -99,6 +91,15 @@ In addition to the ExaEpi inputs, there are also a number of runtime options tha
     Length of the infectious period in days. This counter starts once the incubation phase is over. Before tihs, agents are symptomatic and can spread the disease.
 * ``disease.symptomatic_length`` (`int`, default: ``5``)
     Length of the symptomatic-but-not-infectious stage in days. This counter starts once the infectious phase is complete. During this time agents are symptomatic and may self-withdraw, but they cannot spread the illness.
+* ``agents.size`` (`tuple of 2 integers`: e.g. ``(1, 1)``, default: ``(1, 1)``)
+    This option is deprecated and will removed in a future version of ExaEpi. It controls
+    the number of cells in the domain when running in `demo` mode. During actual usage,
+    this number will be overridden and is irrelevant.
+* ``agent.max_grid_size`` (`integer`, default: ``16``)
+    This option sets the maximum grid size used for MPI domain decomposition. If set to
+    ``16``, for example, the domain will be broken up into grids of `16^2` communities, and
+    these grids will be assigned to different MPI ranks / GPUs.
+
 
 In addition to the ExaEpi inputs, there are also a number of runtime options that can be configured for AMReX itself. Please see <https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters>`__ for more information on these options.
 

etc/lassen_environment.sh

@@ -0,0 +1,9 @@
+module load gcc/12.2.1 cuda/12.2.2 cmake/3.23.1
+export MPICH_GPU_SUPPORT_ENABLED=1
+export CRAY_ACCEL_TARGET=nvidia70
+export AMREX_CUDA_ARCH=7.0
+export CC=gcc
+export CXX=g++
+export FC=gfortran
+export CUDACXX=$(which nvcc)
+export CUDAHOSTCXX=g++

etc/plot.py

@@ -14,5 +14,5 @@
 print(ad["immune"].sum())
 print(ad["previously_infected"].sum())
 
-plt.pcolormesh(ad["total"].reshape(3000, 3000))
+plt.pcolormesh(ad["total"].reshape(ds.domain_dimensions[0:2]))
 plt.savefig("test")

examples/inputs.bay

@@ -0,0 +1,23 @@
+agent.ic_type = "census"
+agent.census_filename = "../../data/CensusData/BayArea.dat"
+agent.workerflow_filename = "../../data/CensusData/BayArea-wf.bin"
+agent.case_filename = "../../data/CaseData/July4.cases"
+
+agent.nsteps = 10
+agent.plot_int = -1
+agent.random_travel_int = -1
+
+agent.aggregated_diag_int = -1
+
+contact.pSC  = 0.2
+contact.pCO  = 1.45
+contact.pNH  = 1.45
+contact.pWO  = 0.5
+contact.pFA  = 1.0
+contact.pBAR = -1.
+
+disease.nstrain = 2
+disease.p_trans = 0.20 0.30
+disease.p_asymp = 0.40 0.40
+disease.reduced_inf = 0.75 0.75
+disease.reinfect_prob = 0.0

examples/inputs → examples/inputs.ca

@@ -1,6 +1,3 @@
-agent.size = (1, 1)
-agent.max_grid_size = 16
-
 agent.ic_type = "census"
 agent.census_filename = "../../data/CensusData/CA.dat"
 agent.workerflow_filename = "../../data/CensusData/CA-wf.bin"