From 0de6c5306b666b0c0630ca1d808b6ac31252be15 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Mon, 6 Feb 2023 11:14:19 -0800 Subject: [PATCH 001/187] update radiuss-spack-config submodule --- .gitlab/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 8194d152ac..6b6532b6cb 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 8194d152acfdfcc8ad5a27051e9988f1e20e8779 +Subproject commit 6b6532b6cbf4684fac4da097b211cad7dbd142f7 From 84629d0555c9701fb178fd055f01c72bbafe7182 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Mon, 6 Feb 2023 11:19:29 -0800 Subject: [PATCH 002/187] update uberenv submodule --- .gitlab/uberenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/uberenv b/.gitlab/uberenv index 4941c237ee..0d00dc8e19 160000 --- a/.gitlab/uberenv +++ b/.gitlab/uberenv @@ -1 +1 @@ -Subproject commit 4941c237eec514d6d68872243efb9f4af8843f4d +Subproject commit 0d00dc8e19a889ba07ae433590b87533c4b5b3da From 6377fc6bc70598159a2b1cd029070637113d1e4c Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Mon, 6 Feb 2023 11:50:15 -0800 Subject: [PATCH 003/187] update spack to v0.19.0 --- .uberenv_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.uberenv_config.json b/.uberenv_config.json index 7c65ba8c6f..a71c61d34f 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,8 +4,8 @@ "package_final_phase": "initconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", - "spack_commit": "13e6f87ef6527954b152eaea303841978e83b992", + "spack_commit": "bb8b4f9979e87ae3fb63a3adcb768eda09e5b059", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", "spack_packages_path": ".gitlab/spack_packages" -} \ No newline at end of file +} From 5a5a55415eaad07146fe356d25e05dda98325a8b Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Mon, 6 Feb 2023 12:06:02 -0800 Subject: [PATCH 004/187] run spack clean --- .gitlab/build_and_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 51d5bc63bf..a7173440d4 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -94,6 +94,8 @@ then upstream_opt="--upstream=${upstream}" fi + spack clean -m + if [[ "${shared_spack}" == "UPSTREAM" ]] then python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" From f2a88451a96fbe223014285baa9d797928b99e4c Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Mon, 6 Feb 2023 12:13:50 -0800 Subject: [PATCH 005/187] Revert "run spack clean" This reverts commit 5a5a55415eaad07146fe356d25e05dda98325a8b. --- .gitlab/build_and_test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index a7173440d4..51d5bc63bf 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -94,8 +94,6 @@ then upstream_opt="--upstream=${upstream}" fi - spack clean -m - if [[ "${shared_spack}" == "UPSTREAM" ]] then python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" From 6dd2a251e3314d2c535407dcd74ab1012ae9a3dc Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 07:54:10 -0800 Subject: [PATCH 006/187] build pipeline with clang 13.0.0 --- .gitlab-ci.yml | 7 ++++--- .gitlab/build_and_test.sh | 16 ++++++++++++++-- .gitlab/corona-jobs.yml | 26 ++++++++++++++++++++++++++ .gitlab/corona-templates.yml | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 .gitlab/corona-jobs.yml create mode 100644 .gitlab/corona-templates.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b6c763515..99a711392f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,6 +75,7 @@ variables: VERBOSE_TEST: "OFF" ON_LASSEN: "ON" ON_QUARTZ: "ON" + ON_CORONA: "ON" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 @@ -89,7 +90,7 @@ stages: - q_build_and_test - l_build_and_test - l_build_and_bench - # - c_build_and_test + - c_build_and_test # These are also templates (.name) that define project specific build commands. # If an allocation exist with the name defined in this pipeline, the job will @@ -101,9 +102,9 @@ stages: --job-name=${ALLOC_NAME} .gitlab/build_and_test.sh # Corona -.build_toss_3_x86_64_ib_corona_script: +.build_toss_4_x86_64_ib_corona_script: script: - - srun -p mi60 --interactive -t ${DEFAULT_TIME} -N 1 .gitlab/build_and_test.sh + - salloc -t ${DEFAULT_TIME} -N 1 .gitlab/build_and_test.sh # CORAL systems use spectrum LSF instead of SLURM .build_blueos_3_ppc64le_ib_script: diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 51d5bc63bf..18ec9e1622 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -46,8 +46,20 @@ hostname=${hostname%%[0-9]*} BUILD_JOBS=${BUILD_JOBS:-"1"} # load newer python to try the clingo concretizer -echo "module load python/3.8.2" -module load python/3.8.2 +# Corona does not have python 3.8.2 +if [[ -n "${AMDGPU_TARGET}" ]]; then + echo "module load python/3.9.12" + module load python/3.9.12 +else + echo "module load python/3.8.2" + module load python/3.8.2 +fi + +# Rocm version specific to Corona. +if [[ -n "${AMDGPU_TARGET}" ]]; then + echo "module load rocm/5.1.1" + module load rocm/5.1.1 +fi if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] then diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml new file mode 100644 index 0000000000..9157163fef --- /dev/null +++ b/.gitlab/corona-jobs.yml @@ -0,0 +1,26 @@ +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2002-2021, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# HIP +# ------------------------------------------------------------------------------ + +# Builds with Hip +corona_clang_hip: + parallel: + matrix: + - COMPILER_SPEC: clang@13.0.0 + AMDGPU_TARGET: gfx906 + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm" + extends: .corona_build_and_test + diff --git a/.gitlab/corona-templates.yml b/.gitlab/corona-templates.yml new file mode 100644 index 0000000000..5413ab8e49 --- /dev/null +++ b/.gitlab/corona-templates.yml @@ -0,0 +1,35 @@ +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2002-2021, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# Tags and rules to run tests on Lassen +# ------------------------------------------------------------------------------ + +# Generic corona build job, extending build script for IBM P9 systems +.corona_build_and_test: + tags: + - shell + - lassen + extends: [.build_toss_4_x86_64_ib_corona_script] + stage: l_build_and_test + needs: [] + artifacts: + paths: + - spack-*.txt + - build_*/* + when: always + rules: + # Don't run if... + - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_CORONA == "OFF" || $BENCHMARK == "ON"' + when: never + # Default is to run if previous stage succeeded + - when: on_success \ No newline at end of file From cf4e427b4021c6d0eeba3d8a4a4d95a7c04cd7eb Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 08:10:53 -0800 Subject: [PATCH 007/187] add corona files to gitlab-ci --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99a711392f..3734b55ce4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,3 +124,5 @@ include: - local: .gitlab/quartz-jobs.yml - local: .gitlab/lassen-templates.yml - local: .gitlab/lassen-jobs.yml + - local: .gitlab/corona-templates.yml + - local: .gitlab/corona-jobs.yml From 00f0931d47641142926d53ab3aeb8e7a96b25f79 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 08:36:44 -0800 Subject: [PATCH 008/187] add corona to proper stage --- .gitlab-ci.yml | 1 + .gitlab/corona-templates.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3734b55ce4..063fd6b8d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,6 +104,7 @@ stages: # Corona .build_toss_4_x86_64_ib_corona_script: script: + - echo ${ALLOC_NAME} - salloc -t ${DEFAULT_TIME} -N 1 .gitlab/build_and_test.sh # CORAL systems use spectrum LSF instead of SLURM diff --git a/.gitlab/corona-templates.yml b/.gitlab/corona-templates.yml index 5413ab8e49..20c52bc0cb 100644 --- a/.gitlab/corona-templates.yml +++ b/.gitlab/corona-templates.yml @@ -20,7 +20,7 @@ - shell - lassen extends: [.build_toss_4_x86_64_ib_corona_script] - stage: l_build_and_test + stage: c_build_and_test needs: [] artifacts: paths: From 4e29127c376a4f9b17360e76fb1a940a240e29c8 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 08:54:23 -0800 Subject: [PATCH 009/187] fix corona template --- .gitlab-ci.yml | 1 - .gitlab/corona-templates.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 063fd6b8d9..510aedb5a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,7 +75,6 @@ variables: VERBOSE_TEST: "OFF" ON_LASSEN: "ON" ON_QUARTZ: "ON" - ON_CORONA: "ON" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 diff --git a/.gitlab/corona-templates.yml b/.gitlab/corona-templates.yml index 20c52bc0cb..73ab216dc6 100644 --- a/.gitlab/corona-templates.yml +++ b/.gitlab/corona-templates.yml @@ -18,7 +18,7 @@ .corona_build_and_test: tags: - shell - - lassen + - corona extends: [.build_toss_4_x86_64_ib_corona_script] stage: c_build_and_test needs: [] From 9c417e80388675b0561adc3967aced9eff3f0610 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 09:34:05 -0800 Subject: [PATCH 010/187] build sundials without mpi --- .gitlab/corona-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 9157163fef..d0516b9dc9 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -21,6 +21,6 @@ corona_clang_hip: - COMPILER_SPEC: clang@13.0.0 AMDGPU_TARGET: gfx906 variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" extends: .corona_build_and_test From f021bead5da06eec50072bef9d3c60dffde89e7e Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 09:52:38 -0800 Subject: [PATCH 011/187] add brackets around amdgpu_target --- .gitlab/corona-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index d0516b9dc9..b0d497e1d1 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -19,7 +19,7 @@ corona_clang_hip: parallel: matrix: - COMPILER_SPEC: clang@13.0.0 - AMDGPU_TARGET: gfx906 + AMDGPU_TARGET: [gfx906] variables: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" extends: .corona_build_and_test From 3e8a5a51deff7bce23fd996827af7fc36270dbef Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 10:11:01 -0800 Subject: [PATCH 012/187] add target feature support to AMDGPU_TARGET --- .gitlab/corona-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index b0d497e1d1..b8b08c75c8 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -19,7 +19,7 @@ corona_clang_hip: parallel: matrix: - COMPILER_SPEC: clang@13.0.0 - AMDGPU_TARGET: [gfx906] + AMDGPU_TARGET: [gfx906:sramexx-:xnack-] variables: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" extends: .corona_build_and_test From c3e55e3a63fbb7c27298ad331e928eb9df1d6725 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 10:40:18 -0800 Subject: [PATCH 013/187] turn on verbose build --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 510aedb5a6..964a5f271c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,7 +71,7 @@ variables: DEFAULT_TIME: 30 BUILD_JOBS: 32 NCPUS: 12 - VERBOSE_BUILD: "OFF" + VERBOSE_BUILD: "ON" VERBOSE_TEST: "OFF" ON_LASSEN: "ON" ON_QUARTZ: "ON" From dcccb4fd88bb77f94856fe29a0f2951c26d6ed21 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 10:46:42 -0800 Subject: [PATCH 014/187] remove extra amd features --- .gitlab/corona-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index b8b08c75c8..b0d497e1d1 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -19,7 +19,7 @@ corona_clang_hip: parallel: matrix: - COMPILER_SPEC: clang@13.0.0 - AMDGPU_TARGET: [gfx906:sramexx-:xnack-] + AMDGPU_TARGET: [gfx906] variables: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" extends: .corona_build_and_test From 02667cbd783177756f8b0a5d995ff5afc43e77f5 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 11:55:54 -0800 Subject: [PATCH 015/187] remove lassen, quartz jobs. Manually set amdgpu_target --- .gitlab-ci.yml | 5 +++-- .gitlab/corona-jobs.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 964a5f271c..d02cdfaf35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,8 +73,9 @@ variables: NCPUS: 12 VERBOSE_BUILD: "ON" VERBOSE_TEST: "OFF" - ON_LASSEN: "ON" - ON_QUARTZ: "ON" + ON_LASSEN: "OFF" + ON_QUARTZ: "OFF" + ON_CORONA: "ON" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index b0d497e1d1..c5a55f3dfb 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -21,6 +21,6 @@ corona_clang_hip: - COMPILER_SPEC: clang@13.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=gfx906 +rocm~mpi" extends: .corona_build_and_test From d2c1546edc3a86363f2edb6127567325ed6f82d6 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 8 Feb 2023 15:36:42 -0800 Subject: [PATCH 016/187] modify sundials package.py to have amdgpu_target addition mimic cuda_arch --- .gitlab/corona-jobs.yml | 2 +- .gitlab/spack_packages/sundials/package.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index c5a55f3dfb..0c5163464a 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -21,6 +21,6 @@ corona_clang_hip: - COMPILER_SPEC: clang@13.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=gfx906 +rocm~mpi" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 +rocm~mpi amdgpu_target=${AMDGPU_TARGET}" extends: .corona_build_and_test diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 4de41139de..33d3de0f8e 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -666,9 +666,14 @@ def initconfig_hardware_entries(self): cmake_cache_path("HIP_PATH", spec["hip"].prefix), cmake_cache_path("HIP_CLANG_INCLUDE_PATH", spec["llvm-amdgpu"].prefix.include), cmake_cache_path("ROCM_PATH", spec["llvm-amdgpu"].prefix), - cmake_cache_string("AMDGPU_TARGETS", spec.variants["amdgpu_target"].value) + # cmake_cache_string("AMDGPU_TARGETS", spec.variants["amdgpu_target"].value) ] ) + if not spec.satisfies("amdgpu_target=none"): + amdgpu_target = spec.variants["amdgpu_target"].value + entries.append( + cmake_cache_string("AMDGPU_TARGETS", "{0}".format(amdgpu_target[0]) + ) return entries From 221c373a3a60303c6497102e93fa5736c24a6447 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 16:10:13 -0800 Subject: [PATCH 017/187] update uberenv to v1.0.0 --- .gitlab/uberenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/uberenv b/.gitlab/uberenv index 4941c237ee..0d00dc8e19 160000 --- a/.gitlab/uberenv +++ b/.gitlab/uberenv @@ -1 +1 @@ -Subproject commit 4941c237eec514d6d68872243efb9f4af8843f4d +Subproject commit 0d00dc8e19a889ba07ae433590b87533c4b5b3da From bcf709ef4a128c01edd55d61c03be62a344cb265 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 16:27:04 -0800 Subject: [PATCH 018/187] update to latest radiuss-spack-configs --- .gitlab/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 8194d152ac..f916c0069a 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 8194d152acfdfcc8ad5a27051e9988f1e20e8779 +Subproject commit f916c0069ac02d8ad5da77be06b383fe4af06ef9 From 2db41e38b51a3035711d12bccec28527361f828d Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 17:30:36 -0800 Subject: [PATCH 019/187] update spack commit to v0.19.0 --- .gitlab/build_and_test.sh | 48 +++++++++++++++++++-------------------- .uberenv_config.json | 4 ++-- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 51d5bc63bf..f3257b978d 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -61,31 +61,31 @@ then exit 1 fi - prefix_opt="" - - if [[ -d /dev/shm ]] - then - prefix="/dev/shm/${hostname}" - if [[ -z ${job_unique_id} ]]; then - job_unique_id=manual_job_$(date +%s) - while [[ -d ${prefix}/${job_unique_id} ]] ; do - sleep 1 - job_unique_id=manual_job_$(date +%s) - done - fi - - prefix="${prefix}/${job_unique_id}" - mkdir -p ${prefix} - prefix_opt="--prefix=${prefix}" - - # We force Spack to put all generated files (cache and configuration of - # all sorts) in a unique location so that there can be no collision - # with existing or concurrent Spack. - spack_user_cache="${prefix}/spack-user-cache" - export SPACK_DISABLE_LOCAL_CONFIG="" - export SPACK_USER_CACHE_PATH="${spack_user_cache}" - mkdir -p ${spack_user_cache} + prefix="/usr/worksapace/sundials/tmp/" + mkdir -p "${prefix}" + chmod g=u "${prefix}" + prefix="${prefix}/${hostname}" + mkdir -p "${prefix}" + chmod g=u "${prefix}" + if [[ -z ${job_unique_id} ]]; then + job_unique_id=manual_job_$(date +%s) + while [[ -d ${prefix}/${job_unique_id} ]] ; do + sleep 1 + job_unique_id=manual_job_$(date +%s) + done fi + prefix="${prefix}/${job_unique_id}" + mkdir -p "${prefix}" + chmod g=u "${prefix}" + prefix_opt="--prefix=${prefix}" + + # We force Spack to put all generated files (cache and configuration of + # all sorts) in a unique location so that there can be no collision + # with existing or concurrent Spack. + spack_user_cache="${prefix}/spack-user-cache" + export SPACK_DISABLE_LOCAL_CONFIG="" + export SPACK_USER_CACHE_PATH="${spack_user_cache}" + mkdir -p ${spack_user_cache} if [[ -d /usr/workspace/sundials ]] then diff --git a/.uberenv_config.json b/.uberenv_config.json index 7c65ba8c6f..a71c61d34f 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,8 +4,8 @@ "package_final_phase": "initconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", - "spack_commit": "13e6f87ef6527954b152eaea303841978e83b992", + "spack_commit": "bb8b4f9979e87ae3fb63a3adcb768eda09e5b059", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", "spack_packages_path": ".gitlab/spack_packages" -} \ No newline at end of file +} From 4e281ba59ed2dcf45498cdd22f41a25238bebcbe Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 17:45:50 -0800 Subject: [PATCH 020/187] fix typo, update permissions --- .gitlab/build_and_test.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index f3257b978d..db8de02a4f 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -61,7 +61,13 @@ then exit 1 fi - prefix="/usr/worksapace/sundials/tmp/" + # temporary to change permissions + chmod -R g=u /usr/worksapace/sundials/tmp-ci-testing + chmod -R g=u /usr/workspace/sundials/spack_installs + chmod g=u /usr/workspace/sundials/make-a-new-spack.sh + chmod -R g=u /usr/workspace/sundials/sunrepo + + prefix="/usr/workspace/sundials/tmp/" mkdir -p "${prefix}" chmod g=u "${prefix}" prefix="${prefix}/${hostname}" @@ -89,8 +95,10 @@ then if [[ -d /usr/workspace/sundials ]] then - upstream="/usr/workspace/sundials/spack_installs/${hostname}" + upstream="/usr/workspace/sundials/spack_installs/v0.19.0/${hostname}" mkdir -p "${upstream}" + chmod g=u /usr/workspace/sundials/spack_installs/v0.19.0 + chmod g=u ${upstream} upstream_opt="--upstream=${upstream}" fi From 4941ea33e4a542d339224ae0b2e3f036776e9a0b Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 18:06:33 -0800 Subject: [PATCH 021/187] fix typo --- .gitlab/build_and_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index db8de02a4f..5ae2133e64 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -62,7 +62,7 @@ then fi # temporary to change permissions - chmod -R g=u /usr/worksapace/sundials/tmp-ci-testing + chmod -R g=u /usr/workspace/sundials/tmp-ci-testing chmod -R g=u /usr/workspace/sundials/spack_installs chmod g=u /usr/workspace/sundials/make-a-new-spack.sh chmod -R g=u /usr/workspace/sundials/sunrepo From f0dce5fb3c7c458e3b866bf01f25081d8460aea0 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 18:31:05 -0800 Subject: [PATCH 022/187] disable some tests, remove chmod update --- .gitlab-ci.yml | 2 +- .gitlab/build_and_test.sh | 1 - .gitlab/quartz-jobs.yml | 36 ++++++++++++++++++------------------ 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b6c763515..32b101d720 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,7 +73,7 @@ variables: NCPUS: 12 VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" - ON_LASSEN: "ON" + ON_LASSEN: "OFF" ON_QUARTZ: "ON" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 5ae2133e64..c684526c2a 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -62,7 +62,6 @@ then fi # temporary to change permissions - chmod -R g=u /usr/workspace/sundials/tmp-ci-testing chmod -R g=u /usr/workspace/sundials/spack_installs chmod g=u /usr/workspace/sundials/make-a-new-spack.sh chmod -R g=u /usr/workspace/sundials/sunrepo diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 6e4b87a236..10e7bdda19 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -52,15 +52,15 @@ # +petsc ~hypre ~superlu-dist ~int64 ~hdf5 # +superlu-dist == segfaults -quartz_clang_tpls: - parallel: - matrix: - - COMPILER_SPEC: clang@12.0.0 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" - extends: .quartz_build_and_test +# quartz_clang_tpls: +# parallel: +# matrix: +# - COMPILER_SPEC: clang@12.0.0 +# INDEX_SPEC: [~int64] +# PRECISION_SPEC: [double] +# variables: +# SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" +# extends: .quartz_build_and_test quartz_gcc_tpls: parallel: @@ -72,12 +72,12 @@ quartz_gcc_tpls: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" extends: .quartz_build_and_test -quartz_intel_tpls: - parallel: - matrix: - - COMPILER_SPEC: intel@19.0.4 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu" - extends: .quartz_build_and_test +# quartz_intel_tpls: +# parallel: +# matrix: +# - COMPILER_SPEC: intel@19.0.4 +# INDEX_SPEC: [~int64] +# PRECISION_SPEC: [double] +# variables: +# SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu" +# extends: .quartz_build_and_test From 2e755e08dbfc2752e9921d44dcc5ae4ed41ace67 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 18:41:22 -0800 Subject: [PATCH 023/187] remove chmod update --- .gitlab/build_and_test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index c684526c2a..e6681a4210 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -62,7 +62,6 @@ then fi # temporary to change permissions - chmod -R g=u /usr/workspace/sundials/spack_installs chmod g=u /usr/workspace/sundials/make-a-new-spack.sh chmod -R g=u /usr/workspace/sundials/sunrepo From 2be344d90be8ee7c51ffca12447244519df1eda8 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 19:50:05 -0800 Subject: [PATCH 024/187] add spack-debug --- .gitlab/build_and_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index e6681a4210..fc928f4dec 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -102,12 +102,12 @@ then if [[ "${shared_spack}" == "UPSTREAM" ]] then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" + python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" --spack-debug elif [[ "${shared_spack}" == "ON" ]] then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" --prefix="${upstream}" + python3 .gitlab/uberenv/uberenv.py --spec="${spec}" --prefix="${upstream}" --spack-debug else - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" + python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" --spack-debug fi # Ensure correct CUDA module is loaded, only works for module naming From 1e0fd82fc7467c27f450fb70010656cb4fab3d50 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 19:57:57 -0800 Subject: [PATCH 025/187] remove chmod commands --- .gitlab/build_and_test.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index fc928f4dec..8dbef1f377 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -61,10 +61,6 @@ then exit 1 fi - # temporary to change permissions - chmod g=u /usr/workspace/sundials/make-a-new-spack.sh - chmod -R g=u /usr/workspace/sundials/sunrepo - prefix="/usr/workspace/sundials/tmp/" mkdir -p "${prefix}" chmod g=u "${prefix}" From 1eb78dfc4ae96480575f046d3ec1ee6bb980b6bb Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 22:03:28 -0800 Subject: [PATCH 026/187] switch to lassen tests --- .gitlab-ci.yml | 4 ++-- .gitlab/lassen-jobs.yml | 16 ++++++++-------- .gitlab/quartz-jobs.yml | 36 ++++++++++++++++++------------------ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32b101d720..46604788dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,8 +73,8 @@ variables: NCPUS: 12 VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" - ON_LASSEN: "OFF" - ON_QUARTZ: "ON" + ON_LASSEN: "ON" + ON_QUARTZ: "OFF" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 7dc51e99d4..6c2174b9d3 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -72,14 +72,14 @@ lassen_xl_cuda_tpls: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test -lassen_gcc_cuda_tpls: - parallel: - matrix: - - COMPILER_SPEC: gcc@8.3.1 - CUDA_SPEC: [cuda@11.5.0] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" - extends: .lassen_build_and_test +# lassen_gcc_cuda_tpls: +# parallel: +# matrix: +# - COMPILER_SPEC: gcc@8.3.1 +# CUDA_SPEC: [cuda@11.5.0] +# variables: +# SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" +# extends: .lassen_build_and_test # ------------------------------------------------------------------------------ # Benchmark jobs diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 10e7bdda19..6e4b87a236 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -52,15 +52,15 @@ # +petsc ~hypre ~superlu-dist ~int64 ~hdf5 # +superlu-dist == segfaults -# quartz_clang_tpls: -# parallel: -# matrix: -# - COMPILER_SPEC: clang@12.0.0 -# INDEX_SPEC: [~int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" -# extends: .quartz_build_and_test +quartz_clang_tpls: + parallel: + matrix: + - COMPILER_SPEC: clang@12.0.0 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" + extends: .quartz_build_and_test quartz_gcc_tpls: parallel: @@ -72,12 +72,12 @@ quartz_gcc_tpls: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" extends: .quartz_build_and_test -# quartz_intel_tpls: -# parallel: -# matrix: -# - COMPILER_SPEC: intel@19.0.4 -# INDEX_SPEC: [~int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu" -# extends: .quartz_build_and_test +quartz_intel_tpls: + parallel: + matrix: + - COMPILER_SPEC: intel@19.0.4 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu" + extends: .quartz_build_and_test From 21c04b3fb32591ab2357b8806f44cb2eb8e371e4 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 22:14:11 -0800 Subject: [PATCH 027/187] comment out chmods --- .gitlab/build_and_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 8dbef1f377..a1278b4ac8 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -63,7 +63,7 @@ then prefix="/usr/workspace/sundials/tmp/" mkdir -p "${prefix}" - chmod g=u "${prefix}" + #chmod g=u "${prefix}" prefix="${prefix}/${hostname}" mkdir -p "${prefix}" chmod g=u "${prefix}" @@ -91,7 +91,7 @@ then then upstream="/usr/workspace/sundials/spack_installs/v0.19.0/${hostname}" mkdir -p "${upstream}" - chmod g=u /usr/workspace/sundials/spack_installs/v0.19.0 + #chmod g=u /usr/workspace/sundials/spack_installs/v0.19.0 chmod g=u ${upstream} upstream_opt="--upstream=${upstream}" fi From 71708b7ad0514fcc454fa077183f6aaabfdc6dc7 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Wed, 8 Feb 2023 22:33:56 -0800 Subject: [PATCH 028/187] comment out chmods --- .gitlab/build_and_test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index a1278b4ac8..8997af7153 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -61,12 +61,12 @@ then exit 1 fi - prefix="/usr/workspace/sundials/tmp/" + prefix="/usr/workspace/sundials/tmp" mkdir -p "${prefix}" #chmod g=u "${prefix}" prefix="${prefix}/${hostname}" mkdir -p "${prefix}" - chmod g=u "${prefix}" + #chmod g=u "${prefix}" if [[ -z ${job_unique_id} ]]; then job_unique_id=manual_job_$(date +%s) while [[ -d ${prefix}/${job_unique_id} ]] ; do @@ -76,7 +76,7 @@ then fi prefix="${prefix}/${job_unique_id}" mkdir -p "${prefix}" - chmod g=u "${prefix}" + #chmod g=u "${prefix}" prefix_opt="--prefix=${prefix}" # We force Spack to put all generated files (cache and configuration of @@ -92,7 +92,7 @@ then upstream="/usr/workspace/sundials/spack_installs/v0.19.0/${hostname}" mkdir -p "${upstream}" #chmod g=u /usr/workspace/sundials/spack_installs/v0.19.0 - chmod g=u ${upstream} + #chmod g=u ${upstream} upstream_opt="--upstream=${upstream}" fi From cd4343cbedecb53d086b7781938afaa0bc247659 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 9 Feb 2023 06:23:14 -0800 Subject: [PATCH 029/187] get the first value in amdgpu_target --- .gitlab/spack_packages/sundials/package.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 33d3de0f8e..d65f72c0f5 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -666,14 +666,10 @@ def initconfig_hardware_entries(self): cmake_cache_path("HIP_PATH", spec["hip"].prefix), cmake_cache_path("HIP_CLANG_INCLUDE_PATH", spec["llvm-amdgpu"].prefix.include), cmake_cache_path("ROCM_PATH", spec["llvm-amdgpu"].prefix), + cmake_cache_string("AMDGPU_TARGETS", spec.variants["amdgpu_target"].value[0]) # cmake_cache_string("AMDGPU_TARGETS", spec.variants["amdgpu_target"].value) ] ) - if not spec.satisfies("amdgpu_target=none"): - amdgpu_target = spec.variants["amdgpu_target"].value - entries.append( - cmake_cache_string("AMDGPU_TARGETS", "{0}".format(amdgpu_target[0]) - ) return entries From 1b3d3389728b1d542136cadfb47c1f75e1c3b3ec Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 9 Feb 2023 06:50:50 -0800 Subject: [PATCH 030/187] remove commented out code, turn off verbose build --- .gitlab-ci.yml | 2 +- .gitlab/corona-jobs.yml | 2 +- .gitlab/spack_packages/sundials/package.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d02cdfaf35..19bb9fc6aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,7 +71,7 @@ variables: DEFAULT_TIME: 30 BUILD_JOBS: 32 NCPUS: 12 - VERBOSE_BUILD: "ON" + VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" ON_LASSEN: "OFF" ON_QUARTZ: "OFF" diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 0c5163464a..b0d497e1d1 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -21,6 +21,6 @@ corona_clang_hip: - COMPILER_SPEC: clang@13.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 +rocm~mpi amdgpu_target=${AMDGPU_TARGET}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" extends: .corona_build_and_test diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index d65f72c0f5..64b41774ee 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -667,10 +667,8 @@ def initconfig_hardware_entries(self): cmake_cache_path("HIP_CLANG_INCLUDE_PATH", spec["llvm-amdgpu"].prefix.include), cmake_cache_path("ROCM_PATH", spec["llvm-amdgpu"].prefix), cmake_cache_string("AMDGPU_TARGETS", spec.variants["amdgpu_target"].value[0]) - # cmake_cache_string("AMDGPU_TARGETS", spec.variants["amdgpu_target"].value) ] ) - return entries def initconfig_package_entries(self): From 8bd8c5ca3bafd56ee81ff2a00153f3338d313c41 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Thu, 9 Feb 2023 07:14:01 -0800 Subject: [PATCH 031/187] use original concretizer --- .uberenv_config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.uberenv_config.json b/.uberenv_config.json index a71c61d34f..20bcbd88e8 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -5,6 +5,7 @@ "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", "spack_commit": "bb8b4f9979e87ae3fb63a3adcb768eda09e5b059", + "spack_concretizer": "original", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", "spack_packages_path": ".gitlab/spack_packages" From ccdbac0f975235e2c049057b10bb4bc69f76d5c8 Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Thu, 9 Feb 2023 09:28:51 -0800 Subject: [PATCH 032/187] Revert "use original concretizer" This reverts commit 8bd8c5ca3bafd56ee81ff2a00153f3338d313c41. --- .uberenv_config.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.uberenv_config.json b/.uberenv_config.json index 20bcbd88e8..a71c61d34f 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -5,7 +5,6 @@ "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", "spack_commit": "bb8b4f9979e87ae3fb63a3adcb768eda09e5b059", - "spack_concretizer": "original", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", "spack_packages_path": ".gitlab/spack_packages" From 0118048f7142c6fcf7de56dcd5647c5e0a306cbc Mon Sep 17 00:00:00 2001 From: "David J. Gardner" Date: Thu, 9 Feb 2023 09:40:33 -0800 Subject: [PATCH 033/187] use spack v0.18.1 --- .gitlab/radiuss-spack-configs | 2 +- .uberenv_config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index f916c0069a..dc67ed7de3 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit f916c0069ac02d8ad5da77be06b383fe4af06ef9 +Subproject commit dc67ed7de39e08a2d63117b21dec495def0558be diff --git a/.uberenv_config.json b/.uberenv_config.json index a71c61d34f..1e9fc78964 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,7 +4,7 @@ "package_final_phase": "initconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", - "spack_commit": "bb8b4f9979e87ae3fb63a3adcb768eda09e5b059", + "spack_commit": "13e6f87ef6527954b152eaea303841978e83b992", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", "spack_packages_path": ".gitlab/spack_packages" From 421dd623ebe13c186c6370c46fb0425fa39e9d4d Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 28 Feb 2023 11:48:21 -0800 Subject: [PATCH 034/187] swap clang version for 15.0.0 --- .gitlab/build_and_test.sh | 4 ++-- .gitlab/corona-jobs.yml | 2 +- .gitlab/radiuss-spack-configs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 18ec9e1622..c060084b8b 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -57,8 +57,8 @@ fi # Rocm version specific to Corona. if [[ -n "${AMDGPU_TARGET}" ]]; then - echo "module load rocm/5.1.1" - module load rocm/5.1.1 + echo "module load rocm/5.4.1" + module load rocm/5.4.1 fi if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index b0d497e1d1..32bf3cbf2d 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -18,7 +18,7 @@ corona_clang_hip: parallel: matrix: - - COMPILER_SPEC: clang@13.0.0 + - COMPILER_SPEC: clang@15.0.0 AMDGPU_TARGET: [gfx906] variables: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 8194d152ac..ff5d106178 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 8194d152acfdfcc8ad5a27051e9988f1e20e8779 +Subproject commit ff5d10617864635b1e3d7eea9e8aa93c70e66a13 From 83bb72fc87581b03d9a832a3a57a5c7cc1a65550 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 28 Feb 2023 13:24:54 -0800 Subject: [PATCH 035/187] update spack to v0.19.1 (latest release as of this commit) --- .uberenv_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.uberenv_config.json b/.uberenv_config.json index 1e9fc78964..67618cca2f 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,7 +4,7 @@ "package_final_phase": "initconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", - "spack_commit": "13e6f87ef6527954b152eaea303841978e83b992", + "spack_commit": "5e0d2107348eed6cbe6deca43a30f5b06c5e40af", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", "spack_packages_path": ".gitlab/spack_packages" From fcc41ae8526a243438e54f541af95f2818358552 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 28 Feb 2023 13:35:59 -0800 Subject: [PATCH 036/187] re-enable quartz --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46604788dc..1b6c763515 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -74,7 +74,7 @@ variables: VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" ON_LASSEN: "ON" - ON_QUARTZ: "OFF" + ON_QUARTZ: "ON" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 From 8f6428205a9650046f639e965c30dfc86ec13b7c Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 28 Feb 2023 13:45:31 -0800 Subject: [PATCH 037/187] revert build_and_test.sh to develop --- .gitlab/build_and_test.sh | 58 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 8997af7153..51d5bc63bf 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -61,49 +61,47 @@ then exit 1 fi - prefix="/usr/workspace/sundials/tmp" - mkdir -p "${prefix}" - #chmod g=u "${prefix}" - prefix="${prefix}/${hostname}" - mkdir -p "${prefix}" - #chmod g=u "${prefix}" - if [[ -z ${job_unique_id} ]]; then - job_unique_id=manual_job_$(date +%s) - while [[ -d ${prefix}/${job_unique_id} ]] ; do - sleep 1 - job_unique_id=manual_job_$(date +%s) - done + prefix_opt="" + + if [[ -d /dev/shm ]] + then + prefix="/dev/shm/${hostname}" + if [[ -z ${job_unique_id} ]]; then + job_unique_id=manual_job_$(date +%s) + while [[ -d ${prefix}/${job_unique_id} ]] ; do + sleep 1 + job_unique_id=manual_job_$(date +%s) + done + fi + + prefix="${prefix}/${job_unique_id}" + mkdir -p ${prefix} + prefix_opt="--prefix=${prefix}" + + # We force Spack to put all generated files (cache and configuration of + # all sorts) in a unique location so that there can be no collision + # with existing or concurrent Spack. + spack_user_cache="${prefix}/spack-user-cache" + export SPACK_DISABLE_LOCAL_CONFIG="" + export SPACK_USER_CACHE_PATH="${spack_user_cache}" + mkdir -p ${spack_user_cache} fi - prefix="${prefix}/${job_unique_id}" - mkdir -p "${prefix}" - #chmod g=u "${prefix}" - prefix_opt="--prefix=${prefix}" - - # We force Spack to put all generated files (cache and configuration of - # all sorts) in a unique location so that there can be no collision - # with existing or concurrent Spack. - spack_user_cache="${prefix}/spack-user-cache" - export SPACK_DISABLE_LOCAL_CONFIG="" - export SPACK_USER_CACHE_PATH="${spack_user_cache}" - mkdir -p ${spack_user_cache} if [[ -d /usr/workspace/sundials ]] then - upstream="/usr/workspace/sundials/spack_installs/v0.19.0/${hostname}" + upstream="/usr/workspace/sundials/spack_installs/${hostname}" mkdir -p "${upstream}" - #chmod g=u /usr/workspace/sundials/spack_installs/v0.19.0 - #chmod g=u ${upstream} upstream_opt="--upstream=${upstream}" fi if [[ "${shared_spack}" == "UPSTREAM" ]] then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" --spack-debug + python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" elif [[ "${shared_spack}" == "ON" ]] then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" --prefix="${upstream}" --spack-debug + python3 .gitlab/uberenv/uberenv.py --spec="${spec}" --prefix="${upstream}" else - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" --spack-debug + python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" fi # Ensure correct CUDA module is loaded, only works for module naming From f6aa8718157abed8bf40a0b04eec625e40e30a11 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 28 Feb 2023 14:02:15 -0800 Subject: [PATCH 038/187] make it easier to update gitlab ci Spack, document it more clearly --- .gitlab-ci.yml | 3 +++ .gitlab/build_and_test.sh | 5 +++-- doc/sundials_developers/source/testing/CI.rst | 16 ++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b6c763515..e5ce0eb8fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,6 +50,8 @@ # ON_QUARTZ: # Should the Quartz pipeline run? Set to "ON" or "OFF" to enable/disable. # +# SPACK_PREFIX: prefix used for shared spack installation, usually this a spack version number that matches the version set in the uberenv_config.json file. Spack installs go in /usr/workspace/sundials/spack_installs/${SPACK_PREFIX}/$(hostname). +# # SHARED_SPACK: # If "ON", then a shared spack install that has been pre-configured is utilized. # If "OFF", then a new spack instance is created for every build (meaning all TPLs have to be installed). @@ -75,6 +77,7 @@ variables: VERBOSE_TEST: "OFF" ON_LASSEN: "ON" ON_QUARTZ: "ON" + SPACK_PREFIX: "v0.19.1" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 51d5bc63bf..8fa8d66ccc 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -20,6 +20,7 @@ job_unique_id=${CI_JOB_ID:-""} sys_type=${SYS_TYPE:-""} py_env_path=${PYTHON_ENVIRONMENT_PATH:-""} +spack_prefix=${SHARED_SPACK_PREFIX:-"v0.19.1"} shared_spack=${SHARED_SPACK:-"UPSTREAM"} # Dependencies @@ -89,7 +90,7 @@ then if [[ -d /usr/workspace/sundials ]] then - upstream="/usr/workspace/sundials/spack_installs/${hostname}" + upstream="/usr/workspace/sundials/spack_installs/${spack_prefix}/${hostname}" mkdir -p "${upstream}" upstream_opt="--upstream=${upstream}" fi @@ -174,7 +175,7 @@ then mkdir -p "${build_dir}" && cd "${build_dir}" date - + $cmake_exe --version # configure diff --git a/doc/sundials_developers/source/testing/CI.rst b/doc/sundials_developers/source/testing/CI.rst index 48de0c5c4f..1e2470b067 100644 --- a/doc/sundials_developers/source/testing/CI.rst +++ b/doc/sundials_developers/source/testing/CI.rst @@ -275,11 +275,11 @@ Caliper, as we need a newer version than in the Spack commit currently used. Updating Spack -------------- -To update the spack commit used for the CI, the first thing to do is update -the spack commit in the ``.uberenv_config.json`` file. Then, a pipeline -should be manually launched with the ``SHARED_SPACK`` CI variable set -to ``ON``. This will cause Spack to re-concretize the specs and ideally -update to newer packages. However, there is no guarantee that individual -dependencies will be updated, so due dilligence is required (i.e., ensure -that the output from the CI job shows that Spack has selected the versions -of dependencies that you expected). +To update the spack commit used for the CI: + +1. The first thing to do is update the spack commit in the ``.uberenv_config.json`` file. +2. Then, a pipeline should be manually launched from the GitLab UI with the ``SHARED_SPACK`` CI variable set +to ``ON`` and the ``SPACK_PREFIX`` variable to the version of spack being set in the uberenv_config.json. + +This will create a new spack installation and rebuild all of the specs. + From 0c9ac8eb6af878adf1bfd96ee2fefc453e0915a2 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 28 Feb 2023 17:26:13 -0800 Subject: [PATCH 039/187] remove raja, camp --- .gitlab/spack_packages/camp/package.py | 72 --------- .gitlab/spack_packages/raja/package.py | 211 ------------------------- 2 files changed, 283 deletions(-) delete mode 100644 .gitlab/spack_packages/camp/package.py delete mode 100644 .gitlab/spack_packages/raja/package.py diff --git a/.gitlab/spack_packages/camp/package.py b/.gitlab/spack_packages/camp/package.py deleted file mode 100644 index aa801d21dc..0000000000 --- a/.gitlab/spack_packages/camp/package.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class Camp(CMakePackage, CudaPackage, ROCmPackage): - """ - Compiler agnostic metaprogramming library providing concepts, - type operations and tuples for C++ and cuda - """ - - homepage = "https://github.com/LLNL/camp" - git = "https://github.com/LLNL/camp.git" - url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" - - maintainers = ['trws'] - - version('main', branch='main', submodules='True') - version('2022.03.2', sha256='e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721') - version('2022.03.0', sha256='e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721') - version('0.3.0', sha256='129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb') - version('0.2.3', sha256='58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7') - version('0.2.2', sha256='194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819') - version('0.1.0', sha256='fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc') - - # TODO: figure out gtest dependency and then set this default True. - variant('tests', default=False, description='Build tests') - - depends_on('cub', when='+cuda') - - depends_on('blt') - - def cmake_args(self): - spec = self.spec - - options = [] - - options.append("-DBLT_SOURCE_DIR={0}".format(spec['blt'].prefix)) - - if '+cuda' in spec: - options.extend([ - '-DENABLE_CUDA=ON', - '-DCUDA_TOOLKIT_ROOT_DIR=%s' % (spec['cuda'].prefix)]) - - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - options.append('-DCUDA_ARCH=sm_{0}'.format(cuda_arch[0])) - flag = '-arch sm_{0}'.format(cuda_arch[0]) - options.append('-DCMAKE_CUDA_FLAGS:STRING={0}'.format(flag)) - else: - options.append('-DENABLE_CUDA=OFF') - - if '+rocm' in spec: - options.extend([ - '-DENABLE_HIP=ON', - '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix) - ]) - archs = self.spec.variants['amdgpu_target'].value - if archs != 'none': - arch_str = ",".join(archs) - options.append( - '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str) - ) - else: - options.append('-DENABLE_HIP=OFF') - - options.append(self.define_from_variant('ENABLE_TESTS', 'tests')) - - return options \ No newline at end of file diff --git a/.gitlab/spack_packages/raja/package.py b/.gitlab/spack_packages/raja/package.py deleted file mode 100644 index db49267ed1..0000000000 --- a/.gitlab/spack_packages/raja/package.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import socket - -from spack import * - - -class Raja(CachedCMakePackage, CudaPackage, ROCmPackage): - """RAJA Parallel Framework.""" - - homepage = "https://software.llnl.gov/RAJA/" - git = "https://github.com/LLNL/RAJA.git" - tags = ['radiuss', 'e4s'] - - maintainers = ['davidbeckingsale'] - - version('develop', branch='develop', submodules=False) - version('main', branch='main', submodules=False) - version('2022.03.1', tag='v2022.03.0', submodules=False) - version('2022.03.0', tag='v2022.03.0', submodules=False) - version('0.14.0', tag='v0.14.0', submodules='True') - version('0.13.0', tag='v0.13.0', submodules='True') - version('0.12.1', tag='v0.12.1', submodules="True") - version('0.12.0', tag='v0.12.0', submodules="True") - version('0.11.0', tag='v0.11.0', submodules="True") - version('0.10.1', tag='v0.10.1', submodules="True") - version('0.10.0', tag='v0.10.0', submodules="True") - version('0.9.0', tag='v0.9.0', submodules="True") - version('0.8.0', tag='v0.8.0', submodules="True") - version('0.7.0', tag='v0.7.0', submodules="True") - version('0.6.0', tag='v0.6.0', submodules="True") - version('0.5.3', tag='v0.5.3', submodules="True") - version('0.5.2', tag='v0.5.2', submodules="True") - version('0.5.1', tag='v0.5.1', submodules="True") - version('0.5.0', tag='v0.5.0', submodules="True") - version('0.4.1', tag='v0.4.1', submodules="True") - version('0.4.0', tag='v0.4.0', submodules="True") - - # export targets when building pre-2.4.0 release with BLT 0.4.0+ - patch('https://github.com/LLNL/RAJA/commit/eca1124ee4af380d6613adc6012c307d1fd4176b.patch?full_index=1', - sha256='12bb78c00b6683ad3e7fd4e3f87f9776bae074b722431b79696bc862816735ef', - when='@:0.13.0 ^blt@0.4:') - - variant('openmp', default=True, description='Build OpenMP backend') - variant('shared', default=True, description='Build Shared Libs') - variant('examples', default=True, description='Build examples.') - variant('exercises', default=True, description='Build exercises.') - # TODO: figure out gtest dependency and then set this default True - # and remove the +tests conflict below. - variant('tests', default=False, description='Build tests') - - depends_on('blt') - depends_on('blt@0.5.0:', type='build', when='@0.14.1:') - depends_on('blt@0.4.1', type='build', when='@0.14.0') - depends_on('blt@0.4.0:', type='build', when='@0.13.0') - depends_on('blt@0.3.6:', type='build', when='@:0.12.0') - - depends_on('camp@0.2.2', when='@0.14.0') - depends_on('camp@0.1.0', when='@0.10.0:0.13.0') - depends_on('camp@2022.03.0:', when='@2022.03.0:') - - depends_on('cmake@:3.20', when='+rocm', type='build') - depends_on('cmake@3.14:', when='@2022.03.0:') - - with when('+rocm @0.12.0:'): - depends_on('camp+rocm') - for arch in ROCmPackage.amdgpu_targets: - depends_on('camp+rocm amdgpu_target={0}'.format(arch), - when='amdgpu_target={0}'.format(arch)) - conflicts('+openmp') - - with when('+cuda @0.12.0:'): - depends_on('camp+cuda') - for sm_ in CudaPackage.cuda_arch_values: - depends_on('camp +cuda cuda_arch={0}'.format(sm_), - when='cuda_arch={0}'.format(sm_)) - - def _get_sys_type(self, spec): - sys_type = spec.architecture - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - @property - def cache_name(self): - hostname = socket.gethostname() - if "SYS_TYPE" in env: - hostname = hostname.rstrip('1234567890') - return "{0}-{1}-{2}@{3}.cmake".format( - hostname, - self._get_sys_type(self.spec), - self.spec.compiler.name, - self.spec.compiler.version - ) - - def initconfig_hardware_entries(self): - spec = self.spec - entries = super(Raja, self).initconfig_hardware_entries() - - entries.append(cmake_cache_option("ENABLE_OPENMP", '+openmp' in spec)) - - if '+cuda' in spec: - entries.append(cmake_cache_option("ENABLE_CUDA", True)) - - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - entries.append(cmake_cache_string( - "CUDA_ARCH", 'sm_{0}'.format(cuda_arch[0]))) - entries.append(cmake_cache_string( - "CMAKE_CUDA_ARCHITECTURES", '{0}'.format(cuda_arch[0]))) - else: - entries.append(cmake_cache_option("ENABLE_CUDA", False)) - - if '+rocm' in spec: - entries.append(cmake_cache_option("ENABLE_HIP", True)) - entries.append(cmake_cache_path( - "HIP_ROOT_DIR", '{0}'.format(spec['hip'].prefix))) - archs = self.spec.variants['amdgpu_target'].value - if archs != 'none': - arch_str = ",".join(archs) - entries.append(cmake_cache_string( - "HIP_HIPCC_FLAGS", '--amdgpu-target={0}'.format(arch_str))) - else: - entries.append(cmake_cache_option("ENABLE_HIP", False)) - - return entries - - def initconfig_package_entries(self): - spec = self.spec - entries = [] - - option_prefix = "RAJA_" if spec.satisfies("@2022.03.0:") else "" - - entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec['blt'].prefix)) - if 'camp' in self.spec: - entries.append(cmake_cache_path("camp_DIR", spec['camp'].prefix)) - entries.append(cmake_cache_option("BUILD_SHARED_LIBS", '+shared' in spec)) - entries.append(cmake_cache_option( - "{}ENABLE_EXAMPLES".format(option_prefix), '+examples' in spec)) - if spec.satisfies('@0.14.0:'): - entries.append(cmake_cache_option( - "{}ENABLE_EXERCISES".format(option_prefix), '+exercises' in spec)) - else: - entries.append(cmake_cache_option("ENABLE_EXERCISES", - '+exercises' in spec)) - - # Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS which - # is used by the spack compiler wrapper. This can go away when BLT - # removes -Werror from GTest flags - if self.spec.satisfies('%clang target=ppc64le:') or not self.run_tests: - entries.append(cmake_cache_option("ENABLE_TESTS", False)) - else: - entries.append(cmake_cache_option("ENABLE_TESTS", True)) - - return entries - - def cmake_args(self): - options = [] - return options - - @property - def build_relpath(self): - """Relative path to the cmake build subdirectory.""" - return join_path('..', self.build_dirname) - - @run_after('install') - def setup_build_tests(self): - """Copy the build test files after the package is installed to a - relative install test subdirectory for use during `spack test run`.""" - # Now copy the relative files - self.cache_extra_test_sources(self.build_relpath) - - # Ensure the path exists since relying on a relative path at the - # same level as the normal stage source path. - mkdirp(self.install_test_root) - - @property - def _extra_tests_path(self): - # TODO: The tests should be converted to re-build and run examples - # TODO: using the installed libraries. - return join_path(self.install_test_root, self.build_relpath, 'bin') - - def _test_examples(self): - """Perform very basic checks on a subset of copied examples.""" - checks = [ - ('ex5_line-of-sight_solution', - [r'RAJA sequential', r'RAJA OpenMP', r'result -- PASS']), - ('ex6_stencil-offset-layout_solution', - [r'RAJA Views \(permuted\)', r'result -- PASS']), - ('ex8_tiled-matrix-transpose_solution', - [r'parallel top inner loop', - r'collapsed inner loops', r'result -- PASS']), - ('kernel-dynamic-tile', [r'Running index', r'(24,24)']), - ('plugin-example', - [r'Launching host kernel for the 10 time']), - ('tut_batched-matrix-multiply', [r'result -- PASS']), - ('wave-eqn', [r'Max Error = 2', r'Evolved solution to time']) - ] - for exe, expected in checks: - reason = 'test: checking output of {0} for {1}' \ - .format(exe, expected) - self.run_test(exe, [], expected, installed=False, - purpose=reason, skip_missing=True, - work_dir=self._extra_tests_path) - - def test(self): - """Perform smoke tests.""" - self._test_examples() \ No newline at end of file From 5d4bcc00f07216aa41c8689d8c309cd0295b1f65 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Wed, 1 Mar 2023 16:34:46 -0800 Subject: [PATCH 040/187] specify suite-sparse version due to spack bug --- .gitlab/quartz-jobs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 6e4b87a236..d526de02ce 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -59,7 +59,7 @@ quartz_clang_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu ^suite-sparse@5.13.0" extends: .quartz_build_and_test quartz_gcc_tpls: @@ -69,7 +69,7 @@ quartz_gcc_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu ^suite-sparse@5.13.0" extends: .quartz_build_and_test quartz_intel_tpls: @@ -79,5 +79,5 @@ quartz_intel_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^suite-sparse@5.13.0" extends: .quartz_build_and_test From 954da46268c28accc8d07b99069ef68cdad04b38 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Wed, 1 Mar 2023 17:20:54 -0800 Subject: [PATCH 041/187] need clang 12.0.1 on quartz --- .gitlab/quartz-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index d526de02ce..40ee07de45 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -55,7 +55,7 @@ quartz_clang_tpls: parallel: matrix: - - COMPILER_SPEC: clang@12.0.0 + - COMPILER_SPEC: clang@12.0.1 INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: From 2c7480d8e20e9f8d56ee6d0b144e1f1c10fa6594 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Wed, 1 Mar 2023 20:27:53 -0800 Subject: [PATCH 042/187] need netlib-lapack from develop due to bug --- .../netlib-lapack/ibm-xl-3.9.1.patch | 108 +++++++++ .../spack_packages/netlib-lapack/ibm-xl.patch | 53 +++++ .../spack_packages/netlib-lapack/package.py | 223 ++++++++++++++++++ .../netlib-lapack/testing.patch | 13 + .../undefined_declarations.patch | 26 ++ 5 files changed, 423 insertions(+) create mode 100644 .gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch create mode 100644 .gitlab/spack_packages/netlib-lapack/ibm-xl.patch create mode 100644 .gitlab/spack_packages/netlib-lapack/package.py create mode 100644 .gitlab/spack_packages/netlib-lapack/testing.patch create mode 100644 .gitlab/spack_packages/netlib-lapack/undefined_declarations.patch diff --git a/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch b/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch new file mode 100644 index 0000000000..c9e1707857 --- /dev/null +++ b/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch @@ -0,0 +1,108 @@ +Fixes for IBM XL and Cray CCE builds: + +* Correct path to the fallback configuration used to handle mangling for + C++/Fortran compatibility (CCE, XL) + +* Change logic for detecting recursive fortran flags to (a) Include XL +(qrecur), and (b) Be explicit, since not every compiler will correctly reject +an incorrect option (ALL) + +NOTE: This patch has been accepted upstream +(see https://github.com/Reference-LAPACK/lapack/pull/621) + +############################################################################## + +diff -Naur a/CBLAS/CMakeLists.txt b/CBLAS/CMakeLists.txt +--- a/CBLAS/CMakeLists.txt 2021-03-25 12:25:15.000000000 -0600 ++++ b/CBLAS/CMakeLists.txt 2021-09-01 16:27:23.561355382 -0600 +@@ -11,9 +11,7 @@ + MACRO_NAMESPACE "F77_" + SYMBOL_NAMESPACE "F77_") + if(NOT FortranCInterface_GLOBAL_FOUND OR NOT FortranCInterface_MODULE_FOUND) +- message(WARNING "Reverting to pre-defined include/lapacke_mangling.h") +- configure_file(include/lapacke_mangling_with_flags.h.in +- ${LAPACK_BINARY_DIR}/include/lapacke_mangling.h) ++ message(WARNING "Reverting to pre-defined include/cblas_mangling.h") + configure_file(include/cblas_mangling_with_flags.h.in + ${LAPACK_BINARY_DIR}/include/cblas_mangling.h) + endif() +diff -Naur a/CMakeLists.txt b/CMakeLists.txt +--- a/CMakeLists.txt 2021-03-25 12:25:15.000000000 -0600 ++++ b/CMakeLists.txt 2021-09-02 09:49:18.070436958 -0600 +@@ -94,16 +94,22 @@ + + # Check if recursive flag exists + include(CheckFortranCompilerFlag) +-check_fortran_compiler_flag("-recursive" _recursiveFlag) +-check_fortran_compiler_flag("-frecursive" _frecursiveFlag) +-check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag) ++if(CMAKE_Fortran_COMPILER_ID STREQUAL Flang) ++ check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag) ++elseif(CMAKE_Fortran_COMPILER_ID STREQUAL GNU) ++ check_fortran_compiler_flag("-frecursive" _frecursiveFlag) ++elseif(CMAKE_Fortran_COMPILER_ID STREQUAL Intel) ++ check_fortran_compiler_flag("-recursive" _recursiveFlag) ++elseif(CMAKE_Fortran_COMPILER_ID STREQUAL XL) ++ check_fortran_compiler_flag("-qrecur" _qrecurFlag) ++endif() + + # Add recursive flag +-if(_recursiveFlag) +- string(REGEX MATCH "-recursive" output_test "${CMAKE_Fortran_FLAGS}") ++if(_MrecursiveFlag) ++ string(REGEX MATCH "-Mrecursive" output_test "${CMAKE_Fortran_FLAGS}") + if(NOT output_test) +- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive" +- CACHE STRING "Recursive flag must be set" FORCE) ++ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive" ++ CACHE STRING "Recursive flag must be set" FORCE) + endif() + elseif(_frecursiveFlag) + string(REGEX MATCH "-frecursive" output_test "${CMAKE_Fortran_FLAGS}") +@@ -111,11 +117,17 @@ + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -frecursive" + CACHE STRING "Recursive flag must be set" FORCE) + endif() +-elseif(_MrecursiveFlag) +- string(REGEX MATCH "-Mrecursive" output_test "${CMAKE_Fortran_FLAGS}") ++elseif(_recursiveFlag) ++ string(REGEX MATCH "-recursive" output_test "${CMAKE_Fortran_FLAGS}") + if(NOT output_test) +- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive" +- CACHE STRING "Recursive flag must be set" FORCE) ++ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive" ++ CACHE STRING "Recursive flag must be set" FORCE) ++ endif() ++elseif(_qrecurFlag) ++ string(REGEX MATCH "-qrecur" output_test "${CMAKE_Fortran_FLAGS}") ++ if(NOT output_test) ++ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qrecur" ++ CACHE STRING "Recursive flag must be set" FORCE) + endif() + endif() + +@@ -124,7 +136,7 @@ + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fp-model strict") + endif() + if(CMAKE_Fortran_COMPILER_ID STREQUAL XL) +- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict=none") ++ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict") + endif() + # Delete libmtsk in linking sequence for Sun/Oracle Fortran Compiler. + # This library is not present in the Sun package SolarisStudio12.3-linux-x86-bin +diff -Naur a/INSTALL/make.inc.XLF b/INSTALL/make.inc.XLF +--- a/INSTALL/make.inc.XLF 2021-03-25 12:25:15.000000000 -0600 ++++ b/INSTALL/make.inc.XLF 2021-09-02 09:50:02.664646455 -0600 +@@ -14,10 +14,10 @@ + # the compiler options desired when NO OPTIMIZATION is selected. + # + FC = xlf +-FFLAGS = -O3 -qfixed -qnosave ++FFLAGS = -O3 -qfixed -qnosave -qrecur + # For -O2, add -qstrict=none + FFLAGS_DRV = $(FFLAGS) +-FFLAGS_NOOPT = -O0 -qfixed -qnosave ++FFLAGS_NOOPT = -O0 -qfixed -qnosave -qrecur + + # Define LDFLAGS to the desired linker options for your machine. + # diff --git a/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch b/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch new file mode 100644 index 0000000000..52b5f19719 --- /dev/null +++ b/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch @@ -0,0 +1,53 @@ +Fixes for IBM XL and Cray CCE builds: + +* Avoid optimizations that would alter program semantics by changing the + qstrict activation threshold from O3 to O2 (XL) + +* Don't assume Fortran code is all in fixed source form; disable qfixed (XL) + +* Correct path to the fallback configuration used to handle mangling for + C++/Fortran compatibility (CCE, XL) +############################################################################## + +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -62,7 +62,7 @@ + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fp-model strict") + endif() + if("${CMAKE_Fortran_COMPILER}" MATCHES "xlf") +- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict=none") ++ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict") + endif() + # Delete libmtsk in linking sequence for Sun/Oracle Fortran Compiler. + # This library is not present in the Sun package SolarisStudio12.3-linux-x86-bin + +--- a/CMAKE/CheckLAPACKCompilerFlags.cmake ++++ b/CMAKE/CheckLAPACKCompilerFlags.cmake +@@ -43,12 +43,6 @@ + if( "${CMAKE_Fortran_FLAGS}" MATCHES "-qflttrap=[a-zA-Z:]:enable" ) + set( FPE_EXIT TRUE ) + endif() +- +- if( NOT ("${CMAKE_Fortran_FLAGS}" MATCHES "-qfixed") ) +- message( STATUS "Enabling fixed format F90/F95 with -qfixed" ) +- set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qfixed" +- CACHE STRING "Flags for Fortran compiler." FORCE ) +- endif() + + # HP Fortran + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "HP" ) + +--- a/CBLAS/CMakeLists.txt ++++ b/CBLAS/CMakeLists.txt +@@ -12,8 +12,8 @@ + SYMBOL_NAMESPACE "F77_") + if(NOT FortranCInterface_GLOBAL_FOUND OR NOT FortranCInterface_MODULE_FOUND) + message(WARNING "Reverting to pre-defined include/lapacke_mangling.h") +- configure_file(include/lapacke_mangling_with_flags.h.in +- ${LAPACK_BINARY_DIR}/include/lapacke_mangling.h) ++ configure_file(include/cblas_mangling_with_flags.h.in ++ ${LAPACK_BINARY_DIR}/include/cblas_mangling.h) + endif() + + include_directories(include ${LAPACK_BINARY_DIR}/include) + diff --git a/.gitlab/spack_packages/netlib-lapack/package.py b/.gitlab/spack_packages/netlib-lapack/package.py new file mode 100644 index 0000000000..cb1832d061 --- /dev/null +++ b/.gitlab/spack_packages/netlib-lapack/package.py @@ -0,0 +1,223 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import spack.build_systems.cmake +from spack.package import * + + +class NetlibLapack(CMakePackage): + """LAPACK version 3.X is a comprehensive FORTRAN library that does + linear algebra operations including matrix inversions, least squared + solutions to linear sets of equations, eigenvector analysis, singular + value decomposition, etc. It is a very comprehensive and reputable + package that has found extensive use in the scientific community. + + """ + + homepage = "https://www.netlib.org/lapack/" + url = "https://www.netlib.org/lapack/lapack-3.5.0.tgz" + tags = ["windows"] + + version( + "3.10.1", + sha256="cd005cd021f144d7d5f7f33c943942db9f03a28d110d6a3b80d718a295f7f714", + url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.1.tar.gz", + ) + version( + "3.10.0", + sha256="328c1bea493a32cac5257d84157dc686cc3ab0b004e2bea22044e0a59f6f8a19", + url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.0.tar.gz", + ) + version( + "3.9.1", + sha256="d0085d2caf997ff39299c05d4bacb6f3d27001d25a4cc613d48c1f352b73e7e0", + url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.9.1.tar.gz", + ) + version( + "3.9.0", + sha256="106087f1bb5f46afdfba7f569d0cbe23dacb9a07cd24733765a0e89dbe1ad573", + url="https://github.com/Reference-LAPACK/lapack/archive/v3.9.0.tar.gz", + ) + version( + "3.8.0", + sha256="deb22cc4a6120bff72621155a9917f485f96ef8319ac074a7afbc68aab88bcf6", + url="https://www.netlib.org/lapack/lapack-3.8.0.tar.gz", + ) + version("3.7.1", sha256="f6c53fd9f56932f3ddb3d5e24c1c07e4cd9b3b08e7f89de9c867125eecc9a1c8") + version("3.7.0", sha256="ed967e4307e986474ab02eb810eed1d1adc73f5e1e3bc78fb009f6fe766db3be") + version("3.6.1", sha256="888a50d787a9d828074db581c80b2d22bdb91435a673b1bf6cd6eb51aa50d1de") + version("3.6.0", sha256="a9a0082c918fe14e377bbd570057616768dca76cbdc713457d8199aaa233ffc3") + version("3.5.0", sha256="9ad8f0d3f3fb5521db49f2dd716463b8fb2b6bc9dc386a9956b8c6144f726352") + version("3.4.2", sha256="60a65daaf16ec315034675942618a2230521ea7adf85eea788ee54841072faf0") + version("3.4.1", sha256="93b910f94f6091a2e71b59809c4db4a14655db527cfc5821ade2e8c8ab75380f") + version("3.4.0", sha256="a7139ef97004d0e3c4c30f1c52d508fd7ae84b5fbaf0dd8e792c167dc306c3e9") + version("3.3.1", sha256="56821ab51c29369a34e5085728f92c549a9aa926f26acf7eeac87b61eed329e4") + + # netlib-lapack is the reference implementation of LAPACK + for ver in [ + "3.10.1", + "3.10.0", + "3.9.1", + "3.9.0", + "3.8.0", + "3.7.1", + "3.7.0", + "3.6.1", + "3.6.0", + "3.5.0", + "3.4.2", + "3.4.1", + "3.4.0", + "3.3.1", + ]: + provides("lapack@" + ver, when="@" + ver) + + variant("shared", default=True, description="Build shared library version") + variant("external-blas", default=False, description="Build lapack with an external blas") + + variant("lapacke", default=True, description="Activates the build of the LAPACKE C interface") + variant("xblas", default=False, description="Builds extended precision routines using XBLAS") + + # Fixes for IBM XL and Cray CCE builds: + # Avoid optimizations that alter program semantics + # Don't assume fixed source form for Fortran + # Correct path to mangling config + patch("ibm-xl.patch", when="@3.7:3.8 %xl") + patch("ibm-xl.patch", when="@3.7:3.8 %xl_r") + patch("ibm-xl.patch", when="@3.7:3.8 %cce@9:") + + # https://github.com/Reference-LAPACK/lapack/pull/621 + # Fixes for IBM XL and Cray CCE builds: + # Correct path to mangling config + # Fix logic for detecting recursive Fortran flags + patch("ibm-xl-3.9.1.patch", when="@3.9.1 %xl") + patch("ibm-xl-3.9.1.patch", when="@3.9.1 %xl_r") + patch("ibm-xl-3.9.1.patch", when="@3.9.1 %cce@13:") + + # https://github.com/Reference-LAPACK/lapack/issues/228 + patch("undefined_declarations.patch", when="@3.8.0:3.8") + + # https://github.com/Reference-LAPACK/lapack/pull/268 + patch("testing.patch", when="@3.7.0:3.8") + + # virtual dependency + provides("blas", when="~external-blas") + provides("lapack") + + depends_on("blas", when="+external-blas") + depends_on("netlib-xblas+fortran+plain_blas", when="+xblas") + depends_on("python@2.7:", type="test") + + # We need to run every phase twice in order to get static and shared + # versions of the libraries. When ~shared, we run the default + # implementations of the CMakePackage's phases and get only one building + # directory 'spack-build-static' with -DBUILD_SHARED_LIBS:BOOL=OFF (see + # implementations of self.build_directory and self.cmake_args() below). + # When +shared, we run the overridden methods for the phases, each + # running the default implementation twice with different values for + # self._building_shared. As a result, we get two building directories: + # 'spack-build-static' with -DBUILD_SHARED_LIBS:BOOL=OFF and + # 'spack-build-shared' with -DBUILD_SHARED_LIBS:BOOL=ON. + _building_shared = False + + def patch(self): + # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. + if self.spec.satisfies("@3.6.0:"): + filter_file( + "${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/", + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/", + "CBLAS/CMakeLists.txt", + string=True, + ) + + # Remove duplicate header file that gets generated during CMake shared + # builds: https://github.com/Reference-LAPACK/lapack/issues/583 + if self.spec.satisfies("platform=windows @0:3.9.1"): + force_remove("LAPACKE/include/lapacke_mangling.h") + + @property + def blas_libs(self): + shared = True if "+shared" in self.spec else False + query_parameters = self.spec.last_query.extra_parameters + query2libraries = { + tuple(): ["libblas"], + ("c", "fortran"): ["libcblas", "libblas"], + ("c",): ["libcblas"], + ("fortran",): ["libblas"], + } + key = tuple(sorted(query_parameters)) + libraries = query2libraries[key] + return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) + + @property + def lapack_libs(self): + shared = True if "+shared" in self.spec else False + query_parameters = self.spec.last_query.extra_parameters + query2libraries = { + tuple(): ["liblapack"], + ("c", "fortran"): ["liblapacke", "liblapack"], + ("c",): ["liblapacke"], + ("fortran",): ["liblapack"], + } + key = tuple(sorted(query_parameters)) + libraries = query2libraries[key] + return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) + + @property + def headers(self): + include_dir = self.spec.prefix.include + cblas_h = join_path(include_dir, "cblas.h") + lapacke_h = join_path(include_dir, "lapacke.h") + return HeaderList([cblas_h, lapacke_h]) + + +class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder): + def cmake_args(self): + args = [ + self.define_from_variant("BUILD_SHARED_LIBS", "shared"), + self.define_from_variant("LAPACKE", "lapacke"), + self.define_from_variant("LAPACKE_WITH_TMG", "lapacke"), + self.define("CBLAS", self.spec.satisfies("@3.6.0:")), + ] + + if self.spec.satisfies("%intel"): + # Intel compiler finds serious syntax issues when trying to + # build CBLAS and LapackE + args.extend([self.define("CBLAS", False), self.define("LAPACKE", False)]) + + if self.spec.satisfies("%xl") or self.spec.satisfies("%xl_r"): + # use F77 compiler if IBM XL + args.extend( + [ + self.define("CMAKE_Fortran_COMPILER", self.pkg.compiler.f77), + self.define( + "CMAKE_Fortran_FLAGS", + " ".join(self.spec.compiler_flags["fflags"]) + " -O3 -qnohot", + ), + ] + ) + + # deprecated routines are commonly needed by, for example, suitesparse + # Note that OpenBLAS spack is built with deprecated routines + args.append(self.define("BUILD_DEPRECATED", True)) + + if self.spec.satisfies("+external-blas"): + args.extend( + [ + self.define("USE_OPTIMIZED_BLAS", True), + self.define("BLAS_LIBRARIES:PATH", self.spec["blas"].libs.joined(";")), + ] + ) + + if self.spec.satisfies("+xblas"): + args.extend( + [ + self.define("XBLAS_INCLUDE_DIR", self.spec["netlib-xblas"].prefix.include), + self.define("XBLAS_LIBRARY", self.spec["netlib-xblas"].libs.joined(";")), + ] + ) + + args.append(self.define("BUILD_TESTING", self.pkg.run_tests)) + + return args diff --git a/.gitlab/spack_packages/netlib-lapack/testing.patch b/.gitlab/spack_packages/netlib-lapack/testing.patch new file mode 100644 index 0000000000..fce18548c4 --- /dev/null +++ b/.gitlab/spack_packages/netlib-lapack/testing.patch @@ -0,0 +1,13 @@ +diff --git a/TESTING/LIN/alahd.f b/TESTING/LIN/alahd.f +index 8f4cd58d..6a4946e0 100644 +--- a/TESTING/LIN/alahd.f ++++ b/TESTING/LIN/alahd.f +@@ -1036,7 +1036,7 @@ + 9929 FORMAT( ' Test ratios (1-3: ', A1, 'TZRZF):' ) + 9920 FORMAT( 3X, ' 7-10: same as 3-6', 3X, ' 11-14: same as 3-6' ) + 9921 FORMAT( ' Test ratios:', / ' (1-2: ', A1, 'GELS, 3-6: ', A1, +- $ 'GELSY, 7-10: ', A1, 'GELSS, 11-14: ', A1, 'GELSD, 15-16: ' ++ $ 'GELSY, 7-10: ', A1, 'GELSS, 11-14: ', A1, 'GELSD, 15-16: ', + $ A1, 'GETSLS)') + 9928 FORMAT( 7X, 'where ALPHA = ( 1 + SQRT( 17 ) ) / 8' ) + 9927 FORMAT( 3X, I2, ': ABS( Largest element in L )', / 12X, diff --git a/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch b/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch new file mode 100644 index 0000000000..9dac2562f7 --- /dev/null +++ b/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch @@ -0,0 +1,26 @@ +diff --git a/SRC/dsytrf_aa_2stage.f b/SRC/dsytrf_aa_2stage.f +index 2991305..f5f06cc 100644 +--- a/SRC/dsytrf_aa_2stage.f ++++ b/SRC/dsytrf_aa_2stage.f +@@ -191,7 +191,7 @@ + EXTERNAL LSAME, ILAENV + * .. + * .. External Subroutines .. +- EXTERNAL XERBLA, DCOPY, DLACGV, DLACPY, ++ EXTERNAL XERBLA, DCOPY, DLACPY, + $ DLASET, DGBTRF, DGEMM, DGETRF, + $ DSYGST, DSWAP, DTRSM + * .. +diff --git a/SRC/ssytrf_aa_2stage.f b/SRC/ssytrf_aa_2stage.f +index be6809d..a929749 100644 +--- a/SRC/ssytrf_aa_2stage.f ++++ b/SRC/ssytrf_aa_2stage.f +@@ -191,7 +191,7 @@ + EXTERNAL LSAME, ILAENV + * .. + * .. External Subroutines .. +- EXTERNAL XERBLA, SCOPY, SLACGV, SLACPY, ++ EXTERNAL XERBLA, SCOPY, SLACPY, + $ SLASET, SGBTRF, SGEMM, SGETRF, + $ SSYGST, SSWAP, STRSM + * .. From 360ee525f5b163b9a9341d49beabde5bf87db51c Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Thu, 2 Mar 2023 13:09:43 -0800 Subject: [PATCH 043/187] try openblas instead of netlib-lapack with clang --- .gitlab/quartz-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 40ee07de45..17e13d33fa 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -59,7 +59,7 @@ quartz_clang_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu ^suite-sparse@5.13.0" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu ^suite-sparse@5.13.0 ^openblas" extends: .quartz_build_and_test quartz_gcc_tpls: From 012c03efefceb4c2c7f3b5b152a8810a4a15a38f Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Thu, 2 Mar 2023 13:21:35 -0800 Subject: [PATCH 044/187] sync sundials spack package --- .gitlab/spack_packages/sundials/package.py | 63 +++++++++++++++++++--- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 4de41139de..3bd56c22a2 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -27,7 +27,8 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # Versions # ========================================================================== version("develop", branch="develop") - version("6.4.0", branch="develop") + version("6.4.1", sha256="7bf10a8d2920591af3fba2db92548e91ad60eb7241ab23350a9b1bc51e05e8d0") + version("6.4.0", sha256="0aff803a12c6d298d05b56839197dd09858631864017e255ed89e28b49b652f1") version("6.3.0", sha256="89a22bea820ff250aa7239f634ab07fa34efe1d2dcfde29cc8d3af11455ba2a7") version("6.2.0", sha256="195d5593772fc483f63f08794d79e4bab30c2ec58e6ce4b0fb6bcc0e0c48f31d") version("6.1.1", sha256="cfaf637b792c330396a25ef787eb59d58726c35918ebbc08e33466e45d50470c") @@ -123,9 +124,17 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): when="@6.0.0: +profiling", description="Enable Caliper instrumentation/profiling", ) + variant("ginkgo", default=False, when="@6.4.0:", description="Enable Ginkgo interfaces") variant("hypre", default=False, when="@2.7.0:", description="Enable Hypre MPI parallel vector") - variant("lapack", default=False, description="Enable LAPACK direct solvers") + variant("kokkos", default=False, when="@6.4.0:", description="Enable Kokkos vector") + variant( + "kokkos-kernels", + default=False, + when="@6.4.0:", + description="Enable KokkosKernels based matrix and linear solver", + ) variant("klu", default=False, description="Enable KLU sparse, direct solver") + variant("lapack", default=False, description="Enable LAPACK direct solvers") variant("petsc", default=False, when="@2.7.0:", description="Enable PETSc interfaces") variant("magma", default=False, when="@5.7.0:", description="Enable MAGMA interface") variant("superlu-mt", default=False, description="Enable SuperLU_MT sparse, direct solver") @@ -190,6 +199,23 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # External libraries depends_on("caliper", when="+caliper") + depends_on("ginkgo@1.5.0:", when="+ginkgo") + depends_on("kokkos", when="+kokkos") + depends_on("kokkos-kernels", when="+kokkos-kernels") + for cuda_arch in CudaPackage.cuda_arch_values: + depends_on( + "kokkos+cuda+cuda_lambda+cuda_constexpr cuda_arch=%s" % cuda_arch, + when="+kokkos +cuda cuda_arch=%s" % cuda_arch, + ) + depends_on( + "kokkos-kernels+cuda cuda_arch=%s" % cuda_arch, + when="+kokkos-kernels +cuda cuda_arch=%s" % cuda_arch, + ) + for rocm_arch in ROCmPackage.amdgpu_targets: + depends_on( + "kokkos+rocm amdgpu_target=%s" % rocm_arch, + when="+kokkos +rocm amdgpu_target=%s" % rocm_arch, + ) depends_on("lapack", when="+lapack") depends_on("hypre+mpi@2.22.1:", when="@5.7.1: +hypre") depends_on("hypre+mpi@:2.22.0", when="@:5.7.0 +hypre") @@ -205,13 +231,13 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # Require that external libraries built with the same precision depends_on("petsc~double~complex", when="+petsc precision=single") depends_on("petsc+double~complex", when="+petsc precision=double") - + # Require that external libraries built with the same index type with when('+int64'): depends_on("hypre+mpi+int64", when="+hypre +int64") depends_on("petsc+int64", when="+petsc +int64") depends_on("superlu-dist+int64", when="+superlu-dist +int64") - + with when('~int64'): depends_on("hypre+mpi~int64", when="+hypre ~int64") depends_on("petsc~int64", when="+petsc ~int64") @@ -735,6 +761,25 @@ def initconfig_package_entries(self): if "+caliper" in spec: entries.append(cmake_cache_path("CALIPER_DIR", spec["caliper"].prefix)) + # Building with Ginkgo + if "+ginkgo" in spec: + gko_backends = ["REF"] + if "+openmp" in spec["ginkgo"] and "+openmp" in spec: + gko_backends.append("OMP") + if "+cuda" in spec["ginkgo"] and "+cuda" in spec: + gko_backends.append("CUDA") + if "+rocm" in spec["ginkgo"] and "+rocm" in spec: + gko_backends.append("HIP") + if "+oneapi" in spec["ginkgo"] and "+sycl" in spec: + gko_backends.append("DPCPP") + entries.extend( + [ + self.cache_option_from_variant("ENABLE_GINKGO", "ginkgo"), + cmake_cache_path("Ginkgo_DIR", spec["ginkgo"].prefix), + cmake_cache_string("SUNDIALS_GINKGO_BACKENDS", ";".join(gko_backends)), + ] + ) + # Building with Hypre if "+hypre" in spec: entries.extend( @@ -747,6 +792,12 @@ def initconfig_package_entries(self): hypre_libs = spec["blas"].libs + spec["lapack"].libs entries.extend([cmake_cache_string("HYPRE_LIBRARIES", hypre_libs.joined(";"))]) + # Building with Kokkos and KokkosKernels + if "+kokkos" in spec: + entries.extend([self.cache_option_from_variant("Kokkos_DIR", spec["kokkos"].prefix)]) + if "+kokkos-kernels" in spec: + entries.extend([self.cache_option_from_variant("KokkosKernels_DIR", spec["kokkos-kernels"].prefix)]) + # Building with KLU if "+klu" in spec: entries.extend( @@ -795,7 +846,7 @@ def initconfig_package_entries(self): entries.extend( [ cmake_cache_path("SUPERLUDIST_DIR", spec["superlu-dist"].prefix), - cmake_cache_string("SUPERLUDIST_OpenMP", "^superlu-dist+openmp" in spec), + cmake_cache_string("SUPERLUDIST_OpenMP", "^superlu-dist+openmp" in spec), ] ) else: @@ -804,7 +855,7 @@ def initconfig_package_entries(self): cmake_cache_path("SUPERLUDIST_INCLUDE_DIR", spec["superlu-dist"].prefix.include), cmake_cache_path("SUPERLUDIST_LIBRARY_DIR", spec["superlu-dist"].prefix.lib), cmake_cache_string("SUPERLUDIST_LIBRARIES", spec["superlu-dist"].libs), - cmake_cache_string("SUPERLUDIST_OpenMP", "^superlu-dist+openmp" in spec), + cmake_cache_string("SUPERLUDIST_OpenMP", "^superlu-dist+openmp" in spec), ] ) From 012bf287116a5c4b60f748921f789ab3fac90f4f Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Thu, 2 Mar 2023 13:22:22 -0800 Subject: [PATCH 045/187] try setting superlu_dist libraries direct --- .gitlab/spack_packages/sundials/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 3bd56c22a2..4972d8bf2f 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -847,6 +847,9 @@ def initconfig_package_entries(self): [ cmake_cache_path("SUPERLUDIST_DIR", spec["superlu-dist"].prefix), cmake_cache_string("SUPERLUDIST_OpenMP", "^superlu-dist+openmp" in spec), + cmake_cache_path("SUPERLUDIST_INCLUDE_DIR", spec["superlu-dist"].prefix.include), + cmake_cache_path("SUPERLUDIST_LIBRARY_DIR", spec["superlu-dist"].prefix.lib), + cmake_cache_string("SUPERLUDIST_LIBRARIES", spec["superlu-dist"].libs), ] ) else: From 281bc27f25ada2cd7498c86974871bc3afa70daa Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Fri, 3 Mar 2023 19:20:21 -0800 Subject: [PATCH 046/187] re-enable gcc build on lassen --- .gitlab/lassen-jobs.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 6c2174b9d3..7dc51e99d4 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -72,14 +72,14 @@ lassen_xl_cuda_tpls: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test -# lassen_gcc_cuda_tpls: -# parallel: -# matrix: -# - COMPILER_SPEC: gcc@8.3.1 -# CUDA_SPEC: [cuda@11.5.0] -# variables: -# SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" -# extends: .lassen_build_and_test +lassen_gcc_cuda_tpls: + parallel: + matrix: + - COMPILER_SPEC: gcc@8.3.1 + CUDA_SPEC: [cuda@11.5.0] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" + extends: .lassen_build_and_test # ------------------------------------------------------------------------------ # Benchmark jobs From 8ef31809ecc50f291e085e5105b8e135aaa584df Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 7 Mar 2023 13:33:50 -0800 Subject: [PATCH 047/187] try old way of enabling superlu-dist --- .gitlab/spack_packages/sundials/package.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 4972d8bf2f..4e95f4457f 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -842,14 +842,12 @@ def initconfig_package_entries(self): # Building with SuperLU_DIST if "+superlu-dist" in spec: - if spec.satisfies("@6.4.0:"): + #if spec.satisfies("@6.4.0:"): + if False: entries.extend( [ cmake_cache_path("SUPERLUDIST_DIR", spec["superlu-dist"].prefix), cmake_cache_string("SUPERLUDIST_OpenMP", "^superlu-dist+openmp" in spec), - cmake_cache_path("SUPERLUDIST_INCLUDE_DIR", spec["superlu-dist"].prefix.include), - cmake_cache_path("SUPERLUDIST_LIBRARY_DIR", spec["superlu-dist"].prefix.lib), - cmake_cache_string("SUPERLUDIST_LIBRARIES", spec["superlu-dist"].libs), ] ) else: From 189819055d6cb1a2ffbc9ff42e8ebcfd07195d31 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 7 Mar 2023 13:34:34 -0800 Subject: [PATCH 048/187] temporarily disable quartz and turn on shared spack --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5ce0eb8fc..32e3afc2e2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,9 +76,9 @@ variables: VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" ON_LASSEN: "ON" - ON_QUARTZ: "ON" + ON_QUARTZ: "OFF" SPACK_PREFIX: "v0.19.1" - SHARED_SPACK: "UPSTREAM" + SHARED_SPACK: "ON" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 BENCHMARK_QUEUE: "pbatch" From 53d85c545b817d82a7f580c71758a01fa6b1dacf Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 7 Mar 2023 16:41:25 -0800 Subject: [PATCH 049/187] Revert "temporarily disable quartz and turn on shared spack" This reverts commit 189819055d6cb1a2ffbc9ff42e8ebcfd07195d31. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32e3afc2e2..e5ce0eb8fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,9 +76,9 @@ variables: VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" ON_LASSEN: "ON" - ON_QUARTZ: "OFF" + ON_QUARTZ: "ON" SPACK_PREFIX: "v0.19.1" - SHARED_SPACK: "ON" + SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" BENCHMARK_NNODES: 4 BENCHMARK_QUEUE: "pbatch" From 7dd3d4f3edfb9a7894fd8aa8a58c46cab7a43b38 Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Tue, 7 Mar 2023 16:43:17 -0800 Subject: [PATCH 050/187] disable superlu-dist on lassen for the moment --- .gitlab/lassen-jobs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 7dc51e99d4..d3e58a641f 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -69,7 +69,7 @@ lassen_xl_cuda_tpls: - COMPILER_SPEC: xl@16.1.1.10 CUDA_SPEC: [cuda@10.1.243] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test lassen_gcc_cuda_tpls: @@ -78,7 +78,7 @@ lassen_gcc_cuda_tpls: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.5.0] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma~superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test # ------------------------------------------------------------------------------ From 3ec6e07a7e9a21acc405ece696dfa34ef205bcad Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 9 Mar 2023 08:53:58 -0800 Subject: [PATCH 051/187] add sundials+magma+raja job for corona --- .gitlab/corona-jobs.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 32bf3cbf2d..06ba5cf296 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -24,3 +24,14 @@ corona_clang_hip: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" extends: .corona_build_and_test +# ------------------------------------------------------------------------------ +# HIP + TPLs +# ------------------------------------------------------------------------------ +corona_clang_hip_tpls: + parallel: + matrix: + - COMPILER_SPEC: clang@15.0.0 + AMDGPU_TARGET: [gfx906] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}" + extends: .corona_build_and_test From 6ca0d2446a8e7ae48ccb9432756637de41beca66 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 10 Mar 2023 12:19:15 -0800 Subject: [PATCH 052/187] add precision to corona tpls job --- .gitlab/corona-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 06ba5cf296..0f522118be 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -21,7 +21,7 @@ corona_clang_hip: - COMPILER_SPEC: clang@15.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" extends: .corona_build_and_test # ------------------------------------------------------------------------------ From e523cbf6e880f7fb22ff21b8a3c3766d13bdac2e Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 10 Mar 2023 12:48:07 -0800 Subject: [PATCH 053/187] remove raja due to cuda error --- .gitlab/corona-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 0f522118be..bb56673c1b 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -33,5 +33,5 @@ corona_clang_hip_tpls: - COMPILER_SPEC: clang@15.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi+magma ^magma+rocm amdgpu_target=${AMDGPU_TARGET}" extends: .corona_build_and_test From b7ce7f508ee2c6d0c1ea3cb09a1b062af4a0a0c7 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 10 Mar 2023 13:56:54 -0800 Subject: [PATCH 054/187] fix raja build issue --- .gitlab/corona-jobs.yml | 2 +- .gitlab/spack_packages/sundials/package.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index bb56673c1b..0f522118be 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -33,5 +33,5 @@ corona_clang_hip_tpls: - COMPILER_SPEC: clang@15.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi+magma ^magma+rocm amdgpu_target=${AMDGPU_TARGET}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}" extends: .corona_build_and_test diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index f888061734..57198e50be 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -838,6 +838,8 @@ def initconfig_package_entries(self): entries.append(cmake_cache_path("RAJA_DIR", spec["raja"].prefix)) if "camp" in spec: entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix.lib.cmake + '/camp')) + if "+rocm" in spec: + entries.append(cmake_cache_string("SUNDIALS_RAJA_BACKENDS", "HIP")) # Building with SuperLU_DIST if "+superlu-dist" in spec: From bb20d20dde5c4267529067f2704453903319eb37 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 13 Mar 2023 16:03:29 -0700 Subject: [PATCH 055/187] switch conditional to check hostname --- .gitlab/build_and_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index a6f8552e4d..44020863bc 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -48,7 +48,7 @@ BUILD_JOBS=${BUILD_JOBS:-"1"} # load newer python to try the clingo concretizer # Corona does not have python 3.8.2 -if [[ -n "${AMDGPU_TARGET}" ]]; then +if [[ "${hostname}" == "corona" ]]; then echo "module load python/3.9.12" module load python/3.9.12 else @@ -57,7 +57,7 @@ else fi # Rocm version specific to Corona. -if [[ -n "${AMDGPU_TARGET}" ]]; then +if [[ "${hostname}" == "corona" ]]; then echo "module load rocm/5.4.1" module load rocm/5.4.1 fi From 41fc36e42317ac85d708bdf81150bc2378f74b00 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 15 Mar 2023 10:54:44 -0700 Subject: [PATCH 056/187] add cmake variables to enable custom MPI executable prefix --- cmake/SundialsBuildOptionsPre.cmake | 4 ++++ cmake/macros/SundialsAddTest.cmake | 20 ++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index d54d7cd266..006ebea2c6 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -292,3 +292,7 @@ sundials_option(SUNDIALS_TEST_DEVTESTS BOOL # Include unit tests in regression tests sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) + +sundials_option(ENABLE_CUSTOM_MPI_RUN_COMMAND BOOL "Enable passing in a custom command prefix to run when running MPI jobs" OFF ADVANCED) + +sundials_option(CUSTOM_MPI_RUN_COMMAND_PREFIX STRING "Executable prefix to append before each MPI job command." "flux mini run" ADVANCED) \ No newline at end of file diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index 40f7d04d8b..0501e2828f 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -135,8 +135,13 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) endif() # check if this test is run with MPI and set the MPI run command - if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND (MPIEXEC_EXECUTABLE)) - set(RUN_COMMAND "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}") + if ((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR ((CUSTOM_MPI_RUN_COMMAND_PREFIX) AND (ENABLE_CUSTOM_MPI_RUN_COMMAND)))) + if (ENABLE_CUSTOM_MPI_RUN_COMMAND) + set(RUN_COMMAND "${CUSTOM_MPI_RUN_COMMAND_PREFIX} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}") + elseif (MPIEXEC_EXECUTABLE) + set(RUN_COMMAND "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}") + endif() + # remove trailing white space (empty MPIEXEC_PREFLAGS) as it can cause # erroneous test failures with some MPI implementations string(STRIP "${RUN_COMMAND}" RUN_COMMAND) @@ -174,11 +179,18 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) endif() # check if this test is run with MPI and add the test run command - if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND (MPIEXEC_EXECUTABLE)) + if ((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR ((CUSTOM_MPI_RUN_COMMAND_PREFIX) AND (ENABLE_CUSTOM_MPI_RUN_COMMAND)))) if(MPIEXEC_PREFLAGS) string(REPLACE " " ";" PREFLAGS "${MPIEXEC_PREFLAGS}") endif() - add_test(NAME ${NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) + if (ENABLE_CUSTOM_MPI_RUN_COMMAND) + if(CUSTOM_MPI_RUN_COMMAND_PREFIX) + string(REPLACE " " ";" MPI_EXEC_ARGS "${CUSTOM_MPI_RUN_COMMAND_PREFIX}") + endif() + add_test(NAME ${NAME} COMMAND ${MPI_EXEC_ARGS} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) + else() + add_test(NAME ${NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) + endif() else() add_test(NAME ${NAME} COMMAND $ ${TEST_ARGS}) endif() From e7f98c225fb042c7216275c76b08e57eed2c7746 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 17 Mar 2023 14:48:10 -0700 Subject: [PATCH 057/187] add SUNDIALS_TEST_MPIRUN_COMMAND to cmake variables --- cmake/SundialsBuildOptionsPre.cmake | 4 +--- cmake/macros/SundialsAddTest.cmake | 20 +++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index 006ebea2c6..ad6e8c6168 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -293,6 +293,4 @@ sundials_option(SUNDIALS_TEST_DEVTESTS BOOL sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) -sundials_option(ENABLE_CUSTOM_MPI_RUN_COMMAND BOOL "Enable passing in a custom command prefix to run when running MPI jobs" OFF ADVANCED) - -sundials_option(CUSTOM_MPI_RUN_COMMAND_PREFIX STRING "Executable prefix to append before each MPI job command." "flux mini run" ADVANCED) \ No newline at end of file +sundials_option(SUNDIALS_TEST_MPIRUN_COMMAND STRING "Job scheduler or mpirun command used to launch SUNDIALS MPI tests." "" ADVANCED) \ No newline at end of file diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index 0501e2828f..299f05e9c9 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -135,11 +135,11 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) endif() # check if this test is run with MPI and set the MPI run command - if ((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR ((CUSTOM_MPI_RUN_COMMAND_PREFIX) AND (ENABLE_CUSTOM_MPI_RUN_COMMAND)))) - if (ENABLE_CUSTOM_MPI_RUN_COMMAND) - set(RUN_COMMAND "${CUSTOM_MPI_RUN_COMMAND_PREFIX} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}") - elseif (MPIEXEC_EXECUTABLE) + if ((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR (SUNDIALS_TEST_MPIRUN_COMMAND))) + if (SUNDIALS_TEST_MPIRUN_COMMAND STREQUAL "") set(RUN_COMMAND "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}") + elseif (MPIEXEC_EXECUTABLE) + set(RUN_COMMAND "${SUNDIALS_TEST_MPIRUN_COMMAND} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${MPIEXEC_PREFLAGS}") endif() # remove trailing white space (empty MPIEXEC_PREFLAGS) as it can cause @@ -179,17 +179,15 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) endif() # check if this test is run with MPI and add the test run command - if ((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR ((CUSTOM_MPI_RUN_COMMAND_PREFIX) AND (ENABLE_CUSTOM_MPI_RUN_COMMAND)))) + if ((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR (SUNDIALS_TEST_MPIRUN_COMMAND))) if(MPIEXEC_PREFLAGS) string(REPLACE " " ";" PREFLAGS "${MPIEXEC_PREFLAGS}") endif() - if (ENABLE_CUSTOM_MPI_RUN_COMMAND) - if(CUSTOM_MPI_RUN_COMMAND_PREFIX) - string(REPLACE " " ";" MPI_EXEC_ARGS "${CUSTOM_MPI_RUN_COMMAND_PREFIX}") - endif() - add_test(NAME ${NAME} COMMAND ${MPI_EXEC_ARGS} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) + if (SUNDIALS_TEST_MPIRUN_COMMAND STREQUAL "") + add_test(NAME ${NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) else() - add_test(NAME ${NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) + string(REPLACE " " ";" MPI_EXEC_ARGS "${SUNDIALS_TEST_MPIRUN_COMMAND}") + add_test(NAME ${NAME} COMMAND ${MPI_EXEC_ARGS} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) endif() else() add_test(NAME ${NAME} COMMAND $ ${TEST_ARGS}) From bc580e767e0be4e0ade2a0fb657d78f592a1e390 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 20 Mar 2023 08:16:20 -0700 Subject: [PATCH 058/187] add scheduler variant to package.py --- .gitlab/corona-jobs.yml | 2 +- .gitlab/spack_packages/sundials/package.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 0f522118be..00d9505f76 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -33,5 +33,5 @@ corona_clang_hip_tpls: - COMPILER_SPEC: clang@15.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} +rocm~mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}" extends: .corona_build_and_test diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 57198e50be..70290544a4 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -178,6 +178,9 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): "profiling", default=False, when="@6.0.0:", description="Build with profiling capabilities" ) + # Scheduler + variant("scheduler", default="slurm", description="Specify which scheduler the system runs on.", values=("flux", "lsf", "slurm")) + # ========================================================================== # Dependencies # ========================================================================== @@ -666,6 +669,9 @@ def initconfig_mpi_entries(self): cmake_cache_path("MPI_MPIF90", spec["mpi"].mpifc) ] ) + if spec.satisfies("scheduler=flux"): + entries.append(cmake_cache_string("SUNDIALS_TEST_MPIRUN_COMMAND", "flux mini run")) + return entries From 7661fadb5f90af63d93cd8f72933d09a007079fe Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 20 Mar 2023 08:41:23 -0700 Subject: [PATCH 059/187] turn on mpi for base corona job --- .gitlab/corona-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 00d9505f76..1de06c9808 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -21,7 +21,7 @@ corona_clang_hip: - COMPILER_SPEC: clang@15.0.0 AMDGPU_TARGET: [gfx906] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm~mpi" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi" extends: .corona_build_and_test # ------------------------------------------------------------------------------ From 2fdfcf06ce9aff0d952824448f0355538aeb9626 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 24 Mar 2023 12:07:28 -0700 Subject: [PATCH 060/187] add petsc to quartz jobs, disable for xl compiler on lassen --- .gitlab/lassen-jobs.yml | 4 ++-- .gitlab/quartz-jobs.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index d3e58a641f..c05fc970a5 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -69,7 +69,7 @@ lassen_xl_cuda_tpls: - COMPILER_SPEC: xl@16.1.1.10 CUDA_SPEC: [cuda@10.1.243] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist~petsc cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test lassen_gcc_cuda_tpls: @@ -78,7 +78,7 @@ lassen_gcc_cuda_tpls: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.5.0] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma~superlu-dist cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma~superlu-dist+petsc cuda_arch=70 ^petsc+cuda cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test # ------------------------------------------------------------------------------ diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 17e13d33fa..ffb5a4b38c 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -59,7 +59,7 @@ quartz_clang_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu ^suite-sparse@5.13.0 ^openblas" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0 ^openblas" extends: .quartz_build_and_test quartz_gcc_tpls: @@ -69,7 +69,7 @@ quartz_gcc_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu ^suite-sparse@5.13.0" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0" extends: .quartz_build_and_test quartz_intel_tpls: @@ -79,5 +79,5 @@ quartz_intel_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^suite-sparse@5.13.0" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0" extends: .quartz_build_and_test From 2458890af1eba4c51f26281572a3cba9cbfcff69 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 24 Mar 2023 14:50:01 -0700 Subject: [PATCH 061/187] turn off petsc for quartz_intel_tpls --- .gitlab/quartz-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index ffb5a4b38c..93599c3188 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -79,5 +79,5 @@ quartz_intel_tpls: INDEX_SPEC: [~int64] PRECISION_SPEC: [double] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^suite-sparse@5.13.0" extends: .quartz_build_and_test From e176be50f75b97cf95a1f98d0d704144b2705589 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 24 Mar 2023 15:06:12 -0700 Subject: [PATCH 062/187] add hypre sub spec to use internal superlu --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index c05fc970a5..f443b23ce7 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -69,7 +69,7 @@ lassen_xl_cuda_tpls: - COMPILER_SPEC: xl@16.1.1.10 CUDA_SPEC: [cuda@10.1.243] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist~petsc cuda_arch=70 ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist~petsc+hypre cuda_arch=70 ^hypre+internal-superlu ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test lassen_gcc_cuda_tpls: From 77d1456109fc66bd3bbfb036611becc4ba52e1bc Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 24 Mar 2023 15:40:31 -0700 Subject: [PATCH 063/187] restrict hypre version --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index f443b23ce7..cf0edb58e4 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -69,7 +69,7 @@ lassen_xl_cuda_tpls: - COMPILER_SPEC: xl@16.1.1.10 CUDA_SPEC: [cuda@10.1.243] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist~petsc+hypre cuda_arch=70 ^hypre+internal-superlu ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist~petsc+hypre cuda_arch=70 ^hypre@2.12.1+internal-superlu ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test lassen_gcc_cuda_tpls: From 66f4a4dc2a379497b179729b49a90f9f1aa48c4d Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 27 Mar 2023 10:42:30 -0700 Subject: [PATCH 064/187] lower hypre version --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index cf0edb58e4..3609412995 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -69,7 +69,7 @@ lassen_xl_cuda_tpls: - COMPILER_SPEC: xl@16.1.1.10 CUDA_SPEC: [cuda@10.1.243] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist~petsc+hypre cuda_arch=70 ^hypre@2.12.1+internal-superlu ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja+magma~superlu-dist~petsc+hypre cuda_arch=70 ^hypre@2.12.0+internal-superlu ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_test lassen_gcc_cuda_tpls: From da4dcfb202553ac8d4a5f26fe1d8e1982102b898 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 27 Mar 2023 15:42:54 -0700 Subject: [PATCH 065/187] store benchmark cali files in sundials workspace --- .gitlab/build_and_bench.sh | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh index 46b7a92e9c..4a031dd525 100644 --- a/.gitlab/build_and_bench.sh +++ b/.gitlab/build_and_bench.sh @@ -7,6 +7,11 @@ benchmark_dir="${build_dir}/benchmarks" ar3d_dir="${benchmark_dir}/advection_reaction_3D" d2d_dir="${benchmark_dir}/diffusion_2D" +job_unique_id=${CI_JOB_ID:-""} +job_timestamp=${CI_JOB_STARTED_AT:-""} + +benchmark_store_dir="/usr/workspace/sundials/benchmarks/$(hostname)/${job_timestamp}_${job_unique_id}" + nresg=$((BENCHMARK_NNODES * 4)) # Lassen has 4 GPUs per node nresc=$((BENCHMARK_NNODES * 40)) # Lassen has 40 cores per node @@ -31,23 +36,23 @@ then if [[ -f ${ar3d_dir}/advection_reaction_3D_mpicuda ]] then - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_arkdirk_newton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkdirk_newton.cali),runtime-report(calc.inclusive)" jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-DIRK --nls newton --tf 10.0 --dont-save - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_cvbdf_newton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_cvbdf_newton.cali),runtime-report(calc.inclusive)" jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method CV-BDF --nls newton --tf 10.0 --dont-save - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_ida_newton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_ida_newton.cali),runtime-report(calc.inclusive)" jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method IDA --nls newton --tf 10.0 --dont-save elif [[ -f ${ar3d_dir}/advection_reaction_3D ]] then - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-IMEX --nls tl-newton --tf 2.0 --dont-save - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_arkdirk_newton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkdirk_newton.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-DIRK --nls newton --tf 2.0 --dont-save - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_cvbdf_newton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_cvbdf_newton.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method CV-BDF --nls newton --tf 2.0 --dont-save - export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 --dont-save fi @@ -61,19 +66,19 @@ then if [[ -d ${d2d_dir}/mpi_gpu ]] then - export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_arkode.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/arkode_diffusion_2D_mpicuda" - export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_cvode.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/cvode_diffusion_2D_mpicuda" - export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_ida.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/ida_diffusion_2D_mpicuda" elif [[ -d ${d2d_dir}/mpi_serial ]] then - export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_arkode.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" - export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_cvode.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" - export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_ida.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/ida_diffusion_2D" fi From 2d3815f99dd9c187976f39c76a3ef6bfed15d008 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 29 Mar 2023 11:51:10 -0700 Subject: [PATCH 066/187] add back benchmarks variant --- .gitlab/build_and_bench.sh | 5 ++++- .gitlab/spack_packages/sundials/package.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh index 4a031dd525..06360b4446 100644 --- a/.gitlab/build_and_bench.sh +++ b/.gitlab/build_and_bench.sh @@ -10,7 +10,10 @@ d2d_dir="${benchmark_dir}/diffusion_2D" job_unique_id=${CI_JOB_ID:-""} job_timestamp=${CI_JOB_STARTED_AT:-""} -benchmark_store_dir="/usr/workspace/sundials/benchmarks/$(hostname)/${job_timestamp}_${job_unique_id}" +# remove tailing number from hostname +hostname=${hostname%%[0-9]*} + +benchmark_store_dir="/usr/workspace/sundials/benchmarks/${hostname}/${job_timestamp}_${job_unique_id}" nresg=$((BENCHMARK_NNODES * 4)) # Lassen has 4 GPUs per node nresc=$((BENCHMARK_NNODES * 40)) # Lassen has 40 cores per node diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 70290544a4..8a8d9a1da9 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -181,6 +181,9 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # Scheduler variant("scheduler", default="slurm", description="Specify which scheduler the system runs on.", values=("flux", "lsf", "slurm")) + # Benchmarking + variant("benchmarks", default=False, description ="Build benchmark programs") + # ========================================================================== # Dependencies # ========================================================================== @@ -730,7 +733,9 @@ def initconfig_package_entries(self): self.cache_option_from_variant("SUNDIALS_BUILD_WITH_MONITORING", "monitoring"), # Profiling self.cache_option_from_variant("SUNDIALS_BUILD_WITH_PROFILING", "profiling"), - self.cache_option_from_variant("ENABLE_CALIPER", "caliper") + self.cache_option_from_variant("ENABLE_CALIPER", "caliper"), + # Benchmarking + self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks") ] ) @@ -912,4 +917,7 @@ def initconfig_package_entries(self): ] ) + if "+benchmarks" in spec: + entries.append(self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks")) + return entries From b533df07b6a51ae9a306e80b319b4cf77b1c1192 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 31 Mar 2023 13:47:37 -0700 Subject: [PATCH 067/187] add profile-examples variant to package.py --- .gitlab/spack_packages/sundials/package.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 8a8d9a1da9..68b28512d9 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -179,10 +179,13 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): ) # Scheduler - variant("scheduler", default="slurm", description="Specify which scheduler the system runs on.", values=("flux", "lsf", "slurm")) + variant("scheduler", default="slurm", description="Specify which scheduler the system runs on", values=("flux", "lsf", "slurm")) # Benchmarking - variant("benchmarks", default=False, description ="Build benchmark programs") + variant("benchmarks", default=False, description="Build benchmark programs") + + # Profiling examples + variant("profile-examples", default=False, when="+caliper +profiling", description="Build examples with profiling capabilities") # ========================================================================== # Dependencies @@ -205,6 +208,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # External libraries depends_on("caliper", when="+caliper") + depends_on("caliper~adiak", when="+profile-examples") depends_on("ginkgo@1.5.0:", when="+ginkgo") depends_on("kokkos", when="+kokkos") depends_on("kokkos-kernels", when="+kokkos-kernels") @@ -673,7 +677,7 @@ def initconfig_mpi_entries(self): ] ) if spec.satisfies("scheduler=flux"): - entries.append(cmake_cache_string("SUNDIALS_TEST_MPIRUN_COMMAND", "flux mini run")) + entries.append(cmake_cache_string("SUNDIALS_TEST_MPIRUN_COMMAND", "flux run")) return entries @@ -735,7 +739,9 @@ def initconfig_package_entries(self): self.cache_option_from_variant("SUNDIALS_BUILD_WITH_PROFILING", "profiling"), self.cache_option_from_variant("ENABLE_CALIPER", "caliper"), # Benchmarking - self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks") + self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks"), + # Profile examples + self.cache_option_from_variant("SUNDIALS_TEST_PROFILE", "profile-examples") ] ) @@ -763,7 +769,8 @@ def initconfig_package_entries(self): self.cache_option_from_variant("RAJA_ENABLE", "raja"), self.cache_option_from_variant("SUPERLUDIST_ENABLE", "superlu-dist"), self.cache_option_from_variant("SUPERLUMT_ENABLE", "superlu-mt"), - self.cache_option_from_variant("Trilinos_ENABLE", "trilinos") + self.cache_option_from_variant("Trilinos_ENABLE", "trilinos"), + self.cache_option_from_variant("ENABLE_GINKGO", "ginkgo") ] ) @@ -784,7 +791,6 @@ def initconfig_package_entries(self): gko_backends.append("DPCPP") entries.extend( [ - self.cache_option_from_variant("ENABLE_GINKGO", "ginkgo"), cmake_cache_path("Ginkgo_DIR", spec["ginkgo"].prefix), cmake_cache_string("SUNDIALS_GINKGO_BACKENDS", ";".join(gko_backends)), ] @@ -916,8 +922,4 @@ def initconfig_package_entries(self): cmake_cache_option("F90_ENABLE", "+examples+fcmix" in spec), ] ) - - if "+benchmarks" in spec: - entries.append(self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks")) - return entries From b7f4f3d88bc7e39862586553913ac3cf9e9cad61 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Apr 2023 08:10:17 -0700 Subject: [PATCH 068/187] add examples test command in build_and_bench --- .gitlab/build_and_bench.sh | 24 ++++++++++++++-------- .gitlab/spack_packages/sundials/package.py | 13 ++++++++++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh index 06360b4446..a177fa7d8d 100644 --- a/.gitlab/build_and_bench.sh +++ b/.gitlab/build_and_bench.sh @@ -33,6 +33,14 @@ cd "${benchmark_dir}" # because we are interested in individual region timings. export CUDA_LAUNCH_BLOCKING=1 +if [[ -d ${build_dir}]] +then + module load cmake/3.23 # --test-dir flag requires cmake 3.20 or higher + date + export CALI_CONFIG="spot(output=${benchmark_store_dir}/example_tests.cali),runtime-report(calc.inclusive)" + ctest --output-on-failure --verbose --test-dir ${build_dir} -T test 2>&1 | tee tests_output.txt +fi + if [[ -d ${ar3d_dir} ]] then date @@ -40,23 +48,23 @@ then if [[ -f ${ar3d_dir}/advection_reaction_3D_mpicuda ]] then export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-IMEX --nls tl-newton --tf 10.0 export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkdirk_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-DIRK --nls newton --tf 10.0 --dont-save + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-DIRK --nls newton --tf 10.0 export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_cvbdf_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method CV-BDF --nls newton --tf 10.0 --dont-save + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method CV-BDF --nls newton --tf 10.0 export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_ida_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method IDA --nls newton --tf 10.0 --dont-save + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method IDA --nls newton --tf 10.0 elif [[ -f ${ar3d_dir}/advection_reaction_3D ]] then export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-IMEX --nls tl-newton --tf 2.0 --dont-save + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-IMEX --nls tl-newton --tf 2.0 export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkdirk_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-DIRK --nls newton --tf 2.0 --dont-save + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-DIRK --nls newton --tf 2.0 export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_cvbdf_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method CV-BDF --nls newton --tf 2.0 --dont-save + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method CV-BDF --nls newton --tf 2.0 export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 --dont-save + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 fi date diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 68b28512d9..81c9214d38 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -179,13 +179,22 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): ) # Scheduler - variant("scheduler", default="slurm", description="Specify which scheduler the system runs on", values=("flux", "lsf", "slurm")) + variant( + "scheduler", + default="slurm", + description="Specify which scheduler the system runs on", + values=("flux", "lsf", "slurm") + ) # Benchmarking variant("benchmarks", default=False, description="Build benchmark programs") # Profiling examples - variant("profile-examples", default=False, when="+caliper +profiling", description="Build examples with profiling capabilities") + variant( + "profile-examples", + default=False, + when="+caliper +profiling", + description="Build examples with profiling capabilities") # ========================================================================== # Dependencies From 986fe05f55c210f00f7c03d9acf0fd3a7e997dd8 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Apr 2023 10:06:46 -0700 Subject: [PATCH 069/187] turn off adiak in benchmark ci spec --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 3609412995..01b4a875eb 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -91,6 +91,6 @@ lassen_gcc_cuda_bench: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.5.0] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double ~int64 +benchmarks+profiling+caliper +mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+cuda cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double ~int64 +benchmarks+profiling+caliper +mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+cuda~adiak cuda_arch=70 ^${CUDA_SPEC}" extends: .lassen_build_and_bench From 24cbee4ff5d23b7a814903a8394d897f82c58e94 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Mon, 3 Apr 2023 10:50:59 -0700 Subject: [PATCH 070/187] update developer docs with info on the CPT suite --- .../source/testing/Benchmarking.rst | 40 +++++++++++++++++++ doc/sundials_developers/source/testing/CI.rst | 20 +++------- .../source/testing/index.rst | 1 + 3 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 doc/sundials_developers/source/testing/Benchmarking.rst diff --git a/doc/sundials_developers/source/testing/Benchmarking.rst b/doc/sundials_developers/source/testing/Benchmarking.rst new file mode 100644 index 0000000000..52c12e0272 --- /dev/null +++ b/doc/sundials_developers/source/testing/Benchmarking.rst @@ -0,0 +1,40 @@ +.. + ----------------------------------------------------------------------------- + SUNDIALS Copyright Start + Copyright (c) 2002-2023, Lawrence Livermore National Security + and Southern Methodist University. + All rights reserved. + + See the top-level LICENSE and NOTICE files for details. + + SPDX-License-Identifier: BSD-3-Clause + SUNDIALS Copyright End + ----------------------------------------------------------------------------- + +Continuous Performance Testing (CPT) +==================================== + +In order to protect against performance regression of SUNDIALS at all scales, we leverage the GitLab CI infrastructure setup for CI testing to perform continuous performance testing. + +The CPT suite consists of the :ref:`SUNDIALS benchmark programs` programs, which can scale up to full supercomputers, and the normal SUNDIALS examples program suite (i.e., the programs in the ``examples`` directory of the SUNDIALS repo). + +The CI suite can run the regular SUNDIALS CI tests, or it can run the SUNDIALS CPT suite by setting the ``BENCHMARK`` variable to ``ON`` when running a pipeline from the GitLab CI UI. +The benchmark problems are run with Caliper and a report for Spot and a human-readable runtime-report are generated. +The runtime-report is printed to the stdout and can be viewed in the GitLab CI job output. The Spot output files are made available as job artifacts. +We maintain a pipeline that runs the CPT suite weekly on a schedule, see ``_. + +FUTURE: WE WILL TRACK PERFORMANCE OVER TIME WITH `CALIPER `_ AND THE `SPOT WEB FRAMEWORK `_. + + +Locally Building and Running the CPT +------------------------------------ + +The SUNDIALS example suite can be run with Caliper profiling enabled for the CPT suite by setting the CMake options + +.. code-block:: cmake + + cmake -DSUNDIALS_BUILD_WITH_PROFILING=ON -DENABLE_CALIPER=ON -DCaliper_DIR=/path/to/caliper -DSUNDIALS_TEST_DEVTESTS=ON -DSUNDIALS_TEST_PROFILE=ON + +This command will result in ``--profile`` option being passed to the SUNDIALS test runner Python script, ``test/testRunner``, which will in turn set the ``CALI_CONFIG`` environment variable before running every test so that when you run ``make test`` the examples will produce `.cali` output files documenting the performance. + +Refer to documentation section :ref:`Benchmarks` for instructions on building and running the ``benchmarks/`` programs locally. diff --git a/doc/sundials_developers/source/testing/CI.rst b/doc/sundials_developers/source/testing/CI.rst index 52626c5301..5382fd9a5e 100644 --- a/doc/sundials_developers/source/testing/CI.rst +++ b/doc/sundials_developers/source/testing/CI.rst @@ -11,8 +11,8 @@ SUNDIALS Copyright End ----------------------------------------------------------------------------- -GitHub -====== +GitHub CI Testing +================= There are two types of CI testing that we run on GitHub via `GitHub actions `_: @@ -95,8 +95,8 @@ GitHub action) every week against the latest Spack develop commit. This allows u the latest versions of dependencies reguarly and detect interface breakages. -GitLab -====== +GitLab CI Testing +================= This section provides an overview of the GitLab continuous integration (CI) features and describes the SUNDIALS GitLab CI setup. For more information see @@ -238,14 +238,7 @@ then used to create an environment variable ``SPEC`` with a Spack spec used by Benchmark Jobs ^^^^^^^^^^^^^^ -The CI suite can run the regular SUNDIALS tests, or it can run the SUNDIALS -benchmark problems in ``becnhmarks/``. The benchmark problems can be -run instead of the regular test suite by setting the ``BENCHMARK`` variable -to ``ON`` when running a pipeline from the GitLab CI UI. The benchmark -problems are run with Caliper and a report for Spot and a human-readable -runtime-report are generated. The runtime-report is printed to the stdout -and can be viewed in the GitLab CI job output. The Spot output files are made -available as job artifacts. +See `SUNDIALS Continuous Performance Testing (CPT)` for more details. GitLab CI Test Script --------------------- @@ -278,8 +271,7 @@ Updating Spack To update the spack commit used for the CI: 1. The first thing to do is update the spack commit in the ``.uberenv_config.json`` file. -2. Then, a pipeline should be manually launched from the GitLab UI with the ``SHARED_SPACK`` CI variable set -to ``ON`` and the ``SPACK_PREFIX`` variable to the version of spack being set in the uberenv_config.json. +2. Then, a pipeline should be manually launched from the GitLab UI with the ``SHARED_SPACK`` CI variable set to ``ON`` and the ``SPACK_PREFIX`` variable to the version of spack being set in the uberenv_config.json. This will create a new spack installation and rebuild all of the specs. diff --git a/doc/sundials_developers/source/testing/index.rst b/doc/sundials_developers/source/testing/index.rst index 305e769952..c1e537e1b3 100644 --- a/doc/sundials_developers/source/testing/index.rst +++ b/doc/sundials_developers/source/testing/index.rst @@ -25,3 +25,4 @@ The following sections describe the SUNDIALS CI suite as well as a testing workf CI Local + Benchmarking From 815ad404a88470aefd8c62fd3d67d11461684467 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Apr 2023 11:16:22 -0700 Subject: [PATCH 071/187] fix syntax error in build_and_bench --- .gitlab/build_and_bench.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh index a177fa7d8d..4c609469d7 100644 --- a/.gitlab/build_and_bench.sh +++ b/.gitlab/build_and_bench.sh @@ -33,12 +33,13 @@ cd "${benchmark_dir}" # because we are interested in individual region timings. export CUDA_LAUNCH_BLOCKING=1 -if [[ -d ${build_dir}]] +if [[ -d ${build_dir} ]] then module load cmake/3.23 # --test-dir flag requires cmake 3.20 or higher date export CALI_CONFIG="spot(output=${benchmark_store_dir}/example_tests.cali),runtime-report(calc.inclusive)" ctest --output-on-failure --verbose --test-dir ${build_dir} -T test 2>&1 | tee tests_output.txt + date fi if [[ -d ${ar3d_dir} ]] From 2545e89a596eeab82760948fec5c88fbeb91de92 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Apr 2023 15:15:09 -0700 Subject: [PATCH 072/187] add ls for debugging --- .gitlab/build_and_bench.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh index 4c609469d7..552f253051 100644 --- a/.gitlab/build_and_bench.sh +++ b/.gitlab/build_and_bench.sh @@ -28,6 +28,7 @@ then fi cd "${benchmark_dir}" +ls # We use synchronous kernel launches for benchmarking # because we are interested in individual region timings. @@ -37,7 +38,7 @@ if [[ -d ${build_dir} ]] then module load cmake/3.23 # --test-dir flag requires cmake 3.20 or higher date - export CALI_CONFIG="spot(output=${benchmark_store_dir}/example_tests.cali),runtime-report(calc.inclusive)" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/test_suite.cali),runtime-report(calc.inclusive)" ctest --output-on-failure --verbose --test-dir ${build_dir} -T test 2>&1 | tee tests_output.txt date fi @@ -45,7 +46,6 @@ fi if [[ -d ${ar3d_dir} ]] then date - if [[ -f ${ar3d_dir}/advection_reaction_3D_mpicuda ]] then export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" @@ -67,7 +67,6 @@ then export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)" jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 fi - date fi From 78a9792b4d417068cb5e6a387e00de0562e4f70e Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Apr 2023 14:00:26 -0700 Subject: [PATCH 073/187] add benchmark cmake macro and cmake options --- .gitlab/radiuss-spack-configs | 2 +- .gitlab/spack_packages/sundials/package.py | 8 ++- cmake/SundialsBuildOptionsPre.cmake | 4 +- cmake/SundialsSetupTesting.cmake | 13 ++++ cmake/macros/SundialsAddBenchmark.cmake | 73 ++++++++++++++++++++++ cmake/macros/SundialsAddTest.cmake | 8 +-- cmake/macros/SundialsCMakeMacros.cmake | 1 + 7 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 cmake/macros/SundialsAddBenchmark.cmake diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index ff5d106178..6f22afc1ec 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit ff5d10617864635b1e3d7eea9e8aa93c70e66a13 +Subproject commit 6f22afc1ece86c479d2d2a64e14736ef00b632d6 diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 81c9214d38..8d2e0e07da 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -685,8 +685,12 @@ def initconfig_mpi_entries(self): cmake_cache_path("MPI_MPIF90", spec["mpi"].mpifc) ] ) - if spec.satisfies("scheduler=flux"): - entries.append(cmake_cache_string("SUNDIALS_TEST_MPIRUN_COMMAND", "flux run")) + if "scheduler=flux" in spec: + entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "flux run")) + if "scheduler=slurm" in spec: + entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "srun")) + if "scheduler=lsf" in spec: + entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "jsrun")) return entries diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index c556d17619..fa2330f456 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -295,4 +295,6 @@ sundials_option(SUNDIALS_TEST_DEVTESTS BOOL sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) -sundials_option(SUNDIALS_TEST_MPIRUN_COMMAND STRING "Job scheduler or mpirun command used to launch SUNDIALS MPI tests." "" ADVANCED) \ No newline at end of file +sundials_option(SUNDIALS_SCHEDULER_COMMAND STIRNG "Job scheduler command to use to launch SUNDIALS MPI tests." "" ADVANCED) + +sundials_option(SUNDIALS_BENCHMARK_OUTPUT_DIR STRING "Specify where to put benchmark output." "/usr/workspace/pan13/sundials/benchmarks" ADVANCED) diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index f3123d3f91..8dfbe116b5 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -99,3 +99,16 @@ if(EXAMPLES_INSTALL) "All installation tests complete.") endif() + +# If benchmarks are enabled, set up `make benchmark` +# Use the test runner to set up benchmarking +if(BUILD_BENCHMARKS) + + message("SUNDIALS Benchmarking") + + # Create benchmark targets + add_custom_target(benchmark + ${CMAKE_COMMAND} -E cmake_echo_color --cyan + "All benchmarks complete." + ) +endif() diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake new file mode 100644 index 0000000000..e776d67b20 --- /dev/null +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -0,0 +1,73 @@ +# Take in an executable, determine how to construct the arguments + +# --------------------------------------------------------------- +# CMake macro for adding benchmarks to `make benchmark`. +# --------------------------------------------------------------- + +macro(sundials_add_benchmark NAME EXECUTABLE) + # Define single value parameters the macro takes in to set up the test runner + # + # NUM_NODES = number of nodes (GPU count or CPU count) to run on or number of resource sets + # BENCHMARK_ARGS = arguments to pass to the executable + # IDENTIFIER = suffix to append to end of benchmark name + set(oneValueArgs NUM_NODES BENCHMARK_ARGS IDENTIFIER) + + # TEST_RUNNER_ARGS = command line arguments to pass to the test executable + set(multiValueArgs TEST_RUNNER_ARGS ) + + # ENABLE_GPU = indicate this benchmark should be run with GPUs + set(options ENABLE_GPU) + + cmake_parse_arguments(sundials_add_benchmark + "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + # command line arguments for the test runner script + set(TEST_RUNNER_ARGS + "--verbose" + "--executablename=$" + "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}" + "--nodiff" + ) + + # set the target name + if(sundials_add_benchmark_IDENTIFIER) + set(TARGET_NAME ${NAME}_${sundials_add_benchmark_IDENTIFIER}) + else() + if(sundials_add_benchmark_BENCHMARK_ARGS) + string(REPLACE " " "_" TEST_SUFFIX "${sundials_add_benchmark_BENCHMARK_ARGS}") + set(TARGET_NAME ${NAME}_${TEST_SUFFIX}) + else() + set(TARGET_NAME ${NAME}_run) + endif() + endif() + + # incorporate scheduler arguments into test_runner + if(SUNDIALS_SCHEDULER_COMMAND STREQUAL "flux run") + set(SCHEDULER_STRING " -nnodes=${sundials_add_benchmark_NUM_NODES}") + elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun" AND ${sundials_add_benchmark_ENABLE_GPU}) + set(SCHEDULER_STRING " --smpiargs=\\\"-gpu\\\" -n${sundials_add_benchmark_NUM_NODES} -a1 -c1 -g1") + elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun") + set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_NODES} -a1 -c1") + elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "srun") + set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_NODES} --cpus-per-task=1 --ntasks-per-node=1") + endif() + string(REPLACE " " ";" SCHEDULER_ARGS "${SCHEDULER_STRING}") + + # taken from SundialsAddTest + string(STRIP "${RUN_COMMAND}" RUN_COMMAND) + set(RUN_COMMAND ${SUNDIALS_SCHEDULER_COMMAND} ${SCHEDULER_ARGS}) + list(APPEND TEST_RUNNER_ARGS "--runcommand=\"${RUN_COMMAND}\"") + + # enable test runner to set up caliper output paths and configs + if(SUNDIALS_TEST_PROFILE) + list(APPEND TEST_RUNNER_ARGS "--profile") + endif() + + list(APPEND TEST_RUNNER_ARGS "--runargs=${sundials_add_benchmark_BENCHMARK_ARGS}" "--testname=${TARGET_NAME}") + add_custom_target(${TARGET_NAME} + COMMENT "Running ${TARGET_NAME}" + COMMAND ${PYTHON_EXECUTABLE} ${TESTRUNNER} ${TEST_RUNNER_ARGS} + ) + add_dependencies(benchmark ${TARGET_NAME}) + +endmacro() diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index 67ad2160a8..5a16f5ecae 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -179,14 +179,12 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) endif() # check if this test is run with MPI and add the test run command - if ((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR (SUNDIALS_TEST_MPIRUN_COMMAND))) + if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND ((MPIEXEC_EXECUTABLE) OR (SUNDIALS_SCHEDULER_COMMAND))) if(MPIEXEC_PREFLAGS) string(REPLACE " " ";" PREFLAGS "${MPIEXEC_PREFLAGS}") endif() - if (SUNDIALS_TEST_MPIRUN_COMMAND STREQUAL "") - add_test(NAME ${NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) - else() - string(REPLACE " " ";" MPI_EXEC_ARGS "${SUNDIALS_TEST_MPIRUN_COMMAND}") + if (SUNDIALS_SCHEDULER_COMMAND) + string(REPLACE " " ";" MPI_EXEC_ARGS "${SUNDIALS_SCHEDULER_COMMAND}") add_test(NAME ${NAME} COMMAND ${MPI_EXEC_ARGS} ${MPIEXEC_NUMPROC_FLAG} ${SUNDIALS_ADD_TEST_MPI_NPROCS} ${PREFLAGS} $ ${TEST_ARGS}) endif() else() diff --git a/cmake/macros/SundialsCMakeMacros.cmake b/cmake/macros/SundialsCMakeMacros.cmake index b2084d789a..8cf64699ca 100644 --- a/cmake/macros/SundialsCMakeMacros.cmake +++ b/cmake/macros/SundialsCMakeMacros.cmake @@ -138,3 +138,4 @@ include(SundialsAddTestInstall) include(SundialsInstallExamples) include(SundialsInstallExamplesGinkgo) include(SundialsOption) +include(SundialsAddBenchmark) From 2b9c44ad76205f79ac591604fc2d96b4dc41e952 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Apr 2023 14:09:30 -0700 Subject: [PATCH 074/187] adjust cali config to be compatible with spot --- test/testRunner | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/testRunner b/test/testRunner index 7d479c89a8..b9d6f13f81 100755 --- a/test/testRunner +++ b/test/testRunner @@ -100,7 +100,7 @@ def main(): answerDir = os.path.join(testDir, "answers") if(args.verbose): - print("testName %s"%testDir) + print("testName %s"%testName) print("outDir %s"%outDir) print("testDir %s"%testDir) print("answerDir %s"%answerDir) @@ -195,13 +195,13 @@ def main(): # If profiling set the CALI_CONFIG environment variable if(args.profile): dateTime = datetime.datetime.now().strftime("%m%d%Y_%H%M%S") - profilePath = os.path.join(outDir, args.testName+".%s.caliper" % dateTime) + profilePath = os.path.join(outDir, args.testName+".%s.cali" % dateTime) os.environ['SUNDIALS_VERSION'] = subprocess.check_output( ['git', 'describe', '--abbrev=12', '--dirty', '--always', '--tags'] ).decode('UTF-8').strip() os.environ['CALI_SERVICES_ENABLE'] = 'env' os.environ['CALI_ENV_EXTRA'] = 'SUNDIALS_VERSION' - os.environ['CALI_CONFIG'] = 'runtime-report,hatchet-region-profile,output=%s' % profilePath + os.environ['CALI_CONFIG'] = 'runtime-report,hatchet-region-profile,spot(output=%s)' % profilePath # if user supplies precision info overide the default choices if(args.floatPrecision >= 0): From 265f5cd46863e1d6b19ebd1a46601b9d9c339403 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Apr 2023 14:10:01 -0700 Subject: [PATCH 075/187] add benchmark macro calls to benchmark CMakeLists --- .../advection_reaction_3D/CMakeLists.txt | 59 +++++++++++++++++++ .../diffusion_2D/mpi_gpu/CMakeLists.txt | 22 +++++++ .../diffusion_2D/mpi_serial/CMakeLists.txt | 8 +++ benchmarks/nvector/cuda/CMakeLists.txt | 7 +++ benchmarks/nvector/hip/CMakeLists.txt | 7 +++ benchmarks/nvector/mpiplusx/CMakeLists.txt | 6 ++ benchmarks/nvector/openmp/CMakeLists.txt | 7 +++ benchmarks/nvector/openmpdev/CMakeLists.txt | 7 +++ benchmarks/nvector/parallel/CMakeLists.txt | 6 ++ benchmarks/nvector/parhyp/CMakeLists.txt | 7 +++ benchmarks/nvector/petsc/CMakeLists.txt | 7 +++ benchmarks/nvector/pthreads/CMakeLists.txt | 6 ++ benchmarks/nvector/raja/CMakeLists.txt | 6 ++ benchmarks/nvector/serial/CMakeLists.txt | 6 ++ 14 files changed, 161 insertions(+) diff --git a/benchmarks/advection_reaction_3D/CMakeLists.txt b/benchmarks/advection_reaction_3D/CMakeLists.txt index e51a95155a..27b3ec8e79 100644 --- a/benchmarks/advection_reaction_3D/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/CMakeLists.txt @@ -56,6 +56,25 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(FILES README.md DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") + set(BENCHMARK_VAR + "--method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save\;arkimex_tlnewton" + "--method ARK-DIRK --nls newton --tf 10.0 --dont-save\;arkdirk_newton" + "--method CV-BDF --nls newton --tf 10.0 --dont-save\;cvbdf_newton" + "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton" + ) + + # NUM_NODES based on Lassen system for 1 node + foreach(benchmark_tuple ${BENCHMARK_VAR}) + list(GET benchmark_tuple 0 benchmark_args) + list(GET benchmark_tuple 1 identifier) + + sundials_add_benchmark(advection_reaction_3D advection_reaction_3D + NUM_NODES 40 + BENCHMARK_ARGS ${benchmark_args} + IDENTIFIER ${identifier} + ) + endforeach() + # ---------------------------------------------------------------------------- # MPI + CUDA # ---------------------------------------------------------------------------- @@ -102,6 +121,25 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(TARGETS advection_reaction_3D_mpicuda DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") + set(BENCHMARK_VAR + "--method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save\;arkimex_tlnewton_cuda" + "--method ARK-DIRK --nls newton --tf 10.0 --dont-save\;arkdirk_newton_cuda" + "--method CV-BDF --nls newton --tf 10.0 --dont-save\;cvbdf_newton_cuda" + "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton_cuda" + ) + # NUM_NODES based on Lassen system for 1 node + foreach(benchmark_tuple ${BENCHMARK_VAR}) + list(GET benchmark_tuple 0 benchmark_args) + list(GET benchmark_tuple 1 identifier) + + sundials_add_benchmark(advection_reaction_3D_mpicuda advection_reaction_3D_mpicuda + ENABLE_GPU + NUM_NODES 4 + BENCHMARK_ARGS ${benchmark_args} + IDENTIFIER ${identifier} + ) + endforeach() + endif() # ---------------------------------------------------------------------------- @@ -109,6 +147,7 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) # ---------------------------------------------------------------------------- if(BUILD_NVECTOR_HIP) + add_dependencies(benchmark advection_reaction_3D_mpihip) add_executable(advection_reaction_3D_mpihip advection_reaction_3D.cpp @@ -141,6 +180,26 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(TARGETS advection_reaction_3D_mpihip DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") + # NUM_NODES based on Corona system for 1 node + set(BENCHMARK_VAR + "--method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save\;arkimex_tlnewton_hip" + "--method ARK-DIRK --nls newton --tf 10.0 --dont-save\;arkdirk_newton_hip" + "--method CV-BDF --nls newton --tf 10.0 --dont-save\;cvbdf_newton_hip" + "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton_hip" + ) + + + foreach(benchmark_tuple ${BENCHMARK_VAR}) + list(GET benchmark_tuple 0 benchmark_args) + list(GET benchmark_tuple 1 identifier) + + sundials_add_benchmark(advection_reaction_3D_mpihip advection_reaction_3D_mpihip + ENABLE_GPU + NUM_NODES 8 + BENCHMARK_ARGS ${benchmark_args} + IDENTIFIER ${identifier} + ) + endforeach() endif() endif() diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index 725b91d93f..52cc60150a 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -74,6 +74,28 @@ foreach(test_tuple ${tests}) add_executable(${target} ${sources}) + if("${backend}" STREQUAL "USE_CUDA") + + # based on Lassen system from build_and_bench (1 nodes). + # sundials_add_benchmark(${target} ${target} + # NUM_NODES 4 + # ENABLE_GPU + # IDENTIFIER "d2d_${package}_cuda" + # ) + + endif() + + if("${backend}" STREQUAL "USE_HIP") + + # based on the Corona system for 1 node + sundials_add_benchmark(${target} ${target} + NUM_NODES 8 + ENABLE_GPU + IDENTIFIER "d2d_${package}_hip" + ) + + endif() + set_target_properties(${target} PROPERTIES FOLDER "Benchmarks") target_compile_definitions(${target} PRIVATE ${problem_type}) diff --git a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt index 304a2d8512..4d19fcf697 100644 --- a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt @@ -48,6 +48,8 @@ foreach(test_tuple ${tests}) # create executable add_executable(${target} ${sources}) + add_dependencies(benchmark ${target}) + set_target_properties(${target} PROPERTIES FOLDER "Benchmarks") target_compile_definitions(${target} PRIVATE ${problem_type}) @@ -71,4 +73,10 @@ foreach(test_tuple ${tests}) install(TARGETS ${target} DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") + # NUM_NODES based 1 node for testing purposes + sundials_add_benchmark(${target} ${target} + NUM_NODES 40 + IDENTIFIER "d2d_${package}_serial" + ) + endforeach() diff --git a/benchmarks/nvector/cuda/CMakeLists.txt b/benchmarks/nvector/cuda/CMakeLists.txt index ee1366befc..3b2d4ceaac 100644 --- a/benchmarks/nvector/cuda/CMakeLists.txt +++ b/benchmarks/nvector/cuda/CMakeLists.txt @@ -19,3 +19,10 @@ sundials_add_nvector_benchmark(nvector_cuda_benchmark SUNDIALS_TARGETS sundials_nveccuda INSTALL_SUBDIR nvector/cuda ) + +# NUM_NODES based on 1 lassen node +sundials_add_benchmark(nvector_cuda_benchmark nvector_cuda_benchmark + NUM_NODES 4 + ENABLE_GPU + BENCHMARK_ARGS "10 10 10 10 10 10" + ) diff --git a/benchmarks/nvector/hip/CMakeLists.txt b/benchmarks/nvector/hip/CMakeLists.txt index 35ab748cf8..01ac8b968e 100644 --- a/benchmarks/nvector/hip/CMakeLists.txt +++ b/benchmarks/nvector/hip/CMakeLists.txt @@ -19,3 +19,10 @@ sundials_add_nvector_benchmark(nvector_hip_benchmark SUNDIALS_TARGETS sundials_nvechip INSTALL_SUBDIR nvector/hip ) + +# NUM_NODES based on Corona system for 1 node +sundials_add_benchmark(nvector_serial_benchmark nvector_serial_benchmark + NUM_NODES 8 + ENABLE_GPU + BENCHMARK_ARGS "10 10 10 10 10 10" + ) diff --git a/benchmarks/nvector/mpiplusx/CMakeLists.txt b/benchmarks/nvector/mpiplusx/CMakeLists.txt index 1018a98ec4..129d557d23 100644 --- a/benchmarks/nvector/mpiplusx/CMakeLists.txt +++ b/benchmarks/nvector/mpiplusx/CMakeLists.txt @@ -20,3 +20,9 @@ sundials_add_nvector_benchmark(nvector_mpiplusx_benchmark LINK_LIBRARIES MPI::MPI_CXX INSTALL_SUBDIR nvector/mpiplusx ) + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(nvector_mpiplusx_benchmark nvector_mpiplusx_benchmark + NUM_NODES 40 + BENCHMARK_ARGS "10 10 10 10 10 10" + ) \ No newline at end of file diff --git a/benchmarks/nvector/openmp/CMakeLists.txt b/benchmarks/nvector/openmp/CMakeLists.txt index 157c2d84e4..bf819ad3c5 100644 --- a/benchmarks/nvector/openmp/CMakeLists.txt +++ b/benchmarks/nvector/openmp/CMakeLists.txt @@ -26,3 +26,10 @@ sundials_add_nvector_benchmark(nvector_openmp_benchmark SUNDIALS_TARGETS sundials_nvecopenmp INSTALL_SUBDIR nvector/openmp ) + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(nvector_openmp_benchmark nvector_openmp_benchmark + NUM_NODES 1 + BENCHMARK_VAR + ";" + ) diff --git a/benchmarks/nvector/openmpdev/CMakeLists.txt b/benchmarks/nvector/openmpdev/CMakeLists.txt index e7a4b46d51..757b32ac90 100644 --- a/benchmarks/nvector/openmpdev/CMakeLists.txt +++ b/benchmarks/nvector/openmpdev/CMakeLists.txt @@ -37,3 +37,10 @@ target_link_libraries(test_nvector_performance_openmpdev ${SUNDIALS_LIBS}) install(TARGETS test_nvector_performance_openmpdev DESTINATION "${BENCHMARKS_INSTALL_PATH}/nvector/openmpdev") + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(test_nvector_performance_openmpdev test_nvector_performance_openmpdev + NUM_NODES 1 + BENCHMARK_VAR + ";" + ) \ No newline at end of file diff --git a/benchmarks/nvector/parallel/CMakeLists.txt b/benchmarks/nvector/parallel/CMakeLists.txt index 0b8f7e33a8..88eed8a401 100644 --- a/benchmarks/nvector/parallel/CMakeLists.txt +++ b/benchmarks/nvector/parallel/CMakeLists.txt @@ -20,3 +20,9 @@ sundials_add_nvector_benchmark(nvector_parallel_benchmark LINK_LIBRARIES MPI::MPI_CXX INSTALL_SUBDIR nvector/parallel ) + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(nvector_parallel_benchmark nvector_parallel_benchmark + NUM_NODES 40 + BENCHMARK_ARGS "10 10 10 10 10 10" + ) \ No newline at end of file diff --git a/benchmarks/nvector/parhyp/CMakeLists.txt b/benchmarks/nvector/parhyp/CMakeLists.txt index 67b74787b6..b6d70b8c45 100644 --- a/benchmarks/nvector/parhyp/CMakeLists.txt +++ b/benchmarks/nvector/parhyp/CMakeLists.txt @@ -22,3 +22,10 @@ sundials_add_nvector_benchmark(nvector_parhyp_benchmark LINK_LIBRARIES MPI::MPI_C SUNDIALS::HYPRE INSTALL_SUBDIR nvector/parhyp ) + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(nvector_parhyp_benchmark nvector_parhyp_benchmark + NUM_NODES 40 + BENCHMARK_VAR + ";" + ) \ No newline at end of file diff --git a/benchmarks/nvector/petsc/CMakeLists.txt b/benchmarks/nvector/petsc/CMakeLists.txt index d4c501fec7..dddd7080be 100644 --- a/benchmarks/nvector/petsc/CMakeLists.txt +++ b/benchmarks/nvector/petsc/CMakeLists.txt @@ -22,3 +22,10 @@ sundials_add_nvector_benchmark(nvector_petsc_benchmark LINK_LIBRARIES MPI::MPI_C PUBLIC SUNDIALS::PETSC INSTALL_SUBDIR nvector/petsc ) + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(nvector_petsc_benchmark nvector_petsc_benchmark + NUM_NODES 40 + BENCHMARK_VAR + ";" + ) \ No newline at end of file diff --git a/benchmarks/nvector/pthreads/CMakeLists.txt b/benchmarks/nvector/pthreads/CMakeLists.txt index b1aa4d2ad0..22b3b67e03 100644 --- a/benchmarks/nvector/pthreads/CMakeLists.txt +++ b/benchmarks/nvector/pthreads/CMakeLists.txt @@ -22,3 +22,9 @@ sundials_add_nvector_benchmark(nvector_pthreads_benchmark LINK_LIBRARIES Threads::Threads INSTALL_SUBDIR nvector/pthreads ) + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(nvector_pthreads_benchmark nvector_pthreads_benchmark + NUM_NODES 40 + BENCHMARK_ARGS "10 10 10 10 10 10" + ) \ No newline at end of file diff --git a/benchmarks/nvector/raja/CMakeLists.txt b/benchmarks/nvector/raja/CMakeLists.txt index 3625506fd8..b8b06969b7 100644 --- a/benchmarks/nvector/raja/CMakeLists.txt +++ b/benchmarks/nvector/raja/CMakeLists.txt @@ -41,6 +41,12 @@ foreach(backend ${SUNDIALS_RAJA_BACKENDS}) SOURCES test_nvector_performance_raja.cpp SUNDIALS_TARGETS sundials_nvecraja ) + + # sundials_add_benchmark(${example_target} ${example_target} + # NUM_NODES 4 + # ENABLE_GPU + # BENCHMARK_ARGS "10 10 10 10 10 10" + # ) target_compile_definitions(${example_target} PRIVATE ${_defines}) diff --git a/benchmarks/nvector/serial/CMakeLists.txt b/benchmarks/nvector/serial/CMakeLists.txt index 1f339c1750..b5d60fa40b 100644 --- a/benchmarks/nvector/serial/CMakeLists.txt +++ b/benchmarks/nvector/serial/CMakeLists.txt @@ -21,3 +21,9 @@ sundials_add_nvector_benchmark(nvector_serial_benchmark SUNDIALS_TARGETS sundials_nvecserial INSTALL_SUBDIR nvector/serial ) + +# NUM_NODES based on Lassen system for 1 node +sundials_add_benchmark(nvector_serial_benchmark nvector_serial_benchmark + NUM_NODES 40 + BENCHMARK_ARGS "10 10 10 10 10 10" + ) \ No newline at end of file From ed35ddc8625d8acc822746ebba3e52ef22d835b9 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Apr 2023 14:46:16 -0700 Subject: [PATCH 076/187] add benchmark macro to nvector raja --- benchmarks/nvector/raja/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/nvector/raja/CMakeLists.txt b/benchmarks/nvector/raja/CMakeLists.txt index b8b06969b7..a915850dfc 100644 --- a/benchmarks/nvector/raja/CMakeLists.txt +++ b/benchmarks/nvector/raja/CMakeLists.txt @@ -42,11 +42,11 @@ foreach(backend ${SUNDIALS_RAJA_BACKENDS}) SUNDIALS_TARGETS sundials_nvecraja ) - # sundials_add_benchmark(${example_target} ${example_target} - # NUM_NODES 4 - # ENABLE_GPU - # BENCHMARK_ARGS "10 10 10 10 10 10" - # ) + sundials_add_benchmark(${example_target} ${example_target} + NUM_NODES 4 + ENABLE_GPU + BENCHMARK_ARGS "10 10 10 10 10 10" + ) target_compile_definitions(${example_target} PRIVATE ${_defines}) From 8b487b64c9fd90cde9078387d196614459a91504 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Apr 2023 14:46:37 -0700 Subject: [PATCH 077/187] separate cali files by benchmark and parameters --- cmake/macros/SundialsAddBenchmark.cmake | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index e776d67b20..971377af64 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -21,14 +21,6 @@ macro(sundials_add_benchmark NAME EXECUTABLE) cmake_parse_arguments(sundials_add_benchmark "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - # command line arguments for the test runner script - set(TEST_RUNNER_ARGS - "--verbose" - "--executablename=$" - "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}" - "--nodiff" - ) - # set the target name if(sundials_add_benchmark_IDENTIFIER) set(TARGET_NAME ${NAME}_${sundials_add_benchmark_IDENTIFIER}) @@ -41,6 +33,19 @@ macro(sundials_add_benchmark NAME EXECUTABLE) endif() endif() + # Make the output directory if it doesn't exist + if(NOT EXISTS ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) + file(MAKE_DIRECTORY ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) + endif() + + # command line arguments for the test runner script + set(TEST_RUNNER_ARGS + "--verbose" + "--executablename=$" + "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" + "--nodiff" + ) + # incorporate scheduler arguments into test_runner if(SUNDIALS_SCHEDULER_COMMAND STREQUAL "flux run") set(SCHEDULER_STRING " -nnodes=${sundials_add_benchmark_NUM_NODES}") From 098c37defc44a087ec4e0dac0e4f4b20aff06df2 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Apr 2023 14:52:07 -0700 Subject: [PATCH 078/187] add benchmark macro to rest of nvector benchmarks --- benchmarks/nvector/kokkos/CMakeLists.txt | 6 ++++++ benchmarks/nvector/openmp/CMakeLists.txt | 5 ++--- benchmarks/nvector/openmpdev/CMakeLists.txt | 5 ++--- benchmarks/nvector/parhyp/CMakeLists.txt | 3 +-- benchmarks/nvector/petsc/CMakeLists.txt | 3 +-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/benchmarks/nvector/kokkos/CMakeLists.txt b/benchmarks/nvector/kokkos/CMakeLists.txt index 5c48b04011..0d275f0eea 100644 --- a/benchmarks/nvector/kokkos/CMakeLists.txt +++ b/benchmarks/nvector/kokkos/CMakeLists.txt @@ -20,6 +20,12 @@ foreach(backend ${KOKKOS_EXAMPLES_BACKENDS}) INSTALL_SUBDIR nvector/kokkos ) + sundials_add_benchmark(test_nvector_performance_kokkos.${backend} test_nvector_performance_kokkos.${backend} + NUM_NODES 4 + ENABLE_GPU + BENCHMARK_ARGS "10 10 10 10 10 10" + ) + target_compile_definitions(test_nvector_performance_kokkos.${backend} PRIVATE USE_${backend}) install(TARGETS test_nvector_performance_kokkos.${backend} diff --git a/benchmarks/nvector/openmp/CMakeLists.txt b/benchmarks/nvector/openmp/CMakeLists.txt index bf819ad3c5..66ed523b6f 100644 --- a/benchmarks/nvector/openmp/CMakeLists.txt +++ b/benchmarks/nvector/openmp/CMakeLists.txt @@ -29,7 +29,6 @@ sundials_add_nvector_benchmark(nvector_openmp_benchmark # NUM_NODES based on Lassen system for 1 node sundials_add_benchmark(nvector_openmp_benchmark nvector_openmp_benchmark - NUM_NODES 1 - BENCHMARK_VAR - ";" + NUM_NODES 40 + BENCHMARK_ARGS "10 10 10 10 10 10" ) diff --git a/benchmarks/nvector/openmpdev/CMakeLists.txt b/benchmarks/nvector/openmpdev/CMakeLists.txt index 757b32ac90..ed9a213ed8 100644 --- a/benchmarks/nvector/openmpdev/CMakeLists.txt +++ b/benchmarks/nvector/openmpdev/CMakeLists.txt @@ -40,7 +40,6 @@ install(TARGETS test_nvector_performance_openmpdev # NUM_NODES based on Lassen system for 1 node sundials_add_benchmark(test_nvector_performance_openmpdev test_nvector_performance_openmpdev - NUM_NODES 1 - BENCHMARK_VAR - ";" + NUM_NODES 40 + BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/parhyp/CMakeLists.txt b/benchmarks/nvector/parhyp/CMakeLists.txt index b6d70b8c45..c0728305d3 100644 --- a/benchmarks/nvector/parhyp/CMakeLists.txt +++ b/benchmarks/nvector/parhyp/CMakeLists.txt @@ -26,6 +26,5 @@ sundials_add_nvector_benchmark(nvector_parhyp_benchmark # NUM_NODES based on Lassen system for 1 node sundials_add_benchmark(nvector_parhyp_benchmark nvector_parhyp_benchmark NUM_NODES 40 - BENCHMARK_VAR - ";" + BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/petsc/CMakeLists.txt b/benchmarks/nvector/petsc/CMakeLists.txt index dddd7080be..73cda72f33 100644 --- a/benchmarks/nvector/petsc/CMakeLists.txt +++ b/benchmarks/nvector/petsc/CMakeLists.txt @@ -26,6 +26,5 @@ sundials_add_nvector_benchmark(nvector_petsc_benchmark # NUM_NODES based on Lassen system for 1 node sundials_add_benchmark(nvector_petsc_benchmark nvector_petsc_benchmark NUM_NODES 40 - BENCHMARK_VAR - ";" + BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file From 417ec4dbbc6371124e635b6cc262d72d48381e68 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 28 Apr 2023 12:58:15 -0700 Subject: [PATCH 079/187] change NUM_NODES to NUM_CORES for clarity --- benchmarks/advection_reaction_3D/CMakeLists.txt | 12 ++++++------ benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt | 4 ++-- .../diffusion_2D/mpi_serial/CMakeLists.txt | 4 ++-- benchmarks/nvector/cuda/CMakeLists.txt | 4 ++-- benchmarks/nvector/hip/CMakeLists.txt | 6 +++--- benchmarks/nvector/kokkos/CMakeLists.txt | 2 +- benchmarks/nvector/mpiplusx/CMakeLists.txt | 4 ++-- benchmarks/nvector/openmp/CMakeLists.txt | 4 ++-- benchmarks/nvector/openmpdev/CMakeLists.txt | 4 ++-- benchmarks/nvector/parallel/CMakeLists.txt | 4 ++-- benchmarks/nvector/parhyp/CMakeLists.txt | 4 ++-- benchmarks/nvector/petsc/CMakeLists.txt | 4 ++-- benchmarks/nvector/pthreads/CMakeLists.txt | 4 ++-- benchmarks/nvector/raja/CMakeLists.txt | 2 +- benchmarks/nvector/serial/CMakeLists.txt | 4 ++-- cmake/macros/SundialsAddBenchmark.cmake | 17 +++++++++-------- 16 files changed, 42 insertions(+), 41 deletions(-) diff --git a/benchmarks/advection_reaction_3D/CMakeLists.txt b/benchmarks/advection_reaction_3D/CMakeLists.txt index 27b3ec8e79..39d6d11e87 100644 --- a/benchmarks/advection_reaction_3D/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/CMakeLists.txt @@ -63,13 +63,13 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton" ) - # NUM_NODES based on Lassen system for 1 node + # NUM_CORES based on Lassen system for 1 node foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) sundials_add_benchmark(advection_reaction_3D advection_reaction_3D - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier} ) @@ -127,14 +127,14 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) "--method CV-BDF --nls newton --tf 10.0 --dont-save\;cvbdf_newton_cuda" "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton_cuda" ) - # NUM_NODES based on Lassen system for 1 node + # NUM_CORES based on Lassen system for 1 node foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) sundials_add_benchmark(advection_reaction_3D_mpicuda advection_reaction_3D_mpicuda ENABLE_GPU - NUM_NODES 4 + NUM_CORES 4 BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier} ) @@ -180,7 +180,6 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(TARGETS advection_reaction_3D_mpihip DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") - # NUM_NODES based on Corona system for 1 node set(BENCHMARK_VAR "--method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save\;arkimex_tlnewton_hip" "--method ARK-DIRK --nls newton --tf 10.0 --dont-save\;arkdirk_newton_hip" @@ -193,9 +192,10 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) + # NUM_CORES based on Corona system for 1 node sundials_add_benchmark(advection_reaction_3D_mpihip advection_reaction_3D_mpihip ENABLE_GPU - NUM_NODES 8 + NUM_CORES 8 BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier} ) diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index 52cc60150a..23b6a97583 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -78,7 +78,7 @@ foreach(test_tuple ${tests}) # based on Lassen system from build_and_bench (1 nodes). # sundials_add_benchmark(${target} ${target} - # NUM_NODES 4 + # NUM_CORES 2 # ENABLE_GPU # IDENTIFIER "d2d_${package}_cuda" # ) @@ -89,7 +89,7 @@ foreach(test_tuple ${tests}) # based on the Corona system for 1 node sundials_add_benchmark(${target} ${target} - NUM_NODES 8 + NUM_CORES 8 ENABLE_GPU IDENTIFIER "d2d_${package}_hip" ) diff --git a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt index 4d19fcf697..2d5d429fa6 100644 --- a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt @@ -73,9 +73,9 @@ foreach(test_tuple ${tests}) install(TARGETS ${target} DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") - # NUM_NODES based 1 node for testing purposes + # NUM_CORES based 1 node for testing purposes sundials_add_benchmark(${target} ${target} - NUM_NODES 40 + NUM_CORES 32 IDENTIFIER "d2d_${package}_serial" ) diff --git a/benchmarks/nvector/cuda/CMakeLists.txt b/benchmarks/nvector/cuda/CMakeLists.txt index 3b2d4ceaac..48571d0203 100644 --- a/benchmarks/nvector/cuda/CMakeLists.txt +++ b/benchmarks/nvector/cuda/CMakeLists.txt @@ -20,9 +20,9 @@ sundials_add_nvector_benchmark(nvector_cuda_benchmark INSTALL_SUBDIR nvector/cuda ) -# NUM_NODES based on 1 lassen node +# NUM_CORES based on 1 lassen node sundials_add_benchmark(nvector_cuda_benchmark nvector_cuda_benchmark - NUM_NODES 4 + NUM_CORES 4 ENABLE_GPU BENCHMARK_ARGS "10 10 10 10 10 10" ) diff --git a/benchmarks/nvector/hip/CMakeLists.txt b/benchmarks/nvector/hip/CMakeLists.txt index 01ac8b968e..6bbfeff68e 100644 --- a/benchmarks/nvector/hip/CMakeLists.txt +++ b/benchmarks/nvector/hip/CMakeLists.txt @@ -20,9 +20,9 @@ sundials_add_nvector_benchmark(nvector_hip_benchmark INSTALL_SUBDIR nvector/hip ) -# NUM_NODES based on Corona system for 1 node -sundials_add_benchmark(nvector_serial_benchmark nvector_serial_benchmark - NUM_NODES 8 +# NUM_CORES based on Corona system for 1 node +sundials_add_benchmark(nvector_hip_benchmark nvector_hip_benchmark + NUM_CORES 8 ENABLE_GPU BENCHMARK_ARGS "10 10 10 10 10 10" ) diff --git a/benchmarks/nvector/kokkos/CMakeLists.txt b/benchmarks/nvector/kokkos/CMakeLists.txt index 0d275f0eea..acb68c378b 100644 --- a/benchmarks/nvector/kokkos/CMakeLists.txt +++ b/benchmarks/nvector/kokkos/CMakeLists.txt @@ -21,7 +21,7 @@ foreach(backend ${KOKKOS_EXAMPLES_BACKENDS}) ) sundials_add_benchmark(test_nvector_performance_kokkos.${backend} test_nvector_performance_kokkos.${backend} - NUM_NODES 4 + NUM_CORES 4 ENABLE_GPU BENCHMARK_ARGS "10 10 10 10 10 10" ) diff --git a/benchmarks/nvector/mpiplusx/CMakeLists.txt b/benchmarks/nvector/mpiplusx/CMakeLists.txt index 129d557d23..dfe1d6de2c 100644 --- a/benchmarks/nvector/mpiplusx/CMakeLists.txt +++ b/benchmarks/nvector/mpiplusx/CMakeLists.txt @@ -21,8 +21,8 @@ sundials_add_nvector_benchmark(nvector_mpiplusx_benchmark INSTALL_SUBDIR nvector/mpiplusx ) -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(nvector_mpiplusx_benchmark nvector_mpiplusx_benchmark - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/openmp/CMakeLists.txt b/benchmarks/nvector/openmp/CMakeLists.txt index 66ed523b6f..9c90eab9b8 100644 --- a/benchmarks/nvector/openmp/CMakeLists.txt +++ b/benchmarks/nvector/openmp/CMakeLists.txt @@ -27,8 +27,8 @@ sundials_add_nvector_benchmark(nvector_openmp_benchmark INSTALL_SUBDIR nvector/openmp ) -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(nvector_openmp_benchmark nvector_openmp_benchmark - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) diff --git a/benchmarks/nvector/openmpdev/CMakeLists.txt b/benchmarks/nvector/openmpdev/CMakeLists.txt index ed9a213ed8..9d48eb65f7 100644 --- a/benchmarks/nvector/openmpdev/CMakeLists.txt +++ b/benchmarks/nvector/openmpdev/CMakeLists.txt @@ -38,8 +38,8 @@ target_link_libraries(test_nvector_performance_openmpdev ${SUNDIALS_LIBS}) install(TARGETS test_nvector_performance_openmpdev DESTINATION "${BENCHMARKS_INSTALL_PATH}/nvector/openmpdev") -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(test_nvector_performance_openmpdev test_nvector_performance_openmpdev - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/parallel/CMakeLists.txt b/benchmarks/nvector/parallel/CMakeLists.txt index 88eed8a401..1e583603a6 100644 --- a/benchmarks/nvector/parallel/CMakeLists.txt +++ b/benchmarks/nvector/parallel/CMakeLists.txt @@ -21,8 +21,8 @@ sundials_add_nvector_benchmark(nvector_parallel_benchmark INSTALL_SUBDIR nvector/parallel ) -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(nvector_parallel_benchmark nvector_parallel_benchmark - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/parhyp/CMakeLists.txt b/benchmarks/nvector/parhyp/CMakeLists.txt index c0728305d3..9b3017f79d 100644 --- a/benchmarks/nvector/parhyp/CMakeLists.txt +++ b/benchmarks/nvector/parhyp/CMakeLists.txt @@ -23,8 +23,8 @@ sundials_add_nvector_benchmark(nvector_parhyp_benchmark INSTALL_SUBDIR nvector/parhyp ) -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(nvector_parhyp_benchmark nvector_parhyp_benchmark - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/petsc/CMakeLists.txt b/benchmarks/nvector/petsc/CMakeLists.txt index 73cda72f33..b7dcc9c5fc 100644 --- a/benchmarks/nvector/petsc/CMakeLists.txt +++ b/benchmarks/nvector/petsc/CMakeLists.txt @@ -23,8 +23,8 @@ sundials_add_nvector_benchmark(nvector_petsc_benchmark INSTALL_SUBDIR nvector/petsc ) -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(nvector_petsc_benchmark nvector_petsc_benchmark - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/pthreads/CMakeLists.txt b/benchmarks/nvector/pthreads/CMakeLists.txt index 22b3b67e03..4475b0e5f4 100644 --- a/benchmarks/nvector/pthreads/CMakeLists.txt +++ b/benchmarks/nvector/pthreads/CMakeLists.txt @@ -23,8 +23,8 @@ sundials_add_nvector_benchmark(nvector_pthreads_benchmark INSTALL_SUBDIR nvector/pthreads ) -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(nvector_pthreads_benchmark nvector_pthreads_benchmark - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/raja/CMakeLists.txt b/benchmarks/nvector/raja/CMakeLists.txt index a915850dfc..7948a0cb72 100644 --- a/benchmarks/nvector/raja/CMakeLists.txt +++ b/benchmarks/nvector/raja/CMakeLists.txt @@ -43,7 +43,7 @@ foreach(backend ${SUNDIALS_RAJA_BACKENDS}) ) sundials_add_benchmark(${example_target} ${example_target} - NUM_NODES 4 + NUM_CORES 4 ENABLE_GPU BENCHMARK_ARGS "10 10 10 10 10 10" ) diff --git a/benchmarks/nvector/serial/CMakeLists.txt b/benchmarks/nvector/serial/CMakeLists.txt index b5d60fa40b..96e0d1319f 100644 --- a/benchmarks/nvector/serial/CMakeLists.txt +++ b/benchmarks/nvector/serial/CMakeLists.txt @@ -22,8 +22,8 @@ sundials_add_nvector_benchmark(nvector_serial_benchmark INSTALL_SUBDIR nvector/serial ) -# NUM_NODES based on Lassen system for 1 node +# NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(nvector_serial_benchmark nvector_serial_benchmark - NUM_NODES 40 + NUM_CORES 40 BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index 971377af64..5fa131a82a 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -7,10 +7,10 @@ macro(sundials_add_benchmark NAME EXECUTABLE) # Define single value parameters the macro takes in to set up the test runner # - # NUM_NODES = number of nodes (GPU count or CPU count) to run on or number of resource sets + # NUM_CORES = number of cores (GPU count or CPU count) to run on or number of resource sets # BENCHMARK_ARGS = arguments to pass to the executable # IDENTIFIER = suffix to append to end of benchmark name - set(oneValueArgs NUM_NODES BENCHMARK_ARGS IDENTIFIER) + set(oneValueArgs NUM_CORES BENCHMARK_ARGS IDENTIFIER) # TEST_RUNNER_ARGS = command line arguments to pass to the test executable set(multiValueArgs TEST_RUNNER_ARGS ) @@ -45,22 +45,23 @@ macro(sundials_add_benchmark NAME EXECUTABLE) "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" "--nodiff" ) - + # incorporate scheduler arguments into test_runner if(SUNDIALS_SCHEDULER_COMMAND STREQUAL "flux run") - set(SCHEDULER_STRING " -nnodes=${sundials_add_benchmark_NUM_NODES}") + set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES}") elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun" AND ${sundials_add_benchmark_ENABLE_GPU}) - set(SCHEDULER_STRING " --smpiargs=\\\"-gpu\\\" -n${sundials_add_benchmark_NUM_NODES} -a1 -c1 -g1") + set(SCHEDULER_STRING " --smpiargs=\\\"-gpu\\\" -n${sundials_add_benchmark_NUM_CORES} -a1 -c1 -g1") elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun") - set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_NODES} -a1 -c1") + set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES} -a1 -c1") elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "srun") - set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_NODES} --cpus-per-task=1 --ntasks-per-node=1") + set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES} --cpus-per-task=1 --ntasks-per-node=1") endif() string(REPLACE " " ";" SCHEDULER_ARGS "${SCHEDULER_STRING}") + string(REPLACE " " ";" SCHEDULER_COMMAND_ARGS "${SUNDIALS_SCHEDULER_COMMAND}") # taken from SundialsAddTest string(STRIP "${RUN_COMMAND}" RUN_COMMAND) - set(RUN_COMMAND ${SUNDIALS_SCHEDULER_COMMAND} ${SCHEDULER_ARGS}) + set(RUN_COMMAND ${SCHEDULER_COMMAND_ARGS} ${SCHEDULER_ARGS}) list(APPEND TEST_RUNNER_ARGS "--runcommand=\"${RUN_COMMAND}\"") # enable test runner to set up caliper output paths and configs From 82a5f9705c844509b6e49b4a484f0ce43ad1bd22 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 28 Apr 2023 12:59:16 -0700 Subject: [PATCH 080/187] add adiak option to cmake and adiak specific code --- cmake/SundialsSetupConfig.cmake | 5 +++++ cmake/SundialsTPLOptions.cmake | 7 +++++++ src/sundials/sundials_context.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 0ac1a9bf8b..c3ae7dc6dd 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -58,6 +58,11 @@ if(ENABLE_CALIPER) set(SUNDIALS_CALIPER_ENABLED TRUE) endif() +# prepare substitution variable SUNDIALS_ADIAK_ENABLED for sundials_config.h +if(ENABLE_ADIAK) + set(SUNDIALS_ADIAK_ENABLED TRUE) +endif() + # prepare substitution variable SUNDIALS_MPI_ENABLED for sundials_config.h if(ENABLE_MPI) set(SUNDIALS_MPI_ENABLED TRUE) diff --git a/cmake/SundialsTPLOptions.cmake b/cmake/SundialsTPLOptions.cmake index f01a0ac14d..9a88dc37e4 100644 --- a/cmake/SundialsTPLOptions.cmake +++ b/cmake/SundialsTPLOptions.cmake @@ -302,6 +302,13 @@ sundials_option(CALIPER_WORKS BOOL "Set to ON to force CMake to accept a given C DEPENDS_ON ENABLE_CALIPER ADVANCED) +# --------------------------------------------------------------- +# Enable Adiak support? +# --------------------------------------------------------------- + +sundials_option(ENABLE_ADIAK BOOL "Enable Adiak support" OFF DEPENDS_ON SUNDIALS_BUILD_WITH_PROFILING) + +sundials_option(adiak_DIR PATH "Path to the root of an Adiak installation" "${ADIAK_DIR}" DEPENDS_ON ENABLE_ADIAK) # --------------------------------------------------------------- # Enable Kokkos support? diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index a1efeed590..e8dcd16bfc 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -30,6 +30,10 @@ int SUNContext_Create(void* comm, SUNContext* sunctx) SUNProfiler profiler = NULL; SUNLogger logger = NULL; +#if defined(SUNDIALS_ADIAK_ENABLED) + adiak_helper(); +#endif + #if defined(SUNDIALS_BUILD_WITH_PROFILING) && !defined(SUNDIALS_CALIPER_ENABLED) if (SUNProfiler_Create(comm, "SUNContext Default", &profiler)) return (-1); #endif @@ -193,6 +197,11 @@ int SUNContext_Free(SUNContext* sunctx) } #endif +#if defined(SUNDIALS_ENABLE_ADIAK) + adiak_fini(); + adiak_clean(); +#endif + if ((*sunctx)->logger && (*sunctx)->own_logger) { SUNLogger_Destroy(&(*sunctx)->logger); @@ -203,3 +212,22 @@ int SUNContext_Free(SUNContext* sunctx) return (0); } + +#if defined(SUNDIALS_ENABLE_ADIAK) +static void adiak_helper() { + adiak_user(); + adiak_uid(); + adiak_launchdate(); + adiak_executable(); + adiak_cmdline(); + adiak_hostname(); + adiak_clustername(); + + adiak_job_size(); + adiak_hostlist(); + + adiak_walltime(); + adiak_cputime(); + adiak_systime(); +} +#endif \ No newline at end of file From 4cc8d45da41dedd4f17f2865e3a35d8586eab627 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 16 May 2023 16:00:31 -0700 Subject: [PATCH 081/187] add Adiak to Sundials --- cmake/SUNDIALSConfig.cmake.in | 4 ++++ cmake/SundialsSetupTPLs.cmake | 9 ++++++++ cmake/macros/SundialsAddExecutable.cmake | 5 ++-- include/sundials/sundials_config.in | 6 +++++ include/sundials/sundials_context.h | 2 ++ src/arkode/CMakeLists.txt | 6 +++++ src/cvode/CMakeLists.txt | 6 +++++ src/cvodes/CMakeLists.txt | 6 +++++ src/ida/CMakeLists.txt | 5 ++++ src/idas/CMakeLists.txt | 6 +++++ src/kinsol/CMakeLists.txt | 6 +++++ src/sundials/CMakeLists.txt | 6 +++++ src/sundials/sundials_context.c | 29 ++++++++++++++---------- 13 files changed, 81 insertions(+), 15 deletions(-) diff --git a/cmake/SUNDIALSConfig.cmake.in b/cmake/SUNDIALSConfig.cmake.in index 56bbe9460b..02108d7b95 100644 --- a/cmake/SUNDIALSConfig.cmake.in +++ b/cmake/SUNDIALSConfig.cmake.in @@ -63,6 +63,10 @@ if("@ENABLE_CALIPER@" AND NOT TARGET caliper) find_dependency(CALIPER PATHS "@CALIPER_DIR@") endif() +if("@ENABLE_ADIAK@" AND NOT TARGET adiak::adiak) + find_dependency(adiak PATHS "@adiak_DIR@") +endif() + if("@ENABLE_CUDA@" AND NOT (TARGET CUDA::cudart AND TARGET CUDA::cublas AND TARGET CUDA::cusparse AND TARGET CUDA::cusolver)) find_dependency(CUDAToolkit) diff --git a/cmake/SundialsSetupTPLs.cmake b/cmake/SundialsSetupTPLs.cmake index 6886a6e33a..c1187dca0d 100644 --- a/cmake/SundialsSetupTPLs.cmake +++ b/cmake/SundialsSetupTPLs.cmake @@ -50,6 +50,15 @@ endif() # Setup other TPLs (listed in alphabetical order) # --------------------------------------------------------------- +# --------------------------------------------------------------- +# Find (and test) the Adiak libraries +# --------------------------------------------------------------- + +if(ENABLE_ADIAK) + include(SundialsAdiak) + list(APPEND SUNDIALS_TPL_LIST "ADIAK") +endif() + # --------------------------------------------------------------- # Find (and test) the Caliper libraries # --------------------------------------------------------------- diff --git a/cmake/macros/SundialsAddExecutable.cmake b/cmake/macros/SundialsAddExecutable.cmake index 9988417774..cfd44e5cb9 100644 --- a/cmake/macros/SundialsAddExecutable.cmake +++ b/cmake/macros/SundialsAddExecutable.cmake @@ -38,10 +38,9 @@ macro(sundials_add_nvector_benchmark NAME) target_link_libraries(${NAME} PRIVATE ${arg_SUNDIALS_TARGETS} ${arg_LINK_LIBRARIES} -lm) - if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - target_include_directories(${NAME} PRIVATE ${caliper_INCLUDE_DIR}) - target_link_libraries(${NAME} PRIVATE caliper) + target_include_directories(${NAME} PRIVATE ${caliper_INCLUDE_DIR} ${adiak_INCLUDE_DIR}) + target_link_libraries(${NAME} PRIVATE caliper adiak::adiak) endif() install(TARGETS ${NAME} diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 545c1aebe8..09cdea24a5 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -112,6 +112,9 @@ /* Caliper */ #cmakedefine SUNDIALS_CALIPER_ENABLED +/* Adiak */ +#cmakedefine SUNDIALS_ADIAK_ENABLED + /* Ginkgo backends */ #cmakedefine SUNDIALS_GINKGO_BACKENDS_CUDA #cmakedefine SUNDIALS_GINKGO_BACKENDS_HIP @@ -143,6 +146,9 @@ #cmakedefine SUNDIALS_RAJA_BACKENDS_HIP #cmakedefine SUNDIALS_RAJA_BACKENDS_SYCL +/* RAJA version number */ +#define RAJA_VERSION @RAJA_PACKAGE_VERSION@ + /* ------------------------------------------------------------------ * SUNDIALS modules enabled * -----------------------------------------------------------------*/ diff --git a/include/sundials/sundials_context.h b/include/sundials/sundials_context.h index a826862dfb..caff5f03ce 100644 --- a/include/sundials/sundials_context.h +++ b/include/sundials/sundials_context.h @@ -36,6 +36,8 @@ SUNDIALS_EXPORT int SUNContext_GetLogger(SUNContext sunctx, SUNLogger* logger); SUNDIALS_EXPORT int SUNContext_SetLogger(SUNContext sunctx, SUNLogger logger); SUNDIALS_EXPORT int SUNContext_Free(SUNContext* ctx); +static void sun_adiak_helper(); + #ifdef __cplusplus } diff --git a/src/arkode/CMakeLists.txt b/src/arkode/CMakeLists.txt index 530ac098cd..2e28e9df29 100644 --- a/src/arkode/CMakeLists.txt +++ b/src/arkode/CMakeLists.txt @@ -60,6 +60,10 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/arkode/ arkode_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) + if(ENABLE_ADIAK) + set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + endif() endif() # Create the sundials_arkode library @@ -72,6 +76,7 @@ sundials_add_library(sundials_arkode arkode INCLUDE_DIRECTORIES ${_include_caliper_if_needed} + ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj @@ -91,6 +96,7 @@ sundials_add_library(sundials_arkode LINK_LIBRARIES # Link to caliper if conditions are met. ${_link_caliper_if_needed} + ${_link_adiak_if_needed} OUTPUT_NAME sundials_arkode VERSION diff --git a/src/cvode/CMakeLists.txt b/src/cvode/CMakeLists.txt index 656c432bb7..48f96e8cda 100644 --- a/src/cvode/CMakeLists.txt +++ b/src/cvode/CMakeLists.txt @@ -104,6 +104,10 @@ endif() if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) + if(ENABLE_ADIAK) + set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + endif() endif() # Create the library @@ -116,6 +120,7 @@ sundials_add_library(sundials_cvode cvode INCLUDE_DIRECTORIES ${_include_caliper_if_needed} + ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj @@ -137,6 +142,7 @@ sundials_add_library(sundials_cvode PRIVATE ${_fused_link_lib} # Link to caliper if conditions are met. ${_link_caliper_if_needed} + ${_link_adiak_if_needed} OUTPUT_NAME sundials_cvode VERSION diff --git a/src/cvodes/CMakeLists.txt b/src/cvodes/CMakeLists.txt index 160bc0755b..9cbbebc8d2 100644 --- a/src/cvodes/CMakeLists.txt +++ b/src/cvodes/CMakeLists.txt @@ -54,6 +54,10 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/cvodes/ cvodes_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) + if(ENABLE_ADIAK) + set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + endif() endif() # Create the library @@ -66,6 +70,7 @@ sundials_add_library(sundials_cvodes cvodes INCLUDE_DIRECTORIES ${_include_caliper_if_needed} + ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj @@ -85,6 +90,7 @@ sundials_add_library(sundials_cvodes LINK_LIBRARIES # Link to caliper if conditions are met. ${_link_caliper_if_needed} + ${_link_adiak_if_needed} OUTPUT_NAME sundials_cvodes VERSION diff --git a/src/ida/CMakeLists.txt b/src/ida/CMakeLists.txt index 3d88513fdd..ac5aea7ee0 100644 --- a/src/ida/CMakeLists.txt +++ b/src/ida/CMakeLists.txt @@ -44,6 +44,10 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/ida/ ida_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) + if(ENABLE_ADIAK) + set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + endif() endif() # Create the library @@ -56,6 +60,7 @@ sundials_add_library(sundials_ida ida INCLUDE_DIRECTORIES ${_include_caliper_if_needed} + ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj diff --git a/src/idas/CMakeLists.txt b/src/idas/CMakeLists.txt index a316b06c08..7a8765c6e0 100644 --- a/src/idas/CMakeLists.txt +++ b/src/idas/CMakeLists.txt @@ -48,6 +48,10 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/idas/ idas_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) + if(ENABLE_ADIAK) + set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + endif() endif() # Create the library @@ -60,6 +64,7 @@ sundials_add_library(sundials_idas idas INCLUDE_DIRECTORIES ${_include_caliper_if_needed} + ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj @@ -79,6 +84,7 @@ sundials_add_library(sundials_idas LINK_LIBRARIES # Link to caliper if conditions are met. ${_link_caliper_if_needed} + ${_link_adiak_if_needed} OUTPUT_NAME sundials_idas VERSION diff --git a/src/kinsol/CMakeLists.txt b/src/kinsol/CMakeLists.txt index c594b5c4bc..3a41643ab0 100644 --- a/src/kinsol/CMakeLists.txt +++ b/src/kinsol/CMakeLists.txt @@ -42,6 +42,10 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/kinsol/ kinsol_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) + if(ENABLE_ADIAK) + set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRe}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + endif() endif() # Create the library @@ -54,6 +58,7 @@ sundials_add_library(sundials_kinsol kinsol INCLUDE_DIRECTORIES ${_include_caliper_if_needed} + ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj @@ -71,6 +76,7 @@ sundials_add_library(sundials_kinsol LINK_LIBRARIES # Link to caliper if conditions are met. ${_link_caliper_if_needed} + ${_link_adiak_if_needed} OUTPUT_NAME sundials_kinsol VERSION diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt index 7feb44067c..1955ff0446 100644 --- a/src/sundials/CMakeLists.txt +++ b/src/sundials/CMakeLists.txt @@ -113,6 +113,10 @@ if(SUNDIALS_BUILD_WITH_PROFILING OR SUNDIALS_LOGGING_ENABLE_MPI) if(ENABLE_CALIPER) set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) + if(ENABLE_ADIAK) + set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + endif() endif() endif() @@ -127,9 +131,11 @@ sundials_add_library(sundials_generic INCLUDE_DIRECTORIES ${_include_mpi_if_needed} ${_include_caliper_if_needed} + ${_include_adiak_if_needed} LINK_LIBRARIES ${_link_mpi_if_needed} ${_link_caliper_if_needed} + ${_link_adiak_if_needed} OUTPUT_NAME sundials_generic VERSION diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index e8dcd16bfc..3b81ceed41 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -15,6 +15,8 @@ * SUNDIALS objects in a simulation share. * ----------------------------------------------------------------*/ +#include "adiak.h" + #include #include #include @@ -29,15 +31,15 @@ int SUNContext_Create(void* comm, SUNContext* sunctx) { SUNProfiler profiler = NULL; SUNLogger logger = NULL; - -#if defined(SUNDIALS_ADIAK_ENABLED) - adiak_helper(); -#endif - #if defined(SUNDIALS_BUILD_WITH_PROFILING) && !defined(SUNDIALS_CALIPER_ENABLED) if (SUNProfiler_Create(comm, "SUNContext Default", &profiler)) return (-1); #endif +#ifdef SUNDIALS_ADIAK_ENABLED + adiak_init(comm); + sun_adiak_helper(); +#endif + #if SUNDIALS_LOGGING_LEVEL > 0 #if defined(SUNDIALS_LOGGING_ENABLE_MPI) if (SUNLogger_CreateFromEnv(comm, &logger)) @@ -197,9 +199,9 @@ int SUNContext_Free(SUNContext* sunctx) } #endif -#if defined(SUNDIALS_ENABLE_ADIAK) +#ifdef SUNDIALS_ADIAK_ENABLED + // adiak_clean(); adiak_fini(); - adiak_clean(); #endif if ((*sunctx)->logger && (*sunctx)->own_logger) @@ -213,8 +215,8 @@ int SUNContext_Free(SUNContext* sunctx) return (0); } -#if defined(SUNDIALS_ENABLE_ADIAK) -static void adiak_helper() { +#ifdef SUNDIALS_ADIAK_ENABLED +static void sun_adiak_helper() { adiak_user(); adiak_uid(); adiak_launchdate(); @@ -225,9 +227,12 @@ static void adiak_helper() { adiak_job_size(); adiak_hostlist(); + + // make the c version + adiak_namevalue("sample_variable", (adiak_category_t)(1), "import.category" "%s", "example"); - adiak_walltime(); - adiak_cputime(); - adiak_systime(); +#ifdef SUNDIALS_RAJA_INCLUDED + adiak_namevalue("raja_version", static_cast(2), "import.category", "%s", RAJA_VERSION); +#endif } #endif \ No newline at end of file From dc56b584955ada4ce41864605df90881bbc04e51 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 17 May 2023 08:55:35 -0700 Subject: [PATCH 082/187] turn off all runs for this branch --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 63622ebb08..bf5aeb3712 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,9 +77,9 @@ variables: NCPUS: 12 VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" - ON_LASSEN: "ON" - ON_QUARTZ: "ON" - ON_CORONA: "ON" + ON_LASSEN: "OFF" + ON_QUARTZ: "OFF" + ON_CORONA: "OFF" SPACK_PREFIX: "v0.19.1" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" From 7472cbfbd0005973e531393733745e8bb99dc333 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 18 May 2023 10:26:31 -0700 Subject: [PATCH 083/187] remove nvector from make: Nothing to be done for 'benchmarks'. --- benchmarks/nvector/cuda/CMakeLists.txt | 6 ------ benchmarks/nvector/hip/CMakeLists.txt | 6 ------ benchmarks/nvector/kokkos/CMakeLists.txt | 6 ------ benchmarks/nvector/mpiplusx/CMakeLists.txt | 6 ------ benchmarks/nvector/openmp/CMakeLists.txt | 5 ----- benchmarks/nvector/openmpdev/CMakeLists.txt | 6 ------ benchmarks/nvector/parallel/CMakeLists.txt | 6 ------ benchmarks/nvector/parhyp/CMakeLists.txt | 6 ------ benchmarks/nvector/petsc/CMakeLists.txt | 6 ------ benchmarks/nvector/pthreads/CMakeLists.txt | 6 ------ benchmarks/nvector/raja/CMakeLists.txt | 6 ------ benchmarks/nvector/serial/CMakeLists.txt | 6 ------ 12 files changed, 71 deletions(-) diff --git a/benchmarks/nvector/cuda/CMakeLists.txt b/benchmarks/nvector/cuda/CMakeLists.txt index 48571d0203..8589be57ed 100644 --- a/benchmarks/nvector/cuda/CMakeLists.txt +++ b/benchmarks/nvector/cuda/CMakeLists.txt @@ -20,9 +20,3 @@ sundials_add_nvector_benchmark(nvector_cuda_benchmark INSTALL_SUBDIR nvector/cuda ) -# NUM_CORES based on 1 lassen node -sundials_add_benchmark(nvector_cuda_benchmark nvector_cuda_benchmark - NUM_CORES 4 - ENABLE_GPU - BENCHMARK_ARGS "10 10 10 10 10 10" - ) diff --git a/benchmarks/nvector/hip/CMakeLists.txt b/benchmarks/nvector/hip/CMakeLists.txt index 6bbfeff68e..a483fdd6a6 100644 --- a/benchmarks/nvector/hip/CMakeLists.txt +++ b/benchmarks/nvector/hip/CMakeLists.txt @@ -20,9 +20,3 @@ sundials_add_nvector_benchmark(nvector_hip_benchmark INSTALL_SUBDIR nvector/hip ) -# NUM_CORES based on Corona system for 1 node -sundials_add_benchmark(nvector_hip_benchmark nvector_hip_benchmark - NUM_CORES 8 - ENABLE_GPU - BENCHMARK_ARGS "10 10 10 10 10 10" - ) diff --git a/benchmarks/nvector/kokkos/CMakeLists.txt b/benchmarks/nvector/kokkos/CMakeLists.txt index acb68c378b..5c48b04011 100644 --- a/benchmarks/nvector/kokkos/CMakeLists.txt +++ b/benchmarks/nvector/kokkos/CMakeLists.txt @@ -20,12 +20,6 @@ foreach(backend ${KOKKOS_EXAMPLES_BACKENDS}) INSTALL_SUBDIR nvector/kokkos ) - sundials_add_benchmark(test_nvector_performance_kokkos.${backend} test_nvector_performance_kokkos.${backend} - NUM_CORES 4 - ENABLE_GPU - BENCHMARK_ARGS "10 10 10 10 10 10" - ) - target_compile_definitions(test_nvector_performance_kokkos.${backend} PRIVATE USE_${backend}) install(TARGETS test_nvector_performance_kokkos.${backend} diff --git a/benchmarks/nvector/mpiplusx/CMakeLists.txt b/benchmarks/nvector/mpiplusx/CMakeLists.txt index dfe1d6de2c..0ee9958c6b 100644 --- a/benchmarks/nvector/mpiplusx/CMakeLists.txt +++ b/benchmarks/nvector/mpiplusx/CMakeLists.txt @@ -19,10 +19,4 @@ sundials_add_nvector_benchmark(nvector_mpiplusx_benchmark SUNDIALS_TARGETS sundials_nvecserial sundials_nvecmpiplusx LINK_LIBRARIES MPI::MPI_CXX INSTALL_SUBDIR nvector/mpiplusx - ) - -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(nvector_mpiplusx_benchmark nvector_mpiplusx_benchmark - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" ) \ No newline at end of file diff --git a/benchmarks/nvector/openmp/CMakeLists.txt b/benchmarks/nvector/openmp/CMakeLists.txt index 9c90eab9b8..f9a8799bc2 100644 --- a/benchmarks/nvector/openmp/CMakeLists.txt +++ b/benchmarks/nvector/openmp/CMakeLists.txt @@ -27,8 +27,3 @@ sundials_add_nvector_benchmark(nvector_openmp_benchmark INSTALL_SUBDIR nvector/openmp ) -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(nvector_openmp_benchmark nvector_openmp_benchmark - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" - ) diff --git a/benchmarks/nvector/openmpdev/CMakeLists.txt b/benchmarks/nvector/openmpdev/CMakeLists.txt index 9d48eb65f7..e7a4b46d51 100644 --- a/benchmarks/nvector/openmpdev/CMakeLists.txt +++ b/benchmarks/nvector/openmpdev/CMakeLists.txt @@ -37,9 +37,3 @@ target_link_libraries(test_nvector_performance_openmpdev ${SUNDIALS_LIBS}) install(TARGETS test_nvector_performance_openmpdev DESTINATION "${BENCHMARKS_INSTALL_PATH}/nvector/openmpdev") - -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(test_nvector_performance_openmpdev test_nvector_performance_openmpdev - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" - ) \ No newline at end of file diff --git a/benchmarks/nvector/parallel/CMakeLists.txt b/benchmarks/nvector/parallel/CMakeLists.txt index 1e583603a6..0b8f7e33a8 100644 --- a/benchmarks/nvector/parallel/CMakeLists.txt +++ b/benchmarks/nvector/parallel/CMakeLists.txt @@ -20,9 +20,3 @@ sundials_add_nvector_benchmark(nvector_parallel_benchmark LINK_LIBRARIES MPI::MPI_CXX INSTALL_SUBDIR nvector/parallel ) - -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(nvector_parallel_benchmark nvector_parallel_benchmark - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" - ) \ No newline at end of file diff --git a/benchmarks/nvector/parhyp/CMakeLists.txt b/benchmarks/nvector/parhyp/CMakeLists.txt index 9b3017f79d..67b74787b6 100644 --- a/benchmarks/nvector/parhyp/CMakeLists.txt +++ b/benchmarks/nvector/parhyp/CMakeLists.txt @@ -22,9 +22,3 @@ sundials_add_nvector_benchmark(nvector_parhyp_benchmark LINK_LIBRARIES MPI::MPI_C SUNDIALS::HYPRE INSTALL_SUBDIR nvector/parhyp ) - -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(nvector_parhyp_benchmark nvector_parhyp_benchmark - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" - ) \ No newline at end of file diff --git a/benchmarks/nvector/petsc/CMakeLists.txt b/benchmarks/nvector/petsc/CMakeLists.txt index b7dcc9c5fc..d4c501fec7 100644 --- a/benchmarks/nvector/petsc/CMakeLists.txt +++ b/benchmarks/nvector/petsc/CMakeLists.txt @@ -22,9 +22,3 @@ sundials_add_nvector_benchmark(nvector_petsc_benchmark LINK_LIBRARIES MPI::MPI_C PUBLIC SUNDIALS::PETSC INSTALL_SUBDIR nvector/petsc ) - -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(nvector_petsc_benchmark nvector_petsc_benchmark - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" - ) \ No newline at end of file diff --git a/benchmarks/nvector/pthreads/CMakeLists.txt b/benchmarks/nvector/pthreads/CMakeLists.txt index 4475b0e5f4..b1aa4d2ad0 100644 --- a/benchmarks/nvector/pthreads/CMakeLists.txt +++ b/benchmarks/nvector/pthreads/CMakeLists.txt @@ -22,9 +22,3 @@ sundials_add_nvector_benchmark(nvector_pthreads_benchmark LINK_LIBRARIES Threads::Threads INSTALL_SUBDIR nvector/pthreads ) - -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(nvector_pthreads_benchmark nvector_pthreads_benchmark - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" - ) \ No newline at end of file diff --git a/benchmarks/nvector/raja/CMakeLists.txt b/benchmarks/nvector/raja/CMakeLists.txt index 7948a0cb72..3625506fd8 100644 --- a/benchmarks/nvector/raja/CMakeLists.txt +++ b/benchmarks/nvector/raja/CMakeLists.txt @@ -41,12 +41,6 @@ foreach(backend ${SUNDIALS_RAJA_BACKENDS}) SOURCES test_nvector_performance_raja.cpp SUNDIALS_TARGETS sundials_nvecraja ) - - sundials_add_benchmark(${example_target} ${example_target} - NUM_CORES 4 - ENABLE_GPU - BENCHMARK_ARGS "10 10 10 10 10 10" - ) target_compile_definitions(${example_target} PRIVATE ${_defines}) diff --git a/benchmarks/nvector/serial/CMakeLists.txt b/benchmarks/nvector/serial/CMakeLists.txt index 96e0d1319f..1f339c1750 100644 --- a/benchmarks/nvector/serial/CMakeLists.txt +++ b/benchmarks/nvector/serial/CMakeLists.txt @@ -21,9 +21,3 @@ sundials_add_nvector_benchmark(nvector_serial_benchmark SUNDIALS_TARGETS sundials_nvecserial INSTALL_SUBDIR nvector/serial ) - -# NUM_CORES based on Lassen system for 1 node -sundials_add_benchmark(nvector_serial_benchmark nvector_serial_benchmark - NUM_CORES 40 - BENCHMARK_ARGS "10 10 10 10 10 10" - ) \ No newline at end of file From 71764b3141ef794d386aa9d1bb3c11b8f2c96c64 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 18 May 2023 10:31:49 -0700 Subject: [PATCH 084/187] add SundialsAdiak.cmake --- cmake/tpl/SundialsAdiak.cmake | 89 +++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 cmake/tpl/SundialsAdiak.cmake diff --git a/cmake/tpl/SundialsAdiak.cmake b/cmake/tpl/SundialsAdiak.cmake new file mode 100644 index 0000000000..7cef96f9f0 --- /dev/null +++ b/cmake/tpl/SundialsAdiak.cmake @@ -0,0 +1,89 @@ +# ----------------------------------------------------------------------------- +# Programmer(s): Cody J. Balos @ LLNL +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2023, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- +# Module to find and setup ADIAK correctly. +# Created from the SundialsTPL.cmake template. +# All SUNDIALS modules that find and setup a TPL must: +# +# 1. Check to make sure the SUNDIALS configuration and the TPL is compatible. +# 2. Find the TPL. +# 3. Check if the TPL works with SUNDIALS, UNLESS the override option +# TPL_WORKS is TRUE - in this case the tests should not be performed and it +# should be assumed that the TPL works with SUNDIALS. +# ----------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- +# Section 1: Include guard +# ----------------------------------------------------------------------------- + +if(NOT DEFINED SUNDIALS_ADIAK_INCLUDED) + set(SUNDIALS_ADIAK_INCLUDED) +else() + return() +endif() + +# ----------------------------------------------------------------------------- +# Section 2: Check to make sure options are compatible +# ----------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- +# Section 3: Find the TPL +# ----------------------------------------------------------------------------- + +find_package(adiak + PATHS "${adiak_DIR}" + NO_DEFAULT_PATH + REQUIRED) +message(STATUS "ADIAK_LIBRARIES: ${adiak_LIBRARIES}") +message(STATUS "ADIAK_INCLUDE_DIR: ${adiak_INCLUDE_DIR}") + + +# ----------------------------------------------------------------------------- +# Section 4: Test the TPL +# ----------------------------------------------------------------------------- + +if(adiak_FOUND AND (NOT adiak_WORKS)) + # Do any checks which don't require compilation first. + + # Create the adiak_TEST directory + set(adiak_TEST_DIR ${PROJECT_BINARY_DIR}/adiak_TEST) + file(MAKE_DIRECTORY ${adiak_TEST_DIR}) + + # Create a CMakeLists.txt file + file(WRITE ${adiak_TEST_DIR}/CMakeLists.txt "") + + # Create a C source file + file(WRITE ${adiak_TEST_DIR}/ltest.c "") + + # To ensure we do not use stuff from the previous attempts, + # we must remove the CMakeFiles directory. + file(REMOVE_RECURSE ${adiak_TEST_DIR}/CMakeFiles) + + # Attempt to build and link the "ltest" executable + try_compile(COMPILE_OK ${adiak_TEST_DIR} ${adiak_TEST_DIR} ltest + OUTPUT_VARIABLE COMPILE_OUTPUT) + + # Process test result + if(COMPILE_OK) + message(STATUS "Checking if adiak works with SUNDIALS... OK") + set(adiak_WORKS TRUE CACHE BOOL "adiak works with SUNDIALS as configured" FORCE) + else() + message(STATUS "Checking if adiak works with SUNDIALS... FAILED") + message(STATUS "Check output: ") + message("${COMPILE_OUTPUT}") + print_error("SUNDIALS interface to adiak is not functional.") + endif() + +elseif(adiak_FOUND AND adiak_WORKS) + message(STATUS "Skipped adiak tests, assuming adiak works with SUNDIALS.") +endif() From 3ab28a8691c31a8e77d35208292a6a88fc6bca15 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 18 May 2023 10:38:05 -0700 Subject: [PATCH 085/187] add output dir for benchmark cali files --- cmake/SundialsBuildOptionsPre.cmake | 3 ++- cmake/macros/SundialsAddBenchmark.cmake | 6 +++--- cmake/macros/SundialsAddTest.cmake | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index fa2330f456..00d819786c 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -297,4 +297,5 @@ sundials_option(SUNDIALS_TEST_UNITTESTS BOOL sundials_option(SUNDIALS_SCHEDULER_COMMAND STIRNG "Job scheduler command to use to launch SUNDIALS MPI tests." "" ADVANCED) -sundials_option(SUNDIALS_BENCHMARK_OUTPUT_DIR STRING "Specify where to put benchmark output." "/usr/workspace/pan13/sundials/benchmarks" ADVANCED) +sundials_option(SUNDIALS_PROFILE_OUTPUT_DIR STRING "Specify where to put benchmark output." "" DEPENDS_ON SUNDIALS_BUILD_WITH_PROFILING ADVANCED) + diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index 5fa131a82a..a3cafa1c70 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -34,15 +34,15 @@ macro(sundials_add_benchmark NAME EXECUTABLE) endif() # Make the output directory if it doesn't exist - if(NOT EXISTS ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) - file(MAKE_DIRECTORY ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) + if(NOT EXISTS ${SUNDIALS_PROFILE_OUTPUT_DIR}/${TARGET_NAME}) + file(MAKE_DIRECTORY ${SUNDIALS_PROFILE_OUTPUT_DIR}/${TARGET_NAME}) endif() # command line arguments for the test runner script set(TEST_RUNNER_ARGS "--verbose" "--executablename=$" - "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" + "--outputdir=${SUNDIALS_PROFILE_OUTPUT_DIR}/${TARGET_NAME}" "--nodiff" ) diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index b93027a017..b229a92ba1 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -94,6 +94,10 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) list(APPEND TEST_ARGS "--profile") endif() + # if(SUNDIALS_PROFILING_OUTPUT_DIR AND SUNDIALS_TEST_PROFILE) + # list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_PROFILING_OUTPUT_DIR}") + # else + # check for a non-default output directory if(SUNDIALS_TEST_OUTPUT_DIR) list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}") From 98a555f45821182435d9f2141a3fe52756aad266 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 18 May 2023 10:39:16 -0700 Subject: [PATCH 086/187] add custom adiak metadata --- cmake/SundialsSetupConfig.cmake | 20 ++++++++++++++++ include/sundials/sundials_config.in | 33 ++++++++++++++++++++++++-- src/sundials/sundials_context.c | 36 +++++++++++++++++++++-------- 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index c3ae7dc6dd..89f1c3a04a 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -39,6 +39,11 @@ else() set(SUNDIALS_USE_GENERIC_MATH TRUE) endif() +if($ENV{CI_JOB_ID}) + set(SUNDIALS_CI_JOB TRUE) + set(SUN_CI_JOB_ID $ENV{CI_JOB_ID}) +endif() + # ============================================================================ # Generate macros and substitution variables related to TPLs # that SUNDIALS is being built with. @@ -73,6 +78,21 @@ if(Trilinos_MPI) set(SUNDIALS_TRILINOS_HAVE_MPI TRUE) endif() +# prepare substitution variable SUNDIALS_RAJA_ENABLED for sundials_config.h +if(ENABLE_RAJA) + set(SUNDIALS_RAJA_ENABLED TRUE) +endif() + +# prepare substitution variable SUNDIALS_CUDA_ENABLED for sundials_config.h +if(ENABLE_CUDA) + set(SUNDIALS_CUDA_ENABLED TRUE) +endif() + +# prepare substitution variable SUNDIALS_MPI_ENABLED for sundials_config.h +if(ENABLE_HIP) + set(SUNDIALS_HIP_ENABLED TRUE) +endif() + # prepare substitution variable(s) SUNDIALS_RAJA_BACKENDS_* foreach(backend ${SUNDIALS_RAJA_BACKENDS}) set(SUNDIALS_RAJA_BACKENDS_${backend} TRUE) diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 09cdea24a5..9f77a066f3 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -105,6 +105,16 @@ #define SUNDIALS_MAX_SPRINTF_SIZE @SUNDIALS_MAX_SPRINTF_SIZE@ #endif +/* Build metadata */ +#define SUN_CXX_COMPILER "@CMAKE_CXX_COMPILER_ID@" +#define SUN_CXX_COMPILER_VERSION "@CMAKE_CXX_COMPILER_VERSION@" +#define SUN_CXX_COMPILER_FLAGS "@CMAKE_CXX_FLAGS@" + +#define SUN_BUILD_TYPE "@CMAKE_BUILD_TYPE@" + +#cmakedefine SUNDIALS_CI_JOB +#cmakedefine SUN_CI_JOB_ID + /* ------------------------------------------------------------------ * SUNDIALS TPL macros * -----------------------------------------------------------------*/ @@ -141,13 +151,32 @@ */ #cmakedefine SUNDIALS_TRILINOS_HAVE_MPI +/* RAJA */ +#cmakedefine SUNDIALS_RAJA_ENABLED + /* RAJA backends */ #cmakedefine SUNDIALS_RAJA_BACKENDS_CUDA #cmakedefine SUNDIALS_RAJA_BACKENDS_HIP #cmakedefine SUNDIALS_RAJA_BACKENDS_SYCL -/* RAJA version number */ -#define RAJA_VERSION @RAJA_PACKAGE_VERSION@ +/* RAJA version number*/ +#define SUN_RAJA_VERSION "@RAJA_VERSION@" + +/* OpenMP * +#cmakedefine SUNDIALS_OPENMP_ENABLED +#define SUN_OPENMP_VERSION "@OpenMP_VERSION@" + +/* ------------------------------------------------------------------ + * SUNDIALS language macros + * -----------------------------------------------------------------*/ + +/* CUDA */ +#cmakedefine SUNDIALS_CUDA_ENABLED +#define SUN_CUDA_VERSION "@CMAKE_CUDA_COMPILER_VERSION@" + +/* HIP */ +#cmakedefine SUNDIALS_HIP_ENABLED +#define SUN_HIP_VERSION "@HIP_VERSION@" /* ------------------------------------------------------------------ * SUNDIALS modules enabled diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 3b81ceed41..02fee306d1 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -15,7 +15,9 @@ * SUNDIALS objects in a simulation share. * ----------------------------------------------------------------*/ +#ifdef SUNDIALS_ADIAK_ENABLED #include "adiak.h" +#endif #include #include @@ -200,7 +202,6 @@ int SUNContext_Free(SUNContext* sunctx) #endif #ifdef SUNDIALS_ADIAK_ENABLED - // adiak_clean(); adiak_fini(); #endif @@ -217,22 +218,39 @@ int SUNContext_Free(SUNContext* sunctx) #ifdef SUNDIALS_ADIAK_ENABLED static void sun_adiak_helper() { - adiak_user(); - adiak_uid(); adiak_launchdate(); adiak_executable(); adiak_cmdline(); - adiak_hostname(); adiak_clustername(); adiak_job_size(); adiak_hostlist(); + adiak_num_hosts(); + + adiak_namevalue("cxx_compiler", 2, NULL, "%s", SUN_CXX_COMPILER); + adiak_namevalue("cxx_compiler_version", 2, NULL, "%s", SUN_CXX_COMPILER_VERSION); + adiak_namevalue("cxx_compiler_flags", 2, NULL, "%s", SUN_CXX_COMPILER_FLAGS); + + adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); + +#ifdef SUNDIALS_CI_JOB + adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); +#endif + +#ifdef SUNDIALS_RAJA_ENABLED + adiak_namevalue("raja_version", 2, NULL, "%s", SUN_RAJA_VERSION); +#endif + +#ifdef SUNDIALS_OPENMP_ENABLED + adiak_namevalue("openmp_version", 2, NULL, "%s", SUN_OPENMP_VERSION); +#endif + +#ifdef SUNDIALS_CUDA_ENABLED + adiak_namevalue("cuda_version", 2, NULL, "%s", SUN_CUDA_VERSION); +#endif - // make the c version - adiak_namevalue("sample_variable", (adiak_category_t)(1), "import.category" "%s", "example"); - -#ifdef SUNDIALS_RAJA_INCLUDED - adiak_namevalue("raja_version", static_cast(2), "import.category", "%s", RAJA_VERSION); +#ifdef SUNDIALS_HIP_ENABLED + adiak_namevalue("hip_version", 2, NULL, "%s", SUN_HIP_VERSION); #endif } #endif \ No newline at end of file From fe718b39aed91002e3c516379a1ef93a9f9b0224 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Fri, 19 May 2023 13:40:36 -0700 Subject: [PATCH 087/187] call SUNMemoryHelper_Create after initlaizing the proflier root in main --- .../advection_reaction_3D.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp b/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp index dc169c5fa1..817321457f 100644 --- a/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp +++ b/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp @@ -72,15 +72,6 @@ int main(int argc, char *argv[]) /* Create SUNDIALS context */ SUNContext_Create((void*) &comm, &ctx); - /* Create SUNDIALS memory helper */ -#if defined(USE_CUDA) - SUNMemoryHelper mem_helper = SUNMemoryHelper_Cuda(ctx); -#elif defined(USE_HIP) - SUNMemoryHelper mem_helper = SUNMemoryHelper_Hip(ctx); -#else - SUNMemoryHelper mem_helper = SUNMemoryHelper_Sys(ctx); -#endif - { /* general problem variables */ N_Vector y = NULL; /* empty solution vector */ @@ -91,6 +82,15 @@ int main(int argc, char *argv[]) SUNDIALS_CXX_MARK_FUNCTION(udata.prof); + /* Create SUNDIALS memory helper */ +#if defined(USE_CUDA) + SUNMemoryHelper mem_helper = SUNMemoryHelper_Cuda(ctx); +#elif defined(USE_HIP) + SUNMemoryHelper mem_helper = SUNMemoryHelper_Hip(ctx); +#else + SUNMemoryHelper mem_helper = SUNMemoryHelper_Sys(ctx); +#endif + /* Process input args and setup the problem */ retval = SetupProblem(argc, argv, &udata, &uopt, mem_helper, ctx); if (check_retval(&retval, "SetupProblem", 1, udata.myid)) MPI_Abort(comm, 1); @@ -130,9 +130,9 @@ int main(int argc, char *argv[]) /* Clean up */ N_VDestroy(N_VGetLocalVector_MPIPlusX(y)); N_VDestroy(y); + SUNMemoryHelper_Destroy(mem_helper); } - SUNMemoryHelper_Destroy(mem_helper); SUNContext_Free(&ctx); MPI_Finalize(); return(0); From e250d6cf682256c7d1b967881a7145c52f24f07e Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Fri, 19 May 2023 15:46:50 -0700 Subject: [PATCH 088/187] use cpp sundials::Context --- .../advection_reaction_3D/advection_reaction_3D.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp b/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp index 817321457f..e6aed834c9 100644 --- a/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp +++ b/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp @@ -63,16 +63,13 @@ /* Main Program */ int main(int argc, char *argv[]) { - SUNContext ctx; - /* Initialize MPI */ MPI_Comm comm = MPI_COMM_WORLD; MPI_Init(&argc, &argv); - /* Create SUNDIALS context */ - SUNContext_Create((void*) &comm, &ctx); - { + sundials::Context ctx; + /* general problem variables */ N_Vector y = NULL; /* empty solution vector */ UserData udata(ctx); /* user data */ @@ -133,7 +130,6 @@ int main(int argc, char *argv[]) SUNMemoryHelper_Destroy(mem_helper); } - SUNContext_Free(&ctx); MPI_Finalize(); return(0); } From 87a3b231bf7673b00e6cb589136d8e909c4d00b9 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 22 May 2023 14:09:56 -0700 Subject: [PATCH 089/187] c++ wrapper for sundials_memory; adjust advection caliper calls to have only one root --- .../advection_reaction_3D/CMakeLists.txt | 24 +++++------ .../advection_reaction_3D.cpp | 41 +++++++++++-------- .../advection_reaction_3D.hpp | 7 ++-- benchmarks/nvector/cuda/CMakeLists.txt | 1 - benchmarks/nvector/hip/CMakeLists.txt | 1 - cmake/SundialsBuildOptionsPre.cmake | 2 +- include/sundials/sundials_memory.hpp | 41 +++++++++++++++++++ src/sundials/CMakeLists.txt | 1 + src/sundials/sundials_memory.c | 2 +- 9 files changed, 83 insertions(+), 37 deletions(-) create mode 100644 include/sundials/sundials_memory.hpp diff --git a/benchmarks/advection_reaction_3D/CMakeLists.txt b/benchmarks/advection_reaction_3D/CMakeLists.txt index 39d6d11e87..2237583e68 100644 --- a/benchmarks/advection_reaction_3D/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/CMakeLists.txt @@ -57,10 +57,10 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save\;arkimex_tlnewton" - "--method ARK-DIRK --nls newton --tf 10.0 --dont-save\;arkdirk_newton" - "--method CV-BDF --nls newton --tf 10.0 --dont-save\;cvbdf_newton" - "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton" + "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton" + "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton" + "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton" + "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton" ) # NUM_CORES based on Lassen system for 1 node @@ -122,10 +122,10 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save\;arkimex_tlnewton_cuda" - "--method ARK-DIRK --nls newton --tf 10.0 --dont-save\;arkdirk_newton_cuda" - "--method CV-BDF --nls newton --tf 10.0 --dont-save\;cvbdf_newton_cuda" - "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton_cuda" + "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_cuda" + "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_cuda" + "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_cuda" + "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_cuda" ) # NUM_CORES based on Lassen system for 1 node foreach(benchmark_tuple ${BENCHMARK_VAR}) @@ -181,10 +181,10 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save\;arkimex_tlnewton_hip" - "--method ARK-DIRK --nls newton --tf 10.0 --dont-save\;arkdirk_newton_hip" - "--method CV-BDF --nls newton --tf 10.0 --dont-save\;cvbdf_newton_hip" - "--method IDA --nls newton --tf 10.0 --dont-save\;ida_newton_hip" + "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_hip" + "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_hip" + "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_hip" + "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_hip" ) diff --git a/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp b/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp index 817321457f..d877bc4ef0 100644 --- a/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp +++ b/benchmarks/advection_reaction_3D/advection_reaction_3D.cpp @@ -60,10 +60,13 @@ #include "advection_reaction_3D.hpp" +using namespace sundials::experimental; + /* Main Program */ int main(int argc, char *argv[]) { SUNContext ctx; + SUNProfiler prof; /* Initialize MPI */ MPI_Comm comm = MPI_COMM_WORLD; @@ -71,28 +74,32 @@ int main(int argc, char *argv[]) /* Create SUNDIALS context */ SUNContext_Create((void*) &comm, &ctx); + SUNContext_GetProfiler(ctx, &prof); { - /* general problem variables */ - N_Vector y = NULL; /* empty solution vector */ - UserData udata(ctx); /* user data */ - UserOptions uopt; /* user options */ - int retval; /* reusable error-checking flag */ - char fname[MXSTR]; - - SUNDIALS_CXX_MARK_FUNCTION(udata.prof); + SUNDIALS_CXX_MARK_FUNCTION(prof); - /* Create SUNDIALS memory helper */ + /* Create SUNDIALS memory helper */ #if defined(USE_CUDA) - SUNMemoryHelper mem_helper = SUNMemoryHelper_Cuda(ctx); + SUNMemoryHelperView mem_helper(SUNMemoryHelper_Cuda(ctx)); + //SUNMemoryHelper mem_helper = SUNMemoryHelper_Cuda(ctx); #elif defined(USE_HIP) - SUNMemoryHelper mem_helper = SUNMemoryHelper_Hip(ctx); + SUNMemoryHelperView mem_helper(SUNMemoryHelper_Hip(ctx)); + //SUNMemoryHelper mem_helper = SUNMemoryHelper_Hip(ctx); #else - SUNMemoryHelper mem_helper = SUNMemoryHelper_Sys(ctx); + SUNMemoryHelperView mem_helper(SUNMemoryHelper_Sys(ctx)); + //SUNMemoryHelper mem_helper = SUNMemoryHelper_Sys(ctx); #endif + /* general problem variables */ + N_Vector y = NULL; /* empty solution vector */ + UserData udata(ctx, mem_helper); /* user data */ + UserOptions uopt; /* user options */ + int retval; /* reusable error-checking flag */ + char fname[MXSTR]; + /* Process input args and setup the problem */ - retval = SetupProblem(argc, argv, &udata, &uopt, mem_helper, ctx); + retval = SetupProblem(argc, argv, &udata, &uopt); if (check_retval(&retval, "SetupProblem", 1, udata.myid)) MPI_Abort(comm, 1); /* Create solution vector */ @@ -130,7 +137,6 @@ int main(int argc, char *argv[]) /* Clean up */ N_VDestroy(N_VGetLocalVector_MPIPlusX(y)); N_VDestroy(y); - SUNMemoryHelper_Destroy(mem_helper); } SUNContext_Free(&ctx); @@ -512,8 +518,7 @@ int ComponentMask(N_Vector mask, int component, const UserData* udata) /* Parses the CLI arguments and sets up the problem */ -int SetupProblem(int argc, char *argv[], UserData* udata, UserOptions* uopt, - SUNMemoryHelper memhelper, SUNContext ctx) +int SetupProblem(int argc, char *argv[], UserData* udata, UserOptions* uopt) { constexpr int STENCIL_WIDTH = 1; @@ -574,12 +579,12 @@ int SetupProblem(int argc, char *argv[], UserData* udata, UserOptions* uopt, const sunindextype npts[] = {uopt->npts, uopt->npts, uopt->npts}; const realtype amax[] = {0.0, 0.0, 0.0}; const realtype bmax[] = {udata->xmax, udata->xmax, udata->xmax}; - udata->grid = new ParallelGrid(memhelper, + udata->grid = new ParallelGrid(udata->helper, &udata->comm, amax, bmax, npts, 3, BoundaryType::PERIODIC, StencilType::UPWIND, STENCIL_WIDTH, uopt->npxyz ); /* Create the solution masks */ - udata->umask = N_VMake_MPIPlusX(udata->comm, LocalNvector(udata->grid->neq, ctx), ctx); + udata->umask = N_VMake_MPIPlusX(udata->comm, LocalNvector(udata->grid->neq, udata->ctx), udata->ctx); udata->vmask = N_VClone(udata->umask); udata->wmask = N_VClone(udata->umask); ComponentMask(udata->umask, 0, udata); diff --git a/benchmarks/advection_reaction_3D/advection_reaction_3D.hpp b/benchmarks/advection_reaction_3D/advection_reaction_3D.hpp index 4396e69eb5..22130c7eda 100644 --- a/benchmarks/advection_reaction_3D/advection_reaction_3D.hpp +++ b/benchmarks/advection_reaction_3D/advection_reaction_3D.hpp @@ -24,6 +24,7 @@ #include #include #include +#include #include "check_retval.h" #include "backends.hpp" @@ -78,6 +79,7 @@ struct UserOptions struct UserData { SUNContext ctx; + SUNMemoryHelper helper; SUNProfiler prof; /* MPI data */ @@ -122,7 +124,7 @@ struct UserData UserOptions* uopt; /* constructor that takes the context */ - UserData(SUNContext ctx) : ctx(ctx) { + UserData(SUNContext ctx, SUNMemoryHelper helper) : ctx(ctx), helper(helper){ SUNContext_GetProfiler(ctx, &prof); } @@ -167,8 +169,7 @@ int ExchangeAllStart(N_Vector y, UserData* udata); int ExchangeAllEnd(UserData* udata); /* functions for processing command line args */ -int SetupProblem(int argc, char *argv[], UserData* udata, UserOptions* uopt, - SUNMemoryHelper memhelper, SUNContext ctx); +int SetupProblem(int argc, char *argv[], UserData* udata, UserOptions* uopt); void InputError(char *name); /* function to write solution to disk */ diff --git a/benchmarks/nvector/cuda/CMakeLists.txt b/benchmarks/nvector/cuda/CMakeLists.txt index 8589be57ed..ee1366befc 100644 --- a/benchmarks/nvector/cuda/CMakeLists.txt +++ b/benchmarks/nvector/cuda/CMakeLists.txt @@ -19,4 +19,3 @@ sundials_add_nvector_benchmark(nvector_cuda_benchmark SUNDIALS_TARGETS sundials_nveccuda INSTALL_SUBDIR nvector/cuda ) - diff --git a/benchmarks/nvector/hip/CMakeLists.txt b/benchmarks/nvector/hip/CMakeLists.txt index a483fdd6a6..35ab748cf8 100644 --- a/benchmarks/nvector/hip/CMakeLists.txt +++ b/benchmarks/nvector/hip/CMakeLists.txt @@ -19,4 +19,3 @@ sundials_add_nvector_benchmark(nvector_hip_benchmark SUNDIALS_TARGETS sundials_nvechip INSTALL_SUBDIR nvector/hip ) - diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index 00d819786c..1c236086ac 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -295,7 +295,7 @@ sundials_option(SUNDIALS_TEST_DEVTESTS BOOL sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) -sundials_option(SUNDIALS_SCHEDULER_COMMAND STIRNG "Job scheduler command to use to launch SUNDIALS MPI tests." "" ADVANCED) +sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests." "" ADVANCED) sundials_option(SUNDIALS_PROFILE_OUTPUT_DIR STRING "Specify where to put benchmark output." "" DEPENDS_ON SUNDIALS_BUILD_WITH_PROFILING ADVANCED) diff --git a/include/sundials/sundials_memory.hpp b/include/sundials/sundials_memory.hpp new file mode 100644 index 0000000000..b410a913ac --- /dev/null +++ b/include/sundials/sundials_memory.hpp @@ -0,0 +1,41 @@ +/* ----------------------------------------------------------------------------- + * Programmer(s): Cody J. Balos @ LLNL + * ----------------------------------------------------------------------------- + * SUNDIALS Copyright Start + * Copyright (c) 2002-2023, Lawrence Livermore National Security + * and Southern Methodist University. + * All rights reserved. + * + * See the top-level LICENSE and NOTICE files for details. + * + * SPDX-License-Identifier: BSD-3-Clause + * SUNDIALS Copyright End + * ----------------------------------------------------------------------------- + * C++ view of SUNDIALS SUNMemoryHelper + * ---------------------------------------------------------------------------*/ + +#ifndef _SUNDIALS_MEMORY_HPP +#define _SUNDIALS_MEMORY_HPP + +#include +#include +#include + +namespace sundials { +namespace impl { +using BaseMemoryHelper = BaseObject<_SUNMemoryHelper, _SUNMemoryHelper_Ops>; +} // namespace impl + +namespace experimental { +struct SUNMemoryHelperDeleter +{ + void operator()(SUNMemoryHelper helper) + { + if (helper) SUNMemoryHelper_Destroy(helper); + } +}; +using SUNMemoryHelperView = ClassView; +} // namespace experimental +} // namespace sundials + +#endif diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt index 1955ff0446..9fcd6fe0a3 100644 --- a/src/sundials/CMakeLists.txt +++ b/src/sundials/CMakeLists.txt @@ -35,6 +35,7 @@ set(sundials_HEADERS sundials_matrix.h sundials_matrix.hpp sundials_memory.h + sundials_memory.hpp sundials_mpi_types.h sundials_nonlinearsolver.h sundials_nonlinearsolver.hpp diff --git a/src/sundials/sundials_memory.c b/src/sundials/sundials_memory.c index 4af4a8eb10..c013f9a977 100644 --- a/src/sundials/sundials_memory.c +++ b/src/sundials/sundials_memory.c @@ -147,7 +147,7 @@ int SUNMemoryHelper_GetAllocStats(SUNMemoryHelper helper, SUNMemoryType mem_type if (helper->ops->getallocstats) { return helper->ops->getallocstats(helper, mem_type, num_allocations, num_deallocations, bytes_allocated, bytes_high_watermark); } else { - ier = -1; + ier = helper->ops->getallocstats(helper, mem_type, num_allocations, num_deallocations, bytes_allocated, bytes_high_watermark); } SUNDIALS_MARK_FUNCTION_END(getSUNProfiler(helper)); return(ier); From cbcbe9d4d4395cff6f9ad7f0d60de393f9c2812b Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 23 May 2023 11:18:17 -0700 Subject: [PATCH 090/187] modify cali_dir to be read from env --- cmake/SundialsBuildOptionsPre.cmake | 3 --- cmake/macros/SundialsAddBenchmark.cmake | 19 ++++++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index 1c236086ac..b9afe2a449 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -296,6 +296,3 @@ sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests." "" ADVANCED) - -sundials_option(SUNDIALS_PROFILE_OUTPUT_DIR STRING "Specify where to put benchmark output." "" DEPENDS_ON SUNDIALS_BUILD_WITH_PROFILING ADVANCED) - diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index a3cafa1c70..ed03a824d2 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -34,15 +34,28 @@ macro(sundials_add_benchmark NAME EXECUTABLE) endif() # Make the output directory if it doesn't exist - if(NOT EXISTS ${SUNDIALS_PROFILE_OUTPUT_DIR}/${TARGET_NAME}) - file(MAKE_DIRECTORY ${SUNDIALS_PROFILE_OUTPUT_DIR}/${TARGET_NAME}) + #if(NOT EXISTS ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) + # file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) + #endif() + + # check if the env variable exists + if(DEFINED ENV{SUNDIALS_CALI_DIR}) + set(SUNDIALS_CALI_DIR $ENV{SUNDIALS_CALI_DIR}) + if(NOT EXISTS ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) + file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) + endif() + else() + set(SUNDIALS_CALI_DIR ${PROJECT_BINARY_DIR}/Benchmarking) + if(NOT EXISTS ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) + file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) + endif() endif() # command line arguments for the test runner script set(TEST_RUNNER_ARGS "--verbose" "--executablename=$" - "--outputdir=${SUNDIALS_PROFILE_OUTPUT_DIR}/${TARGET_NAME}" + "--outputdir=${SUNDIALS_CALI_DIR}/${TARGET_NAME}" "--nodiff" ) From b83194f82683632553eca05bcc6097307c3c2399 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 23 May 2023 16:55:16 -0700 Subject: [PATCH 091/187] add a default dir to SUNDIALS_CALI_DIR --- cmake/SundialsBuildOptionsPre.cmake | 4 +++- cmake/SundialsSetupTesting.cmake | 8 +++++++- cmake/macros/SundialsAddBenchmark.cmake | 23 +++++++---------------- cmake/macros/SundialsAddTest.cmake | 17 +++++++++-------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index b9afe2a449..c36c2bd232 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -295,4 +295,6 @@ sundials_option(SUNDIALS_TEST_DEVTESTS BOOL sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) -sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests." "" ADVANCED) +sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests" "" ADVANCED) + +sundials_option(SUNDIALS_CALI_DIR STRING "Location to write caliper files" "" ADVANCED) diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 8dfbe116b5..2a1dc7e1dd 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -62,6 +62,13 @@ if(SUNDIALS_TEST_DEVTESTS) endif() endif() + # If SUNDIALS_CALI_DIR was provided make sure it exists + if(DEFINED(SUNDIALS_CALI_DIR)) + if(NOT EXISTS ${SUNDIALS_CALI_DIR}/tests) + file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/tests) + endif() + endif() + # Check if using non-default comparison precisions when testing if(SUNDIALS_TEST_FLOAT_PRECISION GREATER_EQUAL "0") message(STATUS "Using non-default float precision: ${SUNDIALS_TEST_FLOAT_PRECISION}") @@ -101,7 +108,6 @@ if(EXAMPLES_INSTALL) endif() # If benchmarks are enabled, set up `make benchmark` -# Use the test runner to set up benchmarking if(BUILD_BENCHMARKS) message("SUNDIALS Benchmarking") diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index ed03a824d2..432f5f88d1 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -7,7 +7,7 @@ macro(sundials_add_benchmark NAME EXECUTABLE) # Define single value parameters the macro takes in to set up the test runner # - # NUM_CORES = number of cores (GPU count or CPU count) to run on or number of resource sets + # NUM_CORES = number of cores (GPU count or CPU count) to run on/number of resource sets # BENCHMARK_ARGS = arguments to pass to the executable # IDENTIFIER = suffix to append to end of benchmark name set(oneValueArgs NUM_CORES BENCHMARK_ARGS IDENTIFIER) @@ -33,22 +33,13 @@ macro(sundials_add_benchmark NAME EXECUTABLE) endif() endif() - # Make the output directory if it doesn't exist - #if(NOT EXISTS ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) - # file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) - #endif() + if(NOT DEFINED SUNDIALS_CALI_DIR) + set(SUNDIALS_CALI_DIR ${PROJECT_BINARY_DIR}/Benchmarking) + endif() - # check if the env variable exists - if(DEFINED ENV{SUNDIALS_CALI_DIR}) - set(SUNDIALS_CALI_DIR $ENV{SUNDIALS_CALI_DIR}) - if(NOT EXISTS ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) - file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) - endif() - else() - set(SUNDIALS_CALI_DIR ${PROJECT_BINARY_DIR}/Benchmarking) - if(NOT EXISTS ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) - file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) - endif() + # make the caliper output directory if it doesn't exist + if(NOT EXISTS ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) + file(MAKE_DIRECTORY ${SUNDIALS_CALI_DIR}/${TARGET_NAME}) endif() # command line arguments for the test runner script diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index b229a92ba1..66701fce3a 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -94,15 +94,16 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) list(APPEND TEST_ARGS "--profile") endif() - # if(SUNDIALS_PROFILING_OUTPUT_DIR AND SUNDIALS_TEST_PROFILE) - # list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_PROFILING_OUTPUT_DIR}") - # else - - # check for a non-default output directory - if(SUNDIALS_TEST_OUTPUT_DIR) - list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}") + # set califile output directory + if(DEFINED(SUNDIALS_CALI_DIR) AND SUNDIALS_TEST_PROFILE) + list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_CALI_DIR}/tests") else() - list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}") + # check for a non-default output directory + if(SUNDIALS_TEST_OUTPUT_DIR) + list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}") + else() + list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}") + endif() endif() # set a non-default answer directory (default is test/answers) From b92c281976df5b09cb5e894c118dbbb41448f090 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 24 May 2023 11:57:45 -0700 Subject: [PATCH 092/187] replace SUNDIALS_CALIPER_OUTPUT_DIR with SUNDIALS_TEST_OUTPUT_DIR --- cmake/SundialsBuildOptionsPre.cmake | 2 -- cmake/SundialsSetupTesting.cmake | 11 ++--------- cmake/macros/SundialsAddBenchmark.cmake | 15 +++++++++------ cmake/macros/SundialsAddTest.cmake | 13 ++++--------- 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index 36a418c913..25d1c4a715 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -296,5 +296,3 @@ sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests" "" ADVANCED) - -sundials_option(SUNDIALS_CALIPER_OUTPUT_DIR STRING "Location to write caliper files" "" ADVANCED) diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 7c915709af..9e67b4dae8 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -49,8 +49,8 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default output directory was provided make sure it exists if(SUNDIALS_TEST_OUTPUT_DIR) message(STATUS "Using non-default test output directory: ${SUNDIALS_TEST_OUTPUT_DIR}") - if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}) - file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}) + if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) + file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) endif() endif() @@ -62,13 +62,6 @@ if(SUNDIALS_TEST_DEVTESTS) endif() endif() - # If SUNDIALS_CALIPER_OUTPUT_DIR was provided make sure it exists - if(SUNDIALS_CALIPER_OUTPUT_DIR) - if(NOT EXISTS ${SUNDIALS_CALIPER_OUTPUT_DIR}/tests) - file(MAKE_DIRECTORY ${SUNDIALS_CALIPER_OUTPUT_DIR}/tests) - endif() - endif() - # Check if using non-default comparison precisions when testing if(SUNDIALS_TEST_FLOAT_PRECISION GREATER_EQUAL "0") message(STATUS "Using non-default float precision: ${SUNDIALS_TEST_FLOAT_PRECISION}") diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index 487a464538..5dc1ceb646 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -33,20 +33,23 @@ macro(sundials_add_benchmark NAME EXECUTABLE) endif() endif() - if(NOT DEFINED SUNDIALS_CALIPER_OUTPUT_DIR) - set(SUNDIALS_CALIPER_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Benchmarking) + # Create default benchmark output directory if custom not defined + if(SUNDIALS_TEST_OUTPUT_DIR) + set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${SUNDIALS_TEST_OUTPUT_DIR}/Benchmarking) + else() + set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Benchmarking/output) endif() - # make the caliper output directory if it doesn't exist - if(NOT EXISTS ${SUNDIALS_CALIPER_OUTPUT_DIR}/${TARGET_NAME}) - file(MAKE_DIRECTORY ${SUNDIALS_CALIPER_OUTPUT_DIR}/${TARGET_NAME}) + # make the output directory if it doesn't exist + if(NOT EXISTS ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) + file(MAKE_DIRECTORY ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) endif() # command line arguments for the test runner script set(TEST_RUNNER_ARGS "--verbose" "--executablename=$" - "--outputdir=${SUNDIALS_CALIPER_OUTPUT_DIR}/${TARGET_NAME}" + "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" "--nodiff" ) diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index f96c2d299c..462488cb1d 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -94,16 +94,11 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) list(APPEND TEST_ARGS "--profile") endif() - # set califile output directory if profiling - if(SUNDIALS_CALIPER_OUTPUT_DIR AND SUNDIALS_TEST_PROFILE) - list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_CALIPER_OUTPUT_DIR}/tests") + # check for a non-default output directory + if(SUNDIALS_TEST_OUTPUT_DIR) + list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing") else() - # check for a non-default output directory - if(SUNDIALS_TEST_OUTPUT_DIR) - list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}") - else() - list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}") - endif() + list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}") endif() # set a non-default answer directory (default is test/answers) From b848d7b6ccdd9cbec2c63be7a91ac9b2a16f9587 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 25 May 2023 09:28:59 -0700 Subject: [PATCH 093/187] add cmake command to reconfigure cali output dir --- .gitlab/build_and_test.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 1bb0aec5bf..d148995af3 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -219,6 +219,14 @@ then cd "${build_dir}" + # configure .cali file output directory + if [[ "${BENCHMARK}" == "ON" ]] && [[ -d /usr/workspace/sundials ]] + then + output_dir="/usr/workspace/sundials/caliper" + $cmake_exe \ + -DSUNDIALS_TEST_OUTPUT_DIR=${output_dir} + fi + VERBOSE_TEST=${VERBOSE_TEST:-"ON"} if [[ "${VERBOSE_TEST}" == "ON" ]]; then verbose_test='--verbose' From 49f5abe8c68c17d38d90f70f389e897e019fb8a8 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 25 May 2023 09:33:15 -0700 Subject: [PATCH 094/187] add docs on SPOT, HIP --- doc/shared/Install.rst | 12 +++++++++++ .../source/testing/Benchmarking.rst | 11 +++++++--- doc/sundials_developers/source/testing/CI.rst | 4 ++-- .../source/testing/Spot.rst | 21 +++++++++++++++++++ .../source/testing/index.rst | 1 + 5 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 doc/sundials_developers/source/testing/Spot.rst diff --git a/doc/shared/Install.rst b/doc/shared/Install.rst index 7ad8c3bcd1..63791c2ea0 100644 --- a/doc/shared/Install.rst +++ b/doc/shared/Install.rst @@ -1445,6 +1445,18 @@ CUDA examples, set ``EXAMPLES_ENABLE_CUDA`` to ``ON``. SUNDIALS has been tested with the CUDA toolkit versions 10 and 11. +.. _Installation.Cmake.ExternalLibraries.HIP: + +Building with HIP +^^^^^^^^^^^^^^^^^^^^^^ +HIP(heterogeneous-compute interface for portability) allows developers to create portable applications for AMD and NVIDIA GPUs. HIP can be obtained from `HIP GitHub repository`_. + +To enable HIP, set ``ENABLE_HIP`` to ``ON`` and set ``AMDGPU_TARGETS`` to the desired target(ex. gfx705). +In addition, set ``CMAKE_C_COMPILER`` and ``CMAKE_CXX_COMPILER`` to point to an installation of ``hipcc``. + +SUNDIALS has been tested with HIP versions between 5.0.0 to 5.4.3. + + .. _Installation.CMake.ExternalLibraries.RAJA: Building with RAJA diff --git a/doc/sundials_developers/source/testing/Benchmarking.rst b/doc/sundials_developers/source/testing/Benchmarking.rst index 52c12e0272..46fe92bb5f 100644 --- a/doc/sundials_developers/source/testing/Benchmarking.rst +++ b/doc/sundials_developers/source/testing/Benchmarking.rst @@ -11,6 +11,9 @@ SUNDIALS Copyright End ----------------------------------------------------------------------------- + +.. _CPT: + Continuous Performance Testing (CPT) ==================================== @@ -29,12 +32,14 @@ FUTURE: WE WILL TRACK PERFORMANCE OVER TIME WITH `CALIPER ` for more details. GitLab CI Test Script --------------------- @@ -258,7 +258,7 @@ build, and test SUNDIALS. This script leverages two Git submodules: repository that provides spack configuration files for various LLNL platfornms i.e., ``compilers.yaml`` and ``packages.yaml`` files for Quartz, Lassen, etc. -These submodues work in conjunction with ``spack_packages/sundials/package.py`` +These submodules work in conjunction with ``spack_packages/sundials/package.py`` to configure and build any third-party libraries needed by the SUNDIALS configuration and generates an initial CMake cache file for building SUNDIALS. Other packages can be added to ``spack_packages//package.py`` diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst new file mode 100644 index 0000000000..dac165f1d3 --- /dev/null +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -0,0 +1,21 @@ +.. + ----------------------------------------------------------------------------- + SUNDIALS Copyright Start + Copyright (c) 2002-2023, Lawrence Livermore National Security + and Southern Methodist University. + All rights reserved. + + See the top-level LICENSE and NOTICE files for details. + + SPDX-License-Identifier: BSD-3-Clause + SUNDIALS Copyright End + ----------------------------------------------------------------------------- + +SPOT Performance Visualization Tool +==================================== + +In order to protect against performance regression of SUNDIALS at all scales, we leverage the GitLab CI infrastructure setup for CI testing to perform continuous performance testing. + +To track and visualize performance over time, we collect data with `Caliper `_ and `Adiak `_. Steps to setting up continuous performance testing(CPT) can be found at :ref:`Continuous Performance Testing`. + +Documentation to using the SPOT performance viisualization tool can be found at `here`_. \ No newline at end of file diff --git a/doc/sundials_developers/source/testing/index.rst b/doc/sundials_developers/source/testing/index.rst index c1e537e1b3..983bdb205c 100644 --- a/doc/sundials_developers/source/testing/index.rst +++ b/doc/sundials_developers/source/testing/index.rst @@ -26,3 +26,4 @@ The following sections describe the SUNDIALS CI suite as well as a testing workf CI Local Benchmarking + Spot From 6e51a8188dd37ba1d4e711b722adadcaa273959d Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 25 May 2023 10:57:11 -0700 Subject: [PATCH 095/187] add adiak to package.py --- .gitlab/spack_packages/sundials/package.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 8fa1bb72a2..92282094a9 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -26,7 +26,8 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # ========================================================================== # Versions # ========================================================================== - version("develop", branch="develop") + version("develop", branch="feature/persistent-benchmarking-test-branch") + # version("develop", branch="develop") version("6.5.1", sha256="4252303805171e4dbdd19a01e52c1dcfe0dafc599c3cfedb0a5c2ffb045a8a75") version("6.5.0", sha256="4e0b998dff292a2617e179609b539b511eb80836f5faacf800e688a886288502") version("6.4.1", sha256="7bf10a8d2920591af3fba2db92548e91ad60eb7241ab23350a9b1bc51e05e8d0") @@ -120,6 +121,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): variant("sycl", default=False, when="@5.7.0:", description="Enable SYCL vector") # External libraries + variant("adiak", default=False, when="+caliper", description="Enable Adiak interfaces") variant( "caliper", default=False, @@ -195,9 +197,12 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): variant( "profile-examples", default=False, - when="+caliper +profiling", + when="+adiak +caliper", description="Build examples with profiling capabilities") + # Benchmarking + variant("benchmarks", default=False, description="Build benchmark programs") + # ========================================================================== # Dependencies # ========================================================================== @@ -218,8 +223,8 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("raja+rocm", when="+raja +rocm") # External libraries + depends_on("adiak", when="+adiak") depends_on("caliper", when="+caliper") - depends_on("caliper~adiak", when="+profile-examples") depends_on("ginkgo@1.5.0:", when="+ginkgo") depends_on("kokkos", when="+kokkos") depends_on("kokkos-kernels", when="+kokkos-kernels") @@ -753,10 +758,12 @@ def initconfig_package_entries(self): # Profiling self.cache_option_from_variant("SUNDIALS_BUILD_WITH_PROFILING", "profiling"), self.cache_option_from_variant("ENABLE_CALIPER", "caliper"), + self.cache_option_from_variant("ENABLE_ADIAK", "adiak"), # Benchmarking self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks"), # Profile examples - self.cache_option_from_variant("SUNDIALS_TEST_PROFILE", "profile-examples") + self.cache_option_from_variant("SUNDIALS_TEST_PROFILE", "profile-examples"), + self.cache_option_from_variant("SUNDIALS_TEST_DEVTESTS", "profile-examples") ] ) @@ -789,9 +796,15 @@ def initconfig_package_entries(self): ] ) + # Building with Adiak + if "+adiak" in spec: + entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix.lib.cmake + "/adiak")) + # Building with Caliper if "+caliper" in spec: entries.append(cmake_cache_path("CALIPER_DIR", spec["caliper"].prefix)) + if "+adiak" in spec["caliper"]: + entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix.lib.cmake + "/adiak")) # Building with Ginkgo if "+ginkgo" in spec: From 3a0d0705cf7926612de393371378f1fdec907121 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 25 May 2023 10:58:32 -0700 Subject: [PATCH 096/187] correct cmake command in build_and_test.sh --- .gitlab/build_and_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index d148995af3..7de9a99622 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -224,7 +224,8 @@ then then output_dir="/usr/workspace/sundials/caliper" $cmake_exe \ - -DSUNDIALS_TEST_OUTPUT_DIR=${output_dir} + -DSUNDIALS_TEST_OUTPUT_DIR=${output_dir} \ + "${project_dir}" fi VERBOSE_TEST=${VERBOSE_TEST:-"ON"} From 859d458fb5caae795bb0c658b66a08327bcaf4cc Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 25 May 2023 11:22:37 -0700 Subject: [PATCH 097/187] add sundials version to set of adiak values --- src/sundials/sundials_context.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 02fee306d1..10bb788136 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -231,6 +231,7 @@ static void sun_adiak_helper() { adiak_namevalue("cxx_compiler_version", 2, NULL, "%s", SUN_CXX_COMPILER_VERSION); adiak_namevalue("cxx_compiler_flags", 2, NULL, "%s", SUN_CXX_COMPILER_FLAGS); + adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); #ifdef SUNDIALS_CI_JOB From d4c8116d1540af190fc3f634525cb69545c56d44 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 26 May 2023 15:12:09 -0700 Subject: [PATCH 098/187] remove sundials version from testRunner --- test/testRunner | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/testRunner b/test/testRunner index b9d6f13f81..2d947f8208 100755 --- a/test/testRunner +++ b/test/testRunner @@ -196,11 +196,7 @@ def main(): if(args.profile): dateTime = datetime.datetime.now().strftime("%m%d%Y_%H%M%S") profilePath = os.path.join(outDir, args.testName+".%s.cali" % dateTime) - os.environ['SUNDIALS_VERSION'] = subprocess.check_output( - ['git', 'describe', '--abbrev=12', '--dirty', '--always', '--tags'] - ).decode('UTF-8').strip() os.environ['CALI_SERVICES_ENABLE'] = 'env' - os.environ['CALI_ENV_EXTRA'] = 'SUNDIALS_VERSION' os.environ['CALI_CONFIG'] = 'runtime-report,hatchet-region-profile,spot(output=%s)' % profilePath # if user supplies precision info overide the default choices From 5873f5ebb8538c759363b4cb53684211c78d347d Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 30 May 2023 11:40:44 -0700 Subject: [PATCH 099/187] swap order of libraries to link with adiak --- src/arkode/CMakeLists.txt | 2 +- src/cvode/CMakeLists.txt | 2 +- src/cvodes/CMakeLists.txt | 2 +- src/ida/CMakeLists.txt | 2 +- src/idas/CMakeLists.txt | 2 +- src/kinsol/CMakeLists.txt | 2 +- src/sundials/CMakeLists.txt | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/arkode/CMakeLists.txt b/src/arkode/CMakeLists.txt index 2e28e9df29..a30fdf7d46 100644 --- a/src/arkode/CMakeLists.txt +++ b/src/arkode/CMakeLists.txt @@ -62,7 +62,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) endif() endif() diff --git a/src/cvode/CMakeLists.txt b/src/cvode/CMakeLists.txt index 48f96e8cda..cca9057533 100644 --- a/src/cvode/CMakeLists.txt +++ b/src/cvode/CMakeLists.txt @@ -106,7 +106,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) endif() endif() diff --git a/src/cvodes/CMakeLists.txt b/src/cvodes/CMakeLists.txt index 9cbbebc8d2..d40a30989c 100644 --- a/src/cvodes/CMakeLists.txt +++ b/src/cvodes/CMakeLists.txt @@ -56,7 +56,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) endif() endif() diff --git a/src/ida/CMakeLists.txt b/src/ida/CMakeLists.txt index ac5aea7ee0..c953f7bae2 100644 --- a/src/ida/CMakeLists.txt +++ b/src/ida/CMakeLists.txt @@ -46,7 +46,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) endif() endif() diff --git a/src/idas/CMakeLists.txt b/src/idas/CMakeLists.txt index 7a8765c6e0..bb499ae044 100644 --- a/src/idas/CMakeLists.txt +++ b/src/idas/CMakeLists.txt @@ -50,7 +50,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) endif() endif() diff --git a/src/kinsol/CMakeLists.txt b/src/kinsol/CMakeLists.txt index 3a41643ab0..3fbd3092f4 100644 --- a/src/kinsol/CMakeLists.txt +++ b/src/kinsol/CMakeLists.txt @@ -44,7 +44,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRe}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) endif() endif() diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt index 9fcd6fe0a3..0681182a00 100644 --- a/src/sundials/CMakeLists.txt +++ b/src/sundials/CMakeLists.txt @@ -116,7 +116,7 @@ if(SUNDIALS_BUILD_WITH_PROFILING OR SUNDIALS_LOGGING_ENABLE_MPI) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) endif() endif() endif() From bb60ebcd819178d69fba41effb595a3b79ca4521 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 30 May 2023 15:03:33 -0700 Subject: [PATCH 100/187] add adiak cmake variables to documentation --- doc/shared/Install.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/shared/Install.rst b/doc/shared/Install.rst index 63791c2ea0..8c46c8dddd 100644 --- a/doc/shared/Install.rst +++ b/doc/shared/Install.rst @@ -1012,6 +1012,22 @@ illustration only. Default: None +.. cmakeoption:: ENABLE_ADIAK + + Enable Adiak support + + Default: OFF + + .. note:: + + Using Adiak requires setting :cmakeop:`ENABLE_CALIPER` and :cmakeop:`SUNDIALS_BUILD_WITH_PROFILING` to ``ON``. + +.. cmakeoption:: adiak_DIR + + Path to the root of an Adiak installation + + Default: None + .. cmakeoption:: SUNDIALS_F77_FUNC_CASE Specify the case to use in the Fortran name-mangling scheme, From 37b1f145df50bed035a788a169bab9b9887cd8d2 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 31 May 2023 09:43:51 -0700 Subject: [PATCH 101/187] change order of dl libs for adiak --- src/arkode/CMakeLists.txt | 2 +- src/cvode/CMakeLists.txt | 2 +- src/cvodes/CMakeLists.txt | 2 +- src/ida/CMakeLists.txt | 2 +- src/idas/CMakeLists.txt | 2 +- src/kinsol/CMakeLists.txt | 2 +- src/sundials/CMakeLists.txt | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/arkode/CMakeLists.txt b/src/arkode/CMakeLists.txt index a30fdf7d46..2e28e9df29 100644 --- a/src/arkode/CMakeLists.txt +++ b/src/arkode/CMakeLists.txt @@ -62,7 +62,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/cvode/CMakeLists.txt b/src/cvode/CMakeLists.txt index cca9057533..48f96e8cda 100644 --- a/src/cvode/CMakeLists.txt +++ b/src/cvode/CMakeLists.txt @@ -106,7 +106,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/cvodes/CMakeLists.txt b/src/cvodes/CMakeLists.txt index d40a30989c..9cbbebc8d2 100644 --- a/src/cvodes/CMakeLists.txt +++ b/src/cvodes/CMakeLists.txt @@ -56,7 +56,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/ida/CMakeLists.txt b/src/ida/CMakeLists.txt index c953f7bae2..ac5aea7ee0 100644 --- a/src/ida/CMakeLists.txt +++ b/src/ida/CMakeLists.txt @@ -46,7 +46,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/idas/CMakeLists.txt b/src/idas/CMakeLists.txt index bb499ae044..7a8765c6e0 100644 --- a/src/idas/CMakeLists.txt +++ b/src/idas/CMakeLists.txt @@ -50,7 +50,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/kinsol/CMakeLists.txt b/src/kinsol/CMakeLists.txt index 3fbd3092f4..3a41643ab0 100644 --- a/src/kinsol/CMakeLists.txt +++ b/src/kinsol/CMakeLists.txt @@ -44,7 +44,7 @@ if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRe}) - set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt index 0681182a00..9fcd6fe0a3 100644 --- a/src/sundials/CMakeLists.txt +++ b/src/sundials/CMakeLists.txt @@ -116,7 +116,7 @@ if(SUNDIALS_BUILD_WITH_PROFILING OR SUNDIALS_LOGGING_ENABLE_MPI) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${CMAKE_DL_LIBS} ${adiak_LIBARIES}) + set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) endif() endif() endif() From 33d2538ef0c474358cec5adba3567f94171a5985 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 1 Jun 2023 10:27:45 -0700 Subject: [PATCH 102/187] a few cmake fixes for setting up adiak --- cmake/SundialsSetupCompilers.cmake | 3 ++- cmake/tpl/SundialsAdiak.cmake | 9 +++------ cmake/tpl/SundialsCaliper.cmake | 1 - include/sundials/sundials_config.in | 2 +- src/sundials/sundials_context.c | 13 +++++++------ 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/cmake/SundialsSetupCompilers.cmake b/cmake/SundialsSetupCompilers.cmake index 0b6b48f5c7..3cd0c055a0 100644 --- a/cmake/SundialsSetupCompilers.cmake +++ b/cmake/SundialsSetupCompilers.cmake @@ -356,7 +356,8 @@ if(BUILD_BENCHMARKS OR EXAMPLES_ENABLE_CXX OR ENABLE_SUPERLUDIST OR ENABLE_MAGMA OR ENABLE_GINKGO OR - ENABLE_KOKKOS) + ENABLE_KOKKOS OR + ENABLE_ADIAK) include(SundialsSetupCXX) endif() diff --git a/cmake/tpl/SundialsAdiak.cmake b/cmake/tpl/SundialsAdiak.cmake index 7cef96f9f0..af7d6f4414 100644 --- a/cmake/tpl/SundialsAdiak.cmake +++ b/cmake/tpl/SundialsAdiak.cmake @@ -1,5 +1,5 @@ # ----------------------------------------------------------------------------- -# Programmer(s): Cody J. Balos @ LLNL +# Programmer(s): Yu Pan @ LLNL # ----------------------------------------------------------------------------- # SUNDIALS Copyright Start # Copyright (c) 2002-2023, Lawrence Livermore National Security @@ -40,11 +40,8 @@ endif() # Section 3: Find the TPL # ----------------------------------------------------------------------------- -find_package(adiak - PATHS "${adiak_DIR}" - NO_DEFAULT_PATH - REQUIRED) -message(STATUS "ADIAK_LIBRARIES: ${adiak_LIBRARIES}") +find_package(adiak REQUIRED) +message(STATUS "ADIAK_LIBRARIES: ${adiak_LIBRARIES}") message(STATUS "ADIAK_INCLUDE_DIR: ${adiak_INCLUDE_DIR}") diff --git a/cmake/tpl/SundialsCaliper.cmake b/cmake/tpl/SundialsCaliper.cmake index 9b883f34f3..a952a2e08b 100644 --- a/cmake/tpl/SundialsCaliper.cmake +++ b/cmake/tpl/SundialsCaliper.cmake @@ -42,7 +42,6 @@ endif() find_package(CALIPER PATHS "${CALIPER_DIR}" - NO_DEFAULT_PATH REQUIRED) message(STATUS "CALIPER_LIB_DIR: ${caliper_LIB_DIR}") diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 9f77a066f3..f9add3144c 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -162,7 +162,7 @@ /* RAJA version number*/ #define SUN_RAJA_VERSION "@RAJA_VERSION@" -/* OpenMP * +/* OpenMP */ #cmakedefine SUNDIALS_OPENMP_ENABLED #define SUN_OPENMP_VERSION "@OpenMP_VERSION@" diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 10bb788136..26c65ad4ec 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -15,10 +15,6 @@ * SUNDIALS objects in a simulation share. * ----------------------------------------------------------------*/ -#ifdef SUNDIALS_ADIAK_ENABLED -#include "adiak.h" -#endif - #include #include #include @@ -29,6 +25,11 @@ #include "sundials_context_impl.h" #include "sundials_debug.h" +#ifdef SUNDIALS_ADIAK_ENABLED +#include +void sunAdiakCollectMetadata(); +#endif + int SUNContext_Create(void* comm, SUNContext* sunctx) { SUNProfiler profiler = NULL; @@ -39,7 +40,7 @@ int SUNContext_Create(void* comm, SUNContext* sunctx) #ifdef SUNDIALS_ADIAK_ENABLED adiak_init(comm); - sun_adiak_helper(); + sunAdiakCollectMetadata(comm); #endif #if SUNDIALS_LOGGING_LEVEL > 0 @@ -217,7 +218,7 @@ int SUNContext_Free(SUNContext* sunctx) } #ifdef SUNDIALS_ADIAK_ENABLED -static void sun_adiak_helper() { +void sunAdiakCollectMetadata() { adiak_launchdate(); adiak_executable(); adiak_cmdline(); From 016fc6ddd3d8c086b0a90956ced07a5bc3b88f81 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 1 Jun 2023 10:27:57 -0700 Subject: [PATCH 103/187] I dont think we need to specify include directories anymore --- src/arkode/CMakeLists.txt | 7 +------ src/cvode/CMakeLists.txt | 7 +------ src/cvodes/CMakeLists.txt | 7 +------ src/ida/CMakeLists.txt | 7 +------ src/idas/CMakeLists.txt | 7 +------ src/kinsol/CMakeLists.txt | 7 +------ src/sundials/CMakeLists.txt | 10 +++------- 7 files changed, 9 insertions(+), 43 deletions(-) diff --git a/src/arkode/CMakeLists.txt b/src/arkode/CMakeLists.txt index 2e28e9df29..0effe06301 100644 --- a/src/arkode/CMakeLists.txt +++ b/src/arkode/CMakeLists.txt @@ -58,11 +58,9 @@ set(arkode_HEADERS add_prefix(${SUNDIALS_SOURCE_DIR}/include/arkode/ arkode_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC adiak::adiak) endif() endif() @@ -74,9 +72,6 @@ sundials_add_library(sundials_arkode ${arkode_HEADERS} INCLUDE_SUBDIR arkode - INCLUDE_DIRECTORIES - ${_include_caliper_if_needed} - ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj diff --git a/src/cvode/CMakeLists.txt b/src/cvode/CMakeLists.txt index 48f96e8cda..210db805cc 100644 --- a/src/cvode/CMakeLists.txt +++ b/src/cvode/CMakeLists.txt @@ -102,11 +102,9 @@ if(SUNDIALS_BUILD_PACKAGE_FUSED_KERNELS) endif() if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC adiak::adiak) endif() endif() @@ -118,9 +116,6 @@ sundials_add_library(sundials_cvode ${cvode_HEADERS} INCLUDE_SUBDIR cvode - INCLUDE_DIRECTORIES - ${_include_caliper_if_needed} - ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj diff --git a/src/cvodes/CMakeLists.txt b/src/cvodes/CMakeLists.txt index 9cbbebc8d2..1083d0a1d0 100644 --- a/src/cvodes/CMakeLists.txt +++ b/src/cvodes/CMakeLists.txt @@ -52,11 +52,9 @@ set(cvodes_HEADERS add_prefix(${SUNDIALS_SOURCE_DIR}/include/cvodes/ cvodes_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC adiak::adiak) endif() endif() @@ -68,9 +66,6 @@ sundials_add_library(sundials_cvodes ${cvodes_HEADERS} INCLUDE_SUBDIR cvodes - INCLUDE_DIRECTORIES - ${_include_caliper_if_needed} - ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj diff --git a/src/ida/CMakeLists.txt b/src/ida/CMakeLists.txt index ac5aea7ee0..a71ab8dae5 100644 --- a/src/ida/CMakeLists.txt +++ b/src/ida/CMakeLists.txt @@ -42,11 +42,9 @@ set(ida_HEADERS add_prefix(${SUNDIALS_SOURCE_DIR}/include/ida/ ida_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC adiak::adiak) endif() endif() @@ -58,9 +56,6 @@ sundials_add_library(sundials_ida ${ida_HEADERS} INCLUDE_SUBDIR ida - INCLUDE_DIRECTORIES - ${_include_caliper_if_needed} - ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj diff --git a/src/idas/CMakeLists.txt b/src/idas/CMakeLists.txt index 7a8765c6e0..a16906b107 100644 --- a/src/idas/CMakeLists.txt +++ b/src/idas/CMakeLists.txt @@ -46,11 +46,9 @@ set(idas_HEADERS add_prefix(${SUNDIALS_SOURCE_DIR}/include/idas/ idas_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC adiak::adiak) endif() endif() @@ -62,9 +60,6 @@ sundials_add_library(sundials_idas ${idas_HEADERS} INCLUDE_SUBDIR idas - INCLUDE_DIRECTORIES - ${_include_caliper_if_needed} - ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj diff --git a/src/kinsol/CMakeLists.txt b/src/kinsol/CMakeLists.txt index 3a41643ab0..fa18e7c30e 100644 --- a/src/kinsol/CMakeLists.txt +++ b/src/kinsol/CMakeLists.txt @@ -40,11 +40,9 @@ set(kinsol_HEADERS add_prefix(${SUNDIALS_SOURCE_DIR}/include/kinsol/ kinsol_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRe}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) + set(_link_adiak_if_needed PUBLIC adiak::adiak) endif() endif() @@ -56,9 +54,6 @@ sundials_add_library(sundials_kinsol ${kinsol_HEADERS} INCLUDE_SUBDIR kinsol - INCLUDE_DIRECTORIES - ${_include_caliper_if_needed} - ${_include_adiak_if_needed} OBJECT_LIBRARIES sundials_generic_obj sundials_sunmemsys_obj diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt index 9fcd6fe0a3..56d0024aed 100644 --- a/src/sundials/CMakeLists.txt +++ b/src/sundials/CMakeLists.txt @@ -112,12 +112,10 @@ if(SUNDIALS_BUILD_WITH_PROFILING OR SUNDIALS_LOGGING_ENABLE_MPI) set(_link_mpi_if_needed PUBLIC ${MPI_C_LIBRARIES}) endif() if(ENABLE_CALIPER) - set(_include_caliper_if_needed PUBLIC ${caliper_INCLUDE_DIR}) set(_link_caliper_if_needed PUBLIC caliper) - if(ENABLE_ADIAK) - set(_include_adiak_if_needed PUBLIC ${adiak_INCLUDE_DIRS}) - set(_link_adiak_if_needed PUBLIC ${adiak_LIBARIES} ${CMAKE_DL_LIBS}) - endif() + endif() + if(ENABLE_ADIAK) + set(_link_adiak_if_needed PRIVATE adiak::adiak) endif() endif() @@ -131,8 +129,6 @@ sundials_add_library(sundials_generic sundials INCLUDE_DIRECTORIES ${_include_mpi_if_needed} - ${_include_caliper_if_needed} - ${_include_adiak_if_needed} LINK_LIBRARIES ${_link_mpi_if_needed} ${_link_caliper_if_needed} From ca487eb866bafa8e782d25b1442bd46428bd1dbe Mon Sep 17 00:00:00 2001 From: "Cody J. Balos" Date: Thu, 1 Jun 2023 13:00:37 -0700 Subject: [PATCH 104/187] move adiak/caliper linking to sundials_add_library --- cmake/macros/SundialsAddExecutable.cmake | 6 +----- cmake/macros/SundialsAddLibrary.cmake | 11 ++++++++++- cmake/tpl/SundialsAdiak.cmake | 17 +++++++++++------ src/arkode/CMakeLists.txt | 11 ----------- src/cvode/CMakeLists.txt | 2 +- src/cvodes/CMakeLists.txt | 2 +- src/ida/CMakeLists.txt | 3 ++- src/idas/CMakeLists.txt | 2 +- src/kinsol/CMakeLists.txt | 2 +- src/sundials/CMakeLists.txt | 2 +- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cmake/macros/SundialsAddExecutable.cmake b/cmake/macros/SundialsAddExecutable.cmake index cfd44e5cb9..705bb0279d 100644 --- a/cmake/macros/SundialsAddExecutable.cmake +++ b/cmake/macros/SundialsAddExecutable.cmake @@ -38,12 +38,8 @@ macro(sundials_add_nvector_benchmark NAME) target_link_libraries(${NAME} PRIVATE ${arg_SUNDIALS_TARGETS} ${arg_LINK_LIBRARIES} -lm) - if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - target_include_directories(${NAME} PRIVATE ${caliper_INCLUDE_DIR} ${adiak_INCLUDE_DIR}) - target_link_libraries(${NAME} PRIVATE caliper adiak::adiak) - endif() - install(TARGETS ${NAME} DESTINATION "${BENCHMARKS_INSTALL_PATH}/${arg_INSTALL_SUBDIR}") endmacro(sundials_add_nvector_benchmark) + diff --git a/cmake/macros/SundialsAddLibrary.cmake b/cmake/macros/SundialsAddLibrary.cmake index 5d734ce545..96d71765ec 100644 --- a/cmake/macros/SundialsAddLibrary.cmake +++ b/cmake/macros/SundialsAddLibrary.cmake @@ -239,7 +239,7 @@ macro(sundials_add_library target) foreach(_tmp ${_all_objs}) # We use target_sources since target_link_libraries does not work # as expected with CMake 3.12 (see CMake issues 18090 and 18692). - # TODO(DJG): Update whenever we require CMake 3.14 or 3.21 + # TODO(DJG): Update whenever we require CMake 3.14 or newer target_sources(${_actual_target_name} PRIVATE $) endforeach() endif() @@ -265,6 +265,15 @@ macro(sundials_add_library target) endif() endif() + if(SUNDIALS_BUILD_WITH_PROFILING) + if(ENABLE_CALIPER) + target_link_libraries(${_actual_target_name} PUBLIC caliper) + endif() + if(ENABLE_ADIAK) + target_link_libraries(${_actual_target_name} PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) + endif() + endif() + # add common includes # Building: public, config/export generated, and shared private headers # Installing: installed include directory diff --git a/cmake/tpl/SundialsAdiak.cmake b/cmake/tpl/SundialsAdiak.cmake index af7d6f4414..5850f3060b 100644 --- a/cmake/tpl/SundialsAdiak.cmake +++ b/cmake/tpl/SundialsAdiak.cmake @@ -56,19 +56,24 @@ if(adiak_FOUND AND (NOT adiak_WORKS)) set(adiak_TEST_DIR ${PROJECT_BINARY_DIR}/adiak_TEST) file(MAKE_DIRECTORY ${adiak_TEST_DIR}) - # Create a CMakeLists.txt file - file(WRITE ${adiak_TEST_DIR}/CMakeLists.txt "") - # Create a C source file - file(WRITE ${adiak_TEST_DIR}/ltest.c "") + file(WRITE ${adiak_TEST_DIR}/ltest.c + "\#include \n" + "int main()\n" + "{\n" + " adiak_init(NULL);\n" + " adiak_fini();\n" + " return 0;\n" + "}\n") # To ensure we do not use stuff from the previous attempts, # we must remove the CMakeFiles directory. file(REMOVE_RECURSE ${adiak_TEST_DIR}/CMakeFiles) # Attempt to build and link the "ltest" executable - try_compile(COMPILE_OK ${adiak_TEST_DIR} ${adiak_TEST_DIR} ltest - OUTPUT_VARIABLE COMPILE_OUTPUT) + try_compile(COMPILE_OK ${adiak_TEST_DIR} ${adiak_TEST_DIR}/ltest.c + OUTPUT_VARIABLE COMPILE_OUTPUT + LINK_LIBRARIES adiak::adiak ${CMAKE_DL_LIBS}) # Process test result if(COMPILE_OK) diff --git a/src/arkode/CMakeLists.txt b/src/arkode/CMakeLists.txt index 0effe06301..ab86aba0c4 100644 --- a/src/arkode/CMakeLists.txt +++ b/src/arkode/CMakeLists.txt @@ -57,13 +57,6 @@ set(arkode_HEADERS # Add prefix with complete path to the ARKODE header files add_prefix(${SUNDIALS_SOURCE_DIR}/include/arkode/ arkode_HEADERS) -if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_link_caliper_if_needed PUBLIC caliper) - if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak) - endif() -endif() - # Create the sundials_arkode library sundials_add_library(sundials_arkode SOURCES @@ -88,10 +81,6 @@ sundials_add_library(sundials_arkode sundials_sunlinsolpcg_obj sundials_sunnonlinsolnewton_obj sundials_sunnonlinsolfixedpoint_obj - LINK_LIBRARIES - # Link to caliper if conditions are met. - ${_link_caliper_if_needed} - ${_link_adiak_if_needed} OUTPUT_NAME sundials_arkode VERSION diff --git a/src/cvode/CMakeLists.txt b/src/cvode/CMakeLists.txt index 210db805cc..1f11a9d853 100644 --- a/src/cvode/CMakeLists.txt +++ b/src/cvode/CMakeLists.txt @@ -104,7 +104,7 @@ endif() if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak) + set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/cvodes/CMakeLists.txt b/src/cvodes/CMakeLists.txt index 1083d0a1d0..1d107c7711 100644 --- a/src/cvodes/CMakeLists.txt +++ b/src/cvodes/CMakeLists.txt @@ -54,7 +54,7 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/cvodes/ cvodes_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak) + set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/ida/CMakeLists.txt b/src/ida/CMakeLists.txt index a71ab8dae5..de686ef067 100644 --- a/src/ida/CMakeLists.txt +++ b/src/ida/CMakeLists.txt @@ -44,7 +44,7 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/ida/ ida_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak) + set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) endif() endif() @@ -75,6 +75,7 @@ sundials_add_library(sundials_ida LINK_LIBRARIES # Link to caliper if conditions are met. ${_link_caliper_if_needed} + ${_link_adiak_if_needed} OUTPUT_NAME sundials_ida VERSION diff --git a/src/idas/CMakeLists.txt b/src/idas/CMakeLists.txt index a16906b107..a41c697871 100644 --- a/src/idas/CMakeLists.txt +++ b/src/idas/CMakeLists.txt @@ -48,7 +48,7 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/idas/ idas_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak) + set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/kinsol/CMakeLists.txt b/src/kinsol/CMakeLists.txt index fa18e7c30e..87ed908a9d 100644 --- a/src/kinsol/CMakeLists.txt +++ b/src/kinsol/CMakeLists.txt @@ -42,7 +42,7 @@ add_prefix(${SUNDIALS_SOURCE_DIR}/include/kinsol/ kinsol_HEADERS) if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) set(_link_caliper_if_needed PUBLIC caliper) if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak) + set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) endif() endif() diff --git a/src/sundials/CMakeLists.txt b/src/sundials/CMakeLists.txt index 56d0024aed..16dd005cee 100644 --- a/src/sundials/CMakeLists.txt +++ b/src/sundials/CMakeLists.txt @@ -115,7 +115,7 @@ if(SUNDIALS_BUILD_WITH_PROFILING OR SUNDIALS_LOGGING_ENABLE_MPI) set(_link_caliper_if_needed PUBLIC caliper) endif() if(ENABLE_ADIAK) - set(_link_adiak_if_needed PRIVATE adiak::adiak) + set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) endif() endif() From dd3a2756def5521f5a610bed562ef31761375c3e Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 1 Jun 2023 13:04:21 -0700 Subject: [PATCH 105/187] remove caliper/adiak linking from individual packages --- src/cvode/CMakeLists.txt | 10 ---------- src/cvodes/CMakeLists.txt | 11 ----------- src/ida/CMakeLists.txt | 11 ----------- src/idas/CMakeLists.txt | 11 ----------- src/kinsol/CMakeLists.txt | 11 ----------- 5 files changed, 54 deletions(-) diff --git a/src/cvode/CMakeLists.txt b/src/cvode/CMakeLists.txt index 1f11a9d853..bcdbe0d513 100644 --- a/src/cvode/CMakeLists.txt +++ b/src/cvode/CMakeLists.txt @@ -101,13 +101,6 @@ if(SUNDIALS_BUILD_PACKAGE_FUSED_KERNELS) set(_fused_link_lib sundials_cvode_fused_stubs) endif() -if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_link_caliper_if_needed PUBLIC caliper) - if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) - endif() -endif() - # Create the library sundials_add_library(sundials_cvode SOURCES @@ -135,9 +128,6 @@ sundials_add_library(sundials_cvode LINK_LIBRARIES # Link to stubs so examples work. PRIVATE ${_fused_link_lib} - # Link to caliper if conditions are met. - ${_link_caliper_if_needed} - ${_link_adiak_if_needed} OUTPUT_NAME sundials_cvode VERSION diff --git a/src/cvodes/CMakeLists.txt b/src/cvodes/CMakeLists.txt index 1d107c7711..480830056d 100644 --- a/src/cvodes/CMakeLists.txt +++ b/src/cvodes/CMakeLists.txt @@ -51,13 +51,6 @@ set(cvodes_HEADERS # Add prefix with complete path to the CVODES header files add_prefix(${SUNDIALS_SOURCE_DIR}/include/cvodes/ cvodes_HEADERS) -if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_link_caliper_if_needed PUBLIC caliper) - if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) - endif() -endif() - # Create the library sundials_add_library(sundials_cvodes SOURCES @@ -82,10 +75,6 @@ sundials_add_library(sundials_cvodes sundials_sunlinsolpcg_obj sundials_sunnonlinsolnewton_obj sundials_sunnonlinsolfixedpoint_obj - LINK_LIBRARIES - # Link to caliper if conditions are met. - ${_link_caliper_if_needed} - ${_link_adiak_if_needed} OUTPUT_NAME sundials_cvodes VERSION diff --git a/src/ida/CMakeLists.txt b/src/ida/CMakeLists.txt index de686ef067..120f3f9687 100644 --- a/src/ida/CMakeLists.txt +++ b/src/ida/CMakeLists.txt @@ -41,13 +41,6 @@ set(ida_HEADERS # Add prefix with complete path to the IDA header files add_prefix(${SUNDIALS_SOURCE_DIR}/include/ida/ ida_HEADERS) -if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_link_caliper_if_needed PUBLIC caliper) - if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) - endif() -endif() - # Create the library sundials_add_library(sundials_ida SOURCES @@ -72,10 +65,6 @@ sundials_add_library(sundials_ida sundials_sunlinsolpcg_obj sundials_sunnonlinsolnewton_obj sundials_sunnonlinsolfixedpoint_obj - LINK_LIBRARIES - # Link to caliper if conditions are met. - ${_link_caliper_if_needed} - ${_link_adiak_if_needed} OUTPUT_NAME sundials_ida VERSION diff --git a/src/idas/CMakeLists.txt b/src/idas/CMakeLists.txt index a41c697871..c825d114c1 100644 --- a/src/idas/CMakeLists.txt +++ b/src/idas/CMakeLists.txt @@ -45,13 +45,6 @@ set(idas_HEADERS # Add prefix with complete path to the IDAS header files add_prefix(${SUNDIALS_SOURCE_DIR}/include/idas/ idas_HEADERS) -if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_link_caliper_if_needed PUBLIC caliper) - if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) - endif() -endif() - # Create the library sundials_add_library(sundials_idas SOURCES @@ -76,10 +69,6 @@ sundials_add_library(sundials_idas sundials_sunlinsolpcg_obj sundials_sunnonlinsolnewton_obj sundials_sunnonlinsolfixedpoint_obj - LINK_LIBRARIES - # Link to caliper if conditions are met. - ${_link_caliper_if_needed} - ${_link_adiak_if_needed} OUTPUT_NAME sundials_idas VERSION diff --git a/src/kinsol/CMakeLists.txt b/src/kinsol/CMakeLists.txt index 87ed908a9d..20fd1572ec 100644 --- a/src/kinsol/CMakeLists.txt +++ b/src/kinsol/CMakeLists.txt @@ -39,13 +39,6 @@ set(kinsol_HEADERS # Add prefix with complete path to the KINSOL header files add_prefix(${SUNDIALS_SOURCE_DIR}/include/kinsol/ kinsol_HEADERS) -if(ENABLE_CALIPER AND SUNDIALS_BUILD_WITH_PROFILING) - set(_link_caliper_if_needed PUBLIC caliper) - if(ENABLE_ADIAK) - set(_link_adiak_if_needed PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) - endif() -endif() - # Create the library sundials_add_library(sundials_kinsol SOURCES @@ -68,10 +61,6 @@ sundials_add_library(sundials_kinsol sundials_sunlinsolspgmr_obj sundials_sunlinsolsptfqmr_obj sundials_sunlinsolpcg_obj - LINK_LIBRARIES - # Link to caliper if conditions are met. - ${_link_caliper_if_needed} - ${_link_adiak_if_needed} OUTPUT_NAME sundials_kinsol VERSION From f362e0f54b46505196f93c790f7d38cc3a91a580 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 5 Jun 2023 09:53:17 -0700 Subject: [PATCH 106/187] adjust CALI_CONFIG to only SPOT for test_runner --- test/testRunner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testRunner b/test/testRunner index 2d947f8208..15af79985e 100755 --- a/test/testRunner +++ b/test/testRunner @@ -197,7 +197,7 @@ def main(): dateTime = datetime.datetime.now().strftime("%m%d%Y_%H%M%S") profilePath = os.path.join(outDir, args.testName+".%s.cali" % dateTime) os.environ['CALI_SERVICES_ENABLE'] = 'env' - os.environ['CALI_CONFIG'] = 'runtime-report,hatchet-region-profile,spot(output=%s)' % profilePath + os.environ['CALI_CONFIG'] = 'spot(output=%s)' % profilePath # if user supplies precision info overide the default choices if(args.floatPrecision >= 0): From 9a9e2d6943406f515d0bf68f84fbe352844eb4e1 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 5 Jun 2023 10:02:58 -0700 Subject: [PATCH 107/187] update for rocmcc 5.5.0 --- .gitlab/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 6f22afc1ec..88cffd08f7 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 6f22afc1ece86c479d2d2a64e14736ef00b632d6 +Subproject commit 88cffd08f7c9a0afc73ff9b47b3abdab8a6affbf From e9bcef492be356cf5fc6ddef91dbcfcbcb501967 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 5 Jun 2023 10:20:02 -0700 Subject: [PATCH 108/187] add job to test CI_JOB_ID to metadata --- .gitlab-ci.yml | 2 +- .gitlab/build_and_test.sh | 1 + .gitlab/corona-jobs.yml | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf5aeb3712..5b7be848cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,7 +79,7 @@ variables: VERBOSE_TEST: "OFF" ON_LASSEN: "OFF" ON_QUARTZ: "OFF" - ON_CORONA: "OFF" + ON_CORONA: "ON" SPACK_PREFIX: "v0.19.1" SHARED_SPACK: "UPSTREAM" BENCHMARK: "OFF" diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 7de9a99622..cadb3149a2 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -190,6 +190,7 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DSUNDIALS_TEST_OUTPUT_DIR="/usr/workspace/sundials/caliper" \ "${project_dir}" # build diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 450a9197ba..03c5d28a50 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -35,3 +35,16 @@ corona_clang_hip_tpls: variables: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}" extends: .corona_build_and_test + +# ------------------------------------------------------------------------------ +# Corona Profile Example Jobs +# ------------------------------------------------------------------------------ + +corona_test_profile: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@5.4.1 + AMDGPU_TARGET: [gfx906] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+adiak+caliper+profile-examples" + extends: .corona_build_and_test \ No newline at end of file From bc3123b5dc7020df7233a4d0e90c74ad88d0caa9 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 5 Jun 2023 13:56:14 -0700 Subject: [PATCH 109/187] fix ci_job_id to be usable as variable in c files --- include/sundials/sundials_config.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index f9add3144c..6f68669eeb 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -113,7 +113,7 @@ #define SUN_BUILD_TYPE "@CMAKE_BUILD_TYPE@" #cmakedefine SUNDIALS_CI_JOB -#cmakedefine SUN_CI_JOB_ID +#define SUN_CI_JOB_ID "@SUN_CI_JOB_ID@" /* ------------------------------------------------------------------ * SUNDIALS TPL macros From 7308c2e8b905d86cef91fc1116ee891863a4c520 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 5 Jun 2023 14:12:33 -0700 Subject: [PATCH 110/187] remove rocm loading; put test output into unique directories --- .gitlab/build_and_test.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index cadb3149a2..d64af2bc81 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -51,8 +51,6 @@ BUILD_JOBS=${BUILD_JOBS:-"1"} if [[ "${hostname}" == "corona" ]]; then echo "module load python/3.9.12" module load python/3.9.12 - echo "module load rocm/5.4.1" - module load rocm/5.4.1 else echo "module load python/3.8.2" module load python/3.8.2 @@ -190,7 +188,7 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ - -DSUNDIALS_TEST_OUTPUT_DIR="/usr/workspace/sundials/caliper" \ + -DSUNDIALS_TEST_OUTPUT_DIR="/usr/workspace/sundials/caliper/run_${job_unique_id}" \ "${project_dir}" # build From 5f4d73028847d7d294c3ff904cd713001a8505e9 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 5 Jun 2023 14:27:34 -0700 Subject: [PATCH 111/187] add HIP cmake options to documentation --- doc/shared/Install.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/shared/Install.rst b/doc/shared/Install.rst index 8c46c8dddd..aff9ee1cb9 100644 --- a/doc/shared/Install.rst +++ b/doc/shared/Install.rst @@ -634,6 +634,18 @@ illustration only. Default: None +.. cmakeoption:: ENABLE_HIP + + Enable HIP Support + + Default: ``OFF`` + +.. cmakeoption:: AMDGPU_TARGETS + + Specify which AMDGPU processor(s) to target. + + Default: None + .. cmakeoption:: ENABLE_HYPRE Flag to enable *hypre* support From b3af82a9336e005521bd54488985b3c05dbff316 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 7 Jun 2023 11:16:46 -0700 Subject: [PATCH 112/187] add spack version to adiak metadata --- .gitlab/spack_packages/sundials/package.py | 3 ++- include/sundials/sundials_config.in | 2 ++ src/sundials/sundials_context.c | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 92282094a9..6697a13247 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -763,7 +763,8 @@ def initconfig_package_entries(self): self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks"), # Profile examples self.cache_option_from_variant("SUNDIALS_TEST_PROFILE", "profile-examples"), - self.cache_option_from_variant("SUNDIALS_TEST_DEVTESTS", "profile-examples") + self.cache_option_from_variant("SUNDIALS_TEST_DEVTESTS", "profile-examples"), + cmake_cache_string("SPACK_VERSION", ".".join(map(str, spack.spack_version_info))) ] ) diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 6f68669eeb..aec69b238d 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -115,6 +115,8 @@ #cmakedefine SUNDIALS_CI_JOB #define SUN_CI_JOB_ID "@SUN_CI_JOB_ID@" +#define SUNDIALS_SPACK_VERSION "@SPACK_VERSION@" + /* ------------------------------------------------------------------ * SUNDIALS TPL macros * -----------------------------------------------------------------*/ diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 26c65ad4ec..5084c541a7 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -235,6 +235,10 @@ void sunAdiakCollectMetadata() { adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); +#ifdef SUNDIALS_SPACK_VERSION + adiak_namevalue("spack_version", 2, NULL, "%s", SUNDIALS_SPACK_VERSION); +#endif + #ifdef SUNDIALS_CI_JOB adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); #endif From 8349dcaf588b40682bd4f272e230b3c720832507 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 7 Jun 2023 15:10:50 -0700 Subject: [PATCH 113/187] move unique run folders to within Testing --- .gitlab/build_and_test.sh | 12 ++---------- cmake/SundialsSetupTesting.cmake | 10 ++++++++-- cmake/macros/SundialsAddTest.cmake | 6 +++++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index d64af2bc81..d85d331190 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -51,6 +51,7 @@ BUILD_JOBS=${BUILD_JOBS:-"1"} if [[ "${hostname}" == "corona" ]]; then echo "module load python/3.9.12" module load python/3.9.12 + module load rocm/5.4.1 else echo "module load python/3.8.2" module load python/3.8.2 @@ -188,7 +189,7 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ - -DSUNDIALS_TEST_OUTPUT_DIR="/usr/workspace/sundials/caliper/run_${job_unique_id}" \ + -DSUNDIALS_TEST_OUTPUT_DIR="/usr/workspace/sundials/caliper/" \ "${project_dir}" # build @@ -218,15 +219,6 @@ then cd "${build_dir}" - # configure .cali file output directory - if [[ "${BENCHMARK}" == "ON" ]] && [[ -d /usr/workspace/sundials ]] - then - output_dir="/usr/workspace/sundials/caliper" - $cmake_exe \ - -DSUNDIALS_TEST_OUTPUT_DIR=${output_dir} \ - "${project_dir}" - fi - VERBOSE_TEST=${VERBOSE_TEST:-"ON"} if [[ "${VERBOSE_TEST}" == "ON" ]]; then verbose_test='--verbose' diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 9e67b4dae8..44f5ad34fa 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -49,8 +49,14 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default output directory was provided make sure it exists if(SUNDIALS_TEST_OUTPUT_DIR) message(STATUS "Using non-default test output directory: ${SUNDIALS_TEST_OUTPUT_DIR}") - if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) - file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) + if(SUN_CI_JOB_ID) + if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) + file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) + endif() + else() + if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) + file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) + endif() endif() endif() diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index 462488cb1d..cfa7a1ba09 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -96,7 +96,11 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) # check for a non-default output directory if(SUNDIALS_TEST_OUTPUT_DIR) - list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing") + if(SUN_CI_JOB_ID) + list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}") + else() + list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing") + endif() else() list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}") endif() From 6c089bd44922c8db48f6e6c999b291b80cb76d24 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 9 Jun 2023 15:15:13 -0700 Subject: [PATCH 114/187] add tpls to adiak metadata --- cmake/SundialsSetupConfig.cmake | 34 ++--------- include/sundials/sundials_config.in | 90 ++++++++++++++++++++++++----- src/sundials/sundials_context.c | 66 ++++++++++++++++++++- 3 files changed, 145 insertions(+), 45 deletions(-) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 89f1c3a04a..230afc08ee 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -58,41 +58,17 @@ foreach(_item ${SUNDIALS_BUILD_LIST}) endif() endforeach() -# prepare substitution variable SUNDIALS_CALIPER_ENABLED for sundials_config.h -if(ENABLE_CALIPER) - set(SUNDIALS_CALIPER_ENABLED TRUE) -endif() - -# prepare substitution variable SUNDIALS_ADIAK_ENABLED for sundials_config.h -if(ENABLE_ADIAK) - set(SUNDIALS_ADIAK_ENABLED TRUE) -endif() - -# prepare substitution variable SUNDIALS_MPI_ENABLED for sundials_config.h -if(ENABLE_MPI) - set(SUNDIALS_MPI_ENABLED TRUE) -endif() +# prepare substitution variable SUNDIALS_${TPL NAME}_ENABLED for sundials_config.h +foreach(tpl ${SUNDIALS_TPL_LIST}) + set(SUNDIALS_${tpl}_ENABLED TRUE) +endforeach() +list(LENGTH SUNDIALS_TPL_LIST SUNDIALS_TPL_LIST_SIZE) # prepare substitution variable SUNDIALS_TRILINOS_HAVE_MPI for sundials_config.h if(Trilinos_MPI) set(SUNDIALS_TRILINOS_HAVE_MPI TRUE) endif() -# prepare substitution variable SUNDIALS_RAJA_ENABLED for sundials_config.h -if(ENABLE_RAJA) - set(SUNDIALS_RAJA_ENABLED TRUE) -endif() - -# prepare substitution variable SUNDIALS_CUDA_ENABLED for sundials_config.h -if(ENABLE_CUDA) - set(SUNDIALS_CUDA_ENABLED TRUE) -endif() - -# prepare substitution variable SUNDIALS_MPI_ENABLED for sundials_config.h -if(ENABLE_HIP) - set(SUNDIALS_HIP_ENABLED TRUE) -endif() - # prepare substitution variable(s) SUNDIALS_RAJA_BACKENDS_* foreach(backend ${SUNDIALS_RAJA_BACKENDS}) set(SUNDIALS_RAJA_BACKENDS_${backend} TRUE) diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index aec69b238d..8306700929 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -115,6 +115,9 @@ #cmakedefine SUNDIALS_CI_JOB #define SUN_CI_JOB_ID "@SUN_CI_JOB_ID@" +#define SUN_TPL_LIST "@SUNDIALS_TPL_LIST@" +#define SUN_TPL_LIST_SIZE "@SUNDIALS_TPL_LIST_SIZE@" + #define SUNDIALS_SPACK_VERSION "@SPACK_VERSION@" /* ------------------------------------------------------------------ @@ -127,6 +130,79 @@ /* Adiak */ #cmakedefine SUNDIALS_ADIAK_ENABLED +/* Ginkgo */ +#cmakedefine SUNDIALS_GINKGO_ENABLED +#define SUN_GINKGO_VERSION "@Ginkgo_VERSION@" + +/* HYPRE */ +#cmakedefine SUNDIALS_HYPRE_ENABLED +#define SUN_HYPRE_VERSION "@HYPRE_VERSION@" + +/* KLU */ +#cmakedefine SUNDIALS_KLU_ENABLED +#define SUN_KLU_VERSION "@KLU_VERSION@" + +/* KOKKOS */ +#cmakedefine SUNDIALS_KOKKOS_ENABLED +#define SUN_KOKKOS_VERSION "@Kokkos_VERSION@" + +/* KOKKOS_KERNELS */ +#cmakedefine SUNDIALS_KOKKOS_KERNELS_ENABLED +#define SUN_KOKKOS_KERNELS_VERSION "@KokkosKernels_VERSION@" + +/* LAPACK */ +#cmakedefine SUNDIALS_BLAS_LAPACK_ENABLED +#define SUN_LAPACK_VERSION "@LAPACK_VERSION@" + +/* MAGMA */ +#cmakedefine SUNDIALS_MAGMA_ENABLED +#define SUN_MAGMA_VERSION "@MAGMA_VERSION@" + +/* MPI */ +#cmakedefine SUNDIALS_MPI_ENABLED +#define SUN_MPI_VERSION "@MPI_VERSION@" + +/* ONEMKL */ +#cmakedefine SUNDIALS_ONEMKL_ENABLED +#define SUN_ONEMKL_VERSION "@MKL_VERSION@" + +/* OpenMP */ +#cmakedefine SUNDIALS_OPENMP_ENABLED +#define SUN_OPENMP_VERSION "@OpenMP_VERSION@" + +/* PETSC */ +#cmakedefine SUNDIALS_PETSC_ENABLED +#define SUN_PETSC_VERSION "@PETSC_VERSION@" + +/* PTHREADS */ +#cmakedefine SUNDIALS_PTHREADS_ENABLED +#define SUN_PTHREADS_VERSION "@Threads_VERSION@" + +/* RAJA */ +#cmakedefine SUNDIALS_RAJA_ENABLED +#define SUN_RAJA_VERSION "@RAJA_VERSION@" + +/* SUPERLUDIST */ +#cmakedefine SUNDIALS_SUPERLUDIST_ENABLED +#define SUN_SUPERLUDIST_VERSION "@SUPERLUDIST_VERSION@" + +/* SUPERLUMT */ +#cmakedefine SUNDIALS_SUPERLUMT_ENABLED +#define SUN_SUPERLUMT_VERSION "@SUPERLUMT_VERSION@" + +/* TRILLINOS */ +#cmakedefine SUNDIALS_TRILLINOS_ENABLED +#define SUN_TRILLINOS_VERSION "@Trillinos_VERSION@" + +/* XBRAID */ +#cmakedefine SUNDIALS_XBRAID_ENABLED +#define SUN_XBRAID_VERSION "@XBRAID_VERSION@" + +/* RAJA backends */ +#cmakedefine SUNDIALS_RAJA_BACKENDS_CUDA +#cmakedefine SUNDIALS_RAJA_BACKENDS_HIP +#cmakedefine SUNDIALS_RAJA_BACKENDS_SYCL + /* Ginkgo backends */ #cmakedefine SUNDIALS_GINKGO_BACKENDS_CUDA #cmakedefine SUNDIALS_GINKGO_BACKENDS_HIP @@ -153,20 +229,6 @@ */ #cmakedefine SUNDIALS_TRILINOS_HAVE_MPI -/* RAJA */ -#cmakedefine SUNDIALS_RAJA_ENABLED - -/* RAJA backends */ -#cmakedefine SUNDIALS_RAJA_BACKENDS_CUDA -#cmakedefine SUNDIALS_RAJA_BACKENDS_HIP -#cmakedefine SUNDIALS_RAJA_BACKENDS_SYCL - -/* RAJA version number*/ -#define SUN_RAJA_VERSION "@RAJA_VERSION@" - -/* OpenMP */ -#cmakedefine SUNDIALS_OPENMP_ENABLED -#define SUN_OPENMP_VERSION "@OpenMP_VERSION@" /* ------------------------------------------------------------------ * SUNDIALS language macros diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 5084c541a7..bd6e1db623 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -234,6 +234,7 @@ void sunAdiakCollectMetadata() { adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); + adiak_namevalue("third_party_libraries", 2, NULL "{}", SUN_TPL_LIST, SUN_TPL_LIST_SIZE); #ifdef SUNDIALS_SPACK_VERSION adiak_namevalue("spack_version", 2, NULL, "%s", SUNDIALS_SPACK_VERSION); @@ -243,14 +244,75 @@ void sunAdiakCollectMetadata() { adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); #endif -#ifdef SUNDIALS_RAJA_ENABLED - adiak_namevalue("raja_version", 2, NULL, "%s", SUN_RAJA_VERSION); +#ifdef SUNDIALS_GINGKO_ENABLED + adiak_namevalue("ginkgo_version", 2, NULL, "%s", SUN_GINKGO_VERSION); +#endif + +#ifdef SUNDIALS_HYPRE_ENABLED + adiak_namevalue("hypre_version", 2, NULL, "%s", SUN_HYPRE_VERSION); +#endif + +#ifdef SUNDIALS_KLU_ENABLED + adiak_namevalue("klu_version", 2, NULL, "%s", SUN_KLU_VERSION); +#endif + +#ifdef SUNDIALS_KOKKOS_ENABLED + adiak_namevalue("kokkos_version", 2, NULL, "%s", SUN_KOKKOS_VERSION); +#endif + +#ifdef SUNDIALS_KOKKOS_KERNELS_ENABLED + adiak_namevalue("kokkos_kernels_version", 2, NULL, "%s", SUN_KOKKOS_KERNELS_VERSION); +#endif + +#ifdef SUNDIALS_BLAS_LAPACK_ENABLED + adiak_namevalue("lapack_version", 2, NULL, "%s", SUN_LAPACK_VERSION); +#endif + +#ifdef SUNDIALS_MAGMA_ENABLED + adiak_namevalue("magma_version", 2, NULL, "%s", SUN_MAGMA_VERSION); +#endif + +// needs a further look +#ifdef SUNDIALS_MPI_ENABLED + adiak_namevalue("mpi_version", 2, NULL, "%s", SUN_MPI_VERSION); +#endif + +#ifdef SUNDIALS_ONEMKL_ENABLED + adiak_namevalue("onemkl_version", 2, NULL, "%s", SUN_ONEMKL_VERSION); #endif #ifdef SUNDIALS_OPENMP_ENABLED adiak_namevalue("openmp_version", 2, NULL, "%s", SUN_OPENMP_VERSION); #endif +#ifdef SUNDIALS_PETSC_ENABLED + adiak_namevalue("petsc_version", 2, NULL, "%s", SUN_PETSC_VERSION); +#endif + +#ifdef SUNDIALS_PTHREADS_ENABLED + adiak_namevalue("pthreads_version", 2, NULL, "%s", SUN_PTHREADS_VERSION); +#endif + +#ifdef SUNDIALS_RAJA_ENABLED + adiak_namevalue("raja_version", 2, NULL, "%s", SUN_RAJA_VERSION); +#endif + +#ifdef SUNDIALS_SUPERLUDIST_ENABLED + adiak_namevalue("superludist_version", 2, NULL, "%s", SUN_SUPERLUDIST_VERSION); +#endif + +#ifdef SUNDIALS_SUPERLUMT_ENABLED + adiak_namevalue("superlumt_version", 2, NULL, "%s", SUN_SUPERLUMT_VERSION); +#endif + +#ifdef SUNDIALS_TRILLINOS_ENABLED + adiak_namevalue("trillinos_version", 2, NULL, "%s", SUN_TRILLINOS_VERSION); +#endif + +#ifdef SUNDIALS_XBRAID_ENABLED + adiak_namevalue("xbraid_version", 2, NULL, "%s", SUN_XBRAID_VERSION); +#endif + #ifdef SUNDIALS_CUDA_ENABLED adiak_namevalue("cuda_version", 2, NULL, "%s", SUN_CUDA_VERSION); #endif From 42352223b9af34f1ca7ab1bfb640f1766bf30ace Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 9 Jun 2023 15:15:52 -0700 Subject: [PATCH 115/187] fix a typo --- src/sundials/sundials_context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index bd6e1db623..42a15932c3 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -234,7 +234,7 @@ void sunAdiakCollectMetadata() { adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); - adiak_namevalue("third_party_libraries", 2, NULL "{}", SUN_TPL_LIST, SUN_TPL_LIST_SIZE); + adiak_namevalue("third_party_libraries", 2, NULL "{%s}", SUN_TPL_LIST, SUN_TPL_LIST_SIZE); #ifdef SUNDIALS_SPACK_VERSION adiak_namevalue("spack_version", 2, NULL, "%s", SUNDIALS_SPACK_VERSION); From 915b9f9448c2ab02158afe65433a600400b04683 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 12 Jun 2023 11:27:07 -0700 Subject: [PATCH 116/187] add tpl list to adiak metadata --- include/sundials/sundials_config.in | 1 - src/sundials/sundials_context.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 8306700929..4d65906f63 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -159,7 +159,6 @@ #define SUN_MAGMA_VERSION "@MAGMA_VERSION@" /* MPI */ -#cmakedefine SUNDIALS_MPI_ENABLED #define SUN_MPI_VERSION "@MPI_VERSION@" /* ONEMKL */ diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 42a15932c3..d54da943f2 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -234,7 +234,7 @@ void sunAdiakCollectMetadata() { adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); - adiak_namevalue("third_party_libraries", 2, NULL "{%s}", SUN_TPL_LIST, SUN_TPL_LIST_SIZE); + adiak_namevalue("third_party_libraries", 2, NULL, "%s", SUN_TPL_LIST); #ifdef SUNDIALS_SPACK_VERSION adiak_namevalue("spack_version", 2, NULL, "%s", SUNDIALS_SPACK_VERSION); From b3532458a48b459776739fd8204ec67891b3f6c5 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 12 Jun 2023 11:37:07 -0700 Subject: [PATCH 117/187] add MPI version and remove irrelevent metadata --- cmake/SundialsSetupConfig.cmake | 1 - include/sundials/sundials_config.in | 2 +- src/sundials/sundials_context.c | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 230afc08ee..32747b308c 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -62,7 +62,6 @@ endforeach() foreach(tpl ${SUNDIALS_TPL_LIST}) set(SUNDIALS_${tpl}_ENABLED TRUE) endforeach() -list(LENGTH SUNDIALS_TPL_LIST SUNDIALS_TPL_LIST_SIZE) # prepare substitution variable SUNDIALS_TRILINOS_HAVE_MPI for sundials_config.h if(Trilinos_MPI) diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 4d65906f63..fbf23cf604 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -159,7 +159,7 @@ #define SUN_MAGMA_VERSION "@MAGMA_VERSION@" /* MPI */ -#define SUN_MPI_VERSION "@MPI_VERSION@" +#define SUN_MPI_VERSION "@MPI_CXX_VERSION@" /* ONEMKL */ #cmakedefine SUNDIALS_ONEMKL_ENABLED diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index d54da943f2..9006e1f22c 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -225,7 +225,6 @@ void sunAdiakCollectMetadata() { adiak_clustername(); adiak_job_size(); - adiak_hostlist(); adiak_num_hosts(); adiak_namevalue("cxx_compiler", 2, NULL, "%s", SUN_CXX_COMPILER); From d91c7d332e277e7c0385388c0f85adc8c6911818 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 12 Jun 2023 13:37:12 -0700 Subject: [PATCH 118/187] update radiuss --- .gitlab/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 88cffd08f7..9096bc19c5 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 88cffd08f7c9a0afc73ff9b47b3abdab8a6affbf +Subproject commit 9096bc19c5a9aad3afb895cee2c93715d4f8e036 From c43f83bcb2e9d70cf20b0b7bbdcc2084f3f89b98 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 12 Jun 2023 15:18:29 -0700 Subject: [PATCH 119/187] fix typo --- src/sundials/sundials_context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 9006e1f22c..8f8c811e23 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -243,7 +243,7 @@ void sunAdiakCollectMetadata() { adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); #endif -#ifdef SUNDIALS_GINGKO_ENABLED +#ifdef SUNDIALS_GINKGO_ENABLED adiak_namevalue("ginkgo_version", 2, NULL, "%s", SUN_GINKGO_VERSION); #endif From 025115824107a1b6888569a8aeee1cc5e5976798 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 15 Jun 2023 16:36:42 -0700 Subject: [PATCH 120/187] add more compiler metadata --- .gitlab/radiuss-spack-configs | 2 +- include/sundials/sundials_config.in | 20 +++++++++++++++++++- src/sundials/sundials_context.c | 22 ++++++++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 9096bc19c5..6534712a8e 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 9096bc19c5a9aad3afb895cee2c93715d4f8e036 +Subproject commit 6534712a8e5e831a9d47738e52c7ab94d2c4f61e diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index fbf23cf604..efde48e0c3 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -106,10 +106,18 @@ #endif /* Build metadata */ +#define SUN_C_COMPILER "@CMAKE_C_COMPILER_ID@" +#define SUN_C_COMPILER_VERSION "@CMAKE_C_COMPILER_VERSION@" +#define SUN_C_COMPILER_FLAGS "@CMAKE_C_FLAGS@" + #define SUN_CXX_COMPILER "@CMAKE_CXX_COMPILER_ID@" #define SUN_CXX_COMPILER_VERSION "@CMAKE_CXX_COMPILER_VERSION@" #define SUN_CXX_COMPILER_FLAGS "@CMAKE_CXX_FLAGS@" +#define SUN_FORTRAN_COMPILER "@CMAKE_FORTRAN_COMPILER_ID@" +#define SUN_FORTRAN_COMPILER_VERSION "@CMAKE_FORTRAN_COMPILER_VERSION@" +#define SUN_FORTRAN_COMPILER_FLAGS "@CMAKE_FORTRAN_FLAGS@" + #define SUN_BUILD_TYPE "@CMAKE_BUILD_TYPE@" #cmakedefine SUNDIALS_CI_JOB @@ -159,7 +167,14 @@ #define SUN_MAGMA_VERSION "@MAGMA_VERSION@" /* MPI */ -#define SUN_MPI_VERSION "@MPI_CXX_VERSION@" +#define SUN_MPI_C_COMPILER "@MPI_C_COMPILER@" +#define SUN_MPI_C_VERSION "@MPI_C_VERSION@" + +#define SUN_MPI_CXX_COMPILER "@MPI_CXX_COMPILER@" +#define SUN_MPI_CXX_VERSION "@MPI_CXX_VERSION@" + +#define SUN_MPI_FORTRAN_COMPILER "@MPI_FORTRAN_COMPILER@" +#define SUN_MPI_FORTRAN_VERSION "@MPI_FORTRAN_VERSION@" /* ONEMKL */ #cmakedefine SUNDIALS_ONEMKL_ENABLED @@ -236,10 +251,13 @@ /* CUDA */ #cmakedefine SUNDIALS_CUDA_ENABLED #define SUN_CUDA_VERSION "@CMAKE_CUDA_COMPILER_VERSION@" +#define SUN_CUDA_COMPILER "@CMAKE_CUDA_COMPILER@" +#define SUN_CUDA_ARCHITECTURES "@CMAKE_CUDA_ARCHITECTURES@" /* HIP */ #cmakedefine SUNDIALS_HIP_ENABLED #define SUN_HIP_VERSION "@HIP_VERSION@" +#define SUN_AMDGPU_TARGETS "@AMDGPU_TARGETS@" /* ------------------------------------------------------------------ * SUNDIALS modules enabled diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 8f8c811e23..0d60543d4d 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -227,10 +227,18 @@ void sunAdiakCollectMetadata() { adiak_job_size(); adiak_num_hosts(); + adiak_namevalue("c_compiler", 2, NULL, "%s", SUN_C_COMPILER); + adiak_namevalue("c_compiler_version", 2, NULL, "%s", SUN_C_COMPILER_VERSION); + adiak_namevalue("c_compiler_flags", 2, NULL, "%s", SUN_C_COMPILER_FLAGS); + adiak_namevalue("cxx_compiler", 2, NULL, "%s", SUN_CXX_COMPILER); adiak_namevalue("cxx_compiler_version", 2, NULL, "%s", SUN_CXX_COMPILER_VERSION); adiak_namevalue("cxx_compiler_flags", 2, NULL, "%s", SUN_CXX_COMPILER_FLAGS); + adiak_namevalue("fortran_compiler", 2, NULL, "%s", SUN_FORTRAN_COMPILER); + adiak_namevalue("fortran_compiler_version", 2, NULL, "%s", SUN_FORTRAN_COMPILER_VERSION); + adiak_namevalue("fortran_compiler_flags", 2, NULL, "%s", SUN_FORTRAN_COMPILER_FLAGS); + adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); adiak_namevalue("third_party_libraries", 2, NULL, "%s", SUN_TPL_LIST); @@ -271,9 +279,15 @@ void sunAdiakCollectMetadata() { adiak_namevalue("magma_version", 2, NULL, "%s", SUN_MAGMA_VERSION); #endif -// needs a further look #ifdef SUNDIALS_MPI_ENABLED - adiak_namevalue("mpi_version", 2, NULL, "%s", SUN_MPI_VERSION); + adiak_namevalue("mpi_c_compiler", 2, NULL, "%s", SUN_MPI_C_COMPILER); + adiak_namevalue("mpi_c_version", 2, NULL, "%s", SUN_MPI_C_VERSION); + + adiak_namevalue("mpi_cxx_compiler", 2, NULL, "%s", SUN_MPI_CXX_COMPILER); + adiak_namevalue("mpi_cxx_version", 2, NULL, "%s", SUN_MPI_CXX_VERSION); + + adiak_namevalue("mpi_fortran_compiler", 2, NULL, "%s", SUN_MPI_FORTRAN_COMPILER); + adiak_namevalue("mpi_fortran_version", 2, NULL, "%s", SUN_MPI_FORTRAN_VERSION); #endif #ifdef SUNDIALS_ONEMKL_ENABLED @@ -314,10 +328,14 @@ void sunAdiakCollectMetadata() { #ifdef SUNDIALS_CUDA_ENABLED adiak_namevalue("cuda_version", 2, NULL, "%s", SUN_CUDA_VERSION); + adiak_namevalue("cuda_compiler", 2, NULL, "%s", SUN_CUDA_COMPILER); + adiak_namevalue("cuda_architectures", 2, NULL, "%s", SUN_CUDA_ARCHITECTURES); #endif #ifdef SUNDIALS_HIP_ENABLED adiak_namevalue("hip_version", 2, NULL, "%s", SUN_HIP_VERSION); + adiak_namevalue("amdgpu_targets", 2, NULL, "%s", SUN_AMDGPU_TARGETS); #endif + } #endif \ No newline at end of file From 59d829e77caf38f8023d8d9eb6c9dbe67495a4b9 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 16 Jun 2023 14:00:44 -0700 Subject: [PATCH 121/187] add docs on local spot usage --- .../source/releases/Checklist.rst | 2 +- .../source/testing/Benchmarking.rst | 72 +++++++++++++------ doc/sundials_developers/source/testing/CI.rst | 2 +- .../source/testing/Spot.rst | 34 ++++++++- 4 files changed, 83 insertions(+), 27 deletions(-) diff --git a/doc/sundials_developers/source/releases/Checklist.rst b/doc/sundials_developers/source/releases/Checklist.rst index 6714487984..db45de136d 100644 --- a/doc/sundials_developers/source/releases/Checklist.rst +++ b/doc/sundials_developers/source/releases/Checklist.rst @@ -96,7 +96,7 @@ web pages. * ``doc/sundials/biblio.bib`` * ``scripts/tarscript`` - The following files are no longer maintaianed: + The following files are no longer maintained: * ``html/main.html`` (This is no longer maintained as of at least 2016) * ``sundialsTB/install_STB.m`` (This is no longer maintained as of 2016) diff --git a/doc/sundials_developers/source/testing/Benchmarking.rst b/doc/sundials_developers/source/testing/Benchmarking.rst index 46fe92bb5f..35390c9d55 100644 --- a/doc/sundials_developers/source/testing/Benchmarking.rst +++ b/doc/sundials_developers/source/testing/Benchmarking.rst @@ -17,29 +17,57 @@ Continuous Performance Testing (CPT) ==================================== -In order to protect against performance regression of SUNDIALS at all scales, we leverage the GitLab CI infrastructure setup for CI testing to perform continuous performance testing. - -The CPT suite consists of the :ref:`SUNDIALS benchmark programs` programs, which can scale up to full supercomputers, and the normal SUNDIALS examples program suite (i.e., the programs in the ``examples`` directory of the SUNDIALS repo). - -The CI suite can run the regular SUNDIALS CI tests, or it can run the SUNDIALS CPT suite by setting the ``BENCHMARK`` variable to ``ON`` when running a pipeline from the GitLab CI UI. -The benchmark problems are run with Caliper and a report for Spot and a human-readable runtime-report are generated. -The runtime-report is printed to the stdout and can be viewed in the GitLab CI job output. The Spot output files are made available as job artifacts. -We maintain a pipeline that runs the CPT suite weekly on a schedule, see ``_. - -FUTURE: WE WILL TRACK PERFORMANCE OVER TIME WITH `CALIPER `_ AND THE `SPOT WEB FRAMEWORK `_. - +In order to protect against performance regression of SUNDIALS at all scales, +we leverage the GitLab CI infrastructure setup for CI testing to perform +continuous performance testing. + +The CPT suite consists of the :ref:`SUNDIALS benchmark programs` +programs, which can scale up to full supercomputers, and the normal SUNDIALS +examples program suite (i.e., the programs in the ``examples`` directory of +the SUNDIALS repo). + +The CI suite can run the regular SUNDIALS CI tests, or it can run the SUNDIALS +CPT suite by setting the ``BENCHMARK`` variable to ``ON`` when running a +pipeline from the GitLab CI UI. +The benchmark problems are run with Caliper and a report for Spot and a +human-readable runtime-report are generated. +The runtime-report is printed to the stdout and can be viewed in the GitLab +CI job output. The Spot output files are made available as job artifacts. +We maintain a pipeline that runs the CPT suite weekly on a schedule, +see ``_. + +Performance over time is tracked with `Caliper `_ +and `SPOT web framework `_. Locally Building and Running the CPT ------------------------------------ -The SUNDIALS example suite can be run with Caliper profiling enabled and Adiak enabled for the CPT suite by setting the CMake options - -.. code-block:: cmake - - cmake -DSUNDIALS_BUILD_WITH_PROFILING=ON -DENABLE_CALIPER=ON -DCaliper_DIR=/path/to/caliper -DENABLE_ADIAK=ON -Dadiak_DIR=/path/to/adiak/lib/cmake/adiak -DSUNDIALS_TEST_DEVTESTS=ON -DSUNDIALS_TEST_PROFILE=ON - -This command will result in ``--profile`` option being passed to the SUNDIALS test runner Python script, ``test/testRunner``, which will in turn set the ``CALI_CONFIG`` environment variable before running every test so that when you run ``make test`` the examples will produce `.cali` output files documenting the performance. Running ``make benchmark`` will execute all the available benchmarks and produce `.cali` output files for each one. - -To specify where `.cali` output files are placed, define the CMake option SUNDIALS_TEST_OUTPUT_DIR with the directory path. By default `.cali` output files are placed in the build directory (:numref:`Installation`) under ``Benchmarking/output`` and ``Testing/output``. - -Refer to documentation section :ref:`Benchmarks` for instructions on building and running the ``benchmarks/`` programs locally. +The SUNDIALS example suite can be run with Caliper profiling enabled and +Adiak enabled for the CPT suite by setting the CMake options + +.. code-block:: bash + + $ cmake \ + > -DSUNDIALS_BUILD_WITH_PROFILING=ON \ + > -DENABLE_CALIPER=ON \ + > -DCaliper_DIR=/path/to/caliper \ + > -DENABLE_ADIAK=ON \ + > -Dadiak_DIR=/path/to/adiak/lib/cmake/adiak \ + > -DSUNDIALS_TEST_DEVTESTS=ON \ + > -DSUNDIALS_TEST_PROFILE=ON \ + +This command will result in ``--profile`` option being passed to the SUNDIALS +test runner Python script, ``test/testRunner``, which will in turn set the +``CALI_CONFIG`` environment variable before running every test so that when +you run ``make test`` the examples will produce `.cali` output files +documenting the performance. Running ``make benchmark`` will execute all the +available benchmarks and produce `.cali` output files for each one. + +To specify where `.cali` output files are placed, define the CMake option +SUNDIALS_TEST_OUTPUT_DIR with the directory path. By default `.cali` output +files are placed in the build directory (:numref:`Installation`) under +``Benchmarking/output`` and ``Testing/output``. + +Refer to section :ref:`Benchmarks` for instructions on building +and running the ``benchmarks/`` programs locally. Refer to section <> +for instructions on build and running examples locally. diff --git a/doc/sundials_developers/source/testing/CI.rst b/doc/sundials_developers/source/testing/CI.rst index fab4e52124..297fe5507d 100644 --- a/doc/sundials_developers/source/testing/CI.rst +++ b/doc/sundials_developers/source/testing/CI.rst @@ -272,7 +272,7 @@ To update the spack commit used for the CI: 1. The first thing to do is update the spack commit in the ``.uberenv_config.json`` file. 2. Then, a pipeline should be manually launched from the GitLab UI with the ``SHARED_SPACK`` CI variable set -to ``ON`` and the ``SPACK_PREFIX`` variable to the version of spack being set in the uberenv_config.json. +to ``ON`` and the ``SPACK_PREFIX`` variable to the version of spack being set in the uberenv_config.json. This will create a new spack installation and rebuild all of the specs. diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst index dac165f1d3..9df83b39e7 100644 --- a/doc/sundials_developers/source/testing/Spot.rst +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -14,8 +14,36 @@ SPOT Performance Visualization Tool ==================================== -In order to protect against performance regression of SUNDIALS at all scales, we leverage the GitLab CI infrastructure setup for CI testing to perform continuous performance testing. +In order to protect against performance regression of SUNDIALS at all scales, +we leverage the GitLab CI infrastructure setup for CI testing to perform +continuous performance testing. -To track and visualize performance over time, we collect data with `Caliper `_ and `Adiak `_. Steps to setting up continuous performance testing(CPT) can be found at :ref:`Continuous Performance Testing`. +To track and visualize performance over time, we collect data with +`Caliper `_ and +`Adiak `_. Steps to setting up continuous +performance testing(CPT) can be found at +:ref:`Continuous Performance Testing`. -Documentation to using the SPOT performance viisualization tool can be found at `here`_. \ No newline at end of file +Documentation to using the SPOT performance visualization tool can be found at +`here`_. + +Setting Up Your Own SPOT Visualizations +--------------------------------------- + +To display data from non-GitLab jobs or create a local collection of runs, +input into the SPOT search bar the absolute file path and refresh the page. +By default `.cali`` files will be generated in the build directory under +``Benchmarking/output`` and ``Testing/output``. To specify where `.cali` output +files are placed, define the CMake option SUNDIALS_TEST_OUTPUT_DIR with the +desired directory path. + +To retain the same filters as a given SPOT visualization link +:ref:`_Quick_Reference`, swap out the ``sf`` value in the URL with the +directory path containing the `.cali` files. + +.. _Quick_Reference: + +Quick Reference to Notable SPOT Visualizations +---------------------------------------------- + +Links will be added as informative SPOT visualizations are found. \ No newline at end of file From a48259d5677bbe0179b0103e49ff313ff3dcc70a Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 20 Jun 2023 11:45:26 -0700 Subject: [PATCH 122/187] add documentation on setting up documentation --- .../source/getting_started/Documentation.rst | 66 +++++++++++++++++++ .../source/getting_started/index.rst | 1 + 2 files changed, 67 insertions(+) create mode 100644 doc/sundials_developers/source/getting_started/Documentation.rst diff --git a/doc/sundials_developers/source/getting_started/Documentation.rst b/doc/sundials_developers/source/getting_started/Documentation.rst new file mode 100644 index 0000000000..bb6678e81a --- /dev/null +++ b/doc/sundials_developers/source/getting_started/Documentation.rst @@ -0,0 +1,66 @@ + ----------------------------------------------------------------------------- + SUNDIALS Copyright Start + Copyright (c) 2002-2023, Lawrence Livermore National Security + and Southern Methodist University. + All rights reserved. + + See the top-level LICENSE and NOTICE files for details. + + SPDX-License-Identifier: BSD-3-Clause + SUNDIALS Copyright End + ----------------------------------------------------------------------------- + +.. _Documentation_Setup: + +Developing Documentation +========================= + +Setting Up Sphinx +----------------- + +The Sundials documentation is generated by Sphinx. To set up Sphinx locally, +run the following commands: + +.. code-block:: bash + + $ cd doc + $ pip install -r requirements.txt + +Running the above will install the necessary Sphinx version. A Python virtual environment can also be used by activating the environment before running the above commands. + +Building and Running Documentation Locally +------------------------------ + +To build the general documentation (also viewable from `here`_), run the following: + +.. code-block:: bash + + $ cd doc/superbuild + $ make html + +To build the developer documentation, run the following: + +.. code-block:: bash + + $ cd doc/sundials_developers + $ make html + +To run documentation locally, open `build/html/index.html`. + +On Windows, use the following command: + +.. code-block:: bash + + $ start build/html/index.html + +On Mac, use the following command: + +.. code-block:: bash + + $ open build/html/index.html + +On Linux, use the following: + +.. code-block:: bash + + $ xdg-open build/html/index.html diff --git a/doc/sundials_developers/source/getting_started/index.rst b/doc/sundials_developers/source/getting_started/index.rst index 373a028cc2..2eb21fb074 100644 --- a/doc/sundials_developers/source/getting_started/index.rst +++ b/doc/sundials_developers/source/getting_started/index.rst @@ -28,5 +28,6 @@ cloning the SUNDIALS repository. A list of helpful commands can be found in the Setup Workflow + Documentation .. Environment \ No newline at end of file From 1bc7937cd2742aee38a0bfec6da81922cd8a4cef Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 20 Jun 2023 14:38:36 -0700 Subject: [PATCH 123/187] update answers --- test/answers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/answers b/test/answers index 72fd01e63e..adc6da31cd 160000 --- a/test/answers +++ b/test/answers @@ -1 +1 @@ -Subproject commit 72fd01e63edeffe39a800c820ac8aa8447270bf7 +Subproject commit adc6da31cd21bfa6e70d6fb026510008643f8ebb From d9feeba6f64a2d12176e3d17c5c08bd2bb3497ab Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 21 Jun 2023 09:15:53 -0700 Subject: [PATCH 124/187] define ci_job_id with timestamp if gitlab id not available --- cmake/SundialsSetupConfig.cmake | 3 ++- cmake/SundialsSetupTesting.cmake | 10 ++-------- include/sundials/sundials_config.in | 1 - src/sundials/sundials_context.c | 6 ++---- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 32747b308c..39b4b6e6c3 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -40,8 +40,9 @@ else() endif() if($ENV{CI_JOB_ID}) - set(SUNDIALS_CI_JOB TRUE) set(SUN_CI_JOB_ID $ENV{CI_JOB_ID}) +else() + string(TIMESTAMP SUN_CI_JOB_ID "%Y%m%d%H%M%S") endif() # ============================================================================ diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 44f5ad34fa..c5b18f2fbe 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -49,14 +49,8 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default output directory was provided make sure it exists if(SUNDIALS_TEST_OUTPUT_DIR) message(STATUS "Using non-default test output directory: ${SUNDIALS_TEST_OUTPUT_DIR}") - if(SUN_CI_JOB_ID) - if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) - file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) - endif() - else() - if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) - file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing) - endif() + if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) + file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) endif() endif() diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index efde48e0c3..c936edcfac 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -120,7 +120,6 @@ #define SUN_BUILD_TYPE "@CMAKE_BUILD_TYPE@" -#cmakedefine SUNDIALS_CI_JOB #define SUN_CI_JOB_ID "@SUN_CI_JOB_ID@" #define SUN_TPL_LIST "@SUNDIALS_TPL_LIST@" diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 0d60543d4d..53c52b7ef3 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -243,14 +243,12 @@ void sunAdiakCollectMetadata() { adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); adiak_namevalue("third_party_libraries", 2, NULL, "%s", SUN_TPL_LIST); + adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); + #ifdef SUNDIALS_SPACK_VERSION adiak_namevalue("spack_version", 2, NULL, "%s", SUNDIALS_SPACK_VERSION); #endif -#ifdef SUNDIALS_CI_JOB - adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); -#endif - #ifdef SUNDIALS_GINKGO_ENABLED adiak_namevalue("ginkgo_version", 2, NULL, "%s", SUN_GINKGO_VERSION); #endif From 52a56ba33ea848d2af2ab5314b99f766db682bdb Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 21 Jun 2023 11:00:15 -0700 Subject: [PATCH 125/187] add ci job start time --- cmake/SundialsSetupConfig.cmake | 5 +++++ include/sundials/sundials_config.in | 3 ++- src/sundials/sundials_context.c | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 39b4b6e6c3..57b8605886 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -41,10 +41,15 @@ endif() if($ENV{CI_JOB_ID}) set(SUN_CI_JOB_ID $ENV{CI_JOB_ID}) +endif() + +if($ENV{CI_JOB_STARTED_AT}) + set(SUN_CI_JOB_START_TIME $ENV{CI_JOB_STARTED_AT}) else() string(TIMESTAMP SUN_CI_JOB_ID "%Y%m%d%H%M%S") endif() + # ============================================================================ # Generate macros and substitution variables related to TPLs # that SUNDIALS is being built with. diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index c936edcfac..58441dd376 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -120,7 +120,8 @@ #define SUN_BUILD_TYPE "@CMAKE_BUILD_TYPE@" -#define SUN_CI_JOB_ID "@SUN_CI_JOB_ID@" +#define SUN_CI_JOB_ID "@CI_JOB_ID@" +#define SUN_CI_JOB_START_TIME "@CI_JOB_START_TIME@" #define SUN_TPL_LIST "@SUNDIALS_TPL_LIST@" #define SUN_TPL_LIST_SIZE "@SUNDIALS_TPL_LIST_SIZE@" diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 53c52b7ef3..14acd8dec1 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -244,6 +244,7 @@ void sunAdiakCollectMetadata() { adiak_namevalue("third_party_libraries", 2, NULL, "%s", SUN_TPL_LIST); adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); + adiak_namevalue("ci_job_start_time", 2, NULL, "%s", SUN_CI_JOB_START_TIME); #ifdef SUNDIALS_SPACK_VERSION adiak_namevalue("spack_version", 2, NULL, "%s", SUNDIALS_SPACK_VERSION); From 92ba518006a7a6d05f717f121a547311bd85a6eb Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 21 Jun 2023 11:05:48 -0700 Subject: [PATCH 126/187] fix default case of ci_job_start_time --- cmake/SundialsSetupConfig.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 57b8605886..85a1a08955 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -40,13 +40,13 @@ else() endif() if($ENV{CI_JOB_ID}) - set(SUN_CI_JOB_ID $ENV{CI_JOB_ID}) + set(CI_JOB_ID $ENV{CI_JOB_ID}) endif() if($ENV{CI_JOB_STARTED_AT}) - set(SUN_CI_JOB_START_TIME $ENV{CI_JOB_STARTED_AT}) + set(CI_JOB_START_TIME $ENV{CI_JOB_STARTED_AT}) else() - string(TIMESTAMP SUN_CI_JOB_ID "%Y%m%d%H%M%S") + string(TIMESTAMP CI_JOB_START_TIME "%Y%m%d%H%M%S") endif() From 5c8fd5800a85718fbd89f877a1e4430636956bbb Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 22 Jun 2023 09:50:35 -0700 Subject: [PATCH 127/187] remove ci from variables to make more generic --- cmake/SundialsSetupConfig.cmake | 8 +++++--- cmake/SundialsSetupTesting.cmake | 4 ++-- cmake/macros/SundialsAddTest.cmake | 4 ++-- include/sundials/sundials_config.in | 4 ++-- src/sundials/sundials_context.c | 7 ++++--- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 85a1a08955..62f54369be 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -40,13 +40,15 @@ else() endif() if($ENV{CI_JOB_ID}) - set(CI_JOB_ID $ENV{CI_JOB_ID}) + set(JOB_ID $ENV{CI_JOB_ID}) +else() + string(TIMESTAMP JOB_ID "%Y%m%d%H%M%S") endif() if($ENV{CI_JOB_STARTED_AT}) - set(CI_JOB_START_TIME $ENV{CI_JOB_STARTED_AT}) + set(JOB_START_TIME $ENV{CI_JOB_STARTED_AT}) else() - string(TIMESTAMP CI_JOB_START_TIME "%Y%m%d%H%M%S") + string(TIMESTAMP JOB_START_TIME "%Y%m%d%H%M%S") endif() diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index c5b18f2fbe..2df6697ca0 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -49,8 +49,8 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default output directory was provided make sure it exists if(SUNDIALS_TEST_OUTPUT_DIR) message(STATUS "Using non-default test output directory: ${SUNDIALS_TEST_OUTPUT_DIR}") - if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) - file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}) + if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${JOB_ID}) + file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${JOB_ID}) endif() endif() diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index cfa7a1ba09..d88c705618 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -96,8 +96,8 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) # check for a non-default output directory if(SUNDIALS_TEST_OUTPUT_DIR) - if(SUN_CI_JOB_ID) - list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${SUN_CI_JOB_ID}") + if(JOB_ID) + list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${JOB_ID}") else() list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing") endif() diff --git a/include/sundials/sundials_config.in b/include/sundials/sundials_config.in index 58441dd376..1360fd6a06 100644 --- a/include/sundials/sundials_config.in +++ b/include/sundials/sundials_config.in @@ -120,8 +120,8 @@ #define SUN_BUILD_TYPE "@CMAKE_BUILD_TYPE@" -#define SUN_CI_JOB_ID "@CI_JOB_ID@" -#define SUN_CI_JOB_START_TIME "@CI_JOB_START_TIME@" +#define SUN_JOB_ID "@JOB_ID@" +#define SUN_JOB_START_TIME "@JOB_START_TIME@" #define SUN_TPL_LIST "@SUNDIALS_TPL_LIST@" #define SUN_TPL_LIST_SIZE "@SUNDIALS_TPL_LIST_SIZE@" diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 14acd8dec1..26ea56f944 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -242,9 +242,10 @@ void sunAdiakCollectMetadata() { adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); adiak_namevalue("third_party_libraries", 2, NULL, "%s", SUN_TPL_LIST); - - adiak_namevalue("ci_job_id", 2, NULL, "%s", SUN_CI_JOB_ID); - adiak_namevalue("ci_job_start_time", 2, NULL, "%s", SUN_CI_JOB_START_TIME); +#ifdef SUN_JOB_ID + adiak_namevalue("job_id", 2, NULL, "%s", SUN_JOB_ID); +#endif + adiak_namevalue("job_start_time", 2, NULL, "%s", SUN_JOB_START_TIME); #ifdef SUNDIALS_SPACK_VERSION adiak_namevalue("spack_version", 2, NULL, "%s", SUNDIALS_SPACK_VERSION); From 19c82b470cfb6a7651e8e9add77fb2e8ea73e844 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 22 Jun 2023 14:59:40 -0700 Subject: [PATCH 128/187] reorganize caliper benchmark structure --- .../advection_reaction_3D/CMakeLists.txt | 22 +++++++++---------- .../diffusion_2D/mpi_gpu/CMakeLists.txt | 4 ++-- .../diffusion_2D/mpi_serial/CMakeLists.txt | 2 +- cmake/macros/SundialsAddBenchmark.cmake | 9 ++++---- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/benchmarks/advection_reaction_3D/CMakeLists.txt b/benchmarks/advection_reaction_3D/CMakeLists.txt index 2237583e68..a58ca74ff9 100644 --- a/benchmarks/advection_reaction_3D/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/CMakeLists.txt @@ -57,10 +57,10 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton" - "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton" - "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton" - "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton" + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton" ) # NUM_CORES based on Lassen system for 1 node @@ -68,7 +68,7 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - sundials_add_benchmark(advection_reaction_3D advection_reaction_3D + sundials_add_benchmark(advection_reaction_3D advection_reaction_3D advection_reaction_3D NUM_CORES 40 BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier} @@ -122,17 +122,17 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_cuda" - "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_cuda" - "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_cuda" - "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_cuda" + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_cuda" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_cuda" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_cuda" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_cuda" ) # NUM_CORES based on Lassen system for 1 node foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - sundials_add_benchmark(advection_reaction_3D_mpicuda advection_reaction_3D_mpicuda + sundials_add_benchmark(advection_reaction_3D_mpicuda advection_reaction_3D_mpicuda advection_reaction_3D ENABLE_GPU NUM_CORES 4 BENCHMARK_ARGS ${benchmark_args} @@ -193,7 +193,7 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) list(GET benchmark_tuple 1 identifier) # NUM_CORES based on Corona system for 1 node - sundials_add_benchmark(advection_reaction_3D_mpihip advection_reaction_3D_mpihip + sundials_add_benchmark(advection_reaction_3D_mpihip advection_reaction_3D_mpihip advection_reaction_3D ENABLE_GPU NUM_CORES 8 BENCHMARK_ARGS ${benchmark_args} diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index 23b6a97583..3164e6d897 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -77,7 +77,7 @@ foreach(test_tuple ${tests}) if("${backend}" STREQUAL "USE_CUDA") # based on Lassen system from build_and_bench (1 nodes). - # sundials_add_benchmark(${target} ${target} + # sundials_add_benchmark(${target} ${target} diffusion_2D # NUM_CORES 2 # ENABLE_GPU # IDENTIFIER "d2d_${package}_cuda" @@ -88,7 +88,7 @@ foreach(test_tuple ${tests}) if("${backend}" STREQUAL "USE_HIP") # based on the Corona system for 1 node - sundials_add_benchmark(${target} ${target} + sundials_add_benchmark(${target} ${target} diffusion_2D NUM_CORES 8 ENABLE_GPU IDENTIFIER "d2d_${package}_hip" diff --git a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt index 2d5d429fa6..94d292fa57 100644 --- a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt @@ -74,7 +74,7 @@ foreach(test_tuple ${tests}) DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") # NUM_CORES based 1 node for testing purposes - sundials_add_benchmark(${target} ${target} + sundials_add_benchmark(${target} ${target} diffusion_2D NUM_CORES 32 IDENTIFIER "d2d_${package}_serial" ) diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index 5dc1ceb646..d652c9c442 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -4,7 +4,8 @@ # CMake macro for adding benchmarks to `make benchmark`. # --------------------------------------------------------------- -macro(sundials_add_benchmark NAME EXECUTABLE) +macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) + # Define single value parameters the macro takes in to set up the test runner # # NUM_CORES = number of cores (GPU count or CPU count) to run on/number of resource sets @@ -33,11 +34,11 @@ macro(sundials_add_benchmark NAME EXECUTABLE) endif() endif() - # Create default benchmark output directory if custom not defined + # Create default benchmark output directory if custom directory not defined if(SUNDIALS_TEST_OUTPUT_DIR) - set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${SUNDIALS_TEST_OUTPUT_DIR}/Benchmarking) + set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${SUNDIALS_TEST_OUTPUT_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) else() - set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Benchmarking/output) + set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) endif() # make the output directory if it doesn't exist From e457298db2f7b2cfb16342e4207c4256433e5d9d Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 22 Jun 2023 15:00:24 -0700 Subject: [PATCH 129/187] add documenation on benchmarking --- .../source/testing/Benchmarking.rst | 14 ++++++++++--- .../source/testing/Spot.rst | 21 +++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/doc/sundials_developers/source/testing/Benchmarking.rst b/doc/sundials_developers/source/testing/Benchmarking.rst index 35390c9d55..f1a00b8965 100644 --- a/doc/sundials_developers/source/testing/Benchmarking.rst +++ b/doc/sundials_developers/source/testing/Benchmarking.rst @@ -60,12 +60,20 @@ This command will result in ``--profile`` option being passed to the SUNDIALS test runner Python script, ``test/testRunner``, which will in turn set the ``CALI_CONFIG`` environment variable before running every test so that when you run ``make test`` the examples will produce `.cali` output files -documenting the performance. Running ``make benchmark`` will execute all the -available benchmarks and produce `.cali` output files for each one. +documenting the performance. + +Note: Caliper prints to the `.out` files by default. Ensure all Caliper configs +requested have the `output` option defined to ensure output data is saved in a +separate location from the test output. Otherwise, the `.out` files for each +test will contain the output and the tests will fail. + +Turning on the ``BUILD_BENCHMARKS`` option will build benchmarks. Running +``make benchmark`` will execute all the available benchmarks and produce +`.cali` output files for each one. To specify where `.cali` output files are placed, define the CMake option SUNDIALS_TEST_OUTPUT_DIR with the directory path. By default `.cali` output -files are placed in the build directory (:numref:`Installation`) under +files are placed in the build directory (:ref:`Installation`) under ``Benchmarking/output`` and ``Testing/output``. Refer to section :ref:`Benchmarks` for instructions on building diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst index 9df83b39e7..c68ffe5e01 100644 --- a/doc/sundials_developers/source/testing/Spot.rst +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -21,29 +21,32 @@ continuous performance testing. To track and visualize performance over time, we collect data with `Caliper `_ and `Adiak `_. Steps to setting up continuous -performance testing(CPT) can be found at +performance testing(CPT) can be found in :ref:`Continuous Performance Testing`. -Documentation to using the SPOT performance visualization tool can be found at -`here`_. +Documentation to using the SPOT performance visualization tool can be found +`here `_. Setting Up Your Own SPOT Visualizations --------------------------------------- To display data from non-GitLab jobs or create a local collection of runs, -input into the SPOT search bar the absolute file path and refresh the page. +input into the SPOT search bar the absolute path to the directory containing the `,cali` files and refresh the page. By default `.cali`` files will be generated in the build directory under ``Benchmarking/output`` and ``Testing/output``. To specify where `.cali` output files are placed, define the CMake option SUNDIALS_TEST_OUTPUT_DIR with the desired directory path. -To retain the same filters as a given SPOT visualization link -:ref:`_Quick_Reference`, swap out the ``sf`` value in the URL with the +To retain the same filters as a given SPOT visualization link in +:ref:`Bookmarks`, swap out the ``sf`` value in the URL with the directory path containing the `.cali` files. -.. _Quick_Reference: +.. _Bookmarks: -Quick Reference to Notable SPOT Visualizations +Bookmarks to Notable SPOT Visualizations ---------------------------------------------- -Links will be added as informative SPOT visualizations are found. \ No newline at end of file +Links will be added as informative SPOT visualizations are found. + +.. future links to add: aggregations of examples +.. \ No newline at end of file From da288cf02bd1a39538514f2014a7c2376a8995cc Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 22 Jun 2023 16:39:07 -0700 Subject: [PATCH 130/187] add calidir option to testRunner --- cmake/SundialsBuildOptionsPre.cmake | 2 + cmake/SundialsSetupTesting.cmake | 12 +++++- cmake/macros/SundialsAddBenchmark.cmake | 9 +++-- cmake/macros/SundialsAddTest.cmake | 17 ++++++--- test/sundials_hatchet.py | 51 +++++++++++++++++++++++++ test/testRunner | 6 ++- 6 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 test/sundials_hatchet.py diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index 25d1c4a715..bf8dea8cff 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -296,3 +296,5 @@ sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" OFF ADVANCED) sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests" "" ADVANCED) + +sundials_option(SUNDIALS_CALIPER_OUTPUT_DIR PATH "Location to write caliper output files" "" ADVANCED) \ No newline at end of file diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 2df6697ca0..4638812cdf 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -49,8 +49,8 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default output directory was provided make sure it exists if(SUNDIALS_TEST_OUTPUT_DIR) message(STATUS "Using non-default test output directory: ${SUNDIALS_TEST_OUTPUT_DIR}") - if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${JOB_ID}) - file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${JOB_ID}) + if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}) + file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}) endif() endif() @@ -62,6 +62,14 @@ if(SUNDIALS_TEST_DEVTESTS) endif() endif() + # If a non-default caliper output directory was provided make sure it exists + if(SUNDIALS_CALIPER_OUTPUT_DIR) + message(STATUS "Using non-default caliper output directory: ${SUNDIALS_CALIPER_OUTPUT_DIR}") + if(NOT EXISTS ${SUNDIALS_CALIPER_OUTPUT_DIR}/Testing/${JOB_ID}) + file(MAKE_DIRECTORY ${SUNDIALS_CALIPER_OUTPUT_DIR}/Testing/${JOB_ID}) + endif() + endif() + # Check if using non-default comparison precisions when testing if(SUNDIALS_TEST_FLOAT_PRECISION GREATER_EQUAL "0") message(STATUS "Using non-default float precision: ${SUNDIALS_TEST_FLOAT_PRECISION}") diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index d652c9c442..31d49ad5ea 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -34,9 +34,9 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) endif() endif() - # Create default benchmark output directory if custom directory not defined - if(SUNDIALS_TEST_OUTPUT_DIR) - set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${SUNDIALS_TEST_OUTPUT_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) + # Create default benchmark caliper output directory if custom directory is not defined + if(SUNDIALS_CALIPER_OUTPUT_DIR) + set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${SUNDIALS_CALIPER_OUTPUT_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) else() set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) endif() @@ -50,7 +50,8 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) set(TEST_RUNNER_ARGS "--verbose" "--executablename=$" - "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" + "--outputdir=${TEST_OUTPUT_DIR}" + "--calidir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" "--nodiff" ) diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index d88c705618..aa70c94637 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -42,6 +42,12 @@ # # -D SUNDIALS_TEST_OUTPUT_DIR= # -D SUNDIALS_TEST_ANSWER_DIR= +# +# By default the caliper output is written to builddir/Caliper. This can be +# changed by setting the cache variable SUNDIALS_CALIPER_OUTPUT_DIR. +# +# -D SUNDIALS_CALIPER_OUTPUT_DIR= +# # ------------------------------------------------------------------------------ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) @@ -92,15 +98,16 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) if(SUNDIALS_TEST_PROFILE) list(APPEND TEST_ARGS "--profile") + if (SUNDIALS_CALIPER_OUTPUT_DIR) + list(APPEND TEST_ARGS "--calidir=${SUNDIALS_CALIPER_OUTPUT_DIR}/Testing/${JOB_ID}") + else() + list(APPEND TEST_ARGS "--calidir=${TEST_OUTPUT_DIR}/Caliper/Testing") + endif() endif() # check for a non-default output directory if(SUNDIALS_TEST_OUTPUT_DIR) - if(JOB_ID) - list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing/${JOB_ID}") - else() - list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}/Testing") - endif() + list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}") else() list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}") endif() diff --git a/test/sundials_hatchet.py b/test/sundials_hatchet.py new file mode 100644 index 0000000000..5ae2ae9ac5 --- /dev/null +++ b/test/sundials_hatchet.py @@ -0,0 +1,51 @@ +import os +import sys +import platform +import datetime as dt +from IPython.display import HTML, display + +import hatchet as ht + +# find way to load in a folder + +# Setup desired cali query. +grouping_attribute = "function" +default_metric = "sum(sum#time.duration),inclusive_sum(sum#time.duration)" +query = "select function,%s group by %s format json-split" % ( + default_metric, + grouping_attribute, +) + +# cali_file_path = "/usr/workspace/pan13/sundials/adiak-testing/Benchmarking/arkode_diffusion_2D_mpi_d2d_arkode_serial/arkode_diffusion_2D_mpi_d2d_arkode_serial.06122023_151704.cali" + +cali_file_path = "/usr/workspace/pan13/sundials/adiak-testing/Benchmarking/advection_reaction_3D_arkdirk_newton/advection_reaction_3D_arkdirk_newton.06092023_143842.cali" + +gf = ht.GraphFrame.from_caliper(cali_file_path, query) + +print(gf.dataframe) + +# use the cali-query to aggregate files into a new file + +# array-to-store-graphframes +# for file in caliper folder (can you do this?) +# create graph frames with cali files + +# sort the graphframes by date +# aggregate all but the last one (most recent one) + +# compare results +# - if past a certain threshold, send an email to some list +# - find a package for this +# - how acquire emails, can one pull from a list, or send to a mailing list? (like archwhisky) + +# do some other analysis cause why not + +# to do +# set up .cali file loads -> likely only do for the latest job (which you can get by checking the environment ID. How would you do locally? Well this wasn't really meant for personal use. one can probably make a second version of this script) +# set up function to aggregate example performance data - how? probably by checking the ci job value in the cali files +# this should be run whenever the benchmark script is run (like after the whole benchmarking and example profiling) +# compare to cached data -> if anomaly in the worst way, send out an email + +# send email for general performance + +# set up a way to store cached performance data. Most likely a CSV or json of some sort with CI job id, date, and relevant data. Or could just calculate straight up \ No newline at end of file diff --git a/test/testRunner b/test/testRunner index 15af79985e..630ec67261 100755 --- a/test/testRunner +++ b/test/testRunner @@ -72,6 +72,8 @@ def main(): parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='run verbosely') + parser.add_argument('--calidir', dest='caliDir', type=str, help='path to directory to write .cali files to') + args = parser.parse_args() if(args.verbose): @@ -85,10 +87,12 @@ def main(): print("buildDir=%s"%(args.buildDir)) print("buildCmd=%s"%(args.buildCmd)) print("profile=%s"%(args.profile)) + print("caliDir=%s"%{args.caliDir}) # regression test name and output directory testName = args.testName outDir = args.outputDir + caliDir = args.caliDir # path to Sundials test directory testDir = os.path.dirname(sys.argv[0]) @@ -195,7 +199,7 @@ def main(): # If profiling set the CALI_CONFIG environment variable if(args.profile): dateTime = datetime.datetime.now().strftime("%m%d%Y_%H%M%S") - profilePath = os.path.join(outDir, args.testName+".%s.cali" % dateTime) + profilePath = os.path.join(caliDir, args.testName+".%s.cali" % dateTime) os.environ['CALI_SERVICES_ENABLE'] = 'env' os.environ['CALI_CONFIG'] = 'spot(output=%s)' % profilePath From 669ee5e4eaf9e4c65a97064cedc225ee758daebb Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 09:52:28 -0700 Subject: [PATCH 131/187] remove hatchet script --- test/sundials_hatchet.py | 51 ---------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 test/sundials_hatchet.py diff --git a/test/sundials_hatchet.py b/test/sundials_hatchet.py deleted file mode 100644 index 5ae2ae9ac5..0000000000 --- a/test/sundials_hatchet.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import sys -import platform -import datetime as dt -from IPython.display import HTML, display - -import hatchet as ht - -# find way to load in a folder - -# Setup desired cali query. -grouping_attribute = "function" -default_metric = "sum(sum#time.duration),inclusive_sum(sum#time.duration)" -query = "select function,%s group by %s format json-split" % ( - default_metric, - grouping_attribute, -) - -# cali_file_path = "/usr/workspace/pan13/sundials/adiak-testing/Benchmarking/arkode_diffusion_2D_mpi_d2d_arkode_serial/arkode_diffusion_2D_mpi_d2d_arkode_serial.06122023_151704.cali" - -cali_file_path = "/usr/workspace/pan13/sundials/adiak-testing/Benchmarking/advection_reaction_3D_arkdirk_newton/advection_reaction_3D_arkdirk_newton.06092023_143842.cali" - -gf = ht.GraphFrame.from_caliper(cali_file_path, query) - -print(gf.dataframe) - -# use the cali-query to aggregate files into a new file - -# array-to-store-graphframes -# for file in caliper folder (can you do this?) -# create graph frames with cali files - -# sort the graphframes by date -# aggregate all but the last one (most recent one) - -# compare results -# - if past a certain threshold, send an email to some list -# - find a package for this -# - how acquire emails, can one pull from a list, or send to a mailing list? (like archwhisky) - -# do some other analysis cause why not - -# to do -# set up .cali file loads -> likely only do for the latest job (which you can get by checking the environment ID. How would you do locally? Well this wasn't really meant for personal use. one can probably make a second version of this script) -# set up function to aggregate example performance data - how? probably by checking the ci job value in the cali files -# this should be run whenever the benchmark script is run (like after the whole benchmarking and example profiling) -# compare to cached data -> if anomaly in the worst way, send out an email - -# send email for general performance - -# set up a way to store cached performance data. Most likely a CSV or json of some sort with CI job id, date, and relevant data. Or could just calculate straight up \ No newline at end of file From af8c80f8ddc190a67d99f6fbb3609e4f10334ef2 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 11:15:42 -0700 Subject: [PATCH 132/187] move parameters to raja CMakeLists.txt --- .../advection_reaction_3D/CMakeLists.txt | 187 +----------------- .../advection_reaction_3D/raja/CMakeLists.txt | 54 ++++- 2 files changed, 56 insertions(+), 185 deletions(-) diff --git a/benchmarks/advection_reaction_3D/CMakeLists.txt b/benchmarks/advection_reaction_3D/CMakeLists.txt index 08c8f50781..0143c1baae 100644 --- a/benchmarks/advection_reaction_3D/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/CMakeLists.txt @@ -16,188 +16,7 @@ if(ENABLE_RAJA) add_subdirectory(raja) endif() - # ---------------------------------------------------------------------------- - # MPI only - # ---------------------------------------------------------------------------- - - add_executable(advection_reaction_3D - advection_reaction_3D.cpp - arkode_driver.cpp - cvode_driver.cpp - ida_driver.cpp - rhs3D.hpp - ParallelGrid.hpp - backends.hpp) - - # ensure the linker language is reset to CXX - set_target_properties(advection_reaction_3D PROPERTIES LINKER_LANGUAGE CXX) - - target_include_directories(advection_reaction_3D - PRIVATE - ${PROJECT_SOURCE_DIR}/utilities - ${MPI_CXX_INCLUDE_DIRS}) - - target_link_libraries(advection_reaction_3D - PRIVATE - sundials_arkode - sundials_cvode - sundials_ida - sundials_nvecmpiplusx - sundials_nvecserial - RAJA - ${MPI_CXX_LIBRARIES} - ${OTHER_LIBS}) - - install(TARGETS advection_reaction_3D - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") - - install(FILES README.md - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") - - set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton" - "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton" - "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton" - "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton" - ) - - # NUM_CORES based on Lassen system for 1 node - foreach(benchmark_tuple ${BENCHMARK_VAR}) - list(GET benchmark_tuple 0 benchmark_args) - list(GET benchmark_tuple 1 identifier) - - sundials_add_benchmark(advection_reaction_3D advection_reaction_3D advection_reaction_3D - NUM_CORES 40 - BENCHMARK_ARGS ${benchmark_args} - IDENTIFIER ${identifier} - ) - endforeach() - - # ---------------------------------------------------------------------------- - # MPI + CUDA - # ---------------------------------------------------------------------------- - - if(BUILD_NVECTOR_CUDA) - - set_source_files_properties(advection_reaction_3D.cpp - PROPERTIES LANGUAGE CUDA) - set_source_files_properties(arkode_driver.cpp PROPERTIES LANGUAGE CUDA) - set_source_files_properties(cvode_driver.cpp PROPERTIES LANGUAGE CUDA) - set_source_files_properties(ida_driver.cpp PROPERTIES LANGUAGE CUDA) - - add_executable(advection_reaction_3D_mpicuda - advection_reaction_3D.cpp - arkode_driver.cpp - cvode_driver.cpp - ida_driver.cpp - rhs3D.hpp - ParallelGrid.hpp - backends.hpp) - - # ensure the linker language is reset to CXX - set_target_properties(advection_reaction_3D_mpicuda - PROPERTIES LINKER_LANGUAGE CXX) - - target_include_directories(advection_reaction_3D_mpicuda - PRIVATE - ${PROJECT_SOURCE_DIR}/utilities - ${MPI_CXX_INCLUDE_DIRS}) - - target_link_libraries(advection_reaction_3D_mpicuda - PRIVATE - sundials_arkode - sundials_cvode - sundials_ida - sundials_nvecmpiplusx - sundials_nveccuda - RAJA - ${MPI_CXX_LIBRARIES} - ${OTHER_LIBS}) - - target_compile_definitions(advection_reaction_3D_mpicuda PRIVATE USE_CUDA_NVEC) - - install(TARGETS advection_reaction_3D_mpicuda - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") - - set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_cuda" - "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_cuda" - "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_cuda" - "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_cuda" - ) - # NUM_CORES based on Lassen system for 1 node - foreach(benchmark_tuple ${BENCHMARK_VAR}) - list(GET benchmark_tuple 0 benchmark_args) - list(GET benchmark_tuple 1 identifier) - - sundials_add_benchmark(advection_reaction_3D_mpicuda advection_reaction_3D_mpicuda advection_reaction_3D - ENABLE_GPU - NUM_CORES 4 - BENCHMARK_ARGS ${benchmark_args} - IDENTIFIER ${identifier} - ) - endforeach() - - endif() - - # ---------------------------------------------------------------------------- - # MPI + HIP - # ---------------------------------------------------------------------------- - - if(BUILD_NVECTOR_HIP) - add_dependencies(benchmark advection_reaction_3D_mpihip) - - add_executable(advection_reaction_3D_mpihip - advection_reaction_3D.cpp - arkode_driver.cpp - cvode_driver.cpp - ida_driver.cpp - rhs3D.hpp - ParallelGrid.hpp - backends.hpp) - - target_include_directories(advection_reaction_3D_mpihip - PRIVATE - ${PROJECT_SOURCE_DIR}/utilities - ${MPI_CXX_INCLUDE_DIRS}) - - target_link_libraries(advection_reaction_3D_mpihip - PRIVATE - sundials_arkode - sundials_cvode - sundials_ida - sundials_nvecmpiplusx - sundials_nvechip - RAJA - hip::device - ${MPI_CXX_LIBRARIES} - ${OTHER_LIBS}) - - target_compile_definitions(advection_reaction_3D_mpihip PRIVATE USE_HIP_NVEC) - - install(TARGETS advection_reaction_3D_mpihip - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D") - - set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_hip" - "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_hip" - "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_hip" - "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_hip" - ) - - - foreach(benchmark_tuple ${BENCHMARK_VAR}) - list(GET benchmark_tuple 0 benchmark_args) - list(GET benchmark_tuple 1 identifier) - - # NUM_CORES based on Corona system for 1 node - sundials_add_benchmark(advection_reaction_3D_mpihip advection_reaction_3D_mpihip advection_reaction_3D - ENABLE_GPU - NUM_CORES 8 - BENCHMARK_ARGS ${benchmark_args} - IDENTIFIER ${identifier} - ) - endforeach() - endif() - +if(ENABLE_KOKKOS AND BUILD_NVECTOR_KOKKOS) + add_subdirectory(kokkos) endif() + diff --git a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt index 0bae78c562..74f966c5ed 100644 --- a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt @@ -58,6 +58,25 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(FILES README.md ../scripts/compare_error.py ../scripts/compute_error.py ../scripts/pickle_solution_output.py DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") + set(BENCHMARK_VAR + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton" + ) + + # NUM_CORES based on Lassen system for 1 node + foreach(benchmark_tuple ${BENCHMARK_VAR}) + list(GET benchmark_tuple 0 benchmark_args) + list(GET benchmark_tuple 1 identifier) + + sundials_add_benchmark(advection_reaction_3D advection_reaction_3D advection_reaction_3D + NUM_CORES 40 + BENCHMARK_ARGS ${benchmark_args} + IDENTIFIER ${identifier} + ) + endforeach() + # ---------------------------------------------------------------------------- # MPI + CUDA # ---------------------------------------------------------------------------- @@ -105,6 +124,23 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(TARGETS advection_reaction_3D_raja_mpicuda DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") + set(BENCHMARK_VAR + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_cuda_raja" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_cuda_raja" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_cuda_raja" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_cuda_raja") + + # NUM_CORES based on Lassen system for 1 node + foreach(benchmark_tuple ${BENCHMARK_VAR}) + list(GET benchmark_tuple 0 benchmark_args) + list(GET benchmark_tuple 1 identifier) + + sundials_add_benchmark(advection_reaction_3D_mpicuda advection_reaction_3D_mpicuda advection_reaction_3D + ENABLE_GPU + NUM_CORES 4 + BENCHMARK_ARGS ${benchmark_args} + IDENTIFIER ${identifier}) + endforeach() endif() # ---------------------------------------------------------------------------- @@ -146,6 +182,22 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(TARGETS advection_reaction_3D_raja_mpihip DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") + set(BENCHMARK_VAR + "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_hip_raja" + "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_hip_raja" + "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_hip_raja" + "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_hip_raja") + + foreach(benchmark_tuple ${BENCHMARK_VAR}) + list(GET benchmark_tuple 0 benchmark_args) + list(GET benchmark_tuple 1 identifier) + + # NUM_CORES based on Corona system for 1 node + sundials_add_benchmark(advection_reaction_3D_mpihip advection_reaction_3D_mpihip advection_reaction_3D + ENABLE_GPU + NUM_CORES 8 + BENCHMARK_ARGS ${benchmark_args} + IDENTIFIER ${identifier}) + endforeach() endif() - endif() From 793f5d2a9b876a28448c763290b66002a9b0f60b Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 11:17:30 -0700 Subject: [PATCH 133/187] update identifiers for MPI only raja benchmarks --- benchmarks/advection_reaction_3D/raja/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt index 74f966c5ed..88da90bdef 100644 --- a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt @@ -59,10 +59,10 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton" - "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton" - "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton" - "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton" + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_raja" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_raja" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_raja" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_raja" ) # NUM_CORES based on Lassen system for 1 node From ff6a14bfceb3aba060d5fd5f0dff9007e7da5bd8 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 11:35:01 -0700 Subject: [PATCH 134/187] fix parameters for make benchmark for advection_reaction_3D_raja --- .../advection_reaction_3D/raja/CMakeLists.txt | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt index 88da90bdef..41db8bedf8 100644 --- a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt @@ -70,7 +70,7 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - sundials_add_benchmark(advection_reaction_3D advection_reaction_3D advection_reaction_3D + sundials_add_benchmark(advection_reaction_3D_raja advection_reaction_3D_raja advection_reaction_3D NUM_CORES 40 BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier} @@ -125,17 +125,17 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_cuda_raja" - "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_cuda_raja" - "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_cuda_raja" - "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_cuda_raja") + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_raja_mpicuda" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_raja_mpicuda" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_raja_mpicuda" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_raja_mpicuda") # NUM_CORES based on Lassen system for 1 node foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - sundials_add_benchmark(advection_reaction_3D_mpicuda advection_reaction_3D_mpicuda advection_reaction_3D + sundials_add_benchmark(advection_reaction_3D_raja_mpicuda advection_reaction_3D_raja_mpicuda advection_reaction_3D ENABLE_GPU NUM_CORES 4 BENCHMARK_ARGS ${benchmark_args} @@ -183,17 +183,17 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_hip_raja" - "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_hip_raja" - "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_hip_raja" - "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_hip_raja") + "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_raja_mpihip" + "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_raja_mpihip" + "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_raja_mpihip" + "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_raja_mpihip") foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) # NUM_CORES based on Corona system for 1 node - sundials_add_benchmark(advection_reaction_3D_mpihip advection_reaction_3D_mpihip advection_reaction_3D + sundials_add_benchmark(advection_reaction_3D_raja_mpihip advection_reaction_3D_raja_mpihip advection_reaction_3D ENABLE_GPU NUM_CORES 8 BENCHMARK_ARGS ${benchmark_args} From 83ea50034a4642c1f12bf00b2922602847036a08 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 11:48:41 -0700 Subject: [PATCH 135/187] update docs --- doc/sundials_developers/source/testing/Benchmarking.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/sundials_developers/source/testing/Benchmarking.rst b/doc/sundials_developers/source/testing/Benchmarking.rst index f1a00b8965..9e7324e26d 100644 --- a/doc/sundials_developers/source/testing/Benchmarking.rst +++ b/doc/sundials_developers/source/testing/Benchmarking.rst @@ -69,13 +69,16 @@ test will contain the output and the tests will fail. Turning on the ``BUILD_BENCHMARKS`` option will build benchmarks. Running ``make benchmark`` will execute all the available benchmarks and produce -`.cali` output files for each one. +`.cali` output files for each one. To change what parameters benchmarks are run +with, edit the respective `CMakeLists.txt`. The ``BENCHMARK_VARS`` variable +determines how many tests to run with different parameters. Arguments passed +into the ``sundials_add_benchmark`` macro change how the benchmark is run. To specify where `.cali` output files are placed, define the CMake option -SUNDIALS_TEST_OUTPUT_DIR with the directory path. By default `.cali` output +SUNDIALS_CALIPER_OUTPUT_DIR with the directory path. By default `.cali` output files are placed in the build directory (:ref:`Installation`) under ``Benchmarking/output`` and ``Testing/output``. -Refer to section :ref:`Benchmarks` for instructions on building +Refer to section :ref:`Benchmarks` for details on instructions on building and running the ``benchmarks/`` programs locally. Refer to section <> for instructions on build and running examples locally. From d127b138b97f16d0ffbc950fdb7073081a825b82 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 14:02:05 -0700 Subject: [PATCH 136/187] fix sphinx warnings --- .../source/getting_started/Documentation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sundials_developers/source/getting_started/Documentation.rst b/doc/sundials_developers/source/getting_started/Documentation.rst index bb6678e81a..97d1b663dc 100644 --- a/doc/sundials_developers/source/getting_started/Documentation.rst +++ b/doc/sundials_developers/source/getting_started/Documentation.rst @@ -29,9 +29,9 @@ run the following commands: Running the above will install the necessary Sphinx version. A Python virtual environment can also be used by activating the environment before running the above commands. Building and Running Documentation Locally ------------------------------- +------------------------------------------- -To build the general documentation (also viewable from `here`_), run the following: +To build the general documentation (also viewable from `here `_), run the following: .. code-block:: bash From 89c1f7a8d0de36b628df26c0ef4eb43d512cf443 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 14:16:34 -0700 Subject: [PATCH 137/187] set caliper output dir in package.py --- .gitlab/spack_packages/sundials/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 6697a13247..4d6d2be2be 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -935,6 +935,9 @@ def initconfig_package_entries(self): if "+trilinos" in spec: entries.append(cmake_cache_path("Trilinos_DIR", spec["trilinos"].prefix)) + if "+profile-examples" in spec: + entries.append(cmake_cache_path("SUNDIALS_CALIPER_OUTPUT_DIR"), "/usr/workspace/sundials/califiles") + # Examples if spec.satisfies("@3:"): entries.extend( From bcd0b90c5b1f796340fdc7d75d51b70a01665eb4 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 14:16:51 -0700 Subject: [PATCH 138/187] fix syntax of docs --- doc/sundials_developers/source/getting_started/Documentation.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sundials_developers/source/getting_started/Documentation.rst b/doc/sundials_developers/source/getting_started/Documentation.rst index 97d1b663dc..4b9c5109da 100644 --- a/doc/sundials_developers/source/getting_started/Documentation.rst +++ b/doc/sundials_developers/source/getting_started/Documentation.rst @@ -1,3 +1,4 @@ +.. ----------------------------------------------------------------------------- SUNDIALS Copyright Start Copyright (c) 2002-2023, Lawrence Livermore National Security From 555662c48ca68cc149ba58ee920e68cb5c8036ef Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 3 Jul 2023 14:21:35 -0700 Subject: [PATCH 139/187] remove profile job --- .gitlab/corona-jobs.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml index 03c5d28a50..450a9197ba 100644 --- a/.gitlab/corona-jobs.yml +++ b/.gitlab/corona-jobs.yml @@ -35,16 +35,3 @@ corona_clang_hip_tpls: variables: SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET}" extends: .corona_build_and_test - -# ------------------------------------------------------------------------------ -# Corona Profile Example Jobs -# ------------------------------------------------------------------------------ - -corona_test_profile: - parallel: - matrix: - - COMPILER_SPEC: rocmcc@5.4.1 - AMDGPU_TARGET: [gfx906] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+adiak+caliper+profile-examples" - extends: .corona_build_and_test \ No newline at end of file From 14e259e6cabd5638ec6a23f1d75fdfcc52f7e6c3 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 5 Jul 2023 11:40:15 -0700 Subject: [PATCH 140/187] add test name to metadata --- test/testRunner | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/testRunner b/test/testRunner index 630ec67261..7f854785e7 100755 --- a/test/testRunner +++ b/test/testRunner @@ -201,6 +201,8 @@ def main(): dateTime = datetime.datetime.now().strftime("%m%d%Y_%H%M%S") profilePath = os.path.join(caliDir, args.testName+".%s.cali" % dateTime) os.environ['CALI_SERVICES_ENABLE'] = 'env' + os.environ['TEST_NAME'] = testName + os.environ['CALI_ENV_EXTRA'] = 'TEST_NAME' os.environ['CALI_CONFIG'] = 'spot(output=%s)' % profilePath # if user supplies precision info overide the default choices From 58de98fe52ff2fb21da568e6cdbfcc8c13934d24 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 6 Jul 2023 17:05:03 -0700 Subject: [PATCH 141/187] add preliminary python script to process caliper files --- test/compare_tests.py | 73 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 test/compare_tests.py diff --git a/test/compare_tests.py b/test/compare_tests.py new file mode 100644 index 0000000000..5f44bd5552 --- /dev/null +++ b/test/compare_tests.py @@ -0,0 +1,73 @@ +import os +import subprocess +import sys +import re +import glob +import argparse +import multiprocessing as mp +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +import hatchet as ht +import thicket as tt + +def main(): + parser = argparse.ArgumentParser(description='Compare Sundials performance results against previous results') + + parser.add_argument('--calidir', dest='caliDir', type=str,help='name of folder containing caliper files', default="/usr/workspace/pan13/shrimp") + + parser.add_argument('--outpath', dest='outpath', type=str, help='path to file to write results to', default="/g/g20/pan13/shrimp/output.out") + + args = parser.parse_args() + + caliDir = args.caliDir + outPath = args.outpath + + # add a thing to check if the output file exists and make sure it does + + # get the latest test run + runDirs = glob.glob("%s/Testing/*" % caliDir, recursive = True) + runDirs.sort(key=os.path.getmtime, reverse=True) + runDir = runDirs[0] + + # Gather files to process + testFiles = glob.glob("%s/*.cali" % (runDir)) + + # Compare test results against past runs. If a test performs below a threshold, output test name to file. + outFile = open(outPath, 'w') + with mp.Pool() as pool: + for res in pool.starmap(process_test, [(caliDir, i) for i in testFiles]): + if res: + outFile.write(res + "\n") + outFile.close() + + outFile = open(outPath, 'r') + try: + outLines = outFile.readlines() + finally: + outFile.close() + + if (len(outLines) == 0): + return -1 + return 0 + +def process_test(caliDir, file): + + th = tt.Thicket.from_caliperreader(file) + + testName = th.metadata['env.TEST_NAME'].values[0] + files = glob.glob("%s/Testing/*/%s.*.cali" % (caliDir, testName), recursive=True) + th_files = tt.Thicket.from_caliperreader(files) + + metrics = ['Max time/rank'] + tt.mean(th_files, columns=metrics) + tt.mean(th, columns=metrics) + + ratio = th.statsframe.dataframe['Max time/rank_mean'] / th_files.statsframe.dataframe['Max time/rank_mean'] + + if 1 - ratio[0] < 0.1: + return testName + +if __name__ == "__main__": + main() From 8c9c921ac08217907c88ea55679dc35dd155422f Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 11 Jul 2023 16:18:52 -0700 Subject: [PATCH 142/187] add file limit and output file check --- test/compare_tests.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/test/compare_tests.py b/test/compare_tests.py index 5f44bd5552..c2ad899f37 100644 --- a/test/compare_tests.py +++ b/test/compare_tests.py @@ -15,17 +15,15 @@ def main(): parser = argparse.ArgumentParser(description='Compare Sundials performance results against previous results') - parser.add_argument('--calidir', dest='caliDir', type=str,help='name of folder containing caliper files', default="/usr/workspace/pan13/shrimp") + parser.add_argument('--calidir', dest='caliDir', type=str,help='name of folder containing caliper files', default="/usr/workspace/pan13/shrimp2") - parser.add_argument('--outpath', dest='outpath', type=str, help='path to file to write results to', default="/g/g20/pan13/shrimp/output.out") + parser.add_argument('--outdir', dest='outDir', type=str, help='path to directory to write results to', default="/g/g20/pan13/shrimp") args = parser.parse_args() caliDir = args.caliDir outPath = args.outpath - # add a thing to check if the output file exists and make sure it does - # get the latest test run runDirs = glob.glob("%s/Testing/*" % caliDir, recursive = True) runDirs.sort(key=os.path.getmtime, reverse=True) @@ -34,14 +32,21 @@ def main(): # Gather files to process testFiles = glob.glob("%s/*.cali" % (runDir)) - # Compare test results against past runs. If a test performs below a threshold, output test name to file. - outFile = open(outPath, 'w') + if not os.path.exists(outPath): + os.mkdirs(outPath) + outFile = open("%s/output.out" % outPath, 'w') + + + # Compare test results against past runs. If a test performs below a threshold, output test name to outFile. with mp.Pool() as pool: for res in pool.starmap(process_test, [(caliDir, i) for i in testFiles]): if res: outFile.write(res + "\n") outFile.close() + # for file in testFiles: + # process_test(caliDir, file) + outFile = open(outPath, 'r') try: outLines = outFile.readlines() @@ -53,13 +58,20 @@ def main(): return 0 def process_test(caliDir, file): - th = tt.Thicket.from_caliperreader(file) - + testName = th.metadata['env.TEST_NAME'].values[0] + + # Gather the last numRuns of tests and remove the current run from set of runs to aggregate + numRuns = 5 files = glob.glob("%s/Testing/*/%s.*.cali" % (caliDir, testName), recursive=True) - th_files = tt.Thicket.from_caliperreader(files) + files.sort(key=os.path.getmtime, reverse=True) + files.pop(0) # remove current run + files = files[:numRuns] + files.sort(key=os.path.ge) + + th_files = tt.Thicket.from_caliperreader(files) metrics = ['Max time/rank'] tt.mean(th_files, columns=metrics) tt.mean(th, columns=metrics) From 0708c4bf2df4a8aa00bf3ef586f68010d52c5c7e Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 14 Jul 2023 15:02:17 -0700 Subject: [PATCH 143/187] add preliminary data visualization for benchmarks --- test/benchmark_analysis.ipynb | 121 ++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 test/benchmark_analysis.ipynb diff --git a/test/benchmark_analysis.ipynb b/test/benchmark_analysis.ipynb new file mode 100644 index 0000000000..3813490634 --- /dev/null +++ b/test/benchmark_analysis.ipynb @@ -0,0 +1,121 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import os\n", + "import subprocess\n", + "import sys\n", + "import re\n", + "import glob\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from IPython.display import display, HTML\n", + "display(HTML(\"\"))\n", + "\n", + "import hatchet as ht\n", + "import thicket as tt" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "# Set desired pandas options\n", + "pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', None)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "caliDir = \"/usr/workspace/pan13/shrimp/Benchmarking/diffusion_2D/arkode_diffusion_2D_mpi_d2d_arkode_serial\"\n", + "caliFiles = glob.glob(\"%s/*.cali\" % caliDir)\n", + "\n", + "th_bench = tt.Thicket.from_caliperreader(caliFiles)" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABa8AAADMCAYAAAB0pV/DAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAACMSElEQVR4nO3dd3hb1f0G8PdqWrLlvbezgeydQJqkJQ00BMpoacIepZSmkAYoe5XZAiGMskfKXg2bH6OBBMiChCSMbG8njveQJVnz/v64kqxlx3Jkydd+P8+jx7bulXR0x7H96tzvEURRFEFERERERERERERENIAoYt0AIiIiIiIiIiIiIqJADK+JiIiIiIiIiIiIaMBheE1EREREREREREREAw7DayIiIiIiIiIiIiIacBheExEREREREREREdGAw/CaiIiIiIiIiIiIiAYchtdERERERERERERENOAwvCYiIiIiIiIiIiKiAUcV6wYQERER0dAkiiIcDgecTmesm0Iyp1QqoVKpIAhCrJtCRERERBHE8JqIiIiIos5ms6G2thZmsznWTaFBQq/XIycnBxqNJtZNISIiIqIIEURRFGPdCCIiIiIaOlwuF/bv3w+lUomMjAxoNBqOmKU+E0URNpsNDQ0NcDqdGDlyJBQKVkckIiIiGgw48pqIiIiIospms8HlcqGgoAB6vT7WzaFBQKfTQa1Wo7KyEjabDXFxcbFuEhERERFFAIckEBEREVFMcHQsRRKPJyIiIqLBh3/hEREREREREREREdGAMyjKhrhcLhw6dAgGg4H1EomIiIgGOE/ZEKfTCafTGevm0CDhdDrhcrnQ0dEBm80W6+YQERERUTdEUYTRaERubu4Rr54bFOH1oUOHUFBQEOtmEBEREVEvFBUV4cknn4TFYol1UwaNadOm4f7778e8efNi3ZSI+eCDD7By5Up8+eWXvX5MY2MjFi1ahMrKyn5sGRERERFFQnV1NfLz83tcZ1CE1waDAYD0hhMTE2PcGiKSM6vDiR9r2uB0iShM1SMrMQ4KBa/oICKKJJvNhrq6OhQXF8tqYr2LL74YL774Ii677DI8/vjjfsv++te/4oknnsD555+P559/vt/acMcdd+D999/Htm3b/O6vqalBSkoKtFptv722h8ViQVZWFr7//nuMGDGi317nhx9+gFKpxKRJk3q1fmdnJyoqKrB161ZoNJp+axcRERERHZ329nYUFBR4M92eDIrw2lMqJDExkeE1EYXF4XThh4Nt2FTahI2ljdha0QKrw+VdrlUpUJSmR1FaPErS41GUpkdJWjyK0uORw2CbiKhPOjs70dDQAKVSCaVSGevm9JogCCgoKMAbb7yBVatWQafTAZDez2uvvYbCwkIIgtCv78lzWWXga+Tl5fXbawb64osvUFRUhNGjRwcts9lsEQuOu3uv3VEqlVAoFEhISJDVhyJEREREQ1Vvyj9zwkYiGlJcLhE/H2rDs1+X4eLV32HiPz7HGY9vxP2f7sWGA02wOlxIT9CiJD0eKoUAq8OFfXUd+HxXHZ7+qgw3vfMTlj67Bcff9wXG3PoJFqxcj0v/sxV3f7QLL2+uxIYDjahpMcPpEmP9VomIZEMURZhtjpjcRDG8/nry5MkoKCjAmjVrvPetWbMGhYWFQSOEP/nkE5xwwglITk5GWloaTjnlFJSWlnqXv/jii0hISMD+/fu9911xxRUYM2YMzGZz0GuvXr0ad9xxB3bu3AlBECAIAlavXg1A+sP/3XffBQBUVFRAEAS8+eabmDNnDnQ6HaZNm4Z9+/bhu+++w9SpU5GQkICTTz4ZDQ0Nfq/x7LPP4phjjkFcXBzGjBkTNMIcAN577z2ceuqpAIDbb78dEydOxLPPPouSkhJvaHyk9+5p45o1azB//nzo9XpMmDABmzZt6nbbNzQ0YOrUqTj99NNhtVq7XY+IiIiIBo9BMfJ6KGvvtOO+/9uDTIMWmYY46Wui9H16ggYqJT+foKFNFEWUNnRgY2kTNh5owubyJrSa7X7rJOnUmDUsDbNHpGHWsDSMyEyAIAiwO1041GpBeaMJlU1mVDSZUOH+vqrZDJvDhf31Hdhf3xH0uhqVAoWpehSn6VHsHqnt+T43WQclR2wTEXlZ7E4ce+unMXntXf9YCL0mvD+JL774Yrzwwgs455xzAADPP/88LrroIqxbt85vPZPJhBUrVmD8+PHo6OjArbfeitNPPx07duyAQqHA+eefjw8//BDnnHMONm7ciE8//RTPPvssNm3aBL1eH/S6Z599Nn766Sd88skn+N///gcASEpK6radt912G1atWoXCwkJcfPHFWLp0KQwGAx5++GHo9Xr8/ve/x6233oonnngCAPDKK6/g1ltvxWOPPYZJkyZh+/bt+OMf/4j4+HhccMEFAKSJ0j/88ENvUA4ABw4cwH//+1+sWbPGO0r6SO/d46abbsIDDzyAkSNH4qabbsKSJUtw4MABqFT++6S6uhoLFizAzJkz8dxzz8lqxD4RERER9R3Da5mrbe3Eq1uqQi4TBCAtXoMMT6jtE2z7fp9h0CJOzX8AaPCobjZjY2mjFFiXNqHB6D86K16jxPSSVMweno5Zw9NwbE5iyPIfaqUCRWnxKEqLD1rmcLpwqLUT5U0mVDaZUNHoDrebTKh2B9sH6jtwIFSwrVSgIFWH4rR4FLtDbU9ZEgbbREQD37nnnosbbrjBOynghg0b8PrrrweF12eeeabfz88//zwyMjKwa9cujB07FgDw1FNPYfz48bjyyiuxZs0a3H777ZgyZUrI19XpdEhISIBKpUJ2dvYR23nNNddg4cKFAICrrroKS5Yswdq1a3H88ccDAC655BLvyG1ACrsffPBBnHHGGQCAkpIS7Nq1C0899ZQ3vN68eTMAYMaMGd7H2Ww2vPjii8jIyAjrvXvauGjRIgBSPe/jjjsOBw4cwJgxY7zr7N27FwsWLMDpp5+OVatW9eryUiIiIiIaHBhey1ySTo0rfzUSDcZO1LdbUW+0ot7YicYOG5wuEY0dNjR22LC79sjPExhuZxi0yEz0Db7jkKDlIUMDz+G2Tmwqa8TGA03YVNaEmhaL33KtSoGpxSnesHpcXhLUR3lVgkqpQGGaHoVpegAZfsscThdq2zq9I7UrmszuryZUN1tgc7pQ2mBCaYMp6HnVSgEFqe7R2ml6d53teJSkxSM3OY5XUxDRoKRTK7HrHwtj9trhysjIwKJFi7B69WqIoohFixYhPT09aL39+/fj1ltvxZYtW9DY2AiXS5pToaqqyhvgpqSk4LnnnsPChQsxe/ZsXH/99Uf3hnyMHz/e+31WVhYAYNy4cX731dfXA5BGSpeWluKSSy7BH//4R+86DofDb3T3e++9h1NOOcVv9HRRUZFfcA307r0HtjEnJwcAUF9f7w2vLRYL5syZg6VLl2LVqlV92xBEREREJFtMImUuOykOKxaMCrrf6RLRbLKh3tiJeqMVDe1W7/d17dLX+nYrGoxW2JwutFnsaLPYQ5Y/8KXXKL0lSjIStSHLlWQatEjWqzkqhvpNs8mGzWVN3tHVZQEhsEohYGJBMmYPT8Os4emYVJgc1asLVEoFClL1KEjVY85I/3/mnS4Rh1otqGwyS6O23aF2RZMZVU1m2JwulDWYgt4TIL2vAncpEr8JJNPjkZesY7BNRLIlCELYpTti7eKLL8ayZcsAAP/+979DrrN48WIUFRXhmWeeQW5uLlwuF8aOHQubzea33ldffQWlUona2lqYTKZezbreG2q12vu95++ywPs8oXJHh/Q34DPPPOM3qhrwnzDx/fffx3333ee3PD4++Aql3r73UG30tAkAtFotTjzxRHz44Ye49tprozoxJRERERHFnrz+S6BeUyoEZLhHTx/Xw3qiKKLNYveG2Z6A2/f7BqMV9e2dMNmcMNuc0ijSpuBJhHxplArv62catMhKDC5VkpmoRVq8liUS6IjaO+34tqwZG0ulkdW7a9v9lgsCMC4vCbOGp2H28HRMLUpB/AC9SkDpDqALUvU4YaT/KD2nS0RtmzvYbnSXI3GP2q50lyIpbzShvNEEwH+CLZVCQH6Kzl2GxF2KxP19foruqEeaExGRv5NOOgk2mw2CIHhLc/hqamrC3r178cwzz2DOnDkAgG+++SZovY0bN+Kf//wnPvjgA1x33XVYtmwZ/vOf/3T7uhqNBk6nM3JvxC0rKwu5ubkoKyvz1vIOtH//flRWVmLBggU9Pldv33tvKBQKvPTSS1i6dCnmz5+PdevWITc3t0/PRURERETy06d059///jfuv/9+HD58GBMmTMCjjz6K6dOnd7v+qlWr8MQTT6Cqqgrp6ek466yzcO+993pnI7/99ttxxx13+D1m9OjR2LNnT1+aR2EQBAHJeg2S9RqMyup5lI/J6nAH2+6A212ipMGnXEm90YpWsx02pwsHWy042Grp8TkVApCeEFCL26BFRkC5kowELTQqhm9DhcXmxNbKZm/N6h9rWuES/dcZnWVwh9VpmFGShiS9OvSTyYhSISA/RY/8FD2OH+EfbLtcImrbO1HZaHLX2e4qRVLZZIbV4fL5YKkhxPPqvKG2J+AuSpNCdAbbREThUyqV2L17t/f7QCkpKUhLS8PTTz+NnJwcVFVVBZUEMRqNOO+883DllVfi5JNPRn5+PqZNm4bFixfjrLPOCvm6xcXFKC8vx44dO5Cfnw+DwQCtVhuR93THHXfgyiuvRFJSEk466SRYrVZs3boVLS0tWLFiBd577z2ceOKJISeTDPe9h0OpVOKVV17BkiVL8Mtf/hLr1q3rVc1vIiIiIpK/sMPrN954AytWrMCTTz6JGTNmYNWqVVi4cCH27t2LzMzMoPVfffVVXH/99Xj++ecxe/Zs7Nu3DxdeeCEEQcDKlSu96x133HHeWdMBBM0wTrEXr1WhRKtCSXrwpaG+Ou1OabS20SrV4g45qtuKJpMVLhHeIBxo7/F5U/RqKeBO9IzoDi5Xkpmold1lxwRYHU7sqGr1jqzeXtUCu9M/rS5Jj/eG1TOHpSE9ITL/qMuFQiEgL1mHvGQdZocItg+3d3qDbE+o7ZlE0upwobLJjMomM9YHPK/S/bz+E0dKXwtS9PzQiIioB4mJid0uUygUeP3113HllVdi7NixGD16NB555BHMmzfPu85VV12F+Ph43HPPPQCketT33HMP/vSnP2HWrFkhS2SceeaZWLNmDebPn4/W1la88MILuPDCCyPyfi699FLo9Xrcf//9uPbaaxEfH49x48Zh+fLlAKR6156JG3vSm/ceLpVKhddeew1nn322N8AO9b8HEREREQ0ugiiK4pFX6zJjxgxMmzYNjz32GACpJl1BQQH++te/hhxRsWzZMuzevRtr16713nf11Vdjy5Yt3ssHb7/9drz77rvYsWNHn95Ee3s7kpKS0NbW1uM/ETSwOJwuNJls3ZYrkWp1d6KhwxoUZPbEoFX1WI9bCr/jkBinYl3uGHE4XfjpUDs2ljZiU2kTvqtoRqfd5bdOblIcZg1Pd9etTkNusq5/G9VaDVR8DVR8A1RuAFwuILUESB0m3dKGS19TigF1P7clglwuEfVGq7cMiVRn2+wNui327i89VwhAnnfEtv8EkgWpOmhV0asjTkSDS2dnJ8rLy1FSUuK9Eo8GtsbGRuTk5KCmpsY7+eNAw+OKiIiISB7CyXLDGqJqs9mwbds23HDDDd77FAoFTjzxRGzatCnkY2bPno2XX34Z3377LaZPn46ysjJ8/PHHOO+88/zW279/P3JzcxEXF4dZs2bh3nvvRWFhYTjNG5ocNqDiKyAhGzDkAPpUqQCwDKiUCmQlxiErMQ5AUrfruVwiWi121Bs7UdfeVbakwVOqxKdsSafdBaPVAWODI+SEd760KkVQuZLMxDhvnW7PKO9UvQYK1uU+Ki6XiD2HjdhU1oRNpY3YUtYMo9Xht056gqYrrB6WhqI0ff9+uNBWIwXV5V9LoXVrZYh1qoDywLHKABLzuwm2SwBNz5dSR5tCISA7KQ7ZSXGYNTzNb5koBgTbjWa/OtsWuxPVzRZUN1vw9f5G/+cVgNxkd7CdrvcG3MXpUikSBttERINLc3MzVq5cOWCDayIiIiIanMIKrxsbG+F0OoP+aM3Kyuq2PvXSpUvR2NiIE044AaIowuFw4PLLL8eNN97oXWfGjBlYvXo1Ro8ejdraWtxxxx2YM2cOfvrpp5CzrVutVlitVu/P7e09l5sY1NprgJfP7PpZqQESsgBDtnRLcH815HTdZ8gBdCmyCbkVCgGp8RqkxmswpofyhqIowmh1eEdvN/QwCaWx0wGrw+UN5nqiUgg+dbmlUdueEdxZBikUHJNtgIp1g71EUURZo0kqA+IeXd1itvutkxinwsxhUhmQ2SPSMTIzoX/D6vZDXUF1xTdAS7n/ckEJ5E4CSuYARScAWgPQXAY0l7q/lgFNZYC1TTrv2muk5wpkyAFShweH2yklgDah/95fHwiC4P0Aaeaw4GC7wRtsm911trtKkZhtTtS0WFDTYsE3BwKfF8hN0gWE2lJZkoJUPeLUDLaJiORm1KhRGDVqVKybQURERERDTL8XB163bh3uuecePP7445gxYwYOHDiAq666CnfeeSduueUWAMDJJ5/sXX/8+PGYMWMGioqK8Oabb+KSSy4Jes577703aILHIcveCWSNBYy1gLkJcNqAtmrp1hOlxifYzvYPtn1DbxmF3IIgIDFOjcQ4NUZk9hwSWmzOrpHbPpNQ1vkG30Yrmk02ONz1hA+3d3b7fAWpOvxxzjD8bkoBdJqhGcxVN5uxyV2zemNpI+rarX7L9RolppekukdWp+PY3EQo+3NEe3utFFJXuAPr5jL/5YJCCquLTwCK5wCFM6XA2lfhDP+fRREwN3eF2X7BdinQ2Sqdi8ZaoPKb4DYlZLmD7WFd4bYn2I4bWCWPBEFAZmIcMhPjMCNUsN1h9QbZvqF2RaMJJpvTO2HrhgNNAc8rBdtF3okjPXW241HIYJuIiIiIiIiIfIRV89pms0Gv1+Ptt9/Gb3/7W+/9F1xwAVpbW/Hee+8FPWbOnDmYOXMm7r//fu99L7/8Mi677DJ0dHRAoQg9WnXatGk48cQTce+99wYtCzXyuqCggDWvHTagow4wHpbCs446d5B22OdWC1iae/+cSi1gyAoOtf1GcmcDccmyCbnDYXO40Nhh9Qu4vRNRusuVVDSZYOyUSmCkxWtw4exinD+rGEl6dYxb37/q2zuloPpAEzaWNQaNYNeoFJhSmOIeWZ2G8fnJUPfn6HTjYZ+w+hugKXA4sALImeAOq38hhdWRDozNzUBzeXCw3VwmfbjUk/gMd6jtE257ypHEdV9WZ6ARRRGNHTZ3GZLgUdsdAeVifAkCkJMYhyKfkdq+wfZQ/WCISC6qmsz462vfw+4UkRqvQUq8Bql6tfQ1XoMUvQZp7vsNahHt9QcxbNgw1iamiGHNayIiIiJ56Lea1xqNBlOmTMHatWu94bXL5cLatWuxbNmykI8xm81BAbVSKQUQ3eXmHR0dKC0tDaqL7aHVaqHVasNp+tCg0gDJBdKtJw6rf8ht9Am5O3xD7hbAaQVaq6Rbj68d5y5XkhM8ktv3/rgkWYXcGpUCucm6HicLtNiceHNrNZ75ugw1LRY8+Pk+PLG+FEunF+KSOSXISZLP5H49aTHZsKW8CRtLpduB+g6/5SqFgAkFyd4JFicXpvTvKFpjnTS62VO3uml/wAqCT1g9Byia1f8hsD5VuuVPCV5mafEJtsv9w21TQ9etekuI500LCLY95UiGSVdHDCCCICDDoEWGQYupxal+y0RRRJPJ5ldf2xNwVzSaYLQ6cKitE4fapA9GAuUkxUkjtn3C7eL0eBSlxjPYJhoA7vxoF3bWtPVq3TyDErfPz4Szzgi11gaVQoBSoXB/Fbq+KoPv52TLRERERERDR9hlQ1asWIELLrgAU6dOxfTp07Fq1SqYTCZcdNFFAIDzzz8feXl53hHTixcvxsqVKzFp0iRv2ZBbbrkFixcv9obY11xzDRYvXoyioiIcOnQIt912G5RKJZYsWRLBt0peKi2QXCjdemLv7N1I7s5WwNEpTXgXatI7v9fW9TCS2+d+baJsQm6dRokLZhfjnBmF+OjHWjyxrhR7Dhvx7Dfl+M+mCvx2Yh7+NHcYRmQG128fyIyddnxX0SyNrC5twu7D7fD9vEkQgONyEzF7eDpmDU/DtOJUJGj7sRJRR4MUVpe7R1Y37g1YQQCyx0lBdckcoHAWoEvuv/aES5cC5KUAeZODl3W2+QTbAbeOOmnUtrkJqPku9PN6A+3h/nW2B1jZH0GQ6senJ2gxpSg42G422byTRXonjnQH3MZOB2rbOlHb1onNZcFXj2QlaoPqaxenx6MoTQ+9pt8rZBENed+WN+PzXXVQKgQ8+LsJECGi2WRHi8mGZrNN+mqyocVsQ7PJDoXCCQBwiSJsDhdsAABnr15LCrEV3jBbpRCgVArdBuBKBt5ERERERLIV9n/0Z599NhoaGnDrrbfi8OHDmDhxIj755BPvJI5VVVV+I61vvvlmCIKAm2++GQcPHkRGRgYWL16Mu+++27tOTU0NlixZgqamJmRkZOCEE07A5s2bkZGREYG3SH2mjgNSiqRbT+yW3o3k7mwDHBagpUK69fja+t6N5NYaBkw4p1IqcNrEPJw6IRfr9jXgyXWl2FLejLe21eCtbTVYcGwW/jxvOCYXDqyRsh6ddie2VbZgY2kjNpY24YeaNjhd/ldHjMpK8IbVM0pSkazX9F+DTI3uMiDuW8Pu4HWyxklBdfEJUlitTw1eJ0yHTYfxfd33cIpOFCYWotBQiGRtcv8GH3FJQO5E6RbI2hE61G4u67pC4uA26RbqeUMF26nDgPj0AXPuAFKwnZagRVqCFlOK/M8RURTRYrZ7a2pXNJndZUikYLu904G6dqlm/Zby4GA706D1C7SL0+K9I7jj+/MDF6IhQhRF3P2x1EefPa0Av52Ud8THmM0WlJeXoyAtHkqNFk6XCIdLhNPlkr46pZ+l+0Q4XC7v7ySnS4TT1bugGwAE+ITZSiEg3Fb43e+5TyGAgTcRERER0QAQVs3rgSqcOikUQ3ZL14ht31DbbyT3YcDau0uOAQDq+N7V5A6ciC9Kvq9qwZPrSvHZrjrvfdNLUvHnucMxb3RGTP8xtjlc2FnT6h5Z3YjtVa2wOV1+6xSl6d1lQNIxc1gqMg39WD/S1ARUbuiqWV2/K3idrLE+ZUBmH3VYLYoiDnYcxNa6rdh6eCu21m3FwY6DQesZ1AYUJBagyFCEgsQCFBoKUZhYiAJDAdLi0mK3H22m7kdstwe/Dz/axK5JIwPD7YTMARVsH0mLyeaeONLsLkNiQrk74G4123t8bIZBixJPmO0bbKfH9++VBESDyAc7D+Gvr22HXqPEumvn9ep3RV9qE4ui6BNye8JtlzfoDhWAO/v4Z64g+IfcUsAdoqyJTwCu6M9JiKlXWPOaiIiISB7CyXIZXstck6UJF396MTL0GcjSZyFDl4FMfab0vV76Pk2XBrVCRpMH2sw+4XZAyO17v7W998+pSeh+JLdv6K1N6Je3dKC+A09/VYp3th+E3SmdcmOyDbh87nCcMj4Hqv6cyNDN6RLx08E2aZLF0iZ8V94Mi91/5Fp2Yhxmj0jzjq7O66HW91EzN7vDavfI6rqfgtfJPFYKqotPAIqOB+LTjuolRVFElbHKG1RvrduKw6bDfusoBSXGpI5BvDoeVcaqoOWB9Cq9d4S252uBoQCFiYXI0MXwAwq7+yqHptLgYLutBkAPXb8moftg25Atq2C71WzzjtT2TiDpDrhbjhBspydoUZIuTRoZOGrbECejPpWoH1kdTpy4cj2qmy3424mjcNWJI3v1uGiFjC5P4O30CbV9A3Bn16huz32uPv5prPANvJWKoJDbG3Qru8qZKI6yPxUEAe+8847fROpyt3r1aixfvhytra1hP5bhNREREZE8MLweQnY17cLZH57d4zoCBKTp0pChy/ALtf1uukwkaZPkdYmszdS7kdw2Y++fU2MIGMntG3j7BN+a+D41ubbNgue/KcerW6pgsknBcV6yDn+cU4KzpxVGdNI5l0vEvnqjt2b1lvImGDsdfuukxWswc3gaZg+XAuviNH3/HQOWFqByY1fN6rqfEBSgZhwjBdUlc9xhdfpRvaQoiihrK/OG1dvqtqHB0uC3jkpQ4bj04zA1ayqmZk/FpMxJiFd37d9ORydqjDWoMlah2liNqvYq7/eHOg5B7CEE1ql0yDfk+4/YdofcmfpMKIT+/9AiJLu7Rn1QsF0qBduiq/vHqvXuIDtEuG3IARQxek990OYpRdJkQoV7AskKd63tZpOtx8emJ2jcobZ/sF2czmCbhpZnvy7DXR/tRqZBi3XXzut1jfmBHDK6Akdwe0Purvu+3bIZf1j8a5ww/0Q8tvrNHn8X9CSofIl3kkr/+++96x/44IP3sWP7dr/f04cPH0ZKSkpUJjK3WCxIT0/Hzp07MWLEiH57HYbXRERERIMfw+shxGQ34afGn1BvrvfeGiwNqDPXocHcgAZzAxyi48hPBECj0HSN4PYE3Dop3Pa9X6fqx9G4/cHaEWKyydqAOt2HAVtH759Tm+gOtrOBxFzgmFOB0b/pdXDXZrbjpc0VeGFDBZrcIVlqvAYXzCrG+bOKkBIffi1pURRR0WT21qzeXNrkfW4PQ5wKM4eluUuBpGFUpqH/LnO2tAJVm9xh9dfA4R8RFFanj+6qWV10ApBwdHXuXaIL+1v2e4PqbXXb0NzpXwNZrVBjXPo4TM2eiqlZUzEhYwL0an2fXs/mtKGmowbV7dWoMlahqt0dcBurcKjjEJxi9zVZNQoNCgwF3nIknjIkhYmFyNZnQ6mI3AcZYXFYgdYqKcwODLdbq4Ae3hNUcUBKiTRZZGC4nZgnr2DbYu+aNLLR5C1LUtFoCjqvAqXFa/zKkPjW205ksE2DSJvZjl/c/yXaLHb888xxOHvaESaC9iH3kPHSSy9FQkICnnvuOezZswfZOTk91uqWRn773x+OJ1behy8//Qhvffo1lIqAUd3esDt0AK4QIjNh5fvvv4/rr78eu3YFl/Wy2WzQaCIzDwbDayIiIqLBj+E1eblEF1o6W7rCbYs74DZ3Bdz15nq0WFt6/ZwGjcGvRInn5lu6JE2XBpVCZvVircauySaDwm6fkNtuCv34jDHACSuAsWcCyl6OPLM78dbWajz9dRmqmy0AAL1GiT9MK8Slc0qQe4SyHQdbLdh4oBGbSpuwqawJtW2dfst1aiWmlaS6R1an4bjcJCj7K6zubAMqN7lrVn8N1P6A4LB6lLtmtbtudULmUb2k0+XEnpY92HZ4G7bWbcX39d+jLaBmepwyDhMyJmBK9hRMzZqK8RnjoVX2/wg1u9OOQ6ZDfiO1PeF2jbGmxw+VVAoV8hPyg8qRFBoKkZOQE7tzy2ED2qq7CbYrAVcPH5QptUBKsTvY9h25PRxIygdiFdb3QXunHZWNZnegbUK5z6jtxo6eg+1Ud7At1dmWRmoXu0dwJ+kZbJO83PPxbjz9VRlGZxnw8VVzwvr9IueQsaOjAzk5Odi6dStuu+02jB8/HjfeeCMAYOnSpXA6nXjjjTe869vtduTk5GDlypU4//zzYTQacfnll+Pdd99FYmIilq+4Gh988AHGjhuPf9x3f1AN77deewk3Lr/Crw3/ePDfOO33SzGhIAUPPfMyfnnSIhysrsJvZk/Avx5/Hq+98DR2/bADI0Yfg3sfewaWjnb84/qrUX5gH6bNmo1HnngGmZlZULknqnz5Py/gsYdXoaKiHMXFxbjyyitxxRX+r3nJJZcgIyMD9913H26//Xa8++67WLZsGe6++25UVlbC5XLhk08+wV133YWffvoJSqUSs2bNwsMPP4zhw4cDACoqKlBSUoL//ve/ePTRR7FlyxaMHDkSTz75JGbNmgUgOLxuaGjAySefjIKCArz++us9jjKX83FFREREvdNhdWDDgUas39eA9Xsb0NhhxeTCFMwcloYZw1IxsSAZcWr5/H85VIWT5cosXaRwKQQF0nRpSNOl4Zi0Y7pdz+a0ocHS4A21/QJuS4M3/LY4LDDajDDajDjQeqDH102PSw8qURJYuiRRkzhwSpVoDdIt/QiXwlqNPmF2HXD4B2DbaqBhD/DOZcCXdwMnLAcmLAXUPf/jFKdW4rxZxVgyvRAf/3QYT6wrxe7adjy/oRwvbqrAaRPzcPncYRiZJU042WC0YlNZEza5R1dXNpn9nk+jVGByUbK3ZvWE/GRoVP002rWzHajaDFR8JZUBqd0ZXG4ibURXUF18gjRS/SjYXXbsbtrtnWBxe/12dNj9R8zrVDpMypzkLQMyNm0s1Mroh4JqpRpFiUUoSiwKWuZwOVBrqu0asW2s8n5fbayG3WVHRXsFKtorgh6rElTITcgNHrFtKEReQl7/vleVRgqf04YDIxf4L3Pau4Lt5nL/cLulAnBagca90i2QQi0F26nDAsLt4UBSQa8/DIqWxDg1xuUnYVx+UtAyY6ddGqEdOIFkoxmNHVY0m2xoNtmwvao16LEpejWK0uJRku6eNNJn1HayPjIjGokipbrZjNUbKgAA1/9mjBRciyJw4H9Sf6BPkybV1acBcclHvvJCFAG7ued1+otaH1Yt/zfffBNjxozB6NGjce6552L58uW44YYbIAgCzjnnHPzud79DR0cHEhKkeTQ+/fRTmM1mnH766QCAFStWYMOGDXj//feRlZWFW2+9FT/s2I6pkyeFnOxy+WUXoqm6FJ988gn+79PP4HSJiE9IhMYd4ibr1UhP0MKok/r/Jx+6DzfccS8yc/Jxy9XLcP1fLoU+IQHX3n4P4nR6/P3PF+Guf9yBm+9dCQD46J038dBdt+P6u/6FMceNx56ff8CNf78KbXYFzvrDOe7a3MD7H3yI1a++gaYOKzrtThw4cABvvv023njrbWjUUj9tMpmwYsUKjB8/Hh0dHbj11ltx+umnY8eOHVD4HAM33XQTHnjgAYwcORI33XQTlixZggMHDkCl8u/vq6ursWDBAsycORPPPfcclEr+I0pERDTUiKKIfXUdWLe3Huv2NmBrZbN3LjGPTWXSgD4A0KgUmFSQ7A2zJxemMMyWuYGVCFDMaJQa5CXkIS8hr9t1RFFEh73DP+C2NKDO5B9wN1oa4RSd0ihvSz1+bvq52+eMU8b1WKLEE3pHY6Rsr3lDbvekVON/B/ziGuC7Z4FNj0sjUD/8G7Dun8DsZcCUi444EaRKqcCpE3KxeHwOvtrfiCfWHcDmsmb89/sa/Pf7Ghw/Ig0NRiv21fkHtUqFgPH5Sd6a1VOK+rFTthrdYfXXUimQ2h3BYXXqcP+wOjHnqF7S5rTh56afvTWrt9dvh8Vh8VsnQZ2AyVmTpbA6ayrGpI0Z8BOUqhQqqWSIoQCzMdtvmdPlRJ25zr8Mic/obavT6g28N2CD32MVggI58Tn+o7XdX/MMef17HinVXWVCAjkdQHtNV5jd5DNiu6UccNqApv3SbX/AYxUqILkoINh235ILpdcdQAxxaozNS8LYvOBgu8PqkEZou0dtV7gnkKxoMqHeaEWL2Y4Wcyt2VLcGPTbZHWwXe0PtrhHbfSkzRHS07v90L2xOF44fkYZ5o9wln7a9IP3+CyQopABbn9Z1SywGchYDpkbApQPsNuChMdF8C11uPBTWXBbPPfcczj33XADASSedhLa2Nqxfvx7z5s3DwoULER8fj3feeQfnnXceAODVV1/FqaeeCoPBAKPRiP/85z949dVX8atf/QoA8MILLyA3N7fb19PpdEhISIBKpUJBXvB6yXoNcpN1sLmv1rrxur/jkiVnAAA6rv4bzjlnKT7+5DOcMHeeNHnzhRfhtVdeQlq8Fg6XC0+u/Ceuv+NunHzKaXC4ROQXFqFs31688dLzWHTmHwAAO7ZugUsUkTdqPA62WtBmscNqs+HGfz6GuDRpfoqfD7bhuOMXQOUua5KWKuCeh/6NcSMKsXHrDowbNw5mq3SFzt9WrMBvfvMbCIKAO+64A8cddxwOHDiAMWO6joG9e/diwYIFOP3007Fq1aqBM9iBiIiI+p2x044NBxqxbm8D1u9rCLrKvDhNj3mjMzF3VAZyk3X4rqIZW8qbsbmsCQ1GK7aUSz9jrTTIb2JBMmYOS8WMYWmYXJgS0fnGqP8xvKZeEwQBBo0BBo0Bw5JDBFRuTpcTLdYWKeA2+dfg9i1d0mZtQ6ezE9XGalQbq3t87SRtknfUdmCJksx4KfhOjUuNXZ3guCRgztXAjD8D218CNjwihXWf3Qx8/SAw43Jg+mXSKLQeCIKAuaMyMHdUBnZUt+LJdaX4dNdhbDjQ5F4OHJuT6A2rpxan9N/kcNYOoHpz1wSLh7YH1ztOKXFPsPgLaYLFpO4//OjVSzqt+KHhB29YvbNhJ6xOq986iZpETMma4h1ZPTpldOz2ez9QKpTITchFbkIuZubM9FvmEl2oN9cHBdqe7y0OCw52HMTBjoPYVLvJ77ECBGTHZ/sF2p5JJPMN+f1by16pkkZWpxQDw3/pv8zlBNoP+pcgafIJth2d0kSSzaXAgc/9HysopQDbE2b7htvJRdJI8QEkQavCcblJOC43ONg2WR3eILuiyYTKRjPK3WVJ6tqtaDXb0Wpuxc4QwXaSTo3iNL27DIn/BJIpejUDH4q4H2pa8f7OQxAE4IaTj5GOsc524Iu7pRXSR0kfSpmbAWu79EGnpVm6Nbk/oUooAFLnAqZ6wCoAdkv3LziA7N27F99++y3eeecdAIBKpcLZZ5+N5557DvPmzYNKpcLvf/97vPLKKzjvvPNgMpnw3nvv4fXXXwcAlJWVwW63Y/r06d7nTEpKwujRoyPWxvHjx3u/z8mRrniaOnmi9++FYYV5aG5sQF6KDiaTCVUVZbh5xTLces2V3sc5HA4kJiWhOD0eTqeIZ9Z/joUnnYx0QxycLhFqpQJ5+QXIzMiE0+WCCMApiijbfwCPP3APftyxDa3NzXC5a3vv3HMAibnDcLBZGl2fkDMcPx5sg0ohoEMhfcC/c18lEjIL0Wq2wWKx4IQT5uB3Z/8B9/zrQdgcLiiVApQRqt9NREREA4soithda8S6ffVYv7cB2ypb4HB1ja7WqhSYNVwaNDFvdCaK0/0HHozONuDcmUUQRRHljSZsLmvGlvImbC5rQl27Fd9WNOPbimbgiwNQKwVMyE/GjGGpmDksDVOKUno96TjFBvcORZxSoUS6Lh3punQcl3Zct+tZnVZveZJ6c33IgLveXA+r04o2axvarG09lipRCkqk6dL86nFnxQfX5k5QJ/TfPz4aPTDjT9Jo6x/eAL55SArd1t0LbHwUmHoRMGtZr8pnTCxIxpPnTUFpQwc+31WH4jQ9ZpSk9d8oS5vJPbL6G3dY/X1wDePkIvcEi+6R1Un5R/WSZrsZOxt2esuA/Nj4I+wuu986qXGpmJI1xRtYj0wZCYUgn4n/IkkhKJAdn43s+GxMy57mt0wURTRaGoMmjvQE2ya7CbWmWtSaarHl8Jag587UZ6LQUIiixCLvxJGFBqkkSV8ntOzdm3IH0MmFwLB5/stcLsB4KCDYLpXKkjSXAQ6LFHC3lAOla/0fKyikkiPdBdtHKOkTbfFaFY7NTcSxucG1vsw2h3eyyIoms7sMiTRq+3B7J9osduysacPOmragxybGqVCcLtXXLgkIuFPjNQyBKGyiKOLuj3YDAE6fmNd1lcE3DwHmRiBtJPDnjV1XRThsgKUFMDdJN0uz+6vFfSVTkvTXqEoHXLpW+kCrp4lhj0ShkvoVhQoQVIBSKf0sqNzL3DdB0bVuGH3cc889B4fD4TdSWhRFaLVaPPbYY0hKSsI555yDuXPnor6+Hp9//jl0Oh1OOumkvr+nMKnVXR9qe87xwPs8oXJHh3RF1zPPPIMZM2b4PY9SqfROMvu/Tz7Cfffdh/wUaVulxmuQnGjAsbmJEEWpPrfTJeKsX52DgsJCPPrvJ5GRnQ2Hw4n5s6ZCDRf0GpW3lJnK3R6HS4TNfdlvh9WOZrMNxk4H1Botph3/C3z44Yc4/YLLkZUjbW8BwRNSeiarVCoEuBw2dNqd2HfYiNQkF1LjNdCplezriIiIBqA2i2d0dT3W72tAXbv/wLVh6fGYO1oKq2eUpPbqKnNBEDAsIwHDMhKwdEYhRFFEZZPZHWRLI7Nr2zqxtbIFWytb8O8vS6FyX9E+Y1gaZg5Lw9SiFMRrGZcOJNwbFDNapdZbOqE7oiii3dbuH3D7lCjxhN+Nne5SJe77eqJT6bz1tzP1mUElSjzLNMqjCIlVGmDyecDEpcCu94CvVwJ1P0oB9pangUnnAsdfKY1EPYLhGQkYPrfnsiN9YjMD1VvcYfXXwMFtIcLqQndQPQcoPl76+Sh02DqwvX47ttZtxba6bfi58eegiQszdBneUdVTs6aiJKmE/3T2giAIyNBnIEOfgSlZU/yWiaKI5s7moEC7ur0alcZKGG1G77mztW5r0HOn69JDjtguNBQiQdMPx6aHQiF9QJKUL43u939TUu355tLQo7btJqmET2slUPZlwBML7mC7JDjcTikG1P04Cr0P9BoVjslJxDE5oYPtqmaztxSJb7Bd29aJ9k4Hfqhpww8hgm1DnArFaVJ97RJ3wO0ZtZ3GYJu6sXZ3PbaUN0OjUuDqhe7Rwq3VwObHpe8X/MO/nI9KAxiypJuvzk6gvFy6YidwYj3RJYXYLofPraefQwTeogsQbUBAdauQfANv7y3wZxUcLhEvvvgiHnzgAfx64UK/p/jtb3+L1157DZdffjlmz56NgoICvPHGG/i///s//O53v/OGx8OGDYNarcZ3332HwkLpd2pbWxv27duHX/ziF0FN89BoNHA6jyLU70ZWVhZyc3NRVlaGc845J+Q6+/fvR2VlJRYsWBByuSBIYXJbaxP279uL5559BnPmzAEAfPPNNwCAzMQ4jMhMgMos/c4YmZmAY3IS4XSJaFRLfwekJ2iQnRiHhDgVlAoFHnnqeVx52UX44x9OxfNvfoj0rGyIkCaxdHSzX0WHDY0dNtz+wTYcNErbS6tSIDVegxS9Rvoar0GqXi199b3fu1wNrWrwXNFFREQUKzaHCyarAyabA2abU/re6sTOmlas21uP76ta4fQZXa1TKzF7eBrmjc7A3FGZKEw7+kFUgiBIg3fS43H2NCnMrm62YLN7VPaWsmYcbLXg+6pWfF/ViifWlUKpEDAuL8k7MntqUT9e8U69wvCaBjRBEJCkTUKSNgkjUrqfSNHhcqC5s9kv1PYLuN2lS4w2IywOCyrbK1HZXtnja6doU4JC7cDa3KlxqT2PAlYogbFnAMedDuz/DPjqAaDmW2Drc9Ikj+N+B5zwNyAzCnU+7RafsPoboGYrEDDKGUkFXaOqi08AUoInGwxHu60d39d97y0Dsrt5N1wBdbKz47O99aqnZk9FoaGQoVmECYLgnbh1YubEoOWtna1BE0d6Qu5WaysaLY1otDTi+/rvgx6bGpfqnTDSt852gaEASdrg8hgRfFNSTfXEHOlY9SWKQEd9iGDbPWrbZgTaqqRb+frg507M7ybYLpGurhhA9BoVxmQnYkx2cLBtsTmlYNtdX7vCPXq7ssmEQ22dMHY68OPBNvx4MESwrVWhKF3vHrHtnkDSXYokPYHB9lDlcLpw7/9Jo64vPr4Eee4ay/jiTqnET9EJwOiTj/6FBAWgVIRX0z5k4H2E0Nvz+8jzM6w9vsSHn3yJlpZmXLJoGpKSRb+Q+8xFC/Dc00/i8nNPBxQqLP39mXjyicexb/8BfLn2c6lfEgQYDAZccMEFuPbaa5GamorMzEzcdtttUCgUPZ5XxcXFKC8vx44dO5Cfnw+DwQCtNjLzGNxxxx248sorkZSUhJNOOglWqxVbt25FS0sLVqxYgffeew8nnngi9Pqe+7+UlBSkpaXh6aefRk5ODqqqqnD99deHXFcQBKiVCqiVUkklAEjSaZCZGIcUvQaCAIzISsT7/30DS5YswRXn/BZffPElMjKz4HCJcLpc7hBbGvHtcIlwOkVYrU5olAIyErRosHTC5nDB6nChtq0zqFZmT+I1Sr9wO80TentDbrVfGJ6sU0OlHJpXhRERkfyJogirw9UVMNukkNns+9XmhNnq+/UI69gcQZMqhjIiMwFzR2Vg3ugMTCvu3ejqoyEIAgrT9ChM0+P3U6WBlNXNZm+97C3lTahutmBHtTQX0VPry6AQgLF5SdIEkCWpmFaS6r06jaKD4TUNCiqFyhsu98TisHSVJvHcLF0juD2lS2wuG1qsLWixtmBfy77uX1eQXndewTyce8y5KEjsZhS5IACjFgIjfw1UbpDqYJd+AfzwunQbc4pUMztv8tFsBn92C1DznRRUl38NHNwq1SD1lZjXFVaXzJHKKRxFKNXS2YJtdduwrW4bttZtxd7mvRDh/wsrPyEfU7OnesuA5CXkMQiLseS4ZCTHJWN8xvigZW3WNtQYa/xHbLvrbDd1NqG5sxnNnc3Y2bAz6LFJ2iRv6ZHAciTJ2uT+2++C0DXSs8h/QkyIojRBXHNZ6FHb1japXn17jXRFQiBDjjQxaWC4nVJyxIlZo02nUWJ0tgGjsw1Byzrt7mC70eSus232TiZ5qM0Co9WBnw6246eD7UGPTdCqpDA7INQuTtcjI0HL83kQe2NrNUobTEjRq3HF/OHSnYe2S2WyAGDhXUf1O+SoRCHwfu6193DiCTOQlGjwCbwlZ/56Nv71yJP4YfM6jD92FM45eSbu/ucDKMrPwfHDE6VJjt0juFfe9Gdc3lqPU05ZhESDAX9f/hdUV5YjTiVIJby8ZU+U3u155plnYs2aNZg/fz5aW1vxwgsv4MILL4zIprv00kuh1+tx//3349prr0V8fDzGjRuH5cuXAwDee+89XHDBBUd8HoVCgddffx1XXnklxo4di9GjR+ORRx7BvHnz+tw2lUqF1157DWeffTZOPPFXWLduHTIzMwGE/se2s1MBR1scXv/TLGi1WphtTjSbbGgx23y+2tFisqHZbJO++t5vtsHpEmGyOWGyWVDT0vta7Ek6tTvcVocY6e0Jv7tC78Q4NRQK9pdERBQeURTRaXdJo5mtTnRYHSGD48BgWVpPCqfNNqf38Z5R0b4joCNNq1IgXquCXqNEvEaFwjS9d66vgtSjHBzkdEjlJLXB//P0VkGqHgWpepw1RSqLerDVgi1l7pHZ5c2obDJ7r2Z9+ispzD42NxEzS9IwY1gaphenIknPMLs/CaIo9t8RGiXt7e1ISkpCW1sbEhODR58RhUMURbRZ24JCbb+A29KAJkuTXzArQMAvC3+JC467ABMzJh45wDn4PfDNSmD3B133Df+lFGIXHR9+AGDvlAJqzwSLNd8BAZMdwpAjhdUl7sA6peSogoZGS6O3XvW2um0ha5IXJxZLQbW7DEh2/JHrfZM8dNg6vKVI/CaRbK9GvaXn8j0GtaGr/EjAiO20uLTYBKCiKE0wFzLYLgU6W3t+fEKWO9ge1hVue4LtOPn8buq0O1HdbA6qr13eaMKhNgt6+qshXqN019X2H7Vdkh6PDAODbTnrsDow7/51aOyw4rbFx+Ki40ukc2b1KUDlN8D4s4Eznu7183V2dqK8vBwlJSWICywbMpCJrl6UMelmhHc3TGYL8qYsxIO3rsAlS37rv7DbMibd/OwTeEdCY2MjcnJyUFNTg6ysrCM/IMaO9rgSRRHtnY7uw22TDU3unz3rtJrtR37iEBQCjhhwJ+nUULrreisV0oSVSoUAhe/3gmc5oFQooBQEKBTwrq/w/equE67wPhbsl4mI+pHLJcJi9w+KTdaA4DhU4OwJmX3CZd/QuT9TPJ1aiXitEnqNO2z2CZ31WiUStCroNSrEa5TQawO+alTexyZopfX1amXkr05yWIGydVKJ1j0fSf+nZY8Hhs+X5lEqnBXRUpC1bRZsKWv2htnljSa/5YIAHJOdKI3MHpaKGSWpSNb301xlg0g4WS7Da6I+srvsaLI0YV/LPry25zV8c/Ab77Jx6eNw/rHn48SiE6FSHOECh/o90kRXP77VVbOzYKYUYo9c0P0/oQ6rVPrDU7O6+tvgsDohuyuoLp4jhWlH8U9KnalOCqvdgXVFe0XQOsOThnuD6ilZU5Chz+jz65F8me1mVBur/epse74/bDrc42P1Kr03yC5KLPKO3i5MLESGLiN2/2ibm7smi/QNt5tKpUnoehKf4Q61Q4Tbcf1YXiXCrA53sO2usS3V2XYH260W9DRgQ6dW+tXXLvEE3OnxyGSwPeCt/HwfHlm7H8Vpenz2t7nSxHt7PgZeXwKo4oBlW7Ff7MSytcvgcDmQHJeMFG2KdGWHNhkpcSlI1rq/16YgUZEINEs1oPW6gVWOJ+ICAu/t33+PPXv2YPrkCWhrbcE//vUQ1n2zGQe2fIr0lMReBd49imDgvW/fPnz66af461//2vf2RFEsPhRxOF1os9i9AbffSG/fENwshd8tJhuMVseRnzhKFAL8Q3BBgFLpH3xLgTmC7vPcfB/rCc4VQteEmopuQ3d34B7iuX1Dd//ngU9gL/h9Hxzuw++1VQFBfqh2+4f7od9j1/aB9zX5O4xI/pwuEebAoLiX5TFMVs+6/uuZbZGfr8KXJ1wODI69X93BckJAAB0fGEy7v+o1KigH6lVBdot09fqu94C9/wdYg68Q9VLFAYUzgWHzpUA7a5w0n1KE1LV3YnOZNAHklvImlDUEh9mjswyYOSwNM4elYnpJGlLjGWYHYnhNFAMHWg7g5d0v44PSD2BzSeU5cuJzcM4x5+CMkWfAoDnCZSwtFcCGh4HtL3eV98gaB8xZARx7mvRP78FtUlDtCasdAfUbE7K6guriOVIwdhR/TB/sOOgdVb21biuqjdV+ywUIGJUyyhtWT86ajNS41D6/Hg0NnY5ObykSvxHbxmoc6jgUVGrGl06lQ74h33/Etvv7TH1mzzXo+5OlxSfYDriZGnp+rD6tqwSJN9weBqQNA3Qp0Wl/BFgdTtS0WPzqa3sC7oMtvQu2i9PiUZSud4/YlkZwZxnieGl9jNW1d2Le/etgsTvx+DmT8ZtxOYDTDjw+E2g6AJywAuKvbsWln12Kbw9/26vnzNHk4LoR1yEzPxNKjRIqQQWlQgmloIRKoYJSUAb97Ht/zM71CNi+fTsuvfRS7N27FxqNBlOmTMHKlSsxbty4rpVcLkDszQhvn/v6HHgLAcF2L0LvCI/wjhS5jOi3OVxoNUvBthRy29FssnpLl3gC8DaLHQ6nCJco1fZ2iiJc3q/w3ud01/52+fzs8n6N9bsdGgTBP4BXKQJDcvgF58oQwbj0WASH8gGj6H3XDQ73Q4XuXaPylQqEHLkfHO4jINz3D+4DR/37vrYq5Icf/oG/SqHw+6DD93WIesPhdPWtPEbASGbf+zrtR/HB8REIAvwC48BwOSh89q4XevRzvEYFnVo5+M8ZmwnY/7kUWO//DLB1dC1LyAaOPVXKSlKHSxlJ6ZdA2ZeAsdb/efTpwLC5XWF2Un5Em1nf3ulTM7sZB+o7gtYZnWXwTgA5vSQV6QmRmbtEzhheE8VQk6UJb+59E6/vfR3NndJozHh1PM4YeQbOOeYc5CXk9fwExsPApseA754H7O5P8BLzpFGfjoDai/GZXZMrlvwCSBvR538mRVFEtbHaO6p6a91W1Jr8O32FoMCY1DHeCRYnZ03u30n5aMixOW2o6ajpmjjSZ8T2oY5DcIrdj17QKDQoMBR4y5H41tnO1mdDqejfyT+61dnWfbDdUdfzY3UpoYPt1GGAPnVAhkeh2Bwu1LR4Jo80+9XZrmmx9FhjL06tQFGqFGQX+4TaxWnxyE5ksB0N1//3B7z+XTUmFybjv3+eLY0w/PYZ4ONrpH8GrtyO9Q3bseyLZdAoNHjsV49BFEW0WFvQam1FS2fX1zZrG1qsLVA71bgs/zJk5GdAoQ4/iFYICm+Q7RtqhwzBB0Hg3SvewDuMsiaDLPCWS3gdTaIoBdgOl0sKvEX/oNsVEIB3Bd/wC8F7Dsm71nV4lgeF7Z77AKfLBacLXc/tCg7oHX6vg160V3ruwNd2+C4PCPV9nyPwuX0/JHC4XPwQIIqCw30EB/2CTxmcgNA9sFxO6HA/xIcEPVwhEBTuB3wQ4f3gQenzIUGoDyh8r0gICPQDX7unKxv8tk+oDyMGWEkgm8PVzYR/PmFzqPIYAeGzb21nm6P/gmaFAPdoZv9RylIZjJ5HOfsH012Pj1P3PDEz+bAagX2fugPrz/0zkMR8Kaw+9lQgf3ro0dSiCDTslULs0i+lK9Xt/iOjkTbSXWJkvpSlRLjEY4PRim/LpVHZm8uasK8uOMwemZngDbNnlKQhwzD0wmyG10QDQKejEx+VfYQXd72IsrYyANI/2ycWnojzjzsfEzIm9PwE5mbg26eBzU901drVp3dNrlg8B0gfdVRhdXlbubcMyLbD24LqFKsEFY5NP9Y7ueKkzElHHkFO1E/sTjsOmQ4FTRxZbaxGjbEGDrH7y7BVChXyE/L96mt7Rm3nJOQcubxPf7EaA4Lt0q6fA0cMBIpL8g+zfcPt+HRZBdsHWy3ekdqeMiSVTSZUHyHY1qoU3hHbxenu+tpp8ShKj0cOg+2I2HvYiJMf/gouEfjvn2dhSlGq9IHMI5MAcxOw6EHYp1yAM98/E+Vt5bh47MX425S/HfF5PSFjYVEh1Fo1nC4nHKIDTpcTTtEJh8sBpxjwvUv6vq8UgsIv4PYNursLvgf9P5qDLPBmeE39xfMhQKig3TcAl4JvBI2ADwrJfUfQ+9zn8AvYQ3ygIIpwOl1dQX03H0Z0fYiA0O0N9cGBz7qer44Qz+37YUKoDxH82x38wQlFh0JA1wj3kGF7YEgeHOh3H+77r6tQCLDa/QNqs60rbLY7+2+/qxRCV9mMkMFyL2sz+4xu1qoYNEedpRXY9wmw633gwP/8y6EmF7kD698CeZPD/x/HYZPmA/OE2Ye+9/9bRqEC8qZ2hdl5UwBlZP83bOrwhNnS6Ow9h41B67y/7HiMz0+O6OsOdAyviQYQl+jCxkMb8eLPL2JT7Sbv/RMzJuL8487HLwt+2fOIUKtR+rQwpRjIGNPnQMolunCg9YB3VPW2um3ekeEeaoUa49LHeSdYnJgxEXr1IK9FSoOCw+VAram2a8S2e+JIT8htd3U/oZZKUCE3ITd4xLahEHkJeVArYzRztM3U/Yjt9oM9P1ab2FVXOzDYTsiUTbBtd7pwsMWC8iYTKj3lSNwBd3WzGY4e/gnWqBQoStUH1dcuStMjN0nHYLuXLnzhW6zb24CTx2bjiXOnSHd+fhuwYZX0AeqfN+H1/W/j7i13I0Wbgo/O+KhXH3L2NWSUAiSXX9DdXfDtG3r3lbd8iSfcPkI5kyETeLsc3Zc1cTqCl0cp8O602VFeWc3wmmgAC1Xexj/cR4iQXwrSg0Ny9PA8wVcIeJ/H5/mDw3331QGex4QcuS/C4fR9nd6P+vdbt4dR/57XdjhdXVcK+KwrxxRHo1QEj1YOUT4j1KjnUOUz4rUqaQ4OkidzM7D3Y2mEdemXgO//a6nDgeN+CxxzKpAzIbL/u1hagPKvu8LslnL/5dpEaaCgJ8w+ylKsoTSbbD4js5tR2WTCjlt/PeSOZ4bXRAPU3ua9eGnXS/io/CPv6LG8hDyce8y5OH3k6YhXx0fstZwuJ/a17POWAfm+/nu0Wlv91tEqtRifMd5bBmR8xnjEqfjPHg0uTpcTdeY6/zIkPqO3rYETnfpQCArkxOf4j9Z2f80z5EGrjNHlXXZLiGDbPWq7rQbooW44NAkBwbZPuG3Ilk2w7XC6R2z71tdulILtql4E24WpehR762zHu+ts65GbrBu4E9VE2YYDjTjn2S1QKQR8vmIuStLjgdYq4NGp0oiYJW/AWHI8Fq1ZhBZrC26ccSOWjFnSq+eO5ghZURS9IXaPo7oDAvG+ClW7u6ea3kMq8PbcRGfowNt3WR8C706HiPKDjSjZfg/ixE5pTgF9asDXNEAXcJ/WIJu+j4gIcP9uCxxF382of7+SPmFeIeA76j+4pE/XY+PUyqCazL7lM3Qa5ZAL5oYcUQTsZmkAjq0DsHa4v3f/bHP/bG0HKjcC5V9Jv+89Msa4R1ifBmQeG73fyy0VXbWyy9Z3XfXukVQADJsnhdkl84D4tIg3wWxzQK+J0ZXAMcTwmmiAazA34PW9r+ONvW+gzdoGADCoDThr1FlYesxSZMdnh/2cDpcDu5t2e8uAbK/bDqPd/3IUnUqHiRkTvRMsjk0fC42Ss97S0OUSXag31wcF2p7vLYF15n0IEJAdn41CQyEKEgtQZCjyjt7ON+RDp9JF8Z34sHcCrZVAU2lwuN1W03MgpNa7w+wQo7YNORGdpbs/OZwuHGrt9E4Y2VVn24TqZnOPl69qlAoUpOq8pUiK07pGbQ+lYNvlErH4sW/w86F2XDi7GLefepy04L+XAj++JY1IueADrPr+YTz303MoTizGmtPWQK3o3ZUKA728g2/gHbKcSYj7XX0eXYywy5kohCFwSbPLGTyy2zf0dgYv67Q7UX6wASUbrkZcR/WRX8NDoQ4It1O6vve9X5/aFXwz8CYiIrkSRcDRGSJo9gmZQy4L+BoYUPc0gCaUrHFdNawzRof10AMtB/BZ5Wf4rOIzNFgaMCVrCmblzsLs3NkoNBT27e8klxOo3eEOs9cBVZv9R4RDAHLGd038WDATUA+8v2PlguE1kUxYHBZ8UPoBXtr1EiraKwBIlyj/uvjXuODYC3Bc+nHdPtbutOPnpp+9I6u312+H2WH2WydeHY9JmZOkkdXZU3Fs2rG9DhaIhjpRFNFoaQyaONITbJsCJ/4IkKnPDDliu8BQELtyPA6rNHI2VLDdWtVzsK2KA1JKpEvnAsPtxDzZBNtOl4hDrRbvSG3PxJHljSZUN1tgc3a/DdRKAQWp7hrb7okji9KkUdu5yXFQKeWxDXpjzfc1WPHmThi0Kqz/+3ykxmuAg9uAZ34JQAD+tB6HDBlY/M5i2Fw2PDL/EcwvnN/r5x/o4XVfuESX3whv34C7uwC8r4G3IAghw+1uQ/AhEnh3mk0oLy9DicGJOHuLVJfd3Cx9tbi/eu9rBsyN0j/vfREYePsG26FGfOtSGXgTEVH4RBFw2tyjlo2hRzN7RzX3JoR2/yz2/SqzI9IkAJr44K9a99e0EVJJkLThYT1taWspPq34FJ9VfIbSttJu18tLyPMG2dOzpyNJm9S392EzAZWbukqM1P/sv1wVBxTN7gqzM4+Tzf9EAwHDayKZcYkufF3zNV7c9SK+Pfyt9/7JmZNxwXEXYG7+XDhEB35o+MFbr3pn/U50Ov3/4TJoDJiSOcU7snp06ujYTURHNIiJoojmzuagQLu6vRqVxkoYbcGTcPhK16V7g+yixK4R24WGQiRoEqL0LgI4bEBbtRRm+4XbpUBLZc9/4Cq1Ul3+tOEBI7eHA0n5Ul1aGfAE25VN5qA621VN5iMH2yl6FKX519cuSY9HXrJOVsF2p92JXz6wDofaOvH3k0bjinkjpH+cXvgNULURmLAEOP1JXPfVdfi4/GNMy56G5379XFjB6GAMr/vCJbq6H9UdWOrE/f3RBt69KWfiuV8hyOe4Bfp4XNnM3QTbTd2E3k1HGXh3E2x7Q+80QO8e+R2X7D95pUIJCAoG4EREA5nT3nPZjHBHM9s6/MtrRJpa333IHCqEPtIylS6iAW5payk+q/gMn1Z86hdYqxQqHJ97PH5d/GsUJRbhu8PfYeOhjdhev91vgm+FoMDY9LGYnTsbs3JmYVzGuL4P6DPWSSOyPWF2x2H/5fEZUokRT5idmNu31xkiGF4Tydiupl14addL+KT8EzhEqdPN1GWi1doKm8vmt26KNsU7ueLUrKkYmTJSdv9oEg1GrZ2tQRNHekLuwNrzgVLjUr0TRnrKkRQmSkF3n0cNHC2n3SfYDpg8sqUi4HK6AAq1FGynDgsIt4dLNeQiPJt3f3G6RNS2ScG276jtikYTKpvNsDm6DxRVCmnEdlGaZ9S23ltnOy9FB/UAC7afWFeKf36yB7lJcfjimnmIUyuB3R8Cb5wjjTD56/f4yd6CJR9J9a3fOOUNHJt2bFivwfC67zyBd2/LmThFJ/r6575CUHQF2p5w+wjBdyz/DonacWUzBwTbgV99g++jDLxDEkIE2kopLOjxPvf9Ct+vAd+HfI6Ax/k+n0IR4j7PeoqA+5RS8B50nyK4nd0+R0/vNeA9hHxsL9vEDwqIhganA7D3dsTykUYzu0dFO21Hft2+UsV1EyR7QuSeQugQy9T6ATnIpKy1TBphXfkZDrQe8N6vUqgwO3c2FhYvxLyCeUjUBOd/ZrsZW+u2YtOhTdh4aCPK2sr8lser4zE9ezpm587G7NzZKDAU9O3KNFEEGvZ01cuu2CAdS77SR3dN/Fh8vHQFFnkxvCYaBOpMdXh1z6t4a99b3lGcaXFp3qB6atZUDEsexrCaSGbarG2oMdb4j9g2VqOyvRLNnc09PjZJm+QdsR1YjiRZmxybkgBOB9Be0xVm+4bbLeU9/wGvUAHJRQHBtvuWXAgo5VHmyOUScbi9syvQ9pk8sqLJBGsPwbZSISA/RecNtaU629Ko7YJUfdSD7WaTDXP/9SWMVgce/N0EnDklXxqV//hMaRT+nGsg/vJmXPjJhfi+/nssHrYY98y5J+zXGSrh9bx58zBx4kSsWrUqos+7bt06zJ8/Hy0tLUhOTu5xXVEUu0qa9FDOJPDnowq8exjNHVjmxFPSBAAuvPBCtLa24t133+3Taw/o48oTeHuD7ZbgEd2e7y3NgKlRmhiVYqw3HxR0E8Dzg4IQHwr09B74QQEdgcsVHB4facTykZZF9IPFAEptL4LkUKOZu1mmjpfNoI++KGstw6eVUkmQUIH1r4t+jfmF80MG1j05bDrsDbI3124OGkiUl5DnDbKn50wP+/m9HDag5tuuMPvQdv+SjAoVkD+ta1R27uRBvT97g+E10SBitpuxrW4b8g35KE4sHvT1KomGsg5bh7cUid8kku3VqLfU9/hYg9rgLT/iKUfiGbGdFpcWm77D5QTaD/oE26VAc3lXsN3TPwyCUgqwPWG2b7idXASo5DHZrMslos7YiXJPmN0oTRzpCbY77T0H23nJOu/Ekb51tgtS9NCoIh9s3/7+z1i9sQLH5iTiw7+eAIVCALY8Bfzf36VLIa/cjrWHv8XydcuhVWrx4ekf9mmS4QEdMvbgwgsvxH/+8x/86U9/wpNPPum37C9/+Qsef/xxXHDBBVi9ejUAoLm5GWq1GgZD30fahArAbTYbmpubkZWV1S/ntm/g7XA5sH79elx64aXYuX9njzW9+0ohKKBSqHD9X66Hsd2IF157occAXCkoQ75vuR5XIYmiNE+B6J64UnRKwY3oCrjP6b7PFeI+z2N81veu5wpxn8/6IZ8v8L5ItCng+/5qpyiGuK/vk6xSfxhkHxQo3Pf3+YOCbt6DXD4ocLkAuzkCo5l9ltvNR37dvlKouoLjbkczx0sjZ7sb6Rz4OJkMwoilsrYybw3rwMB6Vs4s7wjrSF196hJd2N282xtm91RiZHbubIxNH9v3EiOWFqD8q64wu6XCf7k2CSiZI5UZGf5L6X+cIZb19Ht4/e9//xv3338/Dh8+jAkTJuDRRx/F9OnTu11/1apVeOKJJ1BVVYX09HScddZZuPfee0P+UXnffffhhhtuwFVXXdXrUSoMr4mIaLAz282oNlb71dn2fH/YdLjHx+pVem+QXWgolOpsu0dvZ+gyYhRsuwDjIf8SJL7htsPS/WMFhVRyxHektifcTi6SzazfLpeIeqPVHWyb3HW2zd5w22LvPgxUCECed8R2vDfgLkqLR2Fq34LtikYTTly5Hg6XiJcvmYETRqYDllbgkUnSSNBTHoJ90nn47Xu/RZWxCn8c90dcOfnKPr13uYaMF154Ib744gu0t7ejtrYWOp0OgPR+cnJykJiYiPnz53vD60jor9Hb4VixYgU6Ozvx+OOPBy2z2WzQaDQQRdFvdHdvypkEBt43LbsJxnYjHnnxkSO2KdQklS6bC4erD+Ow9jAS9AlIiUtBsjYZKXEpMGgMvFqN/Imi+xZO8B4QrAcF+REI1AND+7CC/1BtCvXhQm/ft+e5Q30I0osPRnzbRANIP3xQ4OgMCKFNAPppnKSgADSGMELmeGn9nkY6y2RQhNyIoohOZyfMdjPMDjPMdjM67B347vB3+LTiU//AWlBhVu4s/Lr415hfMD8q5RI9JUY2HtqIjYc2oryt3G95gjrBv8RIYkHfX6y5vKtWdvl6oLPNf/mla4H8qX1/fhkKJ8sNe4z6G2+8gRUrVuDJJ5/EjBkzsGrVKixcuBB79+5FZmZm0Pqvvvoqrr/+ejz//POYPXs29u3bhwsvvBCCIGDlypV+63733Xd46qmnMH78+HCbRURENKjp1XqMTh2N0amjg5Z1Ojq9pUh8R2xXtVeh1lQLs8OMPc17sKd5T9BjdSod8g353gkjfcuRZOoz+y/sUSikyRyT8oGSX/gvE0XAWNt9sG03Aa2V0q3sy4AnFtzBdklwsJ1SDKh1/fN++kChEJCdFIfspDjMGp7mt0wU/YPtCu+obTMqm0ww25yobragutmCr/c3+j+vAOQm67yTRnYF3FIpEq1KGbI9//p0DxwuEXNHZUjBNQB8/aAUXGeMASadjzf3vYEqYxVS41JxybhLIrYtRFGEpacPLPqRTqUL6wOcyZMno7S0FGvWrME555wDAFizZg0KCwtRUlLit25g8FxcXIzLLrsMBw4cwFtvvYWUlBTcfPPNuOyyy0K+1oUXXoj169dj/fr1ePjhhwEA5eXlqKio8Csbsnr1aixfvhwvv/wyrr76alRXV+M3v/kNXnzxRbz11lu47bbb0NbWhvPOOw8PPfQQlErpGLBarbjpppvw2muvobW1FWPHjsU///lPzJs3z68d77//Ph577DHvexo7dixUKhVefvlljBs3Dl9++SUeeughvPDCCygrK0NqaioWL16Mf/3rX0hJSAEAbxvfeOMNLF++HNXV1TjhhBPwzHPPIDMrE07RiXh1POxKOzL10s/bvtuG8886H5f+9VJceuWl3vAbgPQ9nIBPJuayu9Bua8ejux5Fra3W7z0oBAWStclI0iYhRdsVaof6mqxNRnJcMgxqA694G8wEwT3KTcERktEQdvDeXcjeU/Af7pUGPYT2Rx38R/Lqhx4+GAm17Y5IdE8A6PDrQ/uHEGJUckCw3O1o5m6WqbRDboRqtNhddpjtZlgcFpjsJpjtZpgc7q92EywOS9B9nlDa8zXwvp4mmlYJKszMnYmFxQujFlj70qv1+EX+L/CLfOl/kdqOWmyq7Sox0mZtwxfVX+CL6i8AHGWJkdQS6Tb1YulcPbQDKPsCKF0HNOwGciZG/P0NJmGH1ytXrsQf//hHXHTRRQCAJ598Eh999BGef/55XH/99UHrb9y4EccffzyWLl0KQPqjfcmSJdiyZYvfeh0dHTjnnHPwzDPP4K677urLeyEiIhqS4lRxGJEyAiNSRgQtszltqOmo6Zo40mfE9qGOQ7A4LNjfsh/7W/YHPVaj0KDAUOAtR+I7Yjtbnw1lf03wIgjS7NyJuUDxCf7LRBHoqJfqL4cKt21GoK1KupWvD37uxDz/EduecDulBNDo++f99IEgCMhKjENWYhxmDgsOthuMVp9A231zj9o225yoabGgpsWCr/cHPi+Qm6RDcbreb9S23enCxz8ehkIAbvzNMdLKLRXAFndpjAV3os1hwhM7nwAA/GXiXxCvjo/Y+7U4LJjx6oyIPV84tizdAr06vH1/8cUX44UXXvCG188//zwuuugirFu37oiPffDBB3HnnXfixhtvxNtvv40///nPmDt3LkaPDv5g6uGHH8a+ffswduxY/OMf/wAAZGRkoKKiImhds9mMRx55BK+//jqMRiPOOOMMnH766UhOTsbHH3+MsrIynHnmmTj++ONx9tlnAwCWLVuGXbt24fXXX0dubi7eeecdnHTSSfjxxx8xcuRIAMDPP/+M+vp6/PKXv/S+1n/+8x/8+c9/xoYNG7z3KRQKPPLIIygpKUFZWRmuuOIK/P3vf/cbrW02m/HAAw/gpZdegkKhwLnnnovr/349XnnlFQCAWqmGWqlGhj4DX3zxBc454xz861//8gv3PSO8Q43q7uzsRIeqA1OypqDSUomWzha0WlvRYe+AS3ShubMZzZ3NKIf/yKruqASVFHaHCLdDfZ8SlwK9Ss/AmygUhQL8oCBK+r2cT4hAXR3nHzp7Amq1jkFzP3GJLnQ6OmF2mLuC5h7CZE/47Fk31H02V/9NNqlT6aBX6RGvjkdRYhEWFC3ALwt/GbsJ6UPIScjBGSPPwBkjz4DT5cSe5j3eUdk76nfgYMdBvLXvLby17y0oBAXGpY/zKzGiUvQyYlUogfwp0u0X1wJO+5Cvf30kYW0dm82Gbdu24YYbbvDep1AocOKJJ2LTpk0hHzN79my8/PLL+PbbbzF9+nSUlZXh448/xnnnnee33l/+8hcsWrQIJ5544hHDa6vVCqu1awKT9vb2cN4GERHRkKFRajAsaRiGJQ0LWmZ32nHIdMhv4khPuF1jrIHNZUNpWylK20qDHqtSqJCfkO8dqe2ts20oRE5CTu//eAuXIACGLOlWNNt/mShKk5yFDLbLAGu7VIO7/SBQ8XXwcxtygNThwaO2U0qkEUMDhCAIyEyMQ2ZiHKaXpPotE0URDR1WVDaZu0ZtN3ZNImmyOXGw1YKDrRZsONAU9Ny/m1KA0dkG6Ye1/5Am3Bw2Dxi5AM9uW4k2axuGJw3HGSPPiMI7HbjOPfdc3HDDDaisrAQAbNiwAa+//nqvwuvf/OY3uOKKKwAA1113HR566CF8+eWXIcPrpKQkaDQa6PV6ZGf3XFvcbrfjiSeewPDhwwEAZ511Fl566SXU1dUhISEBxx57LObPn48vv/wSZ599NqqqqvDCCy+gqqoKubm5AIBrrrkGn3zyCV544QXcc480Eed7772HhQsXQqPpuqR65MiR+Ne//uX3+suXL/d+X1xcjLvuuguXX365X3htt9vx5JNPetu4bNkybyjv65133sH555+PZ5991hu0ewiCAJWgCtnHdCo60RHXgdtm3+ZXjsbutKPV2uq9eUJt71drC1o7pa9t1ja0dLbA7DDDITrQ1NmEps7gc6U7aoVaGtkdl4wUbUrI8Nt3eXJcMnSqgXNFCBENAp4PCsIfq0j9yOa0hT1q2TdoDrzP4rBA7KfSLGqFGnq1HvGqeOjVeumm0nvDZ9/74tXxfvfrVLqQ98mtdJdSocRx6cfhuPTj8Mfxf4TJbsLWw1KJkU21m1DeVo6dDTuxs2Enntj5xNGVGOGHekcUVm/W2NgIp9OJrKwsv/uzsrKwZ0/wpcgAsHTpUjQ2NuKEE06AKIpwOBy4/PLLceONN3rXef311/H999/ju+++61U77r33Xtxxxx3hNJ2IiIgCqJVqFCUWoSixKGiZw+VAram2a8S2e+JIT8htd9lR0V6BivaKoMeqBBVyE3K9I7Z9y5HkJeRB3V9/oAkCkJAh3Qpn+i8TRcDc7A60SwOC7VKp7pyxVrpVfhP83AlZ7kA7INxOHQbEDZz5NgRBQKYhDpmGOEwrDg62GztsUn3txq5JIz2jtpN0aqz49Shp5ZqtwE//BSAAC+5ETcdBvLJbGh27YuqKiH84oVPpsGXpliOv2A/6EhxmZGRg0aJFWL16NURRxKJFi5Cent6rx/qWxxMEAdnZ2aiv73lC1t7Q6/XeUBiQ/j4vLi5GQkKC332e1/rxxx/hdDoxatQov+exWq1IS+sa7f/ee+9h2bJlfutMmTIl6PX/97//4d5778WePXvQ3t4Oh8OBzs5OmM1m6PX6kG3MyckJeu9btmzBhx9+iLfffhu//e1vw9wKoXlGc2foM3r9GKvTitbO1qBw2/vV2uq3vKWzBVanFXaXHfWW+iNOsusrThnnH3K7Q+1ETaK3trdCUEg3SF+VCiUECNIyRdf9npvfY3zuEwSh22WB93X3GKXgfm2FsqtNCv+2+baHI9GJSG6cLmfXiOQeRjJ7l/kGyz73eYJqs8PsNzFgJAkQQgfNIe7zhMo9hc96lb7//laXsXh1POYWzMXcgrkAjlxiJD8h3xtkT8uZFl6JEQrS7x/FrVu3Dvfccw8ef/xxzJgxAwcOHMBVV12FO++8E7fccguqq6tx1VVX4fPPP+/1hD033HADVqxY4f25vb0dBQVHUTidiIiI/KgUKqlkiKEAs+E/wtnpcqLOXOdfhsRn9LbVafUG3huwwe+xCkGBnPgcb6DtO2I7z5AHrVLbP29IEID4NOlWMC14ubnZXVM7MNguk2o+d9RJt6oQV5rFZ/gE28O6wu204UDcwLkUUhAEZBi0yDBoMTVEsO1ZB6IIfHqTtGDiUiBnPFatvwZ2lx0zc2ZiTt6cfmlbuKU7Yu3iiy/2hrr//ve/e/04tdr/H0JBEOBydV8P8miet6fX6ujogFKpxLZt27w1sD08gXdtbS22b9+ORYsW+S2Pj/cvGVNRUYFTTjkFf/7zn3H33XcjNTUV33zzDS655BLYbDZveB2qPYFzxw8fPhxpaWl4/vnnsWjRoqDHRItWqUVWfBay4rOOvLKbxWHxC7m9wbfPKO/AENzusqPT2YlOcyfqzHX9+I5ixxOyhwrOfe8PGZIf6THdhOjhvE53Af4R2xbitX3vD/xwoTfvp7ftDevDCfBDBBrcRFGE1WntU8kMv6DZ/bPFYenXeTi0Si3i1fFSGQ2fgNlzn2/A7Bc2BwTMnq/hzt1BkRFYYmR3825viZGd9TtR01GDN/e9iTf3vQmloMTY9LF9KzFCAMIMr9PT06FUKlFX5/+HVV1dXbeXMt5yyy0477zzcOmllwIAxo0bB5PJhMsuuww33XQTtm3bhvr6ekyePNn7GKfTia+++gqPPfYYrFZr0B/UWq0WWm0//XNLREREPVIqlMhNyEVuQi5m5viPcHaJLtSb64MCbc/3FocFBzsO4mDHQWyq9Q+CBQjIjs+WypB4Rm27R2znG/L799J6fap0yw8eUQpLS9dkkYHlSMyNgKlBulWHGDmsT/MZpT3cpxzJMECX0n/vJ0x+//Tsfh+o3gyodMAvb8aO+h34tOJTCBBwzdRr+A+S20knnQSbzQZBELBw4cJ+ex2NRgOnM/Izak2aNAlOpxP19fWYMyf0BxIffPABZs+ejdTU1JDLPbZt2waXy4UHH3wQCoV0WfCbb77Zp3alp6djzZo1mDdvHn7/+9/jzTffjFmAHS6dSgddgg45CTm9Wl8URZgdZr9Q27ekSbutHS7R5b05Rae35rfv/b7LXXDB5XLfBxecLidEuB/jku470nP25nU8y3tzyboIEQ7RAYiAHfaj3czUR75Bdm9G4/dlBH/Q8jCuFOjthxd9DvDdHy6Esw3CuSKhVx+CQODvUEhX9/UqTHYE1Ga2W4JKbljsFpgdZu9kvpGmEBR+o5d7Gt0cGD4HBc3u7xlaDj5KhRROj00fi8vGXwaT3YTvDn+HTYekkdkV7RV+JUYMagOm50glRmblzkKBgYNxjySss0aj0WDKlClYu3at9zI+l8uFtWvXBl1O6GE2m71/xHp4wmhRFPGrX/0KP/74o9/yiy66CGPGjMF1110XFFwTERHRwKUQFMiOz0Z2fDamZfuPcBZFEY2WxqCJIz3BtsluQq2pFrWmWmw5HBwEZ+ozQ47YLjAU9O+oXV0KkJcC5E0OXtbZ5hNsl3Z931QKmOoBc5N0qwlRGk2XEjrYTh0mBemx+AfXYQM+v036fvZfIRpy8MA3fwcAnDbiNIxODa7LPFQplUrs3r3b+31/KS4uxpYtW1BRUYGEhIQjBsm9NWrUKJxzzjk4//zz8eCDD2LSpEloaGjA2rVrMX78eCxatAjvv/8+Tj311CM+14gRI2C32/Hoo49i8eLF2LBhA5588sk+ty0zMxNffPEF5s+fjyVLluD111+HSjX4/tkXBAHx6njEq+ORl5AX6+b0iSiKXcG2Oyz3fO9yubwBd6gQvbuQvK+Bek8hu98yd4DvdDn9vve0r7ev4xfi9/J9+C0L8aGC33bq5fvpbWjnee/9VCKXesk3UO8u1I/0lQK+r3fEKwX6+Drekc+h6jcHjG6OxqSAvqOWdWqdX8Dc3Uhmv/DZva5GoeEHDhS2eHU85hXMw7yCeQCAQx2HvEH25trNaLe1Y23VWqytWgsAKDAU4KF5D/Hv7B6E/VfgihUrcMEFF2Dq1KmYPn06Vq1aBZPJhIsuuggAcP755yMvLw/33nsvAGDx4sVYuXIlJk2a5C0bcsstt2Dx4sVQKpUwGAwYO3as32vEx8cjLS0t6H4iIiKSL0EQvHVnp2T5j3AWRRHNnc1BgXZ1ezUqjZUw2oyoN9ej3lyPrXVbg547XZfuDbI9o7U9IbdBY+i/NxWXBOROlG6BrMaAYLus62djrTSi++A26RbqeX3DbN9wOz69/4Ltrc8BLeVAfCZw/FX4rPIz7GzYCZ1Kh79O+mv/vKaMJSb2f/3Ca665BhdccAGOPfZYWCwWlJeXR+y5X3jhBdx11124+uqrcfDgQaSnp2PmzJk45ZRTYDKZsHbtWqxateqIzzNhwgSsXLkS//znP3HDDTfgF7/4Be69916cf/75fW5bdnY2vvjiC8ybNw/nnHMOXn31VQ5qGYA84ZUS7n3DXRQT3YXovRk9H/RBQw+j9Hv9AUEfw36/Dxi6+/AjjDYd9YcjobZND++xN/zWPfqKUbKmUqj8RykfIWgOus93JLS7fIZSwU6IBp7chFycOepMnDnqTDhdTuxq2uWtl72zfidqO2pl+yF2tAhiYKG5Xnjsscdw//334/Dhw5g4cSIeeeQRzJgxAwAwb948FBcXY/Xq1QAAh8OBu+++Gy+99BIOHjyIjIwMLF68GHfffTeSk5NDPv+8efMwceLEXv2xDEg1r5OSktDW1haVfyKIiIgoulo7W4MmjvSE3K3W1h4fmxqXKoXaPuVIihKLUGAoQJI2RjWpbaaAUiQ+o7bbD/b8WG1i8KSRnnA7IbPvwbalBXhkkvR18cOwTVyK0949DTUdNfjzhD/jiolX9O15Q+js7ER5eTlKSkp6PecJRdeaNWtw8803Y9euXbFuSq/xuCKiWOq3KwjcAX53wb535H44HxD05UOFHtoUp4zrquEcEEh3NzkgJwUkAkx2E/Y07wka2DMUhJPl9im8HmgYXhMREQ1dbdY21Bhr/EdsG6tR2V6J5s7mHh+bpE0KOWK70FCIZG1ybC4VtZmBloqAYNs9arutBj1e861J6D7YNmT3HGx/ehOw6TEg81jg8m/wn90v44GtDyBdl46PTv8ooqVZGDIOfJ999hmsVisWL14c66b0Go8rIiIiInkIJ8sdfMXjiIiIaEhJ0iYhSZuE49KPC1rWYevwliLxm0SyvRr1lnq0Wdvwo/VH/Nj4Y9BjDWqDd6S2J9z2jNhOi0vrv2BboweyjpVugeyd3QTbZUBrNWDrAA7/KN0CqfVASok0WWRguO20At8+La234E602ox46oenAAB/nfTX/q0pTgPSr3/961g3gYiIiIiI4TURERENXgmaBByTdgyOSTsmaJnZbka1sdqvzrbn+8OmwzDajdjVtAu7moLLJuhVem9Nbd/R2oWJhcjQZfRfsK2OAzLHSLdADivQUukTbPuE261VgN0M1P8s3YIIAERg+C+BkSfiqW//CaPNiJEpI3Ha8NP6570QEREREREdAcNrIiIiGpL0aj1Gp44OObN3p6PTW4rEd8R2VXsVak21MDvM2NO8B3ua9wQ9VqfSId+QL4XZ7jrbRYYiFCYWIlOfCYWg6J83pNICGaOkWyCHDWirBppKg4PtlkpAdAJKLbDgTlS1V+H1va8DAK6Zcg0nPyIiIiIiophheE1EREQUIE4VhxEpIzAiZUTQMpvThpqOmq6JI9u7amwfMh2CxWHB/pb92N+yP+ixGoUGBYYCbzkS76jtxEJk67P7LyhWaYC04dItkNMuBdtKLZCUh4e+/BscLgeOzzses/Nm90973AbB1Cs0gPB4IiIiIhp8GF4TERERhUGj1GBY0jAMSxoWtMzutOOQ6ZDfxJGecLvGWAOby4bStlKUtpUGPValUCE/Id9bgsRbZ9tQhJyEHKgU/fRnm1It1b4G8H3d9/hf1f+gEBS4esrV/fN6ANRqNQDAbDZDp9P12+vQ0GI2mwF0HV9EREREJH8Mr4mIiIgiRK1UoyixCEWJRUHLHC4Hak21XSO23RNHekJuu8uOivYKVLRXBD1WJaiQm5AbPGLbUIi8hDyolUcf1rlEFx7Y+gAA4PQRp2Nkysijfs7uKJVKJCcno76+HgCg1+v7r044DXqiKMJsNqO+vh7JyclQKlnqhoiIiGiwYHhNREREFAUqhUoqGWIowGz4l+NwupyoM9f5TxzpM3rb6rR6A+8N2OD3WIWgQE58jjfQ9kwiWZRYhDxDHrRKba/a90n5J/ix8UfoVXosm7QsYu+7O9nZ2QDgDbCJjlZycrL3uCIiIiKiwUEQB0FxuPb2diQlJaGtrQ2JiYmxbg4RERFRxLhEF+rN9UGBtud7i8PS7WMFCMiOz/ZOHOk7ajvfkA+dSirZYXVaceo7p+KQ6RCWTVyGP034U7TeHpxOJ+x2e9RejwYntVrNEddEREREMhFOlsvwmoiIiEimRFFEo6XRf8S2+/sqYxVMdlOPj8/UZ6LQUAiFoMC3h79Fpj4TH57+oTfUJiIiIiIiirRwslyWDSEiIiKSKUEQkKHPQIY+A1OypvgtE0URzZ3NQYF2dXs1Ko2VMNqMqDfXo97cVbbjyklXMrgmIiIiIqIBg+E1ERER0SAkCALSdGlI06VhYuZEv2WiKKLN2uY3cWSCJgGLhy+OTWOJiIiIiIhCYHhNRERENMQIgoDkuGQkxyVjfMb4WDeHiIiIiIgopEERXnvKdre3t8e4JURERERERERERETUHU+G25upGAdFeG00GgEABQUFMW4JERERERERERERER2J0WhEUlJSj+sIYm8i7gHO5XLh0KFDMBgMEAQh1s2Juvb2dhQUFKC6uvqIM3RS5EVj+3MfRxe3NxENNZHq99h/EkUGzyUikgv2VxSOSB8vcj/+5N7+oyGKIoxGI3Jzc6FQKHpcd1CMvFYoFMjPz491M2IuMTFxyB3sA0k0tj/3cXRxexPRUBOpfo/9J1Fk8FwiIrlgf0XhiPTxIvfjT+7t76sjjbj26DnaJiIiIiIiIiIiIiKKAYbXRERERERERERERDTgMLweBLRaLW677TZotdpYN2VIisb25z6OLm5vIhpqItXvsf8kigyeS0QkF+yvKByRPl7kfvzJvf3RMigmbCQiIiIiIiIiIiKiwYUjr4mIiIiIiIiIiIhowGF4TUREREREREREREQDDsNrIiIiIiIiIiIiIhpwGF4TERERERERERER0YDD8JqIiIiIiIiIiIiIBhyG1wNUQ0MDWlpaYt2MISsa25/7OLq4vYloqIlUv8f+kyhyeD4RkRywr6JwRPp4kfvxJ/f2D0SqWDeA/BmNRjz++OP48ssvkZ+fj5tvvhnFxcX47LPPsHv3bsybNw/jx4+HIAixbuqgFI3tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/sHMkEURTHWjaAud9xxB9555x1MmDABmzdvxqmnnorDhw9j+/btiIuLg0ajwUMPPYQZM2bEuqmDUjS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0f0EQaUAoLC8VXX31VFEVRtNlsYm5urnjGGWeI//vf/8Rt27aJEyZMEC+++GLRbDbHuKWDUzS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0fyBheDyCdnZ2iwWAQDx48KDocDlEURVGr1YpfffWVd53169eLY8eOFZuammLVzEErGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39g90nLBxANm/fz8yMzNhNpuhVCpRUVEBnU6HsWPHwuVyAQASExNx+PBhpKamxri1g080tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tH+g4YeMAotFocN5558FqtQIAiouLUV5ejuTkZO8633//PTIyMmLUwsEtGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39g90DK8HkOHDh+OKK66ATqcDAIiiiOTkZIiiCEEQUF5eji+//BInn3xyjFs6OEVj+3MfRxe3NxENNZHq99h/EkUOzycikgP2VRSOSB8vcj/+5N7+gY7h9QCiVCr9PoURBMHvq9FoxPjx47Fw4cKYtG+wi8b25z6OLm5vIhpqItXvsf8kihyeT0QkB+yrKByRPl7kfvzJvf0DnSCKohjrRpDE5XJBFEUoFArvAU7RE43tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/sHOk7YOIAoFAoolcqgA10URYiiiNbWVuzZsydGrRv8orH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb/9Ax7IhA0RVVRX++9//YtSoUcjLy0N2djbS09OhUqm8B//mzZtx7rnnorGxMcatHXyisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3JvvxwwvB4g1qxZg2uvvRbTp0+HRqNBWloasrKykJubi4KCAhQXF+ODDz5AcXFxrJs6KEVj+3MfRxe3NxENNZHq99h/EkUOzycikgP2VRSOSB8vcj/+5N5+OWB4PUDs3r0bv/rVr3D++eejuroae/fuxYEDB7B161Y4HA7o9Xps2bIFf/3rX2Pd1EEpGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssBw+sBQqvVIi0tDUuWLIFC0VWKvK6uDuXl5di1axe2bt2KadOmxbCVg1c0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tlwNBFEUx1o0g4MCBAygtLcX8+fOhVqu9s5R62Gw2xMXFYfv27ZgwYUIMWzo4RWP7cx9HF7c3EQ01ker32H8SRQ7PJyKSA/ZVFI5IHy9yP/7k3n45YHgtA6Ioorq6Gpdeeik+++yzWDdnyInG9uc+ji5ubyIaaiLV77H/JIocnk9EJAfsqygckT5e5H78yb39AwXD6wHK5XL5fVJD0RWN7c99HF3c3kQ01ESq32P/SRQ5PJ+ISA7YV1E4In28yP34k3v7ByJuzQHG5XIBAN555x2sWrXK737PMuo/0dj+3MfRxe1NRENNpPo99p9EkcPziYjkgH0VhSPSx4vcjz+5t38gY3g9wHgO6FdffRXV1dXe+xUKBT+5iYJobH/u4+ji9iaioSZS/R77T6LI4flERHLAvorCEenjRe7Hn9zbP5Bx6w0wgiAAAH788UdMmjQpxq0ZeqKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm//QMbweoBwuVwQRRFKpRKANBvpMcccE+NWDR3R2P7cx9HF7U1EQ02k+j32n0SRw/OJiOSAfRWFI9LHi9yPP7m3Xw44YWOMtbe34/Dhwxg1ahQAFnaPtmhsf+7j6OL2JqKhJlL9HvtPosjh+UREcsC+isIR6eNF7sef3NsvJ9yqMfb2229jzJgxWL58OWpqanigR1k0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tlxNu2RjbsGEDCgsL8fXXX+O0007DSy+9hNbWVgAAB8X3v2hsf+7j6OL2JqKhJlL9HvtPosjh+UREcsC+isIR6eNF7sef3NsvJwyvY+zrr7/GjTfeiNdeew2jRo3CrbfeimuvvRbV1dXeYu+eGUsp8qKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm+/rIgUU3FxceIPP/wgiqIoms1m8amnnhKHDRsmxsfHizfeeKPY3t4e4xYObtHY/tzH0cXtTURDTaT6PfafRJHD84mI5IB9FYUj0seL3I8/ubdfThhex1BdXZ2oUChEo9EoOp1O7/0Oh0O86667xJycHHHSpEniCy+8ELtGDmLR2P7cx9HF7U1EQ02k+j32n0SRw/OJiOSAfRWFI9LHi9yPP7m3X25YNiSGfv75Z+Tn5yMhIQGCIEAURTidTiiVStx00034/PPPMWrUKDz99NOxbuqgFI3tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/vlRhBFVhGPlaamJtTW1mLs2LHeOjiCIHhr4wBAXV0dfvjhByxYsCBWzRy0orH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb7/cMLwmIiIiIiIiIiIiogFHFesGEHDgwAHs27cParUaBQUFKCwshF6vBwA4HA6oVNxN/Ska25/7OLq4vYloqIlUv8f+kyhyeD4RkRywr6JwRPp4kfvxJ/f2ywVHXsfQjh07cMEFF6CsrAzDhw+HVquFRqPBsGHDsHjxYpx11lmxbuKgFo3tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/vlhuF1jBw8eBALFizAxIkTsXz5cjQ3N6O+vh6lpaX49ttv8cUXX+DUU0/FK6+8Ao1GE+vmDjrR2P7cx9HF7U1EQ02k+j32n0SRw/OJiOSAfRWFI9LHi9yPP7m3X5ZEiolXX31VHD16tNjR0RFy+bp168SRI0eKr7zySpRbNjREY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTefjlSxDo8H6rq6uqQk5PjnYnU4XDA4XDA6XQCAObOnYs5c+bg/fffj2UzB61obH/u4+ji9iaioSZS/R77T6LI4flERHLAvorCEenjRe7Hn9zbL0cMr2Nk3rx5qK6uxvLly1FfXw+VSgWVSgWlUgkAsNvtqK2tRVFRUYxbOjhFY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTefjlizesYevvtt3H11VfD6XRi5syZmDZtGkaMGAGr1YqHH34YTqcTTz/9NCZPnhzrpg5K0dj+3MfRxe1NRENNpPo99p9EkcPziYjkgH0VhSPSx4vcjz+5t19uGF7HiCiKEAQBhw8fxttvv43PP/8ce/fuxeHDh6FWq3HyySfj6quvxoQJE2Ld1EEpGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssRw+sBwmazoaOjAwaDAWq1GhaLBTqdLtbNGjKisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jvvxyw5nWMeQq6//zzz3jzzTehUqkAADqdDvxcof9FY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTefjlheB1jngN69erV+O9//+udrRSA3/fUP6Kx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm+/nDC8HiB27NiB2bNnx7oZQ1Y0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tlwOG1zGmUEi7ICMjA8cff3yMWzP0RGP7cx9HF7c3EQ01ker32H8SRQ7PJyKSA/ZVFI5IHy9yP/7k3n454YSNMWK326FWqwEALpcLNTU1KCwsjHGrho5obH/u4+ji9iaioSZS/R77T6LI4flERHLAvorCEenjRe7Hn9zbL0cceR0D27ZtwzXXXIMrrrgCu3btgkKhgEqlwtatW2E2mwGAxd37UTS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u2XK468jrIvvvgC1113HaxWK7RaLUaNGoWTTz4Zjz/+OPbv34/c3Fw88MADWLBgAURRZJH3CIvG9uc+ji5ubyIaaiLV77H/JIocnk9EJAfsqygckT5e5H78yb39csbwOsrOPvtspKSkYOXKlXA6nTjjjDNQX1+PRYsWYdGiRbj55puhVqvxxhtvICUlJdbNHXSisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jvv5yxbEiUbdu2Db/+9a+h1+thMBhQW1uLM844A/fccw+OP/54PPjggzAajfj5559j3dRBKRrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/bLGcPrKLNarUhNTfX+XFZWhrlz5wKQ6uKMGjUKVVVVfutQ5ERj+3MfRxe3NxENNZHq99h/EkUOzycikgP2VRSOSB8vcj/+5N5+OVPFugFDSXt7O0455RRYrVYA0sH96aefYsqUKQAAQRBw8OBBtLW14ZhjjollUwelaGx/7uPo4vYmoqEmUv0e+0+iyOH5RERywL6KwhHp40Xux5/c2y93rHkdRS6XCwcPHoRSqURubm7QcqvViqeeegovvfQSvvvuuxi0cHCLxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/XLHkddRpFAoUFBQ4P1ZFEWIogiFQqreUltbi6amJvztb3+LVRMHtWhsf+7j6OL2JqKhJlL9HvtPosjh+UREcsC+isIR6eNF7sef3Nsvdxx5HSWiKEIQBHR2dsJkMiEtLS3kek6nEw6HA1qtNsotHNyisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jv/2DACRujxFP/5tprr8Vpp52GRYsW4ZtvvglaT6lU8kDvB9HY/tzH0cXtTURDTaT6PfafRJHD84mI5IB9FYUj0seL3I8/ubd/MGB4HSVOpxN/+9vf8Nlnn+HEE09EcnIyTj31VHz++ecApE9y7HY7pk6disbGxhi3dvCJxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YOCSFGxbds2MTc3V6yurhZFURQ7OzvF2267TTzuuOPEqqoqURRFcc+ePaJCoYhlMwetaGx/7uPo4vYmoqEmUv0e+0+iyOH5RERywL6KwhHp40Xux5/c2z8YcOR1lHz//fcoKipCfn4+XC4XtFotVqxYgeHDh+O6664DAGzfvt2vADxFTjS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0fDBheR0lLSwtSU1NRV1cHhUIBURSRmJiI22+/HVu3bsVrr72G3bt349hjj411UwelaGx/7uPo4vYmoqEmUv0e+0+iyOH5RERywL6KwhHp40Xux5/c2z8YMLyOkmnTpiE7Oxvt7e3e+0RRxKRJk/CXv/wFq1atwqOPPooFCxbEsJWDVzS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0fDFSxbsBQMXv2bOh0OhQVFXnvE0URLpcLV111FTZt2oTvvvsOxx9/fAxbOXhFY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTe/sFAEEVRjHUjhhpRFCEIgt99zc3NePTRR3HdddchLi4uRi0bGqKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm+/XDG8jpLm5mZ8+umnOHToECwWC/R6PYqKijBx4kQMHz481s0b9KKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm//YMCyIVHw888/484778Rnn32G5ORkJCQkQBAEOBwOJCUlYcmSJbj00kuh0+li3dRBKRrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/YPFgyvo+CBBx5AQ0MDPvnkE0yfPh0AUF1djV27duH//u//cMstt6CxsRF33HFHyEsQ6OhEY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTe/kFDpH43bNgw8e233xZFURSdTmfQ8qeeekocNWqUuG3btmg3bUiIxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YOFItbh+VAwatQofPjhhzAajVAopE1ut9thtVoBABdffDEaGhqg1Wpj2cxBKxrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/YPFgyvo+Daa6/F2rVrcfvtt2PPnj0AALVaDa1Wi5aWFjz55JPQaDQ49thjY9zSwSka25/7OLq4vYloqIlUv8f+kyhyeD4RkRywr6JwRPp4kfvxJ/f2DxaCKIpirBsx2ImiiOeeew733nsvKisrkZCQgIKCAuTk5KCzsxNVVVVYsWIFrrzyylg3dVCKxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YMFw+soW79+PX766SeUlZWhtbUV8fHx+N3vfoc5c+bEumlDQjS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u2XM4bXMeByueB0OqFWq2PdlCEpGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssVw+t+5nA4sHLlSuzduxfjx4/HBRdcgOTkZADS5QeiKMJut7O4ez+JxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YMJJ2zsRy6XC9dccw1WrVqF6upq3HfffZg/fz6am5u964iiiAceeAAmkymGLR2corH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb/+gI1K/+eGHH8Ti4mLxs88+E0VRFOvr68W5c+eKJ554onedXbt2iYIgxKqJg1o0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tH2w48rofbdy4EdnZ2ViwYAHsdjsyMjLwn//8B3V1dbjtttsAAD/88AOKiopi3NLBKRrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/YPNgyv+9GhQ4cwfPhwWK1WqNVq2O12FBUV4dZbb8WLL76Ibdu24eeff8YxxxwT66YOStHY/tzH0cXtTURDTaT6PfafRJHD84mI5IB9FYUj0seL3I8/ubd/sGF43Y+GDRuG/fv3o7y8HACgVqshiiLOOussLFy4EHfddRfeeecdzJkzJ8YtHZyisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jv/2AjiKIoxroRg5UoimhqakJSUhLUarXfsqamJvz2t7/Fhg0bsHnzZkyfPj1GrRy8orH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb/9gw/A6hvbv349HHnkE9957LxISEmLdnCEnGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssNw2siIiIiIiIiIiIiGnBY85qIiIiIiIiIiIiIBhyG10REREREREREREQ04DC8JiIiIiIiIiIiIqIBh+E1EREREREREREREQ04DK+JiIiIiADMmzcPy5cvj3UzvCoqKiAIAnbs2BHW41avXo3k5OR+aRMRERERUTQxvCYiIiIiGuKKi4uxatWqWDeDiIiIiMgPw2siIiIiIiIiIiIiGnAYXhMRERERBXjppZcwdepUGAwGZGdnY+nSpaivr/cuD1Wa491334UgCN6fb7/9dkycOBEvvfQSiouLkZSUhD/84Q8wGo3edVwuF/71r39hxIgR0Gq1KCwsxN133+33vGVlZZg/fz70ej0mTJiATZs2+S1fvXo1CgsLodfrcfrpp6OpqclveWlpKU477TRkZWUhISEB06ZNw//+9z/v8nnz5qGyshJ/+9vfIAiC33v45ptvMGfOHOh0OhQUFODKK6+EyWQKf4MSEREREfUBw2siIiIiogB2ux133nkndu7ciXfffRcVFRW48MILw36e0tJSvPvuu/jwww/x4YcfYv369bjvvvu8y2+44Qbcd999uOWWW7Br1y68+uqryMrK8nuOm266Cddccw127NiBUaNGYcmSJXA4HACALVu24JJLLsGyZcuwY8cOzJ8/H3fddZff4zs6OvCb3/wGa9euxfbt23HSSSdh8eLFqKqqAgCsWbMG+fn5+Mc//oHa2lrU1tZ6237SSSfhzDPPxA8//IA33ngD33zzDZYtWxb2diAiIiIi6gtBFEUx1o0gIiIiIoq1efPmYeLEiSFrP2/duhXTpk2D0WhEQkICVq9ejeXLl6O1tdW7zrvvvovTTz8dnj+vb7/9dtx///04fPgwDAYDAODvf/87vvrqK2zevBlGoxEZGRl47LHHcOmllwa9ZkVFBUpKSvDss8/ikksuAQDs2rULxx13HHbv3o0xY8Zg6dKlaGtrw0cffeR93B/+8Ad88sknfm0LNHbsWFx++eXeILq4uBjLly/3m7Dy0ksvhVKpxFNPPeW975tvvsHcuXNhMpkQFxd3xG1KRERERHQ0OPKaiIiIiCjAtm3bsHjxYhQWFsJgMGDu3LkA4B2t3FvFxcXe4BoAcnJyvOVHdu/eDavVil/96lc9Psf48eP9Hg/A7zlmzJjht/6sWbP8fu7o6MA111yDY445BsnJyUhISMDu3buP+F527tyJ1atXIyEhwXtbuHAhXC4XysvLj/DOiYiIiIiOnirWDSAiIiIiGkhMJhMWLlyIhQsX4pVXXkFGRgaqqqqwcOFC2Gw2AIBCoUDgBYx2uz3oudRqtd/PgiDA5XIBAHQ6Xa/a4/scnnrUnufojWuuuQaff/45HnjgAYwYMQI6nQ5nnXWW9710p6OjA3/6059w5ZVXBi0rLCzs9esTEREREfUVw2siIiIiIh979uxBU1MT7rvvPhQUFACQyob4ysjIgNFohMlkQnx8PABgx44dYb3OyJEjodPpsHbt2pBlQ3rjmGOOwZYtW/zu27x5s9/PGzZswIUXXojTTz8dgBRKV1RU+K2j0WjgdDr97ps8eTJ27dqFESNG9KltRERERERHi2VDiIiIiIh8FBYWQqPR4NFHH0VZWRnef/993HnnnX7rzJgxA3q9HjfeeCNKS0vx6quvYvXq1WG9TlxcHK677jr8/e9/x4svvojS0lJs3rwZzz33XK+f48orr8Qnn3yCBx54APv378djjz2GTz75xG+dkSNHYs2aNdixYwd27tyJpUuXBo3cLi4uxldffYWDBw+isbERAHDddddh48aN3skg9+/fj/fee48TNhIRERFR1DC8JiIiIiLykZGRgdWrV+Ott97Csccei/vuuw8PPPCA3zqpqal4+eWX8fHHH2PcuHF47bXXcPvtt4f9Wrfccguuvvpq3HrrrTjmmGNw9tlne+tZ98bMmTPxzDPP4OGHH8aECRPw2Wef4eabb/ZbZ+XKlUhJScHs2bOxePFiLFy4EJMnT/Zb5x//+AcqKiowfPhwZGRkAJBqba9fvx779u3DnDlzMGnSJNx6663Izc0N+30SEREREfWFIAYW6yMiIiIiIiIiIiIiijGOvCYiIiIiIiIiIiKiAYfhNRERERERERERERENOAyviYiIiIiIiIiIiGjAYXhNRERERERERERERAMOw2siIiIiIiIiIiIiGnAYXhMRERERERERERHRgMPwmoiIiIiIiIiIiIgGHIbXRERERERERERERDTgMLwmIiIiIiIiIiIiogGH4TURERERERERERERDTgMr4mIiIiIiIiIiIhowGF4TUREREREREREREQDzv8DhjTjP+JMXz8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_merge = th_bench.dataframe.merge(th_bench.metadata, on='profile')\n", + "df_merge['launchdate'] = pd.to_datetime(df_merge['launchdate'], unit='s')\n", + "\n", + "# display only the top-most node\n", + "df_main = df_merge[df_merge['name'] == 'main']\n", + "df_main.set_index('launchdate')\n", + "\n", + "# adjust y-axis to be 2 degrees of precision\n", + "df_main.plot(x='launchdate', y=['Max time/rank', 'Avg time/rank', 'Min time/rank'], kind='line', use_index=True, xticks=df_main['launchdate'], rot=80, x_compat=True, figsize=(18,1))\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 320e3c4ad3e9d94da51d361ec37520c39c07ab4f Mon Sep 17 00:00:00 2001 From: Yu Pan <70486009+yu-nix@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:29:41 -0700 Subject: [PATCH 144/187] Update doc/sundials_developers/source/testing/Spot.rst Co-authored-by: Cody Balos --- doc/sundials_developers/source/testing/Spot.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst index c68ffe5e01..3ce200093d 100644 --- a/doc/sundials_developers/source/testing/Spot.rst +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -31,7 +31,7 @@ Setting Up Your Own SPOT Visualizations --------------------------------------- To display data from non-GitLab jobs or create a local collection of runs, -input into the SPOT search bar the absolute path to the directory containing the `,cali` files and refresh the page. +input into the SPOT search bar the absolute path to the directory containing the `.cali` files and refresh the page. By default `.cali`` files will be generated in the build directory under ``Benchmarking/output`` and ``Testing/output``. To specify where `.cali` output files are placed, define the CMake option SUNDIALS_TEST_OUTPUT_DIR with the From 9f4fcd620b32fa8c00691298f7e6b489f163f45f Mon Sep 17 00:00:00 2001 From: Yu Pan <70486009+yu-nix@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:29:56 -0700 Subject: [PATCH 145/187] Update .gitlab-ci.yml Co-authored-by: Cody Balos --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b7be848cf..71d43768c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ variables: NCPUS: 12 VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" - ON_LASSEN: "OFF" + ON_LASSEN: "ON" ON_QUARTZ: "OFF" ON_CORONA: "ON" SPACK_PREFIX: "v0.19.1" From 18d73af84532fe6c52051b2cbdf09a7b3b8d81eb Mon Sep 17 00:00:00 2001 From: Yu Pan <70486009+yu-nix@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:30:06 -0700 Subject: [PATCH 146/187] Update .gitlab-ci.yml Co-authored-by: Cody Balos --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71d43768c8..63622ebb08 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,7 +78,7 @@ variables: VERBOSE_BUILD: "OFF" VERBOSE_TEST: "OFF" ON_LASSEN: "ON" - ON_QUARTZ: "OFF" + ON_QUARTZ: "ON" ON_CORONA: "ON" SPACK_PREFIX: "v0.19.1" SHARED_SPACK: "UPSTREAM" From fe4abdb8095279414734f00613449c7abe56f3c7 Mon Sep 17 00:00:00 2001 From: Yu Pan <70486009+yu-nix@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:30:46 -0700 Subject: [PATCH 147/187] Update .gitlab/spack_packages/sundials/package.py Co-authored-by: Cody Balos --- .gitlab/spack_packages/sundials/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index eb4e7df1e8..25ec6f5d81 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -26,7 +26,6 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # ========================================================================== # Versions # ========================================================================== - version("develop", branch="feature/persistent-benchmarking-test-branch") # version("develop", branch="develop") version("6.5.1", sha256="4252303805171e4dbdd19a01e52c1dcfe0dafc599c3cfedb0a5c2ffb045a8a75") version("6.5.0", sha256="4e0b998dff292a2617e179609b539b511eb80836f5faacf800e688a886288502") From b8004016658b5cca8c87edc7e8c76ff21a06da4d Mon Sep 17 00:00:00 2001 From: Yu Pan <70486009+yu-nix@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:36:16 -0700 Subject: [PATCH 148/187] Update include/sundials/sundials_context.h Co-authored-by: Cody Balos --- include/sundials/sundials_context.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/sundials/sundials_context.h b/include/sundials/sundials_context.h index caff5f03ce..6fef61b81d 100644 --- a/include/sundials/sundials_context.h +++ b/include/sundials/sundials_context.h @@ -36,7 +36,6 @@ SUNDIALS_EXPORT int SUNContext_GetLogger(SUNContext sunctx, SUNLogger* logger); SUNDIALS_EXPORT int SUNContext_SetLogger(SUNContext sunctx, SUNLogger logger); SUNDIALS_EXPORT int SUNContext_Free(SUNContext* ctx); -static void sun_adiak_helper(); #ifdef __cplusplus } From 02c1ca32269a0afe6d1b8e3d6ccb39c679dfaa5f Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Jul 2023 10:58:02 -0700 Subject: [PATCH 149/187] update SPOT doc --- doc/sundials_developers/source/testing/Spot.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst index 3ce200093d..2d5ff0a5b0 100644 --- a/doc/sundials_developers/source/testing/Spot.rst +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -34,7 +34,7 @@ To display data from non-GitLab jobs or create a local collection of runs, input into the SPOT search bar the absolute path to the directory containing the `.cali` files and refresh the page. By default `.cali`` files will be generated in the build directory under ``Benchmarking/output`` and ``Testing/output``. To specify where `.cali` output -files are placed, define the CMake option SUNDIALS_TEST_OUTPUT_DIR with the +files are placed, define the CMake option SUNDIALS_CALIPER_OUTPUT_DIR with the desired directory path. To retain the same filters as a given SPOT visualization link in From 64fcadecfbf7c438e0c4486267c25cc3af62f7c5 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Jul 2023 11:17:21 -0700 Subject: [PATCH 150/187] adjust benchmark script --- .gitlab/build_and_bench.sh | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh index 552f253051..19c3649c09 100644 --- a/.gitlab/build_and_bench.sh +++ b/.gitlab/build_and_bench.sh @@ -13,7 +13,7 @@ job_timestamp=${CI_JOB_STARTED_AT:-""} # remove tailing number from hostname hostname=${hostname%%[0-9]*} -benchmark_store_dir="/usr/workspace/sundials/benchmarks/${hostname}/${job_timestamp}_${job_unique_id}" +benchmark_store_dir="/usr/workspace/sundials/caliper/benchmarks/${hostname}/${job_timestamp}_${job_unique_id}" nresg=$((BENCHMARK_NNODES * 4)) # Lassen has 4 GPUs per node nresc=$((BENCHMARK_NNODES * 40)) # Lassen has 40 cores per node @@ -34,15 +34,6 @@ ls # because we are interested in individual region timings. export CUDA_LAUNCH_BLOCKING=1 -if [[ -d ${build_dir} ]] -then - module load cmake/3.23 # --test-dir flag requires cmake 3.20 or higher - date - export CALI_CONFIG="spot(output=${benchmark_store_dir}/test_suite.cali),runtime-report(calc.inclusive)" - ctest --output-on-failure --verbose --test-dir ${build_dir} -T test 2>&1 | tee tests_output.txt - date -fi - if [[ -d ${ar3d_dir} ]] then date From f10f04580d8d7c3ebad7a3d7fadf7e695677ee4e Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 21 Jul 2023 13:58:58 -0700 Subject: [PATCH 151/187] remove change to build_and_test --- .gitlab/build_and_test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 365c95f140..a3dc1ba25d 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -191,9 +191,8 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ - -DSUNDIALS_TEST_OUTPUT_DIR="/usr/workspace/sundials/caliper/" \ "${project_dir}" - + # build VERBOSE_BUILD=${VERBOSE_BUILD:-"OFF"} if [[ "${VERBOSE_BUILD}" == "ON" ]]; then From b12239d9b1a865b380e2fdbace0218d8dd2c3213 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 24 Jul 2023 15:00:31 -0700 Subject: [PATCH 152/187] place example cali files in Example folder --- cmake/SundialsSetupTesting.cmake | 4 ++-- cmake/macros/SundialsAddTest.cmake | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 4638812cdf..266533ad88 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -65,8 +65,8 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default caliper output directory was provided make sure it exists if(SUNDIALS_CALIPER_OUTPUT_DIR) message(STATUS "Using non-default caliper output directory: ${SUNDIALS_CALIPER_OUTPUT_DIR}") - if(NOT EXISTS ${SUNDIALS_CALIPER_OUTPUT_DIR}/Testing/${JOB_ID}) - file(MAKE_DIRECTORY ${SUNDIALS_CALIPER_OUTPUT_DIR}/Testing/${JOB_ID}) + if(NOT EXISTS ${SUNDIALS_CALIPER_OUTPUT_DIR}/Example/${JOB_ID}) + file(MAKE_DIRECTORY ${SUNDIALS_CALIPER_OUTPUT_DIR}/Example/${JOB_ID}) endif() endif() diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index aa70c94637..77154f7a4b 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -99,9 +99,9 @@ macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) if(SUNDIALS_TEST_PROFILE) list(APPEND TEST_ARGS "--profile") if (SUNDIALS_CALIPER_OUTPUT_DIR) - list(APPEND TEST_ARGS "--calidir=${SUNDIALS_CALIPER_OUTPUT_DIR}/Testing/${JOB_ID}") + list(APPEND TEST_ARGS "--calidir=${SUNDIALS_CALIPER_OUTPUT_DIR}/Example/${JOB_ID}") else() - list(APPEND TEST_ARGS "--calidir=${TEST_OUTPUT_DIR}/Caliper/Testing") + list(APPEND TEST_ARGS "--calidir=${TEST_OUTPUT_DIR}/Caliper/Example") endif() endif() From ae87999d8b8ba96d8d96b6d1c5c6c39208c86409 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 24 Jul 2023 15:45:43 -0700 Subject: [PATCH 153/187] add SPOT visualization links --- doc/sundials_developers/source/testing/CI.rst | 8 +++-- .../source/testing/Spot.rst | 31 ++++++++++++++----- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/doc/sundials_developers/source/testing/CI.rst b/doc/sundials_developers/source/testing/CI.rst index 297fe5507d..2e5b6e276b 100644 --- a/doc/sundials_developers/source/testing/CI.rst +++ b/doc/sundials_developers/source/testing/CI.rst @@ -270,9 +270,11 @@ Updating Spack To update the spack commit used for the CI: -1. The first thing to do is update the spack commit in the ``.uberenv_config.json`` file. -2. Then, a pipeline should be manually launched from the GitLab UI with the ``SHARED_SPACK`` CI variable set -to ``ON`` and the ``SPACK_PREFIX`` variable to the version of spack being set in the uberenv_config.json. +1. The first thing to do is update the spack commit in the +``.uberenv_config.json`` file. +2. Then, a pipeline should be manually launched from the GitLab UI with the +``SHARED_SPACK`` CI variable set to ``ON`` and the ``SPACK_PREFIX`` variable to +the version of spack being set in the uberenv_config.json. This will create a new spack installation and rebuild all of the specs. diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst index 2d5ff0a5b0..c256e1bcf0 100644 --- a/doc/sundials_developers/source/testing/Spot.rst +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -31,11 +31,11 @@ Setting Up Your Own SPOT Visualizations --------------------------------------- To display data from non-GitLab jobs or create a local collection of runs, -input into the SPOT search bar the absolute path to the directory containing the `.cali` files and refresh the page. -By default `.cali`` files will be generated in the build directory under -``Benchmarking/output`` and ``Testing/output``. To specify where `.cali` output -files are placed, define the CMake option SUNDIALS_CALIPER_OUTPUT_DIR with the -desired directory path. +input into the SPOT search bar the absolute path to the directory containing +the `.cali` files and refresh the page. By default `.cali`` files will be +generated in the build directory under ``Benchmarking/output`` and +``Testing/output``. To specify where `.cali` output files are placed, define +the CMake option SUNDIALS_CALIPER_OUTPUT_DIR with the desired directory path. To retain the same filters as a given SPOT visualization link in :ref:`Bookmarks`, swap out the ``sf`` value in the URL with the @@ -46,7 +46,22 @@ directory path containing the `.cali` files. Bookmarks to Notable SPOT Visualizations ---------------------------------------------- -Links will be added as informative SPOT visualizations are found. +Notes: -.. future links to add: aggregations of examples -.. \ No newline at end of file +SPOT will be put into maintenance mode in the coming months +(as of July 2023). The team behind SPOT plans to bring the functionality to +`Thicket `_ as SPOT's successor. + +Aggregate results for example runs on SPOT may not be accurate. The aggregation method +involves combining `.cali` files in ways that do not necessarily equal the +true sum of the results. + +Examples + +`Aggregated Runs `_ + +Benchmarks + +`Advection Reaction 3D - All Configurations `_ + +`Diffusion 2D - All Configurations `_ From e58dc2e1b64425fedc63fafea1f4a01658a10fa3 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 25 Jul 2023 14:52:37 -0700 Subject: [PATCH 154/187] change folder name to be less redundant --- .../advection_reaction_3D/raja/CMakeLists.txt | 30 ++++++------------- .../source/testing/Spot.rst | 22 ++++++++++++-- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt index 41db8bedf8..e8b3853d59 100644 --- a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt @@ -19,6 +19,13 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) set(OTHER_LIBS OpenMP::OpenMP_CXX) endif() + # Set up parameters to run benchmarks with + set(BENCHMARK_VAR + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton") + # ---------------------------------------------------------------------------- # MPI only # ---------------------------------------------------------------------------- @@ -58,18 +65,11 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(FILES README.md ../scripts/compare_error.py ../scripts/compute_error.py ../scripts/pickle_solution_output.py DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") - set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_raja" - "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_raja" - "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_raja" - "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_raja" - ) - - # NUM_CORES based on Lassen system for 1 node foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) + # NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(advection_reaction_3D_raja advection_reaction_3D_raja advection_reaction_3D NUM_CORES 40 BENCHMARK_ARGS ${benchmark_args} @@ -124,17 +124,11 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(TARGETS advection_reaction_3D_raja_mpicuda DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") - set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton_raja_mpicuda" - "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton_raja_mpicuda" - "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton_raja_mpicuda" - "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton_raja_mpicuda") - - # NUM_CORES based on Lassen system for 1 node foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) + # NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(advection_reaction_3D_raja_mpicuda advection_reaction_3D_raja_mpicuda advection_reaction_3D ENABLE_GPU NUM_CORES 4 @@ -182,12 +176,6 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) install(TARGETS advection_reaction_3D_raja_mpihip DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") - set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 1.0 --dont-save\;arkimex_tlnewton_raja_mpihip" - "--method ARK-DIRK --nls newton --tf 1.0 --dont-save\;arkdirk_newton_raja_mpihip" - "--method CV-BDF --nls newton --tf 1.0 --dont-save\;cvbdf_newton_raja_mpihip" - "--method IDA --nls newton --tf 1.0 --dont-save\;ida_newton_raja_mpihip") - foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst index c256e1bcf0..9362e1ab35 100644 --- a/doc/sundials_developers/source/testing/Spot.rst +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -33,8 +33,8 @@ Setting Up Your Own SPOT Visualizations To display data from non-GitLab jobs or create a local collection of runs, input into the SPOT search bar the absolute path to the directory containing the `.cali` files and refresh the page. By default `.cali`` files will be -generated in the build directory under ``Benchmarking/output`` and -``Testing/output``. To specify where `.cali` output files are placed, define +generated in the build directory under ``Benchmarking`` and +``Testing/output/Caliper/Example``. To specify where `.cali` output files are placed, define the CMake option SUNDIALS_CALIPER_OUTPUT_DIR with the desired directory path. To retain the same filters as a given SPOT visualization link in @@ -65,3 +65,21 @@ Benchmarks `Advection Reaction 3D - All Configurations `_ `Diffusion 2D - All Configurations `_ + +MPI Serial + +`Arkode Diffusion 2D `_ + +`Cvode Diffusion 2D `_ + +`Ida Diffusion 2D `_ + +MPI GPU + +Note: CUDA Diffusion 2D visualizations are not available as the benchmark errors out before completion. + +`Arkode Diffusion 2D - Hip `_ + +`Cvode Diffusion 2D - Hip `_ + +`Ida Diffusion 2D - Hip `_ \ No newline at end of file From d697f310e6677163a461c6b8a5b6a2610a89ed5a Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 25 Jul 2023 14:57:13 -0700 Subject: [PATCH 155/187] add links for SPOT visualization --- .../source/testing/Spot.rst | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/doc/sundials_developers/source/testing/Spot.rst b/doc/sundials_developers/source/testing/Spot.rst index 9362e1ab35..3f647d7fef 100644 --- a/doc/sundials_developers/source/testing/Spot.rst +++ b/doc/sundials_developers/source/testing/Spot.rst @@ -56,30 +56,53 @@ Aggregate results for example runs on SPOT may not be accurate. The aggregation involves combining `.cali` files in ways that do not necessarily equal the true sum of the results. -Examples +**Examples** `Aggregated Runs `_ -Benchmarks +**Benchmarks** -`Advection Reaction 3D - All Configurations `_ +**Advection Reaction 3D** -`Diffusion 2D - All Configurations `_ +- `All configurations `_ -MPI Serial +- By configuration -`Arkode Diffusion 2D `_ + - Serial + - `ARK-DIRK, Newton `_ + - `ARK-IMEX, TL-Newton `_ + - `CV-BDF, Newton `_ + - `IDA, Newton `_ + - CUDA + - `ARK-DIRK, Newton `_ + - `ARK-IMEX, TL-Newton `_ + - `CV-BDF, Newton `_ + - `IDA, Newton `_ + - HIP + - `ARK-DIRK, Newton `_ + - `ARK-IMEX, TL-Newton `_ + - `CV-BDF, Newton `_ + - `IDA, Newton `_ -`Cvode Diffusion 2D `_ -`Ida Diffusion 2D `_ - -MPI GPU +**Diffusion 2D** Note: CUDA Diffusion 2D visualizations are not available as the benchmark errors out before completion. -`Arkode Diffusion 2D - Hip `_ +- `All configurations `_ + +- MPI + Serial + + - `ARKODE `_ + + - `CVODE `_ + + - `IDA `_ + +- MPI + HIP + + - `ARKODE `_ -`Cvode Diffusion 2D - Hip `_ + - `CVODE `_ -`Ida Diffusion 2D - Hip `_ \ No newline at end of file + - `IDA `_ \ No newline at end of file From 330d31e4c1528008b2fb0c76ef821fcd89f60b85 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Tue, 25 Jul 2023 14:58:40 -0700 Subject: [PATCH 156/187] fix hip link --- doc/shared/Install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/shared/Install.rst b/doc/shared/Install.rst index 5bb2d9813c..5cd85d6b10 100644 --- a/doc/shared/Install.rst +++ b/doc/shared/Install.rst @@ -1499,7 +1499,7 @@ SUNDIALS has been tested with the CUDA toolkit versions 10 and 11. Building with HIP ^^^^^^^^^^^^^^^^^^^^^^ -HIP(heterogeneous-compute interface for portability) allows developers to create portable applications for AMD and NVIDIA GPUs. HIP can be obtained from `HIP GitHub repository`_. +HIP(heterogeneous-compute interface for portability) allows developers to create portable applications for AMD and NVIDIA GPUs. HIP can be obtained from `HIP GitHub repository `_. To enable HIP, set ``ENABLE_HIP`` to ``ON`` and set ``AMDGPU_TARGETS`` to the desired target(ex. gfx705). In addition, set ``CMAKE_C_COMPILER`` and ``CMAKE_CXX_COMPILER`` to point to an installation of ``hipcc``. From be627e372fadaf2a620d4ea21cada9d6e46d3b75 Mon Sep 17 00:00:00 2001 From: Yu Pan <70486009+yu-nix@users.noreply.github.com> Date: Wed, 26 Jul 2023 10:03:13 -0700 Subject: [PATCH 157/187] Update .gitlab/spack_packages/sundials/package.py Co-authored-by: Cody Balos --- .gitlab/spack_packages/sundials/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 25ec6f5d81..92d52ce1bd 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -26,7 +26,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): # ========================================================================== # Versions # ========================================================================== - # version("develop", branch="develop") + version("develop", branch="develop") version("6.5.1", sha256="4252303805171e4dbdd19a01e52c1dcfe0dafc599c3cfedb0a5c2ffb045a8a75") version("6.5.0", sha256="4e0b998dff292a2617e179609b539b511eb80836f5faacf800e688a886288502") version("6.4.1", sha256="7bf10a8d2920591af3fba2db92548e91ad60eb7241ab23350a9b1bc51e05e8d0") From 35cf9d3a7d1d68d5bebd3783fec49d00ba82e972 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 26 Jul 2023 10:21:44 -0700 Subject: [PATCH 158/187] add CPU, GPU options --- benchmarks/advection_reaction_3D/raja/CMakeLists.txt | 9 +++------ benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt | 7 ++----- benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt | 3 +-- cmake/SundialsBuildOptionsPre.cmake | 6 +++++- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt index e8b3853d59..a697577573 100644 --- a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt @@ -69,9 +69,8 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - # NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(advection_reaction_3D_raja advection_reaction_3D_raja advection_reaction_3D - NUM_CORES 40 + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_CPUS} BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier} ) @@ -128,10 +127,9 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - # NUM_CORES based on Lassen system for 1 node sundials_add_benchmark(advection_reaction_3D_raja_mpicuda advection_reaction_3D_raja_mpicuda advection_reaction_3D ENABLE_GPU - NUM_CORES 4 + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier}) endforeach() @@ -180,10 +178,9 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - # NUM_CORES based on Corona system for 1 node sundials_add_benchmark(advection_reaction_3D_raja_mpihip advection_reaction_3D_raja_mpihip advection_reaction_3D ENABLE_GPU - NUM_CORES 8 + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier}) endforeach() diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index 3164e6d897..1f507ea574 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -75,11 +75,9 @@ foreach(test_tuple ${tests}) add_executable(${target} ${sources}) if("${backend}" STREQUAL "USE_CUDA") - - # based on Lassen system from build_and_bench (1 nodes). # sundials_add_benchmark(${target} ${target} diffusion_2D - # NUM_CORES 2 # ENABLE_GPU + # NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} # IDENTIFIER "d2d_${package}_cuda" # ) @@ -87,10 +85,9 @@ foreach(test_tuple ${tests}) if("${backend}" STREQUAL "USE_HIP") - # based on the Corona system for 1 node sundials_add_benchmark(${target} ${target} diffusion_2D - NUM_CORES 8 ENABLE_GPU + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} IDENTIFIER "d2d_${package}_hip" ) diff --git a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt index 94d292fa57..8d1e3cdc44 100644 --- a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt @@ -73,9 +73,8 @@ foreach(test_tuple ${tests}) install(TARGETS ${target} DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") - # NUM_CORES based 1 node for testing purposes sundials_add_benchmark(${target} ${target} diffusion_2D - NUM_CORES 32 + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_CPUS} IDENTIFIER "d2d_${package}_serial" ) diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index bf8dea8cff..b9959f745d 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -297,4 +297,8 @@ sundials_option(SUNDIALS_TEST_UNITTESTS BOOL sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests" "" ADVANCED) -sundials_option(SUNDIALS_CALIPER_OUTPUT_DIR PATH "Location to write caliper output files" "" ADVANCED) \ No newline at end of file +sundials_option(SUNDIALS_CALIPER_OUTPUT_DIR PATH "Location to write caliper output files" "" ADVANCED) + +sundials_option(SUNDIALS_BENCHMARK_NUM_CPUS STRING "Number of CPU cores to run benchmarks with" "40" ADVANCED) + +sundials_option(SUNDIALS_BENCHMARK_NUM_GPUS STRING "Number of GPUs to run benchmarks with" "4" ADVANCED) \ No newline at end of file From 430c00b7041c1237cc3b54a84459def84ba1b54c Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 26 Jul 2023 11:36:24 -0700 Subject: [PATCH 159/187] add caliper-dir variant --- .gitlab/spack_packages/sundials/package.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 25ec6f5d81..c75296c13e 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -199,8 +199,8 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): when="+adiak +caliper", description="Build examples with profiling capabilities") - # Benchmarking - variant("benchmarks", default=False, description="Build benchmark programs") + # Caliper Directory + variant("caliper-dir", default="none", description="Specify where to place Caliper files") # ========================================================================== # Dependencies @@ -764,6 +764,7 @@ def initconfig_package_entries(self): self.cache_option_from_variant("SUNDIALS_TEST_PROFILE", "profile-examples"), self.cache_option_from_variant("SUNDIALS_TEST_DEVTESTS", "profile-examples"), cmake_cache_string("SPACK_VERSION", ".".join(map(str, spack.spack_version_info))) + ] ) @@ -808,6 +809,10 @@ def initconfig_package_entries(self): if "+adiak" in spec["caliper"]: entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix.lib.cmake + "/adiak")) + if not "caliper-dir=none" in spec: + entries.append(self.cache_string_from_variant("SUNDIALS_CALIPER_OUTPUT_DIR", "caliper-dir")) + + # Building with Ginkgo if "+ginkgo" in spec: gko_backends = ["REF"] @@ -941,9 +946,6 @@ def initconfig_package_entries(self): if "+trilinos" in spec: entries.append(cmake_cache_path("Trilinos_DIR", spec["trilinos"].prefix)) - if "+profile-examples" in spec: - entries.append(cmake_cache_path("SUNDIALS_CALIPER_OUTPUT_DIR"), "/usr/workspace/sundials/califiles") - # Examples if spec.satisfies("@3:"): entries.extend( From ec78d82fac05c5cb33fd3221968bec5f746f1666 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 26 Jul 2023 13:37:35 -0700 Subject: [PATCH 160/187] add AddBenchmark macro to scripts/shared --- scripts/shared | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/shared b/scripts/shared index 28ecda2d92..670c03fd9d 100755 --- a/scripts/shared +++ b/scripts/shared @@ -76,6 +76,7 @@ $tar $tarfile $distrobase/cmake/SundialsSetupFortran.cmake $tar $tarfile $distrobase/cmake/SundialsSetupTPLs.cmake $tar $tarfile $distrobase/cmake/SundialsSetupTesting.cmake $tar $tarfile $distrobase/cmake/SundialsTPLOptions.cmake +$tar $tarfile $distrobase/cmake/macros/SundialsAddBenchmark.cmake $tar $tarfile $distrobase/cmake/macros/SundialsAddExamplesGinkgo.cmake $tar $tarfile $distrobase/cmake/macros/SundialsAddExecutable.cmake $tar $tarfile $distrobase/cmake/macros/SundialsAddLibrary.cmake From 7f7334ca9ce8309ac11dd8f2005ca28f6aec29d8 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 26 Jul 2023 14:49:29 -0700 Subject: [PATCH 161/187] add sundials_memory.hpp to tarscript --- scripts/shared | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/shared b/scripts/shared index 670c03fd9d..000cee248c 100755 --- a/scripts/shared +++ b/scripts/shared @@ -144,6 +144,7 @@ $tar $tarfile $distrobase/include/sundials/sundials_logger.h $tar $tarfile $distrobase/include/sundials/sundials_math.h $tar $tarfile $distrobase/include/sundials/sundials_matrix.h $tar $tarfile $distrobase/include/sundials/sundials_memory.h +$tar $tarfile $distrobase/include/sundials/sundials_memory.hpp $tar $tarfile $distrobase/include/sundials/sundials_mpi_types.h $tar $tarfile $distrobase/include/sundials/sundials_nonlinearsolver.h $tar $tarfile $distrobase/include/sundials/sundials_nonlinearsolver.hpp From 6e09ca1bba89dc401b71b842e25f34e0c6403ced Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 26 Jul 2023 14:50:21 -0700 Subject: [PATCH 162/187] add SundialsAdiak to tarscript --- scripts/shared | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/shared b/scripts/shared index 000cee248c..2a9cef1731 100755 --- a/scripts/shared +++ b/scripts/shared @@ -95,6 +95,7 @@ $tar $tarfile $distrobase/cmake/tpl/FindSUPERLUMT.cmake $tar $tarfile $distrobase/cmake/tpl/FindSUPERLUDIST.cmake $tar $tarfile $distrobase/cmake/tpl/FindTrilinos.cmake $tar $tarfile $distrobase/cmake/tpl/FindXBRAID.cmake +$tar $tarfile $distrobase/cmake/tpl/SundialsAdiak.cmake $tar $tarfile $distrobase/cmake/tpl/SundialsCaliper.cmake $tar $tarfile $distrobase/cmake/tpl/SundialsGinkgo.cmake $tar $tarfile $distrobase/cmake/tpl/SundialsHypre.cmake From b265d915af3f41979b8f8e8e7ee248042930603d Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 26 Jul 2023 15:24:02 -0700 Subject: [PATCH 163/187] shorten diffusion 2D file names --- benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt | 13 ++++++------- benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index 1f507ea574..57be751aab 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -75,20 +75,19 @@ foreach(test_tuple ${tests}) add_executable(${target} ${sources}) if("${backend}" STREQUAL "USE_CUDA") - # sundials_add_benchmark(${target} ${target} diffusion_2D - # ENABLE_GPU - # NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} - # IDENTIFIER "d2d_${package}_cuda" - # ) + sundials_add_benchmark(${target} ${target} diffusion_2D + ENABLE_GPU + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} + IDENTIFIER "cuda" + ) endif() if("${backend}" STREQUAL "USE_HIP") - sundials_add_benchmark(${target} ${target} diffusion_2D ENABLE_GPU NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} - IDENTIFIER "d2d_${package}_hip" + IDENTIFIER "hip" ) endif() diff --git a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt index 8d1e3cdc44..4c3ae2b577 100644 --- a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt @@ -75,7 +75,7 @@ foreach(test_tuple ${tests}) sundials_add_benchmark(${target} ${target} diffusion_2D NUM_CORES ${SUNDIALS_BENCHMARK_NUM_CPUS} - IDENTIFIER "d2d_${package}_serial" + IDENTIFIER "serial" ) endforeach() From 9219b8198f85dd8c600008d3e65fed8408a57946 Mon Sep 17 00:00:00 2001 From: Yu Pan <70486009+yu-nix@users.noreply.github.com> Date: Thu, 27 Jul 2023 10:31:32 -0700 Subject: [PATCH 164/187] Update cmake/macros/SundialsAddBenchmark.cmake Co-authored-by: Cody Balos --- cmake/macros/SundialsAddBenchmark.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index 31d49ad5ea..6a7e9bcc9c 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -59,7 +59,7 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) if(SUNDIALS_SCHEDULER_COMMAND STREQUAL "flux run") set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES}") elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun" AND ${sundials_add_benchmark_ENABLE_GPU}) - set(SCHEDULER_STRING " --smpiargs=\\\"-gpu\\\" -n${sundials_add_benchmark_NUM_CORES} -a1 -c1 -g1") + set(SCHEDULER_STRING " --smpiargs=\\\"-gpu\\\" -n${sundials_add_benchmark_NUM_GPUS} -a1 -c1 -g1") elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun") set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES} -a1 -c1") elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "srun") From e49f34e96667ee9c6bab34a9eaef3114e98c6e54 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 27 Jul 2023 10:47:35 -0700 Subject: [PATCH 165/187] remove extra identifier for Diffusion 2D benchmark --- benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt | 2 -- benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index 57be751aab..b75bb8063a 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -78,7 +78,6 @@ foreach(test_tuple ${tests}) sundials_add_benchmark(${target} ${target} diffusion_2D ENABLE_GPU NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} - IDENTIFIER "cuda" ) endif() @@ -87,7 +86,6 @@ foreach(test_tuple ${tests}) sundials_add_benchmark(${target} ${target} diffusion_2D ENABLE_GPU NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} - IDENTIFIER "hip" ) endif() diff --git a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt index 4c3ae2b577..f43422e6d5 100644 --- a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt @@ -75,7 +75,6 @@ foreach(test_tuple ${tests}) sundials_add_benchmark(${target} ${target} diffusion_2D NUM_CORES ${SUNDIALS_BENCHMARK_NUM_CPUS} - IDENTIFIER "serial" ) endforeach() From 399e6dbf02b0573463307ae086d331460eecd1bb Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 27 Jul 2023 10:49:24 -0700 Subject: [PATCH 166/187] add copyright to SundialsAddBenchmark.cmake --- cmake/macros/SundialsAddBenchmark.cmake | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index 31d49ad5ea..7c213e36ab 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -1,5 +1,15 @@ -# Take in an executable, determine how to construct the arguments - +# --------------------------------------------------------------- +# Programmer(s): Yu Pan @ LLNL +# --------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2023, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End # --------------------------------------------------------------- # CMake macro for adding benchmarks to `make benchmark`. # --------------------------------------------------------------- From 49abb3fd4a5b178fdd88dd2d10567db54a35d0dc Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 27 Jul 2023 15:20:42 -0700 Subject: [PATCH 167/187] turn on adiak independently --- .gitlab/spack_packages/sundials/package.py | 2 +- doc/shared/Install.rst | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab/spack_packages/sundials/package.py b/.gitlab/spack_packages/sundials/package.py index 6ce6ae60f5..caaf24e494 100644 --- a/.gitlab/spack_packages/sundials/package.py +++ b/.gitlab/spack_packages/sundials/package.py @@ -120,7 +120,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage): variant("sycl", default=False, when="@5.7.0:", description="Enable SYCL vector") # External libraries - variant("adiak", default=False, when="+caliper", description="Enable Adiak interfaces") + variant("adiak", default=False, when="@6.6.0:", description="Enable Adiak interfaces") variant( "caliper", default=False, diff --git a/doc/shared/Install.rst b/doc/shared/Install.rst index 5cd85d6b10..f6126c9bb0 100644 --- a/doc/shared/Install.rst +++ b/doc/shared/Install.rst @@ -1052,10 +1052,6 @@ illustration only. Default: OFF - .. note:: - - Using Adiak requires setting :cmakeop:`ENABLE_CALIPER` and :cmakeop:`SUNDIALS_BUILD_WITH_PROFILING` to ``ON``. - .. cmakeoption:: adiak_DIR Path to the root of an Adiak installation From 821a82078e75e514900407ad78a34b44ffe9c538 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 28 Jul 2023 09:34:36 -0700 Subject: [PATCH 168/187] add +allow-unsupported-compilers --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 5f6f4158a8..b081fb4f68 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -39,6 +39,6 @@ lassen_gcc_cuda_bench: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.8.0] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double ~int64 +benchmarks+profiling+caliper +mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+cuda~adiak cuda_arch=70 ^${CUDA_SPEC}" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double ~int64 +benchmarks+profiling+caliper +mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+cuda~adiak cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" extends: .lassen_build_and_bench From 72e9466a95c9cd33d5f0066ae9cb7db74f208111 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 2 Aug 2023 15:29:55 -0700 Subject: [PATCH 169/187] switch benchmark script to make benchmark --- .gitlab/build_and_bench.sh | 84 +++++--------------------- .gitlab/build_and_bench_original.sh | 94 +++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 69 deletions(-) mode change 100644 => 100755 .gitlab/build_and_bench.sh create mode 100644 .gitlab/build_and_bench_original.sh diff --git a/.gitlab/build_and_bench.sh b/.gitlab/build_and_bench.sh old mode 100644 new mode 100755 index 19c3649c09..99c99be18c --- a/.gitlab/build_and_bench.sh +++ b/.gitlab/build_and_bench.sh @@ -1,20 +1,12 @@ #!/usr/bin/env bash -# Build and test first -source "$BUILD_ROOT/.gitlab/build_and_test.sh" --no-clean - -benchmark_dir="${build_dir}/benchmarks" -ar3d_dir="${benchmark_dir}/advection_reaction_3D" -d2d_dir="${benchmark_dir}/diffusion_2D" +# variables -job_unique_id=${CI_JOB_ID:-""} -job_timestamp=${CI_JOB_STARTED_AT:-""} - -# remove tailing number from hostname -hostname=${hostname%%[0-9]*} +# Build and test first -benchmark_store_dir="/usr/workspace/sundials/caliper/benchmarks/${hostname}/${job_timestamp}_${job_unique_id}" +source "$BUILD_ROOT/.gitlab/build_and_test.sh" --no-clean +# Embed this into `make benchmark` nresg=$((BENCHMARK_NNODES * 4)) # Lassen has 4 GPUs per node nresc=$((BENCHMARK_NNODES * 40)) # Lassen has 40 cores per node @@ -22,69 +14,23 @@ echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~ Benchmarking SUNDIALS" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -if [[ ! -d ${benchmark_dir} ]] -then - echo "ERROR: Benchmark directory not found : ${benchmark_dir}" && exit 1 -fi - -cd "${benchmark_dir}" -ls - # We use synchronous kernel launches for benchmarking # because we are interested in individual region timings. export CUDA_LAUNCH_BLOCKING=1 -if [[ -d ${ar3d_dir} ]] -then - date - if [[ -f ${ar3d_dir}/advection_reaction_3D_mpicuda ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-IMEX --nls tl-newton --tf 10.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkdirk_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-DIRK --nls newton --tf 10.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_cvbdf_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method CV-BDF --nls newton --tf 10.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_ida_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method IDA --nls newton --tf 10.0 - elif [[ -f ${ar3d_dir}/advection_reaction_3D ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-IMEX --nls tl-newton --tf 2.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkdirk_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-DIRK --nls newton --tf 2.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_cvbdf_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method CV-BDF --nls newton --tf 2.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 - fi - date -fi - - -if [[ -d ${d2d_dir} ]] -then - date +if [[ -d ${build_dir} ]] +then - if [[ -d ${d2d_dir}/mpi_gpu ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/arkode_diffusion_2D_mpicuda" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/cvode_diffusion_2D_mpicuda" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/ida_diffusion_2D_mpicuda" - elif [[ -d ${d2d_dir}/mpi_serial ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/ida_diffusion_2D" - fi + # configure for multinode + $cmake_exe \ + -C "${hostconfig_path}" \ + -DSUNDIALS_BENCHMARK_NUM_CPUS=${nresc} \ + -DSUNDIALS_BENCHMARK_NUM_GPUS=${nresg} \ + "${project_dir}" + + cd ${build_dir} + make benchmark - date fi echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" diff --git a/.gitlab/build_and_bench_original.sh b/.gitlab/build_and_bench_original.sh new file mode 100644 index 0000000000..19c3649c09 --- /dev/null +++ b/.gitlab/build_and_bench_original.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# Build and test first +source "$BUILD_ROOT/.gitlab/build_and_test.sh" --no-clean + +benchmark_dir="${build_dir}/benchmarks" +ar3d_dir="${benchmark_dir}/advection_reaction_3D" +d2d_dir="${benchmark_dir}/diffusion_2D" + +job_unique_id=${CI_JOB_ID:-""} +job_timestamp=${CI_JOB_STARTED_AT:-""} + +# remove tailing number from hostname +hostname=${hostname%%[0-9]*} + +benchmark_store_dir="/usr/workspace/sundials/caliper/benchmarks/${hostname}/${job_timestamp}_${job_unique_id}" + +nresg=$((BENCHMARK_NNODES * 4)) # Lassen has 4 GPUs per node +nresc=$((BENCHMARK_NNODES * 40)) # Lassen has 40 cores per node + +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo "~~~~~ Benchmarking SUNDIALS" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + +if [[ ! -d ${benchmark_dir} ]] +then + echo "ERROR: Benchmark directory not found : ${benchmark_dir}" && exit 1 +fi + +cd "${benchmark_dir}" +ls + +# We use synchronous kernel launches for benchmarking +# because we are interested in individual region timings. +export CUDA_LAUNCH_BLOCKING=1 + +if [[ -d ${ar3d_dir} ]] +then + date + if [[ -f ${ar3d_dir}/advection_reaction_3D_mpicuda ]] + then + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-IMEX --nls tl-newton --tf 10.0 + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkdirk_newton.cali),runtime-report(calc.inclusive)" + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-DIRK --nls newton --tf 10.0 + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_cvbdf_newton.cali),runtime-report(calc.inclusive)" + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method CV-BDF --nls newton --tf 10.0 + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_ida_newton.cali),runtime-report(calc.inclusive)" + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method IDA --nls newton --tf 10.0 + elif [[ -f ${ar3d_dir}/advection_reaction_3D ]] + then + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-IMEX --nls tl-newton --tf 2.0 + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkdirk_newton.cali),runtime-report(calc.inclusive)" + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-DIRK --nls newton --tf 2.0 + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_cvbdf_newton.cali),runtime-report(calc.inclusive)" + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method CV-BDF --nls newton --tf 2.0 + export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)" + jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 + fi + date +fi + + +if [[ -d ${d2d_dir} ]] +then + date + + if [[ -d ${d2d_dir}/mpi_gpu ]] + then + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/arkode_diffusion_2D_mpicuda" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/cvode_diffusion_2D_mpicuda" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" + jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/ida_diffusion_2D_mpicuda" + elif [[ -d ${d2d_dir}/mpi_serial ]] + then + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" + jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" + jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" + export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" + jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/ida_diffusion_2D" + fi + + date +fi + +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo "~~~~~ CLEAN UP" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +make clean + From fb119d5a154c84b06d64f21e63e415d2b32db57f Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 3 Aug 2023 15:57:20 -0700 Subject: [PATCH 170/187] update Lassen benchmark job with new variants --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index b081fb4f68..99be2222fb 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -39,6 +39,6 @@ lassen_gcc_cuda_bench: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.8.0] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double ~int64 +benchmarks+profiling+caliper +mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+cuda~adiak cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double caliper-dir=/usr/workspace/sundials/test-job ~int64 +benchmarks+profiling+caliper+adiak+profile-examples+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" extends: .lassen_build_and_bench From 8011685a11a5ccb0b319739fa0d25a2e2d869bb0 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 3 Aug 2023 16:35:49 -0700 Subject: [PATCH 171/187] update caliper file directory --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 99be2222fb..38e8f7071a 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -39,6 +39,6 @@ lassen_gcc_cuda_bench: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.8.0] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double caliper-dir=/usr/workspace/sundials/test-job ~int64 +benchmarks+profiling+caliper+adiak+profile-examples+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double caliper-dir=/usr/workspace/sundials/califiles ~int64 +benchmarks+profiling+caliper+adiak+profile-examples+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" extends: .lassen_build_and_bench From dc9e74361d8a6f7d24187e7a2e5a6453c45503d7 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 4 Aug 2023 14:20:58 -0700 Subject: [PATCH 172/187] separate benchmarks from SUNDIALS_TEST_DEVTEST --- .../diffusion_2D/mpi_gpu/CMakeLists.txt | 13 +++++------ cmake/SundialsSetupTesting.cmake | 15 ++++++++----- cmake/macros/SundialsAddBenchmark.cmake | 22 +++++++++---------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index b75bb8063a..42dee3e3ad 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -74,13 +74,12 @@ foreach(test_tuple ${tests}) add_executable(${target} ${sources}) - if("${backend}" STREQUAL "USE_CUDA") - sundials_add_benchmark(${target} ${target} diffusion_2D - ENABLE_GPU - NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} - ) - - endif() + # if("${backend}" STREQUAL "USE_CUDA") + # sundials_add_benchmark(${target} ${target} diffusion_2D + # ENABLE_GPU + # NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} + # ) + #endif() if("${backend}" STREQUAL "USE_HIP") sundials_add_benchmark(${target} ${target} diffusion_2D diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 266533ad88..d39a9dcc3c 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -17,10 +17,8 @@ # Enable testing with 'make test' include(CTest) -# Check if development tests are enabled -if(SUNDIALS_TEST_DEVTESTS) - - message("SUNDIALS Development testing") +# Set up testRunner +if (SUNDIALS_TEST_DEVTESTS OR BUILD_BENCHMARKS) # Python is needed to use the test runner find_package(PythonInterp REQUIRED) @@ -34,11 +32,18 @@ if(SUNDIALS_TEST_DEVTESTS) # look for the testRunner script in the test directory find_program(TESTRUNNER testRunner PATHS test NO_DEFAULT_PATH) if(NOT TESTRUNNER) - print_error("Could not locate testRunner. Set SUNDIALS_TEST_DEVTESTS=OFF to continue.") + print_error("Could not locate testRunner. Set SUNDIALS_TEST_DEVTESTS=OFF or BUILD_BENCHMARKS=OFF to continue.") endif() message(STATUS "Found testRunner: ${TESTRUNNER}") set(TESTRUNNER ${TESTRUNNER} CACHE INTERNAL "") +endif() + +# Check if development tests are enabled +if(SUNDIALS_TEST_DEVTESTS) + + message("SUNDIALS Development testing") + # Create the default test output directory set(TEST_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Testing/output) diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index 7c213e36ab..d8336e73a8 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -51,19 +51,24 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) endif() - # make the output directory if it doesn't exist + # make the caliper output directory if it doesn't exist if(NOT EXISTS ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) file(MAKE_DIRECTORY ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}) endif() + # make the the output directory if it doesn't exist + if(NOT EXISTS ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/output) + file(MAKE_DIRECTORY ${SUNDIALS_BENCHMARK_OUTPUT_DIR}/output) + endif() + # command line arguments for the test runner script set(TEST_RUNNER_ARGS + "--profile" "--verbose" "--executablename=$" - "--outputdir=${TEST_OUTPUT_DIR}" + "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/output" "--calidir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" - "--nodiff" - ) + "--nodiff") # incorporate scheduler arguments into test_runner if(SUNDIALS_SCHEDULER_COMMAND STREQUAL "flux run") @@ -78,21 +83,14 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) string(REPLACE " " ";" SCHEDULER_ARGS "${SCHEDULER_STRING}") string(REPLACE " " ";" SCHEDULER_COMMAND_ARGS "${SUNDIALS_SCHEDULER_COMMAND}") - # taken from SundialsAddTest string(STRIP "${RUN_COMMAND}" RUN_COMMAND) set(RUN_COMMAND ${SCHEDULER_COMMAND_ARGS} ${SCHEDULER_ARGS}) list(APPEND TEST_RUNNER_ARGS "--runcommand=\"${RUN_COMMAND}\"") - # enable test runner to set up caliper output paths and configs - if(SUNDIALS_TEST_PROFILE) - list(APPEND TEST_RUNNER_ARGS "--profile") - endif() - list(APPEND TEST_RUNNER_ARGS "--runargs=${sundials_add_benchmark_BENCHMARK_ARGS}" "--testname=${TARGET_NAME}") add_custom_target(${TARGET_NAME} COMMENT "Running ${TARGET_NAME}" - COMMAND ${PYTHON_EXECUTABLE} ${TESTRUNNER} ${TEST_RUNNER_ARGS} - ) + COMMAND ${PYTHON_EXECUTABLE} ${TESTRUNNER} ${TEST_RUNNER_ARGS}) add_dependencies(benchmark ${TARGET_NAME}) endmacro() From b6799b26441cdd8ce83c90a117d4aff6865b11ba Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Fri, 4 Aug 2023 14:32:35 -0700 Subject: [PATCH 173/187] remove +profile-examples --- .gitlab/lassen-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 38e8f7071a..92362a3d2a 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -39,6 +39,6 @@ lassen_gcc_cuda_bench: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.8.0] variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double caliper-dir=/usr/workspace/sundials/califiles ~int64 +benchmarks+profiling+caliper+adiak+profile-examples+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=lsf caliper-dir=/usr/workspace/sundials/califiles ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" extends: .lassen_build_and_bench From ffe8a61b3faaf753836b6982a8230286840ddca8 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 7 Aug 2023 16:08:53 -0700 Subject: [PATCH 174/187] add sundials_version adiak metadata --- src/sundials/sundials_context.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sundials/sundials_context.c b/src/sundials/sundials_context.c index 26ea56f944..94f90301b8 100644 --- a/src/sundials/sundials_context.c +++ b/src/sundials/sundials_context.c @@ -239,7 +239,8 @@ void sunAdiakCollectMetadata() { adiak_namevalue("fortran_compiler_version", 2, NULL, "%s", SUN_FORTRAN_COMPILER_VERSION); adiak_namevalue("fortran_compiler_flags", 2, NULL, "%s", SUN_FORTRAN_COMPILER_FLAGS); - adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); + adiak_namevalue("sundials_version", 2, NULL, "%s", SUNDIALS_VERSION); + adiak_namevalue("sundials_git_version", 2, NULL, "%s", SUNDIALS_GIT_VERSION); adiak_namevalue("build_type", 2, NULL, "%s", SUN_BUILD_TYPE); adiak_namevalue("third_party_libraries", 2, NULL, "%s", SUN_TPL_LIST); #ifdef SUN_JOB_ID From 77e6695b46f0e27c33ed9e485c5b5985d6d3199e Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Mon, 7 Aug 2023 16:59:44 -0700 Subject: [PATCH 175/187] modify compare-tests.py for version comparison --- test/compare_tests.py | 64 +++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/test/compare_tests.py b/test/compare_tests.py index c2ad899f37..b7af7ecd31 100644 --- a/test/compare_tests.py +++ b/test/compare_tests.py @@ -15,39 +15,54 @@ def main(): parser = argparse.ArgumentParser(description='Compare Sundials performance results against previous results') - parser.add_argument('--calidir', dest='caliDir', type=str,help='name of folder containing caliper files', default="/usr/workspace/pan13/shrimp2") + parser.add_argument('--release', dest='release', action='store_true', help='indicate if the current run to process is a release') - parser.add_argument('--outdir', dest='outDir', type=str, help='path to directory to write results to', default="/g/g20/pan13/shrimp") + parser.add_argument('--calidir', dest='caliDir', type=str, help='path to directory containing caliper files', default="/usr/workspace/sundials/califiles") + + parser.add_argument('--releasedir', dest='releaseDir', type=str, help='path to directory containing release caliper files', default="/usr/workspace/sundials/califiles/Release") + + parser.add_argument('--outpath', dest='outPath', type=str, help='path to directory to write results to', default="/g/g20/pan13/shrimp") args = parser.parse_args() + release = args.release + releaseDir = args.releaseDir caliDir = args.caliDir - outPath = args.outpath + outPath = args.outPath - # get the latest test run + # Get the latest test run runDirs = glob.glob("%s/Testing/*" % caliDir, recursive = True) runDirs.sort(key=os.path.getmtime, reverse=True) runDir = runDirs[0] + runFile = glob.glob(runDir)[0] + # Load in file to access release value + th_temp = tt.Thicket.from_caliperreader(runFile) + if release: + # Compare against the last release + versionDirs = glob.glob("%s/*" % releaseDir) + versionDirs.sort(key=os.path.getmtime, reverse=True) + versionDir = versionDirs[1] + else: + # Compare against latest release + version = th_temp.metadata['sundials_version'].values[0] + versionDir = "%s/%s" % (releaseDir, version) + # Gather files to process - testFiles = glob.glob("%s/*.cali" % (runDir)) + runFiles = glob.glob("%s/*.cali" % (runDir)) if not os.path.exists(outPath): - os.mkdirs(outPath) + os.makedirs(outPath) outFile = open("%s/output.out" % outPath, 'w') - # Compare test results against past runs. If a test performs below a threshold, output test name to outFile. with mp.Pool() as pool: - for res in pool.starmap(process_test, [(caliDir, i) for i in testFiles]): + for res in pool.starmap(compare_against_release, [(versionDir, i) for i in runFiles]): if res: outFile.write(res + "\n") outFile.close() - # for file in testFiles: - # process_test(caliDir, file) - - outFile = open(outPath, 'r') + outFile = open("%s/output.out" % outPath, 'r') try: outLines = outFile.readlines() finally: @@ -57,28 +72,23 @@ def main(): return -1 return 0 -def process_test(caliDir, file): +def compare_against_release(releaseDir, file): th = tt.Thicket.from_caliperreader(file) - - testName = th.metadata['env.TEST_NAME'].values[0] - # Gather the last numRuns of tests and remove the current run from set of runs to aggregate - numRuns = 5 - files = glob.glob("%s/Testing/*/%s.*.cali" % (caliDir, testName), recursive=True) - files.sort(key=os.path.getmtime, reverse=True) - files.pop(0) # remove current run - files = files[:numRuns] + testName = th.metadata['env.TEST_NAME'].values[0] - files.sort(key=os.path.ge) + # Gather release run + release_file = glob.glob("%s/Testing/*/%s.*.cali" % (releaseDir, testName), recursive=True) + release_th = tt.Thicket.from_caliperreader(release_file) - th_files = tt.Thicket.from_caliperreader(files) metrics = ['Max time/rank'] - tt.mean(th_files, columns=metrics) + tt.mean(release_th, columns=metrics) tt.mean(th, columns=metrics) - ratio = th.statsframe.dataframe['Max time/rank_mean'] / th_files.statsframe.dataframe['Max time/rank_mean'] - - if 1 - ratio[0] < 0.1: + ratio = th.statsframe.dataframe['Max time/rank_mean'] / release_th.statsframe.dataframe['Max time/rank_mean'] + print(ratio[0]) + tolerance = 0.0002 + if 1 - ratio[0] < tolerance: return testName if __name__ == "__main__": From f2e293b648cf4a7ed8a40ebc297e00348f78cfcb Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 9 Aug 2023 14:34:20 -0700 Subject: [PATCH 176/187] convert compare_test.py to compare against releases --- test/compare_tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/compare_tests.py b/test/compare_tests.py index b7af7ecd31..8c93965036 100644 --- a/test/compare_tests.py +++ b/test/compare_tests.py @@ -36,7 +36,6 @@ def main(): runDir = runDirs[0] runFile = glob.glob(runDir)[0] - # Load in file to access release value th_temp = tt.Thicket.from_caliperreader(runFile) if release: # Compare against the last release @@ -44,7 +43,7 @@ def main(): versionDirs.sort(key=os.path.getmtime, reverse=True) versionDir = versionDirs[1] else: - # Compare against latest release + # Compare against the release the run is a part of version = th_temp.metadata['sundials_version'].values[0] versionDir = "%s/%s" % (releaseDir, version) @@ -62,7 +61,7 @@ def main(): outFile.write(res + "\n") outFile.close() - outFile = open("%s/output.out" % outPath, 'r') + outFile = open("%s/example_output.out" % outPath, 'r') try: outLines = outFile.readlines() finally: @@ -72,20 +71,21 @@ def main(): return -1 return 0 + def compare_against_release(releaseDir, file): th = tt.Thicket.from_caliperreader(file) testName = th.metadata['env.TEST_NAME'].values[0] # Gather release run - release_file = glob.glob("%s/Testing/*/%s.*.cali" % (releaseDir, testName), recursive=True) - release_th = tt.Thicket.from_caliperreader(release_file) + releaseFile = glob.glob("%s/Testing/*/%s.*.cali" % (releaseDir, testName), recursive=True) + th_release = tt.Thicket.from_caliperreader(releaseFile) metrics = ['Max time/rank'] - tt.mean(release_th, columns=metrics) + tt.mean(th_release, columns=metrics) tt.mean(th, columns=metrics) - ratio = th.statsframe.dataframe['Max time/rank_mean'] / release_th.statsframe.dataframe['Max time/rank_mean'] + ratio = th.statsframe.dataframe['Max time/rank_mean'] / th_release.statsframe.dataframe['Max time/rank_mean'] print(ratio[0]) tolerance = 0.0002 if 1 - ratio[0] < tolerance: From 525c4a7b8d91619336d5e251f7ba6d41b432953a Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 9 Aug 2023 14:34:44 -0700 Subject: [PATCH 177/187] add benchmark variant of compare_test.py --- test/compare_benchmarks.py | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 test/compare_benchmarks.py diff --git a/test/compare_benchmarks.py b/test/compare_benchmarks.py new file mode 100644 index 0000000000..2472c6d47c --- /dev/null +++ b/test/compare_benchmarks.py @@ -0,0 +1,92 @@ +import os +import glob +import argparse +import multiprocessing as mp + +import thicket as tt + +# Note: script does not work for advection_reaction_3D benchmarks currently. + +def main(): + parser = argparse.ArgumentParser(description='Compare Sundials performance results against previous results') + + parser.add_argument('--release', dest='release', action='store_true', help='indicate if the current run to process is a release') + + parser.add_argument('--calidir', dest='caliDir', type=str, help='path to directory containing caliper files', default="/usr/workspace/sundials/califiles") + + parser.add_argument('--releasedir', dest='releaseDir', type=str, help='path to directory containing release caliper files', default="/usr/workspace/sundials/califiles/Release") + + parser.add_argument('--outpath', dest='outPath', type=str, help='path to directory to write results to', default="/g/g20/pan13/shrimp") + + parser.add_argument('--jobid', dest='jobID', type=int, help='job id of the current run to identify .cali files') + + args = parser.parse_args() + + release = args.release + releaseDir = args.releaseDir + caliDir = args.caliDir + outPath = args.outPath + jobID = args.jobID + + # Get available benchmarks + benchFiles = glob.glob("%s/Benchmarking/*/*" % caliDir) + + if not os.path.exists(outPath): + os.makedirs(outPath) + outFile = open("%s/benchmark_output.out" % outPath, 'w') + + # thread per file + with mp.Pool() as pool: + for res in pool.starmap(process_benchmark, [(jobID, release, releaseDir, i) for i in benchFiles]): + if res: + outFile.write(res + "\n") + outFile.close() + + outFile = open("%s/benchmark_output.out" % outPath, 'r') + try: + outLines = outFile.readlines() + finally: + outFile.close() + + if (len(outLines) == 0): + return -1 + return 0 + +def process_benchmark(jobID, isRelease, releaseDir, benchmarkDir): + + # Get the current benchmark run + benchmarkFiles = glob.glob("%s/*.cali" % benchmarkDir) + th_files = tt.Thicket.from_caliperreader(benchmarkFiles) + curFilter = lambda x: x['job_id'] == jobID + th_current = th_files.filter_metadata(curFilter) + + # Don't compare if the run didn't include this benchmark + if (th_current.dataframe.size == 0): + return + + # Get the release caliper file + if isRelease: + # Get the last release + versionDirs = glob.glob("%s/*" % releaseDir) + versionDirs.sort(key=os.path.getmtime, reverse=True) + versionDir = versionDirs[1] + else: + # Get the release the run is a part of + version = th_current.metadata['sundials_version'].values[0] + versionDir = "%s/%s" % (releaseDir, version) + benchmarkName = th_current.metadata['env.TEST_NAME'].values[0] + releaseFile = glob.glob("%s/Benchmarking/*/%s/*.cali" % (versionDir, benchmarkName), recursive=True) + th_compare = tt.Thicket.from_caliperreader(releaseFile) + + metrics = ['Max time/rank'] + tt.mean(th_current, columns=metrics) + tt.mean(th_compare, columns=metrics) + + ratio = th_current.statsframe.dataframe['Max time/rank_mean'] / th_compare.statsframe.dataframe['Max time/rank_mean'] + + if 1 - ratio[0] < 0.0002: + return benchmarkName + + +if __name__ == "__main__": + main() From c56a3ddf417b598ac9dfb39a775018cc10244554 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 9 Aug 2023 16:39:24 -0700 Subject: [PATCH 178/187] add release directory configuration --- .gitlab/build_and_test.sh | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index a3dc1ba25d..9e0924ad1c 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -1,5 +1,14 @@ #!/usr/bin/bash +CI_COMMIT_BRANCH=main +BUILD_JOBS=32 + +COMPILER_SPEC=rocmcc@5.5.0 +AMDGPU_TARGET=gfx906 +CALIPER_DIR=/g/g20/pan13/test-release/ + +SPEC="%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +benchmarks+profiling+caliper+adiak+rocm+mpi+profile-examples" + # make sure lmod is loaded if test -e /usr/share/lmod/lmod/init/bash then @@ -97,9 +106,9 @@ then mkdir -p ${spack_user_cache} fi - if [[ -d /usr/workspace/sundials ]] + if [[ -d /usr/workspace/pan13 ]] then - upstream="/usr/workspace/sundials/spack_installs/${spack_prefix}/${hostname}" + upstream="/usr/workspace/pan13/spack_installs/${spack_prefix}/${hostname}" mkdir -p "${upstream}" upstream_opt="--upstream=${upstream}" fi @@ -188,11 +197,23 @@ then $cmake_exe --version # configure - $cmake_exe \ - -C "${hostconfig_path}" \ - -DCMAKE_INSTALL_PREFIX=${install_dir} \ - "${project_dir}" - + if [[ "${CI_COMMIT_BRANCH}" == "main" ]] + then + # redirect caliper files to release directory + sundials_version=$(cd ${project_dir}; git describe --abbrev=0) + $cmake_exe \ + -C "${hostconfig_path}" \ + -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DSUNDIALS_CALIPER_OUTPUT_DIR="${CALIPER_DIR}/Release/${hostname}/${sundials_version}" \ + "${project_dir}" + + else + $cmake_exe \ + -C "${hostconfig_path}" \ + -DCMAKE_INSTALL_PREFIX=${install_dir} \ + "${project_dir}" + fi + # build VERBOSE_BUILD=${VERBOSE_BUILD:-"OFF"} if [[ "${VERBOSE_BUILD}" == "ON" ]]; then From 1a0cb8d0d73c8aab663ef5698ce39aea9f320175 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 9 Aug 2023 16:42:01 -0700 Subject: [PATCH 179/187] add caliper dir variable to lassen-jobs.yml --- .gitlab/lassen-jobs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 92362a3d2a..1ffca71d46 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -38,7 +38,8 @@ lassen_gcc_cuda_bench: matrix: - COMPILER_SPEC: gcc@8.3.1 CUDA_SPEC: [cuda@11.8.0] + CALIPER_DIR: /usr/workspace/sundials/califiles variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=lsf caliper-dir=/usr/workspace/sundials/califiles ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=lsf caliper-dir=${CALIPER_DIR} ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" extends: .lassen_build_and_bench From 7ac627695eb858338f0ec61efe9f274e3fa319aa Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Wed, 9 Aug 2023 16:42:46 -0700 Subject: [PATCH 180/187] remove debug variables --- .gitlab/build_and_test.sh | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 9e0924ad1c..b86afc8877 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -1,14 +1,5 @@ #!/usr/bin/bash -CI_COMMIT_BRANCH=main -BUILD_JOBS=32 - -COMPILER_SPEC=rocmcc@5.5.0 -AMDGPU_TARGET=gfx906 -CALIPER_DIR=/g/g20/pan13/test-release/ - -SPEC="%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +benchmarks+profiling+caliper+adiak+rocm+mpi+profile-examples" - # make sure lmod is loaded if test -e /usr/share/lmod/lmod/init/bash then @@ -106,9 +97,9 @@ then mkdir -p ${spack_user_cache} fi - if [[ -d /usr/workspace/pan13 ]] + if [[ -d /usr/workspace/sundials ]] then - upstream="/usr/workspace/pan13/spack_installs/${spack_prefix}/${hostname}" + upstream="/usr/workspace/sundials/spack_installs/${spack_prefix}/${hostname}" mkdir -p "${upstream}" upstream_opt="--upstream=${upstream}" fi From adac1f86f1887671e5b1111c27dedf62b2ea8661 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 10 Aug 2023 16:15:10 -0700 Subject: [PATCH 181/187] compare tests on the same machine --- test/compare_benchmarks.py | 15 +++++++-------- test/compare_tests.py | 6 ++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/test/compare_benchmarks.py b/test/compare_benchmarks.py index 2472c6d47c..644cec3e5a 100644 --- a/test/compare_benchmarks.py +++ b/test/compare_benchmarks.py @@ -53,31 +53,30 @@ def main(): return 0 def process_benchmark(jobID, isRelease, releaseDir, benchmarkDir): - # Get the current benchmark run benchmarkFiles = glob.glob("%s/*.cali" % benchmarkDir) + # Don't compare if the run didn't include this benchmark + if (len(benchmarkFiles) == 0): + return + th_files = tt.Thicket.from_caliperreader(benchmarkFiles) curFilter = lambda x: x['job_id'] == jobID th_current = th_files.filter_metadata(curFilter) - # Don't compare if the run didn't include this benchmark - if (th_current.dataframe.size == 0): - return - # Get the release caliper file + cluster = th_current.metadata['cluster'].values[0] if isRelease: # Get the last release - versionDirs = glob.glob("%s/*" % releaseDir) + versionDirs = glob.glob("%s/%s/*" % (releaseDir, cluster)) versionDirs.sort(key=os.path.getmtime, reverse=True) versionDir = versionDirs[1] else: # Get the release the run is a part of version = th_current.metadata['sundials_version'].values[0] - versionDir = "%s/%s" % (releaseDir, version) + versionDir = "%s/%s/%s" % (releaseDir, cluster, version) benchmarkName = th_current.metadata['env.TEST_NAME'].values[0] releaseFile = glob.glob("%s/Benchmarking/*/%s/*.cali" % (versionDir, benchmarkName), recursive=True) th_compare = tt.Thicket.from_caliperreader(releaseFile) - metrics = ['Max time/rank'] tt.mean(th_current, columns=metrics) tt.mean(th_compare, columns=metrics) diff --git a/test/compare_tests.py b/test/compare_tests.py index 8c93965036..3c74e120db 100644 --- a/test/compare_tests.py +++ b/test/compare_tests.py @@ -37,15 +37,17 @@ def main(): runFile = glob.glob(runDir)[0] th_temp = tt.Thicket.from_caliperreader(runFile) + cluster = th_temp.metadata['cluster'] + # get machine from the file if release: # Compare against the last release - versionDirs = glob.glob("%s/*" % releaseDir) + versionDirs = glob.glob("%s/%s/*" % (releaseDir, cluster)) versionDirs.sort(key=os.path.getmtime, reverse=True) versionDir = versionDirs[1] else: # Compare against the release the run is a part of version = th_temp.metadata['sundials_version'].values[0] - versionDir = "%s/%s" % (releaseDir, version) + versionDir = "%s/%s/%s" % (releaseDir, cluster, version) # Gather files to process runFiles = glob.glob("%s/*.cali" % (runDir)) From 1f7f37ec6a4528d194d012f0d6680b7326dd9666 Mon Sep 17 00:00:00 2001 From: Yu Pan Date: Thu, 10 Aug 2023 17:08:34 -0700 Subject: [PATCH 182/187] rename compare_tests.py to compare_examples.py --- test/{compare_tests.py => compare_examples.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{compare_tests.py => compare_examples.py} (100%) diff --git a/test/compare_tests.py b/test/compare_examples.py similarity index 100% rename from test/compare_tests.py rename to test/compare_examples.py From f12f5431ed8fe61083fada0d8c0368fdfefc1ebf Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 13 Sep 2023 10:31:50 -0700 Subject: [PATCH 183/187] clear notebook outputs --- test/benchmark_analysis.ipynb | 46 +++++------------------------------ 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/test/benchmark_analysis.ipynb b/test/benchmark_analysis.ipynb index 3813490634..9dccfc010d 100644 --- a/test/benchmark_analysis.ipynb +++ b/test/benchmark_analysis.ipynb @@ -2,22 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import os\n", "import subprocess\n", @@ -37,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -48,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -60,30 +47,9 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABa8AAADMCAYAAAB0pV/DAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAACMSElEQVR4nO3dd3hb1f0G8PdqWrLlvbezgeydQJqkJQ00BMpoacIepZSmkAYoe5XZAiGMskfKXg2bH6OBBMiChCSMbG8njveQJVnz/v64kqxlx3Jkydd+P8+jx7bulXR0x7H96tzvEURRFEFERERERERERERENIAoYt0AIiIiIiIiIiIiIqJADK+JiIiIiIiIiIiIaMBheE1EREREREREREREAw7DayIiIiIiIiIiIiIacBheExEREREREREREdGAw/CaiIiIiIiIiIiIiAYchtdERERERERERERENOAwvCYiIiIiIiIiIiKiAUcV6wYQERER0dAkiiIcDgecTmesm0Iyp1QqoVKpIAhCrJtCRERERBHE8JqIiIiIos5ms6G2thZmsznWTaFBQq/XIycnBxqNJtZNISIiIqIIEURRFGPdCCIiIiIaOlwuF/bv3w+lUomMjAxoNBqOmKU+E0URNpsNDQ0NcDqdGDlyJBQKVkckIiIiGgw48pqIiIiIospms8HlcqGgoAB6vT7WzaFBQKfTQa1Wo7KyEjabDXFxcbFuEhERERFFAIckEBEREVFMcHQsRRKPJyIiIqLBh3/hEREREREREREREdGAMyjKhrhcLhw6dAgGg4H1EomIiIgGOE/ZEKfTCafTGevm0CDhdDrhcrnQ0dEBm80W6+YQERERUTdEUYTRaERubu4Rr54bFOH1oUOHUFBQEOtmEBEREVEvFBUV4cknn4TFYol1UwaNadOm4f7778e8efNi3ZSI+eCDD7By5Up8+eWXvX5MY2MjFi1ahMrKyn5sGRERERFFQnV1NfLz83tcZ1CE1waDAYD0hhMTE2PcGiKSM6vDiR9r2uB0iShM1SMrMQ4KBa/oICKKJJvNhrq6OhQXF8tqYr2LL74YL774Ii677DI8/vjjfsv++te/4oknnsD555+P559/vt/acMcdd+D999/Htm3b/O6vqalBSkoKtFptv722h8ViQVZWFr7//nuMGDGi317nhx9+gFKpxKRJk3q1fmdnJyoqKrB161ZoNJp+axcRERERHZ329nYUFBR4M92eDIrw2lMqJDExkeE1EYXF4XThh4Nt2FTahI2ljdha0QKrw+VdrlUpUJSmR1FaPErS41GUpkdJWjyK0uORw2CbiKhPOjs70dDQAKVSCaVSGevm9JogCCgoKMAbb7yBVatWQafTAZDez2uvvYbCwkIIgtCv78lzWWXga+Tl5fXbawb64osvUFRUhNGjRwcts9lsEQuOu3uv3VEqlVAoFEhISJDVhyJEREREQ1Vvyj9zwkYiGlJcLhE/H2rDs1+X4eLV32HiPz7HGY9vxP2f7sWGA02wOlxIT9CiJD0eKoUAq8OFfXUd+HxXHZ7+qgw3vfMTlj67Bcff9wXG3PoJFqxcj0v/sxV3f7QLL2+uxIYDjahpMcPpEmP9VomIZEMURZhtjpjcRDG8/nry5MkoKCjAmjVrvPetWbMGhYWFQSOEP/nkE5xwwglITk5GWloaTjnlFJSWlnqXv/jii0hISMD+/fu9911xxRUYM2YMzGZz0GuvXr0ad9xxB3bu3AlBECAIAlavXg1A+sP/3XffBQBUVFRAEAS8+eabmDNnDnQ6HaZNm4Z9+/bhu+++w9SpU5GQkICTTz4ZDQ0Nfq/x7LPP4phjjkFcXBzGjBkTNMIcAN577z2ceuqpAIDbb78dEydOxLPPPouSkhJvaHyk9+5p45o1azB//nzo9XpMmDABmzZt6nbbNzQ0YOrUqTj99NNhtVq7XY+IiIiIBo9BMfJ6KGvvtOO+/9uDTIMWmYY46Wui9H16ggYqJT+foKFNFEWUNnRgY2kTNh5owubyJrSa7X7rJOnUmDUsDbNHpGHWsDSMyEyAIAiwO1041GpBeaMJlU1mVDSZUOH+vqrZDJvDhf31Hdhf3xH0uhqVAoWpehSn6VHsHqnt+T43WQclR2wTEXlZ7E4ce+unMXntXf9YCL0mvD+JL774Yrzwwgs455xzAADPP/88LrroIqxbt85vPZPJhBUrVmD8+PHo6OjArbfeitNPPx07duyAQqHA+eefjw8//BDnnHMONm7ciE8//RTPPvssNm3aBL1eH/S6Z599Nn766Sd88skn+N///gcASEpK6radt912G1atWoXCwkJcfPHFWLp0KQwGAx5++GHo9Xr8/ve/x6233oonnngCAPDKK6/g1ltvxWOPPYZJkyZh+/bt+OMf/4j4+HhccMEFAKSJ0j/88ENvUA4ABw4cwH//+1+sWbPGO0r6SO/d46abbsIDDzyAkSNH4qabbsKSJUtw4MABqFT++6S6uhoLFizAzJkz8dxzz8lqxD4RERER9R3Da5mrbe3Eq1uqQi4TBCAtXoMMT6jtE2z7fp9h0CJOzX8AaPCobjZjY2mjFFiXNqHB6D86K16jxPSSVMweno5Zw9NwbE5iyPIfaqUCRWnxKEqLD1rmcLpwqLUT5U0mVDaZUNHoDrebTKh2B9sH6jtwIFSwrVSgIFWH4rR4FLtDbU9ZEgbbREQD37nnnosbbrjBOynghg0b8PrrrweF12eeeabfz88//zwyMjKwa9cujB07FgDw1FNPYfz48bjyyiuxZs0a3H777ZgyZUrI19XpdEhISIBKpUJ2dvYR23nNNddg4cKFAICrrroKS5Yswdq1a3H88ccDAC655BLvyG1ACrsffPBBnHHGGQCAkpIS7Nq1C0899ZQ3vN68eTMAYMaMGd7H2Ww2vPjii8jIyAjrvXvauGjRIgBSPe/jjjsOBw4cwJgxY7zr7N27FwsWLMDpp5+OVatW9eryUiIiIiIaHBhey1ySTo0rfzUSDcZO1LdbUW+0ot7YicYOG5wuEY0dNjR22LC79sjPExhuZxi0yEz0Db7jkKDlIUMDz+G2Tmwqa8TGA03YVNaEmhaL33KtSoGpxSnesHpcXhLUR3lVgkqpQGGaHoVpegAZfsscThdq2zq9I7UrmszuryZUN1tgc7pQ2mBCaYMp6HnVSgEFqe7R2ml6d53teJSkxSM3OY5XUxDRoKRTK7HrHwtj9trhysjIwKJFi7B69WqIoohFixYhPT09aL39+/fj1ltvxZYtW9DY2AiXS5pToaqqyhvgpqSk4LnnnsPChQsxe/ZsXH/99Uf3hnyMHz/e+31WVhYAYNy4cX731dfXA5BGSpeWluKSSy7BH//4R+86DofDb3T3e++9h1NOOcVv9HRRUZFfcA307r0HtjEnJwcAUF9f7w2vLRYL5syZg6VLl2LVqlV92xBEREREJFtMImUuOykOKxaMCrrf6RLRbLKh3tiJeqMVDe1W7/d17dLX+nYrGoxW2JwutFnsaLPYQ5Y/8KXXKL0lSjIStSHLlWQatEjWqzkqhvpNs8mGzWVN3tHVZQEhsEohYGJBMmYPT8Os4emYVJgc1asLVEoFClL1KEjVY85I/3/mnS4Rh1otqGwyS6O23aF2RZMZVU1m2JwulDWYgt4TIL2vAncpEr8JJNPjkZesY7BNRLIlCELYpTti7eKLL8ayZcsAAP/+979DrrN48WIUFRXhmWeeQW5uLlwuF8aOHQubzea33ldffQWlUona2lqYTKZezbreG2q12vu95++ywPs8oXJHh/Q34DPPPOM3qhrwnzDx/fffx3333ee3PD4++Aql3r73UG30tAkAtFotTjzxRHz44Ye49tprozoxJRERERHFnrz+S6BeUyoEZLhHTx/Xw3qiKKLNYveG2Z6A2/f7BqMV9e2dMNmcMNuc0ijSpuBJhHxplArv62catMhKDC5VkpmoRVq8liUS6IjaO+34tqwZG0ulkdW7a9v9lgsCMC4vCbOGp2H28HRMLUpB/AC9SkDpDqALUvU4YaT/KD2nS0RtmzvYbnSXI3GP2q50lyIpbzShvNEEwH+CLZVCQH6Kzl2GxF2KxP19foruqEeaExGRv5NOOgk2mw2CIHhLc/hqamrC3r178cwzz2DOnDkAgG+++SZovY0bN+Kf//wnPvjgA1x33XVYtmwZ/vOf/3T7uhqNBk6nM3JvxC0rKwu5ubkoKyvz1vIOtH//flRWVmLBggU9Pldv33tvKBQKvPTSS1i6dCnmz5+PdevWITc3t0/PRURERETy06d059///jfuv/9+HD58GBMmTMCjjz6K6dOnd7v+qlWr8MQTT6Cqqgrp6ek466yzcO+993pnI7/99ttxxx13+D1m9OjR2LNnT1+aR2EQBAHJeg2S9RqMyup5lI/J6nAH2+6A212ipMGnXEm90YpWsx02pwsHWy042Grp8TkVApCeEFCL26BFRkC5kowELTQqhm9DhcXmxNbKZm/N6h9rWuES/dcZnWVwh9VpmFGShiS9OvSTyYhSISA/RY/8FD2OH+EfbLtcImrbO1HZaHLX2e4qRVLZZIbV4fL5YKkhxPPqvKG2J+AuSpNCdAbbREThUyqV2L17t/f7QCkpKUhLS8PTTz+NnJwcVFVVBZUEMRqNOO+883DllVfi5JNPRn5+PqZNm4bFixfjrLPOCvm6xcXFKC8vx44dO5Cfnw+DwQCtVhuR93THHXfgyiuvRFJSEk466SRYrVZs3boVLS0tWLFiBd577z2ceOKJISeTDPe9h0OpVOKVV17BkiVL8Mtf/hLr1q3rVc1vIiIiIpK/sMPrN954AytWrMCTTz6JGTNmYNWqVVi4cCH27t2LzMzMoPVfffVVXH/99Xj++ecxe/Zs7Nu3DxdeeCEEQcDKlSu96x133HHeWdMBBM0wTrEXr1WhRKtCSXrwpaG+Ou1OabS20SrV4g45qtuKJpMVLhHeIBxo7/F5U/RqKeBO9IzoDi5Xkpmold1lxwRYHU7sqGr1jqzeXtUCu9M/rS5Jj/eG1TOHpSE9ITL/qMuFQiEgL1mHvGQdZocItg+3d3qDbE+o7ZlE0upwobLJjMomM9YHPK/S/bz+E0dKXwtS9PzQiIioB4mJid0uUygUeP3113HllVdi7NixGD16NB555BHMmzfPu85VV12F+Ph43HPPPQCketT33HMP/vSnP2HWrFkhS2SceeaZWLNmDebPn4/W1la88MILuPDCCyPyfi699FLo9Xrcf//9uPbaaxEfH49x48Zh+fLlAKR6156JG3vSm/ceLpVKhddeew1nn322N8AO9b8HEREREQ0ugiiK4pFX6zJjxgxMmzYNjz32GACpJl1BQQH++te/hhxRsWzZMuzevRtr16713nf11Vdjy5Yt3ssHb7/9drz77rvYsWNHn95Ee3s7kpKS0NbW1uM/ETSwOJwuNJls3ZYrkWp1d6KhwxoUZPbEoFX1WI9bCr/jkBinYl3uGHE4XfjpUDs2ljZiU2kTvqtoRqfd5bdOblIcZg1Pd9etTkNusq5/G9VaDVR8DVR8A1RuAFwuILUESB0m3dKGS19TigF1P7clglwuEfVGq7cMiVRn2+wNui327i89VwhAnnfEtv8EkgWpOmhV0asjTkSDS2dnJ8rLy1FSUuK9Eo8GtsbGRuTk5KCmpsY7+eNAw+OKiIiISB7CyXLDGqJqs9mwbds23HDDDd77FAoFTjzxRGzatCnkY2bPno2XX34Z3377LaZPn46ysjJ8/PHHOO+88/zW279/P3JzcxEXF4dZs2bh3nvvRWFhYTjNG5ocNqDiKyAhGzDkAPpUqQCwDKiUCmQlxiErMQ5AUrfruVwiWi121Bs7UdfeVbakwVOqxKdsSafdBaPVAWODI+SEd760KkVQuZLMxDhvnW7PKO9UvQYK1uU+Ki6XiD2HjdhU1oRNpY3YUtYMo9Xht056gqYrrB6WhqI0ff9+uNBWIwXV5V9LoXVrZYh1qoDywLHKABLzuwm2SwBNz5dSR5tCISA7KQ7ZSXGYNTzNb5koBgTbjWa/OtsWuxPVzRZUN1vw9f5G/+cVgNxkd7CdrvcG3MXpUikSBttERINLc3MzVq5cOWCDayIiIiIanMIKrxsbG+F0OoP+aM3Kyuq2PvXSpUvR2NiIE044AaIowuFw4PLLL8eNN97oXWfGjBlYvXo1Ro8ejdraWtxxxx2YM2cOfvrpp5CzrVutVlitVu/P7e09l5sY1NprgJfP7PpZqQESsgBDtnRLcH815HTdZ8gBdCmyCbkVCgGp8RqkxmswpofyhqIowmh1eEdvN/QwCaWx0wGrw+UN5nqiUgg+dbmlUdueEdxZBikUHJNtgIp1g71EUURZo0kqA+IeXd1itvutkxinwsxhUhmQ2SPSMTIzoX/D6vZDXUF1xTdAS7n/ckEJ5E4CSuYARScAWgPQXAY0l7q/lgFNZYC1TTrv2muk5wpkyAFShweH2yklgDah/95fHwiC4P0Aaeaw4GC7wRtsm911trtKkZhtTtS0WFDTYsE3BwKfF8hN0gWE2lJZkoJUPeLUDLaJiORm1KhRGDVqVKybQURERERDTL8XB163bh3uuecePP7445gxYwYOHDiAq666CnfeeSduueUWAMDJJ5/sXX/8+PGYMWMGioqK8Oabb+KSSy4Jes577703aILHIcveCWSNBYy1gLkJcNqAtmrp1hOlxifYzvYPtn1DbxmF3IIgIDFOjcQ4NUZk9hwSWmzOrpHbPpNQ1vkG30Yrmk02ONz1hA+3d3b7fAWpOvxxzjD8bkoBdJqhGcxVN5uxyV2zemNpI+rarX7L9RolppekukdWp+PY3EQo+3NEe3utFFJXuAPr5jL/5YJCCquLTwCK5wCFM6XA2lfhDP+fRREwN3eF2X7BdinQ2Sqdi8ZaoPKb4DYlZLmD7WFd4bYn2I4bWCWPBEFAZmIcMhPjMCNUsN1h9QbZvqF2RaMJJpvTO2HrhgNNAc8rBdtF3okjPXW241HIYJuIiIiIiIiIfIRV89pms0Gv1+Ptt9/Gb3/7W+/9F1xwAVpbW/Hee+8FPWbOnDmYOXMm7r//fu99L7/8Mi677DJ0dHRAoQg9WnXatGk48cQTce+99wYtCzXyuqCggDWvHTagow4wHpbCs446d5B22OdWC1iae/+cSi1gyAoOtf1GcmcDccmyCbnDYXO40Nhh9Qu4vRNRusuVVDSZYOyUSmCkxWtw4exinD+rGEl6dYxb37/q2zuloPpAEzaWNQaNYNeoFJhSmOIeWZ2G8fnJUPfn6HTjYZ+w+hugKXA4sALImeAOq38hhdWRDozNzUBzeXCw3VwmfbjUk/gMd6jtE257ypHEdV9WZ6ARRRGNHTZ3GZLgUdsdAeVifAkCkJMYhyKfkdq+wfZQ/WCISC6qmsz462vfw+4UkRqvQUq8Bql6tfQ1XoMUvQZp7vsNahHt9QcxbNgw1iamiGHNayIiIiJ56Lea1xqNBlOmTMHatWu94bXL5cLatWuxbNmykI8xm81BAbVSKQUQ3eXmHR0dKC0tDaqL7aHVaqHVasNp+tCg0gDJBdKtJw6rf8ht9Am5O3xD7hbAaQVaq6Rbj68d5y5XkhM8ktv3/rgkWYXcGpUCucm6HicLtNiceHNrNZ75ugw1LRY8+Pk+PLG+FEunF+KSOSXISZLP5H49aTHZsKW8CRtLpduB+g6/5SqFgAkFyd4JFicXpvTvKFpjnTS62VO3uml/wAqCT1g9Byia1f8hsD5VuuVPCV5mafEJtsv9w21TQ9etekuI500LCLY95UiGSVdHDCCCICDDoEWGQYupxal+y0RRRJPJ5ldf2xNwVzSaYLQ6cKitE4fapA9GAuUkxUkjtn3C7eL0eBSlxjPYJhoA7vxoF3bWtPVq3TyDErfPz4Szzgi11gaVQoBSoXB/Fbq+KoPv52TLRERERERDR9hlQ1asWIELLrgAU6dOxfTp07Fq1SqYTCZcdNFFAIDzzz8feXl53hHTixcvxsqVKzFp0iRv2ZBbbrkFixcv9obY11xzDRYvXoyioiIcOnQIt912G5RKJZYsWRLBt0peKi2QXCjdemLv7N1I7s5WwNEpTXgXatI7v9fW9TCS2+d+baJsQm6dRokLZhfjnBmF+OjHWjyxrhR7Dhvx7Dfl+M+mCvx2Yh7+NHcYRmQG128fyIyddnxX0SyNrC5twu7D7fD9vEkQgONyEzF7eDpmDU/DtOJUJGj7sRJRR4MUVpe7R1Y37g1YQQCyx0lBdckcoHAWoEvuv/aES5cC5KUAeZODl3W2+QTbAbeOOmnUtrkJqPku9PN6A+3h/nW2B1jZH0GQ6senJ2gxpSg42G422byTRXonjnQH3MZOB2rbOlHb1onNZcFXj2QlaoPqaxenx6MoTQ+9pt8rZBENed+WN+PzXXVQKgQ8+LsJECGi2WRHi8mGZrNN+mqyocVsQ7PJDoXCCQBwiSJsDhdsAABnr15LCrEV3jBbpRCgVArdBuBKBt5ERERERLIV9n/0Z599NhoaGnDrrbfi8OHDmDhxIj755BPvJI5VVVV+I61vvvlmCIKAm2++GQcPHkRGRgYWL16Mu+++27tOTU0NlixZgqamJmRkZOCEE07A5s2bkZGREYG3SH2mjgNSiqRbT+yW3o3k7mwDHBagpUK69fja+t6N5NYaBkw4p1IqcNrEPJw6IRfr9jXgyXWl2FLejLe21eCtbTVYcGwW/jxvOCYXDqyRsh6ddie2VbZgY2kjNpY24YeaNjhd/ldHjMpK8IbVM0pSkazX9F+DTI3uMiDuW8Pu4HWyxklBdfEJUlitTw1eJ0yHTYfxfd33cIpOFCYWotBQiGRtcv8GH3FJQO5E6RbI2hE61G4u67pC4uA26RbqeUMF26nDgPj0AXPuAFKwnZagRVqCFlOK/M8RURTRYrZ7a2pXNJndZUikYLu904G6dqlm/Zby4GA706D1C7SL0+K9I7jj+/MDF6IhQhRF3P2x1EefPa0Av52Ud8THmM0WlJeXoyAtHkqNFk6XCIdLhNPlkr46pZ+l+0Q4XC7v7ySnS4TT1bugGwAE+ITZSiEg3Fb43e+5TyGAgTcRERER0QAQVs3rgSqcOikUQ3ZL14ht31DbbyT3YcDau0uOAQDq+N7V5A6ciC9Kvq9qwZPrSvHZrjrvfdNLUvHnucMxb3RGTP8xtjlc2FnT6h5Z3YjtVa2wOV1+6xSl6d1lQNIxc1gqMg39WD/S1ARUbuiqWV2/K3idrLE+ZUBmH3VYLYoiDnYcxNa6rdh6eCu21m3FwY6DQesZ1AYUJBagyFCEgsQCFBoKUZhYiAJDAdLi0mK3H22m7kdstwe/Dz/axK5JIwPD7YTMARVsH0mLyeaeONLsLkNiQrk74G4123t8bIZBixJPmO0bbKfH9++VBESDyAc7D+Gvr22HXqPEumvn9ep3RV9qE4ui6BNye8JtlzfoDhWAO/v4Z64g+IfcUsAdoqyJTwCu6M9JiKlXWPOaiIiISB7CyXIZXstck6UJF396MTL0GcjSZyFDl4FMfab0vV76Pk2XBrVCRpMH2sw+4XZAyO17v7W998+pSeh+JLdv6K1N6Je3dKC+A09/VYp3th+E3SmdcmOyDbh87nCcMj4Hqv6cyNDN6RLx08E2aZLF0iZ8V94Mi91/5Fp2Yhxmj0jzjq7O66HW91EzN7vDavfI6rqfgtfJPFYKqotPAIqOB+LTjuolRVFElbHKG1RvrduKw6bDfusoBSXGpI5BvDoeVcaqoOWB9Cq9d4S252uBoQCFiYXI0MXwAwq7+yqHptLgYLutBkAPXb8moftg25Atq2C71WzzjtT2TiDpDrhbjhBspydoUZIuTRoZOGrbECejPpWoH1kdTpy4cj2qmy3424mjcNWJI3v1uGiFjC5P4O30CbV9A3Bn16huz32uPv5prPANvJWKoJDbG3Qru8qZKI6yPxUEAe+8847fROpyt3r1aixfvhytra1hP5bhNREREZE8MLweQnY17cLZH57d4zoCBKTp0pChy/ALtf1uukwkaZPkdYmszdS7kdw2Y++fU2MIGMntG3j7BN+a+D41ubbNgue/KcerW6pgsknBcV6yDn+cU4KzpxVGdNI5l0vEvnqjt2b1lvImGDsdfuukxWswc3gaZg+XAuviNH3/HQOWFqByY1fN6rqfEBSgZhwjBdUlc9xhdfpRvaQoiihrK/OG1dvqtqHB0uC3jkpQ4bj04zA1ayqmZk/FpMxJiFd37d9ORydqjDWoMlah2liNqvYq7/eHOg5B7CEE1ql0yDfk+4/YdofcmfpMKIT+/9AiJLu7Rn1QsF0qBduiq/vHqvXuIDtEuG3IARQxek990OYpRdJkQoV7AskKd63tZpOtx8emJ2jcobZ/sF2czmCbhpZnvy7DXR/tRqZBi3XXzut1jfmBHDK6Akdwe0Purvu+3bIZf1j8a5ww/0Q8tvrNHn8X9CSofIl3kkr/+++96x/44IP3sWP7dr/f04cPH0ZKSkpUJjK3WCxIT0/Hzp07MWLEiH57HYbXRERERIMfw+shxGQ34afGn1BvrvfeGiwNqDPXocHcgAZzAxyi48hPBECj0HSN4PYE3Dop3Pa9X6fqx9G4/cHaEWKyydqAOt2HAVtH759Tm+gOtrOBxFzgmFOB0b/pdXDXZrbjpc0VeGFDBZrcIVlqvAYXzCrG+bOKkBIffi1pURRR0WT21qzeXNrkfW4PQ5wKM4eluUuBpGFUpqH/LnO2tAJVm9xh9dfA4R8RFFanj+6qWV10ApBwdHXuXaIL+1v2e4PqbXXb0NzpXwNZrVBjXPo4TM2eiqlZUzEhYwL0an2fXs/mtKGmowbV7dWoMlahqt0dcBurcKjjEJxi9zVZNQoNCgwF3nIknjIkhYmFyNZnQ6mI3AcZYXFYgdYqKcwODLdbq4Ae3hNUcUBKiTRZZGC4nZgnr2DbYu+aNLLR5C1LUtFoCjqvAqXFa/zKkPjW205ksE2DSJvZjl/c/yXaLHb888xxOHvaESaC9iH3kPHSSy9FQkICnnvuOezZswfZOTk91uqWRn773x+OJ1behy8//Qhvffo1lIqAUd3esDt0AK4QIjNh5fvvv4/rr78eu3YFl/Wy2WzQaCIzDwbDayIiIqLBj+E1eblEF1o6W7rCbYs74DZ3Bdz15nq0WFt6/ZwGjcGvRInn5lu6JE2XBpVCZvVircauySaDwm6fkNtuCv34jDHACSuAsWcCyl6OPLM78dbWajz9dRmqmy0AAL1GiT9MK8Slc0qQe4SyHQdbLdh4oBGbSpuwqawJtW2dfst1aiWmlaS6R1an4bjcJCj7K6zubAMqN7lrVn8N1P6A4LB6lLtmtbtudULmUb2k0+XEnpY92HZ4G7bWbcX39d+jLaBmepwyDhMyJmBK9hRMzZqK8RnjoVX2/wg1u9OOQ6ZDfiO1PeF2jbGmxw+VVAoV8hPyg8qRFBoKkZOQE7tzy2ED2qq7CbYrAVcPH5QptUBKsTvY9h25PRxIygdiFdb3QXunHZWNZnegbUK5z6jtxo6eg+1Ud7At1dmWRmoXu0dwJ+kZbJO83PPxbjz9VRlGZxnw8VVzwvr9IueQsaOjAzk5Odi6dStuu+02jB8/HjfeeCMAYOnSpXA6nXjjjTe869vtduTk5GDlypU4//zzYTQacfnll+Pdd99FYmIilq+4Gh988AHGjhuPf9x3f1AN77deewk3Lr/Crw3/ePDfOO33SzGhIAUPPfMyfnnSIhysrsJvZk/Avx5/Hq+98DR2/bADI0Yfg3sfewaWjnb84/qrUX5gH6bNmo1HnngGmZlZULknqnz5Py/gsYdXoaKiHMXFxbjyyitxxRX+r3nJJZcgIyMD9913H26//Xa8++67WLZsGe6++25UVlbC5XLhk08+wV133YWffvoJSqUSs2bNwsMPP4zhw4cDACoqKlBSUoL//ve/ePTRR7FlyxaMHDkSTz75JGbNmgUgOLxuaGjAySefjIKCArz++us9jjKX83FFREREvdNhdWDDgUas39eA9Xsb0NhhxeTCFMwcloYZw1IxsSAZcWr5/H85VIWT5cosXaRwKQQF0nRpSNOl4Zi0Y7pdz+a0ocHS4A21/QJuS4M3/LY4LDDajDDajDjQeqDH102PSw8qURJYuiRRkzhwSpVoDdIt/QiXwlqNPmF2HXD4B2DbaqBhD/DOZcCXdwMnLAcmLAXUPf/jFKdW4rxZxVgyvRAf/3QYT6wrxe7adjy/oRwvbqrAaRPzcPncYRiZJU042WC0YlNZEza5R1dXNpn9nk+jVGByUbK3ZvWE/GRoVP002rWzHajaDFR8JZUBqd0ZXG4ibURXUF18gjRS/SjYXXbsbtrtnWBxe/12dNj9R8zrVDpMypzkLQMyNm0s1Mroh4JqpRpFiUUoSiwKWuZwOVBrqu0asW2s8n5fbayG3WVHRXsFKtorgh6rElTITcgNHrFtKEReQl7/vleVRgqf04YDIxf4L3Pau4Lt5nL/cLulAnBagca90i2QQi0F26nDAsLt4UBSQa8/DIqWxDg1xuUnYVx+UtAyY6ddGqEdOIFkoxmNHVY0m2xoNtmwvao16LEpejWK0uJRku6eNNJn1HayPjIjGokipbrZjNUbKgAA1/9mjBRciyJw4H9Sf6BPkybV1acBcclHvvJCFAG7ued1+otaH1Yt/zfffBNjxozB6NGjce6552L58uW44YYbIAgCzjnnHPzud79DR0cHEhKkeTQ+/fRTmM1mnH766QCAFStWYMOGDXj//feRlZWFW2+9FT/s2I6pkyeFnOxy+WUXoqm6FJ988gn+79PP4HSJiE9IhMYd4ibr1UhP0MKok/r/Jx+6DzfccS8yc/Jxy9XLcP1fLoU+IQHX3n4P4nR6/P3PF+Guf9yBm+9dCQD46J038dBdt+P6u/6FMceNx56ff8CNf78KbXYFzvrDOe7a3MD7H3yI1a++gaYOKzrtThw4cABvvv023njrbWjUUj9tMpmwYsUKjB8/Hh0dHbj11ltx+umnY8eOHVD4HAM33XQTHnjgAYwcORI33XQTlixZggMHDkCl8u/vq6ursWDBAsycORPPPfcclEr+I0pERDTUiKKIfXUdWLe3Huv2NmBrZbN3LjGPTWXSgD4A0KgUmFSQ7A2zJxemMMyWuYGVCFDMaJQa5CXkIS8hr9t1RFFEh73DP+C2NKDO5B9wN1oa4RSd0ihvSz1+bvq52+eMU8b1WKLEE3pHY6Rsr3lDbvekVON/B/ziGuC7Z4FNj0sjUD/8G7Dun8DsZcCUi444EaRKqcCpE3KxeHwOvtrfiCfWHcDmsmb89/sa/Pf7Ghw/Ig0NRiv21fkHtUqFgPH5Sd6a1VOK+rFTthrdYfXXUimQ2h3BYXXqcP+wOjHnqF7S5rTh56afvTWrt9dvh8Vh8VsnQZ2AyVmTpbA6ayrGpI0Z8BOUqhQqqWSIoQCzMdtvmdPlRJ25zr8Mic/obavT6g28N2CD32MVggI58Tn+o7XdX/MMef17HinVXWVCAjkdQHtNV5jd5DNiu6UccNqApv3SbX/AYxUqILkoINh235ILpdcdQAxxaozNS8LYvOBgu8PqkEZou0dtV7gnkKxoMqHeaEWL2Y4Wcyt2VLcGPTbZHWwXe0PtrhHbfSkzRHS07v90L2xOF44fkYZ5o9wln7a9IP3+CyQopABbn9Z1SywGchYDpkbApQPsNuChMdF8C11uPBTWXBbPPfcczj33XADASSedhLa2Nqxfvx7z5s3DwoULER8fj3feeQfnnXceAODVV1/FqaeeCoPBAKPRiP/85z949dVX8atf/QoA8MILLyA3N7fb19PpdEhISIBKpUJBXvB6yXoNcpN1sLmv1rrxur/jkiVnAAA6rv4bzjlnKT7+5DOcMHeeNHnzhRfhtVdeQlq8Fg6XC0+u/Ceuv+NunHzKaXC4ROQXFqFs31688dLzWHTmHwAAO7ZugUsUkTdqPA62WtBmscNqs+HGfz6GuDRpfoqfD7bhuOMXQOUua5KWKuCeh/6NcSMKsXHrDowbNw5mq3SFzt9WrMBvfvMbCIKAO+64A8cddxwOHDiAMWO6joG9e/diwYIFOP3007Fq1aqBM9iBiIiI+p2x044NBxqxbm8D1u9rCLrKvDhNj3mjMzF3VAZyk3X4rqIZW8qbsbmsCQ1GK7aUSz9jrTTIb2JBMmYOS8WMYWmYXJgS0fnGqP8xvKZeEwQBBo0BBo0Bw5JDBFRuTpcTLdYWKeA2+dfg9i1d0mZtQ6ezE9XGalQbq3t87SRtknfUdmCJksx4KfhOjUuNXZ3guCRgztXAjD8D218CNjwihXWf3Qx8/SAw43Jg+mXSKLQeCIKAuaMyMHdUBnZUt+LJdaX4dNdhbDjQ5F4OHJuT6A2rpxan9N/kcNYOoHpz1wSLh7YH1ztOKXFPsPgLaYLFpO4//OjVSzqt+KHhB29YvbNhJ6xOq986iZpETMma4h1ZPTpldOz2ez9QKpTITchFbkIuZubM9FvmEl2oN9cHBdqe7y0OCw52HMTBjoPYVLvJ77ECBGTHZ/sF2p5JJPMN+f1by16pkkZWpxQDw3/pv8zlBNoP+pcgafIJth2d0kSSzaXAgc/9HysopQDbE2b7htvJRdJI8QEkQavCcblJOC43ONg2WR3eILuiyYTKRjPK3WVJ6tqtaDXb0Wpuxc4QwXaSTo3iNL27DIn/BJIpejUDH4q4H2pa8f7OQxAE4IaTj5GOsc524Iu7pRXSR0kfSpmbAWu79EGnpVm6Nbk/oUooAFLnAqZ6wCoAdkv3LziA7N27F99++y3eeecdAIBKpcLZZ5+N5557DvPmzYNKpcLvf/97vPLKKzjvvPNgMpnw3nvv4fXXXwcAlJWVwW63Y/r06d7nTEpKwujRoyPWxvHjx3u/z8mRrniaOnmi9++FYYV5aG5sQF6KDiaTCVUVZbh5xTLces2V3sc5HA4kJiWhOD0eTqeIZ9Z/joUnnYx0QxycLhFqpQJ5+QXIzMiE0+WCCMApiijbfwCPP3APftyxDa3NzXC5a3vv3HMAibnDcLBZGl2fkDMcPx5sg0ohoEMhfcC/c18lEjIL0Wq2wWKx4IQT5uB3Z/8B9/zrQdgcLiiVApQRqt9NREREA4soithda8S6ffVYv7cB2ypb4HB1ja7WqhSYNVwaNDFvdCaK0/0HHozONuDcmUUQRRHljSZsLmvGlvImbC5rQl27Fd9WNOPbimbgiwNQKwVMyE/GjGGpmDksDVOKUno96TjFBvcORZxSoUS6Lh3punQcl3Zct+tZnVZveZJ6c33IgLveXA+r04o2axvarG09lipRCkqk6dL86nFnxQfX5k5QJ/TfPz4aPTDjT9Jo6x/eAL55SArd1t0LbHwUmHoRMGtZr8pnTCxIxpPnTUFpQwc+31WH4jQ9ZpSk9d8oS5vJPbL6G3dY/X1wDePkIvcEi+6R1Un5R/WSZrsZOxt2esuA/Nj4I+wuu986qXGpmJI1xRtYj0wZCYUgn4n/IkkhKJAdn43s+GxMy57mt0wURTRaGoMmjvQE2ya7CbWmWtSaarHl8Jag587UZ6LQUIiixCLvxJGFBqkkSV8ntOzdm3IH0MmFwLB5/stcLsB4KCDYLpXKkjSXAQ6LFHC3lAOla/0fKyikkiPdBdtHKOkTbfFaFY7NTcSxucG1vsw2h3eyyIoms7sMiTRq+3B7J9osduysacPOmragxybGqVCcLtXXLgkIuFPjNQyBKGyiKOLuj3YDAE6fmNd1lcE3DwHmRiBtJPDnjV1XRThsgKUFMDdJN0uz+6vFfSVTkvTXqEoHXLpW+kCrp4lhj0ShkvoVhQoQVIBSKf0sqNzL3DdB0bVuGH3cc889B4fD4TdSWhRFaLVaPPbYY0hKSsI555yDuXPnor6+Hp9//jl0Oh1OOumkvr+nMKnVXR9qe87xwPs8oXJHh3RF1zPPPIMZM2b4PY9SqfROMvu/Tz7Cfffdh/wUaVulxmuQnGjAsbmJEEWpPrfTJeKsX52DgsJCPPrvJ5GRnQ2Hw4n5s6ZCDRf0GpW3lJnK3R6HS4TNfdlvh9WOZrMNxk4H1Botph3/C3z44Yc4/YLLkZUjbW8BwRNSeiarVCoEuBw2dNqd2HfYiNQkF1LjNdCplezriIiIBqA2i2d0dT3W72tAXbv/wLVh6fGYO1oKq2eUpPbqKnNBEDAsIwHDMhKwdEYhRFFEZZPZHWRLI7Nr2zqxtbIFWytb8O8vS6FyX9E+Y1gaZg5Lw9SiFMRrGZcOJNwbFDNapdZbOqE7oiii3dbuH3D7lCjxhN+Nne5SJe77eqJT6bz1tzP1mUElSjzLNMqjCIlVGmDyecDEpcCu94CvVwJ1P0oB9pangUnnAsdfKY1EPYLhGQkYPrfnsiN9YjMD1VvcYfXXwMFtIcLqQndQPQcoPl76+Sh02DqwvX47ttZtxba6bfi58eegiQszdBneUdVTs6aiJKmE/3T2giAIyNBnIEOfgSlZU/yWiaKI5s7moEC7ur0alcZKGG1G77mztW5r0HOn69JDjtguNBQiQdMPx6aHQiF9QJKUL43u939TUu355tLQo7btJqmET2slUPZlwBML7mC7JDjcTikG1P04Cr0P9BoVjslJxDE5oYPtqmaztxSJb7Bd29aJ9k4Hfqhpww8hgm1DnArFaVJ97RJ3wO0ZtZ3GYJu6sXZ3PbaUN0OjUuDqhe7Rwq3VwObHpe8X/MO/nI9KAxiypJuvzk6gvFy6YidwYj3RJYXYLofPraefQwTeogsQbUBAdauQfANv7y3wZxUcLhEvvvgiHnzgAfx64UK/p/jtb3+L1157DZdffjlmz56NgoICvPHGG/i///s//O53v/OGx8OGDYNarcZ3332HwkLpd2pbWxv27duHX/ziF0FN89BoNHA6jyLU70ZWVhZyc3NRVlaGc845J+Q6+/fvR2VlJRYsWBByuSBIYXJbaxP279uL5559BnPmzAEAfPPNNwCAzMQ4jMhMgMos/c4YmZmAY3IS4XSJaFRLfwekJ2iQnRiHhDgVlAoFHnnqeVx52UX44x9OxfNvfoj0rGyIkCaxdHSzX0WHDY0dNtz+wTYcNErbS6tSIDVegxS9Rvoar0GqXi199b3fu1wNrWrwXNFFREQUKzaHCyarAyabA2abU/re6sTOmlas21uP76ta4fQZXa1TKzF7eBrmjc7A3FGZKEw7+kFUgiBIg3fS43H2NCnMrm62YLN7VPaWsmYcbLXg+6pWfF/ViifWlUKpEDAuL8k7MntqUT9e8U69wvCaBjRBEJCkTUKSNgkjUrqfSNHhcqC5s9kv1PYLuN2lS4w2IywOCyrbK1HZXtnja6doU4JC7cDa3KlxqT2PAlYogbFnAMedDuz/DPjqAaDmW2Drc9Ikj+N+B5zwNyAzCnU+7RafsPoboGYrEDDKGUkFXaOqi08AUoInGwxHu60d39d97y0Dsrt5N1wBdbKz47O99aqnZk9FoaGQoVmECYLgnbh1YubEoOWtna1BE0d6Qu5WaysaLY1otDTi+/rvgx6bGpfqnTDSt852gaEASdrg8hgRfFNSTfXEHOlY9SWKQEd9iGDbPWrbZgTaqqRb+frg507M7ybYLpGurhhA9BoVxmQnYkx2cLBtsTmlYNtdX7vCPXq7ssmEQ22dMHY68OPBNvx4MESwrVWhKF3vHrHtnkDSXYokPYHB9lDlcLpw7/9Jo64vPr4Eee4ay/jiTqnET9EJwOiTj/6FBAWgVIRX0z5k4H2E0Nvz+8jzM6w9vsSHn3yJlpZmXLJoGpKSRb+Q+8xFC/Dc00/i8nNPBxQqLP39mXjyicexb/8BfLn2c6lfEgQYDAZccMEFuPbaa5GamorMzEzcdtttUCgUPZ5XxcXFKC8vx44dO5Cfnw+DwQCtNjLzGNxxxx248sorkZSUhJNOOglWqxVbt25FS0sLVqxYgffeew8nnngi9Pqe+7+UlBSkpaXh6aefRk5ODqqqqnD99deHXFcQBKiVCqiVUkklAEjSaZCZGIcUvQaCAIzISsT7/30DS5YswRXn/BZffPElMjKz4HCJcLpc7hBbGvHtcIlwOkVYrU5olAIyErRosHTC5nDB6nChtq0zqFZmT+I1Sr9wO80TentDbrVfGJ6sU0OlHJpXhRERkfyJogirw9UVMNukkNns+9XmhNnq+/UI69gcQZMqhjIiMwFzR2Vg3ugMTCvu3ejqoyEIAgrT9ChM0+P3U6WBlNXNZm+97C3lTahutmBHtTQX0VPry6AQgLF5SdIEkCWpmFaS6r06jaKD4TUNCiqFyhsu98TisHSVJvHcLF0juD2lS2wuG1qsLWixtmBfy77uX1eQXndewTyce8y5KEjsZhS5IACjFgIjfw1UbpDqYJd+AfzwunQbc4pUMztv8tFsBn92C1DznRRUl38NHNwq1SD1lZjXFVaXzJHKKRxFKNXS2YJtdduwrW4bttZtxd7mvRDh/wsrPyEfU7OnesuA5CXkMQiLseS4ZCTHJWN8xvigZW3WNtQYa/xHbLvrbDd1NqG5sxnNnc3Y2bAz6LFJ2iRv6ZHAciTJ2uT+2++C0DXSs8h/QkyIojRBXHNZ6FHb1japXn17jXRFQiBDjjQxaWC4nVJyxIlZo02nUWJ0tgGjsw1Byzrt7mC70eSus232TiZ5qM0Co9WBnw6246eD7UGPTdCqpDA7INQuTtcjI0HL83kQe2NrNUobTEjRq3HF/OHSnYe2S2WyAGDhXUf1O+SoRCHwfu6193DiCTOQlGjwCbwlZ/56Nv71yJP4YfM6jD92FM45eSbu/ucDKMrPwfHDE6VJjt0juFfe9Gdc3lqPU05ZhESDAX9f/hdUV5YjTiVIJby8ZU+U3u155plnYs2aNZg/fz5aW1vxwgsv4MILL4zIprv00kuh1+tx//3349prr0V8fDzGjRuH5cuXAwDee+89XHDBBUd8HoVCgddffx1XXnklxo4di9GjR+ORRx7BvHnz+tw2lUqF1157DWeffTZOPPFXWLduHTIzMwGE/se2s1MBR1scXv/TLGi1WphtTjSbbGgx23y+2tFisqHZbJO++t5vtsHpEmGyOWGyWVDT0vta7Ek6tTvcVocY6e0Jv7tC78Q4NRQK9pdERBQeURTRaXdJo5mtTnRYHSGD48BgWVpPCqfNNqf38Z5R0b4joCNNq1IgXquCXqNEvEaFwjS9d66vgtSjHBzkdEjlJLXB//P0VkGqHgWpepw1RSqLerDVgi1l7pHZ5c2obDJ7r2Z9+ispzD42NxEzS9IwY1gaphenIknPMLs/CaIo9t8RGiXt7e1ISkpCW1sbEhODR58RhUMURbRZ24JCbb+A29KAJkuTXzArQMAvC3+JC467ABMzJh45wDn4PfDNSmD3B133Df+lFGIXHR9+AGDvlAJqzwSLNd8BAZMdwpAjhdUl7sA6peSogoZGS6O3XvW2um0ha5IXJxZLQbW7DEh2/JHrfZM8dNg6vKVI/CaRbK9GvaXn8j0GtaGr/EjAiO20uLTYBKCiKE0wFzLYLgU6W3t+fEKWO9ge1hVue4LtOPn8buq0O1HdbA6qr13eaMKhNgt6+qshXqN019X2H7Vdkh6PDAODbTnrsDow7/51aOyw4rbFx+Ki40ukc2b1KUDlN8D4s4Eznu7183V2dqK8vBwlJSWICywbMpCJrl6UMelmhHc3TGYL8qYsxIO3rsAlS37rv7DbMibd/OwTeEdCY2MjcnJyUFNTg6ysrCM/IMaO9rgSRRHtnY7uw22TDU3unz3rtJrtR37iEBQCjhhwJ+nUULrreisV0oSVSoUAhe/3gmc5oFQooBQEKBTwrq/w/equE67wPhbsl4mI+pHLJcJi9w+KTdaA4DhU4OwJmX3CZd/QuT9TPJ1aiXitEnqNO2z2CZ31WiUStCroNSrEa5TQawO+alTexyZopfX1amXkr05yWIGydVKJ1j0fSf+nZY8Hhs+X5lEqnBXRUpC1bRZsKWv2htnljSa/5YIAHJOdKI3MHpaKGSWpSNb301xlg0g4WS7Da6I+srvsaLI0YV/LPry25zV8c/Ab77Jx6eNw/rHn48SiE6FSHOECh/o90kRXP77VVbOzYKYUYo9c0P0/oQ6rVPrDU7O6+tvgsDohuyuoLp4jhWlH8U9KnalOCqvdgXVFe0XQOsOThnuD6ilZU5Chz+jz65F8me1mVBur/epse74/bDrc42P1Kr03yC5KLPKO3i5MLESGLiN2/2ibm7smi/QNt5tKpUnoehKf4Q61Q4Tbcf1YXiXCrA53sO2usS3V2XYH260W9DRgQ6dW+tXXLvEE3OnxyGSwPeCt/HwfHlm7H8Vpenz2t7nSxHt7PgZeXwKo4oBlW7Ff7MSytcvgcDmQHJeMFG2KdGWHNhkpcSlI1rq/16YgUZEINEs1oPW6gVWOJ+ICAu/t33+PPXv2YPrkCWhrbcE//vUQ1n2zGQe2fIr0lMReBd49imDgvW/fPnz66af461//2vf2RFEsPhRxOF1os9i9AbffSG/fENwshd8tJhuMVseRnzhKFAL8Q3BBgFLpH3xLgTmC7vPcfB/rCc4VQteEmopuQ3d34B7iuX1Dd//ngU9gL/h9Hxzuw++1VQFBfqh2+4f7od9j1/aB9zX5O4xI/pwuEebAoLiX5TFMVs+6/uuZbZGfr8KXJ1wODI69X93BckJAAB0fGEy7v+o1KigH6lVBdot09fqu94C9/wdYg68Q9VLFAYUzgWHzpUA7a5w0n1KE1LV3YnOZNAHklvImlDUEh9mjswyYOSwNM4elYnpJGlLjGWYHYnhNFAMHWg7g5d0v44PSD2BzSeU5cuJzcM4x5+CMkWfAoDnCZSwtFcCGh4HtL3eV98gaB8xZARx7mvRP78FtUlDtCasdAfUbE7K6guriOVIwdhR/TB/sOOgdVb21biuqjdV+ywUIGJUyyhtWT86ajNS41D6/Hg0NnY5ObykSvxHbxmoc6jgUVGrGl06lQ74h33/Etvv7TH1mzzXo+5OlxSfYDriZGnp+rD6tqwSJN9weBqQNA3Qp0Wl/BFgdTtS0WPzqa3sC7oMtvQu2i9PiUZSud4/YlkZwZxnieGl9jNW1d2Le/etgsTvx+DmT8ZtxOYDTDjw+E2g6AJywAuKvbsWln12Kbw9/26vnzNHk4LoR1yEzPxNKjRIqQQWlQgmloIRKoYJSUAb97Ht/zM71CNi+fTsuvfRS7N27FxqNBlOmTMHKlSsxbty4rpVcLkDszQhvn/v6HHgLAcF2L0LvCI/wjhS5jOi3OVxoNUvBthRy29FssnpLl3gC8DaLHQ6nCJco1fZ2iiJc3q/w3ud01/52+fzs8n6N9bsdGgTBP4BXKQJDcvgF58oQwbj0WASH8gGj6H3XDQ73Q4XuXaPylQqEHLkfHO4jINz3D+4DR/37vrYq5Icf/oG/SqHw+6DD93WIesPhdPWtPEbASGbf+zrtR/HB8REIAvwC48BwOSh89q4XevRzvEYFnVo5+M8ZmwnY/7kUWO//DLB1dC1LyAaOPVXKSlKHSxlJ6ZdA2ZeAsdb/efTpwLC5XWF2Un5Em1nf3ulTM7sZB+o7gtYZnWXwTgA5vSQV6QmRmbtEzhheE8VQk6UJb+59E6/vfR3NndJozHh1PM4YeQbOOeYc5CXk9fwExsPApseA754H7O5P8BLzpFGfjoDai/GZXZMrlvwCSBvR538mRVFEtbHaO6p6a91W1Jr8O32FoMCY1DHeCRYnZ03u30n5aMixOW2o6ajpmjjSZ8T2oY5DcIrdj17QKDQoMBR4y5H41tnO1mdDqejfyT+61dnWfbDdUdfzY3UpoYPt1GGAPnVAhkeh2Bwu1LR4Jo80+9XZrmmx9FhjL06tQFGqFGQX+4TaxWnxyE5ksB0N1//3B7z+XTUmFybjv3+eLY0w/PYZ4ONrpH8GrtyO9Q3bseyLZdAoNHjsV49BFEW0WFvQam1FS2fX1zZrG1qsLVA71bgs/zJk5GdAoQ4/iFYICm+Q7RtqhwzBB0Hg3SvewDuMsiaDLPCWS3gdTaIoBdgOl0sKvEX/oNsVEIB3Bd/wC8F7Dsm71nV4lgeF7Z77AKfLBacLXc/tCg7oHX6vg160V3ruwNd2+C4PCPV9nyPwuX0/JHC4XPwQIIqCw30EB/2CTxmcgNA9sFxO6HA/xIcEPVwhEBTuB3wQ4f3gQenzIUGoDyh8r0gICPQDX7unKxv8tk+oDyMGWEkgm8PVzYR/PmFzqPIYAeGzb21nm6P/gmaFAPdoZv9RylIZjJ5HOfsH012Pj1P3PDEz+bAagX2fugPrz/0zkMR8Kaw+9lQgf3ro0dSiCDTslULs0i+lK9Xt/iOjkTbSXWJkvpSlRLjEY4PRim/LpVHZm8uasK8uOMwemZngDbNnlKQhwzD0wmyG10QDQKejEx+VfYQXd72IsrYyANI/2ycWnojzjzsfEzIm9PwE5mbg26eBzU901drVp3dNrlg8B0gfdVRhdXlbubcMyLbD24LqFKsEFY5NP9Y7ueKkzElHHkFO1E/sTjsOmQ4FTRxZbaxGjbEGDrH7y7BVChXyE/L96mt7Rm3nJOQcubxPf7EaA4Lt0q6fA0cMBIpL8g+zfcPt+HRZBdsHWy3ekdqeMiSVTSZUHyHY1qoU3hHbxenu+tpp8ShKj0cOg+2I2HvYiJMf/gouEfjvn2dhSlGq9IHMI5MAcxOw6EHYp1yAM98/E+Vt5bh47MX425S/HfF5PSFjYVEh1Fo1nC4nHKIDTpcTTtEJh8sBpxjwvUv6vq8UgsIv4PYNursLvgf9P5qDLPBmeE39xfMhQKig3TcAl4JvBI2ADwrJfUfQ+9zn8AvYQ3ygIIpwOl1dQX03H0Z0fYiA0O0N9cGBz7qer44Qz+37YUKoDxH82x38wQlFh0JA1wj3kGF7YEgeHOh3H+77r6tQCLDa/QNqs60rbLY7+2+/qxRCV9mMkMFyL2sz+4xu1qoYNEedpRXY9wmw633gwP/8y6EmF7kD698CeZPD/x/HYZPmA/OE2Ye+9/9bRqEC8qZ2hdl5UwBlZP83bOrwhNnS6Ow9h41B67y/7HiMz0+O6OsOdAyviQYQl+jCxkMb8eLPL2JT7Sbv/RMzJuL8487HLwt+2fOIUKtR+rQwpRjIGNPnQMolunCg9YB3VPW2um3ekeEeaoUa49LHeSdYnJgxEXr1IK9FSoOCw+VAram2a8S2e+JIT8htd3U/oZZKUCE3ITd4xLahEHkJeVArYzRztM3U/Yjt9oM9P1ab2FVXOzDYTsiUTbBtd7pwsMWC8iYTKj3lSNwBd3WzGY4e/gnWqBQoStUH1dcuStMjN0nHYLuXLnzhW6zb24CTx2bjiXOnSHd+fhuwYZX0AeqfN+H1/W/j7i13I0Wbgo/O+KhXH3L2NWSUAiSXX9DdXfDtG3r3lbd8iSfcPkI5kyETeLsc3Zc1cTqCl0cp8O602VFeWc3wmmgAC1Xexj/cR4iQXwrSg0Ny9PA8wVcIeJ/H5/mDw3331QGex4QcuS/C4fR9nd6P+vdbt4dR/57XdjhdXVcK+KwrxxRHo1QEj1YOUT4j1KjnUOUz4rUqaQ4OkidzM7D3Y2mEdemXgO//a6nDgeN+CxxzKpAzIbL/u1hagPKvu8LslnL/5dpEaaCgJ8w+ylKsoTSbbD4js5tR2WTCjlt/PeSOZ4bXRAPU3ua9eGnXS/io/CPv6LG8hDyce8y5OH3k6YhXx0fstZwuJ/a17POWAfm+/nu0Wlv91tEqtRifMd5bBmR8xnjEqfjPHg0uTpcTdeY6/zIkPqO3rYETnfpQCArkxOf4j9Z2f80z5EGrjNHlXXZLiGDbPWq7rQbooW44NAkBwbZPuG3Ilk2w7XC6R2z71tdulILtql4E24WpehR762zHu+ts65GbrBu4E9VE2YYDjTjn2S1QKQR8vmIuStLjgdYq4NGp0oiYJW/AWHI8Fq1ZhBZrC26ccSOWjFnSq+eO5ghZURS9IXaPo7oDAvG+ClW7u6ea3kMq8PbcRGfowNt3WR8C706HiPKDjSjZfg/ixE5pTgF9asDXNEAXcJ/WIJu+j4gIcP9uCxxF382of7+SPmFeIeA76j+4pE/XY+PUyqCazL7lM3Qa5ZAL5oYcUQTsZmkAjq0DsHa4v3f/bHP/bG0HKjcC5V9Jv+89Msa4R1ifBmQeG73fyy0VXbWyy9Z3XfXukVQADJsnhdkl84D4tIg3wWxzQK+J0ZXAMcTwmmiAazA34PW9r+ONvW+gzdoGADCoDThr1FlYesxSZMdnh/2cDpcDu5t2e8uAbK/bDqPd/3IUnUqHiRkTvRMsjk0fC42Ss97S0OUSXag31wcF2p7vLYF15n0IEJAdn41CQyEKEgtQZCjyjt7ON+RDp9JF8Z34sHcCrZVAU2lwuN1W03MgpNa7w+wQo7YNORGdpbs/OZwuHGrt9E4Y2VVn24TqZnOPl69qlAoUpOq8pUiK07pGbQ+lYNvlErH4sW/w86F2XDi7GLefepy04L+XAj++JY1IueADrPr+YTz303MoTizGmtPWQK3o3ZUKA728g2/gHbKcSYj7XX0eXYywy5kohCFwSbPLGTyy2zf0dgYv67Q7UX6wASUbrkZcR/WRX8NDoQ4It1O6vve9X5/aFXwz8CYiIrkSRcDRGSJo9gmZQy4L+BoYUPc0gCaUrHFdNawzRof10AMtB/BZ5Wf4rOIzNFgaMCVrCmblzsLs3NkoNBT27e8klxOo3eEOs9cBVZv9R4RDAHLGd038WDATUA+8v2PlguE1kUxYHBZ8UPoBXtr1EiraKwBIlyj/uvjXuODYC3Bc+nHdPtbutOPnpp+9I6u312+H2WH2WydeHY9JmZOkkdXZU3Fs2rG9DhaIhjpRFNFoaQyaONITbJsCJ/4IkKnPDDliu8BQELtyPA6rNHI2VLDdWtVzsK2KA1JKpEvnAsPtxDzZBNtOl4hDrRbvSG3PxJHljSZUN1tgc3a/DdRKAQWp7hrb7okji9KkUdu5yXFQKeWxDXpjzfc1WPHmThi0Kqz/+3ykxmuAg9uAZ34JQAD+tB6HDBlY/M5i2Fw2PDL/EcwvnN/r5x/o4XVfuESX3whv34C7uwC8r4G3IAghw+1uQ/AhEnh3mk0oLy9DicGJOHuLVJfd3Cx9tbi/eu9rBsyN0j/vfREYePsG26FGfOtSGXgTEVH4RBFw2tyjlo2hRzN7RzX3JoR2/yz2/SqzI9IkAJr44K9a99e0EVJJkLThYT1taWspPq34FJ9VfIbSttJu18tLyPMG2dOzpyNJm9S392EzAZWbukqM1P/sv1wVBxTN7gqzM4+Tzf9EAwHDayKZcYkufF3zNV7c9SK+Pfyt9/7JmZNxwXEXYG7+XDhEB35o+MFbr3pn/U50Ov3/4TJoDJiSOcU7snp06ujYTURHNIiJoojmzuagQLu6vRqVxkoYbcGTcPhK16V7g+yixK4R24WGQiRoEqL0LgI4bEBbtRRm+4XbpUBLZc9/4Cq1Ul3+tOEBI7eHA0n5Ul1aGfAE25VN5qA621VN5iMH2yl6FKX519cuSY9HXrJOVsF2p92JXz6wDofaOvH3k0bjinkjpH+cXvgNULURmLAEOP1JXPfVdfi4/GNMy56G5379XFjB6GAMr/vCJbq6H9UdWOrE/f3RBt69KWfiuV8hyOe4Bfp4XNnM3QTbTd2E3k1HGXh3E2x7Q+80QO8e+R2X7D95pUIJCAoG4EREA5nT3nPZjHBHM9s6/MtrRJpa333IHCqEPtIylS6iAW5payk+q/gMn1Z86hdYqxQqHJ97PH5d/GsUJRbhu8PfYeOhjdhev91vgm+FoMDY9LGYnTsbs3JmYVzGuL4P6DPWSSOyPWF2x2H/5fEZUokRT5idmNu31xkiGF4Tydiupl14addL+KT8EzhEqdPN1GWi1doKm8vmt26KNsU7ueLUrKkYmTJSdv9oEg1GrZ2tQRNHekLuwNrzgVLjUr0TRnrKkRQmSkF3n0cNHC2n3SfYDpg8sqUi4HK6AAq1FGynDgsIt4dLNeQiPJt3f3G6RNS2ScG276jtikYTKpvNsDm6DxRVCmnEdlGaZ9S23ltnOy9FB/UAC7afWFeKf36yB7lJcfjimnmIUyuB3R8Cb5wjjTD56/f4yd6CJR9J9a3fOOUNHJt2bFivwfC67zyBd2/LmThFJ/r6575CUHQF2p5w+wjBdyz/DonacWUzBwTbgV99g++jDLxDEkIE2kopLOjxPvf9Ct+vAd+HfI6Ax/k+n0IR4j7PeoqA+5RS8B50nyK4nd0+R0/vNeA9hHxsL9vEDwqIhganA7D3dsTykUYzu0dFO21Hft2+UsV1EyR7QuSeQugQy9T6ATnIpKy1TBphXfkZDrQe8N6vUqgwO3c2FhYvxLyCeUjUBOd/ZrsZW+u2YtOhTdh4aCPK2sr8lser4zE9ezpm587G7NzZKDAU9O3KNFEEGvZ01cuu2CAdS77SR3dN/Fh8vHQFFnkxvCYaBOpMdXh1z6t4a99b3lGcaXFp3qB6atZUDEsexrCaSGbarG2oMdb4j9g2VqOyvRLNnc09PjZJm+QdsR1YjiRZmxybkgBOB9Be0xVm+4bbLeU9/wGvUAHJRQHBtvuWXAgo5VHmyOUScbi9syvQ9pk8sqLJBGsPwbZSISA/RecNtaU629Ko7YJUfdSD7WaTDXP/9SWMVgce/N0EnDklXxqV//hMaRT+nGsg/vJmXPjJhfi+/nssHrYY98y5J+zXGSrh9bx58zBx4kSsWrUqos+7bt06zJ8/Hy0tLUhOTu5xXVEUu0qa9FDOJPDnowq8exjNHVjmxFPSBAAuvPBCtLa24t133+3Taw/o48oTeHuD7ZbgEd2e7y3NgKlRmhiVYqw3HxR0E8Dzg4IQHwr09B74QQEdgcsVHB4facTykZZF9IPFAEptL4LkUKOZu1mmjpfNoI++KGstw6eVUkmQUIH1r4t+jfmF80MG1j05bDrsDbI3124OGkiUl5DnDbKn50wP+/m9HDag5tuuMPvQdv+SjAoVkD+ta1R27uRBvT97g+E10SBitpuxrW4b8g35KE4sHvT1KomGsg5bh7cUid8kku3VqLfU9/hYg9rgLT/iKUfiGbGdFpcWm77D5QTaD/oE26VAc3lXsN3TPwyCUgqwPWG2b7idXASo5DHZrMslos7YiXJPmN0oTRzpCbY77T0H23nJOu/Ekb51tgtS9NCoIh9s3/7+z1i9sQLH5iTiw7+eAIVCALY8Bfzf36VLIa/cjrWHv8XydcuhVWrx4ekf9mmS4QEdMvbgwgsvxH/+8x/86U9/wpNPPum37C9/+Qsef/xxXHDBBVi9ejUAoLm5GWq1GgZD30fahArAbTYbmpubkZWV1S/ntm/g7XA5sH79elx64aXYuX9njzW9+0ohKKBSqHD9X66Hsd2IF157occAXCkoQ75vuR5XIYmiNE+B6J64UnRKwY3oCrjP6b7PFeI+z2N81veu5wpxn8/6IZ8v8L5ItCng+/5qpyiGuK/vk6xSfxhkHxQo3Pf3+YOCbt6DXD4ocLkAuzkCo5l9ltvNR37dvlKouoLjbkczx0sjZ7sb6Rz4OJkMwoilsrYybw3rwMB6Vs4s7wjrSF196hJd2N282xtm91RiZHbubIxNH9v3EiOWFqD8q64wu6XCf7k2CSiZI5UZGf5L6X+cIZb19Ht4/e9//xv3338/Dh8+jAkTJuDRRx/F9OnTu11/1apVeOKJJ1BVVYX09HScddZZuPfee0P+UXnffffhhhtuwFVXXdXrUSoMr4mIaLAz282oNlb71dn2fH/YdLjHx+pVem+QXWgolOpsu0dvZ+gyYhRsuwDjIf8SJL7htsPS/WMFhVRyxHektifcTi6SzazfLpeIeqPVHWyb3HW2zd5w22LvPgxUCECed8R2vDfgLkqLR2Fq34LtikYTTly5Hg6XiJcvmYETRqYDllbgkUnSSNBTHoJ90nn47Xu/RZWxCn8c90dcOfnKPr13uYaMF154Ib744gu0t7ejtrYWOp0OgPR+cnJykJiYiPnz53vD60jor9Hb4VixYgU6Ozvx+OOPBy2z2WzQaDQQRdFvdHdvypkEBt43LbsJxnYjHnnxkSO2KdQklS6bC4erD+Ow9jAS9AlIiUtBsjYZKXEpMGgMvFqN/Imi+xZO8B4QrAcF+REI1AND+7CC/1BtCvXhQm/ft+e5Q30I0osPRnzbRANIP3xQ4OgMCKFNAPppnKSgADSGMELmeGn9nkY6y2RQhNyIoohOZyfMdjPMDjPMdjM67B347vB3+LTiU//AWlBhVu4s/Lr415hfMD8q5RI9JUY2HtqIjYc2oryt3G95gjrBv8RIYkHfX6y5vKtWdvl6oLPNf/mla4H8qX1/fhkKJ8sNe4z6G2+8gRUrVuDJJ5/EjBkzsGrVKixcuBB79+5FZmZm0Pqvvvoqrr/+ejz//POYPXs29u3bhwsvvBCCIGDlypV+63733Xd46qmnMH78+HCbRURENKjp1XqMTh2N0amjg5Z1Ojq9pUh8R2xXtVeh1lQLs8OMPc17sKd5T9BjdSod8g353gkjfcuRZOoz+y/sUSikyRyT8oGSX/gvE0XAWNt9sG03Aa2V0q3sy4AnFtzBdklwsJ1SDKh1/fN++kChEJCdFIfspDjMGp7mt0wU/YPtCu+obTMqm0ww25yobragutmCr/c3+j+vAOQm67yTRnYF3FIpEq1KGbI9//p0DxwuEXNHZUjBNQB8/aAUXGeMASadjzf3vYEqYxVS41JxybhLIrYtRFGEpacPLPqRTqUL6wOcyZMno7S0FGvWrME555wDAFizZg0KCwtRUlLit25g8FxcXIzLLrsMBw4cwFtvvYWUlBTcfPPNuOyyy0K+1oUXXoj169dj/fr1ePjhhwEA5eXlqKio8Csbsnr1aixfvhwvv/wyrr76alRXV+M3v/kNXnzxRbz11lu47bbb0NbWhvPOOw8PPfQQlErpGLBarbjpppvw2muvobW1FWPHjsU///lPzJs3z68d77//Ph577DHvexo7dixUKhVefvlljBs3Dl9++SUeeughvPDCCygrK0NqaioWL16Mf/3rX0hJSAEAbxvfeOMNLF++HNXV1TjhhBPwzHPPIDMrE07RiXh1POxKOzL10s/bvtuG8886H5f+9VJceuWl3vAbgPQ9nIBPJuayu9Bua8ejux5Fra3W7z0oBAWStclI0iYhRdsVaof6mqxNRnJcMgxqA694G8wEwT3KTcERktEQdvDeXcjeU/Af7pUGPYT2Rx38R/Lqhx4+GAm17Y5IdE8A6PDrQ/uHEGJUckCw3O1o5m6WqbRDboRqtNhddpjtZlgcFpjsJpjtZpgc7q92EywOS9B9nlDa8zXwvp4mmlYJKszMnYmFxQujFlj70qv1+EX+L/CLfOl/kdqOWmyq7Sox0mZtwxfVX+CL6i8AHGWJkdQS6Tb1YulcPbQDKPsCKF0HNOwGciZG/P0NJmGH1ytXrsQf//hHXHTRRQCAJ598Eh999BGef/55XH/99UHrb9y4EccffzyWLl0KQPqjfcmSJdiyZYvfeh0dHTjnnHPwzDPP4K677urLeyEiIhqS4lRxGJEyAiNSRgQtszltqOmo6Zo40mfE9qGOQ7A4LNjfsh/7W/YHPVaj0KDAUOAtR+I7Yjtbnw1lf03wIgjS7NyJuUDxCf7LRBHoqJfqL4cKt21GoK1KupWvD37uxDz/EduecDulBNDo++f99IEgCMhKjENWYhxmDgsOthuMVp9A231zj9o225yoabGgpsWCr/cHPi+Qm6RDcbreb9S23enCxz8ehkIAbvzNMdLKLRXAFndpjAV3os1hwhM7nwAA/GXiXxCvjo/Y+7U4LJjx6oyIPV84tizdAr06vH1/8cUX44UXXvCG188//zwuuugirFu37oiPffDBB3HnnXfixhtvxNtvv40///nPmDt3LkaPDv5g6uGHH8a+ffswduxY/OMf/wAAZGRkoKKiImhds9mMRx55BK+//jqMRiPOOOMMnH766UhOTsbHH3+MsrIynHnmmTj++ONx9tlnAwCWLVuGXbt24fXXX0dubi7eeecdnHTSSfjxxx8xcuRIAMDPP/+M+vp6/PKXv/S+1n/+8x/8+c9/xoYNG7z3KRQKPPLIIygpKUFZWRmuuOIK/P3vf/cbrW02m/HAAw/gpZdegkKhwLnnnovr/349XnnlFQCAWqmGWqlGhj4DX3zxBc454xz861//8gv3PSO8Q43q7uzsRIeqA1OypqDSUomWzha0WlvRYe+AS3ShubMZzZ3NKIf/yKruqASVFHaHCLdDfZ8SlwK9Ss/AmygUhQL8oCBK+r2cT4hAXR3nHzp7Amq1jkFzP3GJLnQ6OmF2mLuC5h7CZE/47Fk31H02V/9NNqlT6aBX6RGvjkdRYhEWFC3ALwt/GbsJ6UPIScjBGSPPwBkjz4DT5cSe5j3eUdk76nfgYMdBvLXvLby17y0oBAXGpY/zKzGiUvQyYlUogfwp0u0X1wJO+5Cvf30kYW0dm82Gbdu24YYbbvDep1AocOKJJ2LTpk0hHzN79my8/PLL+PbbbzF9+nSUlZXh448/xnnnnee33l/+8hcsWrQIJ5544hHDa6vVCqu1awKT9vb2cN4GERHRkKFRajAsaRiGJQ0LWmZ32nHIdMhv4khPuF1jrIHNZUNpWylK20qDHqtSqJCfkO8dqe2ts20oRE5CTu//eAuXIACGLOlWNNt/mShKk5yFDLbLAGu7VIO7/SBQ8XXwcxtygNThwaO2U0qkEUMDhCAIyEyMQ2ZiHKaXpPotE0URDR1WVDaZu0ZtN3ZNImmyOXGw1YKDrRZsONAU9Ny/m1KA0dkG6Ye1/5Am3Bw2Dxi5AM9uW4k2axuGJw3HGSPPiMI7HbjOPfdc3HDDDaisrAQAbNiwAa+//nqvwuvf/OY3uOKKKwAA1113HR566CF8+eWXIcPrpKQkaDQa6PV6ZGf3XFvcbrfjiSeewPDhwwEAZ511Fl566SXU1dUhISEBxx57LObPn48vv/wSZ599NqqqqvDCCy+gqqoKubm5AIBrrrkGn3zyCV544QXcc480Eed7772HhQsXQqPpuqR65MiR+Ne//uX3+suXL/d+X1xcjLvuuguXX365X3htt9vx5JNPetu4bNkybyjv65133sH555+PZ5991hu0ewiCAJWgCtnHdCo60RHXgdtm3+ZXjsbutKPV2uq9eUJt71drC1o7pa9t1ja0dLbA7DDDITrQ1NmEps7gc6U7aoVaGtkdl4wUbUrI8Nt3eXJcMnSqgXNFCBENAp4PCsIfq0j9yOa0hT1q2TdoDrzP4rBA7KfSLGqFGnq1HvGqeOjVeumm0nvDZ9/74tXxfvfrVLqQ98mtdJdSocRx6cfhuPTj8Mfxf4TJbsLWw1KJkU21m1DeVo6dDTuxs2Enntj5xNGVGOGHekcUVm/W2NgIp9OJrKwsv/uzsrKwZ0/wpcgAsHTpUjQ2NuKEE06AKIpwOBy4/PLLceONN3rXef311/H999/ju+++61U77r33Xtxxxx3hNJ2IiIgCqJVqFCUWoSixKGiZw+VAram2a8S2e+JIT8htd9lR0V6BivaKoMeqBBVyE3K9I7Z9y5HkJeRB3V9/oAkCkJAh3Qpn+i8TRcDc7A60SwOC7VKp7pyxVrpVfhP83AlZ7kA7INxOHQbEDZz5NgRBQKYhDpmGOEwrDg62GztsUn3txq5JIz2jtpN0aqz49Shp5ZqtwE//BSAAC+5ETcdBvLJbGh27YuqKiH84oVPpsGXpliOv2A/6EhxmZGRg0aJFWL16NURRxKJFi5Cent6rx/qWxxMEAdnZ2aiv73lC1t7Q6/XeUBiQ/j4vLi5GQkKC332e1/rxxx/hdDoxatQov+exWq1IS+sa7f/ee+9h2bJlfutMmTIl6PX/97//4d5778WePXvQ3t4Oh8OBzs5OmM1m6PX6kG3MyckJeu9btmzBhx9+iLfffhu//e1vw9wKoXlGc2foM3r9GKvTitbO1qBw2/vV2uq3vKWzBVanFXaXHfWW+iNOsusrThnnH3K7Q+1ETaK3trdCUEg3SF+VCiUECNIyRdf9npvfY3zuEwSh22WB93X3GKXgfm2FsqtNCv+2+baHI9GJSG6cLmfXiOQeRjJ7l/kGyz73eYJqs8PsNzFgJAkQQgfNIe7zhMo9hc96lb7//laXsXh1POYWzMXcgrkAjlxiJD8h3xtkT8uZFl6JEQrS7x/FrVu3Dvfccw8ef/xxzJgxAwcOHMBVV12FO++8E7fccguqq6tx1VVX4fPPP+/1hD033HADVqxY4f25vb0dBQVHUTidiIiI/KgUKqlkiKEAs+E/wtnpcqLOXOdfhsRn9LbVafUG3huwwe+xCkGBnPgcb6DtO2I7z5AHrVLbP29IEID4NOlWMC14ubnZXVM7MNguk2o+d9RJt6oQV5rFZ/gE28O6wu204UDcwLkUUhAEZBi0yDBoMTVEsO1ZB6IIfHqTtGDiUiBnPFatvwZ2lx0zc2ZiTt6cfmlbuKU7Yu3iiy/2hrr//ve/e/04tdr/H0JBEOBydV8P8miet6fX6ujogFKpxLZt27w1sD08gXdtbS22b9+ORYsW+S2Pj/cvGVNRUYFTTjkFf/7zn3H33XcjNTUV33zzDS655BLYbDZveB2qPYFzxw8fPhxpaWl4/vnnsWjRoqDHRItWqUVWfBay4rOOvLKbxWHxC7m9wbfPKO/AENzusqPT2YlOcyfqzHX9+I5ixxOyhwrOfe8PGZIf6THdhOjhvE53Af4R2xbitX3vD/xwoTfvp7ftDevDCfBDBBrcRFGE1WntU8kMv6DZ/bPFYenXeTi0Si3i1fFSGQ2fgNlzn2/A7Bc2BwTMnq/hzt1BkRFYYmR3825viZGd9TtR01GDN/e9iTf3vQmloMTY9LF9KzFCAMIMr9PT06FUKlFX5/+HVV1dXbeXMt5yyy0477zzcOmllwIAxo0bB5PJhMsuuww33XQTtm3bhvr6ekyePNn7GKfTia+++gqPPfYYrFZr0B/UWq0WWm0//XNLREREPVIqlMhNyEVuQi5m5viPcHaJLtSb64MCbc/3FocFBzsO4mDHQWyq9Q+CBQjIjs+WypB4Rm27R2znG/L799J6fap0yw8eUQpLS9dkkYHlSMyNgKlBulWHGDmsT/MZpT3cpxzJMECX0n/vJ0x+//Tsfh+o3gyodMAvb8aO+h34tOJTCBBwzdRr+A+S20knnQSbzQZBELBw4cJ+ex2NRgOnM/Izak2aNAlOpxP19fWYMyf0BxIffPABZs+ejdTU1JDLPbZt2waXy4UHH3wQCoV0WfCbb77Zp3alp6djzZo1mDdvHn7/+9/jzTffjFmAHS6dSgddgg45CTm9Wl8URZgdZr9Q27ekSbutHS7R5b05Rae35rfv/b7LXXDB5XLfBxecLidEuB/jku470nP25nU8y3tzyboIEQ7RAYiAHfaj3czUR75Bdm9G4/dlBH/Q8jCuFOjthxd9DvDdHy6Esw3CuSKhVx+CQODvUEhX9/UqTHYE1Ga2W4JKbljsFpgdZu9kvpGmEBR+o5d7Gt0cGD4HBc3u7xlaDj5KhRROj00fi8vGXwaT3YTvDn+HTYekkdkV7RV+JUYMagOm50glRmblzkKBgYNxjySss0aj0WDKlClYu3at9zI+l8uFtWvXBl1O6GE2m71/xHp4wmhRFPGrX/0KP/74o9/yiy66CGPGjMF1110XFFwTERHRwKUQFMiOz0Z2fDamZfuPcBZFEY2WxqCJIz3BtsluQq2pFrWmWmw5HBwEZ+ozQ47YLjAU9O+oXV0KkJcC5E0OXtbZ5hNsl3Z931QKmOoBc5N0qwlRGk2XEjrYTh0mBemx+AfXYQM+v036fvZfIRpy8MA3fwcAnDbiNIxODa7LPFQplUrs3r3b+31/KS4uxpYtW1BRUYGEhIQjBsm9NWrUKJxzzjk4//zz8eCDD2LSpEloaGjA2rVrMX78eCxatAjvv/8+Tj311CM+14gRI2C32/Hoo49i8eLF2LBhA5588sk+ty0zMxNffPEF5s+fjyVLluD111+HSjX4/tkXBAHx6njEq+ORl5AX6+b0iSiKXcG2Oyz3fO9yubwBd6gQvbuQvK+Bek8hu98yd4DvdDn9vve0r7ev4xfi9/J9+C0L8aGC33bq5fvpbWjnee/9VCKXesk3UO8u1I/0lQK+r3fEKwX6+Drekc+h6jcHjG6OxqSAvqOWdWqdX8Dc3Uhmv/DZva5GoeEHDhS2eHU85hXMw7yCeQCAQx2HvEH25trNaLe1Y23VWqytWgsAKDAU4KF5D/Hv7B6E/VfgihUrcMEFF2Dq1KmYPn06Vq1aBZPJhIsuuggAcP755yMvLw/33nsvAGDx4sVYuXIlJk2a5C0bcsstt2Dx4sVQKpUwGAwYO3as32vEx8cjLS0t6H4iIiKSL0EQvHVnp2T5j3AWRRHNnc1BgXZ1ezUqjZUw2oyoN9ej3lyPrXVbg547XZfuDbI9o7U9IbdBY+i/NxWXBOROlG6BrMaAYLus62djrTSi++A26RbqeX3DbN9wOz69/4Ltrc8BLeVAfCZw/FX4rPIz7GzYCZ1Kh79O+mv/vKaMJSb2f/3Ca665BhdccAGOPfZYWCwWlJeXR+y5X3jhBdx11124+uqrcfDgQaSnp2PmzJk45ZRTYDKZsHbtWqxateqIzzNhwgSsXLkS//znP3HDDTfgF7/4Be69916cf/75fW5bdnY2vvjiC8ybNw/nnHMOXn31VQ5qGYA84ZUS7n3DXRQT3YXovRk9H/RBQw+j9Hv9AUEfw36/Dxi6+/AjjDYd9YcjobZND++xN/zWPfqKUbKmUqj8RykfIWgOus93JLS7fIZSwU6IBp7chFycOepMnDnqTDhdTuxq2uWtl72zfidqO2pl+yF2tAhiYKG5Xnjsscdw//334/Dhw5g4cSIeeeQRzJgxAwAwb948FBcXY/Xq1QAAh8OBu+++Gy+99BIOHjyIjIwMLF68GHfffTeSk5NDPv+8efMwceLEXv2xDEg1r5OSktDW1haVfyKIiIgoulo7W4MmjvSE3K3W1h4fmxqXKoXaPuVIihKLUGAoQJI2RjWpbaaAUiQ+o7bbD/b8WG1i8KSRnnA7IbPvwbalBXhkkvR18cOwTVyK0949DTUdNfjzhD/jiolX9O15Q+js7ER5eTlKSkp6PecJRdeaNWtw8803Y9euXbFuSq/xuCKiWOq3KwjcAX53wb535H44HxD05UOFHtoUp4zrquEcEEh3NzkgJwUkAkx2E/Y07wka2DMUhJPl9im8HmgYXhMREQ1dbdY21Bhr/EdsG6tR2V6J5s7mHh+bpE0KOWK70FCIZG1ybC4VtZmBloqAYNs9arutBj1e861J6D7YNmT3HGx/ehOw6TEg81jg8m/wn90v44GtDyBdl46PTv8ooqVZGDIOfJ999hmsVisWL14c66b0Go8rIiIiInkIJ8sdfMXjiIiIaEhJ0iYhSZuE49KPC1rWYevwliLxm0SyvRr1lnq0Wdvwo/VH/Nj4Y9BjDWqDd6S2J9z2jNhOi0vrv2BboweyjpVugeyd3QTbZUBrNWDrAA7/KN0CqfVASok0WWRguO20At8+La234E602ox46oenAAB/nfTX/q0pTgPSr3/961g3gYiIiIiI4TURERENXgmaBByTdgyOSTsmaJnZbka1sdqvzrbn+8OmwzDajdjVtAu7moLLJuhVem9Nbd/R2oWJhcjQZfRfsK2OAzLHSLdADivQUukTbPuE261VgN0M1P8s3YIIAERg+C+BkSfiqW//CaPNiJEpI3Ha8NP6570QEREREREdAcNrIiIiGpL0aj1Gp44OObN3p6PTW4rEd8R2VXsVak21MDvM2NO8B3ua9wQ9VqfSId+QL4XZ7jrbRYYiFCYWIlOfCYWg6J83pNICGaOkWyCHDWirBppKg4PtlkpAdAJKLbDgTlS1V+H1va8DAK6Zcg0nPyIiIiIiophheE1EREQUIE4VhxEpIzAiZUTQMpvThpqOmq6JI9u7amwfMh2CxWHB/pb92N+yP+ixGoUGBYYCbzkS76jtxEJk67P7LyhWaYC04dItkNMuBdtKLZCUh4e+/BscLgeOzzses/Nm90973AbB1Cs0gPB4IiIiIhp8GF4TERERhUGj1GBY0jAMSxoWtMzutOOQ6ZDfxJGecLvGWAOby4bStlKUtpUGPValUCE/Id9bgsRbZ9tQhJyEHKgU/fRnm1It1b4G8H3d9/hf1f+gEBS4esrV/fN6ANRqNQDAbDZDp9P12+vQ0GI2mwF0HV9EREREJH8Mr4mIiIgiRK1UoyixCEWJRUHLHC4Hak21XSO23RNHekJuu8uOivYKVLRXBD1WJaiQm5AbPGLbUIi8hDyolUcf1rlEFx7Y+gAA4PQRp2Nkysijfs7uKJVKJCcno76+HgCg1+v7r044DXqiKMJsNqO+vh7JyclQKlnqhoiIiGiwYHhNREREFAUqhUoqGWIowGz4l+NwupyoM9f5TxzpM3rb6rR6A+8N2OD3WIWgQE58jjfQ9kwiWZRYhDxDHrRKba/a90n5J/ix8UfoVXosm7QsYu+7O9nZ2QDgDbCJjlZycrL3uCIiIiKiwUEQB0FxuPb2diQlJaGtrQ2JiYmxbg4RERFRxLhEF+rN9UGBtud7i8PS7WMFCMiOz/ZOHOk7ajvfkA+dSirZYXVaceo7p+KQ6RCWTVyGP034U7TeHpxOJ+x2e9RejwYntVrNEddEREREMhFOlsvwmoiIiEimRFFEo6XRf8S2+/sqYxVMdlOPj8/UZ6LQUAiFoMC3h79Fpj4TH57+oTfUJiIiIiIiirRwslyWDSEiIiKSKUEQkKHPQIY+A1OypvgtE0URzZ3NQYF2dXs1Ko2VMNqMqDfXo97cVbbjyklXMrgmIiIiIqIBg+E1ERER0SAkCALSdGlI06VhYuZEv2WiKKLN2uY3cWSCJgGLhy+OTWOJiIiIiIhCYHhNRERENMQIgoDkuGQkxyVjfMb4WDeHiIiIiIgopEERXnvKdre3t8e4JURERERERERERETUHU+G25upGAdFeG00GgEABQUFMW4JERERERERERERER2J0WhEUlJSj+sIYm8i7gHO5XLh0KFDMBgMEAQh1s2Juvb2dhQUFKC6uvqIM3RS5EVj+3MfRxe3NxENNZHq99h/EkUGzyUikgv2VxSOSB8vcj/+5N7+oyGKIoxGI3Jzc6FQKHpcd1CMvFYoFMjPz491M2IuMTFxyB3sA0k0tj/3cXRxexPRUBOpfo/9J1Fk8FwiIrlgf0XhiPTxIvfjT+7t76sjjbj26DnaJiIiIiIiIiIiIiKKAYbXRERERERERERERDTgMLweBLRaLW677TZotdpYN2VIisb25z6OLm5vIhpqItXvsf8kigyeS0QkF+yvKByRPl7kfvzJvf3RMigmbCQiIiIiIiIiIiKiwYUjr4mIiIiIiIiIiIhowGF4TUREREREREREREQDDsNrIiIiIiIiIiIiIhpwGF4TERERERERERER0YDD8JqIiIiIiIiIiIiIBhyG1wNUQ0MDWlpaYt2MISsa25/7OLq4vYloqIlUv8f+kyhyeD4RkRywr6JwRPp4kfvxJ/f2D0SqWDeA/BmNRjz++OP48ssvkZ+fj5tvvhnFxcX47LPPsHv3bsybNw/jx4+HIAixbuqgFI3tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/sHMkEURTHWjaAud9xxB9555x1MmDABmzdvxqmnnorDhw9j+/btiIuLg0ajwUMPPYQZM2bEuqmDUjS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0f0EQaUAoLC8VXX31VFEVRtNlsYm5urnjGGWeI//vf/8Rt27aJEyZMEC+++GLRbDbHuKWDUzS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0fyBheDyCdnZ2iwWAQDx48KDocDlEURVGr1YpfffWVd53169eLY8eOFZuammLVzEErGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39g90nLBxANm/fz8yMzNhNpuhVCpRUVEBnU6HsWPHwuVyAQASExNx+PBhpKamxri1g080tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tH+g4YeMAotFocN5558FqtQIAiouLUV5ejuTkZO8633//PTIyMmLUwsEtGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39g90DK8HkOHDh+OKK66ATqcDAIiiiOTkZIiiCEEQUF5eji+//BInn3xyjFs6OEVj+3MfRxe3NxENNZHq99h/EkUOzycikgP2VRSOSB8vcj/+5N7+gY7h9QCiVCr9PoURBMHvq9FoxPjx47Fw4cKYtG+wi8b25z6OLm5vIhpqItXvsf8kihyeT0QkB+yrKByRPl7kfvzJvf0DnSCKohjrRpDE5XJBFEUoFArvAU7RE43tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/sHOk7YOIAoFAoolcqgA10URYiiiNbWVuzZsydGrRv8orH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb/9Ax7IhA0RVVRX++9//YtSoUcjLy0N2djbS09OhUqm8B//mzZtx7rnnorGxMcatHXyisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3JvvxwwvB4g1qxZg2uvvRbTp0+HRqNBWloasrKykJubi4KCAhQXF+ODDz5AcXFxrJs6KEVj+3MfRxe3NxENNZHq99h/EkUOzycikgP2VRSOSB8vcj/+5N5+OWB4PUDs3r0bv/rVr3D++eejuroae/fuxYEDB7B161Y4HA7o9Xps2bIFf/3rX2Pd1EEpGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssBw+sBQqvVIi0tDUuWLIFC0VWKvK6uDuXl5di1axe2bt2KadOmxbCVg1c0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tlwNBFEUx1o0g4MCBAygtLcX8+fOhVqu9s5R62Gw2xMXFYfv27ZgwYUIMWzo4RWP7cx9HF7c3EQ01ker32H8SRQ7PJyKSA/ZVFI5IHy9yP/7k3n45YHgtA6Ioorq6Gpdeeik+++yzWDdnyInG9uc+ji5ubyIaaiLV77H/JIocnk9EJAfsqygckT5e5H78yb39AwXD6wHK5XL5fVJD0RWN7c99HF3c3kQ01ESq32P/SRQ5PJ+ISA7YV1E4In28yP34k3v7ByJuzQHG5XIBAN555x2sWrXK737PMuo/0dj+3MfRxe1NRENNpPo99p9EkcPziYjkgH0VhSPSx4vcjz+5t38gY3g9wHgO6FdffRXV1dXe+xUKBT+5iYJobH/u4+ji9iaioSZS/R77T6LI4flERHLAvorCEenjRe7Hn9zbP5Bx6w0wgiAAAH788UdMmjQpxq0ZeqKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm//QMbweoBwuVwQRRFKpRKANBvpMcccE+NWDR3R2P7cx9HF7U1EQ02k+j32n0SRw/OJiOSAfRWFI9LHi9yPP7m3Xw44YWOMtbe34/Dhwxg1ahQAFnaPtmhsf+7j6OL2JqKhJlL9HvtPosjh+UREcsC+isIR6eNF7sef3NsvJ9yqMfb2229jzJgxWL58OWpqanigR1k0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tlxNu2RjbsGEDCgsL8fXXX+O0007DSy+9hNbWVgAAB8X3v2hsf+7j6OL2JqKhJlL9HvtPosjh+UREcsC+isIR6eNF7sef3NsvJwyvY+zrr7/GjTfeiNdeew2jRo3CrbfeimuvvRbV1dXeYu+eGUsp8qKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm+/rIgUU3FxceIPP/wgiqIoms1m8amnnhKHDRsmxsfHizfeeKPY3t4e4xYObtHY/tzH0cXtTURDTaT6PfafRJHD84mI5IB9FYUj0seL3I8/ubdfThhex1BdXZ2oUChEo9EoOp1O7/0Oh0O86667xJycHHHSpEniCy+8ELtGDmLR2P7cx9HF7U1EQ02k+j32n0SRw/OJiOSAfRWFI9LHi9yPP7m3X25YNiSGfv75Z+Tn5yMhIQGCIEAURTidTiiVStx00034/PPPMWrUKDz99NOxbuqgFI3tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/vlRhBFVhGPlaamJtTW1mLs2LHeOjiCIHhr4wBAXV0dfvjhByxYsCBWzRy0orH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb7/cMLwmIiIiIiIiIiIiogFHFesGEHDgwAHs27cParUaBQUFKCwshF6vBwA4HA6oVNxN/Ska25/7OLq4vYloqIlUv8f+kyhyeD4RkRywr6JwRPp4kfvxJ/f2ywVHXsfQjh07cMEFF6CsrAzDhw+HVquFRqPBsGHDsHjxYpx11lmxbuKgFo3tz30cXdzeRDTURKrfY/9JFDk8n4hIDthXUTgifbzI/fiTe/vlhuF1jBw8eBALFizAxIkTsXz5cjQ3N6O+vh6lpaX49ttv8cUXX+DUU0/FK6+8Ao1GE+vmDjrR2P7cx9HF7U1EQ02k+j32n0SRw/OJiOSAfRWFI9LHi9yPP7m3X5ZEiolXX31VHD16tNjR0RFy+bp168SRI0eKr7zySpRbNjREY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTefjlSxDo8H6rq6uqQk5PjnYnU4XDA4XDA6XQCAObOnYs5c+bg/fffj2UzB61obH/u4+ji9iaioSZS/R77T6LI4flERHLAvorCEenjRe7Hn9zbL0cMr2Nk3rx5qK6uxvLly1FfXw+VSgWVSgWlUgkAsNvtqK2tRVFRUYxbOjhFY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTefjlizesYevvtt3H11VfD6XRi5syZmDZtGkaMGAGr1YqHH34YTqcTTz/9NCZPnhzrpg5K0dj+3MfRxe1NRENNpPo99p9EkcPziYjkgH0VhSPSx4vcjz+5t19uGF7HiCiKEAQBhw8fxttvv43PP/8ce/fuxeHDh6FWq3HyySfj6quvxoQJE2Ld1EEpGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssRw+sBwmazoaOjAwaDAWq1GhaLBTqdLtbNGjKisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jvvxyw5nWMeQq6//zzz3jzzTehUqkAADqdDvxcof9FY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTefjlheB1jngN69erV+O9//+udrRSA3/fUP6Kx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm+/nDC8HiB27NiB2bNnx7oZQ1Y0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tlwOG1zGmUEi7ICMjA8cff3yMWzP0RGP7cx9HF7c3EQ01ker32H8SRQ7PJyKSA/ZVFI5IHy9yP/7k3n454YSNMWK326FWqwEALpcLNTU1KCwsjHGrho5obH/u4+ji9iaioSZS/R77T6LI4flERHLAvorCEenjRe7Hn9zbL0cceR0D27ZtwzXXXIMrrrgCu3btgkKhgEqlwtatW2E2mwGAxd37UTS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u2XK468jrIvvvgC1113HaxWK7RaLUaNGoWTTz4Zjz/+OPbv34/c3Fw88MADWLBgAURRZJH3CIvG9uc+ji5ubyIaaiLV77H/JIocnk9EJAfsqygckT5e5H78yb39csbwOsrOPvtspKSkYOXKlXA6nTjjjDNQX1+PRYsWYdGiRbj55puhVqvxxhtvICUlJdbNHXSisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jvv5yxbEiUbdu2Db/+9a+h1+thMBhQW1uLM844A/fccw+OP/54PPjggzAajfj5559j3dRBKRrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/bLGcPrKLNarUhNTfX+XFZWhrlz5wKQ6uKMGjUKVVVVfutQ5ERj+3MfRxe3NxENNZHq99h/EkUOzycikgP2VRSOSB8vcj/+5N5+OVPFugFDSXt7O0455RRYrVYA0sH96aefYsqUKQAAQRBw8OBBtLW14ZhjjollUwelaGx/7uPo4vYmoqEmUv0e+0+iyOH5RERywL6KwhHp40Xux5/c2y93rHkdRS6XCwcPHoRSqURubm7QcqvViqeeegovvfQSvvvuuxi0cHCLxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/XLHkddRpFAoUFBQ4P1ZFEWIogiFQqreUltbi6amJvztb3+LVRMHtWhsf+7j6OL2JqKhJlL9HvtPosjh+UREcsC+isIR6eNF7sef3Nsvdxx5HSWiKEIQBHR2dsJkMiEtLS3kek6nEw6HA1qtNsotHNyisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jv/2DACRujxFP/5tprr8Vpp52GRYsW4ZtvvglaT6lU8kDvB9HY/tzH0cXtTURDTaT6PfafRJHD84mI5IB9FYUj0seL3I8/ubd/MGB4HSVOpxN/+9vf8Nlnn+HEE09EcnIyTj31VHz++ecApE9y7HY7pk6disbGxhi3dvCJxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YOCSFGxbds2MTc3V6yurhZFURQ7OzvF2267TTzuuOPEqqoqURRFcc+ePaJCoYhlMwetaGx/7uPo4vYmoqEmUv0e+0+iyOH5RERywL6KwhHp40Xux5/c2z8YcOR1lHz//fcoKipCfn4+XC4XtFotVqxYgeHDh+O6664DAGzfvt2vADxFTjS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0fDBheR0lLSwtSU1NRV1cHhUIBURSRmJiI22+/HVu3bsVrr72G3bt349hjj411UwelaGx/7uPo4vYmoqEmUv0e+0+iyOH5RERywL6KwhHp40Xux5/c2z8YMLyOkmnTpiE7Oxvt7e3e+0RRxKRJk/CXv/wFq1atwqOPPooFCxbEsJWDVzS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u0fDFSxbsBQMXv2bOh0OhQVFXnvE0URLpcLV111FTZt2oTvvvsOxx9/fAxbOXhFY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTe/sFAEEVRjHUjhhpRFCEIgt99zc3NePTRR3HdddchLi4uRi0bGqKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm+/XDG8jpLm5mZ8+umnOHToECwWC/R6PYqKijBx4kQMHz481s0b9KKx/bmPo4vbm4iGmkj1e+w/iSKH5xMRyQH7KgpHpI8XuR9/cm//YMCyIVHw888/484778Rnn32G5ORkJCQkQBAEOBwOJCUlYcmSJbj00kuh0+li3dRBKRrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/YPFgyvo+CBBx5AQ0MDPvnkE0yfPh0AUF1djV27duH//u//cMstt6CxsRF33HFHyEsQ6OhEY/tzH0cXtzcRDTWR6vfYfxJFDs8nIpID9lUUjkgfL3I//uTe/kFDpH43bNgw8e233xZFURSdTmfQ8qeeekocNWqUuG3btmg3bUiIxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YOFItbh+VAwatQofPjhhzAajVAopE1ut9thtVoBABdffDEaGhqg1Wpj2cxBKxrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/YPFgyvo+Daa6/F2rVrcfvtt2PPnj0AALVaDa1Wi5aWFjz55JPQaDQ49thjY9zSwSka25/7OLq4vYloqIlUv8f+kyhyeD4RkRywr6JwRPp4kfvxJ/f2DxaCKIpirBsx2ImiiOeeew733nsvKisrkZCQgIKCAuTk5KCzsxNVVVVYsWIFrrzyylg3dVCKxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YMFw+soW79+PX766SeUlZWhtbUV8fHx+N3vfoc5c+bEumlDQjS2P/dxdHF7E9FQE6l+j/0nUeTwfCIiOWBfReGI9PEi9+NP7u2XM4bXMeByueB0OqFWq2PdlCEpGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssVw+t+5nA4sHLlSuzduxfjx4/HBRdcgOTkZADS5QeiKMJut7O4ez+JxvbnPo4ubm8iGmoi1e+x/ySKHJ5PRCQH7KsoHJE+XuR+/Mm9/YMJJ2zsRy6XC9dccw1WrVqF6upq3HfffZg/fz6am5u964iiiAceeAAmkymGLR2corH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb/+gI1K/+eGHH8Ti4mLxs88+E0VRFOvr68W5c+eKJ554onedXbt2iYIgxKqJg1o0tj/3cXRxexPRUBOpfo/9J1Hk8HwiIjlgX0XhiPTxIvfjT+7tH2w48rofbdy4EdnZ2ViwYAHsdjsyMjLwn//8B3V1dbjtttsAAD/88AOKiopi3NLBKRrbn/s4uri9iWioiVS/x/6TKHJ4PhGRHLCvonBE+niR+/En9/YPNgyv+9GhQ4cwfPhwWK1WqNVq2O12FBUV4dZbb8WLL76Ibdu24eeff8YxxxwT66YOStHY/tzH0cXtTURDTaT6PfafRJHD84mI5IB9FYUj0seL3I8/ubd/sGF43Y+GDRuG/fv3o7y8HACgVqshiiLOOussLFy4EHfddRfeeecdzJkzJ8YtHZyisf25j6OL25uIhppI9XvsP4kih+cTEckB+yoKR6SPF7kff3Jv/2AjiKIoxroRg5UoimhqakJSUhLUarXfsqamJvz2t7/Fhg0bsHnzZkyfPj1GrRy8orH9uY+ji9ubiIaaSPV77D+JIofnExHJAfsqCkekjxe5H39yb/9gw/A6hvbv349HHnkE9957LxISEmLdnCEnGtuf+zi6uL2JaKiJVL/H/pMocng+EZEcsK+icET6eJH78Sf39ssNw2siIiIiIiIiIiIiGnBY85qIiIiIiIiIiIiIBhyG10REREREREREREQ04DC8JiIiIiIiIiIiIqIBh+E1EREREREREREREQ04DK+JiIiIiADMmzcPy5cvj3UzvCoqKiAIAnbs2BHW41avXo3k5OR+aRMRERERUTQxvCYiIiIiGuKKi4uxatWqWDeDiIiIiMgPw2siIiIiIiIiIiIiGnAYXhMRERERBXjppZcwdepUGAwGZGdnY+nSpaivr/cuD1Wa491334UgCN6fb7/9dkycOBEvvfQSiouLkZSUhD/84Q8wGo3edVwuF/71r39hxIgR0Gq1KCwsxN133+33vGVlZZg/fz70ej0mTJiATZs2+S1fvXo1CgsLodfrcfrpp6OpqclveWlpKU477TRkZWUhISEB06ZNw//+9z/v8nnz5qGyshJ/+9vfIAiC33v45ptvMGfOHOh0OhQUFODKK6+EyWQKf4MSEREREfUBw2siIiIiogB2ux133nkndu7ciXfffRcVFRW48MILw36e0tJSvPvuu/jwww/x4YcfYv369bjvvvu8y2+44Qbcd999uOWWW7Br1y68+uqryMrK8nuOm266Cddccw127NiBUaNGYcmSJXA4HACALVu24JJLLsGyZcuwY8cOzJ8/H3fddZff4zs6OvCb3/wGa9euxfbt23HSSSdh8eLFqKqqAgCsWbMG+fn5+Mc//oHa2lrU1tZ6237SSSfhzDPPxA8//IA33ngD33zzDZYtWxb2diAiIiIi6gtBFEUx1o0gIiIiIoq1efPmYeLEiSFrP2/duhXTpk2D0WhEQkICVq9ejeXLl6O1tdW7zrvvvovTTz8dnj+vb7/9dtx///04fPgwDAYDAODvf/87vvrqK2zevBlGoxEZGRl47LHHcOmllwa9ZkVFBUpKSvDss8/ikksuAQDs2rULxx13HHbv3o0xY8Zg6dKlaGtrw0cffeR93B/+8Ad88sknfm0LNHbsWFx++eXeILq4uBjLly/3m7Dy0ksvhVKpxFNPPeW975tvvsHcuXNhMpkQFxd3xG1KRERERHQ0OPKaiIiIiCjAtm3bsHjxYhQWFsJgMGDu3LkA4B2t3FvFxcXe4BoAcnJyvOVHdu/eDavVil/96lc9Psf48eP9Hg/A7zlmzJjht/6sWbP8fu7o6MA111yDY445BsnJyUhISMDu3buP+F527tyJ1atXIyEhwXtbuHAhXC4XysvLj/DOiYiIiIiOnirWDSAiIiIiGkhMJhMWLlyIhQsX4pVXXkFGRgaqqqqwcOFC2Gw2AIBCoUDgBYx2uz3oudRqtd/PgiDA5XIBAHQ6Xa/a4/scnnrUnufojWuuuQaff/45HnjgAYwYMQI6nQ5nnXWW9710p6OjA3/6059w5ZVXBi0rLCzs9esTEREREfUVw2siIiIiIh979uxBU1MT7rvvPhQUFACQyob4ysjIgNFohMlkQnx8PABgx44dYb3OyJEjodPpsHbt2pBlQ3rjmGOOwZYtW/zu27x5s9/PGzZswIUXXojTTz8dgBRKV1RU+K2j0WjgdDr97ps8eTJ27dqFESNG9KltRERERERHi2VDiIiIiIh8FBYWQqPR4NFHH0VZWRnef/993HnnnX7rzJgxA3q9HjfeeCNKS0vx6quvYvXq1WG9TlxcHK677jr8/e9/x4svvojS0lJs3rwZzz33XK+f48orr8Qnn3yCBx54APv378djjz2GTz75xG+dkSNHYs2aNdixYwd27tyJpUuXBo3cLi4uxldffYWDBw+isbERAHDddddh48aN3skg9+/fj/fee48TNhIRERFR1DC8JiIiIiLykZGRgdWrV+Ott97Csccei/vuuw8PPPCA3zqpqal4+eWX8fHHH2PcuHF47bXXcPvtt4f9Wrfccguuvvpq3HrrrTjmmGNw9tlne+tZ98bMmTPxzDPP4OGHH8aECRPw2Wef4eabb/ZbZ+XKlUhJScHs2bOxePFiLFy4EJMnT/Zb5x//+AcqKiowfPhwZGRkAJBqba9fvx779u3DnDlzMGnSJNx6663Izc0N+30SEREREfWFIAYW6yMiIiIiIiIiIiIiijGOvCYiIiIiIiIiIiKiAYfhNRERERERERERERENOAyviYiIiIiIiIiIiGjAYXhNRERERERERERERAMOw2siIiIiIiIiIiIiGnAYXhMRERERERERERHRgMPwmoiIiIiIiIiIiIgGHIbXRERERERERERERDTgMLwmIiIiIiIiIiIiogGH4TURERERERERERERDTgMr4mIiIiIiIiIiIhowGF4TUREREREREREREQDzv8DhjTjP+JMXz8AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "df_merge = th_bench.dataframe.merge(th_bench.metadata, on='profile')\n", "df_merge['launchdate'] = pd.to_datetime(df_merge['launchdate'], unit='s')\n", From 92c30d09e360bb3c55dcab530329ebd80e54f33e Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Wed, 13 Sep 2023 10:34:04 -0700 Subject: [PATCH 184/187] remove old script --- .gitlab/build_and_bench_original.sh | 94 ----------------------------- 1 file changed, 94 deletions(-) delete mode 100644 .gitlab/build_and_bench_original.sh diff --git a/.gitlab/build_and_bench_original.sh b/.gitlab/build_and_bench_original.sh deleted file mode 100644 index 19c3649c09..0000000000 --- a/.gitlab/build_and_bench_original.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -# Build and test first -source "$BUILD_ROOT/.gitlab/build_and_test.sh" --no-clean - -benchmark_dir="${build_dir}/benchmarks" -ar3d_dir="${benchmark_dir}/advection_reaction_3D" -d2d_dir="${benchmark_dir}/diffusion_2D" - -job_unique_id=${CI_JOB_ID:-""} -job_timestamp=${CI_JOB_STARTED_AT:-""} - -# remove tailing number from hostname -hostname=${hostname%%[0-9]*} - -benchmark_store_dir="/usr/workspace/sundials/caliper/benchmarks/${hostname}/${job_timestamp}_${job_unique_id}" - -nresg=$((BENCHMARK_NNODES * 4)) # Lassen has 4 GPUs per node -nresc=$((BENCHMARK_NNODES * 40)) # Lassen has 40 cores per node - -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "~~~~~ Benchmarking SUNDIALS" -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - -if [[ ! -d ${benchmark_dir} ]] -then - echo "ERROR: Benchmark directory not found : ${benchmark_dir}" && exit 1 -fi - -cd "${benchmark_dir}" -ls - -# We use synchronous kernel launches for benchmarking -# because we are interested in individual region timings. -export CUDA_LAUNCH_BLOCKING=1 - -if [[ -d ${ar3d_dir} ]] -then - date - if [[ -f ${ar3d_dir}/advection_reaction_3D_mpicuda ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-IMEX --nls tl-newton --tf 10.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_arkdirk_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-DIRK --nls newton --tf 10.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_cvbdf_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method CV-BDF --nls newton --tf 10.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_ida_newton.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method IDA --nls newton --tf 10.0 - elif [[ -f ${ar3d_dir}/advection_reaction_3D ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkimex_tlnewton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-IMEX --nls tl-newton --tf 2.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_arkdirk_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-DIRK --nls newton --tf 2.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_cvbdf_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method CV-BDF --nls newton --tf 2.0 - export CALI_CONFIG="spot(output=${benchmark_store_dir}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 - fi - date -fi - - -if [[ -d ${d2d_dir} ]] -then - date - - if [[ -d ${d2d_dir}/mpi_gpu ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/arkode_diffusion_2D_mpicuda" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/cvode_diffusion_2D_mpicuda" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" - jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/ida_diffusion_2D_mpicuda" - elif [[ -d ${d2d_dir}/mpi_serial ]] - then - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_arkode.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_cvode.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D" - export CALI_CONFIG="spot(output=${benchmark_store_dir}/d2d_ida.cali),runtime-report(calc.inclusive)" - jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/ida_diffusion_2D" - fi - - date -fi - -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "~~~~~ CLEAN UP" -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -make clean - From 2a4e4426acb1ab851f7402545670cadd7974a1dd Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 14 Sep 2023 12:25:40 -0700 Subject: [PATCH 185/187] remove warning --- test/compare_benchmarks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/compare_benchmarks.py b/test/compare_benchmarks.py index 644cec3e5a..0fd97caa8b 100644 --- a/test/compare_benchmarks.py +++ b/test/compare_benchmarks.py @@ -5,7 +5,6 @@ import thicket as tt -# Note: script does not work for advection_reaction_3D benchmarks currently. def main(): parser = argparse.ArgumentParser(description='Compare Sundials performance results against previous results') From 676e8110d224ee7a7521423b7a2bfdfe193e3dc6 Mon Sep 17 00:00:00 2001 From: "Balos, Cody, J" Date: Thu, 14 Sep 2023 13:51:49 -0700 Subject: [PATCH 186/187] add threshold param --- test/compare_benchmarks.py | 26 ++++++++++++++++++++++---- test/compare_examples.py | 26 ++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/test/compare_benchmarks.py b/test/compare_benchmarks.py index 0fd97caa8b..29b56c3f0c 100644 --- a/test/compare_benchmarks.py +++ b/test/compare_benchmarks.py @@ -1,3 +1,17 @@ +# ----------------------------------------------------------------------------- +# Programmer(s): Yu Pan @ LLNL +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2023, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- + import os import glob import argparse @@ -15,10 +29,12 @@ def main(): parser.add_argument('--releasedir', dest='releaseDir', type=str, help='path to directory containing release caliper files', default="/usr/workspace/sundials/califiles/Release") - parser.add_argument('--outpath', dest='outPath', type=str, help='path to directory to write results to', default="/g/g20/pan13/shrimp") + parser.add_argument('--outpath', dest='outPath', type=str, help='path to directory to write results to', default="/dev/null") parser.add_argument('--jobid', dest='jobID', type=int, help='job id of the current run to identify .cali files') + parser.add_argument('--threshold', dest="threshold", type=float, help='the percentage threshold in performance difference that indicates a regression', default=2.0) + args = parser.parse_args() release = args.release @@ -26,6 +42,7 @@ def main(): caliDir = args.caliDir outPath = args.outPath jobID = args.jobID + threshold = args.threshold # Get available benchmarks benchFiles = glob.glob("%s/Benchmarking/*/*" % caliDir) @@ -36,7 +53,7 @@ def main(): # thread per file with mp.Pool() as pool: - for res in pool.starmap(process_benchmark, [(jobID, release, releaseDir, i) for i in benchFiles]): + for res in pool.starmap(process_benchmark, [(jobID, release, releaseDir, i, threshold) for i in benchFiles]): if res: outFile.write(res + "\n") outFile.close() @@ -51,7 +68,7 @@ def main(): return -1 return 0 -def process_benchmark(jobID, isRelease, releaseDir, benchmarkDir): +def process_benchmark(jobID, isRelease, releaseDir, benchmarkDir, threshold): # Get the current benchmark run benchmarkFiles = glob.glob("%s/*.cali" % benchmarkDir) # Don't compare if the run didn't include this benchmark @@ -82,7 +99,8 @@ def process_benchmark(jobID, isRelease, releaseDir, benchmarkDir): ratio = th_current.statsframe.dataframe['Max time/rank_mean'] / th_compare.statsframe.dataframe['Max time/rank_mean'] - if 1 - ratio[0] < 0.0002: + tolerance = threshold/100 + if 1 - ratio[0] < tolerance: return benchmarkName diff --git a/test/compare_examples.py b/test/compare_examples.py index 3c74e120db..4b792ba03a 100644 --- a/test/compare_examples.py +++ b/test/compare_examples.py @@ -1,3 +1,18 @@ +#!/usr/bin/env python3 +# ----------------------------------------------------------------------------- +# Programmer(s): Yu Pan @ LLNL +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2023, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- + import os import subprocess import sys @@ -21,7 +36,9 @@ def main(): parser.add_argument('--releasedir', dest='releaseDir', type=str, help='path to directory containing release caliper files', default="/usr/workspace/sundials/califiles/Release") - parser.add_argument('--outpath', dest='outPath', type=str, help='path to directory to write results to', default="/g/g20/pan13/shrimp") + parser.add_argument('--outpath', dest='outPath', type=str, help='path to directory to write results to', default="/dev/null") + + parser.add_argument('--threshold', dest="threshold", type=float, help='the percentage threshold in performance difference that indicates a regression', default=2.0) args = parser.parse_args() @@ -29,6 +46,7 @@ def main(): releaseDir = args.releaseDir caliDir = args.caliDir outPath = args.outPath + threshold = args.threshold # Get the latest test run runDirs = glob.glob("%s/Testing/*" % caliDir, recursive = True) @@ -58,7 +76,7 @@ def main(): # Compare test results against past runs. If a test performs below a threshold, output test name to outFile. with mp.Pool() as pool: - for res in pool.starmap(compare_against_release, [(versionDir, i) for i in runFiles]): + for res in pool.starmap(compare_against_release, [(versionDir, i, threshold) for i in runFiles]): if res: outFile.write(res + "\n") outFile.close() @@ -74,7 +92,7 @@ def main(): return 0 -def compare_against_release(releaseDir, file): +def compare_against_release(releaseDir, file, threshold): th = tt.Thicket.from_caliperreader(file) testName = th.metadata['env.TEST_NAME'].values[0] @@ -89,7 +107,7 @@ def compare_against_release(releaseDir, file): ratio = th.statsframe.dataframe['Max time/rank_mean'] / th_release.statsframe.dataframe['Max time/rank_mean'] print(ratio[0]) - tolerance = 0.0002 + tolerance = threshold/100 if 1 - ratio[0] < tolerance: return testName From b932d19951b7f5a08278ef4c836142200b0558cd Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Sep 2023 21:04:20 -0700 Subject: [PATCH 187/187] Update test/compare_benchmarks.py --- test/compare_benchmarks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/compare_benchmarks.py b/test/compare_benchmarks.py index 29b56c3f0c..663f406262 100644 --- a/test/compare_benchmarks.py +++ b/test/compare_benchmarks.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # ----------------------------------------------------------------------------- # Programmer(s): Yu Pan @ LLNL # -----------------------------------------------------------------------------