diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index c51260550..98e38c7c4 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# Copyright (c) 2022-24, Lawrence Livermore National Security, LLC and RADIUSS # project contributors. See the COPYRIGHT file for details. # # SPDX-License-Identifier: (MIT) @@ -16,6 +16,7 @@ variables: # Arguments for top level allocation RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --time=5 --nodes=2" # Arguments for job level allocation +# Note: We repeat the reservation, necessary when jobs are manually re-triggered. RUBY_JOB_ALLOC: "--overlap --reservation=ci --nodes=1" # Project specific variants for ruby PROJECT_RUBY_VARIANTS: "+tests" @@ -69,3 +70,15 @@ variables: artifacts: reports: junit: junit.xml + +.reproducer_vars: + script: + - | + echo -e " + # Required variables \n + export MODULE_LIST=\"${MODULE_LIST}\" \n + export SPEC=\"${SPEC//\"/\\\"}\" \n + # Allow to set job script for debugging (only this differs from CI) \n + export DEBUG_MODE=true \n + # Using the CI build cache is optional and requires a token. Set it like so: \n + # export REGISTRY_TOKEN=\"\" \n" diff --git a/.gitlab/jobs/corona.yml b/.gitlab/jobs/corona.yml index e5cc07ebf..0d5de5629 100644 --- a/.gitlab/jobs/corona.yml +++ b/.gitlab/jobs/corona.yml @@ -8,9 +8,7 @@ # Override reproducer section to define project specific variables. .corona_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] # With GitLab CI, included files cannot be empty. variables: diff --git a/.gitlab/jobs/lassen.yml b/.gitlab/jobs/lassen.yml index a2428f6ee..b083bcf7b 100644 --- a/.gitlab/jobs/lassen.yml +++ b/.gitlab/jobs/lassen.yml @@ -8,9 +8,7 @@ # Override reproducer section to define project specific variables. .lassen_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] # With GitLab CI, included files cannot be empty. variables: diff --git a/.gitlab/jobs/poodle.yml b/.gitlab/jobs/poodle.yml index 2295ba251..d31282227 100644 --- a/.gitlab/jobs/poodle.yml +++ b/.gitlab/jobs/poodle.yml @@ -8,9 +8,7 @@ # Override reproducer section to define project specific variables. .poodle_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] # With GitLab CI, included files cannot be empty. variables: diff --git a/.gitlab/jobs/ruby.yml b/.gitlab/jobs/ruby.yml index e93dc6913..c34e29489 100644 --- a/.gitlab/jobs/ruby.yml +++ b/.gitlab/jobs/ruby.yml @@ -8,9 +8,7 @@ # Override reproducer section to define project specific variables. .ruby_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] # With GitLab CI, included files cannot be empty. variables: diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml index 848d42418..389c8a8b1 100644 --- a/.gitlab/jobs/tioga.yml +++ b/.gitlab/jobs/tioga.yml @@ -8,9 +8,7 @@ # Override reproducer section to define project specific variables. .tioga_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] # With GitLab CI, included files cannot be empty. variables: diff --git a/scripts/gitlab/build-and-test.sh b/scripts/gitlab/build-and-test.sh index 1e109f00e..3416b51a3 100755 --- a/scripts/gitlab/build-and-test.sh +++ b/scripts/gitlab/build-and-test.sh @@ -7,7 +7,7 @@ then fi ############################################################################### -# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC and Caliper +# Copyright (c) 2016-24, Lawrence Livermore National Security, LLC and Caliper # project contributors. See the Caliper/LICENSE file for details. # # SPDX-License-Identifier: (BSD-3-Clause) @@ -26,6 +26,34 @@ spec=${SPEC:-""} module_list=${MODULE_LIST:-""} job_unique_id=${CI_JOB_ID:-""} use_dev_shm=${USE_DEV_SHM:-true} +spack_debug=${SPACK_DEBUG:-false} +debug_mode=${DEBUG_MODE:-false} + +# REGISTRY_TOKEN allows you to provide your own personal access token to the CI +# registry. Be sure to set the token with at least read access to the registry. +registry_token=${REGISTRY_TOKEN:-""} +ci_registry_user=${CI_REGISTRY_USER:-"${USER}"} +ci_registry_image=${CI_REGISTRY_IMAGE:-"czregistry.llnl.gov:5050/radiuss/caliper"} +ci_registry_token=${CI_JOB_TOKEN:-"${registry_token}"} + +timed_message () +{ + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "~ $(date --rfc-3339=seconds) ~ ${1}" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +} + +if [[ ${debug_mode} == true ]] +then + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "~~~~~ Debug mode:" + echo "~~~~~ - Spack debug mode." + echo "~~~~~ - Deactivated shared memory." + echo "~~~~~ - Do not push to buildcache." + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + use_dev_shm=false + spack_debug=true +fi if [[ -n ${module_list} ]] then @@ -63,16 +91,23 @@ echo "project_dir: ${project_dir}" mkdir -p ${prefix} +spack_cmd="${prefix}/spack/bin/spack" +spack_env_path="${prefix}/spack_env" +uberenv_cmd="./scripts/uberenv/uberenv.py" +if [[ ${spack_debug} == true ]] +then + spack_cmd="${spack_cmd} --debug --stacktrace" + uberenv_cmd="${uberenv_cmd} --spack-debug" +fi + # Dependencies if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Building dependencies $(date)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Building dependencies" if [[ -z ${spec} ]] then - echo "SPEC is undefined, aborting..." + echo "[Error]: SPEC is undefined, aborting..." exit 1 fi @@ -87,14 +122,28 @@ then mkdir -p ${spack_user_cache} # generate cmake cache file with uberenv and radiuss spack package - ./scripts/uberenv/uberenv.py --spec="${spec}" ${prefix_opt} + timed_message "Spack setup and environment" + ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt} - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Dependencies built $(date)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + if [[ -n ${ci_registry_token} ]] + then + timed_message "GitLab registry as Spack Buildcache" + ${spack_cmd} -D ${spack_env_path} mirror add --unsigned --oci-username ${ci_registry_user} --oci-password ${ci_registry_token} gitlab_ci oci://${ci_registry_image} + fi + + timed_message "Spack build of dependencies" + ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt} + + if [[ -n ${ci_registry_token} && ${debug_mode} == false ]] + then + timed_message "Push dependencies to buildcache" + ${spack_cmd} -D ${spack_env_path} buildcache push --only dependencies gitlab_ci + fi + + timed_message "Dependencies built" fi -# find cmake cache file (hostconfig) +# Find cmake cache file (hostconfig) if [[ -z ${hostconfig} ]] then # If no host config file was provided, we assume it was generated. @@ -103,16 +152,15 @@ then if [[ ${#hostconfigs[@]} == 1 ]] then hostconfig_path=${hostconfigs[0]} - echo "Found host config file: ${hostconfig_path}" elif [[ ${#hostconfigs[@]} == 0 ]] then - echo "No result for: ${project_dir}/*.cmake" - echo "Spack generated host-config not found." + echo "[Error]: No result for: ${project_dir}/*.cmake" + echo "[Error]: Spack generated host-config not found." exit 1 else - echo "More than one result for: ${project_dir}/*.cmake" - echo "${hostconfigs[@]}" - echo "Please specify one with HOST_CONFIG variable" + echo "[Error]: More than one result for: ${project_dir}/*.cmake" + echo "[Error]: ${hostconfigs[@]}" + echo "[Error]: Please specify one with HOST_CONFIG variable" exit 1 fi else @@ -121,7 +169,7 @@ else fi hostconfig=$(basename ${hostconfig_path}) -echo "Found hostconfig: ${hostconfig}" +echo "[Information]: Found hostconfig ${hostconfig_path}" # Build Directory # When using /dev/shm, we use prefix for both spack builds and source build, unless BUILD_ROOT was defined @@ -143,9 +191,7 @@ then echo "~~~~~ Install Dir: ${install_dir}" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Building Caliper $(date)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Cleaning working directory" # Map CPU core allocations declare -A core_counts=(["lassen"]=40 ["ruby"]=28 ["poodle"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32) @@ -157,7 +203,7 @@ then rm -rf ${build_dir} 2>/dev/null mkdir -p ${build_dir} && cd ${build_dir} - date + timed_message "Building RAJA" if [[ "${truehostname}" == "corona" || "${truehostname}" == "tioga" ]] then module unload rocm @@ -177,26 +223,20 @@ then fi if ! $cmake_exe --build . -j ${core_counts[$truehostname]} then - echo "[Error]: compilation failed, building with verbose output..." - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Running make VERBOSE=1 $(date)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "[Error]: Compilation failed, building with verbose output..." + timed_message "Re-building with --verbose" $cmake_exe --build . --verbose -j 1 else + timed_message "Installing" $cmake_exe --install . fi - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Caliper Built $(date)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Caliper built and installed" fi # Test if [[ "${option}" != "--build-only" ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Testing Caliper $(date)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" if [[ ! -d ${build_dir} ]] then @@ -205,9 +245,8 @@ then cd ${build_dir} - date + timed_message "Testing RAJA" ctest --output-on-failure -T test 2>&1 | tee tests_output.txt - date no_test_str="No tests were found!!!" if [[ "$(tail -n 1 tests_output.txt)" == "${no_test_str}" ]] @@ -215,17 +254,17 @@ then echo "[Error]: No tests were found" && exit 1 fi - echo "Copying Testing xml reports for export" + timed_message "Preparing tests xml reports for export" tree Testing xsltproc -o junit.xml ${project_dir}/scripts/ctest-to-junit.xsl Testing/*/Test.xml mv junit.xml ${project_dir}/junit.xml if grep -q "Errors while running CTest" ./tests_output.txt then - echo "[Error]: failure(s) while running CTest" && exit 1 + echo "[Error]: Failure(s) while running CTest" && exit 1 fi - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Caliper Tests Complete $(date)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Caliper tests completed" fi + +timed_message "Build and test completed" diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs index 7b49b844e..037ee6716 160000 --- a/scripts/radiuss-spack-configs +++ b/scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 7b49b844e90f9b10101bec6236952c1047c1d0f4 +Subproject commit 037ee671622bd3ec2ff955ced356b59dc96fec58 diff --git a/scripts/uberenv b/scripts/uberenv index cf91883ef..205672b8b 160000 --- a/scripts/uberenv +++ b/scripts/uberenv @@ -1 +1 @@ -Subproject commit cf91883ef0500a808338ad6c8b56647da15fa5f3 +Subproject commit 205672b8b2520d7dc69acefe8738960cd5db0937