Use Gitlab Container registry as a Spack binary cache (#564)

* Using Spack build cache

* Fix registry url + unify log

.gitlab/custom-jobs-and-variables.yml

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
+# Copyright (c) 2022-24, Lawrence Livermore National Security, LLC and RADIUSS
 # project contributors. See the COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -16,6 +16,7 @@ variables:
 # Arguments for top level allocation
   RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --time=5 --nodes=2"
 # Arguments for job level allocation
+# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
   RUBY_JOB_ALLOC: "--overlap --reservation=ci --nodes=1"
 # Project specific variants for ruby
   PROJECT_RUBY_VARIANTS: "+tests"
@@ -69,3 +70,15 @@ variables:
   artifacts:
     reports:
       junit: junit.xml
+
+.reproducer_vars:
+  script:
+    - |
+      echo -e "
+      # Required variables \n
+      export MODULE_LIST=\"${MODULE_LIST}\" \n
+      export SPEC=\"${SPEC//\"/\\\"}\" \n
+      # Allow to set job script for debugging (only this differs from CI) \n
+      export DEBUG_MODE=true \n
+      # Using the CI build cache is optional and requires a token. Set it like so: \n
+      # export REGISTRY_TOKEN=\"<your token here>\" \n"

.gitlab/jobs/corona.yml

@@ -8,9 +8,7 @@
 # Override reproducer section to define project specific variables.
 .corona_reproducer_vars:
   script:
-    - |
-      echo -e "export MODULE_LIST=\"${MODULE_LIST}\""
-      echo -e "export SPEC=\"${SPEC//\"/\\\"}\""
+    - !reference [.reproducer_vars, script]
 
 # With GitLab CI, included files cannot be empty.
 variables:

.gitlab/jobs/lassen.yml

@@ -8,9 +8,7 @@
 # Override reproducer section to define project specific variables.
 .lassen_reproducer_vars:
   script:
-    - |
-      echo -e "export MODULE_LIST=\"${MODULE_LIST}\""
-      echo -e "export SPEC=\"${SPEC//\"/\\\"}\""
+    - !reference [.reproducer_vars, script]
 
 # With GitLab CI, included files cannot be empty.
 variables:

.gitlab/jobs/poodle.yml

@@ -8,9 +8,7 @@
 # Override reproducer section to define project specific variables.
 .poodle_reproducer_vars:
   script:
-    - |
-      echo -e "export MODULE_LIST=\"${MODULE_LIST}\""
-      echo -e "export SPEC=\"${SPEC//\"/\\\"}\""
+    - !reference [.reproducer_vars, script]
 
 # With GitLab CI, included files cannot be empty.
 variables:

.gitlab/jobs/ruby.yml

@@ -8,9 +8,7 @@
 # Override reproducer section to define project specific variables.
 .ruby_reproducer_vars:
   script:
-    - |
-      echo -e "export MODULE_LIST=\"${MODULE_LIST}\""
-      echo -e "export SPEC=\"${SPEC//\"/\\\"}\""
+    - !reference [.reproducer_vars, script]
 
 # With GitLab CI, included files cannot be empty.
 variables:

.gitlab/jobs/tioga.yml

@@ -8,9 +8,7 @@
 # Override reproducer section to define project specific variables.
 .tioga_reproducer_vars:
   script:
-    - |
-      echo -e "export MODULE_LIST=\"${MODULE_LIST}\""
-      echo -e "export SPEC=\"${SPEC//\"/\\\"}\""
+    - !reference [.reproducer_vars, script]
 
 # With GitLab CI, included files cannot be empty.
 variables:

scripts/gitlab/build-and-test.sh

@@ -7,7 +7,7 @@ then
 fi
 
 ###############################################################################
-# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC and Caliper
+# Copyright (c) 2016-24, Lawrence Livermore National Security, LLC and Caliper
 # project contributors. See the Caliper/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
@@ -26,6 +26,34 @@ spec=${SPEC:-""}
 module_list=${MODULE_LIST:-""}
 job_unique_id=${CI_JOB_ID:-""}
 use_dev_shm=${USE_DEV_SHM:-true}
+spack_debug=${SPACK_DEBUG:-false}
+debug_mode=${DEBUG_MODE:-false}
+
+# REGISTRY_TOKEN allows you to provide your own personal access token to the CI
+# registry. Be sure to set the token with at least read access to the registry.
+registry_token=${REGISTRY_TOKEN:-""}
+ci_registry_user=${CI_REGISTRY_USER:-"${USER}"}
+ci_registry_image=${CI_REGISTRY_IMAGE:-"czregistry.llnl.gov:5050/radiuss/caliper"}
+ci_registry_token=${CI_JOB_TOKEN:-"${registry_token}"}
+
+timed_message ()
+{
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~ $(date --rfc-3339=seconds) ~ ${1}"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+}
+
+if [[ ${debug_mode} == true ]]
+then
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Debug mode:"
+    echo "~~~~~ - Spack debug mode."
+    echo "~~~~~ - Deactivated shared memory."
+    echo "~~~~~ - Do not push to buildcache."
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    use_dev_shm=false
+    spack_debug=true
+fi
 
 if [[ -n ${module_list} ]]
 then
@@ -63,16 +91,23 @@ echo "project_dir: ${project_dir}"
 
 mkdir -p ${prefix}
 
+spack_cmd="${prefix}/spack/bin/spack"
+spack_env_path="${prefix}/spack_env"
+uberenv_cmd="./scripts/uberenv/uberenv.py"
+if [[ ${spack_debug} == true ]]
+then
+    spack_cmd="${spack_cmd} --debug --stacktrace"
+    uberenv_cmd="${uberenv_cmd} --spack-debug"
+fi
+
 # Dependencies
 if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]]
 then
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Building dependencies $(date)"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Building dependencies"
 
     if [[ -z ${spec} ]]
     then
-        echo "SPEC is undefined, aborting..."
+        echo "[Error]: SPEC is undefined, aborting..."
         exit 1
     fi
 
@@ -87,14 +122,28 @@ then
     mkdir -p ${spack_user_cache}
 
     # generate cmake cache file with uberenv and radiuss spack package
-    ./scripts/uberenv/uberenv.py --spec="${spec}" ${prefix_opt}
+    timed_message "Spack setup and environment"
+    ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt}
 
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Dependencies built $(date)"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    if [[ -n ${ci_registry_token} ]]
+    then
+        timed_message "GitLab registry as Spack Buildcache"
+        ${spack_cmd} -D ${spack_env_path} mirror add --unsigned --oci-username ${ci_registry_user} --oci-password ${ci_registry_token} gitlab_ci oci://${ci_registry_image}
+    fi
+
+    timed_message "Spack build of dependencies"
+    ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt}
+
+    if [[ -n ${ci_registry_token} && ${debug_mode} == false ]]
+    then
+        timed_message "Push dependencies to buildcache"
+        ${spack_cmd} -D ${spack_env_path} buildcache push --only dependencies gitlab_ci
+    fi
+
+    timed_message "Dependencies built"
 fi
 
-# find cmake cache file (hostconfig)
+# Find cmake cache file (hostconfig)
 if [[ -z ${hostconfig} ]]
 then
     # If no host config file was provided, we assume it was generated.
@@ -103,16 +152,15 @@ then
     if [[ ${#hostconfigs[@]} == 1 ]]
     then
         hostconfig_path=${hostconfigs[0]}
-        echo "Found host config file: ${hostconfig_path}"
     elif [[ ${#hostconfigs[@]} == 0 ]]
     then
-        echo "No result for: ${project_dir}/*.cmake"
-        echo "Spack generated host-config not found."
+        echo "[Error]: No result for: ${project_dir}/*.cmake"
+        echo "[Error]: Spack generated host-config not found."
         exit 1
     else
-        echo "More than one result for: ${project_dir}/*.cmake"
-        echo "${hostconfigs[@]}"
-        echo "Please specify one with HOST_CONFIG variable"
+        echo "[Error]: More than one result for: ${project_dir}/*.cmake"
+        echo "[Error]: ${hostconfigs[@]}"
+        echo "[Error]: Please specify one with HOST_CONFIG variable"
         exit 1
     fi
 else
@@ -121,7 +169,7 @@ else
 fi
 
 hostconfig=$(basename ${hostconfig_path})
-echo "Found hostconfig: ${hostconfig}"
+echo "[Information]: Found hostconfig ${hostconfig_path}"
 
 # Build Directory
 # When using /dev/shm, we use prefix for both spack builds and source build, unless BUILD_ROOT was defined
@@ -143,9 +191,7 @@ then
     echo "~~~~~ Install Dir: ${install_dir}"
     echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
     echo ""
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Building Caliper $(date)"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Cleaning working directory"
 
     # Map CPU core allocations
     declare -A core_counts=(["lassen"]=40 ["ruby"]=28 ["poodle"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32)
@@ -157,7 +203,7 @@ then
     rm -rf ${build_dir} 2>/dev/null
     mkdir -p ${build_dir} && cd ${build_dir}
 
-    date
+    timed_message "Building RAJA"
     if [[ "${truehostname}" == "corona" || "${truehostname}" == "tioga" ]]
     then
         module unload rocm
@@ -177,26 +223,20 @@ then
     fi
     if ! $cmake_exe --build . -j ${core_counts[$truehostname]}
     then
-        echo "[Error]: compilation failed, building with verbose output..."
-        echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-        echo "~~~~~ Running make VERBOSE=1 $(date)"
-        echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+        echo "[Error]: Compilation failed, building with verbose output..."
+        timed_message "Re-building with --verbose"
         $cmake_exe --build . --verbose -j 1
     else
+        timed_message "Installing"
         $cmake_exe --install .
     fi
 
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Caliper Built $(date)"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Caliper built and installed"
 fi
 
 # Test
 if [[ "${option}" != "--build-only" ]]
 then
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Testing Caliper $(date)"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 
     if [[ ! -d ${build_dir} ]]
     then
@@ -205,27 +245,26 @@ then
 
     cd ${build_dir}
 
-    date
+    timed_message "Testing RAJA"
     ctest --output-on-failure -T test 2>&1 | tee tests_output.txt
-    date
 
     no_test_str="No tests were found!!!"
     if [[ "$(tail -n 1 tests_output.txt)" == "${no_test_str}" ]]
     then
         echo "[Error]: No tests were found" && exit 1
     fi
 
-    echo "Copying Testing xml reports for export"
+    timed_message "Preparing tests xml reports for export"
     tree Testing
     xsltproc -o junit.xml ${project_dir}/scripts/ctest-to-junit.xsl Testing/*/Test.xml
     mv junit.xml ${project_dir}/junit.xml
 
     if grep -q "Errors while running CTest" ./tests_output.txt
     then
-        echo "[Error]: failure(s) while running CTest" && exit 1
+        echo "[Error]: Failure(s) while running CTest" && exit 1
     fi
 
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Caliper Tests Complete $(date)"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Caliper tests completed"
 fi
+
+timed_message "Build and test completed"