diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 931d1961b..063cabb90 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -71,3 +71,15 @@ variables: artifacts: reports: junit: junit.xml + +.reproducer_vars: + script: + - | + echo -e " + # Required variables \n + export MODULE_LIST=\"${MODULE_LIST}\" \n + export SPEC=\"${SPEC//\"/\\\"}\" \n + # Allow to set job script for debugging (only this differs from CI) \n + export DEBUG_MODE=true \n + # Using the CI build cache is optional and requires a token. Set it like so: \n + # export REGISTRY_TOKEN=\"\" \n" diff --git a/.gitlab/jobs/corona.yml b/.gitlab/jobs/corona.yml index 8fec233c5..9af5ba72b 100644 --- a/.gitlab/jobs/corona.yml +++ b/.gitlab/jobs/corona.yml @@ -9,9 +9,7 @@ # Override reproducer section to define project specific variables. .corona_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] ######################## # Overridden shared jobs diff --git a/.gitlab/jobs/lassen.yml b/.gitlab/jobs/lassen.yml index c6eacf864..1b9bc0eda 100644 --- a/.gitlab/jobs/lassen.yml +++ b/.gitlab/jobs/lassen.yml @@ -9,9 +9,7 @@ # Override reproducer section to define project specific variables. .lassen_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] ######################## # Overridden shared jobs @@ -44,14 +42,14 @@ gcc_8_3_1: gcc_8_3_1_cuda_11_5_0_ats_disabled: extends: .job_on_lassen variables: - SPEC: " ~shared +openmp +cuda %gcc@=8.3.1 cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS}" + SPEC: " ~shared +openmp +cuda %gcc@=8.3.1 cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^blt@develop" MODULE_LIST: "cuda/11.5.0" LASSEN_JOB_ALLOC: "1 --atsdisable -W 30 -q pci" gcc_8_3_1_cuda_11_5_0_ats_disabled_mpi: extends: .job_on_lassen variables: - SPEC: " ~shared +openmp +cuda +mpi %gcc@=8.3.1 cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^spectrum-mpi ${PROJECT_LASSEN_DEPS}" + SPEC: " ~shared +openmp +cuda +mpi %gcc@=8.3.1 cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^spectrum-mpi ^blt@develop" MODULE_LIST: "cuda/11.5.0" LASSEN_JOB_ALLOC: "1 --atsdisable -W 30 -q pci" @@ -61,7 +59,7 @@ gcc_8_3_1_cuda_11_5_0_ats_disabled_mpi: clang_13_0_1_libcpp: variables: - SPEC: " ~shared +openmp %clang@=13.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\" ${PROJECT_LASSEN_DEPS}" + SPEC: " ~shared +openmp %clang@=13.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\" ^blt@develop" extends: .job_on_lassen #clang_14_0_5_asan: diff --git a/.gitlab/jobs/poodle.yml b/.gitlab/jobs/poodle.yml index ed18f60f5..8e86158f0 100644 --- a/.gitlab/jobs/poodle.yml +++ b/.gitlab/jobs/poodle.yml @@ -9,9 +9,7 @@ # Override reproducer section to define projet specific variables. .poodle_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] ######################## # Overridden shared jobs @@ -50,6 +48,6 @@ intel_2022_1_0: intel_2022_1_0_mpi: variables: - SPEC: "~shared +openmp +mpi %intel@=2022.1.0 ^mvapich2 ${PROJECT_POODLE_DEPS}" + SPEC: "~shared +openmp +mpi %intel@=2022.1.0 ^mvapich2 ^blt@develop" allow_failure: true extends: .job_on_poodle diff --git a/.gitlab/jobs/ruby.yml b/.gitlab/jobs/ruby.yml index 3502ed3fb..c19e36d12 100644 --- a/.gitlab/jobs/ruby.yml +++ b/.gitlab/jobs/ruby.yml @@ -9,9 +9,7 @@ # Override reproducer section to define project specific variables. .ruby_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] ####################### # Overridden shared jobs @@ -51,5 +49,5 @@ intel_2022_1_0: intel_2022_1_0_mpi: variables: - SPEC: "~shared +openmp +mpi %intel@=2022.1.0 ^mvapich2 ${PROJECT_RUBY_DEPS}" + SPEC: "~shared +openmp +mpi %intel@=2022.1.0 ^mvapich2 ^blt@develop" extends: .job_on_ruby diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml index bcf9eccb8..00ed3c276 100644 --- a/.gitlab/jobs/tioga.yml +++ b/.gitlab/jobs/tioga.yml @@ -9,9 +9,7 @@ # Override reproducer section to define project specific variables. .tioga_reproducer_vars: script: - - | - echo -e "export MODULE_LIST=\"${MODULE_LIST}\"" - echo -e "export SPEC=\"${SPEC//\"/\\\"}\"" + - !reference [.reproducer_vars, script] ######################## # Overridden shared jobs @@ -31,11 +29,11 @@ rocmcc_6_1_1_hip_openmp: variables: - SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.1.1 ^hip@6.1.1 ${PROJECT_TIOGA_DEPS}" + SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.1.1 ^hip@6.1.1 ^blt@develop" extends: .job_on_tioga rocmcc_6_1_1_hip_openmp_mpi: variables: - SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.1.1 ^hip@6.1.1 ${PROJECT_TIOGA_DEPS}" + SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.1.1 ^hip@6.1.1 ^blt@develop" extends: .job_on_tioga allow_failure: true diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 430d50b4b..f2d020918 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -26,6 +26,8 @@ spec=${SPEC:-""} module_list=${MODULE_LIST:-""} job_unique_id=${CI_JOB_ID:-""} use_dev_shm=${USE_DEV_SHM:-true} +spack_debug=${SPACK_DEBUG:-false} +debug_mode=${DEBUG_MODE:-false} raja_version=${UPDATE_RAJA:-""} sys_type=${SYS_TYPE:-""} @@ -33,6 +35,32 @@ sys_type=${SYS_TYPE:-""} spack_upstream_path=${SPACK_UPSTREAM_PATH:-"/usr/workspace/umdev/RAJAPerf/upstream"} update_spack_upstream=${UPDATE_SPACK_UPSTREAM:-false} +# REGISTRY_TOKEN allows you to provide your own personal access token to the CI +# registry. Be sure to set the token with at least read access to the registry. +registry_token=${REGISTRY_TOKEN:-""} +ci_registry_user=${CI_REGISTRY_USER:-"${USER}"} +ci_registry_image=${CI_REGISTRY_IMAGE:-"czregistry.llnl.gov:5050/radiuss/rajaperf"} +ci_registry_token=${CI_JOB_TOKEN:-"${registry_token}"} + +timed_message () +{ + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "~ $(date --rfc-3339=seconds) ~ ${1}" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +} + +if [[ ${debug_mode} == true ]] +then + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "~~~~~ Debug mode:" + echo "~~~~~ - Spack debug mode." + echo "~~~~~ - Deactivated shared memory." + echo "~~~~~ - Do not push to buildcache." + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + use_dev_shm=false + spack_debug=true +fi + if [[ -n ${module_list} ]] then echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -62,27 +90,33 @@ then fi prefix="${prefix}-${job_unique_id}" - mkdir -p ${prefix} else # We set the prefix in the parent directory so that spack dependencies are not installed inside the source tree. prefix="$(pwd)/../spack-and-build-root" - mkdir -p ${prefix} +fi + +echo "Creating directory ${prefix}" +echo "project_dir: ${project_dir}" + +mkdir -p ${prefix} + +spack_cmd="${prefix}/spack/bin/spack" +spack_env_path="${prefix}/spack_env" +uberenv_cmd="$(pwd)/tpl/RAJA/scripts/uberenv/uberenv.py --project-json=$(pwd)/.uberenv_config.json" +if [[ ${spack_debug} == true ]] +then + spack_cmd="${spack_cmd} --debug --stacktrace" + uberenv_cmd="${uberenv_cmd} --spack-debug" fi # Dependencies -date -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "~~~~~ Build and test started" -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Building Dependencies" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Building dependencies" if [[ -z ${spec} ]] then - echo "SPEC is undefined, aborting..." + echo "[Error]: SPEC is undefined, aborting..." exit 1 fi @@ -102,17 +136,32 @@ then export SPACK_USER_CACHE_PATH="${spack_user_cache}" mkdir -p ${spack_user_cache} - ./tpl/RAJA/scripts/uberenv/uberenv.py --project-json=".uberenv_config.json" --spec="${spec}" ${prefix_opt} ${upstream_opt} + # generate cmake cache file with uberenv and radiuss spack package + timed_message "Spack setup and environment" + ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt} ${upstream_opt} + + if [[ -n ${ci_registry_token} ]] + then + timed_message "GitLab registry as Spack Buildcache" + ${spack_cmd} -D ${spack_env_path} mirror add --unsigned --oci-username ${ci_registry_user} --oci-password ${ci_registry_token} gitlab_ci oci://${ci_registry_image} + fi + + timed_message "Spack build of dependencies" + ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt} ${upstream_opt} + + if [[ -n ${ci_registry_token} && ${debug_mode} == false ]] + then + timed_message "Push dependencies to buildcache" + ${spack_cmd} -D ${spack_env_path} buildcache push --only dependencies gitlab_ci + fi + + timed_message "Dependencies built" mv ${project_dir}/tpl/RAJA/*.cmake ${project_dir}/. fi - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Dependencies Built" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -date -# Host config file +# Find cmake cache file (hostconfig) if [[ -z ${hostconfig} ]] then # If no host config file was provided, we assume it was generated. @@ -121,24 +170,24 @@ then if [[ ${#hostconfigs[@]} == 1 ]] then hostconfig_path=${hostconfigs[0]} - echo "Found host config file: ${hostconfig_path}" elif [[ ${#hostconfigs[@]} == 0 ]] then - echo "No result for: ${project_dir}/*.cmake" - echo "Spack generated host-config not found." + echo "[Error]: No result for: ${project_dir}/*.cmake" + echo "[Error]: Spack generated host-config not found." exit 1 else - echo "More than one result for: ${project_dir}/*.cmake" - echo "${hostconfigs[@]}" - echo "Please specify one with HOST_CONFIG variable" + echo "[Error]: More than one result for: ${project_dir}/*.cmake" + echo "[Error]: ${hostconfigs[@]}" + echo "[Error]: Please specify one with HOST_CONFIG variable" exit 1 fi else # Using provided host-config file. - hostconfig_path="${project_dir}/host-configs/${hostconfig}" + hostconfig_path="${project_dir}/${hostconfig}" fi hostconfig=$(basename ${hostconfig_path}) +echo "[Information]: Found hostconfig ${hostconfig_path}" # Build Directory # When using /dev/shm, we use prefix for both spack builds and source build, unless BUILD_ROOT was defined @@ -151,16 +200,13 @@ cmake_exe=`grep 'CMake executable' ${hostconfig_path} | cut -d ':' -f 2 | xargs` # Build if [[ "${option}" != "--deps-only" && "${option}" != "--test-only" ]] then - date echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~ Host-config: ${hostconfig_path}" echo "~ Build Dir: ${build_dir}" echo "~ Project Dir: ${project_dir}" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Building RAJA Perf Suite" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Cleaning working directory" # Map CPU core allocations declare -A core_counts=(["lassen"]=40 ["ruby"]=28 ["poodle"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32) @@ -169,13 +215,9 @@ then if [[ -n ${raja_version} ]] then cd tpl/RAJA - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~ Updating RAJA Submodule to develop ~~~" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Updating RAJA Submodule to develop" git pull origin develop - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~ Updating Submodules within RAJA ~~~~~~" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Updating Submodules within RAJA" git submodule update --init --recursive cd - fi @@ -207,66 +249,48 @@ then ${project_dir} if ! $cmake_exe --build . -j ${core_counts[$truehostname]} then - echo "ERROR: compilation failed, building with verbose output..." + echo "[Error]: Compilation failed, building with verbose output..." + timed_message "Re-building with --verbose" $cmake_exe --build . --verbose -j 1 fi - date - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ RAJA Perf Suite Built" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "RAJA Perf Suite built" fi -if [[ ! -d ${build_dir} ]] +# Test +if [[ "${option}" != "--build-only" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} then - echo "ERROR: Build directory not found : ${build_dir}" && exit 1 -fi - -cd ${build_dir} - -date -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "~~~~~ TESTING RAJAPERF SUITE" -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -if grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} -then + if [[ ! -d ${build_dir} ]] + then + echo "[Error]: Build directory not found : ${build_dir}" && exit 1 + fi - # - # Maintaining separate, but identical release and debug sections - # in case we want to make them disctinct in the future. - # + cd ${build_dir} - echo "~~~~~~~~~ Run Command: ~~~~~~~~~~~~~~~~~~~~~" - echo "ctest --output-on-failure -T test 2>&1 | tee tests_output.txt" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - date + timed_message "Testing RAJA Perf Suite" ctest --output-on-failure -T test 2>&1 | tee tests_output.txt - date no_test_str="No tests were found!!!" if [[ "$(tail -n 1 tests_output.txt)" == "${no_test_str}" ]] then - echo "ERROR: No tests were found" && exit 1 + echo "[Error]: No tests were found" && exit 1 fi - echo "Copying Testing xml reports for export" + timed_message "Preparing tests xml reports for export" tree Testing xsltproc -o junit.xml ${project_dir}/blt/tests/ctest-to-junit.xsl Testing/*/Test.xml mv junit.xml ${project_dir}/junit.xml if grep -q "Errors while running CTest" ./tests_output.txt then - echo "ERROR: failure(s) while running CTest" && exit 1 + echo "[Error]: Failure(s) while running CTest" && exit 1 fi + + timed_message "RAJA Perf Suite tests completed" fi -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "~~~~~ CLEAN UP" -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +timed_message "Cleaning up" make clean -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "~~~~~ Build and test completed" -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -date +timed_message "Build and test completed"