diff --git a/host-configs b/host-configs new file mode 120000 index 000000000..c6592a966 --- /dev/null +++ b/host-configs @@ -0,0 +1 @@ +tpl/RAJA/host-configs \ No newline at end of file diff --git a/scripts/lc-builds/blueos_spectrum_nvcc_gcc_caliper.sh b/scripts/lc-builds/blueos_spectrum_nvcc_gcc_caliper.sh new file mode 100755 index 000000000..7877c1719 --- /dev/null +++ b/scripts/lc-builds/blueos_spectrum_nvcc_gcc_caliper.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2017-24, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJAPerf/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [[ $# -lt 6 ]]; then + echo + echo "You must pass 4 arguments to the script (in this order): " + echo " 1) compiler version number for spectrum mpi" + echo " 2) compiler version number for nvcc (number only, not 'sm_70' for example)" + echo " 3) CUDA compute architecture" + echo " 4) compiler version number for gcc. " + echo " 5) path to caliper cmake directory" + echo " 6) path to adiak cmake directory" + echo " 7...) optional arguments to cmake" + echo + echo "For example: " + echo " blueos_spectrum_nvcc_gcc.sh rolling-release 10.2.89 70 8.3.1" + exit +fi + +COMP_MPI_VER=$1 +COMP_NVCC_VER=$2 +COMP_ARCH=$3 +COMP_GCC_VER=$4 +CALI_DIR=$5 +ADIAK_DIR=$6 +shift 6 + +BUILD_SUFFIX=lc_blueos-spectrum-${COMP_MPI_VER}-nvcc-${COMP_NVCC_VER}-${COMP_ARCH}-gcc-${COMP_GCC_VER}_caliper +RAJA_HOSTCONFIG=../tpl/RAJA/host-configs/lc-builds/blueos/nvcc_gcc_X.cmake + +echo +echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo + +rm -rf build_${BUILD_SUFFIX} >/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + +module load cmake/3.23.1 + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DMPI_CXX_COMPILER=/usr/tce/packages/spectrum-mpi/spectrum-mpi-${COMP_MPI_VER}-gcc-${COMP_GCC_VER}/bin/mpig++ \ + -DCMAKE_CXX_COMPILER=/usr/tce/packages/gcc/gcc-${COMP_GCC_VER}/bin/g++ \ + -DBLT_CXX_STD=c++14 \ + -C ${RAJA_HOSTCONFIG} \ + -DENABLE_MPI=On \ + -DENABLE_OPENMP=On \ + -DENABLE_CUDA=On \ + -DCUDA_SEPARABLE_COMPILATION=On \ + -DCUDA_TOOLKIT_ROOT_DIR=/usr/tce/packages/cuda/cuda-${COMP_NVCC_VER} \ + -DCMAKE_CUDA_COMPILER=/usr/tce/packages/cuda/cuda-${COMP_NVCC_VER}/bin/nvcc \ + -DCMAKE_CUDA_ARCHITECTURES=${COMP_ARCH} \ + -DRAJA_PERFSUITE_USE_CALIPER=ON \ + -Dcaliper_DIR=${CALI_DIR} \ + -Dadiak_DIR=${ADIAK_DIR} \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. + +echo +echo "***********************************************************************" +echo +echo "cd into directory build_${BUILD_SUFFIX} and run make to build RAJA Perf Suite" +echo +echo " Please note that you have to run with mpi when you run" +echo " the RAJA Perf Suite; for example," +echo +echo " lrun -n4 ./bin/raja-perf.exe" +echo +echo "***********************************************************************" diff --git a/scripts/lc-builds/toss4_cray-mpich_amdclang_caliper.sh b/scripts/lc-builds/toss4_cray-mpich_amdclang_caliper.sh new file mode 100755 index 000000000..bbded66fa --- /dev/null +++ b/scripts/lc-builds/toss4_cray-mpich_amdclang_caliper.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2017-24, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJAPerf/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [[ $# -lt 5 ]]; then + echo + echo "You must pass 2 or more arguments to the script (in this order): " + echo " 1) cray-mpich compiler version number" + echo " 2) HIP compiler version number" + echo " 3) HIP compute architecture" + echo " 4) path to caliper cmake directory" + echo " 5) path to adiak cmake directory" + echo " 6...) optional arguments to cmake" + echo + echo "For example: " + echo " toss4_cray-mpich_amdclang.sh 8.1.14 4.1.0 gfx906 /usr/workspace/wsb/asde/caliper-quartz/share/cmake/caliper /usr/workspace/wsb/asde/caliper-quartz/lib/cmake/adiak" + exit +fi + +MPI_VER=$1 +COMP_VER=$2 +COMP_ARCH=$3 +CALI_DIR=$4 +ADIAK_DIR=$5 +shift 5 + +HOSTCONFIG="hip_3_X" + +if [[ ${COMP_VER} == 4.* ]] +then +##HIP_CLANG_FLAGS="-mllvm -amdgpu-fixed-function-abi=1" + HOSTCONFIG="hip_4_link_X" +elif [[ ${COMP_VER} == 3.* ]] +then + HOSTCONFIG="hip_3_X" +else + echo "Unknown hip version, using ${HOSTCONFIG} host-config" +fi + +# if [[ ${COMP_ARCH} == gfx90a ]] +# then + # note that unsafe atomics require use of coarse grain memory +##HIP_CLANG_FLAGS="-munsafe-fp-atomics" +# fi + +BUILD_SUFFIX=lc_toss4-cray-mpich-${MPI_VER}-amdclang-${COMP_VER}-${COMP_ARCH}_caliper +RAJA_HOSTCONFIG=../tpl/RAJA/host-configs/lc-builds/toss4/${HOSTCONFIG}.cmake + +echo +echo "Creating build directory ${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo +echo "To get cmake to work you may have to configure with" +echo " -DHIP_PLATFORM=amd" +echo +echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2" +echo " -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\"" +echo +echo "To work around some issues where *_FUSED kernels crash add these options" +echo " -DCMAKE_CXX_FLAGS=\"-fgpu-rdc\"" +echo " -DCMAKE_EXE_LINKER_FLAGS=\"-fgpu-rdc\"" +echo +echo "To work around some issues where *_FUSED kernels perform poorly use this environment variable" +echo " env HSA_SCRATCH_SINGLE_LIMIT=4000000000" +echo +echo "To work around some issues where the build fails with a weird error about max or fmax add these options" +echo " -DCMAKE_CXX_FLAGS=\"--hip-version={hip_version:ex=6.1.2}\"" +echo " -DCMAKE_EXE_LINKER_FLAGS=\"--hip-version={hip_version:ex=6.1.2}\"" +echo + + + +rm -rf build_${BUILD_SUFFIX} >/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + + +module load cmake/3.23.1 + +# unload rocm to avoid configuration problems where the loaded rocm and COMP_VER +# are inconsistent causing the rocprim from the module to be used unexpectedly +module unload rocm rocmcc + +if [[ "${COMP_VER}" == *-magic ]]; then + ROCM_PATH="/usr/tce/packages/rocmcc/rocmcc-${COMP_VER}" + MPI_ROCM_PATH="/usr/tce/packages/cray-mpich/cray-mpich-${MPI_VER}-rocmcc-${COMP_VER}" +else + ROCM_PATH="/opt/rocm-${COMP_VER}" + MPI_ROCM_PATH=/usr/tce/packages/cray-mpich-tce/cray-mpich-${MPI_VER}-rocmcc-${COMP_VER} +fi + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DMPI_C_COMPILER="${MPI_ROCM_PATH}/bin/mpiamdclang" \ + -DMPI_CXX_COMPILER="${MPI_ROCM_PATH}/bin/mpiamdclang++" \ + -DCMAKE_PREFIX_PATH="${ROCM_PATH}/lib/cmake" \ + -DHIP_PLATFORM=amd \ + -DROCM_ROOT_DIR="${ROCM_PATH}" \ + -DHIP_ROOT_DIR="${ROCM_PATH}/hip" \ + -DHIP_PATH="${ROCM_PATH}/llvm/bin" \ + -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/amdclang" \ + -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/amdclang++" \ + -DCMAKE_HIP_ARCHITECTURES="${COMP_ARCH}" \ + -DGPU_TARGETS="${COMP_ARCH}" \ + -DAMDGPU_TARGETS="${COMP_ARCH}" \ + -DBLT_CXX_STD=c++14 \ + -C ${RAJA_HOSTCONFIG} \ + -DENABLE_MPI=ON \ + -DENABLE_HIP=ON \ + -DENABLE_OPENMP=OFF \ + -DENABLE_CUDA=OFF \ + -DRAJA_PERFSUITE_USE_CALIPER=ON \ + -Dcaliper_DIR=${CALI_DIR} \ + -Dadiak_DIR=${ADIAK_DIR} \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. + +echo +echo "***********************************************************************" +echo +echo "cd into directory build_${BUILD_SUFFIX} and run make to build RAJA" +echo +echo " Please note that you have to have a consistent build environment" +echo " when you make RAJA as cmake may reconfigure; unload the rocm module" +echo " or load the appropriate rocm module (${COMP_VER}) when building." +echo +echo " module unload rocm" +echo " srun -n1 make" +echo +echo " Please note that rocm requires libpgmath.so from rocm/llvm to run." +echo " Until this is handled transparently in the build system you may add " +echo " rocm/llvm to your LD_LIBRARY_PATH." +echo +echo " export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rocm-${COMP_VER}/llvm/lib" +echo +echo "***********************************************************************"