From dcbc5f6cf77d5516266f5c0b9bc7c91c94f3b67f Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <harmenstoppels@gmail.com>
Date: Mon, 8 Feb 2021 08:50:04 +0100
Subject: [PATCH] Make nvToolsExt conditional on WITH_CUDA_PROFILING (#428)

* Make nvToolsExt conditional on WITH_CUDA_PROFILING

* no need for these flags on mc partition w/o calls to srun

* Use --hint=nomultithread and work around --exclusive bug

* make mkdir consistent with other test.sh files

* Bring back -Werror in Release

* Revert "no need for these flags on mc partition w/o calls to srun"

This reverts commit a6fa6a83b18b5855b28c9bbee6802d8ed635e2b0.

* Make the build flags similar to the run flags again
---
 .ci/daint.cscs.ch/cray.build.sh   | 3 +--
 .ci/daint.cscs.ch/cray.test.sh    | 3 +--
 .ci/daint.cscs.ch/gnu.build.sh    | 3 +--
 .ci/daint.cscs.ch/gnu.test.sh     | 3 +--
 .ci/daint.cscs.ch/intel.build.sh  | 3 +--
 .ci/daint.cscs.ch/intel.test.sh   | 3 +--
 .ci/daint.cscs.ch/ocl.build.sh    | 3 +--
 .ci/daint.cscs.ch/ocl.test.sh     | 6 ++----
 cmake/CompilerConfiguration.cmake | 2 +-
 src/CMakeLists.txt                | 2 +-
 10 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/.ci/daint.cscs.ch/cray.build.sh b/.ci/daint.cscs.ch/cray.build.sh
index 82016b666a0..bbc3d113988 100755
--- a/.ci/daint.cscs.ch/cray.build.sh
+++ b/.ci/daint.cscs.ch/cray.build.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="mc"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
diff --git a/.ci/daint.cscs.ch/cray.test.sh b/.ci/daint.cscs.ch/cray.test.sh
index ee5df55151b..f98212e1e06 100755
--- a/.ci/daint.cscs.ch/cray.test.sh
+++ b/.ci/daint.cscs.ch/cray.test.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="gpu"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
diff --git a/.ci/daint.cscs.ch/gnu.build.sh b/.ci/daint.cscs.ch/gnu.build.sh
index 76984955c40..4683e776836 100755
--- a/.ci/daint.cscs.ch/gnu.build.sh
+++ b/.ci/daint.cscs.ch/gnu.build.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="mc"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
diff --git a/.ci/daint.cscs.ch/gnu.test.sh b/.ci/daint.cscs.ch/gnu.test.sh
index 2ed9c13b050..49f7aba7d63 100755
--- a/.ci/daint.cscs.ch/gnu.test.sh
+++ b/.ci/daint.cscs.ch/gnu.test.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="gpu"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
diff --git a/.ci/daint.cscs.ch/intel.build.sh b/.ci/daint.cscs.ch/intel.build.sh
index 4a92733d87d..4b33b585269 100755
--- a/.ci/daint.cscs.ch/intel.build.sh
+++ b/.ci/daint.cscs.ch/intel.build.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="mc"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
diff --git a/.ci/daint.cscs.ch/intel.test.sh b/.ci/daint.cscs.ch/intel.test.sh
index 0e3497a7cf0..b4f211382d6 100755
--- a/.ci/daint.cscs.ch/intel.test.sh
+++ b/.ci/daint.cscs.ch/intel.test.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="gpu"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
diff --git a/.ci/daint.cscs.ch/ocl.build.sh b/.ci/daint.cscs.ch/ocl.build.sh
index 52c153ee929..c846815ff1f 100755
--- a/.ci/daint.cscs.ch/ocl.build.sh
+++ b/.ci/daint.cscs.ch/ocl.build.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="mc"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
diff --git a/.ci/daint.cscs.ch/ocl.test.sh b/.ci/daint.cscs.ch/ocl.test.sh
index c7a180bd22e..797e2d60dca 100755
--- a/.ci/daint.cscs.ch/ocl.test.sh
+++ b/.ci/daint.cscs.ch/ocl.test.sh
@@ -1,13 +1,12 @@
 #!/bin/bash -l
 
 #SBATCH --export=ALL
-#SBATCH --exclusive
 #SBATCH --constraint="gpu"
 #SBATCH --partition="cscsci"
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=4
 #SBATCH --cpus-per-task=3
-#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT
+#SBATCH --hint=nomultithread
 
 set -o errexit
 set -o nounset
@@ -22,8 +21,7 @@ set -o xtrace  # do not set earlier to avoid noise from module
 
 umask 0002  # make sure group members can access the data
 
-mkdir -p "${SCRATCH}/${BUILD_TAG}.ocl"
-chmod 0775 "${SCRATCH}/${BUILD_TAG}.ocl"
+mkdir --mode=0775 -p "${SCRATCH}/${BUILD_TAG}.ocl"
 cd "${SCRATCH}/${BUILD_TAG}.ocl"
 
 export CRAY_CUDA_MPS=1 # enable the CUDA proxy for MPI+CUDA
diff --git a/cmake/CompilerConfiguration.cmake b/cmake/CompilerConfiguration.cmake
index 351ea8d53b7..af29b4dd0a2 100644
--- a/cmake/CompilerConfiguration.cmake
+++ b/cmake/CompilerConfiguration.cmake
@@ -41,7 +41,7 @@ Please open an issue at https://github.com/cp2k/dbcsr/issues with the reported c
 endif ()
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  set(CMAKE_CXX_FLAGS_RELEASE      "-O3 -g -funroll-loops -Wall")
+  set(CMAKE_CXX_FLAGS_RELEASE      "-O3 -g -funroll-loops -Wall -Werror")
   set(CMAKE_CXX_FLAGS_COVERAGE     "-O0 -g --coverage -Wall -Werror")
   set(CMAKE_CXX_FLAGS_DEBUG        "-O2 -ggdb -Wall -Werror -fsanitize=undefined -fsanitize=address -fsanitize-recover=all")
   if ((NOT (USE_MPI)) OR (NOT ("${MPI_Fortran_LIBRARY_VERSION_STRING}" MATCHES "Open MPI")))
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 351373e29fa..763df46897f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -233,8 +233,8 @@ if (USE_ACCEL)
     dbcsr
     PRIVATE $<$<STREQUAL:${USE_ACCEL},cuda>:CUDA::cudart>
             $<$<STREQUAL:${USE_ACCEL},cuda>:CUDA::cublas>
-            $<$<STREQUAL:${USE_ACCEL},cuda>:CUDA::nvToolsExt>
             $<$<STREQUAL:${USE_ACCEL},cuda>:CUDA::nvrtc>
+            $<$<BOOL:${WITH_CUDA_PROFILING}>:CUDA::nvToolsExt>
             $<$<STREQUAL:${USE_ACCEL},hip>:roc::hipblas>
             $<$<STREQUAL:${USE_ACCEL},hip>:hip::host>
             $<$<STREQUAL:${USE_ACCEL},opencl>:OpenCL::OpenCL>)