omp statement around loop

env/levante.dkrz.de/shell

@@ -0,0 +1 @@
+shell.gnu

env/levante.dkrz.de/shell.gnu

@@ -0,0 +1,45 @@
+# make the contents as shell agnostic as possible so we can include them with bash, zsh and others
+export LC_ALL=en_US.UTF-8
+export CPU_MODEL=AMD_EPYC_ZEN3
+
+module --force purge
+
+module load git
+
+module load gcc/11.2.0-gcc-11.2.0
+
+module load openmpi/4.1.2-gcc-11.2.0
+
+module load netcdf-c/4.8.1-gcc-11.2.0
+module load netcdf-fortran/4.5.3-gcc-11.2.0
+
+export FC=mpif90 CC=mpicc CXX=mpicxx 
+
+# following is only needed for libblas which is needed by params lib and often provided by lapack
+#module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0
+# so use the LD_LIBRARY_PATH or other paths like prefix paths etc for cmake
+#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH
+spack load intel-oneapi-mkl@2022.0.1%gcc@11.2.0
+
+#other alternative blas
+#spack load [email protected]%[email protected]
+
+ulimit -s unlimited # without setting the stack size we get a segfault from the levante netcdf library at runtime
+ulimit -c 0 # do not create a coredump after a crash
+
+#OPENMPI specific runtime settings some deviation from ref:https://docs.dkrz.de/doc/levante/running-jobs/runtime-settings.html
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+# for most runs one may or may not want to disable HCOLL
+export OMPI_MCA_coll="^ml,hcoll"
+export OMPI_MCA_coll_hcoll_enable="0"
+export HCOLL_ENABLE_MCAST_ALL="0"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export UCX_HANDLE_ERRORS=bt

env/levante.dkrz.de/shell.intel

@@ -0,0 +1,36 @@
+# make the contents as shell agnostic as possible so we can include them with bash, zsh and others
+export LC_ALL=en_US.UTF-8
+export CPU_MODEL=AMD_EPYC_ZEN3
+
+module --force purge 
+
+module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0
+module load openmpi/4.1.2-intel-2021.5.0
+export FC=mpif90 CC=mpicc CXX=mpicxx ;
+spack load [email protected]%[email protected] # this handles adding to path elegantly then using hardcoded path below
+#module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0
+#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH
+
+module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
+module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0
+module load git # to be able to determine the fesom git SHA when compiling
+
+ulimit -s unlimited # without setting the stack size we get a segfault from the levante netcdf library at runtime
+ulimit -c 0 # do not create a coredump after a crash
+
+# environment for Open MPI 4.0.0 and later from https://docs.dkrz.de/doc/levante/running-jobs/runtime-settings.html
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+# for most runs one may or may not want to disable HCOLL
+export OMPI_MCA_coll="^ml,hcoll"
+export OMPI_MCA_coll_hcoll_enable="0"
+export HCOLL_ENABLE_MCAST_ALL="0"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export UCX_HANDLE_ERRORS=bt

env/levante.dkrz.de/shell.nvhpc

@@ -0,0 +1,29 @@
+# make the contents as shell agnostic as possible so we can include them with bash, zsh and others
+export LC_ALL=en_US.UTF-8
+export CPU_MODEL=AMD_EPYC_ZEN3
+
+# module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0
+# module load openmpi/4.1.2-intel-2021.5.0
+spack load [email protected]%[email protected]
+export FC=mpif90 CC=mpicc CXX=mpicxx;
+
+module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
+module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0
+module load git # to be able to determine the fesom git SHA when compiling
+
+# environment for Open MPI 4.0.0 and later from https://docs.dkrz.de/doc/levante/running-jobs/runtime-settings.html
+export OMPI_MCA_pml="ucx"
+export OMPI_MCA_btl=self
+export OMPI_MCA_osc="pt2pt"
+export UCX_IB_ADDR_TYPE=ib_global
+# for most runs one may or may not want to disable HCOLL
+export OMPI_MCA_coll="^ml,hcoll"
+export OMPI_MCA_coll_hcoll_enable="0"
+export HCOLL_ENABLE_MCAST_ALL="0"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
+export UCX_UNIFIED_MODE=y
+export HDF5_USE_FILE_LOCKING=FALSE
+export OMPI_MCA_io="romio321"
+export UCX_HANDLE_ERRORS=bt

src/oce_ale_tracer.F90

@@ -278,19 +278,19 @@ subroutine solve_tracers_ale(ice, dynamics, tracers, partit, mesh)
     end do
 !$OMP END PARALLEL DO
 
-!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nzmin, nzmax)
     !---age-code-begin
     if (use_age_tracer) then
+!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nzmin, nzmax)
       do node=1,myDim_nod2D+eDim_nod2D
         nzmax=nlevels_nod2D(node)-1
         nzmin=ulevels_nod2D(node)
         where (tracers%data(index_age_tracer)%values(nzmin:nzmax,node) < 0._WP )
                tracers%data(index_age_tracer)%values(nzmin:nzmax,node) = 0._WP
         end where
       end do
+!$OMP END PARALLEL DO
     end if
     !---age-code-end
-!$OMP END PARALLEL DO
 end subroutine solve_tracers_ale
 !
 !