Merge branch 'refactoring' into refactoring_update_partialcells

CMakeLists.txt

@@ -14,6 +14,7 @@ set(OIFS_COUPLED OFF CACHE BOOL "compile fesom coupled to OpenIFS. (Also needs F
 set(CRAY OFF CACHE BOOL "compile with cray ftn")
 set(USE_ICEPACK OFF CACHE BOOL "compile fesom with the Iceapck modules for sea ice column physics.")
 set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical for reproducible results")
+
 #set(VERBOSE OFF CACHE BOOL "toggle debug output")
 #add_subdirectory(oasis3-mct/lib/psmile)
 add_subdirectory(src)

config/namelist.io

@@ -7,6 +7,7 @@ ldiag_salt3D     =.false.
 ldiag_dMOC       =.false.
 ldiag_DVD        =.false.
 ldiag_forc       =.false.
+ldiag_extflds    =.false.
 /
 
 &nml_general

configure_any.sh

@@ -18,7 +18,7 @@ source env.sh # source this from your run script too
 if [[ ${LIB} = yes ]]; then
   mkdir build.lib || true # build dir for library
   cd build.lib
-  cmake -DBUILD_FESOM_AS_LIBRARY=ON .. # not required when re-compiling
+  cmake -DBUILD_FESOM_AS_LIBRARY=ON -DFESOM_INSTALL_PREFIX=/perm/${USER}/fesom2/ -DBUILD_SHARED_LIBS=ON -DDISABLE_MULTITHREADING=ON -DENABLE_OPENMP=ON .. # not required when re-compiling
   sed -i -e 's/-lFALSE//g' src/CMakeFiles/fesom.dir/link.txt # workaround for the moment on cray
 else
   mkdir build || true # build dir for binary

env.sh

@@ -27,7 +27,7 @@ fi
 
 if [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then
    STRATEGY="mistral.dkrz.de"
-elif [[ $LOGINHOST =~ ^l[A-Za-z0-9]+\.atos\.local$ ]]; then
+elif [[ $LOGINHOST =~ ^l[A-Za-z0-9]+\.lvt\.dkrz\.de$ ]]; then
    STRATEGY="levante.dkrz.de"
 elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then
    STRATEGY="ollie"

env/albedo/shell

@@ -1,9 +1,23 @@
 # make the contents as shell agnostic as possible so we can include them with bash, zsh and others
-
 module load intel-oneapi-compilers 
-module load intel-oneapi-mkl/2022.1.0
+export FC="mpiifort -qmkl" CC=mpiicc CXX=mpiicpc
 module load intel-oneapi-mpi/2021.6.0
-export FC=mpiifort CC=mpiicc CXX=mpiicpc
 
+module load intel-oneapi-mkl/2022.1.0
 module load netcdf-fortran/4.5.4-intel-oneapi-mpi2021.6.0-oneapi2022.1.0
-module load netcdf-c/4.8.1-intel-oneapi-mpi2021.6.0-oneapi2022.1.0
+module load netcdf-c/4.8.1-intel-oneapi-mpi2021.6.0-oneapi2022.1.0
+
+# from DKRZ recommented environment variables on levante
+# (https://docs.dkrz.de/doc/levante/running-jobs/runtime-settings.html) 
+export HCOLL_ENABLE_MCAST_ALL="0"
+export HCOLL_MAIN_IB=mlx5_0:1
+export UCX_IB_ADDR_TYPE=ib_global
+export UCX_NET_DEVICES=mlx5_0:1
+export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self # this line here brings the most speedup factor ~1.5
+export UCX_UNIFIED_MODE=y
+export UCX_HANDLE_ERRORS=bt
+export HDF5_USE_FILE_LOCKING=FALSE
+export I_MPI_PMI=pmi2
+export I_MPI_PMI_LIBRARY=/usr/lib64/libpmi2.so
+
+export ENABLE_ALBEDO_INTELMPI_WORKAROUNDS=''

env/atosecmwf/shell

@@ -21,7 +21,7 @@ module unload gcc
 
 # Load modules
 module load prgenv/intel
-module load intel/2021.2.0
+module load intel/2021.4.0
 module load hpcx-openmpi/2.9.0
 module load intel-mkl/19.0.5
 module load fftw/3.3.9

src/CMakeLists.txt

@@ -13,6 +13,12 @@ if(DEFINED ENV{ENABLE_ALEPH_CRAYMPICH_WORKAROUNDS}) # be able to set the initial
 else()
    option(ALEPH_CRAYMPICH_WORKAROUNDS "workaround for performance issues on aleph" OFF)
 endif()
+if(DEFINED ENV{ENABLE_ALBEDO_INTELMPI_WORKAROUNDS}) # be able to set the initial cache value from our env settings for aleph, not only via cmake command
+   option(ALBEDO_INTELMPI_WORKAROUNDS "workaround for performance issues on albedo" ON)
+else()
+   option(ALBEDO_INTELMPI_WORKAROUNDS "workaround for performance issues on albedo" OFF)
+endif()
+
 
 if(ALEPH_CRAYMPICH_WORKAROUNDS)
    # todo: enable these options only for our targets
@@ -24,8 +30,11 @@ if(ALEPH_CRAYMPICH_WORKAROUNDS)
    #add_compile_options(-DDISABLE_PARALLEL_RESTART_READ) # reading restarts is slow when doing it on parallel on aleph, switch it off for now
    add_compile_options(-DENABLE_ALEPH_CRAYMPICH_WORKAROUNDS)
 endif()
+if(ALBEDO_INTELMPI_WORKAROUNDS)
+   add_compile_options(-DENABLE_ALBEDO_INTELMPI_WORKAROUNDS)
+endif()
 
-option(DISABLE_MULTITHREADING "disable asynchronous operations" OFF)
+option(DISABLE_MULTITHREADING "disable asynchronous operations" ON)
 option(ENABLE_OPENACC "compile with OpenACC support" OFF)
 set(NV_GPU_ARCH "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)")
 
@@ -129,13 +138,17 @@ if(${CMAKE_Fortran_COMPILER_ID} STREQUAL  Intel )
    else()
         target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin)
    endif()
-#   target_compile_options(${PROJECT_NAME} PRIVATE -qopenmp -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
-#   target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
    if(${FESOM_PLATFORM_STRATEGY} STREQUAL  levante.dkrz.de )
       target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2)
+   elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL  albedo)
+      target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -O3 -ip -fPIC -qopt-malloc-options=2 -qopt-prefetch=5 -unroll-aggressive) #NEC mpi option
    else()
       target_compile_options(${PROJECT_NAME} PRIVATE -xHost)
    endif()
+#    target_compile_options(${PROJECT_NAME} PRIVATE -g -traceback ) #-check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
+#    target_compile_options(${PROJECT_NAME} PRIVATE -qopenmp -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
+#    target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -g -traceback -check all,noarg_temp_created,bounds,uninit ) #-ftrapuv ) #-init=zero)
+
 elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL  GNU )
 #    target_compile_options(${PROJECT_NAME} PRIVATE -O3 -finit-local-zero  -finline-functions -fimplicit-none  -fdefault-real-8 -ffree-line-length-none)
    target_compile_options(${PROJECT_NAME} PRIVATE -O2 -g -ffloat-store -finit-local-zero  -finline-functions -fimplicit-none  -fdefault-real-8 -ffree-line-length-none)

src/MOD_PARTIT.F90

@@ -64,6 +64,7 @@ module MOD_PARTIT
   integer            :: npes
   integer            :: mype
   integer            :: maxPEnum=100
+!PS   logical            :: flag_debug=.false.
   integer, allocatable, dimension(:)  :: part
 
   ! Mesh partition
@@ -151,6 +152,7 @@ subroutine WRITE_T_PARTIT(partit, unit, iostat, iomsg)
     write(unit, iostat=iostat, iomsg=iomsg) partit%npes
     write(unit, iostat=iostat, iomsg=iomsg) partit%mype
     write(unit, iostat=iostat, iomsg=iomsg) partit%maxPEnum
+!PS     write(unit, iostat=iostat, iomsg=iomsg) partit%flag_debug
     call write_bin_array(partit%part,           unit, iostat, iomsg)
 
     write(unit, iostat=iostat, iomsg=iomsg) partit%myDim_nod2D
@@ -182,6 +184,7 @@ subroutine READ_T_PARTIT(partit, unit, iostat, iomsg)
     read(unit, iostat=iostat, iomsg=iomsg) partit%npes
     read(unit, iostat=iostat, iomsg=iomsg) partit%mype
     read(unit, iostat=iostat, iomsg=iomsg) partit%maxPEnum
+!PS     read(unit, iostat=iostat, iomsg=iomsg) partit%flag_debug
     call read_bin_array(partit%part,           unit, iostat, iomsg)
 
     read(unit, iostat=iostat, iomsg=iomsg) partit%myDim_nod2D

src/gen_model_setup.F90

@@ -30,6 +30,9 @@ subroutine setup_model(partit)
   read (fileunit, NML=run_config)
 !!$  read (fileunit, NML=machine)
   close (fileunit)
+
+!PS   partit%flag_debug=flag_debug
+
   ! ==========
   ! compute dt
   ! ========== 

src/gen_modules_diag.F90

@@ -16,14 +16,14 @@ module diagnostics
   implicit none
 
   private
-  public :: ldiag_solver, lcurt_stress_surf, ldiag_energy, ldiag_dMOC, ldiag_DVD,       &
-            ldiag_forc, ldiag_salt3D, ldiag_curl_vel3, diag_list, ldiag_vorticity,      &
-            compute_diagnostics, rhs_diag, curl_stress_surf, curl_vel3, wrhof, rhof,    &
-            u_x_u, u_x_v, v_x_v, v_x_w, u_x_w, dudx, dudy, dvdx, dvdy, dudz, dvdz,      & 
-            utau_surf, utau_bott, av_dudz_sq, av_dudz, av_dvdz, stress_bott, u_surf,    &
-            v_surf, u_bott, v_bott, std_dens_min, std_dens_max, std_dens_N, std_dens,   &
-            std_dens_UVDZ, std_dens_DIV, std_dens_Z, std_dens_dVdT, std_dens_flux,      &
-            dens_flux_e, vorticity, compute_diag_dvd_2ndmoment_klingbeil_etal_2014,                &
+  public :: ldiag_solver, lcurt_stress_surf, ldiag_energy, ldiag_dMOC, ldiag_DVD,                  &
+            ldiag_forc, ldiag_salt3D, ldiag_curl_vel3, diag_list, ldiag_vorticity, ldiag_extflds,  &
+            compute_diagnostics, rhs_diag, curl_stress_surf, curl_vel3, wrhof, rhof,               &
+            u_x_u, u_x_v, v_x_v, v_x_w, u_x_w, dudx, dudy, dvdx, dvdy, dudz, dvdz,                 & 
+            utau_surf, utau_bott, av_dudz_sq, av_dudz, av_dvdz, stress_bott, u_surf,               &
+            v_surf, u_bott, v_bott, std_dens_min, std_dens_max, std_dens_N, std_dens,              &
+            std_dens_UVDZ, std_dens_DIV, std_dens_Z, std_dens_dVdT, std_dens_flux,                 &
+            dens_flux_e, vorticity, zisotherm, compute_diag_dvd_2ndmoment_klingbeil_etal_2014,  &
             compute_diag_dvd_2ndmoment_burchard_etal_2008, compute_diag_dvd
   ! Arrays used for diagnostics, some shall be accessible to the I/O
   ! 1. solver diagnostics: A*x=rhs? 
@@ -37,7 +37,8 @@ module diagnostics
   real(kind=WP),  save, allocatable, target      :: utau_surf(:), utau_bott(:)
   real(kind=WP),  save, allocatable, target      :: stress_bott(:,:), u_bott(:), v_bott(:), u_surf(:), v_surf(:)
   real(kind=WP),  save, allocatable, target      :: vorticity(:,:)
-
+  real(kind=WP),  save, allocatable, target      :: zisotherm(:)             !target temperature is specified as whichtemp in compute_extflds
+  real(kind=WP),  save, allocatable, target      :: tempzavg(:), saltzavg(:)  !target depth for averaging is specified as whichdepth in compute_extflds
 ! defining a set of standard density bins which will be used for computing densMOC
 ! integer,        parameter                      :: std_dens_N  = 100
 ! real(kind=WP),  save, target                   :: std_dens(std_dens_N)
@@ -74,9 +75,10 @@ module diagnostics
   logical                                       :: ldiag_forc       =.false.
 
   logical                                       :: ldiag_vorticity  =.false.
+  logical                                       :: ldiag_extflds    =.false.
 
   namelist /diag_list/ ldiag_solver, lcurt_stress_surf, ldiag_curl_vel3, ldiag_energy, &
-                       ldiag_dMOC, ldiag_DVD, ldiag_salt3D, ldiag_forc, ldiag_vorticity
+                       ldiag_dMOC, ldiag_DVD, ldiag_salt3D, ldiag_forc, ldiag_vorticity, ldiag_extflds
 
   contains
 
@@ -780,6 +782,81 @@ subroutine relative_vorticity(mode, dynamics, partit, mesh)
 
 ! Now it the relative vorticity known on neighbors too
 end subroutine relative_vorticity
+!
+!
+!_______________________________________________________________________________
+subroutine compute_extflds(mode, dynamics, tracers, partit, mesh)
+    IMPLICIT NONE
+    integer,        intent(in)              :: mode
+    logical,        save                    :: firstcall=.true.
+    type(t_dyn)   , intent(in),     target  :: dynamics
+    type(t_tracer), intent(in)   ,  target  :: tracers
+    type(t_partit), intent(inout),  target  :: partit
+    type(t_mesh)  , intent(in)   ,  target  :: mesh
+    real(kind=WP),  dimension(:,:), pointer :: temp, salt
+    real(kind=WP)                           :: zn, zint, tup, tlo
+    integer                                 :: n, nz, nzmin, nzmax
+    real(kind=WP)                           :: whichtemp=  20.0_WP ! which isotherm to compute (set 20 per default)
+    real(kind=WP)                           :: whichdepth=300.0_WP ! for which tepth to average for tempzavg & saltzavg
+
+#include "associate_part_def.h"
+#include "associate_mesh_def.h"
+#include "associate_part_ass.h"
+#include "associate_mesh_ass.h" 
+    if (firstcall) then  !allocate the stuff at the first call
+        allocate(zisotherm(myDim_nod2D+eDim_nod2D))
+        allocate(tempzavg(myDim_nod2D+eDim_nod2D), saltzavg(myDim_nod2D+eDim_nod2D))
+        zisotherm=0.0_WP
+        tempzavg =0.0_WP
+        saltzavg =0.0_WP
+        firstcall=.false.
+        if (mode==0) return
+    end if  
+    temp   => tracers%data(1)%values(:,:)
+    salt   => tracers%data(2)%values(:,:)    
+
+!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax, zn, tup, tlo)
+    DO n=1, myDim_nod2D
+       saltzavg(n) =0.0_WP
+       nzmax=nlevels_nod2D(n)
+       nzmin=ulevels_nod2D(n)
+       zn  =0.0_WP
+       do nz=nzmin+1, nzmax-1
+          tup=temp(nz-1, n)
+          if (tup < whichtemp) exit
+          tlo=temp(nz,   n)                 
+          if ((tup-whichtemp)*(tlo-whichtemp)<0) then
+             zn=zn+0.5_WP*(hnode(nz-1, n)+(whichtemp-tup)*sum(hnode(nz-1:nz, n))/(tlo-tup))
+             exit
+          end if
+          zn=zn+hnode(nz-1, n) 
+       end do
+       zisotherm(n)=zn
+    END DO
+!$OMP END PARALLEL DO 
+
+!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax, zint)
+    DO n=1, myDim_nod2D
+       tempzavg(n) =0.0_WP
+       saltzavg(n) =0.0_WP
+       nzmax=nlevels_nod2D(n)
+       nzmin=ulevels_nod2D(n)
+       zint=0.0_WP
+       do nz=nzmin, nzmax-1
+          zint=zint+hnode(nz, n) 
+          tempzavg(n)=tempzavg(n)+temp(nz,   n)*hnode(nz, n)
+          saltzavg(n)=saltzavg(n)+salt(nz,   n)*hnode(nz, n)
+          if (zint>=whichdepth) exit
+       end do
+       tempzavg(n)=tempzavg(n)/zint
+       saltzavg(n)=saltzavg(n)/zint
+    END DO
+!$OMP END PARALLEL DO 
+
+  call exchange_nod(zisotherm, partit)
+  call exchange_nod(tempzavg, partit)
+  call exchange_nod(saltzavg, partit)
+end subroutine compute_extflds
 
 
 
@@ -813,6 +890,8 @@ subroutine compute_diagnostics(mode, dynamics, tracers, partit, mesh)
 
   ! compute relative vorticity
   if (ldiag_vorticity)   call relative_vorticity(mode, dynamics, partit, mesh)
+  ! soe exchanged fields requested by IFS/FESOM in NextGEMS.
+  if (ldiag_extflds)     call compute_extflds(mode, dynamics, tracers, partit, mesh)
 
 end subroutine compute_diagnostics