refactoring with icebergs

CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.9)
 
 # set default build type cache entry (do so before project(...) is called, which would create this cache entry on its own)
 if(NOT CMAKE_BUILD_TYPE)
-	message(STATUS "setting default build type: Release")
-	set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.")
+  message(STATUS "setting default build type: Release")
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.")
 endif()
 
 project(FESOM2.0)
@@ -19,7 +19,7 @@ set(USE_ICEPACK OFF CACHE BOOL "compile fesom with the Iceapck modules for sea i
 set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical for reproducible results")
 set(USE_MULTIO OFF CACHE BOOL "Use MULTIO for IO, either grib or binary for now. This also means path to MULTIO installation has to provided using env MULTIO_INSTALL_PATH='..' and multio configuration yamls must be present to run the model with MULTIO")
 set(OASIS_WITH_YAC OFF CACHE BOOL "Useing a version of OASIS compiled with YAC instead of SCRIP for interpolation?")
-
-#set(VERBOSE OFF CACHE BOOL "toggle debug output")
+set(ASYNC_ICEBERGS ON CACHE BOOL "compile fesom with or without support for asynchronous iceberg computations")
+set(VERBOSE OFF CACHE BOOL "toggle debug output")
 #add_subdirectory(oasis3-mct/lib/psmile)
 add_subdirectory(src)

config/namelist.config

@@ -65,3 +65,11 @@ flag_debug=.false.
 n_levels=2
 n_part= 12, 36          ! 432 number of partitions on each hierarchy level
 /
+
+&icebergs
+use_icesheet_coupling=.false.
+ib_num=1
+use_icebergs=.false.
+steps_per_ib_step=8
+ib_async_mode=0
+/

config/namelist.forcing

@@ -23,6 +23,16 @@ ncar_bulk_z_shum=10.0 ! height at which humi forcing is located (CORE, JRA-do: 1
 use_landice_water=.false.
 landice_start_mon=5
 landice_end_mon=10
+fwf_path='./mesh/'
+
+/
+
+&age_tracer
+use_age_tracer=.false.
+use_age_mask=.false.
+age_tracer_path='./mesh/'
+age_start_year=2000
+
 /
 
 &nam_sbc

config/namelist.io

@@ -13,7 +13,7 @@ ldiag_trflx      =.false.
 /
 
 &nml_general
-io_listsize    =100 !number of streams to allocate. shallbe large or equal to the number of streams in &nml_list
+io_listsize    =120 !number of streams to allocate. shallbe large or equal to the number of streams in &nml_list
 vec_autorotate =.false.
 /
 
@@ -50,4 +50,7 @@ io_list =  'sst       ',1, 'm', 4,
            'bolus_u   ',1, 'y', 4,
            'bolus_v   ',1, 'y', 4,
            'bolus_w   ',1, 'y', 4,
+           'icb       ',1, 'm', 4,
+           'fw        ',1, 'm', 4,
+           'fh        ',1, 'm', 4,
 /

env/levante.dkrz.de/shell

@@ -1 +1 @@
-shell.intel
+shell.gnu

env/mistral.dkrz.de/shell-intel+openmpi

@@ -42,9 +42,5 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${NETCDFFROOT}/lib:${HDF5ROOT}/lib:${NET
 
 export FESOM_USE_CPLNG='active'
 
-export DR_HOOK=1
-export DR_HOOK_IGNORE_SIGNALS='-1'
-export DR_HOOK_OPT=prof
-export DR_HOOK_PROFILE_LIMIT=0.5
 export OIFS_DUMMY_ACTION=ABORT
 export HDF5_DISABLE_VERSION_CHECK=1

src/CMakeLists.txt

@@ -152,6 +152,9 @@ if(${FESOM_COUPLED})
    endif()
    target_compile_definitions(${PROJECT_NAME} PRIVATE __oasis)
 endif()
+if(${ASYNC_ICEBERGS})
+   target_compile_definitions(${PROJECT_NAME} PRIVATE __async_icebergs)
+endif()
 if(${OIFS_COUPLED})
    target_compile_definitions(${PROJECT_NAME} PRIVATE __oifs)
 endif()

src/MOD_DYN.F90

@@ -62,7 +62,11 @@ MODULE MOD_DYN
 
     ! sea surface height arrays
     real(kind=WP), allocatable, dimension(:)    :: eta_n, d_eta, ssh_rhs, ssh_rhs_old
-
+
+    ! LA: 2023-05-17 iceberg arrays
+    real(kind=WP), allocatable, dimension(:)    :: eta_n_ib ! kh 18.03.21 additional array for asynchronous iceberg computations
+    real(kind=WP), allocatable, dimension(:,:,:):: uv_ib    ! kh 18.03.21 additional array for asynchronous iceberg computations
+
     !___________________________________________________________________________
     ! summarizes solver input parameter
     type(t_solverinfo)                          :: solverinfo

src/MOD_ICE.F90

@@ -49,6 +49,7 @@ MODULE MOD_ICE
     !___________________________________________________________________________
     real(kind=WP) :: rhoair=1.3  , inv_rhoair=1./1.3  ! Air density & inverse ,  LY2004 !1.3 AOMIP
     real(kind=WP) :: rhowat=1025., inv_rhowat=1./1025.! Water density & inverse
+    real(kind=WP) :: rhofwt=1000., inv_rhofwt=1./1000.! Freshwter density & inverse
     real(kind=WP) :: rhoice=910. , inv_rhoice=1./910. ! Ice density & inverse, AOMIP
     real(kind=WP) :: rhosno=290. , inv_rhosno=1./290. ! Snow density & inverse, AOMIP
     ! Specific heat of air, ice, snow [J/(kg * K)]
@@ -111,9 +112,9 @@ MODULE MOD_ICE
 
     !___________________________________________________________________________
     ! zonal & merdional ice velocity
-    real(kind=WP), allocatable, dimension(:)    :: uice, uice_rhs, uice_old, uice_aux
-    real(kind=WP), allocatable, dimension(:)    :: vice, vice_rhs, vice_old, vice_aux
-
+    real(kind=WP), allocatable, dimension(:)    :: uice, uice_rhs, uice_old, uice_aux, uice_ib
+    real(kind=WP), allocatable, dimension(:)    :: vice, vice_rhs, vice_old, vice_aux, vice_ib
+    
     ! surface stess atm<-->ice, oce<-->ice
     real(kind=WP), allocatable, dimension(:)    :: stress_atmice_x, stress_iceoce_x
     real(kind=WP), allocatable, dimension(:)    :: stress_atmice_y, stress_iceoce_y
@@ -133,9 +134,17 @@ MODULE MOD_ICE
     ! total number of ice tracers (default=3, 1=area, 2=mice, 3=msnow, (4=ice_temp)
 #if defined (__oifs) || defined (__ifsinterface)
     integer                                     :: num_itracers=4
+#else
+!    integer                                     :: num_itracers=3
+    !------------------------------
+    ! LA 2023-01-31 add icebergs
+#if defined(__async_icebergs)
+    integer                                     :: num_itracers=5
 #else
     integer                                     :: num_itracers=3
-#endif
+#endif 
+    !------------------------------
+#endif 
 
     ! put ice tracers data arrays
     type(t_ice_data), allocatable, dimension(:) :: data
@@ -751,17 +760,19 @@ subroutine ice_init(ice, partit, mesh)
     ! to here since namelist.ice is now read in ice_init where whichEVP is not available
     ! when  mesh_auxiliary_arrays is called
     !array of 2D boundary conditions is used in ice_maEVP
-    if (ice%whichEVP > 0) then
+
+    ! LA 2023-05-24 initiate bc_index_nod2D also for whichEVP==0
+    !if (ice%whichEVP > 0) then
         allocate(mesh%bc_index_nod2D(myDim_nod2D+eDim_nod2D))
         mesh%bc_index_nod2D=1._WP
         do n=1, myDim_edge2D
             ed=mesh%edges(:, n)
             if (myList_edge2D(n) <= mesh%edge2D_in) cycle
             mesh%bc_index_nod2D(ed)=0._WP
         end do
-    end if
-
-end subroutine ice_init
+    !end if
+    
+end subroutine ice_init  
 !
 !
 !

src/MOD_MESH.F90

@@ -118,11 +118,15 @@ MODULE MOD_MESH
 !_______________________________________________________________________________
 ! Arrays added for ALE implementation:
 ! --> layer thinkness at node and depthlayer for t=n and t=n+1
+!------------------------------
 real(kind=WP), allocatable,dimension(:,:)   :: hnode
 real(kind=WP), allocatable,dimension(:,:)   :: hnode_new
 real(kind=WP), allocatable,dimension(:,:)   :: zbar_3d_n
 real(kind=WP), allocatable,dimension(:,:)   :: Z_3d_n
+! LA 2023-01-31 add icebergs
+real(kind=WP), allocatable,dimension(:,:)   :: Z_3d_n_ib
 
+!------------------------------
 ! --> layer thinkness at elements, interpolated from hnode
 real(kind=WP), allocatable,dimension(:,:)   :: helem
 
@@ -250,6 +254,7 @@ subroutine write_t_mesh(mesh, unit, iostat, iomsg)
     call write_bin_array(mesh%hnode_new,               unit, iostat, iomsg)
     call write_bin_array(mesh%zbar_3d_n,               unit, iostat, iomsg)
     call write_bin_array(mesh%Z_3d_n,                  unit, iostat, iomsg)
+    call write_bin_array(mesh%Z_3d_n_ib,               unit, iostat, iomsg)
     call write_bin_array(mesh%helem,                   unit, iostat, iomsg)
     call write_bin_array(mesh%bottom_elem_thickness,   unit, iostat, iomsg)
     call write_bin_array(mesh%bottom_node_thickness,   unit, iostat, iomsg)
@@ -346,6 +351,7 @@ subroutine read_t_mesh(mesh, unit, iostat, iomsg)
     call read_bin_array(mesh%hnode_new,               unit, iostat, iomsg)
     call read_bin_array(mesh%zbar_3d_n,               unit, iostat, iomsg)
     call read_bin_array(mesh%Z_3d_n,                  unit, iostat, iomsg)
+    call read_bin_array(mesh%Z_3d_n_ib,               unit, iostat, iomsg)
     call read_bin_array(mesh%helem,                   unit, iostat, iomsg)
     call read_bin_array(mesh%bottom_elem_thickness,   unit, iostat, iomsg)
     call read_bin_array(mesh%bottom_node_thickness,   unit, iostat, iomsg)

src/MOD_PARTIT.F90

@@ -33,11 +33,22 @@ module MOD_PARTIT
 end type com_struct
 
 TYPE T_PARTIT
+
+  !---------------------------------------------------
+  !LA 2023-01-31 add asynchronous icebergs
+  ! kh 10.02.21 communicator for async iceberg computations based on OpenMP
+  integer              :: MPI_COMM_FESOM_IB
+  !---------------------------------------------------
 
   type(com_struct) :: com_nod2D
   type(com_struct) :: com_elem2D
   type(com_struct) :: com_elem2D_full
 
+  !---------------------------------------------------
+  !LA 2023-01-31 add asynchronous icebergs
+  ! kh 11.02.21
+  integer            :: MPIERR_IB
+  !---------------------------------------------------
   integer              :: npes
   integer              :: mype
   integer              :: maxPEnum=100

src/associate_mesh_ass.h

@@ -54,6 +54,9 @@ hnode(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D)               => mesh%hnode(:,:)
 hnode_new(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D)           => mesh%hnode_new(:,:)
 zbar_3d_n(1:mesh%nl, 1:myDim_nod2D+eDim_nod2D)             => mesh%zbar_3d_n(:,:)
 Z_3d_n(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D)              => mesh%Z_3d_n(:,:)
+#if defined(__async_icebergs)
+Z_3d_n_ib(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D)           => mesh%Z_3d_n_ib(:,:)
+#endif
 helem(1:mesh%nl-1, 1:myDim_elem2D)                         => mesh%helem(:,:)
 bottom_elem_thickness(1:myDim_elem2D)                      => mesh%bottom_elem_thickness(:)
 bottom_node_thickness(1:myDim_nod2D+eDim_nod2D)            => mesh%bottom_node_thickness(:)

src/associate_mesh_def.h

@@ -38,6 +38,7 @@ real(kind=WP), dimension(:,:), pointer :: hnode
 real(kind=WP), dimension(:,:), pointer :: hnode_new
 real(kind=WP), dimension(:,:), pointer :: zbar_3d_n
 real(kind=WP), dimension(:,:), pointer :: Z_3d_n
+real(kind=WP), dimension(:,:), pointer :: Z_3d_n_ib
 real(kind=WP), dimension(:,:), pointer :: helem
 real(kind=WP), dimension(:)  , pointer :: bottom_elem_thickness
 real(kind=WP), dimension(:)  , pointer :: bottom_node_thickness

src/associate_part_ass.h

@@ -1,4 +1,5 @@
 MPI_COMM_FESOM  => partit%MPI_COMM_FESOM
+MPI_COMM_FESOM_IB       => partit%MPI_COMM_FESOM_IB
 com_nod2D       => partit%com_nod2D
 com_elem2D      => partit%com_elem2D
 com_elem2D_full => partit%com_elem2D_full
@@ -12,6 +13,7 @@ eDim_edge2D     => partit%eDim_edge2D
 pe_status       => partit%pe_status
 elem_full_flag  => partit%elem_full_flag
 MPIERR          => partit%MPIERR
+MPIERR_IB               => partit%MPIERR_IB
 npes            => partit%npes
 mype            => partit%mype
 maxPEnum        => partit%maxPEnum

src/associate_part_def.h

@@ -1,5 +1,5 @@
-
-  integer,          pointer     :: MPI_COMM_FESOM ! FESOM communicator (for ocean only runs if often a copy of MPI_COMM_WORLD)
+  integer,          pointer     :: MPI_COMM_FESOM       ! FESOM communicator (for ocean only runs if often a copy of MPI_COMM_WORLD)
+  integer,          pointer     :: MPI_COMM_FESOM_IB    ! FESOM communicator copy for icebergs LA: 2023-05-22
   type(com_struct), pointer     :: com_nod2D
   type(com_struct), pointer     :: com_elem2D
   type(com_struct), pointer     :: com_elem2D_full
@@ -17,6 +17,7 @@
   integer, dimension(:,:,:), pointer :: s_mpitype_nod3D, r_mpitype_nod3D
 
   integer, pointer   :: MPIERR
+  integer, pointer   :: MPIERR_IB       ! copy for icebergs LA: 2023-05-22
   integer, pointer   :: npes
   integer, pointer   :: mype
   integer, pointer   :: maxPEnum

src/cpl_driver.F90

@@ -13,7 +13,7 @@ module cpl_driver
   ! Modules used
   !
   use mod_oasis                    ! oasis module
-  use g_config, only : dt
+  use g_config, only : dt, use_icebergs, lwiso
   use o_param,  only : rad
   USE MOD_PARTIT
   implicit none
@@ -22,19 +22,27 @@ module cpl_driver
   ! Exchange parameters for coupling FESOM with ECHAM6
   !
 
+  !---wiso-code
+  ! define nsend and nrecv as variables instead of fixed parameters
+  ! (final number of fields depends now on lwiso switch and is set in subroutine cpl_oasis3mct_define_unstr)
+
 #if defined (__oifs)
-  integer, parameter         :: nsend = 7
-  integer, parameter         :: nrecv = 13
+  integer                    :: nsend = 7
+  integer                    :: nrecv = 13
 #else
-  integer, parameter         :: nsend = 4
-  integer, parameter         :: nrecv = 12
+  integer                    :: nsend = 4
+  integer                    :: nrecv = 12
 #endif
 
-  integer, dimension(nsend)  :: send_id
-  integer, dimension(nrecv)  :: recv_id
+  ! define send_id and recv_id with variable dimension as nsend and nrecv are now variables)
+  integer, allocatable, dimension(:) :: send_id
+  integer, allocatable, dimension(:) :: recv_id
+
+  ! define cpl_send and cpl_recv with variable dimension as nsend and nrecv are now variables)
+  character(len=32), allocatable, dimension(:) :: cpl_send
+  character(len=32), allocatable, dimension(:) :: cpl_recv
 
-  character(len=32)          :: cpl_send(nsend)
-  character(len=32)          :: cpl_recv(nrecv)
+  !---wiso-code-end
 
   character(len=16)          :: appl_name      ! application name for OASIS use
   character(len=16)          :: comp_name      ! name of this component
@@ -462,6 +470,14 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh)
     ! ... Some initialisation
     ! -----------------------------------------------------------------
 
+!---wiso-code
+    ALLOCATE(cpl_send(nsend))
+    ALLOCATE(cpl_recv(nrecv))
+
+    ALLOCATE(send_id(nsend))
+    ALLOCATE(recv_id(nrecv))
+!---wiso-code-end
+
     send_id = 0
     recv_id = 0
 
@@ -637,6 +653,17 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh)
     cpl_send( 2)='sit_feom' ! 2. sea ice thickness [m]             ->
     cpl_send( 3)='sie_feom' ! 3. sea ice extent [%-100]            ->
     cpl_send( 4)='snt_feom' ! 4. snow thickness [m]                ->
+!---wiso-code
+! add isotope coupling fields
+    IF (lwiso) THEN
+      cpl_send( 5)='o18w_oce' !                 -> h2o18 of ocean water
+      cpl_send( 6)='hdow_oce' !                 -> hdo16 of ocean water
+      cpl_send( 7)='o16w_oce' !                 -> h2o16 of ocean water
+      cpl_send( 8)='o18i_oce' !                 -> h2o18 of sea ice
+      cpl_send( 9)='hdoi_oce' !                 -> hdo16 of sea ice
+      cpl_send(10)='o16i_oce' !                 -> h2o16 of sea ice
+    END IF
+!---wiso-code-end
 #endif
 
 
@@ -672,6 +699,23 @@ subroutine cpl_oasis3mct_define_unstr(partit, mesh)
     cpl_recv(10) = 'heat_ico'
     cpl_recv(11) = 'heat_swo'    
     cpl_recv(12) = 'hydr_oce'
+! --- icebergs ---
+    IF (lwiso) THEN
+      cpl_recv(13) = 'w1_oce'
+      cpl_recv(14) = 'w2_oce'
+      cpl_recv(15) = 'w3_oce'
+      cpl_recv(16) = 'i1_oce'
+      cpl_recv(17) = 'i2_oce'
+      cpl_recv(18) = 'i3_oce'
+      IF (use_icebergs) THEN
+        cpl_recv(19) = 'u10w_oce'
+        cpl_recv(20) = 'v10w_oce'
+      END IF
+    ELSE IF (use_icebergs) THEN
+      cpl_recv(13) = 'u10w_oce'
+      cpl_recv(14) = 'v10w_oce'
+    END IF
+! --- icebergs ---
 #endif
 
     if (mype .eq. 0) then