From 9213bca0ab94fca5c27302a213ede8afab9d8c71 Mon Sep 17 00:00:00 2001 From: Patrick Scholz Date: Wed, 10 May 2023 11:01:51 +0200 Subject: [PATCH 01/50] add fortran file for ssh split explicite subcycling --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/oce_ale_ssh_splitexpl_subcycl.F90 diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 new file mode 100644 index 000000000..e69de29bb From 8210cc572b89d0d3724c0118f3775a59228eaf63 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 10 May 2023 14:17:08 +0200 Subject: [PATCH 02/50] add additional variables for split-explicite ssh subcycling computation --- src/MOD_DYN.F90 | 27 ++++++++++++++++++++++-- src/oce_setup_step.F90 | 47 +++++++++++++++++++++++++++--------------- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index 67e780a18..5c3f05356 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -49,7 +49,7 @@ MODULE MOD_DYN ! set main structure for dynamicss, contains viscosity options and parameters + ! option for momentum advection TYPE T_DYN -!___________________________________________________________________________ + !___________________________________________________________________________ ! instant zonal merdional velocity & Adams-Bashfort rhs real(kind=WP), allocatable, dimension(:,:,:):: uv, uv_rhs, uv_rhsAB, fer_uv @@ -62,6 +62,19 @@ MODULE MOD_DYN ! sea surface height arrays real(kind=WP), allocatable, dimension(:) :: eta_n, d_eta, ssh_rhs, ssh_rhs_old + !___arrays for split explicite ssh computation______________________________ + ! se_uvh...transport velocity, + real(kind=WP), allocatable, dimension(:,:,:):: se_uvh + !se_uv_rhs...vertical integral of transport velocity rhs, se_uvh_BT4AB... + ! barotropic transport velocities (vertically integrated), contains actual + ! timestep (1:2) and previous timestep (3:4) for adams-bashfort interpolation + real(kind=WP), allocatable, dimension(:,:) :: se_uvh_rhs, se_uvh_BT4AB + + ! se_uvBT...barotropic trnasport velocities from barotropic time stepping + ! se_uvBT_theta...velocities for dissipative time stepping of thickness equation + ! UBTmean_mean... Mean BT velocity to trim 3D velocity in tracers + real(kind=WP), allocatable, dimension(:,:) :: se_uvBT, se_uvBT_theta, se_uvBT_mean + !___________________________________________________________________________ ! summarizes solver input parameter type(t_solverinfo) :: solverinfo @@ -100,7 +113,17 @@ MODULE MOD_DYN logical :: use_wsplit = .false. ! maximum allowed CFL criteria in vertical (0.5 < w_max_cfl < 1.) ! in older FESOM it used to be w_exp_max=1.e-3 - real(kind=WP) :: wsplit_maxcfl= 1.0 + real(kind=WP) :: wsplit_maxcfl = 1.0 + + ! switch between ssh computation, by solver or split explicite subcycling + ! use_ssh_splitexpl_subcycl = .false. --> solver + ! use_ssh_splitexpl_subcycl = .true. --> split explicite subcycling + logical :: use_ssh_splitexpl_subcycl = .false. + + ! barotropic subcycling time-steps + integer :: splitexpl_BTsteps = 40 + + !___________________________________________________________________________ ! energy diagnostic part: will be computed inside the model ("hard integration"): logical :: ldiag_ke = .true. ! different contributions to velocity change. will be computed inside the code. diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index c8eb89913..773fa2672 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -393,7 +393,8 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) real(kind=WP) :: wsplit_maxcfl namelist /dynamics_visc / opt_visc, visc_gamma0, visc_gamma1, visc_gamma2, & use_ivertvisc, visc_easybsreturn - namelist /dynamics_general/ momadv_opt, use_freeslip, use_wsplit, wsplit_maxcfl, ldiag_KE + namelist /dynamics_general/ momadv_opt, use_freeslip, use_wsplit, wsplit_maxcfl, & + ldiag_KE, use_ssh_splitexpl_subcycl !___________________________________________________________________________ ! pointer on necessary derived types #include "associate_part_def.h" @@ -417,17 +418,18 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) !___________________________________________________________________________ ! set parameters in derived type - dynamics%opt_visc = opt_visc - dynamics%visc_gamma0 = visc_gamma0 - dynamics%visc_gamma1 = visc_gamma1 - dynamics%visc_gamma2 = visc_gamma2 - dynamics%visc_easybsreturn = visc_easybsreturn - dynamics%use_ivertvisc = use_ivertvisc - dynamics%momadv_opt = momadv_opt - dynamics%use_freeslip = use_freeslip - dynamics%use_wsplit = use_wsplit - dynamics%wsplit_maxcfl = wsplit_maxcfl - dynamics%ldiag_KE = ldiag_KE + dynamics%opt_visc = opt_visc + dynamics%visc_gamma0 = visc_gamma0 + dynamics%visc_gamma1 = visc_gamma1 + dynamics%visc_gamma2 = visc_gamma2 + dynamics%visc_easybsreturn = visc_easybsreturn + dynamics%use_ivertvisc = use_ivertvisc + dynamics%momadv_opt = momadv_opt + dynamics%use_freeslip = use_freeslip + dynamics%use_wsplit = use_wsplit + dynamics%wsplit_maxcfl = wsplit_maxcfl + dynamics%ldiag_KE = ldiag_KE + dynamics%use_ssh_splitexpl_subcycl = use_ssh_splitexpl_subcycl !___________________________________________________________________________ ! define local vertice & elem array size elem_size=myDim_elem2D+eDim_elem2D @@ -469,14 +471,25 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) !___________________________________________________________________________ ! allocate/initialise ssh arrays in derived type allocate(dynamics%eta_n( node_size)) - allocate(dynamics%d_eta( node_size)) - allocate(dynamics%ssh_rhs( node_size)) dynamics%eta_n = 0.0_WP - dynamics%d_eta = 0.0_WP - dynamics%ssh_rhs = 0.0_WP + !!PS allocate(dynamics%ssh_rhs_old(node_size)) !!PS dynamics%ssh_rhs_old= 0.0_WP - + + if (dynamics%use_ssh_splitexpl_subcycl) then + allocate(dynamics%se_uvh( 2, nl-1, elem_size)) + allocate(dynamics%se_uvh_rhs( 2, elem_size)) + allocate(dynamics%se_uvh_BT4AB(4, elem_size)) + dynamics%se_uvh = 0.0_WP + dynamics%se_uvh_rhs = 0.0_WP + dynamics%se_uvh_BT4AB = 0.0_WP + else + allocate(dynamics%d_eta( node_size)) + allocate(dynamics%ssh_rhs( node_size)) + dynamics%d_eta = 0.0_WP + dynamics%ssh_rhs = 0.0_WP + end if + !___________________________________________________________________________ ! inititalise working arrays allocate(dynamics%work%uvnode_rhs(2, nl-1, node_size)) From f4cc3d1937e3eb20e483019ba491f0bf7da9099d Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 10 May 2023 14:50:17 +0200 Subject: [PATCH 03/50] add additional variables for split-explicite ssh subcycling computation --- src/MOD_DYN.F90 | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index 5c3f05356..da5c1bded 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -253,7 +253,8 @@ subroutine WRITE_T_DYN(dynamics, unit, iostat, iomsg) write(unit, iostat=iostat, iomsg=iomsg) dynamics%use_freeslip write(unit, iostat=iostat, iomsg=iomsg) dynamics%use_wsplit write(unit, iostat=iostat, iomsg=iomsg) dynamics%wsplit_maxcfl - + write(unit, iostat=iostat, iomsg=iomsg) dynamics%use_ssh_splitexpl_subcycl + !___________________________________________________________________________ call dynamics%solverinfo%WRITE_T_SOLVERINFO(unit) @@ -273,6 +274,14 @@ subroutine WRITE_T_DYN(dynamics, unit, iostat, iomsg) call write_bin_array(dynamics%fer_w , unit, iostat, iomsg) call write_bin_array(dynamics%fer_uv, unit, iostat, iomsg) end if + if (dynamics%use_ssh_splitexpl_subcycl) then + call write_bin_array(dynamics%se_uvh , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvh_rhs , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvh_BT4AB , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) + end if end subroutine WRITE_T_DYN @@ -299,6 +308,7 @@ subroutine READ_T_DYN(dynamics, unit, iostat, iomsg) read(unit, iostat=iostat, iomsg=iomsg) dynamics%use_freeslip read(unit, iostat=iostat, iomsg=iomsg) dynamics%use_wsplit read(unit, iostat=iostat, iomsg=iomsg) dynamics%wsplit_maxcfl + read(unit, iostat=iostat, iomsg=iomsg) dynamics%use_ssh_splitexpl_subcycl !___________________________________________________________________________ call dynamics%solverinfo%READ_T_SOLVERINFO(unit) @@ -319,6 +329,14 @@ subroutine READ_T_DYN(dynamics, unit, iostat, iomsg) call read_bin_array(dynamics%fer_w , unit, iostat, iomsg) call read_bin_array(dynamics%fer_uv , unit, iostat, iomsg) end if + if (dynamics%use_ssh_splitexpl_subcycl) then + call read_bin_array(dynamics%se_uvh , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvh_rhs , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvh_BT4AB , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) + end if end subroutine READ_T_DYN From 1d781cae61939e6feacdcd942a88104e5e05f904 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 12 May 2023 10:44:08 +0200 Subject: [PATCH 04/50] improve flag for use_ssh_splitexpl_subcycl=.false. --- src/oce_setup_step.F90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index 773fa2672..4aaf5c6b6 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -391,6 +391,7 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) logical :: use_wsplit =.false. logical :: ldiag_KE =.false. real(kind=WP) :: wsplit_maxcfl + logical :: use_ssh_splitexpl_subcycl=.false. namelist /dynamics_visc / opt_visc, visc_gamma0, visc_gamma1, visc_gamma2, & use_ivertvisc, visc_easybsreturn namelist /dynamics_general/ momadv_opt, use_freeslip, use_wsplit, wsplit_maxcfl, & @@ -486,8 +487,8 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) else allocate(dynamics%d_eta( node_size)) allocate(dynamics%ssh_rhs( node_size)) - dynamics%d_eta = 0.0_WP - dynamics%ssh_rhs = 0.0_WP + dynamics%d_eta = 0.0_WP + dynamics%ssh_rhs = 0.0_WP end if !___________________________________________________________________________ From 8f7e0fdffc624e1b37efee7c252fc309c63d467e Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 12 May 2023 10:45:26 +0200 Subject: [PATCH 05/50] add subroutine momentum_adv_scalar_4splitexpl for momentum advection when split-explicite scheme is used --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 433 ++++++++++++++++++++++++++ 1 file changed, 433 insertions(+) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index e69de29bb..9bafb91e1 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -0,0 +1,433 @@ +module momentum_adv_scalar_4splitexpl_interface + interface + subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) + use mod_mesh + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + end subroutine + end interface +end module + +! +! +!_______________________________________________________________________________ +! Transports are used instead of velocities, Urhs, Vrhs are also for transports. +subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + use MOD_DYN + USE o_PARAM + use g_comm_auto + IMPLICIT NONE + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + !___________________________________________________________________________ + integer :: node, elem, ed, nz + integer :: nl1, ul1, nl2, ul2, nl12, ul12 + real(kind=WP) :: uv12, uv1, uv2, qc, qu, qd, num_ord=0.95_WP + integer :: ednodes(2), edelem(2) + real(kind=WP) :: dx1, dy1, dx2, dy2, uu, vv + real(kind=WP) :: wu(mesh%nl), wv(mesh%nl), un1(mesh%nl), un2(mesh%nl) + + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:,:), pointer :: UV, UV_rhsAB, UVnode_rhs, UVnode, UVh + real(kind=WP), dimension(:,:) , pointer :: Wvel_e +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + UV =>dynamics%uv(:,:,:) + UV_rhsAB =>dynamics%uv_rhsAB(:,:,:) + UVnode_rhs=>dynamics%work%uvnode_rhs(:,:,:) + UVnode =>dynamics%uvnode(:,:,:) + Wvel_e =>dynamics%w_e(:,:) + UVh =>dynamics%se_uvh(:,:,:) + !___________________________________________________________________________ + ! 1st. compute vertical momentum advection component: w * du/dz, w*dv/dz +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(node, elem, ed, nz, nl1, ul1, nl2, ul2, nl12, ul12, & +!$OMP uv1, uv2, uv12, qc, qu, qd, wu, wv, & +!$OMP ednodes, edelem, dx1, dy1, dx2, dy2, un, uu, vv) + +!$OMP DO + do node=1, myDim_nod2D + nl1 = nlevels_nod2D(node) + ul1 = ulevels_nod2D(node) + + !_______________________________________________________________________ + ! surface + nz=ul1 + wu(nz)=UVnode(1, nz, node)*Wvel_e(nz, node)*area(nz, node) + wv(nz)=UVnode(2, nz, node)*Wvel_e(nz, node)*area(nz, node) + + !_______________________________________________________________________ + ! subsurface --> centered 2nd order + nz=ul1+1 ! Central differences at the second and last but one levels + wu(nz)=0.5_WP*(UVnode(1, nz, node)+UVnode(1, nz-1, node))*Wvel_e(nz, node)*area(nz, node) + wv(nz)=0.5_WP*(UVnode(2, nz, node)+UVnode(2, nz-1, node))*Wvel_e(nz, node)*area(nz, node) + + !_______________________________________________________________________ + ! bulk --> centered 4th order (num_ord=1), upwind 3rd order (num_ord=0) + do nz=ul1+2, nl1-2 + qc = (UVnode(1, nz-1, node)-UVnode(1, nz , node))/(hnode(nz-1, node)+hnode(nz , node)) ! factor 2 is accounted for later + qu = (UVnode(1, nz , node)-UVnode(1, nz+1, node))/(hnode(nz , node)+hnode(nz+1, node)) + qd = (UVnode(1, nz-2, node)-UVnode(1, nz-1, node))/(hnode(nz-2, node)+hnode(nz-1, node)) + + uv1 = UVnode(1, nz , node)+(2*qc+qu)*hnode(nz , node)/6.0_WP ! Gradient reconstruction 2(2qc+qu)(h/2)(1/6) + uv2 = UVnode(1, nz-1, node)-(2*qc+qd)*hnode(nz-1, node)/6.0_WP + uv12 = (Wvel_e(nz, node)+abs(Wvel_e(nz, node)))*uv1+ & + (Wvel_e(nz, node)-abs(Wvel_e(nz, node)))*uv2 + wu(nz)=0.5_WP*(num_ord*(uv1+uv2)*Wvel_e(nz, node)+(1.0_WP-num_ord)*uv12)*area(nz, node) + + qc = (UVnode(2, nz-1, node)-UVnode(2, nz , node))/(hnode(nz-1, node)+hnode(nz , node)) + qu = (UVnode(2, nz , node)-UVnode(2, nz+1, node))/(hnode(nz , node)+hnode(nz+1, node)) + qd = (UVnode(2, nz-2, node)-UVnode(2, nz-1, node))/(hnode(nz-2, node)+hnode(nz-1, node)) + + uv1 = UVnode(2, nz , node)+(2*qc+qu)*hnode(nz , node)/6.0_WP + uv2 = UVnode(2, nz-1, node)-(2*qc+qd)*hnode(nz-1, node)/6.0_WP + uv12 = (Wvel_e(nz, node)+abs(Wvel_e(nz, node)))*uv1+ & + (Wvel_e(nz, node)-abs(Wvel_e(nz, node)))*uv2 + wv(nz)=0.5_WP*(num_ord*(uv1+uv2)*Wvel_e(nz, node)+(1.0_WP-num_ord)*uv12)*area(nz, node) + end do ! --> do nz=ul1+2, nl1-2 + + !_______________________________________________________________________ + ! one layer above bottom --> centered 2nd order + nz=nl1-1 + wu(nz)=0.5_WP*(UVnode(1, nz, node)+UVnode(1, nz-1, node))*Wvel_e(nz, node)*area(nz, node) + wv(nz)=0.5_WP*(UVnode(2, nz, node)+UVnode(2, nz-1, node))*Wvel_e(nz, node)*area(nz, node) + + !_______________________________________________________________________ + ! bottom layer --> boundary condition + nz=nl1 + wu(nz)=0.0_WP + wv(nz)=0.0_WP + + !_______________________________________________________________________ + ! set to the rhs for transports, not velocities!!! --> No division by h + do nz=1, nl1-1 + UVnode_rhs(1, nz, node)= -(wu(nz)-wu(nz+1)) + UVnode_rhs(2, nz, node)= -(wv(nz)-wv(nz+1)) + end do + + end do ! --> do node=1, myDim_nod2D +!$OMP END DO + + + + !___________________________________________________________________________ + ! 2nd. compute horizontal advection component: u*du/dx, u*dv/dx & v*du/dy, v*dv/dy + ! loop over triangle edges +!$OMP DO + do ed=1, myDim_edge2D + ! local indice of nodes that span up edge ed + ednodes = edges(:,ed) + + ! local index of element that contribute to edge + edelem = edge_tri(1:2,ed) + + !_______________________________________________________________________ + ! index off surface layer in case of cavity !=1 and index of mid depth + ! bottom layer + nl1 = nlevels(edelem(1))-1 + ul1 = ulevels(edelem(1)) + + !_______________________________________________________________________ + !NR --> Natalja Style + un1 = 0.0_WP + un1(ul1:nl1) = ( UVh(2, ul1:nl1, edelem(1))*edge_cross_dxdy(1,ed) & + - UVh(1, ul1:nl1, edelem(1))*edge_cross_dxdy(2,ed)) + + !_______________________________________________________________________ + ! if edelem(2)==0 than edge is boundary edge + if(edelem(2)>0) then + nl2 = nlevels(edelem(2))-1 + ul2 = ulevels(edelem(2)) + + !___________________________________________________________________ + !NR --> Natalja Style + un2 = 0.0_WP + un2(ul2:nl2) = -( UVh(2, ul2:nl2, edelem(2))*edge_cross_dxdy(3,ed) & + - UVh(1, ul2:nl2, edelem(2))*edge_cross_dxdy(4,ed)) + + !___________________________________________________________________ + ! nl12 ... minimum number of layers -1 between element edelem(1) & edelem(2) that + ! contribute to edge ed + ! nu12 ... upper index of layers between element edelem(1) & edelem(2) that + ! contribute to edge ed + ! be carefull !!! --> if ed is a boundary edge than edelem(1)~=0 and edelem(2)==0 + ! that means nl1>0, nl2==0, nl12=min(nl1,nl2)=0 !!! + ul12 = max(ul1, ul2) + nl12 = min(nl1, nl2) + + !___________________________________________________________________ + ! ensure openmp numerical reproducability +#if defined(__openmp_reproducible) +!$OMP ORDERED +#endif + !___________________________________________________________________ + !NR add contribution to first edge node --> ednodes(1) + !NR Do not calculate on Halo nodes, as the result will not be used. + !NR The "if" is cheaper than the avoided computiations. + if (ednodes(1) <= myDim_nod2d) then +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock(partit%plock(ednodes(1))) +#endif + ! cavity domain where only edelem(1) exist + do nz=ul1 , ul12-1 + UVnode_rhs(1, nz, ednodes(1)) = UVnode_rhs(1, nz, ednodes(1)) + un1(nz)*UV(1, nz, edelem(1)) + UVnode_rhs(2, nz, ednodes(1)) = UVnode_rhs(2, nz, ednodes(1)) + un1(nz)*UV(2, nz, edelem(1)) + end do + ! cavity domain where only edelem(2) exist + do nz=ul2 , ul12-1 + UVnode_rhs(1, nz, ednodes(1)) = UVnode_rhs(1, nz, ednodes(1)) + un2(nz)*UV(1, nz, edelem(2)) + UVnode_rhs(2, nz, ednodes(1)) = UVnode_rhs(2, nz, ednodes(1)) + un2(nz)*UV(2, nz, edelem(2)) + end do + ! bulk domain where edelem(1) and edelem(2) exist + do nz=ul12, nl12 + UVnode_rhs(1, nz, ednodes(1)) = UVnode_rhs(1, nz, ednodes(1)) + (un1(nz) + un2(nz))*(UV(1, nz, edelem(1))+UV(1, nz, edelem(2)))*0.5_WP + UVnode_rhs(2, nz, ednodes(1)) = UVnode_rhs(2, nz, ednodes(1)) + (un1(nz) + un2(nz))*(UV(2, nz, edelem(1))+UV(2, nz, edelem(2)))*0.5_WP + end do + ! bottom domain where only edelem(1) exist + do nz=nl12+1, nl1 + UVnode_rhs(1, nz, ednodes(1)) = UVnode_rhs(1, nz, ednodes(1)) + un1(nz)*UV(1, nz, edelem(1)) + UVnode_rhs(2, nz, ednodes(1)) = UVnode_rhs(2, nz, ednodes(1)) + un1(nz)*UV(2, nz, edelem(1)) + end do + ! bottom domain where only edelem(2) exist + do nz=nl12+1, nl2 + UVnode_rhs(1, nz, ednodes(1)) = UVnode_rhs(1, nz, ednodes(1)) + un2(nz)*UV(1, nz, edelem(2)) + UVnode_rhs(2, nz, ednodes(1)) = UVnode_rhs(2, nz, ednodes(1)) + un2(nz)*UV(2, nz, edelem(2)) + end do +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(1))) +#endif + end if + !___________________________________________________________________ + !NR add contribution to second edge node --> ednodes(2) + !NR Do not calculate on Halo nodes, as the result will not be used. + !NR The "if" is cheaper than the avoided computiations. + if (ednodes(2) <= myDim_nod2d) then +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock(partit%plock(ednodes(2))) +#endif + ! cavity domain where only edelem(1) exist + do nz=ul1 , ul12-1 + UVnode_rhs(1, nz, ednodes(2)) = UVnode_rhs(1, nz, ednodes(2)) - un1(nz)*UV(1, nz, edelem(1)) + UVnode_rhs(2, nz, ednodes(2)) = UVnode_rhs(2, nz, ednodes(2)) - un1(nz)*UV(2, nz, edelem(1)) + end do + ! cavity domain where only edelem(2) exist + do nz=ul2 , ul12-1 + UVnode_rhs(1, nz, ednodes(2)) = UVnode_rhs(1, nz, ednodes(2)) - un2(nz)*UV(1, nz, edelem(2)) + UVnode_rhs(2, nz, ednodes(2)) = UVnode_rhs(2, nz, ednodes(2)) - un2(nz)*UV(2, nz, edelem(2)) + end do + ! bulk domain where edelem(1) and edelem(2) exist + do nz=ul12, nl12 + UVnode_rhs(1, nz, ednodes(2)) = UVnode_rhs(1, nz, ednodes(2)) - (un1(nz) + un2(nz))*(UV(1, nz, edelem(1))+UV(1, nz, edelem(2)))*0.5_WP + UVnode_rhs(2, nz, ednodes(2)) = UVnode_rhs(2, nz, ednodes(2)) - (un1(nz) + un2(nz))*(UV(2, nz, edelem(1))+UV(2, nz, edelem(2)))*0.5_WP + end do + ! bottom domain where only edelem(1) exist + do nz=nl12+1, nl1 + UVnode_rhs(1, nz, ednodes(2)) = UVnode_rhs(1, nz, ednodes(2)) - un1(nz)*UV(1, nz, edelem(1)) + UVnode_rhs(2, nz, ednodes(2)) = UVnode_rhs(2, nz, ednodes(2)) - un1(nz)*UV(2, nz, edelem(1)) + end do + ! bottom domain where only edelem(2) exist + do nz=nl12+1, nl2 + UVnode_rhs(1, nz, ednodes(2)) = UVnode_rhs(1, nz, ednodes(2)) - un2(nz)*UV(1, nz, edelem(2)) + UVnode_rhs(2, nz, ednodes(2)) = UVnode_rhs(2, nz, ednodes(2)) - un2(nz)*UV(2, nz, edelem(2)) + end do +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(2))) +#endif + end if + + !_______________________________________________________________________ + ! if edelem(2)==0 than edge is boundary edge + else ! --> if(edelem(2)>0) then + !___________________________________________________________________ + !NR add contribution to first edge node --> ednodes(1) + !NR Do not calculate on Halo nodes, as the result will not be used. + !NR The "if" is cheaper than the avoided computiations. + if (ednodes(1) <= myDim_nod2d) then +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock(partit%plock(ednodes(1))) +#endif + ! bulk domain where only edelem(1) exist + do nz=ul1 , nl1 + UVnode_rhs(1, nz, ednodes(1)) = UVnode_rhs(1, nz, ednodes(1)) + un1(nz)*UV(1, nz, edelem(1)) + UVnode_rhs(2, nz, ednodes(1)) = UVnode_rhs(2, nz, ednodes(1)) + un1(nz)*UV(2, nz, edelem(1)) + end do +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(1))) +#endif + end if + !___________________________________________________________________ + !NR add contribution to second edge node --> ednodes(2) + !NR Do not calculate on Halo nodes, as the result will not be used. + !NR The "if" is cheaper than the avoided computiations. + if (ednodes(2) <= myDim_nod2d) then +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock(partit%plock(ednodes(2))) +#endif + ! bulk domain where only edelem(1) exist + do nz=ul1 , nl1-1 + UVnode_rhs(1, nz, ednodes(2)) = UVnode_rhs(1, nz, ednodes(2)) - un1(nz)*UV(1, nz, edelem(1)) + UVnode_rhs(2, nz, ednodes(2)) = UVnode_rhs(2, nz, ednodes(2)) - un1(nz)*UV(2, nz, edelem(1)) + end do +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(2))) +#endif + end if + end if ! --> if(edelem(2)>0) then + +#if defined(__openmp_reproducible) +!$OMP END ORDERED +#endif + + end do ! --> do ed=1, myDim_edge2D +!$OMP END DO + + +!PS !_______________________________________________________________________ +!PS ! nl12 ... minimum number of layers -1 between element edelem(1) & edelem(2) that +!PS ! contribute to edge ed +!PS ! nu12 ... upper index of layers between element edelem(1) & edelem(2) that +!PS ! contribute to edge ed +!PS ! be carefull !!! --> if ed is a boundary edge than edelem(1)~=0 and edelem(2)==0 +!PS ! that means nl1>0, nl2==0, nl12=min(nl1,nl2)=0 !!! +!PS ul12 = max(ul1, ul2) +!PS nl12 = min(nl1, nl2) +!PS +!PS !_______________________________________________________________________ +!PS ! (A1) goes only into this loop when the edge has only facing element +!PS ! edelem(1) --> so the edge is a boundary edge --> this is for ocean +!PS ! surface in case of cavity +!PS do nz=nu1, nu12-1 +!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) +!PS uu=un*UV(1, nz, edelem(1)) ! the momentum to be carried depends on velocities +!PS vv=un*UV(2, nz, edelem(1)) +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=nu1, nu12-1 +!PS +!PS !_______________________________________________________________________ +!PS ! (A2) goes only into this loop when the edge has a facing elemenmt +!PS ! edelem(2) --> so the edge is a boundary edge --> this is for ocean +!PS ! surface in case of cavity +!PS if (nu2 > 0) then +!PS do nz=nu2, nu12-1 +!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) +!PS uu=un*UV(1, nz, edelem(2)) +!PS vv=un*UV(2, nz, edelem(2)) +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=nu2, nu12-1 +!PS end if +!PS +!PS !_______________________________________________________________________ +!PS ! (B) Both segments +!PS ! loop over depth layers from shared upper layer index nu12 to shared +!PS ! lower layer index nl12 +!PS do nz=nu12, nl12 +!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) & +!PS -(UVh(2, nz, edelem(2))*x2 - UVh(1, nz, edelem(2))*y2) +!PS uu=un*(UV(1, nz, edelem(1)) + UV(1, nz, edelem(2)))*0.5_WP! the momentum to be carried depends on velocities +!PS vv=un*(UV(2, nz, edelem(1)) + UV(2, nz, edelem(2)))*0.5_WP +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=nu12, nl12 +!PS +!PS !_______________________________________________________________________ +!PS ! (C1) remaining segments from the shared lower lyer index nl12 to bottom +!PS ! of element edelem(1) +!PS do nz=nl12+1, nl1 +!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) +!PS uu=un*UV(1, nz, edelem(1)) ! the momentum to be carried depends on velocities +!PS vv=un*UV(2, nz, edelem(1)) +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=nl12+1, nl1 +!PS +!PS !_______________________________________________________________________ +!PS ! (C2) remaining segments from the shared lower lyer index nl12 to bottom +!PS ! of element edelem(1) +!PS do nz=nl12+1, nl2 +!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) +!PS uu=un*UV(1, nz, edelem(2)) +!PS vv=un*UV(2, nz, edelem(2)) +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=nl12+1, nl2 +!PS end do ! --> do ed=1, myDim_edge2D +!PS !$OMP END DO + + + !___________________________________________________________________________ + ! divide total nodal momentum advection by scalar area +!$OMP DO + do node=1,myDim_nod2d + nl1 = nlevels_nod2D(node)-1 + ul1 = ulevels_nod2D(node) + UVnode_rhs(1, ul1:nl1, node) = UVnode_rhs(1, ul1:nl1, node) * areasvol_inv(ul1:nl1, node) + UVnode_rhs(2, ul1:nl1, node) = UVnode_rhs(2, ul1:nl1, node) * areasvol_inv(ul1:nl1, node) + end do ! --> do node=1,myDim_nod2d +!$OMP END DO + + !___________________________________________________________________________ +!$OMP MASTER + call exchange_nod(UVnode_rhs, partit) +!$OMP END MASTER +!$OMP BARRIER + + !___________________________________________________________________________ + ! convert total nodal advection from vertice --> elements +!$OMP DO + do elem=1, myDim_elem2D + nl1 = nlevels(elem)-1 + ul1 = ulevels(elem) + UV_rhsAB(1:2, ul1:nl1, elem) = UV_rhsAB(1:2, ul1:nl1, elem) + elem_area(elem)* & + ( UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(1, elem)) & + + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(2, elem)) & + + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(3, elem))) / 3.0_WP + end do ! --> do edelem=1, myDim_elem2D +!$OMP END DO + + !___________________________________________________________________________ + ! for energz diagnostic + if (dynamics%ldiag_ke) then !we repeat the computation here and there are multiple ways to speed it up +!$OMP DO + do elem=1, myDim_elem2D + nl1 = nlevels(elem)-1 + ul1 = ulevels(elem) + dynamics%ke_adv_AB(1:2, ul1:nl1, elem) = dynamics%ke_adv_AB(1:2, ul1:nl1, elem) + elem_area(elem)* & + ( UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(1, elem)) & + + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(2, elem)) & + + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(3, elem))) / 3.0_WP + end do +!$OMP END DO + end if +!$OMP END PARALLEL +end subroutine + + + + + + + + + From e524698be6c45483166ed3a037c7c1cb5979d2e9 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 12 May 2023 10:52:05 +0200 Subject: [PATCH 06/50] add transport velocties --- src/oce_ale_vel_rhs.F90 | 44 ++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index b951767cb..7ec1a8d1e 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -46,6 +46,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) use g_comm_auto use g_sbf, only: l_mslp use momentum_adv_scalar_interface + use momentum_adv_scalar_4splitexpl_interface implicit none type(t_ice) , intent(inout), target :: ice type(t_dyn) , intent(inout), target :: dynamics @@ -64,6 +65,8 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) real(kind=WP), dimension(:) , pointer :: eta_n real(kind=WP), dimension(:) , pointer :: m_ice, m_snow, a_ice real(kind=WP) , pointer :: rhoice, rhosno, inv_rhowat + real(kind=WP), dimension(:,:,:), pointer :: UVh + real(kind=WP), dimension(:,:) , pointer :: UVh_BT4AB #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" @@ -78,6 +81,12 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) rhosno => ice%thermo%rhosno inv_rhowat=> ice%thermo%inv_rhowat + ! if split-explicite ssh subcycling is used + if (dynamics%use_ssh_splitexpl_subcycl) then + UVh => dynamics%se_uvh + UVh_BT4AB => dynamics%se_uvh_BT4AB + end if + !___________________________________________________________________________ use_pice=0 if (use_floatice .and. .not. trim(which_ale)=='linfs') use_pice=1 @@ -136,26 +145,39 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) ! apply pressure gradient force, as well as contributions from gradient of ! the sea surface height as well as ice pressure in case of floating sea ice ! to velocity rhs - pre = -(p_eta+p_ice+p_air) - if (use_global_tides) then pre=pre-ssh_gp(elnodes) end if Fx = sum(gradient_sca(1:3,elem)*pre) Fy = sum(gradient_sca(4:6,elem)*pre) - do nz=nzmin,nzmax-1 ! add pressure gradient terms UV_rhs(1,nz,elem) = UV_rhs(1,nz,elem) + (Fx-pgf_x(nz,elem))*elem_area(elem) UV_rhs(2,nz,elem) = UV_rhs(2,nz,elem) + (Fy-pgf_y(nz,elem))*elem_area(elem) - - ! add coriolis force - UV_rhsAB(1,nz,elem) = UV(2,nz,elem)*ff! + mm*UV(1,nz,elem)*UV(2,nz,elem) - UV_rhsAB(2,nz,elem) =-UV(1,nz,elem)*ff! - mm*UV(1,nz,elem)*UV(2,nz,elem) end do - + + !____________________________________________________________________________ + ! when ssh split-explicite subcycling method is setted use transport velocities + ! u*h, v*h instead of u,v + if (.not. dynamics%use_ssh_splitexpl_subcycl) then + do nz=nzmin,nzmax-1 + ! add coriolis force + UV_rhsAB(1,nz,elem) = UV(2,nz,elem)*ff! + mm*UV(1,nz,elem)*UV(2,nz,elem) + UV_rhsAB(2,nz,elem) =-UV(1,nz,elem)*ff! - mm*UV(1,nz,elem)*UV(2,nz,elem) + end do + else + UVh_BT4AB(1:2,elem) = 0.0_WP + do nz=nzmin,nzmax-1 + UV_rhsAB(1, nz ,elem) = UVh(1, nz, elem)*ff! + mm*UV(1,nz,elem)*UVh(1, nz, elem) + UV_rhsAB(2, nz ,elem) =-UVh(2, nz, elem)*ff! - mm*UV(1,nz,elem)*UVh(2, nz, elem) + UVh_BT4AB(1,elem) = UVh_BT4AB(1, elem) + UVh(1, nz, elem) ! Barotropic velocity + UVh_BT4AB(2,elem) = UVh_BT4AB(2, elem) + UVh(2, nz, elem) ! + end do + end if + + !___________________________________________________________________________ if (dynamics%ldiag_ke) then do nz=nzmin,nzmax-1 dynamics%ke_pre(1,nz,elem)= (Fx-pgf_x(nz,elem))*dt!*elem_area(elem) !not to divide it aterwards (at the end of this subroutine) @@ -178,7 +200,11 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) if (mype==0) write(*,*) 'in moment not adapted mom_adv advection typ for ALE, check your namelist' call par_ex(partit%MPI_COMM_FESOM, partit%mype, 1) elseif (dynamics%momadv_opt==2) then - call momentum_adv_scalar(dynamics, partit, mesh) + if (.not. dynamics%use_ssh_splitexpl_subcycl) then + call momentum_adv_scalar(dynamics, partit, mesh) + else + call momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) + end if end if !___________________________________________________________________________ ! Update the rhs From 9260839cc30b8972780fcd203b387ce0815d235e Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 12 May 2023 15:33:03 +0200 Subject: [PATCH 07/50] addapt visc filter towards transport velocity update --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 7 +- src/oce_dyn.F90 | 105 +++++++++++++++++++------- 2 files changed, 78 insertions(+), 34 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 9bafb91e1..a81b1b26b 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -20,9 +20,9 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT USE MOD_PARSUP - use MOD_DYN + USE MOD_DYN USE o_PARAM - use g_comm_auto + USE g_comm_auto IMPLICIT NONE type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit @@ -32,7 +32,6 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) integer :: nl1, ul1, nl2, ul2, nl12, ul12 real(kind=WP) :: uv12, uv1, uv2, qc, qu, qd, num_ord=0.95_WP integer :: ednodes(2), edelem(2) - real(kind=WP) :: dx1, dy1, dx2, dy2, uu, vv real(kind=WP) :: wu(mesh%nl), wv(mesh%nl), un1(mesh%nl), un2(mesh%nl) !___________________________________________________________________________ @@ -119,7 +118,6 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) !$OMP END DO - !___________________________________________________________________________ ! 2nd. compute horizontal advection component: u*du/dx, u*dv/dx & v*du/dy, v*dv/dy ! loop over triangle edges @@ -291,7 +289,6 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) end do ! --> do ed=1, myDim_edge2D !$OMP END DO - !PS !_______________________________________________________________________ !PS ! nl12 ... minimum number of layers -1 between element edelem(1) & edelem(2) that !PS ! contribute to edge ed diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index 67d34e777..585836159 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -424,6 +424,8 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) END DO !$OMP END PARALLEL DO + !___________________________________________________________________________ + ! Sum up velocity differences over edge with respect to elemtnal index !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(u1, v1, len, vi, ed, el, nz, nzmin, nzmax) !$OMP DO DO ed=1, myDim_edge2D+eDim_edge2D @@ -436,26 +438,28 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) update_v(nz)=(UV(2,nz,el(1))-UV(2,nz,el(2))) END DO #if defined(_OPENMP) && !defined(__openmp_reproducible) - call omp_set_lock(partit%plock(el(1))) + call omp_set_lock(partit%plock(el(1))) #else !$OMP ORDERED #endif - U_c(nzmin:nzmax-1, el(1))=U_c(nzmin:nzmax-1, el(1))-update_u(nzmin:nzmax-1) - V_c(nzmin:nzmax-1, el(1))=V_c(nzmin:nzmax-1, el(1))-update_v(nzmin:nzmax-1) + U_c(nzmin:nzmax-1, el(1))=U_c(nzmin:nzmax-1, el(1))-update_u(nzmin:nzmax-1) + V_c(nzmin:nzmax-1, el(1))=V_c(nzmin:nzmax-1, el(1))-update_v(nzmin:nzmax-1) #if defined(_OPENMP) && !defined(__openmp_reproducible) - call omp_unset_lock(partit%plock(el(1))) - call omp_set_lock (partit%plock(el(2))) + call omp_unset_lock(partit%plock(el(1))) + call omp_set_lock (partit%plock(el(2))) #endif - U_c(nzmin:nzmax-1, el(2))=U_c(nzmin:nzmax-1, el(2))+update_u(nzmin:nzmax-1) - V_c(nzmin:nzmax-1, el(2))=V_c(nzmin:nzmax-1, el(2))+update_v(nzmin:nzmax-1) + U_c(nzmin:nzmax-1, el(2))=U_c(nzmin:nzmax-1, el(2))+update_u(nzmin:nzmax-1) + V_c(nzmin:nzmax-1, el(2))=V_c(nzmin:nzmax-1, el(2))+update_v(nzmin:nzmax-1) #if defined(_OPENMP) && !defined(__openmp_reproducible) - call omp_unset_lock(partit%plock(el(2))) + call omp_unset_lock(partit%plock(el(2))) #else !$OMP END ORDERED #endif END DO !$OMP END DO + !___________________________________________________________________________ + ! compute viscosity on element !$OMP DO DO ed=1,myDim_elem2D len=sqrt(elem_area(ed)) @@ -474,21 +478,37 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) END DO END DO !$OMP END DO + + !___________________________________________________________________________ !$OMP MASTER call exchange_elem(U_c, partit) call exchange_elem(V_c, partit) !$OMP END MASTER !$OMP BARRIER + + !___________________________________________________________________________ !$OMP DO DO ed=1, myDim_edge2D+eDim_edge2D if(myList_edge2D(ed)>edge2D_in) cycle el=edge_tri(:,ed) nzmin = maxval(ulevels(el)) nzmax = minval(nlevels(el)) - DO nz=nzmin,nzmax-1 - update_u(nz)=(U_c(nz,el(1))-U_c(nz,el(2))) - update_v(nz)=(V_c(nz,el(1))-V_c(nz,el(2))) - END DO + !_______________________________________________________________________ + if (dynamics%use_ssh_splitexpl_subcycl) then + !SD Approximate update for transports. We do not care about accuracy + !SD here. --> of course, helem will be better. + do nz=nzmin,nzmax-1 + update_u(nz)=(U_c(nz,el(1))-U_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + update_v(nz)=(V_c(nz,el(1))-V_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + end do + else + do nz=nzmin,nzmax-1 + update_u(nz)=(U_c(nz,el(1))-U_c(nz,el(2))) + update_v(nz)=(V_c(nz,el(1))-V_c(nz,el(2))) + end do + end if + + !_______________________________________________________________________ #if defined(_OPENMP) && ! defined(__openmp_reproducible) call omp_set_lock(partit%plock(el(1))) #else @@ -556,6 +576,8 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) V_c(:, elem) = 0.0_WP END DO !$OMP END PARALLEL DO + + !___________________________________________________________________________ !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(u1, v1, len, vi, ed, el, nz, nzmin, nzmax) !$OMP DO DO ed=1, myDim_edge2D+eDim_edge2D @@ -594,14 +616,17 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) #else !$OMP END ORDERED #endif - END DO !$OMP END DO + + !___________________________________________________________________________ !$OMP MASTER call exchange_elem(U_c, partit) call exchange_elem(V_c, partit) !$OMP END MASTER !$OMP BARRIER + + !___________________________________________________________________________ !$OMP DO DO ed=1, myDim_edge2D+eDim_edge2D if(myList_edge2D(ed)>edge2D_in) cycle @@ -609,18 +634,38 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) len=sqrt(sum(elem_area(el))) nzmin = maxval(ulevels(el)) nzmax = minval(nlevels(el)) - DO nz=nzmin,nzmax-1 - u1=(UV(1,nz,el(1))-UV(1,nz,el(2))) - v1=(UV(2,nz,el(1))-UV(2,nz,el(2))) - vi=u1*u1+v1*v1 - vi=-dt*sqrt(max(dynamics%visc_gamma0, & - max(dynamics%visc_gamma1*sqrt(vi), & - dynamics%visc_gamma2*vi) & - )*len) - ! vi=-dt*sqrt(max(dynamics%visc_gamma0, dynamics%visc_gamma1*max(sqrt(vi), dynamics%visc_gamma2*vi))*len) - update_u(nz)=vi*(U_c(nz,el(1))-U_c(nz,el(2))) - update_v(nz)=vi*(V_c(nz,el(1))-V_c(nz,el(2))) - END DO + !_______________________________________________________________________ + if (dynamics%use_ssh_splitexpl_subcycl) then + !SD Approximate update for transports. We do not care about accuracy + !SD here. --> of course, helem will be better. + do nz=nzmin,nzmax-1 + u1=(UV(1,nz,el(1))-UV(1,nz,el(2))) + v1=(UV(2,nz,el(1))-UV(2,nz,el(2))) + vi=u1*u1+v1*v1 + vi=-dt*sqrt(max(dynamics%visc_gamma0, & + max(dynamics%visc_gamma1*sqrt(vi), & + dynamics%visc_gamma2*vi) & + )*len) + ! vi=-dt*sqrt(max(dynamics%visc_gamma0, dynamics%visc_gamma1*max(sqrt(vi), dynamics%visc_gamma2*vi))*len) + update_u(nz)=vi*(U_c(nz,el(1))-U_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + update_v(nz)=vi*(V_c(nz,el(1))-V_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + end do + else + do nz=nzmin,nzmax-1 + u1=(UV(1,nz,el(1))-UV(1,nz,el(2))) + v1=(UV(2,nz,el(1))-UV(2,nz,el(2))) + vi=u1*u1+v1*v1 + vi=-dt*sqrt(max(dynamics%visc_gamma0, & + max(dynamics%visc_gamma1*sqrt(vi), & + dynamics%visc_gamma2*vi) & + )*len) + ! vi=-dt*sqrt(max(dynamics%visc_gamma0, dynamics%visc_gamma1*max(sqrt(vi), dynamics%visc_gamma2*vi))*len) + update_u(nz)=vi*(U_c(nz,el(1))-U_c(nz,el(2))) + update_v(nz)=vi*(V_c(nz,el(1))-V_c(nz,el(2))) + end do + end if + + !_______________________________________________________________________ #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_set_lock(partit%plock(el(1))) #else @@ -628,15 +673,12 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) #endif UV_rhs(1, nzmin:nzmax-1, el(1))=UV_rhs(1, nzmin:nzmax-1, el(1))-update_u(nzmin:nzmax-1)/elem_area(el(1)) UV_rhs(2, nzmin:nzmax-1, el(1))=UV_rhs(2, nzmin:nzmax-1, el(1))-update_v(nzmin:nzmax-1)/elem_area(el(1)) - #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(el(1))) call omp_set_lock (partit%plock(el(2))) #endif - UV_rhs(1, nzmin:nzmax-1, el(2))=UV_rhs(1, nzmin:nzmax-1, el(2))+update_u(nzmin:nzmax-1)/elem_area(el(2)) UV_rhs(2, nzmin:nzmax-1, el(2))=UV_rhs(2, nzmin:nzmax-1, el(2))+update_v(nzmin:nzmax-1)/elem_area(el(2)) - #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(el(2))) #else @@ -646,7 +688,9 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) !$OMP END DO !$OMP END PARALLEL end subroutine visc_filt_bidiff - +! +! +!_______________________________________________________________________________ SUBROUTINE compute_ke_wrho(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT @@ -689,6 +733,9 @@ SUBROUTINE compute_ke_wrho(dynamics, partit, mesh) END DO call exchange_nod(dynamics%ke_wrho, partit) END SUBROUTINE compute_ke_wrho +! +! +!_______________________________________________________________________________ ! APE generation stuff SUBROUTINE compute_apegen(dynamics, tracers, partit, mesh) USE MOD_MESH From 56f5a410d981887e6eddf67c124aab8f4ac29da9 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 22 May 2023 11:42:25 +0200 Subject: [PATCH 08/50] add and rename some variables that are need for split-explicite subsampling --- src/MOD_DYN.F90 | 16 +++++++++------- src/oce_setup_step.F90 | 18 +++++++++++++----- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index da5c1bded..900ed561d 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -65,15 +65,15 @@ MODULE MOD_DYN !___arrays for split explicite ssh computation______________________________ ! se_uvh...transport velocity, real(kind=WP), allocatable, dimension(:,:,:):: se_uvh - !se_uv_rhs...vertical integral of transport velocity rhs, se_uvh_BT4AB... + !se_uv_rhs...vertical integral of transport velocity rhs, se_uvhBT_4AB... ! barotropic transport velocities (vertically integrated), contains actual ! timestep (1:2) and previous timestep (3:4) for adams-bashfort interpolation - real(kind=WP), allocatable, dimension(:,:) :: se_uvh_rhs, se_uvh_BT4AB + real(kind=WP), allocatable, dimension(:,:) :: se_uvhBC_rhs, se_uvhBT_4AB ! se_uvBT...barotropic trnasport velocities from barotropic time stepping ! se_uvBT_theta...velocities for dissipative time stepping of thickness equation ! UBTmean_mean... Mean BT velocity to trim 3D velocity in tracers - real(kind=WP), allocatable, dimension(:,:) :: se_uvBT, se_uvBT_theta, se_uvBT_mean + real(kind=WP), allocatable, dimension(:,:) :: se_uvBT, se_uvBT_theta, se_uvBT_mean, se_uvBT_12 !___________________________________________________________________________ ! summarizes solver input parameter @@ -276,11 +276,12 @@ subroutine WRITE_T_DYN(dynamics, unit, iostat, iomsg) end if if (dynamics%use_ssh_splitexpl_subcycl) then call write_bin_array(dynamics%se_uvh , unit, iostat, iomsg) - call write_bin_array(dynamics%se_uvh_rhs , unit, iostat, iomsg) - call write_bin_array(dynamics%se_uvh_BT4AB , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvhBC_rhs , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvhBT_4AB , unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT , unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT_12 , unit, iostat, iomsg) end if @@ -331,11 +332,12 @@ subroutine READ_T_DYN(dynamics, unit, iostat, iomsg) end if if (dynamics%use_ssh_splitexpl_subcycl) then call read_bin_array(dynamics%se_uvh , unit, iostat, iomsg) - call read_bin_array(dynamics%se_uvh_rhs , unit, iostat, iomsg) - call read_bin_array(dynamics%se_uvh_BT4AB , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvhBC_rhs , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvhBT_4AB , unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT , unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT_12 , unit, iostat, iomsg) end if end subroutine READ_T_DYN diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index 4aaf5c6b6..59498a14a 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -478,12 +478,20 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) !!PS dynamics%ssh_rhs_old= 0.0_WP if (dynamics%use_ssh_splitexpl_subcycl) then - allocate(dynamics%se_uvh( 2, nl-1, elem_size)) - allocate(dynamics%se_uvh_rhs( 2, elem_size)) - allocate(dynamics%se_uvh_BT4AB(4, elem_size)) + allocate(dynamics%se_uvh( 2, nl-1, elem_size)) + allocate(dynamics%se_uvhBC_rhs( 2, elem_size)) + allocate(dynamics%se_uvhBT_4AB( 4, elem_size)) + allocate(dynamics%se_uvBT( 2, elem_size)) + allocate(dynamics%se_uvBT_theta(2, elem_size)) + allocate(dynamics%se_uvBT_mean( 2, elem_size)) + allocate(dynamics%se_uvBT_12( 2, elem_size)) dynamics%se_uvh = 0.0_WP - dynamics%se_uvh_rhs = 0.0_WP - dynamics%se_uvh_BT4AB = 0.0_WP + dynamics%se_uvhBC_rhs = 0.0_WP + dynamics%se_uvhBT_4AB = 0.0_WP + dynamics%se_uvBT = 0.0_WP + dynamics%se_uvBT_theta = 0.0_WP + dynamics%se_uvBT_mean = 0.0_WP + dynamics%se_uvBT_12 = 0.0_WP else allocate(dynamics%d_eta( node_size)) allocate(dynamics%ssh_rhs( node_size)) From 5e46284be0d520b592da07c5b585dcae10f4021a Mon Sep 17 00:00:00 2001 From: Patrick Scholz Date: Mon, 22 May 2023 11:44:25 +0200 Subject: [PATCH 09/50] add viscfilter option 5 for transport velocities --- src/oce_dyn.F90 | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index 585836159..d26cc84a6 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -296,6 +296,7 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) END DO !$OMP END PARALLEL DO + !___________________________________________________________________________ !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(u1, v1, len, vi, nz, ed, el, nelem, k, elem, nzmin, nzmax, update_u, update_v) !$OMP DO DO ed=1, myDim_edge2D+eDim_edge2D @@ -334,14 +335,16 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) #endif END DO !$OMP END DO + + !___________________________________________________________________________ !$OMP MASTER call exchange_elem(U_b, partit) call exchange_elem(V_b, partit) !$OMP END MASTER !$OMP BARRIER - ! =========== - ! Compute smoothed viscous term: - ! =========== + + !___________________________________________________________________________ + ! Compute smoothed viscous term, smoth from elements to nodes !$OMP DO DO ed=1, myDim_nod2D nzmin = ulevels_nod2D(ed) @@ -361,19 +364,35 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) END DO END DO !$OMP END DO + + !___________________________________________________________________________ !$OMP MASTER call exchange_nod(U_c, V_c, partit) !$OMP END MASTER !$OMP BARRIER + + !___________________________________________________________________________ + ! smooth from nodes back to elements, take unsmooth viscos term minus + ! smoothed viscos term multiplied backscatter parameter !$OMP DO do ed=1, myDim_elem2D nelem=elem2D_nodes(:,ed) nzmin = ulevels(ed) nzmax = nlevels(ed) - Do nz=nzmin, nzmax-1 - UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP - UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP - END DO + !_______________________________________________________________________ + if (dynamics%use_ssh_splitexpl_subcycl) then + !SD Approximate update for transports. We do not care about accuracy + !SD here. --> of course, helem will be better. + Do nz=nzmin, nzmax-1 + UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+(U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) + UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+(V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) + END DO + else + Do nz=nzmin, nzmax-1 + UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP + UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP + END DO + end if end do !$OMP END DO !$OMP END PARALLEL From e6c83c8e8d9c1e2247aa4492f2af6d72e3279b49 Mon Sep 17 00:00:00 2001 From: Patrick Scholz Date: Mon, 22 May 2023 11:53:45 +0200 Subject: [PATCH 10/50] restructure/comment oce_timestep for split-explicite subcycling routines --- src/oce_ale.F90 | 228 ++++++++++++++++++++++++++++++------------------ 1 file changed, 141 insertions(+), 87 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 5f5f05470..07d6f5ff5 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2919,6 +2919,8 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) use write_step_info_interface use check_blowup_interface use fer_solve_interface + use impl_vert_visc_ale_vtransp_interface + use compute_ssh_split_explicit_interface IMPLICIT NONE integer , intent(in) :: n type(t_dyn) , intent(inout), target :: dynamics @@ -3003,9 +3005,9 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call oce_mixing_KPP'//achar(27)//'[0m' call oce_mixing_KPP(Av, Kv_double, dynamics, tracers, partit, mesh) !$OMP PARALLEL DO - DO node=1, myDim_nod2D+eDim_nod2D + do node=1, myDim_nod2D+eDim_nod2D Kv(:, node)=Kv_double(:, node, 1) - END DO + end do !$OMP END PARALLEL DO call mo_convect(ice, partit, mesh) @@ -3056,58 +3058,75 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) t1=MPI_Wtime() !___________________________________________________________________________ + ! add contribution from momentum advection, coriolis and pressure gradient | + ! force to UV_rhs + ! UV_rhs = dt*[ (R_advec + R_coriolis)_AB2^n + R_pressure^n ] if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call compute_vel_rhs'//achar(27)//'[0m' call compute_vel_rhs(ice, dynamics, partit, mesh) !___________________________________________________________________________ - if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call viscosity_filter'//achar(27)//'[0m' if (dynamics%ldiag_ke) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) - end do - end do + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) + end do + end do !$OMP END PARALLEL DO end if + + !___________________________________________________________________________ + ! add contribution from horizontal viscosity to UV_rhs + ! UV_rhs = dt*[ (R_advec + R_coriolis)^n + R_pressure + R_hviscos] + ! UV_rhs = UV_rhs + dt*R_hviscos + if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call viscosity_filter'//achar(27)//'[0m' call viscosity_filter(dynamics%opt_visc, dynamics, partit, mesh) + + !___________________________________________________________________________ if (dynamics%ldiag_ke) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_hvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)-dynamics%ke_rhs_bak(:,nz,elem) - end do - end do + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_hvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)-dynamics%ke_rhs_bak(:,nz,elem) + end do + end do +!$OMP END PARALLEL DO + +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) + end do + end do !$OMP END PARALLEL DO end if !___________________________________________________________________________ if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call impl_vert_visc_ale'//achar(27)//'[0m' - if (dynamics%ldiag_ke) then -!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) - end do - end do -!$OMP END PARALLEL DO + if(dynamics%use_ivertvisc) then + if (dynamics%use_ssh_splitexpl_subcycl) then + call impl_vert_visc_ale(dynamics,partit, mesh) + else + call impl_vert_visc_ale_vtransp(dynamics, partit, mesh) + end if end if - if(dynamics%use_ivertvisc) call impl_vert_visc_ale(dynamics,partit, mesh) + + !___________________________________________________________________________ if (dynamics%ldiag_ke) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_vvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)-dynamics%ke_rhs_bak(:,nz,elem) - end do - end do + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_vvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)-dynamics%ke_rhs_bak(:,nz,elem) + end do + end do !$OMP END PARALLEL DO end if t2=MPI_Wtime() @@ -3115,56 +3134,76 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ ! >->->->->->->->->->->->-> ALE-part starts <-<-<-<-<-<-<-<-<-<-<-<- !___________________________________________________________________________ + + !___________________________________________________________________________ + ! Compute SSH via solver ! Update stiffness matrix by dhe=hbar(n+1/2)-hbar(n-1/2) on elements, only ! needed for zlevel and zstar - if (.not. trim(which_ale)=='linfs') call update_stiff_mat_ale(partit, mesh) - if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call compute_ssh_rhs_ale'//achar(27)//'[0m' - ! ssh_rhs=-alpha*\nabla\int(U_n+U_rhs)dz-(1-alpha)*... - ! see "FESOM2: from finite elements to finte volumes, S. Danilov..." eq. (18) rhs - call compute_ssh_rhs_ale(dynamics, partit, mesh) - - ! Take updated ssh matrix and solve --> new ssh! - t30=MPI_Wtime() - call solve_ssh_ale(dynamics, partit, mesh) - - if ((toy_ocean) .AND. (TRIM(which_toy)=="soufflet")) then - if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call relax_zonal_vel'//achar(27)//'[0m' - call relax_zonal_vel(dynamics, partit, mesh) - end if - t3=MPI_Wtime() - - ! estimate new horizontal velocity u^(n+1) - ! u^(n+1) = u* + [-g * tau * theta * grad(eta^(n+1)-eta^(n)) ] - if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call update_vel'//achar(27)//'[0m' - ! ke will be computed inside there if dynamics%ldiag_ke is .TRUE. - call update_vel(dynamics, partit, mesh) - - ! --> eta_(n) --> eta_(n+1) = eta_(n) + deta = eta_(n) + (eta_(n+1) + eta_(n)) - t4=MPI_Wtime() - - ! Update to hbar(n+3/2) and compute dhe to be used on the next step - if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call compute_hbar_ale'//achar(27)//'[0m' - call compute_hbar_ale(dynamics, partit, mesh) - - !___________________________________________________________________________ - ! - Current dynamic elevation alpha*hbar(n+1/2)+(1-alpha)*hbar(n-1/2) - ! equation (14) Danlov et.al "the finite volume sea ice ocean model FESOM2 - ! ...if we do it here we don't need to write hbar_old into a restart file... - ! - where(ulevels_nod2D==1) is used here because of the rigid lid - ! approximation under the cavity - ! - at points in the cavity the time derivative term in ssh matrix will be - ! omitted; and (14) will not be applied at cavity points. Additionally, - ! since there is no real elevation, but only surface pressure, there is - ! no layer motion under the cavity. In this case the ice sheet acts as a - ! rigid lid. + if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. trim(which_ale)=='linfs') call update_stiff_mat_ale(partit, mesh) + if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call compute_ssh_rhs_ale'//achar(27)//'[0m' + ! ssh_rhs=-alpha*\nabla\int(U_n+U_rhs)dz-(1-alpha)*... + ! see "FESOM2: from finite elements to finte volumes, S. Danilov..." eq. (18) rhs + call compute_ssh_rhs_ale(dynamics, partit, mesh) + + ! Take updated ssh matrix and solve --> new ssh! + t30=MPI_Wtime() + call solve_ssh_ale(dynamics, partit, mesh) + + if ((toy_ocean) .AND. (TRIM(which_toy)=="soufflet")) then + if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call relax_zonal_vel'//achar(27)//'[0m' + call relax_zonal_vel(dynamics, partit, mesh) + end if + t3=MPI_Wtime() + + ! estimate new horizontal velocity u^(n+1) + ! u^(n+1) = u* + [-g * tau * theta * grad(eta^(n+1)-eta^(n)) ] + if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call update_vel'//achar(27)//'[0m' + ! ke will be computed inside there if dynamics%ldiag_ke is .TRUE. + call update_vel(dynamics, partit, mesh) + + ! --> eta_(n) --> eta_(n+1) = eta_(n) + deta = eta_(n) + (eta_(n+1) + eta_(n)) + t4=MPI_Wtime() + + ! Update to hbar(n+3/2) and compute dhe to be used on the next step + if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call compute_hbar_ale'//achar(27)//'[0m' + call compute_hbar_ale(dynamics, partit, mesh) + + !___________________________________________________________________________ + ! - Current dynamic elevation alpha*hbar(n+1/2)+(1-alpha)*hbar(n-1/2) + ! equation (14) Danlov et.al "the finite volume sea ice ocean model FESOM2 + ! ...if we do it here we don't need to write hbar_old into a restart file... + ! - where(ulevels_nod2D==1) is used here because of the rigid lid + ! approximation under the cavity + ! - at points in the cavity the time derivative term in ssh matrix will be + ! omitted; and (14) will not be applied at cavity points. Additionally, + ! since there is no real elevation, but only surface pressure, there is + ! no layer motion under the cavity. In this case the ice sheet acts as a + ! rigid lid. !$OMP PARALLEL DO - do node=1, myDim_nod2D+eDim_nod2D - if (ulevels_nod2D(node)==1) eta_n(node)=alpha*hbar(node)+(1.0_WP-alpha)*hbar_old(node) - end do + do node=1, myDim_nod2D+eDim_nod2D + if (ulevels_nod2D(node)==1) eta_n(node)=alpha*hbar(node)+(1.0_WP-alpha)*hbar_old(node) + end do !$OMP END PARALLEL DO - ! --> eta_(n) - ! call zero_dynamics !DS, zeros several dynamical variables; to be used for testing new implementations! - t5=MPI_Wtime() + ! --> eta_(n) + ! call zero_dynamics !DS, zeros several dynamical variables; to be used for testing new implementations! + t5=MPI_Wtime() + + !___________________________________________________________________________ + ! Compute SSH via split-explicite subcycling + else + ! Compute vertical integral of transport velocity rhs omitting the contributions from + ! the elevation and Coriolis. + call compute_BC_BT_SE_vtransp(dynamics, partit, mesh) + + ! Do barotropic step, get eta_{n+1} and BT transport + call compute_BT_step_SE_ale(dynamics, partit, mesh) + + ! Trim U to be consistent with BT transport + call update_trim_vel_ale_vtransp(1, dynamics, partit, mesh) + + end if ! --> if (.not. dynamics%use_ssh_splitexpl_subcycl) then + !___________________________________________________________________________ ! Do horizontal and vertical scaling of GM/Redi diffusivity if (Fer_GM .or. Redi) then @@ -3181,23 +3220,29 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) t6=MPI_Wtime() !___________________________________________________________________________ - ! The main step of ALE procedure --> this is were the magic happens --> here - ! is decided how change in hbar is distributed over the vertical layers - if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call vert_vel_ale'//achar(27)//'[0m' ! keep the old vertical velocity for computation of the mean between the timesteps (is used in compute_ke_wrho) if (dynamics%ldiag_ke) then !$OMP PARALLEL DO - do node=1, myDim_nod2D+eDim_nod2D - dynamics%w_old(:, node)=dynamics%w(:, node) - end do + do node=1, myDim_nod2D+eDim_nod2D + dynamics%w_old(:, node)=dynamics%w(:, node) + end do !$OMP END PARALLEL DO end if + + !___________________________________________________________________________ + ! The main step of ALE procedure --> this is were the magic happens --> here + ! is decided how change in hbar is distributed over the vertical layers + if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call vert_vel_ale'//achar(27)//'[0m' call vert_vel_ale(dynamics, partit, mesh) t7=MPI_Wtime() + + !___________________________________________________________________________ + ! energy diagnostic computation if (dynamics%ldiag_ke) then call compute_ke_wrho(dynamics, partit, mesh) call compute_apegen (dynamics, tracers, partit, mesh) end if + !___________________________________________________________________________ ! solve tracer equation if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call solve_tracers_ale'//achar(27)//'[0m' @@ -3209,10 +3254,19 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call update_thickness_ale'//achar(27)//'[0m' call update_thickness_ale(partit, mesh) t9=MPI_Wtime() + + !___________________________________________________________________________ + ! Trim to make velocity consistent with BT velocity at n+1/2 + ! Velocity will be used to advance momentum + if (dynamics%use_ssh_splitexpl_subcycl) then + call update_trim_vel_ale_vtransp(2, dynamics, partit, mesh) + end if + !___________________________________________________________________________ ! write out global fields for debugging if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call write_step_info'//achar(27)//'[0m' call write_step_info(n,logfile_outfreq, ice, dynamics, tracers, partit, mesh) + !___________________________________________________________________________ ! write energy diagnostic info (dynamics%ldiag_ke = .true.) if (dynamics%ldiag_ke) then From 83a55669aa0f4c4d96530eaed42bafe64ea2b4db Mon Sep 17 00:00:00 2001 From: Patrick Scholz Date: Mon, 22 May 2023 11:56:41 +0200 Subject: [PATCH 11/50] add routines for split-explicite subcycling from sergeys prototyp routines --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 574 +++++++++++++++++++++++++- 1 file changed, 570 insertions(+), 4 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index a81b1b26b..af618aaf0 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1,7 +1,52 @@ +! +! +!_______________________________________________________________________________ module momentum_adv_scalar_4splitexpl_interface interface subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) - use mod_mesh + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + end subroutine + end interface +end module +! +! +!_______________________________________________________________________________ +module impl_vert_visc_ale_vtransp_interface + interface + subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + end subroutine + end interface +end module +! +! +!_______________________________________________________________________________ +module compute_ssh_split_explicit_interface + interface + subroutine compute_BC_BT_SE_vtransp(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + end subroutine + + subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) + USE MOD_MESH USE MOD_PARTIT USE MOD_PARSUP USE MOD_DYN @@ -9,13 +54,24 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(in) , target :: mesh end subroutine + + subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + integer , intent(in) :: mode + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + end subroutine end interface end module ! ! !_______________________________________________________________________________ -! Transports are used instead of velocities, Urhs, Vrhs are also for transports. +! Transports are used instead of velocities, Urhs, Vrhs are also for transports. subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT @@ -420,11 +476,521 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) !$OMP END PARALLEL end subroutine +! +! +!_______________________________________________________________________________ +!SD Transport velocity version (T in the name) +!SD Solve U**-U*=dt*D_vert u**, where U*=Uh+rhs (rhs contains dt/elem_area) +!SD Express through Delta U=U**-U* and then introduce +!SD Delta u= Delta U/helem; after Delta u is found, rhs=rhs +h*Delta u. +!SD The rhs accumulated previously is not affected in this step, this is different +!SD from FESOM, and I would recomment this variant. +! +! +! solve equation vertical viscosity implicite part: +! --> U^(n+0.5,**) - U^(n+0.5,*) = dt*( Av * d/dz * u^(n+0.5,**) )|^t_b +! | +! +-> du = u^(n+0.5,**) - u^(n+0.5,*) = dU/h^(*) +! | +! +-> du * h^(*) = dU +! | u^(n+0.5,**) = du + u^(n+0.5,*) +! V +! du * h^(*) = dt*( Av*d/dz*du )|^t_b + dt*( Av*d/dz*u^(n+0.5,*) )|^t_b +! du - dt/h^(*)*( Av*d/dz*du ) = dt/h^(*)*( Av*d/dz*u^(n+0.5,*) ) +! +! --> 1st. solve homogenouse part: +! f(du) = du - dt/h^(*)*( Av*d/dz*du ) = 0 +! +! --> 2nd. compute difference quotient at du_i using Gauss-Theorem --> flux form +! V_i * du_i = dt * [ Av_i/h_i * (du_i-1 - du_i)/(Z_i-1 - Z_i) * A_i * nvec_up(+1) +! +Av_i+1/h_i * (du_i - du_i+1)/(Z_i - Z_i+1) * A_i+1 * nvec_dwn(-1) ] +! +! f(du_i) = du_i - dt*Av_i/h_i * (du_i-1 - du_i)/(Z_i-1 - Z_i) * A_i/V_i+1 +! + dt*Av_i+1/h_i * (du_i - du_i+1)/(Z_i - Z_i+1) * A_i+1/V_i+1 +! = 0 +! +! --> 3rd. solve for coefficents a, b, c (homogenous part): +! f(du_i) = [ a*du_i-1 + b*du_i + c*du_i+1 ] +! | +! +-> estimate a, b, c by derivation of d(du_i) +! | +! +-> a = d[f(du_i)]/d[du_i-1] = - dt*Av_i/h_i / (Z_i-1 - Z_i) +! | +! +-> c = d[f(du_i)]/d[du_i+1] = - dt*Av_i+1/h_i / (Z_i - Z_i+1) +! | +! +-> b = d[f(du_i)]/d[du_i] = 1 + dt*Av_i/h_i / (Z_i-1 - Z_i) + dt*Av_i+1/h_i / (Z_i - Z_i+1) +! 1 - a - c +! +! --> 4th. solve inhomogenous part: +! [ a*du_i-1 + b*du_i + c*du_i+1 ] = RHS/V_i +! +! RHS/V_i = dt* [ Av_i /h_i * (u^(n+0.5,*)_i-1 - u^(n+0.5,*)_i)/(Z_i-1 - Z_i) * A_i * nvec_up(+1) +! | +Av_i+1/h_i * (u^(n+0.5,*)_i - u^(n+0.5,*)_i+1)/(Z_i - Z_i+1) * A_i+1 * nvec_dwn(-1) ] +! | +! V +! = -a*u^(n+0.5,*)_i-1 + (a+c)*u^(n+0.5,*)_i - c*u^(n+0.5,*)_i+1 +! +! --> 5th. solve for du_i --> forward sweep algorithm --> see lower +! | b_1 c_1 ... | |du_1| +! | a_2 b_2 c_2 ... | |du_2| +! | a_3 b_3 c_3 ... | * |du_3| = RHS/V_i +! | a_4 b_4 c_4 ...| |du_4| +! | : | | : | +! +subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) + USE MOD_MESH + USE o_PARAM + USE o_ARRAYS, only: Av, stress_surf + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + USE g_CONFIG, only: dt + IMPLICIT NONE + !___________________________________________________________________________ + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + !___________________________________________________________________________ + real(kind=WP) :: a(mesh%nl-1), b(mesh%nl-1), c(mesh%nl-1), ur(mesh%nl-1), vr(mesh%nl-1) + real(kind=WP) :: cp(mesh%nl-1), up(mesh%nl-1), vp(mesh%nl-1), uu(mesh%nl-1), vv(mesh%nl-1) + integer :: elem, nz, nzmax, nzmin, elnodes(3) + real(kind=WP) :: zinv, m, friction, wu, wd + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:,:), pointer :: UV, UV_rhs, UVh + real(kind=WP), dimension(:,:) , pointer :: Wvel_i +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + UV =>dynamics%uv(:,:,:) + UV_rhs =>dynamics%uv_rhs(:,:,:) + Wvel_i =>dynamics%w_i(:,:) + UVh =>dynamics%se_uvh(:,:,:) + !___________________________________________________________________________ +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(a, b, c, ur, vr, cp, up, vp, elem, & +!$OMP nz, nzmin, nzmax, elnodes, & +!$OMP zinv, m, friction, wu, wv, uu, vv) + + !___________________________________________________________________________ +!$OMP DO + do elem=1,myDim_elem2D + elnodes= elem2D_nodes(:,elem) + nzmin = ulevels(elem) + nzmax = nlevels(elem) + + !_______________________________________________________________________ + ! New velocities: compute u_k^(n+1/2, *) = u_k^(n-1/2) + UV_rhs/helem + ! UV_rhs is here in terms of transport velocity, we need real velocity + ! therefor divide with helem + uu(nzmin:nzmax-1)=(UVh(1, nzmin:nzmax-1, elem)+UV_rhs(1, nzmin:nzmax-1, elem))/helem(nzmin:nzmax-1, elem) !! u*=U*/h + vv(nzmin:nzmax-1)=(UVh(2, nzmin:nzmax-1, elem)+UV_rhs(2, nzmin:nzmax-1, elem))/helem(nzmin:nzmax-1, elem) + + !_______________________________________________________________________ + ! Operator + rhs + ! Regular part of coefficients: + + !_____1st layer (surface)_____ + nz = nzmin + zinv = 2.0_WP*dt/helem(nz, elem) + c( nz)= -Av(nz+1,elem)/(helem(nz,elem)+helem(nz+1,elem))*zinv + a( nz)= 0.0_WP + b( nz)= -c(nz)+1.0_WP + + ! Update from the vertical advection + wu = sum(Wvel_i(nz , elnodes))/3._WP + wd = sum(Wvel_i(nz+1, elnodes))/3._WP + b( nz)= b(nz)+wu*zinv + b( nz)= b(nz)-min(0._WP, wd)*zinv + c( nz)= c(nz)-max(0._WP, wd)*zinv + + ur(nz)= +c(nz)*uu(nz)-c(nz)*uu(nz+1) + vr(nz)= +c(nz)*vv(nz)-c(nz)*vv(nz+1) + + !_____bulk layers_____ + do nz=nzmin+1, nzmax-2 + zinv = 2.0_WP*dt/helem(nz, elem) + a( nz)= -Av(nz , elem)/(helem(nz-1, elem)+helem(nz , elem))*zinv + c( nz)= -Av(nz+1, elem)/(helem(nz , elem)+helem(nz+1, elem))*zinv + b( nz)= -a(nz)-c(nz)+1.0_WP + + ! Update from the vertical advection + wu=sum(Wvel_i(nz , elnodes))/3._WP + wd=sum(Wvel_i(nz+1, elnodes))/3._WP + a( nz)= a(nz)+min(0._WP, wu)*zinv + b( nz)= b(nz)+max(0._WP, wu)*zinv + + ur(nz)= -a(nz)*uu(nz-1)+(a(nz)+c(nz))*uu(nz)-c(nz)*uu(nz+1) !! add dt*D_vert u* + vr(nz)= -a(nz)*vv(nz-1)+(a(nz)+c(nz))*vv(nz)-c(nz)*vv(nz+1) !! to the rhs (because of Delta u) + end do + + !_____bottom layer_____ + nz = nzmax-1 + zinv = 2.0_WP*dt/helem(nz,elem) + a( nz)= -Av(nz, elem)/(helem(nz-1, elem)+helem(nz,elem))*zinv + b( nz)= -a(nz)+1.0_WP + c( nz)= 0.0_WP + + ! Update from the vertical advection + wu = sum(Wvel_i(nz-1, elnodes))/3._WP + a( nz)= a(nz)+min(0._WP, wu)*zinv + b( nz)= b(nz)+max(0._WP, wu)*zinv + + ur(nz)= -a(nz)*uu(nz-1)+a(nz)*uu(nz) + vr(nz)= -a(nz)*vv(nz-1)+a(nz)*vv(nz) + !_______________________________________________________________________ + ! Forcing to the rhs: + ! The first row contains surface forcing + nz = nzmin + zinv = 1.0_WP*dt/helem(nz, elem) + ur(nz)= ur(nz) + zinv*stress_surf(1, elem)/density_0 + vr(nz)= vr(nz) + zinv*stress_surf(2, elem)/density_0 + if (dynamics%ldiag_ke) then + dynamics%ke_wind(1,elem)=stress_surf(1, elem)/density_0*dt + dynamics%ke_wind(2,elem)=stress_surf(2, elem)/density_0*dt + end if + + ! The last row contains bottom friction + nz = nzmax-1 + zinv = 1.0_WP*dt/helem(nz, elem) + friction=-C_d*sqrt(UV(1, nz, elem)**2 + UV(2, nz, elem)**2) + ur(nz)= ur(nz) + zinv*friction*UV(1, nz, elem) + vr(nz)= vr(nz) + zinv*friction*UV(2, nz, elem) + if (dynamics%ldiag_ke) then + dynamics%ke_drag(1, elem)=friction*UV(1, nz, elem)*dt + dynamics%ke_drag(2, elem)=friction*UV(2, nz, elem)*dt + end if + + !_______________________________________________________________________ + ! The sweep algorithm --> solve for du_i + ! initialize c-prime and s,t-prime + cp(nzmin) = c(nzmin)/b(nzmin) + up(nzmin) = ur(nzmin)/b(nzmin) + vp(nzmin) = vr(nzmin)/b(nzmin) + + ! sove for vectors c-prime and t, s-prime + do nz=nzmin+1, nzmax-1 + m = b(nz)-cp(nz-1)*a(nz) + cp(nz) = c(nz)/m + up(nz) = (ur(nz)-up(nz-1)*a(nz))/m + vp(nz) = (vr(nz)-vp(nz-1)*a(nz))/m + end do + + ! initialize x + ur(nzmax-1) = up(nzmax-1) + vr(nzmax-1) = vp(nzmax-1) + + ! solve for x from the vectors c-prime and d-prime + do nz=nzmax-2, nzmin, -1 + ur(nz) = up(nz)-cp(nz)*ur(nz+1) + vr(nz) = vp(nz)-cp(nz)*vr(nz+1) + end do + + !_______________________________________________________________________ + ! RHS update + ! ur and vr are in terms of du velocity difference, UV_rhs are in terms + ! of transport velocity + do nz=nzmin, nzmax-1 + UV_rhs(1, nz, elem)=UV_rhs(1, nz, elem)+ur(nz)*helem(nz, elem) !! The rhs is for transport + UV_rhs(2, nz, elem)=UV_rhs(2, nz, elem)+vr(nz)*helem(nz, elem) + end do + end do ! --> do elem=1,myDim_elem2D +!$OMP END DO +!$OMP END PARALLEL +end subroutine impl_vert_visc_ale_vtransp +! +! +!_______________________________________________________________________________ +!SD Compute vertical integral of transport velocity rhs omitting the contributions from +!SD the elevation and Coriolis. The elevation and Coriolis are accounted for +!SD explicitly in BT equations, and should therefore be removed from the vertically integrated rhs. +subroutine compute_BC_BT_SE_vtransp(dynamics, partit, mesh) + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_MESH + USE MOD_DYN + USE g_config, only: dt, r_restart + IMPLICIT NONE + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + !___________________________________________________________________________ + real(kind=WP) :: vert_sum_u, vert_sum_v, Fx, Fy, ab1, ab2, hh + integer :: elem, nz, nzmin, nzmax, elnodes(4) + logical, save :: sfirst + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:,:), pointer :: UV_rhs + real(kind=WP), dimension(:) , pointer :: eta_n + real(kind=WP), dimension(:,:) , pointer :: UVhBT_4AB, UVhBC_rhs +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + eta_n =>dynamics%eta_n(:) + UV_rhs =>dynamics%uv_rhs(:,:,:) + UVhBC_rhs =>dynamics%se_uvhBC_rhs(:,:) + UVhBT_4AB =>dynamics%se_uvhBT_4AB(:,:) + + !___________________________________________________________________________ + ab1=(1.5_WP+epsilon) + if(sfirst .and. (.not. r_restart)) then + ab1=1.0_WP + sfirst=.false. + end if + ab2=1.0_WP-ab1 + !___________________________________________________________________________ +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, Fx, Fy, & +!$OMP vert_sum_u, vert_sum_v, hh) - - + !___________________________________________________________________________ +!$OMP DO + do elem=1, myDim_elem2D + elnodes= elem2D_nodes(:,elem) + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + + !_______________________________________________________________________ + Fx=g*dt*sum(gradient_sca(1:4,elem)*eta_n(elnodes)) + Fy=g*dt*sum(gradient_sca(5:8,elem)*eta_n(elnodes)) + + !_______________________________________________________________________ + ! Sum 3D rhs + vert_sum_u=0.0_WP + vert_sum_v=0.0_WP + do nz=nzmin, nzmax + vert_sum_u=vert_sum_u + UV_rhs(1, nz, elem) + vert_sum_v=vert_sum_v + UV_rhs(2, nz, elem) + end do + + !_______________________________________________________________________ + ! Remove the contribution from the elevation + hh=sum(helem(nzmin:nzmax, elem)) + vert_sum_u=vert_sum_u + Fx*hh + vert_sum_v=vert_sum_v + Fy*hh + + !_______________________________________________________________________ + ! Remove the contribution from the Coriolis + UVhBC_rhs(1, elem)=vert_sum_u - dt*mesh%coriolis(elem)* & + (ab1*UVhBT_4AB(2,elem)+ab2*UVhBT_4AB(4,elem)) ! for AB-interpolated + UVhBC_rhs(2, elem)=vert_sum_v + dt*mesh%coriolis(elem)* & + (ab1*UVhBT_4AB(1,elem)+ab2*UVhBT_4AB(3,elem)) + + !_______________________________________________________________________ + ! save actual of vertical integrated transport velocity UVhBT_4AB(1:2,elem) + ! timestep for next time loop step + UVhBT_4AB(3:4,elem)=UVhBT_4AB(1:2,elem) + end do +!PS !$OMP END DO +!PS !$OMP END PARALLEL +end subroutine compute_BC_BT_SE_vtransp +! +! +!_______________________________________________________________________________ +! Barotropic time stepping with Forward-Backward dissipative method +! (Demange et al. 2019) +subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_MESH + USE MOD_DYN + use g_comm_auto + USE g_config, only: dt + IMPLICIT NONE + !___________________________________________________________________________ + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + !___________________________________________________________________________ + real(kind=WP) :: dtBT, BT_inv, hh, f, rx, ry, a, d, c1, c2, ax, ay + real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, thetaBT + integer :: step, elem, elnodes(3), edge, ednodes(2), edelem(2) + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:) , pointer :: eta_n + real(kind=WP), dimension(:,:) , pointer :: UVhBC_rhs, UVBT, UVBT_theta, UVBT_mean, UVBT_12 +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + eta_n =>dynamics%eta_n(:) + UVhBC_rhs =>dynamics%se_uvhBC_rhs(:,:) + UVBT =>dynamics%se_uvBT(:,:) + UVBT_theta=>dynamics%se_uvBT_theta(:,:) + UVBT_mean =>dynamics%se_uvBT_mean(:,:) + UVBT_12 =>dynamics%se_uvBT_12(:,:) + + !___________________________________________________________________________ + ! Dissipation parameter of FB dissipative method 0.14 is the default value + ! from Demange et al. + thetaBT= 0.14_WP + + ! BTsteps should be 30 or 40. + dtBT = dt/dynamics%splitexpl_BTsteps + BT_inv = 1.0_WP/(1.0_WP*dynamics%splitexpl_BTsteps) + + !___________________________________________________________________________ + ! eta_n elevation used in BT stepping, it is just a copy of eta_n + ! UBT and VBT are transport velocities + UVBT_mean= 0.0_WP + do step=1, dynamics%splitexpl_BTsteps + ! + ! Dissipative forward--backward time stepping + ! + !_______________________________________________________________________ + ! Advance velocities. I use SI stepping for the Coriolis + do elem=1, myDim_elem2D + elnodes=elem2D_nodes(:,elem) + !PS hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP + hh = -zbar_e_bot(elem)+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth + f = mesh%coriolis(elem) + rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(1, elem)) + BT_inv*UVhBC_rhs(1, elem) + ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(2, elem)) + BT_inv*UVhBC_rhs(2, elem) + + ! Semi-Implicit Coriolis + a = dtBT*f*0.5_WP + d = 1.0_WP/(1.0_WP+a*a) + ax = d*(rx+a*ry) + ay = d*(-a*rx+ry) + UVBT( 1, elem) = UVBT( 1, elem) + ax ! New velocity + UVBT( 2, elem) = UVBT( 2, elem) + ay + + ! velocities for dissipative time stepping of thickness equation + UVBT_theta(1, elem) = UVBT_theta(1, elem) + thetaBT*ax ! New vel*(1+thetaBT)-old vel *thetaBT + UVBT_theta(2, elem) = UVBT_theta(2, elem) + thetaBT*ay + + ! Mean BT velocity to trim 3D velocity in tracers + UVBT_mean( 1, elem) = UVBT_mean( 1, elem) + UVBT_theta(1, elem)*BT_inv + UVBT_mean( 2, elem) = UVBT_mean( 2, elem) + UVBT_theta(2, elem)*BT_inv + end do + + !_______________________________________________________________________ + ! Store mid-step velocity (to trim 3D velocities in momentum) + if(step==dynamics%splitexpl_BTsteps/2) then + UVBT_12=UVBT + end if + + !_______________________________________________________________________ + ! Advance thickness + do edge=1, myDim_edge2D + ednodes = edges(:,edge) + edelem = edge_tri(:,edge) + deltaX1 = edge_cross_dxdy(1,edge) + deltaY1 = edge_cross_dxdy(2,edge) + + c1 = UVBT_theta(2, edelem(1))*deltaX1 - UVBT_theta(1, edelem(1))*deltaY1 + c2 = 0.0_WP + if(edelem(2)>0) then + deltaX2=edge_cross_dxdy(3,edge) + deltaY2=edge_cross_dxdy(4,edge) + c2=-(UVBT_theta(2, edelem(2))*deltaX2 - UVBT_theta(1, edelem(2))*deltaY2) + end if + eta_n(ednodes(1))=eta_n(ednodes(1))+(c1+c2)*dtBT/area(1,ednodes(1)) + eta_n(ednodes(2))=eta_n(ednodes(2))-(c1+c2)*dtBT/area(1,ednodes(2)) + end do + + !_______________________________________________________________________ + call exchange_nod(eta_n, partit) + + end do ! --> do step=1, dynamics%splitexpl_BTsteps + + hbar_old = hbar + hbar = eta_n +end subroutine compute_BT_step_SE_ale +! +! +!_______________________________________________________________________________ +! Trim U and Uh to be consistent with BT transport +subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_MESH + USE MOD_DYN + use g_comm_auto + IMPLICIT NONE + !___________________________________________________________________________ + integer , intent(in) :: mode + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + integer :: i,elem, elnodes(4), nz, m + real(kind=WP) :: eta(4), ff + real(kind=WP) :: Fx, Fy, ubar, vbar, hh_inv + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:,:), pointer :: UVh, UV, UV_rhs + real(kind=WP), dimension(:,:) , pointer :: UVBT_mean, UVBT_12 +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + UVh =>dynamics%se_uvh(:,:,:) + UV_rhs =>dynamics%uv_rhs(:,:,:) + UVBT_mean =>dynamics%se_uvBT_mean(:,:) + UVBT_12 =>dynamics%se_uvBT_12(:,:) + if (mode==2) then + UV =>dynamics%uv(:,:,:) + end if + + !___________________________________________________________________________ + ! + ! Transport version + ! + !___________________________________________________________________________ + ! Trim full velocity to ensure that its vertical sum equals to the mean BT velocity + ! The trimmed Uh,Vh are consistent with new total height defined by eta_n + if (mode==1) then + do elem=1, myDim_elem2D + ubar=0.0_WP + vbar=0.0_WP + hh_inv=0.0_WP + do nz=1, nlevels(elem)-1 + ! Update transport velocities + UVh(1, nz, elem)=UVh(1, nz, elem)+UV_rhs(1, nz, elem) + UVh(2, nz, elem)=UVh(2, nz, elem)+UV_rhs(2, nz, elem) + ubar=ubar+UVh(1, nz, elem) + vbar=vbar+UVh(2, nz, elem) + hh_inv=hh_inv+helem(nz,elem) + end do + + hh_inv=1.0_WP/hh_inv ! Postpone AB and 2nd order, just use available thickness + DO nz=1, nlevels(elem)-1 + UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) + end do + end do + call exchange_elem(UVh, partit) ! This exchange can be avoided, but test first. + + !___________________________________________________________________________ + ! Trim to make velocity consistent with BT velocity at n+1/2 + ! Velocity will be used to advance momentum + else + do elem=1, myDim_elem2D + ubar=0.0_WP + vbar=0.0_WP + hh_inv=0.0_WP + do nz=1, nlevels(elem)-1 + ubar=ubar+UVh(1, nz, elem) + vbar=vbar+UVh(2, nz, elem) + hh_inv=hh_inv+helem(nz,elem) + end do + + hh_inv=1.0_WP/hh_inv ! Postpone 2nd order, just use available thickness + do nz=1, nlevels(elem)-1 + UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_12(1, elem)-ubar)*helem(nz,elem)*hh_inv + UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_12(2, elem)-vbar)*helem(nz,elem)*hh_inv + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection + end do + end do + call exchange_elem(UVh, partit) ! + call exchange_elem(UV , partit) ! Check if this is needed + end if ! --> if (mode==1) then +end subroutine update_trim_vel_ale_vtransp From fc4897bb790eb15731e4f59b54633c1cf193f768 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 22 May 2023 12:05:22 +0200 Subject: [PATCH 12/50] add comments and explanations to adams-bashfort --- src/oce_ale_vel_rhs.F90 | 96 ++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 34 deletions(-) diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index 7ec1a8d1e..e62ff1a93 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -66,7 +66,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) real(kind=WP), dimension(:) , pointer :: m_ice, m_snow, a_ice real(kind=WP) , pointer :: rhoice, rhosno, inv_rhowat real(kind=WP), dimension(:,:,:), pointer :: UVh - real(kind=WP), dimension(:,:) , pointer :: UVh_BT4AB + real(kind=WP), dimension(:,:) , pointer :: UVhBT_4AB #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" @@ -84,7 +84,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) ! if split-explicite ssh subcycling is used if (dynamics%use_ssh_splitexpl_subcycl) then UVh => dynamics%se_uvh - UVh_BT4AB => dynamics%se_uvh_BT4AB + UVhBT_4AB => dynamics%se_uvhBT_4AB end if !___________________________________________________________________________ @@ -96,13 +96,18 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) do elem=1, myDim_elem2D nzmax = nlevels(elem) nzmin = ulevels(elem) - !___________________________________________________________________________ + !_______________________________________________________________________ ! Take care of the AB part - !!PS do nz=1,nl-1 do nz=nzmin,nzmax-1 UV_rhs(1,nz,elem)=-(0.5_WP+epsilon)*UV_rhsAB(1,nz,elem) UV_rhs(2,nz,elem)=-(0.5_WP+epsilon)*UV_rhsAB(2,nz,elem) - end do + ! | + ! V + ! Here Adams-Bashfort from previouse time n-0.5 + ! Thats why (0.5_WP+epsilon)*... to achieve second + ! order AB2 --> fAB2 = 3/2*fab_n - 1/2*fab_n-1 + end do + if (dynamics%ldiag_ke) then do nz=nzmin,nzmax-1 dynamics%ke_adv(:,nz,elem)=-(0.5_WP+epsilon)*dynamics%ke_adv_AB(:,nz,elem) @@ -110,7 +115,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) end do end if - !___________________________________________________________________________ + !_______________________________________________________________________ ! Sea level and pressure contribution -\nabla(\eta +hpressure/rho_0) ! and the Coriolis force + metric terms elnodes=elem2D_nodes(:,elem) @@ -119,9 +124,9 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) p_eta = g*eta_n(elnodes) ff = mesh%coriolis(elem)*elem_area(elem) - !mm=metric_factor(elem)*gg + !mm=metric_factor(elem)*elem_area(elem) - !___________________________________________________________________________ + !_______________________________________________________________________ ! contribution from sea level pressure if (l_mslp) then p_air = press_air(elnodes)/1000 @@ -129,7 +134,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) p_air = 0.0_WP end if - !___________________________________________________________________________ + !_______________________________________________________________________ ! in case of ALE zlevel and zstar add pressure from ice to atmospheric pressure ! to account for floating ice if (use_pice > 0) then @@ -141,7 +146,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) p_ice = 0.0_WP endif - !____________________________________________________________________________ + !_______________________________________________________________________ ! apply pressure gradient force, as well as contributions from gradient of ! the sea surface height as well as ice pressure in case of floating sea ice ! to velocity rhs @@ -149,35 +154,45 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) if (use_global_tides) then pre=pre-ssh_gp(elnodes) end if - Fx = sum(gradient_sca(1:3,elem)*pre) Fy = sum(gradient_sca(4:6,elem)*pre) - do nz=nzmin,nzmax-1 - ! add pressure gradient terms - UV_rhs(1,nz,elem) = UV_rhs(1,nz,elem) + (Fx-pgf_x(nz,elem))*elem_area(elem) - UV_rhs(2,nz,elem) = UV_rhs(2,nz,elem) + (Fy-pgf_y(nz,elem))*elem_area(elem) - end do - !____________________________________________________________________________ + !_______________________________________________________________________ ! when ssh split-explicite subcycling method is setted use transport velocities ! u*h, v*h instead of u,v if (.not. dynamics%use_ssh_splitexpl_subcycl) then - do nz=nzmin,nzmax-1 - ! add coriolis force - UV_rhsAB(1,nz,elem) = UV(2,nz,elem)*ff! + mm*UV(1,nz,elem)*UV(2,nz,elem) - UV_rhsAB(2,nz,elem) =-UV(1,nz,elem)*ff! - mm*UV(1,nz,elem)*UV(2,nz,elem) + do nz=nzmin, nzmax-1 + ! add pressure gradient terms + UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx-pgf_x(nz, elem))*elem_area(elem) + UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (Fy-pgf_y(nz, elem))*elem_area(elem) + + ! add coriolis force, initialise AB2 array of actual timestep + ! with coriolis term + UV_rhsAB(1, nz, elem) = UV(2, nz, elem)*ff! + mm*UV(1,nz,elem)*UV(2,nz,elem) + UV_rhsAB(2, nz, elem) =-UV(1, nz, elem)*ff! - mm*UV(1,nz,elem)*UV(2,nz,elem) end do else - UVh_BT4AB(1:2,elem) = 0.0_WP + UVhBT_4AB(1:2,elem) = 0.0_WP do nz=nzmin,nzmax-1 + ! add pressure gradient terms + UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx-pgf_x(nz, elem))*elem_area(elem)*helem(nz,elem) + UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (Fy-pgf_y(nz, elem))*elem_area(elem)*helem(nz,elem) + + ! add coriolis force, initialise AB2 array of actual timestep + ! with coriolis term UV_rhsAB(1, nz ,elem) = UVh(1, nz, elem)*ff! + mm*UV(1,nz,elem)*UVh(1, nz, elem) UV_rhsAB(2, nz ,elem) =-UVh(2, nz, elem)*ff! - mm*UV(1,nz,elem)*UVh(2, nz, elem) - UVh_BT4AB(1,elem) = UVh_BT4AB(1, elem) + UVh(1, nz, elem) ! Barotropic velocity - UVh_BT4AB(2,elem) = UVh_BT4AB(2, elem) + UVh(2, nz, elem) ! + + ! compute barotropic velocity for adams-bashfort time stepping + ! UVhBT_4AB(1:2, elem)--> actual timestep, + ! UVhBT_4AB(3:4, elem)--> previous timestep (is setted in + ! call compute_BC_BT_SE_vtransp) + UVhBT_4AB(1, elem) = UVhBT_4AB(1, elem) + UVh(1, nz, elem) ! + UVhBT_4AB(2, elem) = UVhBT_4AB(2, elem) + UVh(2, nz, elem) ! end do end if - !___________________________________________________________________________ + !_______________________________________________________________________ if (dynamics%ldiag_ke) then do nz=nzmin,nzmax-1 dynamics%ke_pre(1,nz,elem)= (Fx-pgf_x(nz,elem))*dt!*elem_area(elem) !not to divide it aterwards (at the end of this subroutine) @@ -195,7 +210,8 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) !$OMP END PARALLEL DO !___________________________________________________________________________ - ! advection + ! advection --> add momentum advection to actual timerstep adams-bashfort + ! array UV_rhsAB if (dynamics%momadv_opt==1) then if (mype==0) write(*,*) 'in moment not adapted mom_adv advection typ for ALE, check your namelist' call par_ex(partit%MPI_COMM_FESOM, partit%mype, 1) @@ -206,6 +222,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) call momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) end if end if + !___________________________________________________________________________ ! Update the rhs ff=(1.5_WP+epsilon) @@ -220,20 +237,30 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) do nz=nzmin,nzmax-1 UV_rhs(1,nz,elem)=dt*(UV_rhs(1,nz,elem)+UV_rhsAB(1,nz,elem)*ff)/elem_area(elem) UV_rhs(2,nz,elem)=dt*(UV_rhs(2,nz,elem)+UV_rhsAB(2,nz,elem)*ff)/elem_area(elem) + ! | | + ! V V + ! fAB = (f_pgf - 1/2*fab_n-1) +3/2*fab_n + ! + ! until here: UV_rhs = dt*[ (R_advec + R_coriolis)^n + R_pressure] + ! --> horizontal viscosity contribution still missing is added in + ! call viscosity_filter + ! --> vertical viscosity contribution still missing is added in + ! call impl_vert_visc_ale end do end do !$OMP END PARALLEL DO + !___________________________________________________________________________ if (dynamics%ldiag_ke) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_adv(:,nz,elem)=dt*(dynamics%ke_adv(:,nz,elem)+dynamics%ke_adv_AB(:,nz,elem)*ff)/elem_area(elem) - dynamics%ke_cor(:,nz,elem)=dt*(dynamics%ke_cor(:,nz,elem)+dynamics%ke_cor_AB(:,nz,elem)*ff)/elem_area(elem) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_adv(:,nz,elem)=dt*(dynamics%ke_adv(:,nz,elem)+dynamics%ke_adv_AB(:,nz,elem)*ff)/elem_area(elem) + dynamics%ke_cor(:,nz,elem)=dt*(dynamics%ke_cor(:,nz,elem)+dynamics%ke_cor_AB(:,nz,elem)*ff)/elem_area(elem) + end do end do - end do !$OMP END PARALLEL DO end if @@ -243,9 +270,10 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) END SUBROUTINE compute_vel_rhs ! +! +!_______________________________________________________________________________ ! Momentum advection on scalar control volumes with ALE adaption--> exchange zinv(nz) ! against hnode(nz,node) -!_______________________________________________________________________________ subroutine momentum_adv_scalar(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT From 466e3d7c507cc76fc5195756e1948b9aefd1485d Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 22 May 2023 14:29:47 +0200 Subject: [PATCH 13/50] actualise and add some comments and explanations --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 56 +++++++++--- src/oce_ale_tracer.F90 | 119 +++++++++++++++----------- 2 files changed, 116 insertions(+), 59 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index af618aaf0..334d91951 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -497,22 +497,54 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) ! V ! du * h^(*) = dt*( Av*d/dz*du )|^t_b + dt*( Av*d/dz*u^(n+0.5,*) )|^t_b ! du - dt/h^(*)*( Av*d/dz*du ) = dt/h^(*)*( Av*d/dz*u^(n+0.5,*) ) +! --> solve for du +! ^ +! /|\ nvec_up (+1) +! | +! ----------- zbar_1, A_1 *----|----* +! Z_1 o T_1, Ac_1 |\ | ./| +! ----------- zbar_2, A_2 | \ ./ | Gaus Theorem: +! Z_2 o T_2, Ac_2 | \ / | --> Flux form +! ----------- zbar_3, A_3 | | | --> normal vec outwards facing +! Z_3 o T_3, Ac_3 *---|-----* +! ----------- zbar_4 \ | | ./ +! : \ | |/ +! \|/| +! * | +! V nvec_dwn (-1) ! ! --> 1st. solve homogenouse part: ! f(du) = du - dt/h^(*)*( Av*d/dz*du ) = 0 ! ! --> 2nd. compute difference quotient at du_i using Gauss-Theorem --> flux form -! V_i * du_i = dt * [ Av_i/h_i * (du_i-1 - du_i)/(Z_i-1 - Z_i) * A_i * nvec_up(+1) -! +Av_i+1/h_i * (du_i - du_i+1)/(Z_i - Z_i+1) * A_i+1 * nvec_dwn(-1) ] -! -! f(du_i) = du_i - dt*Av_i/h_i * (du_i-1 - du_i)/(Z_i-1 - Z_i) * A_i/V_i+1 -! + dt*Av_i+1/h_i * (du_i - du_i+1)/(Z_i - Z_i+1) * A_i+1/V_i+1 +! | +! +-> Gauss THeorem: int(V', div(F_vec))dV = intcircle(A', F_vec*n_vec)dA +! | +! +-> du = dt/h^(*)*( Av*d/dz*du ) | *div()_z=d/dz, *int(V',)dV +! | int(V', d/dz *du)dV = int(V', d/dz *dt/h^(*)*( Av*d/dz*du ) )dV +! | ... = intcircle(A', dt/h^(*)*( Av*d/dz*du )*n_vec)dA +! | int(V', d/dz *du)dz*dA = ... +! | int(A', du)dA = intcircle(A', dt/h^(*)*( Av*d/dz*du )*n_vec)dA +! | +! +-> Ac_i area of vector cell = are of triangle, A_i, A_i+1 area of top +! | and bottom face +! V +! +! du_i * Ac_i = dt * [ Av_i /h_i * (du_i-1 - du_i )/(Z_i-1 - Z_i ) * A_i * nvec_up(+1) +! +Av_i+1/h_i * (du_i - du_i+1)/(Z_i - Z_i+1) * A_i+1 * nvec_dwn(-1) ] +! | +! +-> since we are on triangles Ac_i = A_i = A_i+1 --> can kick out A_i/Ac_i +! | and A_i+1/Ac_i +! +-> take into account normal vector direction +! V +! f(du_i) = du_i - dt*Av_i /h_i * (du_i-1 - du_i )/(Z_i-1 - Z_i ) +! + dt*Av_i+1/h_i * (du_i - du_i+1)/(Z_i - Z_i+1) ! = 0 ! ! --> 3rd. solve for coefficents a, b, c (homogenous part): ! f(du_i) = [ a*du_i-1 + b*du_i + c*du_i+1 ] ! | -! +-> estimate a, b, c by derivation of d(du_i) +! +-> estimate a, b, c by derivation of f(du_i) ! | ! +-> a = d[f(du_i)]/d[du_i-1] = - dt*Av_i/h_i / (Z_i-1 - Z_i) ! | @@ -522,18 +554,20 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) ! 1 - a - c ! ! --> 4th. solve inhomogenous part: -! [ a*du_i-1 + b*du_i + c*du_i+1 ] = RHS/V_i +! [ a*du_i-1 + b*du_i + c*du_i+1 ] = RHS/A_i ! -! RHS/V_i = dt* [ Av_i /h_i * (u^(n+0.5,*)_i-1 - u^(n+0.5,*)_i)/(Z_i-1 - Z_i) * A_i * nvec_up(+1) -! | +Av_i+1/h_i * (u^(n+0.5,*)_i - u^(n+0.5,*)_i+1)/(Z_i - Z_i+1) * A_i+1 * nvec_dwn(-1) ] -! | +! RHS/A_i = dt* [ Av_i /h_i * (u^(n+0.5,*)_i-1 - u^(n+0.5,*)_i )/(Z_i-1 - Z_i ) * A_i /A_i * nvec_up(+1) +! | +Av_i+1/h_i * (u^(n+0.5,*)_i - u^(n+0.5,*)_i+1)/(Z_i - Z_i+1) * A_i+1/A_i * nvec_dwn(-1) ] +! | +! +-> since we are on triangles A_i = A_i+1 --> can kick out A_i +! +-> take into account normal vector direction ! V ! = -a*u^(n+0.5,*)_i-1 + (a+c)*u^(n+0.5,*)_i - c*u^(n+0.5,*)_i+1 ! ! --> 5th. solve for du_i --> forward sweep algorithm --> see lower ! | b_1 c_1 ... | |du_1| ! | a_2 b_2 c_2 ... | |du_2| -! | a_3 b_3 c_3 ... | * |du_3| = RHS/V_i +! | a_3 b_3 c_3 ... | * |du_3| = RHS/A_i ! | a_4 b_4 c_4 ...| |du_4| ! | : | | : | ! diff --git a/src/oce_ale_tracer.F90 b/src/oce_ale_tracer.F90 index 34b9528ff..dc1201fe2 100644 --- a/src/oce_ale_tracer.F90 +++ b/src/oce_ale_tracer.F90 @@ -485,69 +485,90 @@ subroutine diff_ver_part_impl_ale(tr_num, dynamics, tracers, partit, mesh) if (Redi) isredi=1._WP Ty =0.0_WP Ty1 =0.0_WP - + ! solve equation diffusion equation implicite part: ! --> h^(n+0.5)* (T^(n+0.5)-Tstar) = dt*( K_33*d/dz*(T^(n+0.5)-Tstar) + K_33*d/dz*Tstar ) ! --> Tnew = T^(n+0.5)-Tstar ! --> h^(n+0.5)* (Tnew) = dt*(K_33*d/dz*Tnew) + K_33*dt*d/dz*Tstar ! --> h^(n+0.5)* (Tnew) = dt*(K_33*d/dz*Tnew) + RHS ! --> solve for T_new - ! --> V_1 (Skalar Volume), A_1 (Area of edge), . - ! no Cavity A1==V1, yes Cavity A1 !=V1 /I\ nvec_up (+1) - ! I - ! ----------- zbar_1, A_1 *----I----* - ! Z_1 o T_1, V1 |\ I ./| - ! ----------- zbar_2, A_2 | \ ./ | Gaus Theorem: - ! Z_2 o T_2, V2 | \ / | --> Flux form - ! ----------- zbar_3, A_3 | | | --> normal vec outwards facing - ! Z_3 o T_3, V3 *---|-----* - ! ----------- zbar_4 \ | I ./ - ! : \ | I/ - ! \|/I - ! * I - ! \I/ - ! * nvec_dwn (-1) + ! --> As_1 (Skalar Area), A_1 (Area of edge), + ! no Cavity A_1==As_1, yes Cavity A1 !=As_1 + ! + ! ----------- zbar_1, A_1 nvec_up (+1) + ! Z_1 o T_1, As_1 ^ + ! ----------- zbar_2, A_2 _____|_____ Gaus Theorem: + ! Z_2 o T_2, As_2 /| | |\ --> Flux form + ! ----------- zbar_3, A_3 * | | | * --> normal vec outwards facing + ! Z_3 o T_3, As_3 |\___________/| + ! ----------- zbar_4 | | | | + ! : | |_________| | + ! |/ \| + ! * | * + ! \_____|_____/ + ! | + ! V nvec_dwn (-1) + ! ! --> 1st. solve homogenouse part: - ! f(Tnew) = h^(n+0.5)* (Tnew) - dt*(K_33*dTnew/dz) = 0 + ! f(Tnew) = h^(n+0.5)* (Tnew) - dt*(K_33*dTnew/dz) = 0 ! ! --> 2nd. Difference Quotient at Tnew_i in flux form (Gaus Theorem, dont forget normal vectors!!!): - ! V_i*Tnew_i *h_i = -dt * [ K_33 * (Tnew_i-1 - Tnew_i)/(Z_i-1 - Z_i) * A_i * nvec_up - ! +K_33 * (Tnew_i - Tnew_i+1)/(Z_i - Z_i+1) * A_i+1 * nvec_dwn ] - ! Tnew_i *h_i = -dt * [ K_33 * (Tnew_i-1 - Tnew_i)/(Z_i-1 - Z_i) * A_i /V_i * nvec_up - ! +K_33 * (Tnew_i - Tnew_i+1)/(Z_i - Z_i+1) * A_i+1/V_i * nvec_dwn ] - ! - ! --> 3rd. solve for coefficents a, b, c: - ! f(Tnew) = [ a*dTnew_i-1 + b*dTnew_i + c*dTnew_i+1 ] - ! - ! df(Tnew)/dTnew_i-1 = a = -dt*K_33/(Z_i-1 - Z_i) * A_i/V_i * (nvec_up =1) + ! | + ! +-> Gauss THeorem: int(V', div(F_vec))dV = intcircle(A', F_vec*n_vec)dA + ! | + ! +-> du = dt*( K_33*d/dz*dTnew ) | *div()_z=d/dz, *int(V',)dV + ! | int(V', d/dz *du)dV = int(V', d/dz *dt*( K_33*d/dz*dTnew ) )dV + ! | ... = intcircle(A', dt*( K_33*d/dz*dTnew )*n_vec)dA + ! | int(V', d/dz *du)dz*dA = ... + ! | int(A', du)dA = intcircle(A', dt*( K_33*d/dz*dTnew )*n_vec)dA + ! | + ! +-> As_i area of scalar cell = A_i, A_i+1 area of top + ! | and bottom face of scalar cell + ! V ! - ! df(Tnew)/dTnew_i+1 = c = dt * K_33 * 1/(Z_i - Z_i+1) * A_i+1/V_i * (nvec_dwn=-1) - ! = -dt * K_33 * 1/(Z_i - Z_i+1) * A_i+1/V_i + ! As_i*Tnew_i *h_i = dt * [ K_33 * (Tnew_i-1 - Tnew_i )/(Z_i-1 - Z_i ) * A_i * nvec_up(+1) + ! +K_33 * (Tnew_i - Tnew_i+1)/(Z_i - Z_i+1) * A_i+1 * nvec_dwn(-1) ] + ! Tnew_i *h_i = dt * [ K_33 * (Tnew_i-1 - Tnew_i )/(Z_i-1 - Z_i ) * A_i /As_i * nvec_up(+1) + ! +K_33 * (Tnew_i - Tnew_i+1)/(Z_i - Z_i+1) * A_i+1/As_i * nvec_dwn(-1) ] + ! | + ! +-> since we are on scalar cell As_i != A_i != A_i+1 + ! +-> take into account normal vector direction + ! V + ! f(Tnew) = Tnew_i*h_i - dt*K_33 * (Tnew_i-1 - Tnew_i )/(Z_i-1 - Z_i ) * A_i /As_i + ! + dt*K_33 * (Tnew_i - Tnew_i+1)/(Z_i - Z_i+1) * A_i+1/As_i + ! = 0 ! - ! df(Tnew)/dTnew_i = b = h_i + dt*K_33/(Z_i-1 - Z_i) * A_i/V_i * (nvec_up=+1) - ! - dt*K_33/(Z_i - Z_i+1) * A_i+1/V_i * (nvec_dwn=-1) - ! = h_i + dt*K_33/(Z_i-1 - Z_i) * A_i/V_i - ! + dt*K_33/(Z_i - Z_i+1) * A_i+1/V_i - ! = h_i -(a+c) + ! --> 3rd. solve for coefficents a, b, c: + ! f(Tnew) = [ a*dTnew_i-1 + b*dTnew_i + c*dTnew_i+1 ] + ! | + ! +-> estimate a, b, c by derivation of f(Tnew_i) + ! | + ! +-> a = df(Tnew)/dTnew_i-1 = -dt*K_33/(Z_i-1 - Z_i) * A_i/As_i + ! | + ! +-> c = df(Tnew)/dTnew_i+1 = -dt * K_33 * 1/(Z_i - Z_i+1) * A_i+1/As_i + ! | + ! +-> b = df(Tnew)/dTnew_i = h_i + dt*K_33/(Z_i-1 - Z_i) * A_i /As_i + ! + dt*K_33/(Z_i - Z_i+1) * A_i+1/As_i + ! = h_i -(a+c) ! ! --> 4th. solve inhomogenous part: - ! [ a*dTnew_i-1 + b*dTnew_i + c*dTnew_i+1 ] = RHS/V_i + ! [ a*dTnew_i-1 + b*dTnew_i + c*dTnew_i+1 ] = RHS/As_i ! - ! RHS = K_33*dt*d/dz*Tstar + ! RHS = K_33*dt*d/dz*Tstar ! ! --> write as Difference Quotient in flux form - ! RHS/V_i = K_33 * dt * (Tstar_i-1 - Tstar_i)/(Z_i-1 - Z_i) * A_i/V_i * (nvec_up=1) - ! + K_33 * dt * (Tstar_i - Tstar_i+1)/(Z_i - Z_i+1) * A_i+1/V_i * (nvec_dwn=-1) + ! RHS/As_i = K_33 * dt * (Tstar_i-1 - Tstar_i)/(Z_i-1 - Z_i) * A_i/As_i * (nvec_up=1) + ! + K_33 * dt * (Tstar_i - Tstar_i+1)/(Z_i - Z_i+1) * A_i+1/As_i * (nvec_dwn=-1) ! - ! = K_33*dt/(Z_i-1 - Z_i) * A_i/V_i * Tstar_i-1 - ! - K_33*dt/(Z_i-1 - Z_i) * A_i/V_i * Tstar_i - ! - K_33*dt/(Z_i - Z_i+1) * A_i+1/V_i * Tstar_i - ! + K_33*dt/(Z_i - Z_i+1) * A_i+1/V_i * Tstar_i+1 + ! = K_33*dt/(Z_i-1 - Z_i) * A_i/As_i * Tstar_i-1 + ! - K_33*dt/(Z_i-1 - Z_i) * A_i/As_i * Tstar_i + ! - K_33*dt/(Z_i - Z_i+1) * A_i+1/As_i * Tstar_i + ! + K_33*dt/(Z_i - Z_i+1) * A_i+1/As_i * Tstar_i+1 ! - ! = -a*Tstar_i-1 + (a+c)*Tstar_i - c * Tstar_i+1 - ! |-> b = h_i - (a+c), a+c = h_i-b + ! = -a*Tstar_i-1 + (a+c)*Tstar_i - c * Tstar_i+1 + ! |-> b = h_i - (a+c), a+c = h_i-b ! - ! = -a*Tstar_i-1 - (b-h_i)*Tstar_i - c * Tstar_i+1 + ! = -a*Tstar_i-1 - (b-h_i)*Tstar_i - c * Tstar_i+1 ! ! --> 5th. solve for Tnew_i --> forward sweep algorithm --> see lower ! | b_1 c_1 ... | |dTnew_1| @@ -556,12 +577,14 @@ subroutine diff_ver_part_impl_ale(tr_num, dynamics, tracers, partit, mesh) ! | a_4 b_4 c_4 ...| |dTnew_4| ! | : | | : | ! - ! --> a = -dt*K_33 / (Z_i-1 - Z_i) * A_i/V_i + ! --> a = -dt*K_33 / (Z_i-1 - Z_i) * A_i/As_i ! - ! --> c = -dt*K_33 / (Z_i - Z_i+1) * A_i+1/V_i + ! --> c = -dt*K_33 / (Z_i - Z_i+1) * A_i+1/As_i + ! + ! --> b = h^(n+0.5) -[ dt*K_33/(Z_i-1 - Z_i ) * A_i /As_i + ! +dt*K_33/(Z_i - Z_i+1) * A_i+1/As_i ] + ! = -(a+c) + h^(n+0.5) ! - ! --> b = h^(n+0.5) -[ dt*K_33/(Z_i-1 - Z_i)*A_i/V_i + dt*K_33/(Z_i - Z_i+1) * A_i+1/V_i ] = -(a+c) + h^(n+0.5) - !___________________________________________________________________________ ! loop over local nodes From 07282eb31720dac60d6106e31328b3139da3cfcc Mon Sep 17 00:00:00 2001 From: Patrick Scholz Date: Tue, 23 May 2023 11:51:36 +0200 Subject: [PATCH 14/50] add some more comments and change some varaible names --- src/MOD_DYN.F90 | 12 +-- src/oce_ale.F90 | 2 +- src/oce_ale_ssh_splitexpl_subcycl.F90 | 148 +++++++++++++++++++------- src/oce_ale_vel_rhs.F90 | 16 +-- src/oce_setup_step.F90 | 14 ++- 5 files changed, 129 insertions(+), 63 deletions(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index 900ed561d..e95fae68d 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -65,10 +65,10 @@ MODULE MOD_DYN !___arrays for split explicite ssh computation______________________________ ! se_uvh...transport velocity, real(kind=WP), allocatable, dimension(:,:,:):: se_uvh - !se_uv_rhs...vertical integral of transport velocity rhs, se_uvhBT_4AB... + !se_uv_rhs...vertical integral of transport velocity rhs, se_uvBT_4AB... ! barotropic transport velocities (vertically integrated), contains actual ! timestep (1:2) and previous timestep (3:4) for adams-bashfort interpolation - real(kind=WP), allocatable, dimension(:,:) :: se_uvhBC_rhs, se_uvhBT_4AB + real(kind=WP), allocatable, dimension(:,:) :: se_uvBT_rhs, se_uvBT_4AB ! se_uvBT...barotropic trnasport velocities from barotropic time stepping ! se_uvBT_theta...velocities for dissipative time stepping of thickness equation @@ -276,8 +276,8 @@ subroutine WRITE_T_DYN(dynamics, unit, iostat, iomsg) end if if (dynamics%use_ssh_splitexpl_subcycl) then call write_bin_array(dynamics%se_uvh , unit, iostat, iomsg) - call write_bin_array(dynamics%se_uvhBC_rhs , unit, iostat, iomsg) - call write_bin_array(dynamics%se_uvhBT_4AB , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT_rhs , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT_4AB , unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT , unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) @@ -332,8 +332,8 @@ subroutine READ_T_DYN(dynamics, unit, iostat, iomsg) end if if (dynamics%use_ssh_splitexpl_subcycl) then call read_bin_array(dynamics%se_uvh , unit, iostat, iomsg) - call read_bin_array(dynamics%se_uvhBC_rhs , unit, iostat, iomsg) - call read_bin_array(dynamics%se_uvhBT_4AB , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT_rhs , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT_4AB , unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT , unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 07d6f5ff5..825e3c097 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -3194,7 +3194,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) else ! Compute vertical integral of transport velocity rhs omitting the contributions from ! the elevation and Coriolis. - call compute_BC_BT_SE_vtransp(dynamics, partit, mesh) + call compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ! Do barotropic step, get eta_{n+1} and BT transport call compute_BT_step_SE_ale(dynamics, partit, mesh) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 334d91951..c0b2ef63f 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -35,7 +35,7 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) !_______________________________________________________________________________ module compute_ssh_split_explicit_interface interface - subroutine compute_BC_BT_SE_vtransp(dynamics, partit, mesh) + subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT USE MOD_PARSUP @@ -739,7 +739,7 @@ end subroutine impl_vert_visc_ale_vtransp !SD Compute vertical integral of transport velocity rhs omitting the contributions from !SD the elevation and Coriolis. The elevation and Coriolis are accounted for !SD explicitly in BT equations, and should therefore be removed from the vertically integrated rhs. -subroutine compute_BC_BT_SE_vtransp(dynamics, partit, mesh) +subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) USE MOD_PARTIT USE MOD_PARSUP USE MOD_MESH @@ -757,15 +757,15 @@ subroutine compute_BC_BT_SE_vtransp(dynamics, partit, mesh) ! pointer on necessary derived types real(kind=WP), dimension(:,:,:), pointer :: UV_rhs real(kind=WP), dimension(:) , pointer :: eta_n - real(kind=WP), dimension(:,:) , pointer :: UVhBT_4AB, UVhBC_rhs + real(kind=WP), dimension(:,:) , pointer :: UVBT_4AB, UVBT_rhs #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" #include "associate_mesh_ass.h" eta_n =>dynamics%eta_n(:) UV_rhs =>dynamics%uv_rhs(:,:,:) - UVhBC_rhs =>dynamics%se_uvhBC_rhs(:,:) - UVhBT_4AB =>dynamics%se_uvhBT_4AB(:,:) + UVBT_rhs =>dynamics%se_uvBT_rhs(:,:) + UVBT_4AB =>dynamics%se_uvBT_4AB(:,:) !___________________________________________________________________________ ab1=(1.5_WP+epsilon) @@ -791,7 +791,7 @@ subroutine compute_BC_BT_SE_vtransp(dynamics, partit, mesh) Fy=g*dt*sum(gradient_sca(5:8,elem)*eta_n(elnodes)) !_______________________________________________________________________ - ! Sum 3D rhs + ! vertically integrate UV_rhs --> for barotropic equatiobn vert_sum_u=0.0_WP vert_sum_v=0.0_WP do nz=nzmin, nzmax @@ -800,31 +800,53 @@ subroutine compute_BC_BT_SE_vtransp(dynamics, partit, mesh) end do !_______________________________________________________________________ - ! Remove the contribution from the elevation + ! Remove the contribution from the elevation will be accounted explicitely + ! for in the barotropic equation hh=sum(helem(nzmin:nzmax, elem)) vert_sum_u=vert_sum_u + Fx*hh vert_sum_v=vert_sum_v + Fy*hh !_______________________________________________________________________ - ! Remove the contribution from the Coriolis - UVhBC_rhs(1, elem)=vert_sum_u - dt*mesh%coriolis(elem)* & - (ab1*UVhBT_4AB(2,elem)+ab2*UVhBT_4AB(4,elem)) ! for AB-interpolated - UVhBC_rhs(2, elem)=vert_sum_v + dt*mesh%coriolis(elem)* & - (ab1*UVhBT_4AB(1,elem)+ab2*UVhBT_4AB(3,elem)) + ! Remove the contribution from the Coriolis will be accounted explicitely + ! for in the barotropic equation + ! UVBT_rhs ... baroclinic forcing term in barotropic equation R_b + ! --> d/dt*U_bt + f*e_z x U_bt + g*H* grad(eta) = R_bt + UVBT_rhs(1, elem)=vert_sum_u - dt*mesh%coriolis(elem)* & + (ab1*UVBT_4AB(2,elem)+ab2*UVBT_4AB(4,elem)) ! for AB-interpolated + UVBT_rhs(2, elem)=vert_sum_v + dt*mesh%coriolis(elem)* & + (ab1*UVBT_4AB(1,elem)+ab2*UVBT_4AB(3,elem)) !_______________________________________________________________________ - ! save actual of vertical integrated transport velocity UVhBT_4AB(1:2,elem) + ! save actual of vertical integrated transport velocity UVBT_4AB(1:2,elem) ! timestep for next time loop step - UVhBT_4AB(3:4,elem)=UVhBT_4AB(1:2,elem) + ! UVBT_4AB ... is U_bt in barotropic equation + ! --> d/dt*U_bt + f*e_z x U_bt + g*H*grad_H(eta) = R_bt + UVBT_4AB(3:4,elem)=UVBT_4AB(1:2,elem) end do !PS !$OMP END DO !PS !$OMP END PARALLEL -end subroutine compute_BC_BT_SE_vtransp +end subroutine compute_BT_rhs_SE_vtransp ! ! !_______________________________________________________________________________ -! Barotropic time stepping with Forward-Backward dissipative method -! (Demange et al. 2019) +! Barotropic time stepping with Forward-Backward dissipative method +! (Demange et al. 2019) is used where eta and U_bt = sum_k(U_k) = sum_k(u_k*h_k) +! are estimated from the equations ... +! --> d/dt*U_bt + f*e_z x U_bt + g*H*grad_H(eta) = R_bt +! --> d/dt*eta + div_h(U_bt) + FW = 0 +! +! +! --> forward-backward dissipative time stepping by Demagne et al. 2019 +! --> equation (6) in T. Banerjee et al.,Split-Explicite external +! mode solver in FESOM2, +! +! Ubt^(n+(m+1)/M) = Ubt^(n+(m)/M) - dt/M*[ +! + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+(m)/M)) +! - h*H^m*grad_H*eta^((n+m)/M) +! - Rbt-->UVBT_rhs ] +! +! eta^(n+(m+1)/M) = eta^(n+(m)/M) - dt/M * div_H * [(1+theta)*Ubt^(n+(m+1)/M) - theta*Ubt^(n+(m)/M)] +! subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) USE MOD_PARTIT USE MOD_PARSUP @@ -844,13 +866,13 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:) , pointer :: eta_n - real(kind=WP), dimension(:,:) , pointer :: UVhBC_rhs, UVBT, UVBT_theta, UVBT_mean, UVBT_12 + real(kind=WP), dimension(:,:) , pointer :: UVBT_rhs, UVBT, UVBT_theta, UVBT_mean, UVBT_12 #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" #include "associate_mesh_ass.h" eta_n =>dynamics%eta_n(:) - UVhBC_rhs =>dynamics%se_uvhBC_rhs(:,:) + UVBT_rhs =>dynamics%se_uvBT_rhs(:,:) UVBT =>dynamics%se_uvBT(:,:) UVBT_theta=>dynamics%se_uvBT_theta(:,:) UVBT_mean =>dynamics%se_uvBT_mean(:,:) @@ -877,25 +899,44 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! Advance velocities. I use SI stepping for the Coriolis do elem=1, myDim_elem2D elnodes=elem2D_nodes(:,elem) - !PS hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP + !___________________________________________________________________ + ! compute term: + ! AAA = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+(m)/M)) + ! - h*H^m*grad_H*eta^((n+m)/M) + ! - Rbt-->UVBT_rhs ] hh = -zbar_e_bot(elem)+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth f = mesh%coriolis(elem) - rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(1, elem)) + BT_inv*UVhBC_rhs(1, elem) - ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(2, elem)) + BT_inv*UVhBC_rhs(2, elem) + rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(1, elem)) + BT_inv*UVBT_rhs(1, elem) + ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(2, elem)) + BT_inv*UVBT_rhs(2, elem) ! Semi-Implicit Coriolis a = dtBT*f*0.5_WP d = 1.0_WP/(1.0_WP+a*a) ax = d*(rx+a*ry) ay = d*(-a*rx+ry) - UVBT( 1, elem) = UVBT( 1, elem) + ax ! New velocity - UVBT( 2, elem) = UVBT( 2, elem) + ay + !___________________________________________________________________ + ! compute new velocities Ubt^(n+(m+1)/M) at barotropic time step (n+(m+1)/M) ... + ! Ubt^(n+(m+1)/M) = Ubt^(n+(m)/M) + AAA + ! equation (6) in T. Banerjee et al.,Split-Explicite external + ! mode solver in FESOM2, + UVBT( 1, elem) = UVBT(1, elem) + ax + UVBT( 2, elem) = UVBT(2, elem) + ay + + !___________________________________________________________________ ! velocities for dissipative time stepping of thickness equation - UVBT_theta(1, elem) = UVBT_theta(1, elem) + thetaBT*ax ! New vel*(1+thetaBT)-old vel *thetaBT - UVBT_theta(2, elem) = UVBT_theta(2, elem) + thetaBT*ay + ! compute: [(1+theta)*Ubt^(n+(m+1)/M) - theta*Ubt^(n+(m)/M)] by ... + ! + ! --> AAA = Ubt^(n+(m+1)/M) - Ubt^(n+(m)/M) | *theta + ! AAA * theta = (Ubt^(n+(m+1)/M) - Ubt^(n+(m)/M) )*theta + ! --> Ubt^(n+(m+1)/M) + AAA*theta = Ubt^(n+(m+1)/M) + (Ubt^(n+(m+1)/M) - Ubt^(n+(m)/M) )*theta + ! (1+theta)*Ubt^(n+(m+1)/M) + theta*Ubt^(n+(m)/M) + UVBT_theta(1, elem) = UVBT(1, elem) + thetaBT*ax ! New vel*(1+thetaBT)-old vel *thetaBT + UVBT_theta(2, elem) = UVBT(2, elem) + thetaBT*ay - ! Mean BT velocity to trim 3D velocity in tracers + !___________________________________________________________________ + ! Mean BT velocity to trim 3D velocity in tracers, equation (10) in + ! T. Banerjee et al.,Split-Explicite external mode solver in FESOM2, UVBT_mean( 1, elem) = UVBT_mean( 1, elem) + UVBT_theta(1, elem)*BT_inv UVBT_mean( 2, elem) = UVBT_mean( 2, elem) + UVBT_theta(2, elem)*BT_inv end do @@ -908,12 +949,16 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !_______________________________________________________________________ ! Advance thickness + ! compute: dt/M * div_H * [(1+theta)*Ubt^(n+(m+1)/M) - theta*Ubt^(n+(m)/M)] + ! and advance ssh --> eta^(n+(m+1)/M) = eta^(n+(m)/M) - dt/M * div_H * [...] do edge=1, myDim_edge2D ednodes = edges(:,edge) edelem = edge_tri(:,edge) + + !___________________________________________________________________ + ! compute divergence div_H * [(1+theta)*Ubt^(n+(m+1)/M) - theta*Ubt^(n+(m)/M)] deltaX1 = edge_cross_dxdy(1,edge) deltaY1 = edge_cross_dxdy(2,edge) - c1 = UVBT_theta(2, edelem(1))*deltaX1 - UVBT_theta(1, edelem(1))*deltaY1 c2 = 0.0_WP if(edelem(2)>0) then @@ -921,6 +966,11 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) deltaY2=edge_cross_dxdy(4,edge) c2=-(UVBT_theta(2, edelem(2))*deltaX2 - UVBT_theta(1, edelem(2))*deltaY2) end if + + !___________________________________________________________________ + ! advance ssh --> eta^(n+(m+1)/M) = eta^(n+(m)/M) - dt/M * div_H * [...] + ! equation (6) in T. Banerjee et al.,Split-Explicite external + ! mode solver in FESOM2, eta_n(ednodes(1))=eta_n(ednodes(1))+(c1+c2)*dtBT/area(1,ednodes(1)) eta_n(ednodes(2))=eta_n(ednodes(2))-(c1+c2)*dtBT/area(1,ednodes(2)) end do @@ -949,7 +999,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(in) , target :: mesh - integer :: i,elem, elnodes(4), nz, m + integer :: i,elem, elnodes(4), nz, m, nzmin, nzmax real(kind=WP) :: eta(4), ff real(kind=WP) :: Fx, Fy, ubar, vbar, hh_inv !___________________________________________________________________________ @@ -977,20 +1027,35 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! The trimmed Uh,Vh are consistent with new total height defined by eta_n if (mode==1) then do elem=1, myDim_elem2D - ubar=0.0_WP - vbar=0.0_WP - hh_inv=0.0_WP - do nz=1, nlevels(elem)-1 - ! Update transport velocities + ubar = 0.0_WP + vbar = 0.0_WP + hh_inv = 0.0_WP + + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + do nz=nzmin, nzmax + !_______________________________________________________________ + ! Update transport velocities: U^(n+1/2,**) = U^(n+1/2,*) + U_rhs UVh(1, nz, elem)=UVh(1, nz, elem)+UV_rhs(1, nz, elem) UVh(2, nz, elem)=UVh(2, nz, elem)+UV_rhs(2, nz, elem) + + !_______________________________________________________________ + ! vertically integrate updated transport velocity: sum(k, U_k^(n+1/2,**) ) ubar=ubar+UVh(1, nz, elem) vbar=vbar+UVh(2, nz, elem) hh_inv=hh_inv+helem(nz,elem) end do + !___________________________________________________________________ + ! finalize horizontal transport by making vertically integrated + ! transport equal to the value obtained from the barotropic solution + ! + ! --> equation (11) in T. Banerjee et al.,Split-Explicite external + ! mode solver in FESOM2, + ! U_k^(n+1/2) = U^(n+1/2,**) + ! + [<>^(n+1/2) - sum(k, U_k^(n+1/2,**)] * h_k^(n+1/2)/sum(k,h_k^(n+1/2)) hh_inv=1.0_WP/hh_inv ! Postpone AB and 2nd order, just use available thickness - DO nz=1, nlevels(elem)-1 + DO nz=nzmin, nzmax UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed @@ -1004,17 +1069,20 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! Velocity will be used to advance momentum else do elem=1, myDim_elem2D - ubar=0.0_WP - vbar=0.0_WP - hh_inv=0.0_WP - do nz=1, nlevels(elem)-1 + ubar = 0.0_WP + vbar = 0.0_WP + hh_inv = 0.0_WP + + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + do nz=nzmin, nzmax ubar=ubar+UVh(1, nz, elem) vbar=vbar+UVh(2, nz, elem) hh_inv=hh_inv+helem(nz,elem) end do hh_inv=1.0_WP/hh_inv ! Postpone 2nd order, just use available thickness - do nz=1, nlevels(elem)-1 + do nz=nzmin, nzmax UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_12(1, elem)-ubar)*helem(nz,elem)*hh_inv UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_12(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index e62ff1a93..771dc8303 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -66,7 +66,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) real(kind=WP), dimension(:) , pointer :: m_ice, m_snow, a_ice real(kind=WP) , pointer :: rhoice, rhosno, inv_rhowat real(kind=WP), dimension(:,:,:), pointer :: UVh - real(kind=WP), dimension(:,:) , pointer :: UVhBT_4AB + real(kind=WP), dimension(:,:) , pointer :: UVBT_4AB #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" @@ -83,8 +83,8 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) ! if split-explicite ssh subcycling is used if (dynamics%use_ssh_splitexpl_subcycl) then - UVh => dynamics%se_uvh - UVhBT_4AB => dynamics%se_uvhBT_4AB + UVh => dynamics%se_uvh + UVBT_4AB => dynamics%se_uvBT_4AB end if !___________________________________________________________________________ @@ -172,7 +172,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) UV_rhsAB(2, nz, elem) =-UV(1, nz, elem)*ff! - mm*UV(1,nz,elem)*UV(2,nz,elem) end do else - UVhBT_4AB(1:2,elem) = 0.0_WP + UVBT_4AB(1:2,elem) = 0.0_WP do nz=nzmin,nzmax-1 ! add pressure gradient terms UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx-pgf_x(nz, elem))*elem_area(elem)*helem(nz,elem) @@ -184,11 +184,11 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) UV_rhsAB(2, nz ,elem) =-UVh(2, nz, elem)*ff! - mm*UV(1,nz,elem)*UVh(2, nz, elem) ! compute barotropic velocity for adams-bashfort time stepping - ! UVhBT_4AB(1:2, elem)--> actual timestep, - ! UVhBT_4AB(3:4, elem)--> previous timestep (is setted in + ! UVBT_4AB(1:2, elem)--> actual timestep, + ! UVBT_4AB(3:4, elem)--> previous timestep (is setted in ! call compute_BC_BT_SE_vtransp) - UVhBT_4AB(1, elem) = UVhBT_4AB(1, elem) + UVh(1, nz, elem) ! - UVhBT_4AB(2, elem) = UVhBT_4AB(2, elem) + UVh(2, nz, elem) ! + UVBT_4AB(1, elem) = UVBT_4AB(1, elem) + UVh(1, nz, elem) ! + UVBT_4AB(2, elem) = UVBT_4AB(2, elem) + UVh(2, nz, elem) ! end do end if diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index 59498a14a..c8b4bf2ef 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -473,21 +473,17 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) ! allocate/initialise ssh arrays in derived type allocate(dynamics%eta_n( node_size)) dynamics%eta_n = 0.0_WP - - !!PS allocate(dynamics%ssh_rhs_old(node_size)) - !!PS dynamics%ssh_rhs_old= 0.0_WP - if (dynamics%use_ssh_splitexpl_subcycl) then allocate(dynamics%se_uvh( 2, nl-1, elem_size)) - allocate(dynamics%se_uvhBC_rhs( 2, elem_size)) - allocate(dynamics%se_uvhBT_4AB( 4, elem_size)) + allocate(dynamics%se_uvBT_rhs( 2, elem_size)) + allocate(dynamics%se_uvBT_4AB( 4, elem_size)) allocate(dynamics%se_uvBT( 2, elem_size)) allocate(dynamics%se_uvBT_theta(2, elem_size)) allocate(dynamics%se_uvBT_mean( 2, elem_size)) allocate(dynamics%se_uvBT_12( 2, elem_size)) dynamics%se_uvh = 0.0_WP - dynamics%se_uvhBC_rhs = 0.0_WP - dynamics%se_uvhBT_4AB = 0.0_WP + dynamics%se_uvBT_rhs = 0.0_WP + dynamics%se_uvBT_4AB = 0.0_WP dynamics%se_uvBT = 0.0_WP dynamics%se_uvBT_theta = 0.0_WP dynamics%se_uvBT_mean = 0.0_WP @@ -497,6 +493,8 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) allocate(dynamics%ssh_rhs( node_size)) dynamics%d_eta = 0.0_WP dynamics%ssh_rhs = 0.0_WP + !!PS allocate(dynamics%ssh_rhs_old(node_size)) + !!PS dynamics%ssh_rhs_old= 0.0_WP end if !___________________________________________________________________________ From 7636ceb3cb7b831de86160c1d9cc9d38149b41bb Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 2 Jun 2023 14:59:02 +0200 Subject: [PATCH 15/50] .... --- src/io_blowup.F90 | 19 +- src/io_meandata.F90 | 24 +- src/oce_ale.F90 | 290 +++++++++++++----- src/oce_ale_ssh_splitexpl_subcycl.F90 | 425 ++++++++++++++++++++++++-- src/oce_ale_vel_rhs.F90 | 39 +-- src/write_step_info.F90 | 67 +++- 6 files changed, 729 insertions(+), 135 deletions(-) diff --git a/src/io_blowup.F90 b/src/io_blowup.F90 index 60832dd37..b672392c8 100644 --- a/src/io_blowup.F90 +++ b/src/io_blowup.F90 @@ -102,12 +102,25 @@ subroutine ini_blowup_io(year, ice, dynamics, tracers, partit, mesh) !___Define the netCDF variables for 2D fields_______________________________ !___SSH_____________________________________________________________________ call def_variable(bid, 'eta_n' , (/nod2D/) , 'sea surface elevation', 'm', dynamics%eta_n); - call def_variable(bid, 'd_eta' , (/nod2D/) , 'change in ssh from solver', 'm', dynamics%d_eta); !___ALE related fields______________________________________________________ call def_variable(bid, 'hbar' , (/nod2D/) , 'ALE surface elevation hbar_n+0.5', 'm', hbar); !!PS call def_variable(bid, 'hbar_old' , (/nod2D/) , 'ALE surface elevation hbar_n-0.5', 'm', hbar_old); - call def_variable(bid, 'ssh_rhs' , (/nod2D/) , 'RHS for the elevation', '?', dynamics%ssh_rhs); - call def_variable(bid, 'ssh_rhs_old', (/nod2D/) , 'RHS for the elevation', '?', dynamics%ssh_rhs_old); + if (.not. dynamics%use_ssh_splitexpl_subcycl) then + call def_variable(bid, 'd_eta' , (/nod2D/) , 'change in ssh from solver', 'm', dynamics%d_eta); + call def_variable(bid, 'ssh_rhs' , (/nod2D/) , 'RHS for the elevation', '?', dynamics%ssh_rhs); + call def_variable(bid, 'ssh_rhs_old', (/nod2D/) , 'RHS for the elevation', '?', dynamics%ssh_rhs_old); + + else + call def_variable(bid, 'ubt_rhs' , (/elem2D/), 'zonal RHS barotr. transp. equation' , '?', dynamics%se_uvBT_rhs( 1,:)); + call def_variable(bid, 'vbt_rhs' , (/elem2D/), 'merid. RHS barotr. transp. equation', '?', dynamics%se_uvBT_rhs( 2,:)); + call def_variable(bid, 'ubt' , (/elem2D/), 'zonal barotr. transp.' , '?', dynamics%se_uvBT( 1,:)); + call def_variable(bid, 'vbt' , (/elem2D/), 'merid. barotr. transp.' , '?', dynamics%se_uvBT( 2,:)); + call def_variable(bid, 'ubt_theta', (/elem2D/), 'zonal barotr. theta term.' , '?', dynamics%se_uvBT_theta(1,:)); + call def_variable(bid, 'vbt_theta', (/elem2D/), 'merid. barotr. theta term' , '?', dynamics%se_uvBT_theta(2,:)); + call def_variable(bid, 'ubt_mean' , (/elem2D/), 'zonal barotr. mean term.' , '?', dynamics%se_uvBT_mean( 1,:)); + call def_variable(bid, 'vbt_mean' , (/elem2D/), 'merid. barotr. mean term' , '?', dynamics%se_uvBT_mean( 2,:)); + end if + !___Define the netCDF variables for 3D fields_______________________________ call def_variable(bid, 'hnode' , (/nl-1, nod2D/) , 'ALE stuff', '?', hnode); call def_variable(bid, 'helem' , (/nl-1, elem2D/) , 'Element layer thickness', 'm/s', helem(:,:)); diff --git a/src/io_meandata.F90 b/src/io_meandata.F90 index 1edb8067c..37819c762 100644 --- a/src/io_meandata.F90 +++ b/src/io_meandata.F90 @@ -613,7 +613,28 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh) if (sel_forcvar(14)==0) call def_stream(nod2D , myDim_nod2D , 'relaxsalt', 'relaxation salt flux' , 'm/s*psu', relax_salt(:) , io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) #endif if (sel_forcvar(15)==0) call def_stream(nod2D , myDim_nod2D , 'realsalt' , 'real salt flux from sea ice' , 'm/s*psu', real_salt_flux(:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) - end if + end if +!_______________________________________________________________________________ +! Split-Explicite subcycling varaibles +CASE ('SPLIT-EXPL') + if (dynamics%use_ssh_splitexpl_subcycl) then + call def_stream(elem2D, myDim_elem2D , 'ubt' , 'zonal barotrop. transport' , '?' , dynamics%se_uvBT(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'vbt' , 'merid. barotrop. transport', '?' , dynamics%se_uvBT(2,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'ubt_rhs', 'zonal barotrop. transport RHS' , '?' , dynamics%se_uvBT_rhs(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'vbt_rhs', 'merid. barotrop. transport RHS', '?' , dynamics%se_uvBT_rhs(2,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'ubt_12' , 'zonal barotrop. transport 12' , '?' , dynamics%se_uvBT_12(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'vbt_12' , 'merid. barotrop. transport 12', '?' , dynamics%se_uvBT_12(2,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'ubt_theta', 'zonal barotrop. transport theta' , '?' , dynamics%se_uvBT_theta(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'vbt_theta', 'merid. barotrop. transport theta', '?' , dynamics%se_uvBT_theta(2,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'ubt_mean' , 'zonal barotrop. transport mean' , '?' , dynamics%se_uvBT_mean(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'vbt_mean' , 'merid. barotrop. transport mean', '?' , dynamics%se_uvBT_mean(2,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream((/nl-1,elem2D/), (/nl-1,myDim_elem2D/) , 'u_rhs' , 'zonal transport rhs' , '?' , dynamics%uv_rhs(1,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream((/nl-1,elem2D/), (/nl-1,myDim_elem2D/) , 'v_rhs' , 'merid. transport rhs', '?' , dynamics%uv_rhs(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream((/nl-1,elem2D/), (/nl-1,myDim_elem2D/) , 'uh' , 'zonal transport' , '?' , dynamics%se_uvh(1,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream((/nl-1,elem2D/), (/nl-1,myDim_elem2D/) , 'vh' , 'merid. transport', '?' , dynamics%se_uvh(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + end if + + !_______________________________________________________________________________ CASE DEFAULT if (mype==0) write(*,*) 'stream ', io_list(i)%id, ' is not defined !' @@ -762,6 +783,7 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh) call def_stream(nod2D , myDim_nod2D , 'ke_swA', 'Thermal expansion coeff (alpha)', '1/°C', dynamics%ke_swA(:), io_list(i)%freq, 'm', 8, partit, mesh) call def_stream(nod2D , myDim_nod2D , 'ke_swB', 'Taline contraction coeff (beta)', '1/PSU', dynamics%ke_swB(:), io_list(i)%freq, 'm', 8, partit, mesh) end if + end subroutine ! ! diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 825e3c097..e692b44a4 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -1991,7 +1991,7 @@ subroutine vert_vel_ale(dynamics, partit, mesh) integer :: el(2), enodes(2), n, nz, ed, nzmin, nzmax, uln1, uln2, nln1, nln2 real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, dd, dd1, dddt, cflmax ! still to be understood but if you allocate these arrays statically the results will be different: - real(kind=WP) :: c1(mesh%nl-1), c2(mesh%nl-1) + real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1), e2c1(mesh%nl-1), e2c2(mesh%nl-1) ! --> zlevel with local zstar real(kind=WP) :: dhbar_total, dhbar_rest, distrib_dhbar_int real(kind=WP), dimension(:), allocatable :: max_dhbar2distr, cumsum_maxdhbar, distrib_dhbar @@ -2011,14 +2011,20 @@ subroutine vert_vel_ale(dynamics, partit, mesh) Wvel_e => dynamics%w_e(:,:) Wvel_i => dynamics%w_i(:,:) CFL_z => dynamics%cfl_z(:,:) - ssh_rhs => dynamics%ssh_rhs(:) - ssh_rhs_old => dynamics%ssh_rhs_old(:) eta_n => dynamics%eta_n(:) - d_eta => dynamics%d_eta(:) if (Fer_GM) then fer_UV => dynamics%fer_uv(:,:,:) fer_Wvel=> dynamics%fer_w(:,:) - end if + end if + + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + d_eta => dynamics%d_eta(:) + ssh_rhs => dynamics%ssh_rhs(:) + ssh_rhs_old => dynamics%ssh_rhs_old(:) + else + + end if + !___________________________________________________________________________ ! Contributions from levels in divergence !$OMP PARALLEL DO @@ -2054,11 +2060,11 @@ subroutine vert_vel_ale(dynamics, partit, mesh) ! --> h * u_vec * n_vec ! --> e_vec = (dx,dy), n_vec = (-dy,dx); ! --> h * u*(-dy) + v*dx - c1(nz)=( UV(2,nz,el(1))*deltaX1 - UV(1,nz,el(1))*deltaY1 )*helem(nz,el(1)) + e1c1(nz)=( UV(2,nz,el(1))*deltaX1 - UV(1,nz,el(1))*deltaY1 )*helem(nz,el(1)) ! inflow(outflow) "flux" to control volume of node enodes1 ! is equal to outflow(inflow) "flux" to control volume of node enodes2 if (Fer_GM) then - c2(nz)=(fer_UV(2,nz,el(1))*deltaX1- fer_UV(1,nz,el(1))*deltaY1)*helem(nz,el(1)) + e1c2(nz)=(fer_UV(2,nz,el(1))*deltaX1- fer_UV(1,nz,el(1))*deltaY1)*helem(nz,el(1)) end if end do #if defined(_OPENMP) && !defined(__openmp_reproducible) @@ -2066,17 +2072,17 @@ subroutine vert_vel_ale(dynamics, partit, mesh) #else !$OMP ORDERED #endif - Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+e1c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+e1c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(1))) call omp_set_lock (partit%plock(enodes(2))) #endif - Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-e1c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-e1c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(2))) @@ -2086,33 +2092,49 @@ subroutine vert_vel_ale(dynamics, partit, mesh) !_______________________________________________________________________ ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer ! for el(2) + e2c1 = 0.0_WP + e2c2 = 0.0_WP if(el(2)>0)then deltaX2=edge_cross_dxdy(3,ed) deltaY2=edge_cross_dxdy(4,ed) nzmin = ulevels(el(2)) nzmax = nlevels(el(2))-1 do nz = nzmax, nzmin, -1 - c1(nz)=-(UV(2,nz,el(2))*deltaX2 - UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) + e2c1(nz)=-(UV(2,nz,el(2))*deltaX2 - UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) if (Fer_GM) then - c2(nz)=-(fer_UV(2,nz,el(2))*deltaX2-fer_UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) + e2c2(nz)=-(fer_UV(2,nz,el(2))*deltaX2-fer_UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) end if + + !_______________________________________________________________ + if ( e2c1(nz)/=e2c1(nz) ) then + write(*,*) ' --> subroutine vert_vel_ale --> found Nan in e2c1(nz)' + write(*,*) ' mype =', mype + write(*,*) ' edge =', ed + write(*,*) ' el =', el + write(*,*) ' nz =', nz + write(*,*) ' e2c1(nz) =', e2c1(nz) + write(*,*) ' UV(1,nz,el(2)) =', UV(1,nz,el(2)) + write(*,*) ' UV(2,nz,el(2)) =', UV(2,nz,el(2)) + write(*,*) ' helem(nz,el(2))=', helem(nz,el(2)) + + end if end do #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_set_lock (partit%plock(enodes(1))) #else !$OMP ORDERED #endif - Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+e2c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+e2c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(1))) call omp_set_lock (partit%plock(enodes(2))) #endif - Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-e2c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-e2c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(2))) @@ -2120,6 +2142,23 @@ subroutine vert_vel_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end if + + !_______________________________________________________________________ + if ( any(Wvel(nzmin:nzmax, enodes(1))/=Wvel(nzmin:nzmax, enodes(1))) .or. & + any(Wvel(nzmin:nzmax, enodes(2))/=Wvel(nzmin:nzmax, enodes(2)))) then + write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' + write(*,*) ' mype =', mype + write(*,*) ' edge =', ed + write(*,*) ' enodes =', enodes + write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, enodes(1)) + write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, enodes(2)) + write(*,*) ' e1c1', e1c1(nzmin:nzmax) + write(*,*) ' e1c2', e1c2(nzmin:nzmax) + write(*,*) ' e2c1', e2c1(nzmin:nzmax) + write(*,*) ' e2c2', e2c2(nzmin:nzmax) + + end if + end do ! --> do ed=1, myDim_edge2D !$OMP END DO !$OMP END PARALLEL @@ -2142,6 +2181,14 @@ subroutine vert_vel_ale(dynamics, partit, mesh) fer_Wvel(nz,n)=fer_Wvel(nz,n)+fer_Wvel(nz+1,n) end if end do + + !_______________________________________________________________________ + if ( any(Wvel(nzmin:nzmax, n)/=Wvel(nzmin:nzmax, n)) ) then + write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after cumulativ summation(...)' + write(*,*) ' mype =', mype + write(*,*) ' node =', n + write(*,*) ' Wvel(nzmin:nzmax, n)=', Wvel(nzmin:nzmax, n) + end if end do !$OMP END PARALLEL DO !___________________________________________________________________________ @@ -2159,6 +2206,14 @@ subroutine vert_vel_ale(dynamics, partit, mesh) end if end do + + !_______________________________________________________________________ + if ( any(Wvel(nzmin:nzmax, n)/=Wvel(nzmin:nzmax, n)) ) then + write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after divide with area' + write(*,*) ' mype =', mype + write(*,*) ' node =', n + write(*,*) ' Wvel(nzmin:nzmax, n)=', Wvel(nzmin:nzmax, n) + end if end do !$OMP END PARALLEL DO ! | @@ -2434,6 +2489,14 @@ subroutine vert_vel_ale(dynamics, partit, mesh) ! continutity Wvel(nzmin,n)=Wvel(nzmin,n)-water_flux(n) + !_______________________________________________________________________ + if ( any(Wvel(nzmin:nzmax, n)/=Wvel(nzmin:nzmax, n)) ) then + write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after ssh is distributed' + write(*,*) ' mype =', mype + write(*,*) ' node =', n + write(*,*) ' Wvel(nzmin:nzmax, n)=', Wvel(nzmin:nzmax, n) + end if + endif ! --> if (nzmin==1) then end do ! --> do n=1, myDim_nod2D @@ -2455,10 +2518,12 @@ subroutine vert_vel_ale(dynamics, partit, mesh) write(*,*) write(*,*) 'water_flux = ', water_flux(n) write(*,*) - write(*,*) "ssh_rhs = ", ssh_rhs(n) - write(*,*) "ssh_rhs_old = ", ssh_rhs_old(n) write(*,*) "eta_n = ", eta_n(n) - write(*,*) "d_eta = ", d_eta(n) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + write(*,*) "d_eta = ", d_eta(n) + write(*,*) "ssh_rhs = ", ssh_rhs(n) + write(*,*) "ssh_rhs_old = ", ssh_rhs_old(n) + end if write(*,*) write(*,*) "zbar_3d_n(1,n) = ", zbar_3d_n(1,n) write(*,*) "dd1 = ", dd1 @@ -2487,65 +2552,139 @@ subroutine vert_vel_ale(dynamics, partit, mesh) call exchange_nod(hnode_new, partit) ! Or extend cycles above if (Fer_GM) call exchange_nod(fer_Wvel, partit) !$OMP BARRIER + + !___________________________________________________________________________ + ! compute vertical CFL_z criteria + call compute_CFLz(dynamics, partit, mesh) + + !___________________________________________________________________________ + ! compute implicite explicite splitting of vetical velocity Wvel according + ! to CFL_z criteria + call compute_Wvel_split(dynamics, partit, mesh) + +end subroutine vert_vel_ale +! +! +!_______________________________________________________________________________ +! compute vertical CFL_z criteria and print out warning when critical value over +! stepped +subroutine compute_CFLz(dynamics, partit, mesh) + use g_config,only: dt, flag_warn_cflz + use MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + use o_PARAM + use g_comm_auto + implicit none + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh), intent(inout), target :: mesh + !___________________________________________________________________________ + integer :: node, nz, nzmin, nzmax + real(kind=WP) :: cflmax, c1, c2 + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:) , pointer :: Wvel, CFL_z +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + Wvel => dynamics%w(:,:) + CFL_z => dynamics%cfl_z(:,:) + !___________________________________________________________________________ ! calc vertical CFL criteria for debugging purpose and vertical Wvel splitting !$OMP PARALLEL DO - do n=1, myDim_nod2D+eDim_nod2D - CFL_z(1,n)=0._WP + do node=1, myDim_nod2D+eDim_nod2D + CFL_z(1,node)=0._WP end do !$OMP END PARALLEL DO -!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax) - do n=1, myDim_nod2D+eDim_nod2D - nzmin = ulevels_nod2D(n) - nzmax = nlevels_nod2D(n)-1 + !___________________________________________________________________________ +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax) + do node=1, myDim_nod2D+eDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2D(node)-1 do nz=nzmin,nzmax - c1(1)=abs(Wvel(nz,n) *dt/hnode_new(nz,n)) !c1->c1(1) is made for the sake of reproducibility with the master branch (rounding error) - c2(1)=abs(Wvel(nz+1,n)*dt/hnode_new(nz,n)) !otherwise just add these terms (c(1) & c(2)) to CFL_z, respectively! + c1=abs(Wvel(nz,node) *dt/hnode_new(nz,node)) !c1->c1(1) is made for the sake of reproducibility with the master branch (rounding error) + c2=abs(Wvel(nz+1,node)*dt/hnode_new(nz,node)) !otherwise just add these terms (c(1) & c(2)) to CFL_z, respectively! ! strong condition: ! total volume change induced by the vertical motion ! no matter, upwind or downwind ! - CFL_z(nz, n)=CFL_z(nz,n)+c1(1) - CFL_z(nz+1,n)= c2(1) - end do - end do + CFL_z(nz, node)=CFL_z(nz,node)+c1 + CFL_z(nz+1,node)= c2 + end do ! --> do nz=nzmin,nzmax + end do ! --> do node=1, myDim_nod2D+eDim_nod2D !$OMP END PARALLEL DO -cflmax=0. -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(n) REDUCTION(max:cflmax) + + !___________________________________________________________________________ + cflmax=0. +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(node) REDUCTION(max:cflmax) !$OMP DO - do n=1, myDim_nod2D+eDim_nod2D - cflmax=max(cflmax, maxval(CFL_z(:, n))) - end do + do node=1, myDim_nod2D+eDim_nod2D + cflmax=max(cflmax, maxval(CFL_z(:, node))) + end do ! --> do node=1, myDim_nod2D+eDim_nod2D !$OMP END DO !$OMP END PARALLEL + !___________________________________________________________________________ if (cflmax > 1.0_WP .and. flag_warn_cflz) then -!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax) - do n=1, myDim_nod2D - nzmin = ulevels_nod2D(n) - nzmax = nlevels_nod2D(n)-1 - do nz=nzmin,nzmax - if (abs(CFL_z(nz,n)-cflmax) < 1.e-12 .and. CFL_z(nz,n) > 1.75_WP .and. CFL_z(nz,n)<=2.5_WP ) then +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax) + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2D(node)-1 + do nz=nzmin, nzmax + if (abs(CFL_z(nz,node)-cflmax) < 1.e-12 .and. CFL_z(nz,node) > 1.75_WP .and. CFL_z(nz,node)<=2.5_WP ) then print '(A, A, F4.2, A, I6, A, F7.2,A,F6.2, A, I3,I3)', achar(27)//'[33m'//' --> WARNING CFLz>1.75:'//achar(27)//'[0m',& - 'CFLz_max=',cflmax,',mstep=',mstep,',glon/glat=',geo_coord_nod2D(1,n)/rad,'/',geo_coord_nod2D(2,n)/rad,& + 'CFLz_max=',cflmax,',mstep=',mstep,',glon/glat=',geo_coord_nod2D(1,node)/rad,'/',geo_coord_nod2D(2,node)/rad,& ',nz/nzmin=',nz,nzmin - elseif (abs(CFL_z(nz,n)-cflmax) < 1.e-12 .and. CFL_z(nz,n) > 2.5_WP) then + elseif (abs(CFL_z(nz,node)-cflmax) < 1.e-12 .and. CFL_z(nz,node) > 2.5_WP) then print '(A, A, F4.2, A, I6, A, F7.2,A,F6.2, A, I3,I3)', achar(27)//'[31m'//' --> WARNING CFLz>2.5:'//achar(27)//'[0m',& - 'CFLz_max=',cflmax,',mstep=',mstep,',glon/glat=',geo_coord_nod2D(1,n)/rad,'/',geo_coord_nod2D(2,n)/rad,& + 'CFLz_max=',cflmax,',mstep=',mstep,',glon/glat=',geo_coord_nod2D(1,node)/rad,'/',geo_coord_nod2D(2,node)/rad,& ',nz/nzmin=',nz,nzmin !!PS write(*,*) '***********************************************************' !!PS write(*,*) 'max. CFL_z = ', cflmax, ' mype = ', mype !!PS write(*,*) 'mstep = ', mstep - !!PS write(*,*) 'glon, glat = ', geo_coord_nod2D(:,n)/rad - !!PS write(*,*) '2D node = ', myList_nod2D(n) + !!PS write(*,*) 'glon, glat = ', geo_coord_nod2D(:,node)/rad + !!PS write(*,*) '2D node = ', myList_nod2D(node) !!PS write(*,*) 'nz = ', nz !!PS write(*,*) '***********************************************************' end if - end do - end do + end do ! --> do nz=nzmin,nzmax + end do ! --> do node=1, myDim_nod2D !$OMP END PARALLEL DO - end if - + end if ! --> if (cflmax > 1.0_WP .and. flag_warn_cflz) then +end subroutine compute_CFLz +! +! +!_______________________________________________________________________________ +subroutine compute_Wvel_split(dynamics, partit, mesh) + use MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + use o_PARAM + use g_comm_auto + implicit none + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh), intent(inout), target :: mesh + !___________________________________________________________________________ + integer :: node, nz, nzmin, nzmax + real(kind=WP) :: dd + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, Wvel_i, CFL_z +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + Wvel => dynamics%w( :,:) + Wvel_e => dynamics%w_e( :,:) + Wvel_i => dynamics%w_i( :,:) + CFL_z => dynamics%cfl_z(:,:) + !___________________________________________________________________________ ! Split implicit vertical velocity onto implicit and explicit components using CFL criteria: ! wsplit_maxcfl constrains the allowed explicit w according to the CFL at this place @@ -2553,23 +2692,23 @@ subroutine vert_vel_ale(dynamics, partit, mesh) ! wsplit_maxcfl=0 means w_exp is zero (everything computed implicitly) ! wsplit_maxcfl=inf menas w_impl is zero (everything computed explicitly) ! a guess for optimal choice of wsplit_maxcfl would be 0.95 -!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax, dd) - do n=1, myDim_nod2D+eDim_nod2D - nzmin = ulevels_nod2D(n) - nzmax = nlevels_nod2D(n) - do nz=nzmin,nzmax - Wvel_e(nz,n)=Wvel(nz,n) - Wvel_i(nz,n)=0.0_WP - if (dynamics%use_wsplit .and. (CFL_z(nz, n) > dynamics%wsplit_maxcfl)) then - dd=max((CFL_z(nz, n)-dynamics%wsplit_maxcfl), 0.0_WP)/max(dynamics%wsplit_maxcfl, 1.e-12) - Wvel_e(nz,n)=(1.0_WP/(1.0_WP+dd))*Wvel(nz,n) !explicit part =1. if dd=0. - Wvel_i(nz,n)=(dd /(1.0_WP+dd))*Wvel(nz,n) !implicit part =1. if dd=inf +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax, dd) + do node=1, myDim_nod2D+eDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2D(node) + do nz=nzmin, nzmax + Wvel_e(nz, node)=Wvel(nz, node) + Wvel_i(nz, node)=0.0_WP + if (dynamics%use_wsplit .and. (CFL_z(nz, node) > dynamics%wsplit_maxcfl)) then + dd=max((CFL_z(nz, node)-dynamics%wsplit_maxcfl), 0.0_WP)/max(dynamics%wsplit_maxcfl, 1.e-12) + Wvel_e(nz, node)=(1.0_WP/(1.0_WP+dd))*Wvel(nz, node) !explicit part =1. if dd=0. + Wvel_i(nz, node)=(dd /(1.0_WP+dd))*Wvel(nz, node) !implicit part =1. if dd=inf end if - end do - end do + end do ! --> do nz=nzmin,nzmax + end do ! --> do node=1, myDim_nod2D+eDim_nod2D !$OMP END PARALLEL DO -end subroutine vert_vel_ale - +end subroutine compute_Wvel_split +! ! ! !=============================================================================== @@ -2943,10 +3082,10 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ t0=MPI_Wtime() -! water_flux = 0.0_WP -! heat_flux = 0.0_WP -! stress_surf= 0.0_WP -! stress_node_surf= 0.0_WP + water_flux = 0.0_WP + heat_flux = 0.0_WP + stress_surf= 0.0_WP + stress_node_surf= 0.0_WP !___________________________________________________________________________ ! calculate equation of state, density, pressure and mixed layer depths @@ -3110,7 +3249,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call impl_vert_visc_ale'//achar(27)//'[0m' if(dynamics%use_ivertvisc) then - if (dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_splitexpl_subcycl ) then call impl_vert_visc_ale(dynamics,partit, mesh) else call impl_vert_visc_ale_vtransp(dynamics, partit, mesh) @@ -3233,7 +3372,12 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) ! The main step of ALE procedure --> this is were the magic happens --> here ! is decided how change in hbar is distributed over the vertical layers if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call vert_vel_ale'//achar(27)//'[0m' - call vert_vel_ale(dynamics, partit, mesh) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + call vert_vel_ale(dynamics, partit, mesh) + else + call compute_thickness_zstar(dynamics, partit, mesh) + call compute_vert_vel_transpv(dynamics, partit, mesh) + end if t7=MPI_Wtime() !___________________________________________________________________________ diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index c0b2ef63f..2dcc21991 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1,9 +1,9 @@ ! ! !_______________________________________________________________________________ -module momentum_adv_scalar_4splitexpl_interface +module momentum_adv_scalar_transpv_interface interface - subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) + subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT USE MOD_PARSUP @@ -65,6 +65,16 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(in) , target :: mesh end subroutine + + subroutine compute_vert_vel_transpv(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + end subroutine end interface end module @@ -72,7 +82,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! !_______________________________________________________________________________ ! Transports are used instead of velocities, Urhs, Vrhs are also for transports. -subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) +subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT USE MOD_PARSUP @@ -188,8 +198,8 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) !_______________________________________________________________________ ! index off surface layer in case of cavity !=1 and index of mid depth ! bottom layer - nl1 = nlevels(edelem(1))-1 ul1 = ulevels(edelem(1)) + nl1 = nlevels(edelem(1))-1 !_______________________________________________________________________ !NR --> Natalja Style @@ -200,9 +210,9 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) !_______________________________________________________________________ ! if edelem(2)==0 than edge is boundary edge if(edelem(2)>0) then - nl2 = nlevels(edelem(2))-1 ul2 = ulevels(edelem(2)) - + nl2 = nlevels(edelem(2))-1 + !___________________________________________________________________ !NR --> Natalja Style un2 = 0.0_WP @@ -433,8 +443,8 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) ! divide total nodal momentum advection by scalar area !$OMP DO do node=1,myDim_nod2d - nl1 = nlevels_nod2D(node)-1 ul1 = ulevels_nod2D(node) + nl1 = nlevels_nod2D(node)-1 UVnode_rhs(1, ul1:nl1, node) = UVnode_rhs(1, ul1:nl1, node) * areasvol_inv(ul1:nl1, node) UVnode_rhs(2, ul1:nl1, node) = UVnode_rhs(2, ul1:nl1, node) * areasvol_inv(ul1:nl1, node) end do ! --> do node=1,myDim_nod2d @@ -450,8 +460,8 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) ! convert total nodal advection from vertice --> elements !$OMP DO do elem=1, myDim_elem2D - nl1 = nlevels(elem)-1 ul1 = ulevels(elem) + nl1 = nlevels(elem)-1 UV_rhsAB(1:2, ul1:nl1, elem) = UV_rhsAB(1:2, ul1:nl1, elem) + elem_area(elem)* & ( UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(1, elem)) & + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(2, elem)) & @@ -474,7 +484,7 @@ subroutine momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) !$OMP END DO end if !$OMP END PARALLEL -end subroutine +end subroutine momentum_adv_scalar_transpv ! ! @@ -620,6 +630,21 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) uu(nzmin:nzmax-1)=(UVh(1, nzmin:nzmax-1, elem)+UV_rhs(1, nzmin:nzmax-1, elem))/helem(nzmin:nzmax-1, elem) !! u*=U*/h vv(nzmin:nzmax-1)=(UVh(2, nzmin:nzmax-1, elem)+UV_rhs(2, nzmin:nzmax-1, elem))/helem(nzmin:nzmax-1, elem) + !_______________________________________________________________ + if ( any(uu(nzmin:nzmax-1)/=uu(nzmin:nzmax-1)) .or. & + any(vv(nzmin:nzmax-1)/=vv(nzmin:nzmax-1)) ) then + write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in uu=UVh=UV_rhs' + write(*,*) ' mype =', mype + write(*,*) ' elem =', elem + write(*,*) ' uu(nzmin:nzmax-1)=', uu(nzmin:nzmax-1) + write(*,*) ' vv(nzmin:nzmax-1)=', vv(nzmin:nzmax-1) + write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) + write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) + write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) + write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) + write(*,*) ' helem(nz, elem) =', helem(nz, elem) + end if + !_______________________________________________________________________ ! Operator + rhs ! Regular part of coefficients: @@ -729,6 +754,25 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) UV_rhs(1, nz, elem)=UV_rhs(1, nz, elem)+ur(nz)*helem(nz, elem) !! The rhs is for transport UV_rhs(2, nz, elem)=UV_rhs(2, nz, elem)+vr(nz)*helem(nz, elem) end do + + !_______________________________________________________________ + if ( any(UV_rhs(1, nzmin:nzmax-1, elem)/=UV_rhs(1, nzmin:nzmax-1, elem)) .or. & + any(UV_rhs(2, nzmin:nzmax-1, elem)/=UV_rhs(2, nzmin:nzmax-1, elem))) then + write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in UV_rhs=UVh_rhs+uvr*helem' + write(*,*) ' mype =', mype + write(*,*) ' elem =', elem + write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nzmin:nzmax-1, elem) + write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nzmin:nzmax-1, elem) + write(*,*) ' ur(nzmin:nzmax-1)=', ur(nzmin:nzmax-1) + write(*,*) ' vr(nzmin:nzmax-1)=', vr(nzmin:nzmax-1) + write(*,*) ' helem(nz, elem) =', helem(nzmin:nzmax-1, elem) + write(*,*) ' a(nzmin:nzmax-1) =', a(nzmin:nzmax-1) + write(*,*) ' b(nzmin:nzmax-1) =', b(nzmin:nzmax-1) + write(*,*) ' c(nzmin:nzmax-1) =', c(nzmin:nzmax-1) + write(*,*) ' Av(nzmin:nzmax-1,elem) =', Av(nzmin:nzmax-1, elem) + write(*,*) ' Wvel_i(nzmin:nzmax-1,elem) =',Wvel_i(nzmin:nzmax-1, elnodes) + end if + end do ! --> do elem=1,myDim_elem2D !$OMP END DO !$OMP END PARALLEL @@ -751,7 +795,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) type(t_mesh) , intent(in) , target :: mesh !___________________________________________________________________________ real(kind=WP) :: vert_sum_u, vert_sum_v, Fx, Fy, ab1, ab2, hh - integer :: elem, nz, nzmin, nzmax, elnodes(4) + integer :: elem, nz, nzmin, nzmax, elnodes(3) logical, save :: sfirst !___________________________________________________________________________ ! pointer on necessary derived types @@ -787,8 +831,8 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) nzmax = nlevels(elem)-1 !_______________________________________________________________________ - Fx=g*dt*sum(gradient_sca(1:4,elem)*eta_n(elnodes)) - Fy=g*dt*sum(gradient_sca(5:8,elem)*eta_n(elnodes)) + Fx=g*dt*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + Fy=g*dt*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) !_______________________________________________________________________ ! vertically integrate UV_rhs --> for barotropic equatiobn @@ -863,6 +907,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) real(kind=WP) :: dtBT, BT_inv, hh, f, rx, ry, a, d, c1, c2, ax, ay real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, thetaBT integer :: step, elem, elnodes(3), edge, ednodes(2), edelem(2) + integer :: nzmin1, nzmax1, nzmin2, nzmax2 !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:) , pointer :: eta_n @@ -881,7 +926,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !___________________________________________________________________________ ! Dissipation parameter of FB dissipative method 0.14 is the default value ! from Demange et al. - thetaBT= 0.14_WP +!PS thetaBT= 0.14_WP + thetaBT= 0.5_WP ! BTsteps should be 30 or 40. dtBT = dt/dynamics%splitexpl_BTsteps @@ -904,16 +950,17 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! AAA = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+(m)/M)) ! - h*H^m*grad_H*eta^((n+m)/M) ! - Rbt-->UVBT_rhs ] - hh = -zbar_e_bot(elem)+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth +!PS hh = -zbar_e_bot(elem)+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth + hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth f = mesh%coriolis(elem) - rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(1, elem)) + BT_inv*UVBT_rhs(1, elem) - ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(2, elem)) + BT_inv*UVBT_rhs(2, elem) + rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(2, elem)) + BT_inv*UVBT_rhs(1, elem) + ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(1, elem)) + BT_inv*UVBT_rhs(2, elem) ! Semi-Implicit Coriolis a = dtBT*f*0.5_WP d = 1.0_WP/(1.0_WP+a*a) - ax = d*(rx+a*ry) - ay = d*(-a*rx+ry) + ax = d*( rx + a*ry) + ay = d*(-a*rx + ry) !___________________________________________________________________ ! compute new velocities Ubt^(n+(m+1)/M) at barotropic time step (n+(m+1)/M) ... @@ -971,8 +1018,51 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! advance ssh --> eta^(n+(m+1)/M) = eta^(n+(m)/M) - dt/M * div_H * [...] ! equation (6) in T. Banerjee et al.,Split-Explicite external ! mode solver in FESOM2, - eta_n(ednodes(1))=eta_n(ednodes(1))+(c1+c2)*dtBT/area(1,ednodes(1)) - eta_n(ednodes(2))=eta_n(ednodes(2))-(c1+c2)*dtBT/area(1,ednodes(2)) + eta_n(ednodes(1))=eta_n(ednodes(1)) + (c1+c2)*dtBT/area(1,ednodes(1)) + eta_n(ednodes(2))=eta_n(ednodes(2)) - (c1+c2)*dtBT/area(1,ednodes(2)) + + +!PS if ( (mype==398) .and. ((abs(eta_n(ednodes(1)))>10) .or. (abs(eta_n(ednodes(1)))>10)) ) then + if ((abs(eta_n(ednodes(1)))>100) .or. (abs(eta_n(ednodes(2)))>100)) then + write(*,*) '-------------------------------------------------' + write(*,*) ' --> subroutine compute_BT_step_SE_ale --> found eta_n>100 in barotrop. subcycling' + write(*,*) ' mype = ', mype + write(*,*) ' mstep = ', mstep + write(*,*) ' btstep = ', step + write(*,*) ' edge = ', edge + write(*,*) ' ednodes(1:2) = ', ednodes + write(*,*) ' edelem(1:2) = ', edelem + write(*,*) ' glon,glat = ', geo_coord_nod2D(:,ednodes)/rad + write(*,*) + write(*,*) ' eta_n(ednodes) = ', eta_n(ednodes) + write(*,*) ' c1, c2 = ', c1, c2 + if(edelem(2)>0) then + write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem) + write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem) + write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem) + write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem) + nzmin1 = ulevels(edelem(1)) + nzmax1 = nlevels(edelem(1))-1 + nzmin2 = ulevels(edelem(2)) + nzmax2 = nlevels(edelem(2))-1 + write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) + write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%se_uvh(1:2, nzmin2:nzmax2, edelem(2)) + write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) + write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%uv_rhs(1:2, nzmin2:nzmax2, edelem(2)) + else + write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem(1)) + write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem(1)) + write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem(1)) + write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem(1)) + nzmin1 = ulevels(edelem(1)) + nzmax1 = nlevels(edelem(1))-1 + write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) + write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) + + end if + write(*,*) + end if + end do !_______________________________________________________________________ @@ -1010,13 +1100,11 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) #include "associate_mesh_def.h" #include "associate_part_ass.h" #include "associate_mesh_ass.h" + UV =>dynamics%uv(:,:,:) UVh =>dynamics%se_uvh(:,:,:) UV_rhs =>dynamics%uv_rhs(:,:,:) UVBT_mean =>dynamics%se_uvBT_mean(:,:) UVBT_12 =>dynamics%se_uvBT_12(:,:) - if (mode==2) then - UV =>dynamics%uv(:,:,:) - end if !___________________________________________________________________________ ! @@ -1060,6 +1148,27 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) + + !_______________________________________________________________ + if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & + UV( 2, nz, elem)/=UV( 2, nz, elem)) then + write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=1) --> found Nan in UV after update UV with barotr. term' + write(*,*) ' mype =', mype + write(*,*) ' elem =', elem + write(*,*) ' nz =', nz + write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) + write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) + write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) + write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) + write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) + write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) + write(*,*) ' helem(nz, elem) =', helem(nz, elem) + write(*,*) ' UVBT_mean(1,elem)=', UVBT_mean(1,elem) + write(*,*) ' UVBT_mean(2,elem)=', UVBT_mean(2,elem) + write(*,*) ' ubar, vbar =', ubar, vbar + write(*,*) ' hh_inv =', hh_inv + end if + end do end do call exchange_elem(UVh, partit) ! This exchange can be avoided, but test first. @@ -1087,12 +1196,280 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_12(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection + + !_______________________________________________________________ + if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & + UV( 2, nz, elem)/=UV( 2, nz, elem)) then + write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=2) --> found Nan in UV after update UV with barotr. term' + write(*,*) ' mype =', mype + write(*,*) ' elem =', elem + write(*,*) ' nz =', nz + write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) + write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) + write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) + write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) + write(*,*) ' helem(nz, elem) =', helem(nz, elem) + write(*,*) ' UVBT_12(1,elem) =', UVBT_12(1,elem) + write(*,*) ' UVBT_12(2,elem) =', UVBT_12(2,elem) + write(*,*) ' ubar, vbar =', ubar, vbar + write(*,*) ' hh_inv =', hh_inv + end if end do end do call exchange_elem(UVh, partit) ! call exchange_elem(UV , partit) ! Check if this is needed end if ! --> if (mode==1) then end subroutine update_trim_vel_ale_vtransp - +! +! +!_______________________________________________________________________________ +! Trim U and Uh to be consistent with BT transport +subroutine compute_thickness_zstar(dynamics, partit, mesh) + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_MESH + USE MOD_DYN + use g_comm_auto + !___________________________________________________________________________ + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + integer :: node, elem, nz, nzmin, nzmax, elnodes(3) + real(kind=WP) :: hh_inv + + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:), pointer :: eta_n +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + eta_n => dynamics%eta_n(:) + + !___________________________________________________________________________ + ! leave bottom layer again untouched + do node=1, myDim_nod2D+eDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2D_min(node)-1 + hh_inv=-1.0_WP/zbar(nzmax) + do nz=nzmin, nzmax-1 + hnode_new(nz,node)=(zbar(nz)-zbar(nz+1))*(1.0_WP+hh_inv*eta_n(node)) + end do + end do + + !___________________________________________________________________________ + ! --> update mean layer thinkness at element + do elem=1, myDim_elem2D + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + elnodes=elem2D_nodes(:,elem) + do nz=nzmin, nzmax + helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP + end do + end do + +end subroutine compute_thickness_zstar +! +! +!_______________________________________________________________________________ +! calculate vertical velocity from eq.3 in S. Danilov et al. : FESOM2: from +! finite elements to finite volumes. +! +! dh_k/dt + grad(u*h)_k + (w^t - w^b) + water_flux_k=1 = 0 +! +! w^t = w^b - dh_k/dt - grad(u*h)_k - water_flux=1 +! --> do cumulativ summation from bottom to top +subroutine compute_vert_vel_transpv(dynamics, partit, mesh) + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_MESH + USE MOD_DYN + use o_ARRAYS, only: water_flux + use g_config, only: dt + use g_comm_auto + !___________________________________________________________________________ + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(in) , target :: mesh + integer :: node, elem, nz, nzmin, nzmax, ednodes(2), edelem(2) + real(kind=WP) :: hh_inv + real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) + real(kind=WP) :: e2c1(mesh%nl-1), e2c2(mesh%nl-1) + + + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:,:), pointer :: UVh, fer_UV + real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, Wvel_i, CFL_z, fer_Wvel +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + UVh => dynamics%se_uvh(:,:,:) + Wvel => dynamics%w(:,:) + Wvel_e => dynamics%w_e(:,:) + Wvel_i => dynamics%w_i(:,:) + CFL_z => dynamics%cfl_z(:,:) + if (Fer_GM) then + fer_UV => dynamics%fer_uv(:,:,:) + fer_Wvel=> dynamics%fer_w(:,:) + end if + + !___________________________________________________________________________ + do node=1, myDim_nod2D+eDim_nod2D + Wvel(:, node)=0.0_WP + end do ! --> do node=1, myDim_nod2D+eDim_nod2D + + !___________________________________________________________________________ + do ed=1, myDim_edge2D + ! local indice of nodes that span up edge ed + ednodes=edges(:,ed) + + ! local index of element that contribute to edge + edelem=edge_tri(:,ed) + + ! edge_cross_dxdy(1:2,ed)... dx,dy distance from element centroid edelem(1) to + ! center of edge --> needed to calc flux perpedicular to edge from elem edelem(1) + deltaX1=edge_cross_dxdy(1,ed) + deltaY1=edge_cross_dxdy(2,ed) + + !_______________________________________________________________________ + ! calc div(u_vec*h) for every layer + ! do it with gauss-law: int( div(u_vec)*dV) = int( u_vec * n_vec * dS ) + nzmin = ulevels(edelem(1)) + nzmax = nlevels(edelem(1))-1 + ! we introduced c1 & c2 as arrays here to avoid deadlocks when in OpenMP mode + do nz = nzmax, nzmin, -1 + ! --> h * u_vec * n_vec + ! --> e_vec = (dx,dy), n_vec = (-dy,dx); + ! --> h * u*(-dy) + v*dx + e1c1(nz)=( UVh(2, nz, edelem(1))*deltaX1 - UVh(1, nz, edelem(1))*deltaY1 ) + ! inflow(outflow) "flux" to control volume of node enodes1 + ! is equal to outflow(inflow) "flux" to control volume of node enodes2 + end do ! --> do nz=nzmax,nzmin,-1 + Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e1c1(nzmin:nzmax) + Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e1c1(nzmin:nzmax) + + if (Fer_GM) then + do nz = nzmax, nzmin, -1 + e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1- fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) + end do ! --> do nz=nzmax,nzmin,-1 + fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e1c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e1c2(nzmin:nzmax) + end if + + !_______________________________________________________________________ + ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer + ! for edelem(2) + e2c1 = 0.0_WP + e2c2 = 0.0_WP + if(edelem(2)>0)then + deltaX2=edge_cross_dxdy(3,ed) + deltaY2=edge_cross_dxdy(4,ed) + nzmin = ulevels(edelem(2)) + nzmax = nlevels(edelem(2))-1 + do nz = nzmax, nzmin, -1 + e2c1(nz)=-(UVh(2, nz, edelem(2))*deltaX2 - UVh(1, nz, edelem(2))*deltaY2) + end do ! --> do nz=nzmax,nzmin,-1 + Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e2c1(nzmin:nzmax) + Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e2c1(nzmin:nzmax) + + if (Fer_GM) then + do nz = nzmax, nzmin, -1 + e2c2(nz)=-(fer_UV(2, nz, edelem(2))*deltaX2-fer_UV(1, nz, edelem(2))*deltaY2)*helem(nz, edelem(2)) + end do ! --> do nz=nzmax,nzmin,-1 + fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e2c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e2c2(nzmin:nzmax) + end if + end if + + !_______________________________________________________________________ + if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & + any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then + write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' + write(*,*) ' mype =', mype + write(*,*) ' edge =', ed + write(*,*) ' enodes =', enodes + write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) + write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) + write(*,*) ' e1c1', e1c1(nzmin:nzmax) + write(*,*) ' e1c2', e1c2(nzmin:nzmax) + write(*,*) ' e2c1', e2c1(nzmin:nzmax) + write(*,*) ' e2c2', e2c2(nzmin:nzmax) + end if + end do ! --> do ed=1, myDim_edge2D + + !___________________________________________________________________________ + ! add the contribution from -dh/dt * area + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2d(node)-1 + do nz=nzmax,nzmin,-1 + Wvel(nz, node)=Wvel(nz, node)-(hnode_new(nz, node)-hnode(nz, node))*area(nz, node)/dt + end do ! --> do nz=nzmax,nzmin,-1 + + !_______________________________________________________________________ + if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then + write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' + write(*,*) ' mype =', mype + write(*,*) ' node =', node + write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) + write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) + write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) + end if + end do ! --> do node=1, myDim_nod2D + + !___________________________________________________________________________ + ! Sum up to get W*area + ! cumulative summation of div(u_vec*h) vertically + ! W_k = W_k+1 - div(h_k*u_k) + ! W_k ... vertical flux troughdo node=1, myDim_nod2D + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2d(node)-1 + do nz=nzmax,nzmin,-1 + Wvel(nz, node)=Wvel(nz, node)+Wvel(nz+1, node) + if (Fer_GM) then + fer_Wvel(nz, node)=fer_Wvel(nz, node)+fer_Wvel(nz+1, node) + end if + end do ! --> do nz=nzmax,nzmin,-1 + end do ! --> do node=1, myDim_nod2D + + !___________________________________________________________________________ + ! divide with depth dependent cell area to convert from Vertical flux to + ! physical vertical velocities in units m/s + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2d(node)-1 + do nz=nzmin,nzmax + Wvel(nz, node)=Wvel(nz, node)/area(nz, node) + if (Fer_GM) then + fer_Wvel(nz, node)=fer_Wvel(nz, node)/area(nz, node) + end if + end do ! --> do nz=nzmax,nzmin,-1 + end do ! --> do node=1, myDim_nod2D + + !___________________________________________________________________________ + ! Add surface fresh water flux as upper boundary condition for + ! continutity + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) + end do ! --> do node=1, myDim_nod2D + + !___________________________________________________________________________ + call exchange_nod(Wvel, partit) + if (Fer_GM) call exchange_nod(fer_Wvel, partit) + + !___________________________________________________________________________ + ! compute vertical CFL_z criteria + call compute_CFLz(dynamics, partit, mesh) + + !___________________________________________________________________________ + ! compute implicite explicite splitting of vetical velocity Wvel according + ! to CFL_z criteria + call compute_Wvel_split(dynamics, partit, mesh) + +end subroutine compute_vert_vel_transpv diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index 771dc8303..dc334a1be 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -46,7 +46,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) use g_comm_auto use g_sbf, only: l_mslp use momentum_adv_scalar_interface - use momentum_adv_scalar_4splitexpl_interface + use momentum_adv_scalar_transpv_interface implicit none type(t_ice) , intent(inout), target :: ice type(t_dyn) , intent(inout), target :: dynamics @@ -154,8 +154,8 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) if (use_global_tides) then pre=pre-ssh_gp(elnodes) end if - Fx = sum(gradient_sca(1:3,elem)*pre) - Fy = sum(gradient_sca(4:6,elem)*pre) + Fx = sum(gradient_sca(1:3, elem)*pre) + Fy = sum(gradient_sca(4:6, elem)*pre) !_______________________________________________________________________ ! when ssh split-explicite subcycling method is setted use transport velocities @@ -172,37 +172,40 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) UV_rhsAB(2, nz, elem) =-UV(1, nz, elem)*ff! - mm*UV(1,nz,elem)*UV(2,nz,elem) end do else - UVBT_4AB(1:2,elem) = 0.0_WP do nz=nzmin,nzmax-1 ! add pressure gradient terms UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx-pgf_x(nz, elem))*elem_area(elem)*helem(nz,elem) UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (Fy-pgf_y(nz, elem))*elem_area(elem)*helem(nz,elem) - +!PS UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (pgf_x(nz, elem))*elem_area(elem)*helem(nz,elem) +!PS UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (pgf_y(nz, elem))*elem_area(elem)*helem(nz,elem) +!PS UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx)*elem_area(elem)*helem(nz,elem) +!PS UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (Fy)*elem_area(elem)*helem(nz,elem) + ! add coriolis force, initialise AB2 array of actual timestep ! with coriolis term - UV_rhsAB(1, nz ,elem) = UVh(1, nz, elem)*ff! + mm*UV(1,nz,elem)*UVh(1, nz, elem) - UV_rhsAB(2, nz ,elem) =-UVh(2, nz, elem)*ff! - mm*UV(1,nz,elem)*UVh(2, nz, elem) + UV_rhsAB(1, nz ,elem) = UVh(2, nz, elem)*ff! + mm*UV(1,nz,elem)*UVh(2, nz, elem) + UV_rhsAB(2, nz ,elem) =-UVh(1, nz, elem)*ff! - mm*UV(1,nz,elem)*UVh(1, nz, elem) ! compute barotropic velocity for adams-bashfort time stepping ! UVBT_4AB(1:2, elem)--> actual timestep, ! UVBT_4AB(3:4, elem)--> previous timestep (is setted in ! call compute_BC_BT_SE_vtransp) - UVBT_4AB(1, elem) = UVBT_4AB(1, elem) + UVh(1, nz, elem) ! - UVBT_4AB(2, elem) = UVBT_4AB(2, elem) + UVh(2, nz, elem) ! + UVBT_4AB(1, elem) = UVh(1, nz, elem) ! + UVBT_4AB(2, elem) = UVh(2, nz, elem) ! end do end if !_______________________________________________________________________ if (dynamics%ldiag_ke) then do nz=nzmin,nzmax-1 - dynamics%ke_pre(1,nz,elem)= (Fx-pgf_x(nz,elem))*dt!*elem_area(elem) !not to divide it aterwards (at the end of this subroutine) - dynamics%ke_pre(2,nz,elem)= (Fy-pgf_y(nz,elem))*dt!*elem_area(elem) !but account for DT here + dynamics%ke_pre( 1, nz, elem)= (Fx-pgf_x(nz, elem))*dt!*elem_area(elem) !not to divide it aterwards (at the end of this subroutine) + dynamics%ke_pre( 2, nz, elem)= (Fy-pgf_y(nz, elem))*dt!*elem_area(elem) !but account for DT here - dynamics%ke_cor_AB(1,nz,elem)= UV(2,nz,elem)*ff - dynamics%ke_cor_AB(2,nz,elem)=-UV(1,nz,elem)*ff + dynamics%ke_cor_AB(1, nz, elem)= UV(2, nz, elem)*ff + dynamics%ke_cor_AB(2, nz, elem)=-UV(1, nz, elem)*ff - dynamics%ke_adv_AB(1,nz,elem)= 0.0_WP - dynamics%ke_adv_AB(2,nz,elem)= 0.0_WP + dynamics%ke_adv_AB(1, nz, elem)= 0.0_WP + dynamics%ke_adv_AB(2, nz, elem)= 0.0_WP end do end if @@ -219,7 +222,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) if (.not. dynamics%use_ssh_splitexpl_subcycl) then call momentum_adv_scalar(dynamics, partit, mesh) else - call momentum_adv_scalar_4splitexpl(dynamics, partit, mesh) + call momentum_adv_scalar_transpv(dynamics, partit, mesh) end if end if @@ -232,8 +235,8 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) end if !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) do elem=1, myDim_elem2D - nzmax = nlevels(elem) nzmin = ulevels(elem) + nzmax = nlevels(elem) do nz=nzmin,nzmax-1 UV_rhs(1,nz,elem)=dt*(UV_rhs(1,nz,elem)+UV_rhsAB(1,nz,elem)*ff)/elem_area(elem) UV_rhs(2,nz,elem)=dt*(UV_rhs(2,nz,elem)+UV_rhsAB(2,nz,elem)*ff)/elem_area(elem) @@ -254,8 +257,8 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) if (dynamics%ldiag_ke) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) do elem=1, myDim_elem2D - nzmax = nlevels(elem) nzmin = ulevels(elem) + nzmax = nlevels(elem) do nz=nzmin,nzmax-1 dynamics%ke_adv(:,nz,elem)=dt*(dynamics%ke_adv(:,nz,elem)+dynamics%ke_adv_AB(:,nz,elem)*ff)/elem_area(elem) dynamics%ke_cor(:,nz,elem)=dt*(dynamics%ke_cor(:,nz,elem)+dynamics%ke_cor_AB(:,nz,elem)*ff)/elem_area(elem) diff --git a/src/write_step_info.F90 b/src/write_step_info.F90 index 3ebfb5953..498b08750 100644 --- a/src/write_step_info.F90 +++ b/src/write_step_info.F90 @@ -91,7 +91,7 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) Wvel => dynamics%w(:,:) CFL_z => dynamics%cfl_z(:,:) eta_n => dynamics%eta_n(:) - d_eta => dynamics%d_eta(:) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) d_eta => dynamics%d_eta(:) m_ice => ice%data(2)%values(:) if (mod(istep,outfreq)==0) then @@ -117,23 +117,26 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) do n=1, myDim_nod2D loc_eta = loc_eta + areasvol(ulevels_nod2D(n), n)*eta_n(n) loc_hbar = loc_hbar + areasvol(ulevels_nod2D(n), n)*hbar(n) - loc_deta = loc_deta + areasvol(ulevels_nod2D(n), n)*d_eta(n) loc_dhbar = loc_dhbar + areasvol(ulevels_nod2D(n), n)*(hbar(n)-hbar_old(n)) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + loc_deta = loc_deta + areasvol(ulevels_nod2D(n), n)*d_eta(n) + end if loc_wflux = loc_wflux + areasvol(ulevels_nod2D(n), n)*water_flux(n) end do + if (dynamics%use_ssh_splitexpl_subcycl) loc_deta=loc_dhbar #if !defined(__openmp_reproducible) !$OMP END PARALLEL DO #endif !_______________________________________________________________________ call MPI_AllREDUCE(loc_eta , int_eta , 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) call MPI_AllREDUCE(loc_hbar , int_hbar , 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) - call MPI_AllREDUCE(loc_deta , int_deta , 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) +!PS call MPI_AllREDUCE(loc_deta , int_deta , 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) call MPI_AllREDUCE(loc_dhbar, int_dhbar, 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) call MPI_AllREDUCE(loc_wflux, int_wflux, 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) int_eta = int_eta /ocean_areawithcav int_hbar = int_hbar /ocean_areawithcav - int_deta = int_deta /ocean_areawithcav +!PS int_deta = int_deta /ocean_areawithcav int_dhbar= int_dhbar/ocean_areawithcav int_wflux= int_wflux/ocean_areawithcav !_______________________________________________________________________ @@ -161,7 +164,11 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) call MPI_AllREDUCE(loc , min_vvel , 1, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_FESOM, MPIerr) loc=omp_min_max_sum1(UVnode(2,2,:), 1, myDim_nod2D, 'min', partit) call MPI_AllREDUCE(loc , min_vvel2 , 1, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_FESOM, MPIerr) - loc=omp_min_max_sum1(d_eta, 1, myDim_nod2D, 'min', partit) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + loc=omp_min_max_sum1(d_eta, 1, myDim_nod2D, 'min', partit) + else + loc=omp_min_max_sum1(hbar-hbar_old, 1, myDim_nod2D, 'min', partit) + end if call MPI_AllREDUCE(loc , min_deta , 1, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_FESOM, MPIerr) loc=omp_min_max_sum1(hnode(1,:), 1, myDim_nod2D, 'min', partit) call MPI_AllREDUCE(loc , min_hnode , 1, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_FESOM, MPIerr) @@ -193,7 +200,11 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) call MPI_AllREDUCE(loc , max_vvel , 1, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_FESOM, MPIerr) loc=omp_min_max_sum1(UVnode(2,2,:), 1, myDim_nod2D, 'max', partit) call MPI_AllREDUCE(loc , max_vvel2 , 1, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_FESOM, MPIerr) - loc=omp_min_max_sum1(d_eta, 1, myDim_nod2D, 'max', partit) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + loc=omp_min_max_sum1(d_eta, 1, myDim_nod2D, 'max', partit) + else + loc=omp_min_max_sum1(hbar-hbar_old, 1, myDim_nod2D, 'max', partit) + end if call MPI_AllREDUCE(loc , max_deta , 1, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_FESOM, MPIerr) loc=omp_min_max_sum1(hnode(1, :), 1, myDim_nod2D, 'max', partit) call MPI_AllREDUCE(loc , max_hnode , 1, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_FESOM, MPIerr) @@ -296,6 +307,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) real(kind=WP), dimension(:) , pointer :: u_ice, v_ice real(kind=WP), dimension(:) , pointer :: a_ice, m_ice, m_snow real(kind=WP), dimension(:) , pointer :: a_ice_old, m_ice_old, m_snow_old + real(kind=WP), dimension(:), allocatable, target :: dhbar #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" @@ -303,10 +315,8 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) UV => dynamics%uv(:,:,:) Wvel => dynamics%w(:,:) CFL_z => dynamics%cfl_z(:,:) - ssh_rhs => dynamics%ssh_rhs(:) - ssh_rhs_old => dynamics%ssh_rhs_old(:) + eta_n => dynamics%eta_n(:) - d_eta => dynamics%d_eta(:) u_ice => ice%uice(:) v_ice => ice%vice(:) a_ice => ice%data(1)%values(:) @@ -315,6 +325,15 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) a_ice_old => ice%data(1)%values_old(:) m_ice_old => ice%data(2)%values_old(:) m_snow_old => ice%data(3)%values_old(:) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + d_eta => dynamics%d_eta(:) + ssh_rhs => dynamics%ssh_rhs(:) + ssh_rhs_old => dynamics%ssh_rhs_old(:) + else + allocate(dhbar(myDim_nod2D+eDim_nod2D)) + dhbar = hbar-hbar_old + d_eta => dhbar + end if !___________________________________________________________________________ !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz) @@ -338,7 +357,9 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'zbar_3d_n = ',zbar_3d_n(:,n) write(*,*) 'Z_3d_n = ',Z_3d_n(:,n) write(*,*) - write(*,*) 'ssh_rhs = ',ssh_rhs(n),', ssh_rhs_old = ',ssh_rhs_old(n) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + write(*,*) 'ssh_rhs = ',ssh_rhs(n),', ssh_rhs_old = ',ssh_rhs_old(n) + end if write(*,*) write(*,*) 'hbar = ',hbar(n),', hbar_old = ',hbar_old(n) write(*,*) @@ -389,8 +410,10 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'd_eta(n) = ',d_eta(n) write(*,*) 'hbar = ',hbar(n) write(*,*) 'hbar_old = ',hbar_old(n) - write(*,*) 'ssh_rhs = ',ssh_rhs(n) - write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + write(*,*) 'ssh_rhs = ',ssh_rhs(n) + write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) + end if write(*,*) write(*,*) 'CFL_z(:,n) = ',CFL_z(:,n) write(*,*) @@ -411,6 +434,14 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'hnode(1, n) = ',hnode(1,n) write(*,*) 'hnode(:, n) = ',hnode(:,n) write(*,*) + write(*,*) 'eta_n = ',eta_n(n) + write(*,*) 'd_eta(n) = ',d_eta(n) + write(*,*) 'hbar = ',hbar(n) + write(*,*) 'hbar_old = ',hbar_old(n) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + write(*,*) 'ssh_rhs = ',ssh_rhs(n) + write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) + end if write(*,*) 'glon,glat = ',geo_coord_nod2D(:,n)/rad write(*,*) !$OMP END CRITICAL @@ -445,8 +476,10 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'd_eta(n) = ',d_eta(n) write(*,*) 'hbar = ',hbar(n) write(*,*) 'hbar_old = ',hbar_old(n) - write(*,*) 'ssh_rhs = ',ssh_rhs(n) - write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + write(*,*) 'ssh_rhs = ',ssh_rhs(n) + write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) + end if write(*,*) write(*,*) 'm_ice = ',m_ice(n) write(*,*) 'm_ice_old = ',m_ice_old(n) @@ -493,8 +526,10 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'd_eta(n) = ',d_eta(n) write(*,*) 'hbar = ',hbar(n) write(*,*) 'hbar_old = ',hbar_old(n) - write(*,*) 'ssh_rhs = ',ssh_rhs(n) - write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) + if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + write(*,*) 'ssh_rhs = ',ssh_rhs(n) + write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) + end if write(*,*) write(*,*) 'hnode = ',hnode(:,n) write(*,*) 'hnode_new = ',hnode_new(:,n) From a494f987e9263aa3b5a4a3b62594a95d174d08b5 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 23 Jun 2023 10:17:54 +0200 Subject: [PATCH 16/50] add split-explicite blowup variables --- src/io_blowup.F90 | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/io_blowup.F90 b/src/io_blowup.F90 index b672392c8..ec42e1067 100644 --- a/src/io_blowup.F90 +++ b/src/io_blowup.F90 @@ -111,14 +111,16 @@ subroutine ini_blowup_io(year, ice, dynamics, tracers, partit, mesh) call def_variable(bid, 'ssh_rhs_old', (/nod2D/) , 'RHS for the elevation', '?', dynamics%ssh_rhs_old); else - call def_variable(bid, 'ubt_rhs' , (/elem2D/), 'zonal RHS barotr. transp. equation' , '?', dynamics%se_uvBT_rhs( 1,:)); - call def_variable(bid, 'vbt_rhs' , (/elem2D/), 'merid. RHS barotr. transp. equation', '?', dynamics%se_uvBT_rhs( 2,:)); - call def_variable(bid, 'ubt' , (/elem2D/), 'zonal barotr. transp.' , '?', dynamics%se_uvBT( 1,:)); - call def_variable(bid, 'vbt' , (/elem2D/), 'merid. barotr. transp.' , '?', dynamics%se_uvBT( 2,:)); - call def_variable(bid, 'ubt_theta', (/elem2D/), 'zonal barotr. theta term.' , '?', dynamics%se_uvBT_theta(1,:)); - call def_variable(bid, 'vbt_theta', (/elem2D/), 'merid. barotr. theta term' , '?', dynamics%se_uvBT_theta(2,:)); - call def_variable(bid, 'ubt_mean' , (/elem2D/), 'zonal barotr. mean term.' , '?', dynamics%se_uvBT_mean( 1,:)); - call def_variable(bid, 'vbt_mean' , (/elem2D/), 'merid. barotr. mean term' , '?', dynamics%se_uvBT_mean( 2,:)); + call def_variable(bid, 'ubt_rhs' , (/elem2D/), 'zonal RHS barotr. transp. equation' , '?' , dynamics%se_uvBT_rhs( 1,:)); + call def_variable(bid, 'vbt_rhs' , (/elem2D/), 'merid. RHS barotr. transp. equation', '?' , dynamics%se_uvBT_rhs( 2,:)); + call def_variable(bid, 'ubt' , (/elem2D/), 'zonal barotr. transp.' , '?' , dynamics%se_uvBT( 1,:)); + call def_variable(bid, 'vbt' , (/elem2D/), 'merid. barotr. transp.' , '?' , dynamics%se_uvBT( 2,:)); + call def_variable(bid, 'ubt_theta', (/elem2D/), 'zonal barotr. theta term.' , '?' , dynamics%se_uvBT_theta(1,:)); + call def_variable(bid, 'vbt_theta', (/elem2D/), 'merid. barotr. theta term' , '?' , dynamics%se_uvBT_theta(2,:)); + call def_variable(bid, 'ubt_mean' , (/elem2D/), 'zonal barotr. mean term.' , '?' , dynamics%se_uvBT_mean( 1,:)); + call def_variable(bid, 'vbt_mean' , (/elem2D/), 'merid. barotr. mean term' , '?' , dynamics%se_uvBT_mean( 2,:)); + call def_variable(bid, 'uh' , (/nl-1, elem2D/), 'zonal velocity' , 'm/s', dynamics%se_uvh(1,:,:)); + call def_variable(bid, 'vh' , (/nl-1, elem2D/), 'meridional velocity' , 'm/s', dynamics%se_uvh(2,:,:)); end if !___Define the netCDF variables for 3D fields_______________________________ From 4c7f9a32841fcc33b4bf5291e836875130640e7b Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 7 Jul 2023 15:50:15 +0200 Subject: [PATCH 17/50] fix big in vertical integral of UVBT_4AB, fix bug in SE namelist parameter, switch back on tracer solver + some cosmetics --- src/MOD_DYN.F90 | 3 +- src/oce_ale.F90 | 6 +- src/oce_ale_ssh_splitexpl_subcycl.F90 | 329 +++++++++++++------------- src/oce_ale_vel_rhs.F90 | 11 +- src/oce_dyn.F90 | 8 +- src/oce_setup_step.F90 | 11 +- src/write_step_info.F90 | 2 +- 7 files changed, 194 insertions(+), 176 deletions(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index e95fae68d..b9afadaba 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -120,8 +120,9 @@ MODULE MOD_DYN ! use_ssh_splitexpl_subcycl = .true. --> split explicite subcycling logical :: use_ssh_splitexpl_subcycl = .false. - ! barotropic subcycling time-steps + ! barotropic subcycling time-steps and dissipation parameter integer :: splitexpl_BTsteps = 40 + real(kind=WP) :: splitexpl_BTtheta = 0.14_WP !___________________________________________________________________________ ! energy diagnostic part: will be computed inside the model ("hard integration"): diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index e692b44a4..505f1de62 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -3375,7 +3375,11 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) if ( .not. dynamics%use_ssh_splitexpl_subcycl) then call vert_vel_ale(dynamics, partit, mesh) else - call compute_thickness_zstar(dynamics, partit, mesh) + if (trim(which_ale)=='zstar' ) then + call compute_thickness_zstar(dynamics, partit, mesh) + else + hnode_new = hnode + end if call compute_vert_vel_transpv(dynamics, partit, mesh) end if t7=MPI_Wtime() diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 2dcc21991..1d3b86d04 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -122,9 +122,11 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) !$OMP DO do node=1, myDim_nod2D - nl1 = nlevels_nod2D(node) ul1 = ulevels_nod2D(node) - + nl1 = nlevels_nod2D(node) + wu(1:nl1+1) = 0.0_WP + wv(1:nl1+1) = 0.0_WP + UVnode_rhs(:,:,node) = 0.0_WP !_______________________________________________________________________ ! surface nz=ul1 @@ -144,8 +146,8 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) qu = (UVnode(1, nz , node)-UVnode(1, nz+1, node))/(hnode(nz , node)+hnode(nz+1, node)) qd = (UVnode(1, nz-2, node)-UVnode(1, nz-1, node))/(hnode(nz-2, node)+hnode(nz-1, node)) - uv1 = UVnode(1, nz , node)+(2*qc+qu)*hnode(nz , node)/6.0_WP ! Gradient reconstruction 2(2qc+qu)(h/2)(1/6) - uv2 = UVnode(1, nz-1, node)-(2*qc+qd)*hnode(nz-1, node)/6.0_WP + uv1 = UVnode(1, nz , node)+(2*qc+qu)*hnode(nz , node)/6.0_WP ! Gradient reconstruction 2(2qc+qu)(h/2)(1/6) + uv2 = UVnode(1, nz-1, node)-(2*qc+qd)*hnode(nz-1, node)/6.0_WP uv12 = (Wvel_e(nz, node)+abs(Wvel_e(nz, node)))*uv1+ & (Wvel_e(nz, node)-abs(Wvel_e(nz, node)))*uv2 wu(nz)=0.5_WP*(num_ord*(uv1+uv2)*Wvel_e(nz, node)+(1.0_WP-num_ord)*uv12)*area(nz, node) @@ -154,8 +156,8 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) qu = (UVnode(2, nz , node)-UVnode(2, nz+1, node))/(hnode(nz , node)+hnode(nz+1, node)) qd = (UVnode(2, nz-2, node)-UVnode(2, nz-1, node))/(hnode(nz-2, node)+hnode(nz-1, node)) - uv1 = UVnode(2, nz , node)+(2*qc+qu)*hnode(nz , node)/6.0_WP - uv2 = UVnode(2, nz-1, node)-(2*qc+qd)*hnode(nz-1, node)/6.0_WP + uv1 = UVnode(2, nz , node)+(2*qc+qu)*hnode(nz , node)/6.0_WP + uv2 = UVnode(2, nz-1, node)-(2*qc+qd)*hnode(nz-1, node)/6.0_WP uv12 = (Wvel_e(nz, node)+abs(Wvel_e(nz, node)))*uv1+ & (Wvel_e(nz, node)-abs(Wvel_e(nz, node)))*uv2 wv(nz)=0.5_WP*(num_ord*(uv1+uv2)*Wvel_e(nz, node)+(1.0_WP-num_ord)*uv12)*area(nz, node) @@ -175,9 +177,9 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) !_______________________________________________________________________ ! set to the rhs for transports, not velocities!!! --> No division by h - do nz=1, nl1-1 - UVnode_rhs(1, nz, node)= -(wu(nz)-wu(nz+1)) - UVnode_rhs(2, nz, node)= -(wv(nz)-wv(nz+1)) + do nz=ul1, nl1-1 + UVnode_rhs(1, nz, node)= UVnode_rhs(1, nz, node) - (wu(nz)-wu(nz+1)) + UVnode_rhs(2, nz, node)= UVnode_rhs(2, nz, node) - (wv(nz)-wv(nz+1)) end do end do ! --> do node=1, myDim_nod2D @@ -611,6 +613,7 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) UV_rhs =>dynamics%uv_rhs(:,:,:) Wvel_i =>dynamics%w_i(:,:) UVh =>dynamics%se_uvh(:,:,:) + !___________________________________________________________________________ !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(a, b, c, ur, vr, cp, up, vp, elem, & !$OMP nz, nzmin, nzmax, elnodes, & @@ -622,6 +625,8 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) elnodes= elem2D_nodes(:,elem) nzmin = ulevels(elem) nzmax = nlevels(elem) + uu = 0.0_WP + vv = 0.0_WP !_______________________________________________________________________ ! New velocities: compute u_k^(n+1/2, *) = u_k^(n-1/2) + UV_rhs/helem @@ -631,19 +636,19 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) vv(nzmin:nzmax-1)=(UVh(2, nzmin:nzmax-1, elem)+UV_rhs(2, nzmin:nzmax-1, elem))/helem(nzmin:nzmax-1, elem) !_______________________________________________________________ - if ( any(uu(nzmin:nzmax-1)/=uu(nzmin:nzmax-1)) .or. & - any(vv(nzmin:nzmax-1)/=vv(nzmin:nzmax-1)) ) then - write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in uu=UVh=UV_rhs' - write(*,*) ' mype =', mype - write(*,*) ' elem =', elem - write(*,*) ' uu(nzmin:nzmax-1)=', uu(nzmin:nzmax-1) - write(*,*) ' vv(nzmin:nzmax-1)=', vv(nzmin:nzmax-1) - write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) - write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) - write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) - write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) - write(*,*) ' helem(nz, elem) =', helem(nz, elem) - end if +!PS if ( any(uu(nzmin:nzmax-1)/=uu(nzmin:nzmax-1)) .or. & +!PS any(vv(nzmin:nzmax-1)/=vv(nzmin:nzmax-1)) ) then +!PS write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in uu=UVh=UV_rhs' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' elem =', elem +!PS write(*,*) ' uu(nzmin:nzmax-1)=', uu(nzmin:nzmax-1) +!PS write(*,*) ' vv(nzmin:nzmax-1)=', vv(nzmin:nzmax-1) +!PS write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) +!PS write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) +!PS write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) +!PS write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) +!PS write(*,*) ' helem(nz, elem) =', helem(nz, elem) +!PS end if !_______________________________________________________________________ ! Operator + rhs @@ -756,22 +761,22 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) end do !_______________________________________________________________ - if ( any(UV_rhs(1, nzmin:nzmax-1, elem)/=UV_rhs(1, nzmin:nzmax-1, elem)) .or. & - any(UV_rhs(2, nzmin:nzmax-1, elem)/=UV_rhs(2, nzmin:nzmax-1, elem))) then - write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in UV_rhs=UVh_rhs+uvr*helem' - write(*,*) ' mype =', mype - write(*,*) ' elem =', elem - write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nzmin:nzmax-1, elem) - write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nzmin:nzmax-1, elem) - write(*,*) ' ur(nzmin:nzmax-1)=', ur(nzmin:nzmax-1) - write(*,*) ' vr(nzmin:nzmax-1)=', vr(nzmin:nzmax-1) - write(*,*) ' helem(nz, elem) =', helem(nzmin:nzmax-1, elem) - write(*,*) ' a(nzmin:nzmax-1) =', a(nzmin:nzmax-1) - write(*,*) ' b(nzmin:nzmax-1) =', b(nzmin:nzmax-1) - write(*,*) ' c(nzmin:nzmax-1) =', c(nzmin:nzmax-1) - write(*,*) ' Av(nzmin:nzmax-1,elem) =', Av(nzmin:nzmax-1, elem) - write(*,*) ' Wvel_i(nzmin:nzmax-1,elem) =',Wvel_i(nzmin:nzmax-1, elnodes) - end if +!PS if ( any(UV_rhs(1, nzmin:nzmax-1, elem)/=UV_rhs(1, nzmin:nzmax-1, elem)) .or. & +!PS any(UV_rhs(2, nzmin:nzmax-1, elem)/=UV_rhs(2, nzmin:nzmax-1, elem))) then +!PS write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in UV_rhs=UVh_rhs+uvr*helem' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' elem =', elem +!PS write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nzmin:nzmax-1, elem) +!PS write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nzmin:nzmax-1, elem) +!PS write(*,*) ' ur(nzmin:nzmax-1)=', ur(nzmin:nzmax-1) +!PS write(*,*) ' vr(nzmin:nzmax-1)=', vr(nzmin:nzmax-1) +!PS write(*,*) ' helem(nz, elem) =', helem(nzmin:nzmax-1, elem) +!PS write(*,*) ' a(nzmin:nzmax-1) =', a(nzmin:nzmax-1) +!PS write(*,*) ' b(nzmin:nzmax-1) =', b(nzmin:nzmax-1) +!PS write(*,*) ' c(nzmin:nzmax-1) =', c(nzmin:nzmax-1) +!PS write(*,*) ' Av(nzmin:nzmax-1,elem) =', Av(nzmin:nzmax-1, elem) +!PS write(*,*) ' Wvel_i(nzmin:nzmax-1,elem) =',Wvel_i(nzmin:nzmax-1, elnodes) +!PS end if end do ! --> do elem=1,myDim_elem2D !$OMP END DO @@ -830,10 +835,6 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) nzmin = ulevels(elem) nzmax = nlevels(elem)-1 - !_______________________________________________________________________ - Fx=g*dt*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) - Fy=g*dt*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - !_______________________________________________________________________ ! vertically integrate UV_rhs --> for barotropic equatiobn vert_sum_u=0.0_WP @@ -846,7 +847,9 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) !_______________________________________________________________________ ! Remove the contribution from the elevation will be accounted explicitely ! for in the barotropic equation - hh=sum(helem(nzmin:nzmax, elem)) + Fx = g*dt*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + Fy = g*dt*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) + hh = sum(helem(nzmin:nzmax, elem)) vert_sum_u=vert_sum_u + Fx*hh vert_sum_v=vert_sum_v + Fy*hh @@ -855,6 +858,8 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ! for in the barotropic equation ! UVBT_rhs ... baroclinic forcing term in barotropic equation R_b ! --> d/dt*U_bt + f*e_z x U_bt + g*H* grad(eta) = R_bt + ! --> from AB2 in oce_ale_vel_rhs.F90 --> ab2=-0.5 (from previouse time + ! step) --> ab1=1.5 or 0.0 (first tstep) UVBT_rhs(1, elem)=vert_sum_u - dt*mesh%coriolis(elem)* & (ab1*UVBT_4AB(2,elem)+ab2*UVBT_4AB(4,elem)) ! for AB-interpolated UVBT_rhs(2, elem)=vert_sum_v + dt*mesh%coriolis(elem)* & @@ -867,8 +872,8 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ! --> d/dt*U_bt + f*e_z x U_bt + g*H*grad_H(eta) = R_bt UVBT_4AB(3:4,elem)=UVBT_4AB(1:2,elem) end do -!PS !$OMP END DO -!PS !$OMP END PARALLEL +!$OMP END DO +!$OMP END PARALLEL end subroutine compute_BT_rhs_SE_vtransp ! ! @@ -927,7 +932,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! Dissipation parameter of FB dissipative method 0.14 is the default value ! from Demange et al. !PS thetaBT= 0.14_WP - thetaBT= 0.5_WP +!PS thetaBT= 0.5_WP + thetaBT= dynamics%splitexpl_BTtheta ! BTsteps should be 30 or 40. dtBT = dt/dynamics%splitexpl_BTsteps @@ -1023,45 +1029,45 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !PS if ( (mype==398) .and. ((abs(eta_n(ednodes(1)))>10) .or. (abs(eta_n(ednodes(1)))>10)) ) then - if ((abs(eta_n(ednodes(1)))>100) .or. (abs(eta_n(ednodes(2)))>100)) then - write(*,*) '-------------------------------------------------' - write(*,*) ' --> subroutine compute_BT_step_SE_ale --> found eta_n>100 in barotrop. subcycling' - write(*,*) ' mype = ', mype - write(*,*) ' mstep = ', mstep - write(*,*) ' btstep = ', step - write(*,*) ' edge = ', edge - write(*,*) ' ednodes(1:2) = ', ednodes - write(*,*) ' edelem(1:2) = ', edelem - write(*,*) ' glon,glat = ', geo_coord_nod2D(:,ednodes)/rad - write(*,*) - write(*,*) ' eta_n(ednodes) = ', eta_n(ednodes) - write(*,*) ' c1, c2 = ', c1, c2 - if(edelem(2)>0) then - write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem) - write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem) - write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem) - write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem) - nzmin1 = ulevels(edelem(1)) - nzmax1 = nlevels(edelem(1))-1 - nzmin2 = ulevels(edelem(2)) - nzmax2 = nlevels(edelem(2))-1 - write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) - write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%se_uvh(1:2, nzmin2:nzmax2, edelem(2)) - write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) - write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%uv_rhs(1:2, nzmin2:nzmax2, edelem(2)) - else - write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem(1)) - write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem(1)) - write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem(1)) - write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem(1)) - nzmin1 = ulevels(edelem(1)) - nzmax1 = nlevels(edelem(1))-1 - write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) - write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) - - end if - write(*,*) - end if +!PS if ((abs(eta_n(ednodes(1)))>100) .or. (abs(eta_n(ednodes(2)))>100)) then +!PS write(*,*) '-------------------------------------------------' +!PS write(*,*) ' --> subroutine compute_BT_step_SE_ale --> found eta_n>100 in barotrop. subcycling' +!PS write(*,*) ' mype = ', mype +!PS write(*,*) ' mstep = ', mstep +!PS write(*,*) ' btstep = ', step +!PS write(*,*) ' edge = ', edge +!PS write(*,*) ' ednodes(1:2) = ', ednodes +!PS write(*,*) ' edelem(1:2) = ', edelem +!PS write(*,*) ' glon,glat = ', geo_coord_nod2D(:,ednodes)/rad +!PS write(*,*) +!PS write(*,*) ' eta_n(ednodes) = ', eta_n(ednodes) +!PS write(*,*) ' c1, c2 = ', c1, c2 +!PS if(edelem(2)>0) then +!PS write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem) +!PS write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem) +!PS write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem) +!PS write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem) +!PS nzmin1 = ulevels(edelem(1)) +!PS nzmax1 = nlevels(edelem(1))-1 +!PS nzmin2 = ulevels(edelem(2)) +!PS nzmax2 = nlevels(edelem(2))-1 +!PS write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) +!PS write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%se_uvh(1:2, nzmin2:nzmax2, edelem(2)) +!PS write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) +!PS write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%uv_rhs(1:2, nzmin2:nzmax2, edelem(2)) +!PS else +!PS write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem(1)) +!PS write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem(1)) +!PS write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem(1)) +!PS write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem(1)) +!PS nzmin1 = ulevels(edelem(1)) +!PS nzmax1 = nlevels(edelem(1))-1 +!PS write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) +!PS write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) +!PS +!PS end if +!PS write(*,*) +!PS end if end do @@ -1089,9 +1095,8 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(in) , target :: mesh - integer :: i,elem, elnodes(4), nz, m, nzmin, nzmax - real(kind=WP) :: eta(4), ff - real(kind=WP) :: Fx, Fy, ubar, vbar, hh_inv + integer :: elem, nz, nzmin, nzmax + real(kind=WP) :: ubar, vbar, hh_inv !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:,:,:), pointer :: UVh, UV, UV_rhs @@ -1115,12 +1120,12 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! The trimmed Uh,Vh are consistent with new total height defined by eta_n if (mode==1) then do elem=1, myDim_elem2D + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - - nzmin = ulevels(elem) - nzmax = nlevels(elem)-1 do nz=nzmin, nzmax !_______________________________________________________________ ! Update transport velocities: U^(n+1/2,**) = U^(n+1/2,*) + U_rhs @@ -1129,9 +1134,9 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) !_______________________________________________________________ ! vertically integrate updated transport velocity: sum(k, U_k^(n+1/2,**) ) - ubar=ubar+UVh(1, nz, elem) - vbar=vbar+UVh(2, nz, elem) - hh_inv=hh_inv+helem(nz,elem) + ubar = ubar+UVh(1, nz, elem) + vbar = vbar+UVh(2, nz, elem) + hh_inv= hh_inv+helem(nz,elem) end do !___________________________________________________________________ @@ -1149,30 +1154,31 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) - !_______________________________________________________________ - if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & - UV( 2, nz, elem)/=UV( 2, nz, elem)) then - write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=1) --> found Nan in UV after update UV with barotr. term' - write(*,*) ' mype =', mype - write(*,*) ' elem =', elem - write(*,*) ' nz =', nz - write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) - write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) - write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) - write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) - write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) - write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) - write(*,*) ' helem(nz, elem) =', helem(nz, elem) - write(*,*) ' UVBT_mean(1,elem)=', UVBT_mean(1,elem) - write(*,*) ' UVBT_mean(2,elem)=', UVBT_mean(2,elem) - write(*,*) ' ubar, vbar =', ubar, vbar - write(*,*) ' hh_inv =', hh_inv - end if +!PS !_______________________________________________________________ +!PS if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & +!PS UV( 2, nz, elem)/=UV( 2, nz, elem)) then +!PS write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=1) --> found Nan in UV after update UV with barotr. term' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' elem =', elem +!PS write(*,*) ' nz =', nz +!PS write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) +!PS write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) +!PS write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) +!PS write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) +!PS write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) +!PS write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) +!PS write(*,*) ' helem(nz, elem) =', helem(nz, elem) +!PS write(*,*) ' UVBT_mean(1,elem)=', UVBT_mean(1,elem) +!PS write(*,*) ' UVBT_mean(2,elem)=', UVBT_mean(2,elem) +!PS write(*,*) ' ubar, vbar =', ubar, vbar +!PS write(*,*) ' hh_inv =', hh_inv +!PS end if end do end do call exchange_elem(UVh, partit) ! This exchange can be avoided, but test first. - + call exchange_elem(UV, partit) + !___________________________________________________________________________ ! Trim to make velocity consistent with BT velocity at n+1/2 ! Velocity will be used to advance momentum @@ -1197,23 +1203,24 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection - !_______________________________________________________________ - if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & - UV( 2, nz, elem)/=UV( 2, nz, elem)) then - write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=2) --> found Nan in UV after update UV with barotr. term' - write(*,*) ' mype =', mype - write(*,*) ' elem =', elem - write(*,*) ' nz =', nz - write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) - write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) - write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) - write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) - write(*,*) ' helem(nz, elem) =', helem(nz, elem) - write(*,*) ' UVBT_12(1,elem) =', UVBT_12(1,elem) - write(*,*) ' UVBT_12(2,elem) =', UVBT_12(2,elem) - write(*,*) ' ubar, vbar =', ubar, vbar - write(*,*) ' hh_inv =', hh_inv - end if +!PS !_______________________________________________________________ +!PS if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & +!PS UV( 2, nz, elem)/=UV( 2, nz, elem)) then +!PS write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=2) --> found Nan in UV after update UV with barotr. term' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' elem =', elem +!PS write(*,*) ' nz =', nz +!PS write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) +!PS write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) +!PS write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) +!PS write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) +!PS write(*,*) ' helem(nz, elem) =', helem(nz, elem) +!PS write(*,*) ' UVBT_12(1,elem) =', UVBT_12(1,elem) +!PS write(*,*) ' UVBT_12(2,elem) =', UVBT_12(2,elem) +!PS write(*,*) ' ubar, vbar =', ubar, vbar +!PS write(*,*) ' hh_inv =', hh_inv +!PS end if + end do end do call exchange_elem(UVh, partit) ! @@ -1285,7 +1292,7 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) USE MOD_MESH USE MOD_DYN use o_ARRAYS, only: water_flux - use g_config, only: dt + use g_config, only: dt, which_ale use g_comm_auto !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics @@ -1300,7 +1307,7 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:,:,:), pointer :: UVh, fer_UV - real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, Wvel_i, CFL_z, fer_Wvel + real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, fer_Wvel #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" @@ -1308,8 +1315,6 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) UVh => dynamics%se_uvh(:,:,:) Wvel => dynamics%w(:,:) Wvel_e => dynamics%w_e(:,:) - Wvel_i => dynamics%w_i(:,:) - CFL_z => dynamics%cfl_z(:,:) if (Fer_GM) then fer_UV => dynamics%fer_uv(:,:,:) fer_Wvel=> dynamics%fer_w(:,:) @@ -1352,7 +1357,7 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) if (Fer_GM) then do nz = nzmax, nzmin, -1 - e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1- fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) + e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1 - fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) end do ! --> do nz=nzmax,nzmin,-1 fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e1c2(nzmin:nzmax) fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e1c2(nzmin:nzmax) @@ -1384,19 +1389,19 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) end if !_______________________________________________________________________ - if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & - any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then - write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' - write(*,*) ' mype =', mype - write(*,*) ' edge =', ed - write(*,*) ' enodes =', enodes - write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) - write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) - write(*,*) ' e1c1', e1c1(nzmin:nzmax) - write(*,*) ' e1c2', e1c2(nzmin:nzmax) - write(*,*) ' e2c1', e2c1(nzmin:nzmax) - write(*,*) ' e2c2', e2c2(nzmin:nzmax) - end if +!PS if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & +!PS any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then +!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' edge =', ed +!PS write(*,*) ' enodes =', enodes +!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) +!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) +!PS write(*,*) ' e1c1', e1c1(nzmin:nzmax) +!PS write(*,*) ' e1c2', e1c2(nzmin:nzmax) +!PS write(*,*) ' e2c1', e2c1(nzmin:nzmax) +!PS write(*,*) ' e2c2', e2c2(nzmin:nzmax) +!PS end if end do ! --> do ed=1, myDim_edge2D !___________________________________________________________________________ @@ -1409,14 +1414,14 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) end do ! --> do nz=nzmax,nzmin,-1 !_______________________________________________________________________ - if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then - write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' - write(*,*) ' mype =', mype - write(*,*) ' node =', node - write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) - write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) - write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) - end if +!PS if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then +!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' node =', node +!PS write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) +!PS write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) +!PS write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) +!PS end if end do ! --> do node=1, myDim_nod2D !___________________________________________________________________________ @@ -1452,10 +1457,12 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) !___________________________________________________________________________ ! Add surface fresh water flux as upper boundary condition for ! continutity - do node=1, myDim_nod2D - nzmin = ulevels_nod2D(node) - if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) - end do ! --> do node=1, myDim_nod2D + if (.not. (trim(which_ale)=='linfs' )) then + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) + end do ! --> do node=1, myDim_nod2D + end if !___________________________________________________________________________ call exchange_nod(Wvel, partit) diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index dc334a1be..a2a4cf7db 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -123,7 +123,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) ! p_eta=g*eta_n(elnodes)*(1-theta) !! this place needs update (1-theta)!!! p_eta = g*eta_n(elnodes) - ff = mesh%coriolis(elem)*elem_area(elem) + ff = mesh%coriolis(elem)*elem_area(elem) !mm=metric_factor(elem)*elem_area(elem) !_______________________________________________________________________ @@ -172,14 +172,11 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) UV_rhsAB(2, nz, elem) =-UV(1, nz, elem)*ff! - mm*UV(1,nz,elem)*UV(2,nz,elem) end do else + UVBT_4AB(1:2, elem) = 0.0_WP do nz=nzmin,nzmax-1 ! add pressure gradient terms UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx-pgf_x(nz, elem))*elem_area(elem)*helem(nz,elem) UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (Fy-pgf_y(nz, elem))*elem_area(elem)*helem(nz,elem) -!PS UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (pgf_x(nz, elem))*elem_area(elem)*helem(nz,elem) -!PS UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (pgf_y(nz, elem))*elem_area(elem)*helem(nz,elem) -!PS UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx)*elem_area(elem)*helem(nz,elem) -!PS UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (Fy)*elem_area(elem)*helem(nz,elem) ! add coriolis force, initialise AB2 array of actual timestep ! with coriolis term @@ -190,8 +187,8 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) ! UVBT_4AB(1:2, elem)--> actual timestep, ! UVBT_4AB(3:4, elem)--> previous timestep (is setted in ! call compute_BC_BT_SE_vtransp) - UVBT_4AB(1, elem) = UVh(1, nz, elem) ! - UVBT_4AB(2, elem) = UVh(2, nz, elem) ! + UVBT_4AB(1, elem) = UVBT_4AB(1, elem) + UVh(1, nz, elem) ! + UVBT_4AB(2, elem) = UVBT_4AB(2, elem) + UVh(2, nz, elem) ! end do end if diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index d26cc84a6..92faffb55 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -449,10 +449,10 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) !$OMP DO DO ed=1, myDim_edge2D+eDim_edge2D if(myList_edge2D(ed)>edge2D_in) cycle - el=edge_tri(:,ed) + el = edge_tri(:,ed) nzmin = maxval(ulevels(el)) nzmax = minval(nlevels(el)) - DO nz=nzmin,nzmax-1 + DO nz=nzmin, nzmax-1 update_u(nz)=(UV(1,nz,el(1))-UV(1,nz,el(2))) update_v(nz)=(UV(2,nz,el(1))-UV(2,nz,el(2))) END DO @@ -481,10 +481,10 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) ! compute viscosity on element !$OMP DO DO ed=1,myDim_elem2D - len=sqrt(elem_area(ed)) + len = sqrt(elem_area(ed)) nzmin = ulevels(ed) nzmax = nlevels(ed) - Do nz=nzmin,nzmax-1 + Do nz=nzmin, nzmax-1 ! vi has the sense of harmonic viscosity coef. because of ! division by area in the end u1=U_c(nz,ed)**2+V_c(nz,ed)**2 diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index c8b4bf2ef..b4393674e 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -392,10 +392,14 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) logical :: ldiag_KE =.false. real(kind=WP) :: wsplit_maxcfl logical :: use_ssh_splitexpl_subcycl=.false. + integer :: splitexpl_BTsteps + real(kind=WP) :: splitexpl_BTtheta + namelist /dynamics_visc / opt_visc, visc_gamma0, visc_gamma1, visc_gamma2, & use_ivertvisc, visc_easybsreturn namelist /dynamics_general/ momadv_opt, use_freeslip, use_wsplit, wsplit_maxcfl, & - ldiag_KE, use_ssh_splitexpl_subcycl + ldiag_KE, use_ssh_splitexpl_subcycl, splitexpl_BTsteps, & + splitexpl_BTtheta !___________________________________________________________________________ ! pointer on necessary derived types #include "associate_part_def.h" @@ -431,6 +435,11 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) dynamics%wsplit_maxcfl = wsplit_maxcfl dynamics%ldiag_KE = ldiag_KE dynamics%use_ssh_splitexpl_subcycl = use_ssh_splitexpl_subcycl + if (dynamics%use_ssh_splitexpl_subcycl) then + dynamics%splitexpl_BTsteps = splitexpl_BTsteps + dynamics%splitexpl_BTtheta = splitexpl_BTtheta + end if + !___________________________________________________________________________ ! define local vertice & elem array size elem_size=myDim_elem2D+eDim_elem2D diff --git a/src/write_step_info.F90 b/src/write_step_info.F90 index 498b08750..33517654e 100644 --- a/src/write_step_info.F90 +++ b/src/write_step_info.F90 @@ -340,7 +340,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) do n=1, myDim_nod2d !___________________________________________________________________ ! check ssh - if ( ((eta_n(n) /= eta_n(n)) .or. eta_n(n)<-50.0 .or. eta_n(n)>50.0 .or. (d_eta(n) /= d_eta(n)) ) ) then + if ( ((eta_n(n) /= eta_n(n)) .or. eta_n(n)<-10.0 .or. eta_n(n)>10.0 .or. (d_eta(n) /= d_eta(n)) ) ) then !$OMP CRITICAL found_blowup_loc=1 write(*,*) '___CHECK FOR BLOW UP___________ --> mstep=',istep From cdb4c4ee9ace2e1138a36f198fd1f439797baa00 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 7 Jul 2023 17:06:33 +0200 Subject: [PATCH 18/50] fix small bug in vertical loop --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 149 +++++++++++++++++--------- 1 file changed, 101 insertions(+), 48 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 1d3b86d04..0a83a6a32 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -99,7 +99,7 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) real(kind=WP) :: uv12, uv1, uv2, qc, qu, qd, num_ord=0.95_WP integer :: ednodes(2), edelem(2) real(kind=WP) :: wu(mesh%nl), wv(mesh%nl), un1(mesh%nl), un2(mesh%nl) - +!PS real(kind=WP) :: un, uu, vv, x1, x2, y1, y2 !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:,:,:), pointer :: UV, UV_rhsAB, UVnode_rhs, UVnode, UVh @@ -340,7 +340,7 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) call omp_set_lock(partit%plock(ednodes(2))) #endif ! bulk domain where only edelem(1) exist - do nz=ul1 , nl1-1 + do nz=ul1 , nl1 UVnode_rhs(1, nz, ednodes(2)) = UVnode_rhs(1, nz, ednodes(2)) - un1(nz)*UV(1, nz, edelem(1)) UVnode_rhs(2, nz, ednodes(2)) = UVnode_rhs(2, nz, ednodes(2)) - un1(nz)*UV(2, nz, edelem(1)) end do @@ -357,6 +357,43 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) end do ! --> do ed=1, myDim_edge2D !$OMP END DO + + + + + + + + +!PS !___________________________________________________________________________ +!PS ! 2nd. compute horizontal advection component: u*du/dx, u*dv/dx & v*du/dy, v*dv/dy +!PS ! loop over triangle edges +!PS !$OMP DO +!PS do ed=1, myDim_edge2D +!PS ! local indice of nodes that span up edge ed +!PS ednodes = edges(:,ed) +!PS +!PS ! local index of element that contribute to edge +!PS edelem = edge_tri(1:2,ed) +!PS +!PS !_______________________________________________________________________ +!PS ! index off surface layer in case of cavity !=1 and index of mid depth +!PS ! bottom layer +!PS ul1 = ulevels(edelem(1)) +!PS nl1 = nlevels(edelem(1))-1 +!PS x1 = edge_cross_dxdy(1,ed) +!PS y1 = edge_cross_dxdy(2,ed) +!PS ul2 = 0 +!PS nl2 = nl +!PS +!PS !_______________________________________________________________________ +!PS if (edelem(2) > 0) then +!PS ul2 = ulevels(edelem(2)) +!PS nl2 = nlevels(edelem(2))-1 +!PS x2 = edge_cross_dxdy(3,ed) +!PS y2 = edge_cross_dxdy(4,ed) +!PS end if +!PS !PS !_______________________________________________________________________ !PS ! nl12 ... minimum number of layers -1 between element edelem(1) & edelem(2) that !PS ! contribute to edge ed @@ -371,7 +408,7 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) !PS ! (A1) goes only into this loop when the edge has only facing element !PS ! edelem(1) --> so the edge is a boundary edge --> this is for ocean !PS ! surface in case of cavity -!PS do nz=nu1, nu12-1 +!PS do nz=ul1, ul12-1 !PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) !PS uu=un*UV(1, nz, edelem(1)) ! the momentum to be carried depends on velocities !PS vv=un*UV(2, nz, edelem(1)) @@ -379,38 +416,7 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) !PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu !PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv !PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=nu1, nu12-1 -!PS -!PS !_______________________________________________________________________ -!PS ! (A2) goes only into this loop when the edge has a facing elemenmt -!PS ! edelem(2) --> so the edge is a boundary edge --> this is for ocean -!PS ! surface in case of cavity -!PS if (nu2 > 0) then -!PS do nz=nu2, nu12-1 -!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) -!PS uu=un*UV(1, nz, edelem(2)) -!PS vv=un*UV(2, nz, edelem(2)) -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=nu2, nu12-1 -!PS end if -!PS -!PS !_______________________________________________________________________ -!PS ! (B) Both segments -!PS ! loop over depth layers from shared upper layer index nu12 to shared -!PS ! lower layer index nl12 -!PS do nz=nu12, nl12 -!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) & -!PS -(UVh(2, nz, edelem(2))*x2 - UVh(1, nz, edelem(2))*y2) -!PS uu=un*(UV(1, nz, edelem(1)) + UV(1, nz, edelem(2)))*0.5_WP! the momentum to be carried depends on velocities -!PS vv=un*(UV(2, nz, edelem(1)) + UV(2, nz, edelem(2)))*0.5_WP -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=nu12, nl12 +!PS end do ! --> do nz=ul1, ul12-1 !PS !PS !_______________________________________________________________________ !PS ! (C1) remaining segments from the shared lower lyer index nl12 to bottom @@ -425,18 +431,65 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) !PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv !PS end do ! --> do nz=nl12+1, nl1 !PS -!PS !_______________________________________________________________________ -!PS ! (C2) remaining segments from the shared lower lyer index nl12 to bottom -!PS ! of element edelem(1) -!PS do nz=nl12+1, nl2 -!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) -!PS uu=un*UV(1, nz, edelem(2)) -!PS vv=un*UV(2, nz, edelem(2)) -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=nl12+1, nl2 +!PS if (edelem(2) > 0) then +!PS !_______________________________________________________________________ +!PS ! (A2) goes only into this loop when the edge has a facing elemenmt +!PS ! edelem(2) --> so the edge is a boundary edge --> this is for ocean +!PS ! surface in case of cavity +!PS do nz=ul2, ul12-1 +!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) +!PS uu=un*UV(1, nz, edelem(2)) +!PS vv=un*UV(2, nz, edelem(2)) +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=ul2, ul12-1 +!PS +!PS !_______________________________________________________________________ +!PS ! (B) Both segments +!PS ! loop over depth layers from shared upper layer index ul12 to shared +!PS ! lower layer index nl12 +!PS do nz=ul12, nl12 +!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) & +!PS -(UVh(2, nz, edelem(2))*x2 - UVh(1, nz, edelem(2))*y2) +!PS uu=un*(UV(1, nz, edelem(1)) + UV(1, nz, edelem(2)))*0.5_WP! the momentum to be carried depends on velocities +!PS vv=un*(UV(2, nz, edelem(1)) + UV(2, nz, edelem(2)))*0.5_WP +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=ul12, nl12 +!PS +!PS !_______________________________________________________________________ +!PS ! (C2) remaining segments from the shared lower lyer index nl12 to bottom +!PS ! of element edelem(1) +!PS do nz=nl12+1, nl2 +!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) +!PS uu=un*UV(1, nz, edelem(2)) +!PS vv=un*UV(2, nz, edelem(2)) +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=nl12+1, nl2 +!PS else +!PS +!PS !_______________________________________________________________________ +!PS ! (B) Both segments +!PS ! loop over depth layers from shared upper layer index ul12 to shared +!PS ! lower layer index nl12 +!PS do nz=ul12, nl12 +!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) +!PS uu=un*UV(1, nz, edelem(1))! the momentum to be carried depends on velocities +!PS vv=un*UV(2, nz, edelem(1)) +!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu +!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu +!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv +!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv +!PS end do ! --> do nz=ul12, nl12 +!PS end if +!PS !PS end do ! --> do ed=1, myDim_edge2D !PS !$OMP END DO @@ -476,8 +529,8 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) if (dynamics%ldiag_ke) then !we repeat the computation here and there are multiple ways to speed it up !$OMP DO do elem=1, myDim_elem2D - nl1 = nlevels(elem)-1 ul1 = ulevels(elem) + nl1 = nlevels(elem)-1 dynamics%ke_adv_AB(1:2, ul1:nl1, elem) = dynamics%ke_adv_AB(1:2, ul1:nl1, elem) + elem_area(elem)* & ( UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(1, elem)) & + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(2, elem)) & From ae5bf22e3d5cb00ddd4a3f5bef28c121ca3f4558 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 12 Jul 2023 12:03:17 +0200 Subject: [PATCH 19/50] fix declaration bug --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 4 ++-- src/oce_ale_vel_rhs.F90 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 0a83a6a32..97cc5b603 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1351,8 +1351,8 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(in) , target :: mesh - integer :: node, elem, nz, nzmin, nzmax, ednodes(2), edelem(2) - real(kind=WP) :: hh_inv + integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) + real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) real(kind=WP) :: e2c1(mesh%nl-1), e2c2(mesh%nl-1) diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index a2a4cf7db..261d82a4f 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -173,7 +173,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) end do else UVBT_4AB(1:2, elem) = 0.0_WP - do nz=nzmin,nzmax-1 + do nz=nzmin, nzmax-1 ! add pressure gradient terms UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx-pgf_x(nz, elem))*elem_area(elem)*helem(nz,elem) UV_rhs( 2, nz, elem) = UV_rhs(2, nz, elem) + (Fy-pgf_y(nz, elem))*elem_area(elem)*helem(nz,elem) From 042a7be21f3228b1fddf327d3ef2008af941b3b3 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 12 Jul 2023 12:35:52 +0200 Subject: [PATCH 20/50] fix interfasce problem --- src/oce_ale.F90 | 31 ++++++++++++++++++++++++++- src/oce_ale_ssh_splitexpl_subcycl.F90 | 1 + 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 505f1de62..cc66b8698 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -1,3 +1,27 @@ +module oce_wsplit_interface + interface + subroutine compute_CFLz(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + end subroutine + + subroutine compute_Wvel_split(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + end subroutine + end interface +end module + module oce_ale_interfaces interface subroutine init_bottom_elem_thickness(partit, mesh) @@ -146,6 +170,9 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) end subroutine end interface end module + + + ! CONTENT: ! ------------ ! subroutine ale_init @@ -1983,6 +2010,7 @@ subroutine vert_vel_ale(dynamics, partit, mesh) use g_comm_auto use io_RESTART !!PS use g_forcing_arrays !!PS +!PS use oce_wsplit_interface implicit none type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit @@ -2569,7 +2597,7 @@ end subroutine vert_vel_ale ! compute vertical CFL_z criteria and print out warning when critical value over ! stepped subroutine compute_CFLz(dynamics, partit, mesh) - use g_config,only: dt, flag_warn_cflz + use g_config, only: dt, flag_warn_cflz use MOD_MESH USE MOD_PARTIT USE MOD_PARSUP @@ -3049,6 +3077,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) use g_cvmix_kpp use g_cvmix_tidal use Toy_Channel_Soufflet + use oce_wsplit_interface use oce_ale_interfaces use pressure_bv_interface use pressure_force_4_linfs_interface diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 97cc5b603..914aa9654 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1347,6 +1347,7 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) use o_ARRAYS, only: water_flux use g_config, only: dt, which_ale use g_comm_auto +!PS use oce_wsplit_interface !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit From edba3665d778e080e303f6cea975d4945088d2ce Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 12 Jul 2023 13:08:41 +0200 Subject: [PATCH 21/50] still try to fix this interface --- src/oce_ale.F90 | 56 ++++++++++++++------------- src/oce_ale_ssh_splitexpl_subcycl.F90 | 23 +++++------ 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index cc66b8698..53f3e22a3 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -1,27 +1,3 @@ -module oce_wsplit_interface - interface - subroutine compute_CFLz(dynamics, partit, mesh) - USE MOD_MESH - USE MOD_PARTIT - USE MOD_PARSUP - USE MOD_DYN - type(t_dyn) , intent(inout), target :: dynamics - type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(inout), target :: mesh - end subroutine - - subroutine compute_Wvel_split(dynamics, partit, mesh) - USE MOD_MESH - USE MOD_PARTIT - USE MOD_PARSUP - USE MOD_DYN - type(t_dyn) , intent(inout), target :: dynamics - type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(inout), target :: mesh - end subroutine - end interface -end module - module oce_ale_interfaces interface subroutine init_bottom_elem_thickness(partit, mesh) @@ -171,7 +147,33 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) end interface end module +module compute_CFLz_interface + interface + subroutine compute_CFLz(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + end subroutine + end interface +end module +module compute_Wvel_split_interface + interface + subroutine compute_Wvel_split(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + end subroutine + end interface +end module ! CONTENT: ! ------------ @@ -2010,7 +2012,8 @@ subroutine vert_vel_ale(dynamics, partit, mesh) use g_comm_auto use io_RESTART !!PS use g_forcing_arrays !!PS -!PS use oce_wsplit_interface +!PS use compute_Wvel_split_interaface +!PS use compute_CFLz_interface implicit none type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit @@ -3077,7 +3080,8 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) use g_cvmix_kpp use g_cvmix_tidal use Toy_Channel_Soufflet - use oce_wsplit_interface + use compute_Wvel_split_interface + use compute_CFLz_interface use oce_ale_interfaces use pressure_bv_interface use pressure_force_4_linfs_interface diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 914aa9654..34cced534 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -26,7 +26,7 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) USE MOD_DYN type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout), target :: mesh end subroutine end interface end module @@ -42,7 +42,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) USE MOD_DYN type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout), target :: mesh end subroutine subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) @@ -52,7 +52,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) USE MOD_DYN type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout), target :: mesh end subroutine subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) @@ -63,7 +63,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) integer , intent(in) :: mode type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout), target :: mesh end subroutine subroutine compute_vert_vel_transpv(dynamics, partit, mesh) @@ -73,7 +73,7 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) USE MOD_DYN type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout), target :: mesh end subroutine end interface end module @@ -850,7 +850,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) IMPLICIT NONE type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout) , target :: mesh !___________________________________________________________________________ real(kind=WP) :: vert_sum_u, vert_sum_v, Fx, Fy, ab1, ab2, hh integer :: elem, nz, nzmin, nzmax, elnodes(3) @@ -960,7 +960,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout) , target :: mesh !___________________________________________________________________________ real(kind=WP) :: dtBT, BT_inv, hh, f, rx, ry, a, d, c1, c2, ax, ay real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, thetaBT @@ -1147,7 +1147,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) integer , intent(in) :: mode type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout) , target :: mesh integer :: elem, nz, nzmin, nzmax real(kind=WP) :: ubar, vbar, hh_inv !___________________________________________________________________________ @@ -1293,7 +1293,7 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout) , target :: mesh integer :: node, elem, nz, nzmin, nzmax, elnodes(3) real(kind=WP) :: hh_inv @@ -1347,11 +1347,12 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) use o_ARRAYS, only: water_flux use g_config, only: dt, which_ale use g_comm_auto -!PS use oce_wsplit_interface +!PS use compute_Wvel_split_interface +!PS use compute_CFLz_interface !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(in) , target :: mesh + type(t_mesh) , intent(inout) , target :: mesh integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) From 748f801e6f606000a5ac348418381240221a1822 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 12 Jul 2023 14:06:45 +0200 Subject: [PATCH 22/50] still try to fix this interface --- src/oce_ale.F90 | 67 ++++++++++++++------------- src/oce_ale_ssh_splitexpl_subcycl.F90 | 13 ++++-- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 53f3e22a3..9d26d3c13 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -1,3 +1,31 @@ + +module compute_CFLz_interface + interface + subroutine compute_CFLz(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + end subroutine + end interface +end module +module compute_Wvel_split_interface + interface + subroutine compute_Wvel_split(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + end subroutine + end interface +end module + module oce_ale_interfaces interface subroutine init_bottom_elem_thickness(partit, mesh) @@ -147,33 +175,6 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) end interface end module -module compute_CFLz_interface - interface - subroutine compute_CFLz(dynamics, partit, mesh) - USE MOD_MESH - USE MOD_PARTIT - USE MOD_PARSUP - USE MOD_DYN - type(t_dyn) , intent(inout), target :: dynamics - type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(inout), target :: mesh - end subroutine - end interface -end module - -module compute_Wvel_split_interface - interface - subroutine compute_Wvel_split(dynamics, partit, mesh) - USE MOD_MESH - USE MOD_PARTIT - USE MOD_PARSUP - USE MOD_DYN - type(t_dyn) , intent(inout), target :: dynamics - type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(inout), target :: mesh - end subroutine - end interface -end module ! CONTENT: ! ------------ @@ -2012,8 +2013,8 @@ subroutine vert_vel_ale(dynamics, partit, mesh) use g_comm_auto use io_RESTART !!PS use g_forcing_arrays !!PS -!PS use compute_Wvel_split_interaface -!PS use compute_CFLz_interface + use compute_Wvel_split_interface + use compute_CFLz_interface implicit none type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit @@ -3080,9 +3081,11 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) use g_cvmix_kpp use g_cvmix_tidal use Toy_Channel_Soufflet - use compute_Wvel_split_interface - use compute_CFLz_interface +!PS use compute_Wvel_split_interface +!PS use compute_CFLz_interface use oce_ale_interfaces + use compute_vert_vel_transpv_interface + use compute_ssh_split_explicit_interface use pressure_bv_interface use pressure_force_4_linfs_interface use pressure_force_4_zxxxx_interface @@ -3092,7 +3095,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) use check_blowup_interface use fer_solve_interface use impl_vert_visc_ale_vtransp_interface - use compute_ssh_split_explicit_interface + IMPLICIT NONE integer , intent(in) :: n type(t_dyn) , intent(inout), target :: dynamics diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 34cced534..b28dfd8c5 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -66,6 +66,11 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) type(t_mesh) , intent(inout), target :: mesh end subroutine + end interface +end module + +module compute_vert_vel_transpv_interface + interface subroutine compute_vert_vel_transpv(dynamics, partit, mesh) USE MOD_MESH USE MOD_PARTIT @@ -1290,6 +1295,7 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) USE MOD_MESH USE MOD_DYN use g_comm_auto + implicit none !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit @@ -1347,12 +1353,13 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) use o_ARRAYS, only: water_flux use g_config, only: dt, which_ale use g_comm_auto -!PS use compute_Wvel_split_interface -!PS use compute_CFLz_interface + use compute_Wvel_split_interface + use compute_CFLz_interface + implicit none !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(inout) , target :: mesh + type(t_mesh) , intent(inout), target :: mesh integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) From ec17f242201e2c744b56a4de338cf8802d18177f Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 12 Jul 2023 14:23:46 +0200 Subject: [PATCH 23/50] still try to fix this interface --- src/oce_ale.F90 | 221 +++++++++++++ src/oce_ale_ssh_splitexpl_subcycl.F90 | 432 +++++++++++++------------- 2 files changed, 437 insertions(+), 216 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 9d26d3c13..926c82237 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -12,6 +12,7 @@ subroutine compute_CFLz(dynamics, partit, mesh) end subroutine end interface end module + module compute_Wvel_split_interface interface subroutine compute_Wvel_split(dynamics, partit, mesh) @@ -26,6 +27,20 @@ subroutine compute_Wvel_split(dynamics, partit, mesh) end interface end module +module compute_vert_vel_transpv_interface + interface + subroutine compute_vert_vel_transpv(dynamics, partit, mesh) + USE MOD_MESH + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_DYN + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + end subroutine + end interface +end module + module oce_ale_interfaces interface subroutine init_bottom_elem_thickness(partit, mesh) @@ -2595,6 +2610,212 @@ subroutine vert_vel_ale(dynamics, partit, mesh) call compute_Wvel_split(dynamics, partit, mesh) end subroutine vert_vel_ale + +! +! +!_______________________________________________________________________________ +! calculate vertical velocity from eq.3 in S. Danilov et al. : FESOM2: from +! finite elements to finite volumes. +! +! dh_k/dt + grad(u*h)_k + (w^t - w^b) + water_flux_k=1 = 0 +! +! w^t = w^b - dh_k/dt - grad(u*h)_k - water_flux=1 +! --> do cumulativ summation from bottom to top +subroutine compute_vert_vel_transpv(dynamics, partit, mesh) + USE MOD_PARTIT + USE MOD_PARSUP + USE MOD_MESH + USE MOD_DYN + use o_ARRAYS, only: water_flux + use g_config, only: dt, which_ale + use g_comm_auto + use compute_Wvel_split_interface + use compute_CFLz_interface + implicit none + !___________________________________________________________________________ + type(t_dyn) , intent(inout), target :: dynamics + type(t_partit), intent(inout), target :: partit + type(t_mesh) , intent(inout), target :: mesh + integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) + real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 + real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) + real(kind=WP) :: e2c1(mesh%nl-1), e2c2(mesh%nl-1) + + + !___________________________________________________________________________ + ! pointer on necessary derived types + real(kind=WP), dimension(:,:,:), pointer :: UVh, fer_UV + real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, fer_Wvel +#include "associate_part_def.h" +#include "associate_mesh_def.h" +#include "associate_part_ass.h" +#include "associate_mesh_ass.h" + UVh => dynamics%se_uvh(:,:,:) + Wvel => dynamics%w(:,:) + Wvel_e => dynamics%w_e(:,:) + if (Fer_GM) then + fer_UV => dynamics%fer_uv(:,:,:) + fer_Wvel=> dynamics%fer_w(:,:) + end if + + !___________________________________________________________________________ + do node=1, myDim_nod2D+eDim_nod2D + Wvel(:, node)=0.0_WP + end do ! --> do node=1, myDim_nod2D+eDim_nod2D + + !___________________________________________________________________________ + do ed=1, myDim_edge2D + ! local indice of nodes that span up edge ed + ednodes=edges(:,ed) + + ! local index of element that contribute to edge + edelem=edge_tri(:,ed) + + ! edge_cross_dxdy(1:2,ed)... dx,dy distance from element centroid edelem(1) to + ! center of edge --> needed to calc flux perpedicular to edge from elem edelem(1) + deltaX1=edge_cross_dxdy(1,ed) + deltaY1=edge_cross_dxdy(2,ed) + + !_______________________________________________________________________ + ! calc div(u_vec*h) for every layer + ! do it with gauss-law: int( div(u_vec)*dV) = int( u_vec * n_vec * dS ) + nzmin = ulevels(edelem(1)) + nzmax = nlevels(edelem(1))-1 + ! we introduced c1 & c2 as arrays here to avoid deadlocks when in OpenMP mode + do nz = nzmax, nzmin, -1 + ! --> h * u_vec * n_vec + ! --> e_vec = (dx,dy), n_vec = (-dy,dx); + ! --> h * u*(-dy) + v*dx + e1c1(nz)=( UVh(2, nz, edelem(1))*deltaX1 - UVh(1, nz, edelem(1))*deltaY1 ) + ! inflow(outflow) "flux" to control volume of node enodes1 + ! is equal to outflow(inflow) "flux" to control volume of node enodes2 + end do ! --> do nz=nzmax,nzmin,-1 + Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e1c1(nzmin:nzmax) + Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e1c1(nzmin:nzmax) + + if (Fer_GM) then + do nz = nzmax, nzmin, -1 + e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1 - fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) + end do ! --> do nz=nzmax,nzmin,-1 + fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e1c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e1c2(nzmin:nzmax) + end if + + !_______________________________________________________________________ + ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer + ! for edelem(2) + e2c1 = 0.0_WP + e2c2 = 0.0_WP + if(edelem(2)>0)then + deltaX2=edge_cross_dxdy(3,ed) + deltaY2=edge_cross_dxdy(4,ed) + nzmin = ulevels(edelem(2)) + nzmax = nlevels(edelem(2))-1 + do nz = nzmax, nzmin, -1 + e2c1(nz)=-(UVh(2, nz, edelem(2))*deltaX2 - UVh(1, nz, edelem(2))*deltaY2) + end do ! --> do nz=nzmax,nzmin,-1 + Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e2c1(nzmin:nzmax) + Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e2c1(nzmin:nzmax) + + if (Fer_GM) then + do nz = nzmax, nzmin, -1 + e2c2(nz)=-(fer_UV(2, nz, edelem(2))*deltaX2-fer_UV(1, nz, edelem(2))*deltaY2)*helem(nz, edelem(2)) + end do ! --> do nz=nzmax,nzmin,-1 + fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e2c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e2c2(nzmin:nzmax) + end if + end if + + !_______________________________________________________________________ +!PS if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & +!PS any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then +!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' edge =', ed +!PS write(*,*) ' enodes =', enodes +!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) +!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) +!PS write(*,*) ' e1c1', e1c1(nzmin:nzmax) +!PS write(*,*) ' e1c2', e1c2(nzmin:nzmax) +!PS write(*,*) ' e2c1', e2c1(nzmin:nzmax) +!PS write(*,*) ' e2c2', e2c2(nzmin:nzmax) +!PS end if + end do ! --> do ed=1, myDim_edge2D + + !___________________________________________________________________________ + ! add the contribution from -dh/dt * area + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2d(node)-1 + do nz=nzmax,nzmin,-1 + Wvel(nz, node)=Wvel(nz, node)-(hnode_new(nz, node)-hnode(nz, node))*area(nz, node)/dt + end do ! --> do nz=nzmax,nzmin,-1 + + !_______________________________________________________________________ +!PS if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then +!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' +!PS write(*,*) ' mype =', mype +!PS write(*,*) ' node =', node +!PS write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) +!PS write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) +!PS write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) +!PS end if + end do ! --> do node=1, myDim_nod2D + + !___________________________________________________________________________ + ! Sum up to get W*area + ! cumulative summation of div(u_vec*h) vertically + ! W_k = W_k+1 - div(h_k*u_k) + ! W_k ... vertical flux troughdo node=1, myDim_nod2D + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2d(node)-1 + do nz=nzmax,nzmin,-1 + Wvel(nz, node)=Wvel(nz, node)+Wvel(nz+1, node) + if (Fer_GM) then + fer_Wvel(nz, node)=fer_Wvel(nz, node)+fer_Wvel(nz+1, node) + end if + end do ! --> do nz=nzmax,nzmin,-1 + end do ! --> do node=1, myDim_nod2D + + !___________________________________________________________________________ + ! divide with depth dependent cell area to convert from Vertical flux to + ! physical vertical velocities in units m/s + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + nzmax = nlevels_nod2d(node)-1 + do nz=nzmin,nzmax + Wvel(nz, node)=Wvel(nz, node)/area(nz, node) + if (Fer_GM) then + fer_Wvel(nz, node)=fer_Wvel(nz, node)/area(nz, node) + end if + end do ! --> do nz=nzmax,nzmin,-1 + end do ! --> do node=1, myDim_nod2D + + !___________________________________________________________________________ + ! Add surface fresh water flux as upper boundary condition for + ! continutity + if (.not. (trim(which_ale)=='linfs' )) then + do node=1, myDim_nod2D + nzmin = ulevels_nod2D(node) + if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) + end do ! --> do node=1, myDim_nod2D + end if + + !___________________________________________________________________________ + call exchange_nod(Wvel, partit) + if (Fer_GM) call exchange_nod(fer_Wvel, partit) + + !___________________________________________________________________________ + ! compute vertical CFL_z criteria + call compute_CFLz(dynamics, partit, mesh) + + !___________________________________________________________________________ + ! compute implicite explicite splitting of vetical velocity Wvel according + ! to CFL_z criteria + call compute_Wvel_split(dynamics, partit, mesh) + +end subroutine compute_vert_vel_transpv ! ! !_______________________________________________________________________________ diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index b28dfd8c5..b8cf97a98 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -69,19 +69,19 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) end interface end module -module compute_vert_vel_transpv_interface - interface - subroutine compute_vert_vel_transpv(dynamics, partit, mesh) - USE MOD_MESH - USE MOD_PARTIT - USE MOD_PARSUP - USE MOD_DYN - type(t_dyn) , intent(inout), target :: dynamics - type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(inout), target :: mesh - end subroutine - end interface -end module +!PS module compute_vert_vel_transpv_interface +!PS interface +!PS subroutine compute_vert_vel_transpv(dynamics, partit, mesh) +!PS USE MOD_MESH +!PS USE MOD_PARTIT +!PS USE MOD_PARSUP +!PS USE MOD_DYN +!PS type(t_dyn) , intent(inout), target :: dynamics +!PS type(t_partit), intent(inout), target :: partit +!PS type(t_mesh) , intent(inout), target :: mesh +!PS end subroutine +!PS end interface +!PS end module ! ! @@ -1335,210 +1335,210 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) end do end subroutine compute_thickness_zstar -! -! -!_______________________________________________________________________________ -! calculate vertical velocity from eq.3 in S. Danilov et al. : FESOM2: from -! finite elements to finite volumes. -! -! dh_k/dt + grad(u*h)_k + (w^t - w^b) + water_flux_k=1 = 0 -! -! w^t = w^b - dh_k/dt - grad(u*h)_k - water_flux=1 -! --> do cumulativ summation from bottom to top -subroutine compute_vert_vel_transpv(dynamics, partit, mesh) - USE MOD_PARTIT - USE MOD_PARSUP - USE MOD_MESH - USE MOD_DYN - use o_ARRAYS, only: water_flux - use g_config, only: dt, which_ale - use g_comm_auto - use compute_Wvel_split_interface - use compute_CFLz_interface - implicit none - !___________________________________________________________________________ - type(t_dyn) , intent(inout), target :: dynamics - type(t_partit), intent(inout), target :: partit - type(t_mesh) , intent(inout), target :: mesh - integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) - real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 - real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) - real(kind=WP) :: e2c1(mesh%nl-1), e2c2(mesh%nl-1) - - - !___________________________________________________________________________ - ! pointer on necessary derived types - real(kind=WP), dimension(:,:,:), pointer :: UVh, fer_UV - real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, fer_Wvel -#include "associate_part_def.h" -#include "associate_mesh_def.h" -#include "associate_part_ass.h" -#include "associate_mesh_ass.h" - UVh => dynamics%se_uvh(:,:,:) - Wvel => dynamics%w(:,:) - Wvel_e => dynamics%w_e(:,:) - if (Fer_GM) then - fer_UV => dynamics%fer_uv(:,:,:) - fer_Wvel=> dynamics%fer_w(:,:) - end if - - !___________________________________________________________________________ - do node=1, myDim_nod2D+eDim_nod2D - Wvel(:, node)=0.0_WP - end do ! --> do node=1, myDim_nod2D+eDim_nod2D - - !___________________________________________________________________________ - do ed=1, myDim_edge2D - ! local indice of nodes that span up edge ed - ednodes=edges(:,ed) - - ! local index of element that contribute to edge - edelem=edge_tri(:,ed) - - ! edge_cross_dxdy(1:2,ed)... dx,dy distance from element centroid edelem(1) to - ! center of edge --> needed to calc flux perpedicular to edge from elem edelem(1) - deltaX1=edge_cross_dxdy(1,ed) - deltaY1=edge_cross_dxdy(2,ed) - - !_______________________________________________________________________ - ! calc div(u_vec*h) for every layer - ! do it with gauss-law: int( div(u_vec)*dV) = int( u_vec * n_vec * dS ) - nzmin = ulevels(edelem(1)) - nzmax = nlevels(edelem(1))-1 - ! we introduced c1 & c2 as arrays here to avoid deadlocks when in OpenMP mode - do nz = nzmax, nzmin, -1 - ! --> h * u_vec * n_vec - ! --> e_vec = (dx,dy), n_vec = (-dy,dx); - ! --> h * u*(-dy) + v*dx - e1c1(nz)=( UVh(2, nz, edelem(1))*deltaX1 - UVh(1, nz, edelem(1))*deltaY1 ) - ! inflow(outflow) "flux" to control volume of node enodes1 - ! is equal to outflow(inflow) "flux" to control volume of node enodes2 - end do ! --> do nz=nzmax,nzmin,-1 - Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e1c1(nzmin:nzmax) - Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e1c1(nzmin:nzmax) - - if (Fer_GM) then - do nz = nzmax, nzmin, -1 - e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1 - fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) - end do ! --> do nz=nzmax,nzmin,-1 - fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e1c2(nzmin:nzmax) - fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e1c2(nzmin:nzmax) - end if - - !_______________________________________________________________________ - ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer - ! for edelem(2) - e2c1 = 0.0_WP - e2c2 = 0.0_WP - if(edelem(2)>0)then - deltaX2=edge_cross_dxdy(3,ed) - deltaY2=edge_cross_dxdy(4,ed) - nzmin = ulevels(edelem(2)) - nzmax = nlevels(edelem(2))-1 - do nz = nzmax, nzmin, -1 - e2c1(nz)=-(UVh(2, nz, edelem(2))*deltaX2 - UVh(1, nz, edelem(2))*deltaY2) - end do ! --> do nz=nzmax,nzmin,-1 - Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e2c1(nzmin:nzmax) - Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e2c1(nzmin:nzmax) - - if (Fer_GM) then - do nz = nzmax, nzmin, -1 - e2c2(nz)=-(fer_UV(2, nz, edelem(2))*deltaX2-fer_UV(1, nz, edelem(2))*deltaY2)*helem(nz, edelem(2)) - end do ! --> do nz=nzmax,nzmin,-1 - fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e2c2(nzmin:nzmax) - fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e2c2(nzmin:nzmax) - end if - end if - - !_______________________________________________________________________ -!PS if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & -!PS any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then -!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' edge =', ed -!PS write(*,*) ' enodes =', enodes -!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) -!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) -!PS write(*,*) ' e1c1', e1c1(nzmin:nzmax) -!PS write(*,*) ' e1c2', e1c2(nzmin:nzmax) -!PS write(*,*) ' e2c1', e2c1(nzmin:nzmax) -!PS write(*,*) ' e2c2', e2c2(nzmin:nzmax) +!PS ! +!PS ! +!PS !_______________________________________________________________________________ +!PS ! calculate vertical velocity from eq.3 in S. Danilov et al. : FESOM2: from +!PS ! finite elements to finite volumes. +!PS ! +!PS ! dh_k/dt + grad(u*h)_k + (w^t - w^b) + water_flux_k=1 = 0 +!PS ! +!PS ! w^t = w^b - dh_k/dt - grad(u*h)_k - water_flux=1 +!PS ! --> do cumulativ summation from bottom to top +!PS subroutine compute_vert_vel_transpv(dynamics, partit, mesh) +!PS USE MOD_PARTIT +!PS USE MOD_PARSUP +!PS USE MOD_MESH +!PS USE MOD_DYN +!PS use o_ARRAYS, only: water_flux +!PS use g_config, only: dt, which_ale +!PS use g_comm_auto +!PS use compute_Wvel_split_interface +!PS use compute_CFLz_interface +!PS implicit none +!PS !___________________________________________________________________________ +!PS type(t_dyn) , intent(inout), target :: dynamics +!PS type(t_partit), intent(inout), target :: partit +!PS type(t_mesh) , intent(inout), target :: mesh +!PS integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) +!PS real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 +!PS real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) +!PS real(kind=WP) :: e2c1(mesh%nl-1), e2c2(mesh%nl-1) +!PS +!PS +!PS !___________________________________________________________________________ +!PS ! pointer on necessary derived types +!PS real(kind=WP), dimension(:,:,:), pointer :: UVh, fer_UV +!PS real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, fer_Wvel +!PS #include "associate_part_def.h" +!PS #include "associate_mesh_def.h" +!PS #include "associate_part_ass.h" +!PS #include "associate_mesh_ass.h" +!PS UVh => dynamics%se_uvh(:,:,:) +!PS Wvel => dynamics%w(:,:) +!PS Wvel_e => dynamics%w_e(:,:) +!PS if (Fer_GM) then +!PS fer_UV => dynamics%fer_uv(:,:,:) +!PS fer_Wvel=> dynamics%fer_w(:,:) +!PS end if +!PS +!PS !___________________________________________________________________________ +!PS do node=1, myDim_nod2D+eDim_nod2D +!PS Wvel(:, node)=0.0_WP +!PS end do ! --> do node=1, myDim_nod2D+eDim_nod2D +!PS +!PS !___________________________________________________________________________ +!PS do ed=1, myDim_edge2D +!PS ! local indice of nodes that span up edge ed +!PS ednodes=edges(:,ed) +!PS +!PS ! local index of element that contribute to edge +!PS edelem=edge_tri(:,ed) +!PS +!PS ! edge_cross_dxdy(1:2,ed)... dx,dy distance from element centroid edelem(1) to +!PS ! center of edge --> needed to calc flux perpedicular to edge from elem edelem(1) +!PS deltaX1=edge_cross_dxdy(1,ed) +!PS deltaY1=edge_cross_dxdy(2,ed) +!PS +!PS !_______________________________________________________________________ +!PS ! calc div(u_vec*h) for every layer +!PS ! do it with gauss-law: int( div(u_vec)*dV) = int( u_vec * n_vec * dS ) +!PS nzmin = ulevels(edelem(1)) +!PS nzmax = nlevels(edelem(1))-1 +!PS ! we introduced c1 & c2 as arrays here to avoid deadlocks when in OpenMP mode +!PS do nz = nzmax, nzmin, -1 +!PS ! --> h * u_vec * n_vec +!PS ! --> e_vec = (dx,dy), n_vec = (-dy,dx); +!PS ! --> h * u*(-dy) + v*dx +!PS e1c1(nz)=( UVh(2, nz, edelem(1))*deltaX1 - UVh(1, nz, edelem(1))*deltaY1 ) +!PS ! inflow(outflow) "flux" to control volume of node enodes1 +!PS ! is equal to outflow(inflow) "flux" to control volume of node enodes2 +!PS end do ! --> do nz=nzmax,nzmin,-1 +!PS Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e1c1(nzmin:nzmax) +!PS Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e1c1(nzmin:nzmax) +!PS +!PS if (Fer_GM) then +!PS do nz = nzmax, nzmin, -1 +!PS e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1 - fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) +!PS end do ! --> do nz=nzmax,nzmin,-1 +!PS fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e1c2(nzmin:nzmax) +!PS fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e1c2(nzmin:nzmax) !PS end if - end do ! --> do ed=1, myDim_edge2D - - !___________________________________________________________________________ - ! add the contribution from -dh/dt * area - do node=1, myDim_nod2D - nzmin = ulevels_nod2D(node) - nzmax = nlevels_nod2d(node)-1 - do nz=nzmax,nzmin,-1 - Wvel(nz, node)=Wvel(nz, node)-(hnode_new(nz, node)-hnode(nz, node))*area(nz, node)/dt - end do ! --> do nz=nzmax,nzmin,-1 - - !_______________________________________________________________________ -!PS if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then -!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' node =', node -!PS write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) -!PS write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) -!PS write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) +!PS +!PS !_______________________________________________________________________ +!PS ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer +!PS ! for edelem(2) +!PS e2c1 = 0.0_WP +!PS e2c2 = 0.0_WP +!PS if(edelem(2)>0)then +!PS deltaX2=edge_cross_dxdy(3,ed) +!PS deltaY2=edge_cross_dxdy(4,ed) +!PS nzmin = ulevels(edelem(2)) +!PS nzmax = nlevels(edelem(2))-1 +!PS do nz = nzmax, nzmin, -1 +!PS e2c1(nz)=-(UVh(2, nz, edelem(2))*deltaX2 - UVh(1, nz, edelem(2))*deltaY2) +!PS end do ! --> do nz=nzmax,nzmin,-1 +!PS Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e2c1(nzmin:nzmax) +!PS Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e2c1(nzmin:nzmax) +!PS +!PS if (Fer_GM) then +!PS do nz = nzmax, nzmin, -1 +!PS e2c2(nz)=-(fer_UV(2, nz, edelem(2))*deltaX2-fer_UV(1, nz, edelem(2))*deltaY2)*helem(nz, edelem(2)) +!PS end do ! --> do nz=nzmax,nzmin,-1 +!PS fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e2c2(nzmin:nzmax) +!PS fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e2c2(nzmin:nzmax) +!PS end if !PS end if - end do ! --> do node=1, myDim_nod2D - - !___________________________________________________________________________ - ! Sum up to get W*area - ! cumulative summation of div(u_vec*h) vertically - ! W_k = W_k+1 - div(h_k*u_k) - ! W_k ... vertical flux troughdo node=1, myDim_nod2D - do node=1, myDim_nod2D - nzmin = ulevels_nod2D(node) - nzmax = nlevels_nod2d(node)-1 - do nz=nzmax,nzmin,-1 - Wvel(nz, node)=Wvel(nz, node)+Wvel(nz+1, node) - if (Fer_GM) then - fer_Wvel(nz, node)=fer_Wvel(nz, node)+fer_Wvel(nz+1, node) - end if - end do ! --> do nz=nzmax,nzmin,-1 - end do ! --> do node=1, myDim_nod2D - - !___________________________________________________________________________ - ! divide with depth dependent cell area to convert from Vertical flux to - ! physical vertical velocities in units m/s - do node=1, myDim_nod2D - nzmin = ulevels_nod2D(node) - nzmax = nlevels_nod2d(node)-1 - do nz=nzmin,nzmax - Wvel(nz, node)=Wvel(nz, node)/area(nz, node) - if (Fer_GM) then - fer_Wvel(nz, node)=fer_Wvel(nz, node)/area(nz, node) - end if - end do ! --> do nz=nzmax,nzmin,-1 - end do ! --> do node=1, myDim_nod2D - - !___________________________________________________________________________ - ! Add surface fresh water flux as upper boundary condition for - ! continutity - if (.not. (trim(which_ale)=='linfs' )) then - do node=1, myDim_nod2D - nzmin = ulevels_nod2D(node) - if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) - end do ! --> do node=1, myDim_nod2D - end if - - !___________________________________________________________________________ - call exchange_nod(Wvel, partit) - if (Fer_GM) call exchange_nod(fer_Wvel, partit) - - !___________________________________________________________________________ - ! compute vertical CFL_z criteria - call compute_CFLz(dynamics, partit, mesh) - - !___________________________________________________________________________ - ! compute implicite explicite splitting of vetical velocity Wvel according - ! to CFL_z criteria - call compute_Wvel_split(dynamics, partit, mesh) - -end subroutine compute_vert_vel_transpv +!PS +!PS !_______________________________________________________________________ +!PS !PS if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & +!PS !PS any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then +!PS !PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' +!PS !PS write(*,*) ' mype =', mype +!PS !PS write(*,*) ' edge =', ed +!PS !PS write(*,*) ' enodes =', enodes +!PS !PS write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) +!PS !PS write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) +!PS !PS write(*,*) ' e1c1', e1c1(nzmin:nzmax) +!PS !PS write(*,*) ' e1c2', e1c2(nzmin:nzmax) +!PS !PS write(*,*) ' e2c1', e2c1(nzmin:nzmax) +!PS !PS write(*,*) ' e2c2', e2c2(nzmin:nzmax) +!PS !PS end if +!PS end do ! --> do ed=1, myDim_edge2D +!PS +!PS !___________________________________________________________________________ +!PS ! add the contribution from -dh/dt * area +!PS do node=1, myDim_nod2D +!PS nzmin = ulevels_nod2D(node) +!PS nzmax = nlevels_nod2d(node)-1 +!PS do nz=nzmax,nzmin,-1 +!PS Wvel(nz, node)=Wvel(nz, node)-(hnode_new(nz, node)-hnode(nz, node))*area(nz, node)/dt +!PS end do ! --> do nz=nzmax,nzmin,-1 +!PS +!PS !_______________________________________________________________________ +!PS !PS if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then +!PS !PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' +!PS !PS write(*,*) ' mype =', mype +!PS !PS write(*,*) ' node =', node +!PS !PS write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) +!PS !PS write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) +!PS !PS write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) +!PS !PS end if +!PS end do ! --> do node=1, myDim_nod2D +!PS +!PS !___________________________________________________________________________ +!PS ! Sum up to get W*area +!PS ! cumulative summation of div(u_vec*h) vertically +!PS ! W_k = W_k+1 - div(h_k*u_k) +!PS ! W_k ... vertical flux troughdo node=1, myDim_nod2D +!PS do node=1, myDim_nod2D +!PS nzmin = ulevels_nod2D(node) +!PS nzmax = nlevels_nod2d(node)-1 +!PS do nz=nzmax,nzmin,-1 +!PS Wvel(nz, node)=Wvel(nz, node)+Wvel(nz+1, node) +!PS if (Fer_GM) then +!PS fer_Wvel(nz, node)=fer_Wvel(nz, node)+fer_Wvel(nz+1, node) +!PS end if +!PS end do ! --> do nz=nzmax,nzmin,-1 +!PS end do ! --> do node=1, myDim_nod2D +!PS +!PS !___________________________________________________________________________ +!PS ! divide with depth dependent cell area to convert from Vertical flux to +!PS ! physical vertical velocities in units m/s +!PS do node=1, myDim_nod2D +!PS nzmin = ulevels_nod2D(node) +!PS nzmax = nlevels_nod2d(node)-1 +!PS do nz=nzmin,nzmax +!PS Wvel(nz, node)=Wvel(nz, node)/area(nz, node) +!PS if (Fer_GM) then +!PS fer_Wvel(nz, node)=fer_Wvel(nz, node)/area(nz, node) +!PS end if +!PS end do ! --> do nz=nzmax,nzmin,-1 +!PS end do ! --> do node=1, myDim_nod2D +!PS +!PS !___________________________________________________________________________ +!PS ! Add surface fresh water flux as upper boundary condition for +!PS ! continutity +!PS if (.not. (trim(which_ale)=='linfs' )) then +!PS do node=1, myDim_nod2D +!PS nzmin = ulevels_nod2D(node) +!PS if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) +!PS end do ! --> do node=1, myDim_nod2D +!PS end if +!PS +!PS !___________________________________________________________________________ +!PS call exchange_nod(Wvel, partit) +!PS if (Fer_GM) call exchange_nod(fer_Wvel, partit) +!PS +!PS !___________________________________________________________________________ +!PS ! compute vertical CFL_z criteria +!PS call compute_CFLz(dynamics, partit, mesh) +!PS +!PS !___________________________________________________________________________ +!PS ! compute implicite explicite splitting of vetical velocity Wvel according +!PS ! to CFL_z criteria +!PS call compute_Wvel_split(dynamics, partit, mesh) +!PS +!PS end subroutine compute_vert_vel_transpv From 83b7381c2b843fa8528982e18a63d9e2ab3996e1 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 12 Jul 2023 15:05:04 +0200 Subject: [PATCH 24/50] fix interface problem for the moment --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 219 -------------------------- 1 file changed, 219 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index b8cf97a98..278676fca 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -69,20 +69,6 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) end interface end module -!PS module compute_vert_vel_transpv_interface -!PS interface -!PS subroutine compute_vert_vel_transpv(dynamics, partit, mesh) -!PS USE MOD_MESH -!PS USE MOD_PARTIT -!PS USE MOD_PARSUP -!PS USE MOD_DYN -!PS type(t_dyn) , intent(inout), target :: dynamics -!PS type(t_partit), intent(inout), target :: partit -!PS type(t_mesh) , intent(inout), target :: mesh -!PS end subroutine -!PS end interface -!PS end module - ! ! !_______________________________________________________________________________ @@ -1335,210 +1321,5 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) end do end subroutine compute_thickness_zstar -!PS ! -!PS ! -!PS !_______________________________________________________________________________ -!PS ! calculate vertical velocity from eq.3 in S. Danilov et al. : FESOM2: from -!PS ! finite elements to finite volumes. -!PS ! -!PS ! dh_k/dt + grad(u*h)_k + (w^t - w^b) + water_flux_k=1 = 0 -!PS ! -!PS ! w^t = w^b - dh_k/dt - grad(u*h)_k - water_flux=1 -!PS ! --> do cumulativ summation from bottom to top -!PS subroutine compute_vert_vel_transpv(dynamics, partit, mesh) -!PS USE MOD_PARTIT -!PS USE MOD_PARSUP -!PS USE MOD_MESH -!PS USE MOD_DYN -!PS use o_ARRAYS, only: water_flux -!PS use g_config, only: dt, which_ale -!PS use g_comm_auto -!PS use compute_Wvel_split_interface -!PS use compute_CFLz_interface -!PS implicit none -!PS !___________________________________________________________________________ -!PS type(t_dyn) , intent(inout), target :: dynamics -!PS type(t_partit), intent(inout), target :: partit -!PS type(t_mesh) , intent(inout), target :: mesh -!PS integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) -!PS real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 -!PS real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) -!PS real(kind=WP) :: e2c1(mesh%nl-1), e2c2(mesh%nl-1) -!PS -!PS -!PS !___________________________________________________________________________ -!PS ! pointer on necessary derived types -!PS real(kind=WP), dimension(:,:,:), pointer :: UVh, fer_UV -!PS real(kind=WP), dimension(:,:) , pointer :: Wvel, Wvel_e, fer_Wvel -!PS #include "associate_part_def.h" -!PS #include "associate_mesh_def.h" -!PS #include "associate_part_ass.h" -!PS #include "associate_mesh_ass.h" -!PS UVh => dynamics%se_uvh(:,:,:) -!PS Wvel => dynamics%w(:,:) -!PS Wvel_e => dynamics%w_e(:,:) -!PS if (Fer_GM) then -!PS fer_UV => dynamics%fer_uv(:,:,:) -!PS fer_Wvel=> dynamics%fer_w(:,:) -!PS end if -!PS -!PS !___________________________________________________________________________ -!PS do node=1, myDim_nod2D+eDim_nod2D -!PS Wvel(:, node)=0.0_WP -!PS end do ! --> do node=1, myDim_nod2D+eDim_nod2D -!PS -!PS !___________________________________________________________________________ -!PS do ed=1, myDim_edge2D -!PS ! local indice of nodes that span up edge ed -!PS ednodes=edges(:,ed) -!PS -!PS ! local index of element that contribute to edge -!PS edelem=edge_tri(:,ed) -!PS -!PS ! edge_cross_dxdy(1:2,ed)... dx,dy distance from element centroid edelem(1) to -!PS ! center of edge --> needed to calc flux perpedicular to edge from elem edelem(1) -!PS deltaX1=edge_cross_dxdy(1,ed) -!PS deltaY1=edge_cross_dxdy(2,ed) -!PS -!PS !_______________________________________________________________________ -!PS ! calc div(u_vec*h) for every layer -!PS ! do it with gauss-law: int( div(u_vec)*dV) = int( u_vec * n_vec * dS ) -!PS nzmin = ulevels(edelem(1)) -!PS nzmax = nlevels(edelem(1))-1 -!PS ! we introduced c1 & c2 as arrays here to avoid deadlocks when in OpenMP mode -!PS do nz = nzmax, nzmin, -1 -!PS ! --> h * u_vec * n_vec -!PS ! --> e_vec = (dx,dy), n_vec = (-dy,dx); -!PS ! --> h * u*(-dy) + v*dx -!PS e1c1(nz)=( UVh(2, nz, edelem(1))*deltaX1 - UVh(1, nz, edelem(1))*deltaY1 ) -!PS ! inflow(outflow) "flux" to control volume of node enodes1 -!PS ! is equal to outflow(inflow) "flux" to control volume of node enodes2 -!PS end do ! --> do nz=nzmax,nzmin,-1 -!PS Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e1c1(nzmin:nzmax) -!PS Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e1c1(nzmin:nzmax) -!PS -!PS if (Fer_GM) then -!PS do nz = nzmax, nzmin, -1 -!PS e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1 - fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) -!PS end do ! --> do nz=nzmax,nzmin,-1 -!PS fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e1c2(nzmin:nzmax) -!PS fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e1c2(nzmin:nzmax) -!PS end if -!PS -!PS !_______________________________________________________________________ -!PS ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer -!PS ! for edelem(2) -!PS e2c1 = 0.0_WP -!PS e2c2 = 0.0_WP -!PS if(edelem(2)>0)then -!PS deltaX2=edge_cross_dxdy(3,ed) -!PS deltaY2=edge_cross_dxdy(4,ed) -!PS nzmin = ulevels(edelem(2)) -!PS nzmax = nlevels(edelem(2))-1 -!PS do nz = nzmax, nzmin, -1 -!PS e2c1(nz)=-(UVh(2, nz, edelem(2))*deltaX2 - UVh(1, nz, edelem(2))*deltaY2) -!PS end do ! --> do nz=nzmax,nzmin,-1 -!PS Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e2c1(nzmin:nzmax) -!PS Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e2c1(nzmin:nzmax) -!PS -!PS if (Fer_GM) then -!PS do nz = nzmax, nzmin, -1 -!PS e2c2(nz)=-(fer_UV(2, nz, edelem(2))*deltaX2-fer_UV(1, nz, edelem(2))*deltaY2)*helem(nz, edelem(2)) -!PS end do ! --> do nz=nzmax,nzmin,-1 -!PS fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e2c2(nzmin:nzmax) -!PS fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e2c2(nzmin:nzmax) -!PS end if -!PS end if -!PS -!PS !_______________________________________________________________________ -!PS !PS if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & -!PS !PS any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then -!PS !PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' -!PS !PS write(*,*) ' mype =', mype -!PS !PS write(*,*) ' edge =', ed -!PS !PS write(*,*) ' enodes =', enodes -!PS !PS write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) -!PS !PS write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) -!PS !PS write(*,*) ' e1c1', e1c1(nzmin:nzmax) -!PS !PS write(*,*) ' e1c2', e1c2(nzmin:nzmax) -!PS !PS write(*,*) ' e2c1', e2c1(nzmin:nzmax) -!PS !PS write(*,*) ' e2c2', e2c2(nzmin:nzmax) -!PS !PS end if -!PS end do ! --> do ed=1, myDim_edge2D -!PS -!PS !___________________________________________________________________________ -!PS ! add the contribution from -dh/dt * area -!PS do node=1, myDim_nod2D -!PS nzmin = ulevels_nod2D(node) -!PS nzmax = nlevels_nod2d(node)-1 -!PS do nz=nzmax,nzmin,-1 -!PS Wvel(nz, node)=Wvel(nz, node)-(hnode_new(nz, node)-hnode(nz, node))*area(nz, node)/dt -!PS end do ! --> do nz=nzmax,nzmin,-1 -!PS -!PS !_______________________________________________________________________ -!PS !PS if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then -!PS !PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' -!PS !PS write(*,*) ' mype =', mype -!PS !PS write(*,*) ' node =', node -!PS !PS write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) -!PS !PS write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) -!PS !PS write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) -!PS !PS end if -!PS end do ! --> do node=1, myDim_nod2D -!PS -!PS !___________________________________________________________________________ -!PS ! Sum up to get W*area -!PS ! cumulative summation of div(u_vec*h) vertically -!PS ! W_k = W_k+1 - div(h_k*u_k) -!PS ! W_k ... vertical flux troughdo node=1, myDim_nod2D -!PS do node=1, myDim_nod2D -!PS nzmin = ulevels_nod2D(node) -!PS nzmax = nlevels_nod2d(node)-1 -!PS do nz=nzmax,nzmin,-1 -!PS Wvel(nz, node)=Wvel(nz, node)+Wvel(nz+1, node) -!PS if (Fer_GM) then -!PS fer_Wvel(nz, node)=fer_Wvel(nz, node)+fer_Wvel(nz+1, node) -!PS end if -!PS end do ! --> do nz=nzmax,nzmin,-1 -!PS end do ! --> do node=1, myDim_nod2D -!PS -!PS !___________________________________________________________________________ -!PS ! divide with depth dependent cell area to convert from Vertical flux to -!PS ! physical vertical velocities in units m/s -!PS do node=1, myDim_nod2D -!PS nzmin = ulevels_nod2D(node) -!PS nzmax = nlevels_nod2d(node)-1 -!PS do nz=nzmin,nzmax -!PS Wvel(nz, node)=Wvel(nz, node)/area(nz, node) -!PS if (Fer_GM) then -!PS fer_Wvel(nz, node)=fer_Wvel(nz, node)/area(nz, node) -!PS end if -!PS end do ! --> do nz=nzmax,nzmin,-1 -!PS end do ! --> do node=1, myDim_nod2D -!PS -!PS !___________________________________________________________________________ -!PS ! Add surface fresh water flux as upper boundary condition for -!PS ! continutity -!PS if (.not. (trim(which_ale)=='linfs' )) then -!PS do node=1, myDim_nod2D -!PS nzmin = ulevels_nod2D(node) -!PS if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) -!PS end do ! --> do node=1, myDim_nod2D -!PS end if -!PS -!PS !___________________________________________________________________________ -!PS call exchange_nod(Wvel, partit) -!PS if (Fer_GM) call exchange_nod(fer_Wvel, partit) -!PS -!PS !___________________________________________________________________________ -!PS ! compute vertical CFL_z criteria -!PS call compute_CFLz(dynamics, partit, mesh) -!PS -!PS !___________________________________________________________________________ -!PS ! compute implicite explicite splitting of vetical velocity Wvel according -!PS ! to CFL_z criteria -!PS call compute_Wvel_split(dynamics, partit, mesh) -!PS -!PS end subroutine compute_vert_vel_transpv From e989cd5a5a02c75592fcf394a43b425bb164cbf1 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 3 Aug 2023 12:14:37 +0200 Subject: [PATCH 25/50] fix indices problem in curl computation --- src/gen_modules_diag.F90 | 89 ++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/src/gen_modules_diag.F90 b/src/gen_modules_diag.F90 index e0156774e..cc678bf56 100644 --- a/src/gen_modules_diag.F90 +++ b/src/gen_modules_diag.F90 @@ -175,69 +175,78 @@ subroutine diag_curl_vel3(mode, dynamics, partit, mesh) #include "associate_mesh_ass.h" UV => dynamics%uv(:,:,:) - !===================== + !___________________________________________________________________________ if (firstcall) then !allocate the stuff at the first call allocate(curl_vel3(nl-1, myDim_nod2D+eDim_nod2D)) firstcall=.false. if (mode==0) return end if + !___________________________________________________________________________ curl_vel3=0. - - DO ed=1,myDim_edge2D + do ed=1,myDim_edge2D enodes=edges(:,ed) el=edge_tri(:,ed) + !_______________________________________________________________________ nl1=nlevels(el(1))-1 nu1=ulevels(el(1)) deltaX1=edge_cross_dxdy(1,ed) deltaY1=edge_cross_dxdy(2,ed) - nl2=0 - nu2=0 + + !_______________________________________________________________________ if (el(2)>0) then deltaX2=edge_cross_dxdy(3,ed) deltaY2=edge_cross_dxdy(4,ed) nl2=nlevels(el(2))-1 nu2=ulevels(el(2)) - end if - - nl12 = min(nl1,nl2) - nu12 = min(nu1,nu2) - DO nz=nu1,nu12-1 - c1=deltaX1*UV(1,nz,el(1))+deltaY1*UV(2,nz,el(1)) - curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 - curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 - END DO - if (nu2>0) then - DO nz=nu2,nu12-1 + nl12 = min(nl1,nl2) + nu12 = max(nu1,nu2) + !___________________________________________________________________ + do nz=nu1,nu12-1 + c1=deltaX1*UV(1,nz,el(1))+deltaY1*UV(2,nz,el(1)) + curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 + curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 + end do + do nz=nu2,nu12-1 c1= -deltaX2*UV(1,nz,el(2))-deltaY2*UV(2,nz,el(2)) curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 - END DO + end do + !___________________________________________________________________ + do nz=nu12,nl12 + c1=deltaX1*UV(1,nz,el(1))+deltaY1*UV(2,nz,el(1))- & + deltaX2*UV(1,nz,el(2))-deltaY2*UV(2,nz,el(2)) + curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 + curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 + end do + !___________________________________________________________________ + do nz=nl12+1,nl1 + c1=deltaX1*UV(1,nz,el(1))+deltaY1*UV(2,nz,el(1)) + curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 + curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 + end do + do nz=nl12+1,nl2 + c1= -deltaX2*UV(1,nz,el(2))-deltaY2*UV(2,nz,el(2)) + curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 + curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 + end do + !_______________________________________________________________________ + else + do nz=nu1,nl1 + c1=deltaX1*UV(1,nz,el(1))+deltaY1*UV(2,nz,el(1)) + curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 + curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 + end do end if - DO nz=nu12,nl12 - c1=deltaX1*UV(1,nz,el(1))+deltaY1*UV(2,nz,el(1))- & - deltaX2*UV(1,nz,el(2))-deltaY2*UV(2,nz,el(2)) - curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 - curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 - END DO - DO nz=nl12+1,nl1 - c1=deltaX1*UV(1,nz,el(1))+deltaY1*UV(2,nz,el(1)) - curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 - curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 - END DO - DO nz=nl12+1,nl2 - c1= -deltaX2*UV(1,nz,el(2))-deltaY2*UV(2,nz,el(2)) - curl_vel3(nz,enodes(1))=curl_vel3(nz,enodes(1))+c1 - curl_vel3(nz,enodes(2))=curl_vel3(nz,enodes(2))-c1 - END DO - END DO - - DO n=1, myDim_nod2D - !!PS DO nz=1, nlevels_nod2D(n)-1 - DO nz=ulevels_nod2D(n), nlevels_nod2D(n)-1 + end do + + !___________________________________________________________________________ + do n=1, myDim_nod2D + do nz=ulevels_nod2D(n), nlevels_nod2D(n)-1 curl_vel3(nz,n)=curl_vel3(nz,n)/areasvol(nz,n) - END DO - END DO + end do + end do + end subroutine diag_curl_vel3 ! ============================================================== ! From cb952357f41d8a9c3482459403acf2abb958bb8f Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 3 Aug 2023 14:07:39 +0200 Subject: [PATCH 26/50] small fix indices limit --- src/oce_ale.F90 | 2 -- src/oce_ale_ssh_splitexpl_subcycl.F90 | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 926c82237..95c40e829 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -3302,8 +3302,6 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) use g_cvmix_kpp use g_cvmix_tidal use Toy_Channel_Soufflet -!PS use compute_Wvel_split_interface -!PS use compute_CFLz_interface use oce_ale_interfaces use compute_vert_vel_transpv_interface use compute_ssh_split_explicit_interface diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 278676fca..3908a5a72 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -115,8 +115,8 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) do node=1, myDim_nod2D ul1 = ulevels_nod2D(node) nl1 = nlevels_nod2D(node) - wu(1:nl1+1) = 0.0_WP - wv(1:nl1+1) = 0.0_WP + wu(1:nl1) = 0.0_WP + wv(1:nl1) = 0.0_WP UVnode_rhs(:,:,node) = 0.0_WP !_______________________________________________________________________ ! surface From 9d48d8d9a43217c0c7595e88c3bc034386baa250 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 18 Aug 2023 12:02:34 +0200 Subject: [PATCH 27/50] add stabilisation for splitexplicite, add longer term stable core2 parameters for split expl --- src/MOD_DYN.F90 | 10 +- src/oce_ale_ssh_splitexpl_subcycl.F90 | 311 ++++++++++++++++---------- src/oce_setup_step.F90 | 15 +- 3 files changed, 216 insertions(+), 120 deletions(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index b9afadaba..bcd8aa49f 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -121,8 +121,14 @@ MODULE MOD_DYN logical :: use_ssh_splitexpl_subcycl = .false. ! barotropic subcycling time-steps and dissipation parameter - integer :: splitexpl_BTsteps = 40 - real(kind=WP) :: splitexpl_BTtheta = 0.14_WP + integer :: splitexpl_BTsteps = 50 + real(kind=WP) :: splitexpl_BTtheta = 0.14_WP + logical :: splitexpl_bottdrag = .true. + logical :: splitexpl_bdrag_si = .true. + logical :: splitexpl_visc = .true. + real(kind=WP) :: splitexpl_visc_gamma0 = 10 + real(kind=WP) :: splitexpl_visc_gamma1 = 2750 + real(kind=WP) :: splitexpl_visc_gamma2 = 0 !___________________________________________________________________________ ! energy diagnostic part: will be computed inside the model ("hard integration"): diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 3908a5a72..94d8c5d8c 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -805,23 +805,7 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) end do !_______________________________________________________________ -!PS if ( any(UV_rhs(1, nzmin:nzmax-1, elem)/=UV_rhs(1, nzmin:nzmax-1, elem)) .or. & -!PS any(UV_rhs(2, nzmin:nzmax-1, elem)/=UV_rhs(2, nzmin:nzmax-1, elem))) then -!PS write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in UV_rhs=UVh_rhs+uvr*helem' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' elem =', elem -!PS write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nzmin:nzmax-1, elem) -!PS write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nzmin:nzmax-1, elem) -!PS write(*,*) ' ur(nzmin:nzmax-1)=', ur(nzmin:nzmax-1) -!PS write(*,*) ' vr(nzmin:nzmax-1)=', vr(nzmin:nzmax-1) -!PS write(*,*) ' helem(nz, elem) =', helem(nzmin:nzmax-1, elem) -!PS write(*,*) ' a(nzmin:nzmax-1) =', a(nzmin:nzmax-1) -!PS write(*,*) ' b(nzmin:nzmax-1) =', b(nzmin:nzmax-1) -!PS write(*,*) ' c(nzmin:nzmax-1) =', c(nzmin:nzmax-1) -!PS write(*,*) ' Av(nzmin:nzmax-1,elem) =', Av(nzmin:nzmax-1, elem) -!PS write(*,*) ' Wvel_i(nzmin:nzmax-1,elem) =',Wvel_i(nzmin:nzmax-1, elnodes) -!PS end if - + end do ! --> do elem=1,myDim_elem2D !$OMP END DO !$OMP END PARALLEL @@ -844,8 +828,10 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) type(t_mesh) , intent(inout) , target :: mesh !___________________________________________________________________________ real(kind=WP) :: vert_sum_u, vert_sum_v, Fx, Fy, ab1, ab2, hh - integer :: elem, nz, nzmin, nzmax, elnodes(3) + integer :: elem, nz, nzmin, nzmax, elnodes(3), ed, el(2) logical, save :: sfirst + real(kind=WP) :: update_ubt, update_vbt, vi, len + !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:,:,:), pointer :: UV_rhs @@ -898,7 +884,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) vert_sum_v=vert_sum_v + Fy*hh !_______________________________________________________________________ - ! Remove the contribution from the Coriolis will be accounted explicitely + ! Remove the contribution from the Coriolis will be accounted explicitely ! for in the barotropic equation ! UVBT_rhs ... baroclinic forcing term in barotropic equation R_b ! --> d/dt*U_bt + f*e_z x U_bt + g*H* grad(eta) = R_bt @@ -953,14 +939,18 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(inout) , target :: mesh !___________________________________________________________________________ - real(kind=WP) :: dtBT, BT_inv, hh, f, rx, ry, a, d, c1, c2, ax, ay + real(kind=WP) :: dtBT, BT_inv, hh, f, rx, ry, a, b, d, c1, c2, ax, ay real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, thetaBT integer :: step, elem, elnodes(3), edge, ednodes(2), edelem(2) - integer :: nzmin1, nzmax1, nzmin2, nzmax2 + real(kind=WP) :: update_ubt, update_vbt, vi, len, nzmax + real(kind=WP), allocatable :: bottomdrag(:), UVBT_harmvisc(:,:) + + !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:) , pointer :: eta_n real(kind=WP), dimension(:,:) , pointer :: UVBT_rhs, UVBT, UVBT_theta, UVBT_mean, UVBT_12 + real(kind=WP), dimension(:,:,:), pointer :: UV #include "associate_part_def.h" #include "associate_mesh_def.h" #include "associate_part_ass.h" @@ -971,26 +961,156 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) UVBT_theta=>dynamics%se_uvBT_theta(:,:) UVBT_mean =>dynamics%se_uvBT_mean(:,:) UVBT_12 =>dynamics%se_uvBT_12(:,:) + if (dynamics%splitexpl_bottdrag) then + UV =>dynamics%uv(:,:,:) + end if + + allocate(bottomdrag( myDim_elem2D+eDim_elem2D)) + allocate(UVBT_harmvisc(2, myDim_elem2D+eDim_elem2D)) + bottomdrag = 0.0_WP + UVBT_harmvisc = 0.0_WP !___________________________________________________________________________ ! Dissipation parameter of FB dissipative method 0.14 is the default value ! from Demange et al. -!PS thetaBT= 0.14_WP -!PS thetaBT= 0.5_WP thetaBT= dynamics%splitexpl_BTtheta ! BTsteps should be 30 or 40. dtBT = dt/dynamics%splitexpl_BTsteps BT_inv = 1.0_WP/(1.0_WP*dynamics%splitexpl_BTsteps) + + !___SPLIT-EXPLICITE STABILIZATION___________________________________________ + ! trim R (UVBT_rhs): + ! UVBT_rhs --> UVBT_rhs - div_h Ah H^n div_h(Ubt^n/H^n) + Cd*|Ubot|* Ubt^n/H^n + ! The intention here is to approximately remove the term thatwill be + ! added later to the barotropic momentum equation (see subroutine + ! compute_BT_step_SE_ale) + ! --> use only harmonmic viscosity operator applied to the barotropic + ! velocity + if (dynamics%splitexpl_visc) then + + !_______________________________________________________________________ + ! remove viscosity + do edge=1, myDim_edge2D+eDim_edge2D + + ! if ed is an outer boundary edge, skip it + if(myList_edge2D(edge)>edge2D_in) cycle + + ! elem indices that participate in edge + edelem= edge_tri(:,edge) + nzmax = minval(nlevels(edelem)) + hh = -zbar(nzmax) + len = sqrt(sum(elem_area(edelem))) + + update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh + update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh + vi=update_ubt*update_ubt + update_vbt*update_vbt + vi=-dt*sqrt(max(dynamics%splitexpl_visc_gamma0, & + max(dynamics%splitexpl_visc_gamma1*sqrt(vi), & + dynamics%splitexpl_visc_gamma2*vi) & + )*len) + update_ubt=update_ubt*vi + update_vbt=update_vbt*vi + + !___________________________________________________________________ +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock(partit%plock(edelem(1))) +#else +!$OMP ORDERED +#endif + UVBT_rhs(1, edelem(1))=UVBT_rhs(1, edelem(1))-update_ubt/elem_area(edelem(1))*hh + UVBT_rhs(2, edelem(1))=UVBT_rhs(2, edelem(1))-update_vbt/elem_area(edelem(1))*hh +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(edelem(1))) + call omp_set_lock (partit%plock(edelem(2))) +#endif + UVBT_rhs(1, edelem(2))=UVBT_rhs(1, edelem(2))+update_ubt/elem_area(edelem(2))*hh + UVBT_rhs(2, edelem(2))=UVBT_rhs(2, edelem(2))+update_vbt/elem_area(edelem(2))*hh +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(edelem(2))) +#else +!$OMP END ORDERED +#endif + end do ! --> do edge=1, myDim_edge2D+eDim_edge2D + + call exchange_elem_begin(UVBT_rhs, partit) + call exchange_elem_end(partit) + end if ! --> if (dynamics%splitexpl_visc) then + + !___________________________________________________________________________ + ! remove bottom drag + if (dynamics%splitexpl_bottdrag) then + do elem=1, myDim_elem2D + elnodes= elem2D_nodes(:,elem) + nzmax = nlevels(elem) +!PS hh = -zbar(nzmax)+sum(eta_n(elnodes))/3.0_WP + hh = -zbar(nzmax) + bottomdrag(elem) = dt*C_d*sqrt(UV(1, nzmax-1, elem)**2 + UV(2, nzmax-1, elem)**2) + UVBT_rhs(1, elem)=UVBT_rhs(1, elem) + bottomdrag(elem)*UVBT(1, elem)/hh + UVBT_rhs(2, elem)=UVBT_rhs(2, elem) + bottomdrag(elem)*UVBT(2, elem)/hh + end do + end if ! --> if (dynamics%splitexpl_bottdrag) then + !___________________________________________________________________________ ! eta_n elevation used in BT stepping, it is just a copy of eta_n ! UBT and VBT are transport velocities UVBT_mean= 0.0_WP do step=1, dynamics%splitexpl_BTsteps - ! - ! Dissipative forward--backward time stepping - ! + !####################################################################### + !########## Dissipative forward--backward time stepping ########## + !####################################################################### + + !_______________________________________________________________________ + ! compute harmonic viscosity for stability + if (dynamics%splitexpl_visc) then + UVBT_harmvisc = 0.0_WP + do edge=1, myDim_edge2D+eDim_edge2D + + ! if ed is an outer boundary edge, skip it + if(myList_edge2D(edge)>edge2D_in) cycle + + ! elem indices that participate in edge + edelem = edge_tri(:,edge) + nzmax = minval(nlevels(edelem)) + hh = -zbar(nzmax) + len = sqrt(sum(elem_area(edelem))) + + update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh + update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh + vi=update_ubt*update_ubt + update_vbt*update_vbt + vi=dt*sqrt(max(dynamics%splitexpl_visc_gamma0, & + max(dynamics%splitexpl_visc_gamma1*sqrt(vi), & + dynamics%splitexpl_visc_gamma2*vi) & + )*len) + update_ubt=update_ubt*vi + update_vbt=update_vbt*vi + + !_______________________________________________________________ +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock(partit%plock(edelem(1))) +#else +!$OMP ORDERED +#endif + UVBT_harmvisc(1, edelem(1))=UVBT_harmvisc(1, edelem(1))-update_ubt/elem_area(edelem(1))*hh + UVBT_harmvisc(2, edelem(1))=UVBT_harmvisc(2, edelem(1))-update_vbt/elem_area(edelem(1))*hh +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(edelem(1))) + call omp_set_lock (partit%plock(edelem(2))) +#endif + UVBT_harmvisc(1, edelem(2))=UVBT_harmvisc(1, edelem(2))+update_ubt/elem_area(edelem(2))*hh + UVBT_harmvisc(2, edelem(2))=UVBT_harmvisc(2, edelem(2))+update_vbt/elem_area(edelem(2))*hh +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(edelem(2))) +#else +!$OMP END ORDERED +#endif + end do ! --> do edge=1, myDim_edge2D+eDim_edge2D + + call exchange_elem_begin(UVBT_harmvisc, partit) + call exchange_elem_end(partit) + end if ! -> if (dynamics%splitexpl_visc) then + !_______________________________________________________________________ ! Advance velocities. I use SI stepping for the Coriolis do elem=1, myDim_elem2D @@ -1000,17 +1120,58 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! AAA = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+(m)/M)) ! - h*H^m*grad_H*eta^((n+m)/M) ! - Rbt-->UVBT_rhs ] -!PS hh = -zbar_e_bot(elem)+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth - hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth +!PS hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth + hh = -zbar(nlevels(elem)) ! Total fluid depth f = mesh%coriolis(elem) - rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(2, elem)) + BT_inv*UVBT_rhs(1, elem) - ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(1, elem)) + BT_inv*UVBT_rhs(2, elem) + rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(2, elem)) & + + BT_inv*UVBT_rhs(1, elem) & + + BT_inv*(UVBT_harmvisc(1, elem) - bottomdrag(elem)*UVBT(1, elem)/hh) ! <-- stabilization terms + + ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(1, elem)) & + + BT_inv*UVBT_rhs(2, elem) & + + BT_inv*(UVBT_harmvisc(2, elem) - bottomdrag(elem)*UVBT(2, elem)/hh) ! <-- stabilization terms + + ! compute new velocity Ubt^(n+(m+1)/M), Vbt^(n+(m+1)/M) considering + ! in terms of Increments (deltaU) and semi-Implicit Coriolis + ! (We do it here based on increments since it saves us some significant + ! digits for the accuracy) + ! Increments: + ! deltaU = Ubt^(n+(m+1)/M)-Ubt^(n+m/M) + ! Ubt^(n+(m+1)/M)+Ubt^(n+m/M) = Ubt^(n+(m+1)/M)-Ubt^(n+m/M)+2*Ubt^(n+m/M) + ! = deltaU + 2*Ubt^(n+m/M) + ! + ! Ubt^(n+(m+1)/M)-Ubt^(n+m/M) = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+m/M)) + ! - h*H^m*grad_H*eta^(n+m/M) + ! - Rbt-->UVBT_rhs ] + ! + ! deltaU - dt/(2*M)*f*deltaV = dt/M*f*Vbt^(n+m/M) - h*H^m*gradx_H*eta^(n+m/M) + Rbtx + ! deltaV + dt/(2*M)*f*deltaU = -dt/M*f*Ubt^(n+m/M) - h*H^m*grady_H*eta^(n+m/M) + Rbty + ! \________________________v___________________________/ + ! --> a = dt/(2*M)*f Rx, Ry + ! + ! | 1 -a | * | deltaU | = MAT* | deltaU | = | Rx | + ! | a 1 | | deltaV | = | deltaV | = | Ry | + ! + ! --> d = 1/(1 + a^2) ; inv(MAT) = d* | 1 a | + ! | -a 1 | + ! + ! | deltaU | = inv(MAT) * | Rx | + ! | deltaV | = | Ry | + ! + ! Ubt^(n+(m+1)/M) = Ubt^(n+m/M) + d*( Rx + a*Ry) + ! Vbt^(n+(m+1)/M) = Vbt^(n+m/M) + d*(-a*Rx + Ry) + ! ! Semi-Implicit Coriolis a = dtBT*f*0.5_WP - d = 1.0_WP/(1.0_WP+a*a) - ax = d*( rx + a*ry) - ay = d*(-a*rx + ry) + if (dynamics%splitexpl_bdrag_si) then + b = 1.0_WP+BT_inv*bottomdrag(elem)/hh + else + b = 1.0_WP + end if + d = 1.0_WP/(b*b + a*a) + ax = d*( b*rx + a*ry ) + ay = d*( -a*rx + b*ry ) !___________________________________________________________________ ! compute new velocities Ubt^(n+(m+1)/M) at barotropic time step (n+(m+1)/M) ... @@ -1036,6 +1197,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! T. Banerjee et al.,Split-Explicite external mode solver in FESOM2, UVBT_mean( 1, elem) = UVBT_mean( 1, elem) + UVBT_theta(1, elem)*BT_inv UVBT_mean( 2, elem) = UVBT_mean( 2, elem) + UVBT_theta(2, elem)*BT_inv + end do !_______________________________________________________________________ @@ -1071,55 +1233,13 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) eta_n(ednodes(1))=eta_n(ednodes(1)) + (c1+c2)*dtBT/area(1,ednodes(1)) eta_n(ednodes(2))=eta_n(ednodes(2)) - (c1+c2)*dtBT/area(1,ednodes(2)) - -!PS if ( (mype==398) .and. ((abs(eta_n(ednodes(1)))>10) .or. (abs(eta_n(ednodes(1)))>10)) ) then -!PS if ((abs(eta_n(ednodes(1)))>100) .or. (abs(eta_n(ednodes(2)))>100)) then -!PS write(*,*) '-------------------------------------------------' -!PS write(*,*) ' --> subroutine compute_BT_step_SE_ale --> found eta_n>100 in barotrop. subcycling' -!PS write(*,*) ' mype = ', mype -!PS write(*,*) ' mstep = ', mstep -!PS write(*,*) ' btstep = ', step -!PS write(*,*) ' edge = ', edge -!PS write(*,*) ' ednodes(1:2) = ', ednodes -!PS write(*,*) ' edelem(1:2) = ', edelem -!PS write(*,*) ' glon,glat = ', geo_coord_nod2D(:,ednodes)/rad -!PS write(*,*) -!PS write(*,*) ' eta_n(ednodes) = ', eta_n(ednodes) -!PS write(*,*) ' c1, c2 = ', c1, c2 -!PS if(edelem(2)>0) then -!PS write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem) -!PS write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem) -!PS write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem) -!PS write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem) -!PS nzmin1 = ulevels(edelem(1)) -!PS nzmax1 = nlevels(edelem(1))-1 -!PS nzmin2 = ulevels(edelem(2)) -!PS nzmax2 = nlevels(edelem(2))-1 -!PS write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) -!PS write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%se_uvh(1:2, nzmin2:nzmax2, edelem(2)) -!PS write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) -!PS write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(2)) = ', dynamics%uv_rhs(1:2, nzmin2:nzmax2, edelem(2)) -!PS else -!PS write(*,*) ' UVBT_rhs( 1:2,edelem) = ', UVBT_rhs( 1:2, edelem(1)) -!PS write(*,*) ' UVBT( 1:2,edelem) = ', UVBT( 1:2, edelem(1)) -!PS write(*,*) ' UVBT_theta(1:2,edelem) = ', UVBT_theta(1:2, edelem(1)) -!PS write(*,*) ' UVBT_mean( 1:2,edelem) = ', UVBT_mean( 1:2, edelem(1)) -!PS nzmin1 = ulevels(edelem(1)) -!PS nzmax1 = nlevels(edelem(1))-1 -!PS write(*,*) ' UVh(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%se_uvh(1:2, nzmin1:nzmax1, edelem(1)) -!PS write(*,*) ' UV_rhs(1:2,nzmin:nzmax,edelem(1)) = ', dynamics%uv_rhs(1:2, nzmin1:nzmax1, edelem(1)) -!PS -!PS end if -!PS write(*,*) -!PS end if - end do !_______________________________________________________________________ call exchange_nod(eta_n, partit) end do ! --> do step=1, dynamics%splitexpl_BTsteps - + deallocate(bottomdrag, UVBT_harmvisc) hbar_old = hbar hbar = eta_n end subroutine compute_BT_step_SE_ale @@ -1198,26 +1318,6 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) -!PS !_______________________________________________________________ -!PS if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & -!PS UV( 2, nz, elem)/=UV( 2, nz, elem)) then -!PS write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=1) --> found Nan in UV after update UV with barotr. term' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' elem =', elem -!PS write(*,*) ' nz =', nz -!PS write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) -!PS write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) -!PS write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) -!PS write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) -!PS write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) -!PS write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) -!PS write(*,*) ' helem(nz, elem) =', helem(nz, elem) -!PS write(*,*) ' UVBT_mean(1,elem)=', UVBT_mean(1,elem) -!PS write(*,*) ' UVBT_mean(2,elem)=', UVBT_mean(2,elem) -!PS write(*,*) ' ubar, vbar =', ubar, vbar -!PS write(*,*) ' hh_inv =', hh_inv -!PS end if - end do end do call exchange_elem(UVh, partit) ! This exchange can be avoided, but test first. @@ -1246,25 +1346,6 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_12(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection - -!PS !_______________________________________________________________ -!PS if ( UV( 1, nz, elem)/=UV( 1, nz, elem) .or. & -!PS UV( 2, nz, elem)/=UV( 2, nz, elem)) then -!PS write(*,*) ' --> subroutine update_trim_vel_ale_vtransp(mode=2) --> found Nan in UV after update UV with barotr. term' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' elem =', elem -!PS write(*,*) ' nz =', nz -!PS write(*,*) ' UV( 1, nz, elem) =', UV( 1, nz, elem) -!PS write(*,*) ' UV( 2, nz, elem) =', UV( 2, nz, elem) -!PS write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) -!PS write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) -!PS write(*,*) ' helem(nz, elem) =', helem(nz, elem) -!PS write(*,*) ' UVBT_12(1,elem) =', UVBT_12(1,elem) -!PS write(*,*) ' UVBT_12(2,elem) =', UVBT_12(2,elem) -!PS write(*,*) ' ubar, vbar =', ubar, vbar -!PS write(*,*) ' hh_inv =', hh_inv -!PS end if - end do end do call exchange_elem(UVh, partit) ! diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index b4393674e..9984328ab 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -394,12 +394,15 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) logical :: use_ssh_splitexpl_subcycl=.false. integer :: splitexpl_BTsteps real(kind=WP) :: splitexpl_BTtheta + logical :: splitexpl_visc, splitexpl_bottdrag, splitexpl_bdrag_si + real(kind=WP) :: splitexpl_visc_gamma0, splitexpl_visc_gamma1, splitexpl_visc_gamma2 namelist /dynamics_visc / opt_visc, visc_gamma0, visc_gamma1, visc_gamma2, & use_ivertvisc, visc_easybsreturn namelist /dynamics_general/ momadv_opt, use_freeslip, use_wsplit, wsplit_maxcfl, & ldiag_KE, use_ssh_splitexpl_subcycl, splitexpl_BTsteps, & - splitexpl_BTtheta + splitexpl_BTtheta, splitexpl_bottdrag, splitexpl_bdrag_si, & + splitexpl_visc, splitexpl_visc_gamma0, splitexpl_visc_gamma1, splitexpl_visc_gamma2 !___________________________________________________________________________ ! pointer on necessary derived types #include "associate_part_def.h" @@ -436,8 +439,14 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) dynamics%ldiag_KE = ldiag_KE dynamics%use_ssh_splitexpl_subcycl = use_ssh_splitexpl_subcycl if (dynamics%use_ssh_splitexpl_subcycl) then - dynamics%splitexpl_BTsteps = splitexpl_BTsteps - dynamics%splitexpl_BTtheta = splitexpl_BTtheta + dynamics%splitexpl_BTsteps = splitexpl_BTsteps + dynamics%splitexpl_BTtheta = splitexpl_BTtheta + dynamics%splitexpl_bottdrag = splitexpl_bottdrag + dynamics%splitexpl_bdrag_si = splitexpl_bdrag_si + dynamics%splitexpl_visc = splitexpl_visc + dynamics%splitexpl_visc_gamma0 = splitexpl_visc_gamma0 + dynamics%splitexpl_visc_gamma1 = splitexpl_visc_gamma1 + dynamics%splitexpl_visc_gamma2 = splitexpl_visc_gamma2 end if !___________________________________________________________________________ From f7b7dcd5d2fc436f2d499a4335df37361aba1f2b Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 12 Sep 2023 15:15:46 +0200 Subject: [PATCH 28/50] update changes to subcycling, try to make energy diagnostic consistent with subcycling --- src/gen_modules_diag.F90 | 4 +- src/io_meandata.F90 | 94 ++++---- src/oce_ale.F90 | 119 +++++++--- src/oce_ale_ssh_splitexpl_subcycl.F90 | 301 ++++++++++---------------- src/oce_dyn.F90 | 66 +++--- 5 files changed, 290 insertions(+), 294 deletions(-) diff --git a/src/gen_modules_diag.F90 b/src/gen_modules_diag.F90 index cc678bf56..c8ac0eca3 100644 --- a/src/gen_modules_diag.F90 +++ b/src/gen_modules_diag.F90 @@ -76,7 +76,9 @@ module diagnostics logical :: ldiag_vorticity =.false. logical :: ldiag_extflds =.false. - namelist /diag_list/ ldiag_solver, lcurt_stress_surf, ldiag_curl_vel3, ldiag_Ri, ldiag_TurbFlux, ldiag_dMOC, ldiag_DVD, ldiag_salt3D, ldiag_forc, ldiag_vorticity, ldiag_extflds + namelist /diag_list/ ldiag_solver, lcurt_stress_surf, ldiag_curl_vel3, ldiag_Ri, & + ldiag_TurbFlux, ldiag_dMOC, ldiag_DVD, ldiag_salt3D, ldiag_forc, & + ldiag_vorticity, ldiag_extflds contains diff --git a/src/io_meandata.F90 b/src/io_meandata.F90 index 37819c762..314ee73a2 100644 --- a/src/io_meandata.F90 +++ b/src/io_meandata.F90 @@ -487,6 +487,8 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh) if (Fer_GM) then call def_stream( nod2D , myDim_nod2D , 'fer_C', 'GM, depth independent speed', 'm/s' , fer_c(:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) end if +CASE ('cfl_z ') + call def_stream((/nl, nod2D/), (/nl, myDim_nod2D/), 'cfl_z', 'vertical CFL criteria', '?', dynamics%cfl_z(:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) !_______________________________________________________________________________ ! Density MOC diagnostics @@ -716,72 +718,72 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh) end if if (dynamics%ldiag_ke) then - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_u_xVEL', 'work of advection [u]', 'm2/s2', dynamics%ke_adv_xVEL(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_v_xVEL', 'work of advection [v]', 'm2/s2', dynamics%ke_adv_xVEL(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_u_xVEL', 'work of advection [u]', 'm2/s2', dynamics%ke_adv_xVEL(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_v_xVEL', 'work of advection [v]', 'm2/s2', dynamics%ke_adv_xVEL(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_u_xVEL', 'work Coriolis :) [u]', 'm2/s2', dynamics%ke_cor_xVEL(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_v_xVEL', 'work Coriolis :) [v]', 'm2/s2', dynamics%ke_cor_xVEL(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_u_xVEL', 'work Coriolis :) [u]', 'm2/s2', dynamics%ke_cor_xVEL(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_v_xVEL', 'work Coriolis :) [v]', 'm2/s2', dynamics%ke_cor_xVEL(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_u_xVEL', 'work of pressure gradient [x]', 'm2/s2', dynamics%ke_pre_xVEL(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_v_xVEL', 'work of pressure gradient [y]', 'm2/s2', dynamics%ke_pre_xVEL(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_u_xVEL', 'work of pressure gradient [x]', 'm2/s2', dynamics%ke_pre_xVEL(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_v_xVEL', 'work of pressure gradient [y]', 'm2/s2', dynamics%ke_pre_xVEL(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_u_xVEL', 'work of hor. visc. [x]', 'm2/s2', dynamics%ke_hvis_xVEL(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_v_xVEL', 'work of hor. visc. [y]', 'm2/s2', dynamics%ke_hvis_xVEL(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_u_xVEL', 'work of hor. visc. [x]', 'm2/s2', dynamics%ke_hvis_xVEL(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_v_xVEL', 'work of hor. visc. [y]', 'm2/s2', dynamics%ke_hvis_xVEL(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_u_xVEL', 'work of ver. visc. [x]', 'm2/s2', dynamics%ke_vvis_xVEL(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_v_xVEL', 'work of ver. visc. [y]', 'm2/s2', dynamics%ke_vvis_xVEL(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_u_xVEL', 'work of ver. visc. [x]', 'm2/s2', dynamics%ke_vvis_xVEL(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_v_xVEL', 'work of ver. visc. [y]', 'm2/s2', dynamics%ke_vvis_xVEL(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_du2', 'KE change [0.5 du2]', 'm2/s2', dynamics%ke_du2(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_dv2', 'KE change [0.5 dv2]', 'm2/s2', dynamics%ke_du2(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_du2', 'KE change [0.5 du2]', 'm2/s2', dynamics%ke_du2(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_dv2', 'KE change [0.5 dv2]', 'm2/s2', dynamics%ke_du2(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'W_x_RHO', 'buoyancy work = ke_pre x VEL', 'm2/s2', dynamics%ke_wrho(:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'W_x_RHO', 'buoyancy work = ke_pre x VEL', 'm2/s2', dynamics%ke_wrho(:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_wind_x_xVEL', 'work of wind [x]', 'm2/s2', dynamics%ke_wind_xVEL(1,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_wind_y_xVEL', 'work of wind [y]', 'm2/s2', dynamics%ke_wind_xVEL(2,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_wind_x_xVEL', 'work of wind [x]', 'm2/s2', dynamics%ke_wind_xVEL(1,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_wind_y_xVEL', 'work of wind [y]', 'm2/s2', dynamics%ke_wind_xVEL(2,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_drag_x_xVEL', 'work of drag [x]', 'm2/s2', dynamics%ke_drag_xVEL(1,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_drag_y_xVEL', 'work of drag [y]', 'm2/s2', dynamics%ke_drag_xVEL(2,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_drag_x_xVEL', 'work of drag [x]', 'm2/s2', dynamics%ke_drag_xVEL(1,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_drag_y_xVEL', 'work of drag [y]', 'm2/s2', dynamics%ke_drag_xVEL(2,:), io_list(i)%freq, 'y', 8, partit, mesh) ! same as above but without multiplying with UMEAN (for later computation of turbulence fluxes) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_u', 'advection *dT [u]', 'm/s', dynamics%ke_adv(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_v', 'advection *dT [v]', 'm/s', dynamics%ke_adv(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_u', 'advection *dT [u]', 'm/s', dynamics%ke_adv(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_adv_v', 'advection *dT [v]', 'm/s', dynamics%ke_adv(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_u', 'Coriolis *dT [X]', 'm/s', dynamics%ke_cor(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_v', 'Coriolis *dT [Y]', 'm/s', dynamics%ke_cor(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_u', 'Coriolis *dT [X]', 'm/s', dynamics%ke_cor(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_cor_v', 'Coriolis *dT [Y]', 'm/s', dynamics%ke_cor(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_u', 'pressure gradient *dT [x]', 'm/s', dynamics%ke_pre(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_v', 'pressure gradient *dT [y]', 'm/s', dynamics%ke_pre(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_u', 'pressure gradient *dT [x]', 'm/s', dynamics%ke_pre(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_pre_v', 'pressure gradient *dT [y]', 'm/s', dynamics%ke_pre(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_u', 'hor. visc. [x] *dT', 'm/s', dynamics%ke_hvis(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_v', 'hor. visc. [y] *dT', 'm/s', dynamics%ke_hvis(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_u', 'hor. visc. [x] *dT', 'm/s', dynamics%ke_hvis(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_hvis_v', 'hor. visc. [y] *dT', 'm/s', dynamics%ke_hvis(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_u', 'ver. visc. [x] *dT', 'm/s', dynamics%ke_vvis(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_v', 'ver. visc. [y] *dT', 'm/s', dynamics%ke_vvis(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_u', 'ver. visc. [x] *dT', 'm/s', dynamics%ke_vvis(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_vvis_v', 'ver. visc. [y] *dT', 'm/s', dynamics%ke_vvis(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_Umean', 'mean U', 'm/s', dynamics%ke_umean(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_Vmean', 'mean V', 'm/s', dynamics%ke_umean(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_Umean', 'mean U', 'm/s', dynamics%ke_umean(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_Vmean', 'mean V', 'm/s', dynamics%ke_umean(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_U2mean', 'U2 mean', 'm/s', dynamics%ke_u2mean(1,:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_V2mean', 'V2 mean', 'm/s', dynamics%ke_u2mean(2,:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_U2mean', 'U2 mean', 'm/s', dynamics%ke_u2mean(1,:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'ke_V2mean', 'V2 mean', 'm/s', dynamics%ke_u2mean(2,:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'ke_dW', 'd/dz (vertical VEL)', 'm/s', dynamics%ke_dW(:,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'ke_PFULL', 'full Pressure', 'm/s', dynamics%ke_Pfull(:,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'ke_dW', 'd/dz (vertical VEL)', 'm/s', dynamics%ke_dW(:,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'ke_PFULL', 'full Pressure', 'm/s', dynamics%ke_Pfull(:,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_wind_x', 'wind [x] *dT', 'm/s', dynamics%ke_wind(1,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_wind_y', 'wind [y] *dT', 'm/s', dynamics%ke_wind(2,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_wind_x', 'wind [x] *dT', 'm/s', dynamics%ke_wind(1,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_wind_y', 'wind [y] *dT', 'm/s', dynamics%ke_wind(2,:), io_list(i)%freq, 'y', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_drag_x', 'drag [x] *dT', 'm/s', dynamics%ke_drag(1,:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(elem2D, myDim_elem2D, 'ke_drag_y', 'drag [y] *dT', 'm/s', dynamics%ke_drag(2,:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_drag_x', 'drag [x] *dT', 'm/s', dynamics%ke_drag(1,:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(elem2D, myDim_elem2D, 'ke_drag_y', 'drag [y] *dT', 'm/s', dynamics%ke_drag(2,:), io_list(i)%freq, 'y', 8, partit, mesh) ! surface fields for APE input computations... - call def_stream(nod2D , myDim_nod2D , 'ke_J', 'surface temperature flux [Js]','°C/s', dynamics%ke_J(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_G', 'surface salinity flux [Gs]','PSU/s', dynamics%ke_G(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_D', 'surface density', 'kg/m^3', dynamics%ke_D(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_D2', 'surface density squared', 'kg^2/m^6', dynamics%ke_D2(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_n0', 'dRHO/dz', 'kg/m^4', dynamics%ke_n0(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_JD', 'surface temperature flux [Js] * RHO','°C*kg/s/m^3', dynamics%ke_JD(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_GD', 'surface salinity flux [Gs] * RHO','PSU*kg/s/m^3', dynamics%ke_GD(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_swA', 'Thermal expansion coeff (alpha)', '1/°C', dynamics%ke_swA(:), io_list(i)%freq, 'm', 8, partit, mesh) - call def_stream(nod2D , myDim_nod2D , 'ke_swB', 'Taline contraction coeff (beta)', '1/PSU', dynamics%ke_swB(:), io_list(i)%freq, 'm', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_J', 'surface temperature flux [Js]','°C/s', dynamics%ke_J(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_G', 'surface salinity flux [Gs]','PSU/s', dynamics%ke_G(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_D', 'surface density', 'kg/m^3', dynamics%ke_D(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_D2', 'surface density squared', 'kg^2/m^6', dynamics%ke_D2(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_n0', 'dRHO/dz', 'kg/m^4', dynamics%ke_n0(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_JD', 'surface temperature flux [Js] * RHO','°C*kg/s/m^3', dynamics%ke_JD(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_GD', 'surface salinity flux [Gs] * RHO','PSU*kg/s/m^3', dynamics%ke_GD(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_swA', 'Thermal expansion coeff (alpha)', '1/°C', dynamics%ke_swA(:), io_list(i)%freq, 'y', 8, partit, mesh) + call def_stream(nod2D , myDim_nod2D , 'ke_swB', 'Taline contraction coeff (beta)', '1/PSU', dynamics%ke_swB(:), io_list(i)%freq, 'y', 8, partit, mesh) end if end subroutine diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 95c40e829..5d1d64778 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -3337,10 +3337,10 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ t0=MPI_Wtime() - water_flux = 0.0_WP - heat_flux = 0.0_WP - stress_surf= 0.0_WP - stress_node_surf= 0.0_WP +!PS water_flux = 0.0_WP +!PS heat_flux = 0.0_WP +!PS stress_surf= 0.0_WP +!PS stress_node_surf= 0.0_WP !___________________________________________________________________________ ! calculate equation of state, density, pressure and mixed layer depths @@ -3459,16 +3459,32 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) call compute_vel_rhs(ice, dynamics, partit, mesh) !___________________________________________________________________________ + ! Energy diagnostic contribution if (dynamics%ldiag_ke) then + ! if use solver + if (.not. dynamics%use_ssh_splitexpl_subcycl) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) + end do end do - end do !$OMP END PARALLEL DO + + ! if use splitexpl subcycl. UV_rhs in units of transport --> therefor *1/helem + else +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)/helem(nz,elem) + end do + end do +!$OMP END PARALLEL DO + end if end if !___________________________________________________________________________ @@ -3479,26 +3495,51 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) call viscosity_filter(dynamics%opt_visc, dynamics, partit, mesh) !___________________________________________________________________________ + ! Energy diagnostic contribution if (dynamics%ldiag_ke) then + ! if use solver + if (.not. dynamics%use_ssh_splitexpl_subcycl) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_hvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)-dynamics%ke_rhs_bak(:,nz,elem) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_hvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) - dynamics%ke_rhs_bak(:,nz,elem) + end do end do - end do !$OMP END PARALLEL DO - !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) + end do end do - end do !$OMP END PARALLEL DO + + ! if use splitexpl subcycl. UV_rhs in units of transport --> therefor *1/helem + else +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_hvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)/helem(nz,elem) - dynamics%ke_rhs_bak(:,nz,elem) + end do + end do +!$OMP END PARALLEL DO +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_rhs_bak(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)/helem(nz,elem) + end do + end do +!$OMP END PARALLEL DO + + end if end if !___________________________________________________________________________ @@ -3513,15 +3554,28 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ if (dynamics%ldiag_ke) then + ! if use solver + if (.not. dynamics%use_ssh_splitexpl_subcycl) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) - do elem=1, myDim_elem2D - nzmax = nlevels(elem) - nzmin = ulevels(elem) - do nz=nzmin,nzmax-1 - dynamics%ke_vvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)-dynamics%ke_rhs_bak(:,nz,elem) + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_vvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem) - dynamics%ke_rhs_bak(:,nz,elem) + end do end do - end do !$OMP END PARALLEL DO + + ! if use splitexpl subcycl. UV_rhs in units of transport --> therefor *1/helem + else + do elem=1, myDim_elem2D + nzmax = nlevels(elem) + nzmin = ulevels(elem) + do nz=nzmin,nzmax-1 + dynamics%ke_vvis(:,nz,elem)=dynamics%UV_rhs(:,nz,elem)/helem(nz,elem) - dynamics%ke_rhs_bak(:,nz,elem) + end do + end do + end if end if t2=MPI_Wtime() @@ -3588,14 +3642,17 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) else ! Compute vertical integral of transport velocity rhs omitting the contributions from ! the elevation and Coriolis. + t30=MPI_Wtime() call compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ! Do barotropic step, get eta_{n+1} and BT transport call compute_BT_step_SE_ale(dynamics, partit, mesh) + t3=MPI_Wtime() ! Trim U to be consistent with BT transport call update_trim_vel_ale_vtransp(1, dynamics, partit, mesh) - + t4=MPI_Wtime() + t5=t4 end if ! --> if (.not. dynamics%use_ssh_splitexpl_subcycl) then !___________________________________________________________________________ @@ -3672,7 +3729,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ ! write energy diagnostic info (dynamics%ldiag_ke = .true.) - if (dynamics%ldiag_ke) then + if ( (dynamics%ldiag_ke) .and. (mod(n,logfile_outfreq)==0) ) then call write_enegry_info(dynamics, partit, mesh) end if diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 94d8c5d8c..17d0e8a86 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -348,143 +348,6 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) end do ! --> do ed=1, myDim_edge2D !$OMP END DO - - - - - - - - -!PS !___________________________________________________________________________ -!PS ! 2nd. compute horizontal advection component: u*du/dx, u*dv/dx & v*du/dy, v*dv/dy -!PS ! loop over triangle edges -!PS !$OMP DO -!PS do ed=1, myDim_edge2D -!PS ! local indice of nodes that span up edge ed -!PS ednodes = edges(:,ed) -!PS -!PS ! local index of element that contribute to edge -!PS edelem = edge_tri(1:2,ed) -!PS -!PS !_______________________________________________________________________ -!PS ! index off surface layer in case of cavity !=1 and index of mid depth -!PS ! bottom layer -!PS ul1 = ulevels(edelem(1)) -!PS nl1 = nlevels(edelem(1))-1 -!PS x1 = edge_cross_dxdy(1,ed) -!PS y1 = edge_cross_dxdy(2,ed) -!PS ul2 = 0 -!PS nl2 = nl -!PS -!PS !_______________________________________________________________________ -!PS if (edelem(2) > 0) then -!PS ul2 = ulevels(edelem(2)) -!PS nl2 = nlevels(edelem(2))-1 -!PS x2 = edge_cross_dxdy(3,ed) -!PS y2 = edge_cross_dxdy(4,ed) -!PS end if -!PS -!PS !_______________________________________________________________________ -!PS ! nl12 ... minimum number of layers -1 between element edelem(1) & edelem(2) that -!PS ! contribute to edge ed -!PS ! nu12 ... upper index of layers between element edelem(1) & edelem(2) that -!PS ! contribute to edge ed -!PS ! be carefull !!! --> if ed is a boundary edge than edelem(1)~=0 and edelem(2)==0 -!PS ! that means nl1>0, nl2==0, nl12=min(nl1,nl2)=0 !!! -!PS ul12 = max(ul1, ul2) -!PS nl12 = min(nl1, nl2) -!PS -!PS !_______________________________________________________________________ -!PS ! (A1) goes only into this loop when the edge has only facing element -!PS ! edelem(1) --> so the edge is a boundary edge --> this is for ocean -!PS ! surface in case of cavity -!PS do nz=ul1, ul12-1 -!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) -!PS uu=un*UV(1, nz, edelem(1)) ! the momentum to be carried depends on velocities -!PS vv=un*UV(2, nz, edelem(1)) -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=ul1, ul12-1 -!PS -!PS !_______________________________________________________________________ -!PS ! (C1) remaining segments from the shared lower lyer index nl12 to bottom -!PS ! of element edelem(1) -!PS do nz=nl12+1, nl1 -!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) -!PS uu=un*UV(1, nz, edelem(1)) ! the momentum to be carried depends on velocities -!PS vv=un*UV(2, nz, edelem(1)) -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=nl12+1, nl1 -!PS -!PS if (edelem(2) > 0) then -!PS !_______________________________________________________________________ -!PS ! (A2) goes only into this loop when the edge has a facing elemenmt -!PS ! edelem(2) --> so the edge is a boundary edge --> this is for ocean -!PS ! surface in case of cavity -!PS do nz=ul2, ul12-1 -!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) -!PS uu=un*UV(1, nz, edelem(2)) -!PS vv=un*UV(2, nz, edelem(2)) -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=ul2, ul12-1 -!PS -!PS !_______________________________________________________________________ -!PS ! (B) Both segments -!PS ! loop over depth layers from shared upper layer index ul12 to shared -!PS ! lower layer index nl12 -!PS do nz=ul12, nl12 -!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) & -!PS -(UVh(2, nz, edelem(2))*x2 - UVh(1, nz, edelem(2))*y2) -!PS uu=un*(UV(1, nz, edelem(1)) + UV(1, nz, edelem(2)))*0.5_WP! the momentum to be carried depends on velocities -!PS vv=un*(UV(2, nz, edelem(1)) + UV(2, nz, edelem(2)))*0.5_WP -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=ul12, nl12 -!PS -!PS !_______________________________________________________________________ -!PS ! (C2) remaining segments from the shared lower lyer index nl12 to bottom -!PS ! of element edelem(1) -!PS do nz=nl12+1, nl2 -!PS un=-(UVh(2, nz, edelem(2))*x2- UVh(1, nz, edelem(2))*y2) -!PS uu=un*UV(1, nz, edelem(2)) -!PS vv=un*UV(2, nz, edelem(2)) -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=nl12+1, nl2 -!PS else -!PS -!PS !_______________________________________________________________________ -!PS ! (B) Both segments -!PS ! loop over depth layers from shared upper layer index ul12 to shared -!PS ! lower layer index nl12 -!PS do nz=ul12, nl12 -!PS un= (UVh(2, nz, edelem(1))*x1 - UVh(1, nz, edelem(1))*y1) -!PS uu=un*UV(1, nz, edelem(1))! the momentum to be carried depends on velocities -!PS vv=un*UV(2, nz, edelem(1)) -!PS UVnode_rhs(1, nz, ednodes(1))=UVnode_rhs(1, nz, ednodes(1))+uu -!PS UVnode_rhs(1, nz, ednodes(2))=UVnode_rhs(1, nz, ednodes(2))-uu -!PS UVnode_rhs(2, nz, ednodes(1))=UVnode_rhs(2, nz, ednodes(1))+vv -!PS UVnode_rhs(2, nz, ednodes(2))=UVnode_rhs(2, nz, ednodes(2))-vv -!PS end do ! --> do nz=ul12, nl12 -!PS end if -!PS -!PS end do ! --> do ed=1, myDim_edge2D -!PS !$OMP END DO - - !___________________________________________________________________________ ! divide total nodal momentum advection by scalar area !$OMP DO @@ -678,22 +541,7 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) ! therefor divide with helem uu(nzmin:nzmax-1)=(UVh(1, nzmin:nzmax-1, elem)+UV_rhs(1, nzmin:nzmax-1, elem))/helem(nzmin:nzmax-1, elem) !! u*=U*/h vv(nzmin:nzmax-1)=(UVh(2, nzmin:nzmax-1, elem)+UV_rhs(2, nzmin:nzmax-1, elem))/helem(nzmin:nzmax-1, elem) - - !_______________________________________________________________ -!PS if ( any(uu(nzmin:nzmax-1)/=uu(nzmin:nzmax-1)) .or. & -!PS any(vv(nzmin:nzmax-1)/=vv(nzmin:nzmax-1)) ) then -!PS write(*,*) ' --> subroutine impl_vert_visc_ale_vtransp --> found Nan in uu=UVh=UV_rhs' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' elem =', elem -!PS write(*,*) ' uu(nzmin:nzmax-1)=', uu(nzmin:nzmax-1) -!PS write(*,*) ' vv(nzmin:nzmax-1)=', vv(nzmin:nzmax-1) -!PS write(*,*) ' UV_rhs(1,nz,elem)=', UV_rhs( 1, nz, elem) -!PS write(*,*) ' UV_rhs(2,nz,elem)=', UV_rhs( 2, nz, elem) -!PS write(*,*) ' UVh( 1, nz, elem)=', UVh( 1, nz, elem) -!PS write(*,*) ' UVh( 2, nz, elem)=', UVh( 2, nz, elem) -!PS write(*,*) ' helem(nz, elem) =', helem(nz, elem) -!PS end if - + !_______________________________________________________________________ ! Operator + rhs ! Regular part of coefficients: @@ -822,6 +670,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) USE MOD_MESH USE MOD_DYN USE g_config, only: dt, r_restart + USE g_comm_auto IMPLICIT NONE type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit @@ -901,6 +750,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ! UVBT_4AB ... is U_bt in barotropic equation ! --> d/dt*U_bt + f*e_z x U_bt + g*H*grad_H(eta) = R_bt UVBT_4AB(3:4,elem)=UVBT_4AB(1:2,elem) + end do !$OMP END DO !$OMP END PARALLEL @@ -931,8 +781,10 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) USE MOD_PARSUP USE MOD_MESH USE MOD_DYN - use g_comm_auto - USE g_config, only: dt + USE g_comm_auto + USE g_config, only: dt, which_ALE, use_cavity_fw2press + USE g_support, only: integrate_nod + use o_ARRAYS, only: water_flux IMPLICIT NONE !___________________________________________________________________________ type(t_dyn) , intent(inout), target :: dynamics @@ -941,11 +793,10 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !___________________________________________________________________________ real(kind=WP) :: dtBT, BT_inv, hh, f, rx, ry, a, b, d, c1, c2, ax, ay real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, thetaBT - integer :: step, elem, elnodes(3), edge, ednodes(2), edelem(2) + integer :: step, elem, edge, node, elnodes(3), ednodes(2), edelem(2) real(kind=WP) :: update_ubt, update_vbt, vi, len, nzmax real(kind=WP), allocatable :: bottomdrag(:), UVBT_harmvisc(:,:) - !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:) , pointer :: eta_n @@ -979,7 +830,6 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) dtBT = dt/dynamics%splitexpl_BTsteps BT_inv = 1.0_WP/(1.0_WP*dynamics%splitexpl_BTsteps) - !___SPLIT-EXPLICITE STABILIZATION___________________________________________ ! trim R (UVBT_rhs): ! UVBT_rhs --> UVBT_rhs - div_h Ah H^n div_h(Ubt^n/H^n) + Cd*|Ubot|* Ubt^n/H^n @@ -989,7 +839,6 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! --> use only harmonmic viscosity operator applied to the barotropic ! velocity if (dynamics%splitexpl_visc) then - !_______________________________________________________________________ ! remove viscosity do edge=1, myDim_edge2D+eDim_edge2D @@ -1033,9 +882,6 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do ! --> do edge=1, myDim_edge2D+eDim_edge2D - - call exchange_elem_begin(UVBT_rhs, partit) - call exchange_elem_end(partit) end if ! --> if (dynamics%splitexpl_visc) then !___________________________________________________________________________ @@ -1072,6 +918,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! elem indices that participate in edge edelem = edge_tri(:,edge) + nzmax = minval(nlevels(edelem)) hh = -zbar(nzmax) len = sqrt(sum(elem_area(edelem))) @@ -1079,7 +926,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh vi=update_ubt*update_ubt + update_vbt*update_vbt - vi=dt*sqrt(max(dynamics%splitexpl_visc_gamma0, & + vi=dt*sqrt(max(dynamics%splitexpl_visc_gamma0, & max(dynamics%splitexpl_visc_gamma1*sqrt(vi), & dynamics%splitexpl_visc_gamma2*vi) & )*len) @@ -1106,9 +953,6 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do ! --> do edge=1, myDim_edge2D+eDim_edge2D - - call exchange_elem_begin(UVBT_harmvisc, partit) - call exchange_elem_end(partit) end if ! -> if (dynamics%splitexpl_visc) then !_______________________________________________________________________ @@ -1137,13 +981,13 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! (We do it here based on increments since it saves us some significant ! digits for the accuracy) ! Increments: - ! deltaU = Ubt^(n+(m+1)/M)-Ubt^(n+m/M) - ! Ubt^(n+(m+1)/M)+Ubt^(n+m/M) = Ubt^(n+(m+1)/M)-Ubt^(n+m/M)+2*Ubt^(n+m/M) - ! = deltaU + 2*Ubt^(n+m/M) + ! deltaU = Ubt^(n+(m+1)/M)-Ubt^(n+m/M) + ! Ubt^(n+(m+1)/M)+Ubt^(n+m/M) = Ubt^(n+(m+1)/M)-Ubt^(n+m/M)+2*Ubt^(n+m/M) + ! = deltaU + 2*Ubt^(n+m/M) ! - ! Ubt^(n+(m+1)/M)-Ubt^(n+m/M) = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+m/M)) - ! - h*H^m*grad_H*eta^(n+m/M) - ! - Rbt-->UVBT_rhs ] + ! Ubt^(n+(m+1)/M)-Ubt^(n+m/M) = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+m/M)) + ! - h*H^m*grad_H*eta^(n+m/M) + ! - Rbt-->UVBT_rhs ] ! ! deltaU - dt/(2*M)*f*deltaV = dt/M*f*Vbt^(n+m/M) - h*H^m*gradx_H*eta^(n+m/M) + Rbtx ! deltaV + dt/(2*M)*f*deltaU = -dt/M*f*Ubt^(n+m/M) - h*H^m*grady_H*eta^(n+m/M) + Rbty @@ -1178,6 +1022,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! Ubt^(n+(m+1)/M) = Ubt^(n+(m)/M) + AAA ! equation (6) in T. Banerjee et al.,Split-Explicite external ! mode solver in FESOM2, + + ! compute barotropic velocity at time step (n+(m+1)/M) UVBT( 1, elem) = UVBT(1, elem) + ax UVBT( 2, elem) = UVBT(2, elem) + ay @@ -1199,6 +1045,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) UVBT_mean( 2, elem) = UVBT_mean( 2, elem) + UVBT_theta(2, elem)*BT_inv end do + call exchange_elem_begin(UVBT, partit) + call exchange_elem_end(partit) !_______________________________________________________________________ ! Store mid-step velocity (to trim 3D velocities in momentum) @@ -1230,18 +1078,28 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! advance ssh --> eta^(n+(m+1)/M) = eta^(n+(m)/M) - dt/M * div_H * [...] ! equation (6) in T. Banerjee et al.,Split-Explicite external ! mode solver in FESOM2, - eta_n(ednodes(1))=eta_n(ednodes(1)) + (c1+c2)*dtBT/area(1,ednodes(1)) - eta_n(ednodes(2))=eta_n(ednodes(2)) - (c1+c2)*dtBT/area(1,ednodes(2)) + eta_n(ednodes(1))=eta_n(ednodes(1)) + (c1+c2)*dtBT/areasvol(1,ednodes(1)) + eta_n(ednodes(2))=eta_n(ednodes(2)) - (c1+c2)*dtBT/areasvol(1,ednodes(2)) end do + !_______________________________________________________________________ + ! Apply freshwater boundary condition + if ( .not. trim(which_ALE)=='linfs') then + do node=1,myDim_nod2D + eta_n(node)=eta_n(node) - dtBT*water_flux(node) + end do + end if + !_______________________________________________________________________ call exchange_nod(eta_n, partit) end do ! --> do step=1, dynamics%splitexpl_BTsteps + deallocate(bottomdrag, UVBT_harmvisc) hbar_old = hbar - hbar = eta_n + hbar = eta_n + end subroutine compute_BT_step_SE_ale ! ! @@ -1261,6 +1119,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) type(t_mesh) , intent(inout) , target :: mesh integer :: elem, nz, nzmin, nzmax real(kind=WP) :: ubar, vbar, hh_inv + real(kind=WP) :: usum(2), udiff(2) !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:,:,:), pointer :: UVh, UV, UV_rhs @@ -1287,21 +1146,77 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) nzmin = ulevels(elem) nzmax = nlevels(elem)-1 + !___________________________________________________________________ ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP do nz=nzmin, nzmax - !_______________________________________________________________ - ! Update transport velocities: U^(n+1/2,**) = U^(n+1/2,*) + U_rhs - UVh(1, nz, elem)=UVh(1, nz, elem)+UV_rhs(1, nz, elem) - UVh(2, nz, elem)=UVh(2, nz, elem)+UV_rhs(2, nz, elem) - + !PS !___________________________________________________________ + !PS ! Update transport velocities: U^(n+1/2,**) = U^(n+1/2,*) + U_rhs + !PS UVh(1, nz, elem)=UVh(1, nz, elem)+UV_rhs(1, nz, elem) + !PS UVh(2, nz, elem)=UVh(2, nz, elem)+UV_rhs(2, nz, elem) + !PS + !PS !___________________________________________________________ + !PS ! vertically integrate updated transport velocity: sum(k, U_k^(n+1/2,**) ) + !PS ubar = ubar+UVh(1, nz, elem) + !PS vbar = vbar+UVh(2, nz, elem) + !PS hh_inv= hh_inv+helem(nz,elem) !_______________________________________________________________ ! vertically integrate updated transport velocity: sum(k, U_k^(n+1/2,**) ) - ubar = ubar+UVh(1, nz, elem) - vbar = vbar+UVh(2, nz, elem) + ! --> the actual update of the transport velocity is done after the + ! if (dynamics%ldiag_ke) block + ubar = ubar+UVh(1, nz, elem) + UV_rhs(1, nz, elem) + vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) hh_inv= hh_inv+helem(nz,elem) end do + hh_inv=1.0_WP/hh_inv + + !___________________________________________________________________ + if (dynamics%ldiag_ke) then + DO nz=nzmin, nzmax + ! U_(n+1) - U_n = Urhs_n |* (U_(n+1)+U_n) + ! U_(n+1)^2 - U_n^2 = Urhs_n * (U_(n+1)+U_n) + ! | + ! +-> U_(n+1) = U_n+Urhs_n + ! U_(n+1)^2 - U_n^2 = Urhs_n * (2*U_n + Urhs) + ! | | + ! v v + ! udiff usum + usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + ! transform: transport vel --> velocity + usum(1) = usum(1)/helem(nz,elem) + usum(2) = usum(2)/helem(nz,elem) + + udiff(1) = UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + udiff(2) = UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + ! transform: transport vel --> velocity + udiff(1) = udiff(1)/helem(nz,elem) + udiff(2) = udiff(2)/helem(nz,elem) + + ! (U_(n+1)^2 - U_n^2)/2 = usum*udiff/2 + dynamics%ke_du2( :,nz,elem) = usum*udiff/2.0_WP + + dynamics%ke_pre_xVEL( :,nz,elem) = usum*dynamics%ke_pre (:,nz,elem)/2.0_WP + dynamics%ke_adv_xVEL( :,nz,elem) = usum*dynamics%ke_adv (:,nz,elem)/2.0_WP + dynamics%ke_cor_xVEL( :,nz,elem) = usum*dynamics%ke_cor (:,nz,elem)/2.0_WP + dynamics%ke_hvis_xVEL(:,nz,elem) = usum*dynamics%ke_hvis(:,nz,elem)/2.0_WP + dynamics%ke_vvis_xVEL(:,nz,elem) = usum*dynamics%ke_vvis(:,nz,elem)/2.0_WP + + ! U_(n+0.5) = U_n + 0.5*Urhs + dynamics%ke_umean( :,nz,elem) = usum/2.0_WP + ! U_(n+0.5)^2 + dynamics%ke_u2mean( :,nz,elem) = (usum*usum)/4.0_WP + + if (nz==nzmin) then + dynamics%ke_wind_xVEL(:,elem) = usum*dynamics%ke_wind(:,elem)/2.0_WP + end if + + if (nz==nzmax) then + dynamics%ke_drag_xVEL(:,elem) = usum*dynamics%ke_drag(:,elem)/2.0_WP + end if + END DO + end if !___________________________________________________________________ ! finalize horizontal transport by making vertically integrated @@ -1311,15 +1226,17 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! mode solver in FESOM2, ! U_k^(n+1/2) = U^(n+1/2,**) ! + [<>^(n+1/2) - sum(k, U_k^(n+1/2,**)] * h_k^(n+1/2)/sum(k,h_k^(n+1/2)) - hh_inv=1.0_WP/hh_inv ! Postpone AB and 2nd order, just use available thickness DO nz=nzmin, nzmax - UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv - UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + !PS UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + !PS UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) - end do end do + + !_______________________________________________________________________ call exchange_elem(UVh, partit) ! This exchange can be avoided, but test first. call exchange_elem(UV, partit) @@ -1328,18 +1245,20 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! Velocity will be used to advance momentum else do elem=1, myDim_elem2D + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + + !___________________________________________________________________ ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - - nzmin = ulevels(elem) - nzmax = nlevels(elem)-1 do nz=nzmin, nzmax ubar=ubar+UVh(1, nz, elem) vbar=vbar+UVh(2, nz, elem) hh_inv=hh_inv+helem(nz,elem) end do + !___________________________________________________________________ hh_inv=1.0_WP/hh_inv ! Postpone 2nd order, just use available thickness do nz=nzmin, nzmax UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_12(1, elem)-ubar)*helem(nz,elem)*hh_inv @@ -1348,8 +1267,10 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do end do + !___________________________________________________________________ call exchange_elem(UVh, partit) ! call exchange_elem(UV , partit) ! Check if this is needed + end if ! --> if (mode==1) then end subroutine update_trim_vel_ale_vtransp ! @@ -1396,7 +1317,7 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) nzmin = ulevels(elem) nzmax = nlevels(elem)-1 elnodes=elem2D_nodes(:,elem) - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP end do end do diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index 92faffb55..57ac24fe7 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -99,32 +99,46 @@ SUBROUTINE update_vel(dynamics, partit, mesh) nzmax = nlevels(elem) if (dynamics%ldiag_ke) then - DO nz=nzmin, nzmax-1 - dynamics%ke_pre(1,nz,elem) =dynamics%ke_pre(1,nz,elem) + Fx - dynamics%ke_pre(2,nz,elem) =dynamics%ke_pre(2,nz,elem) + Fy - - usum(1)=2.0_WP*UV(1,nz,elem)+(UV_rhs(1,nz,elem) + Fx) - usum(2)=2.0_WP*UV(2,nz,elem)+(UV_rhs(2,nz,elem) + Fy) - - udiff(1)=UV_rhs(1,nz,elem) + Fx - udiff(2)=UV_rhs(2,nz,elem) + Fy - dynamics%ke_du2 (:,nz,elem) = usum*udiff/2.0_WP - dynamics%ke_pre_xVEL (:,nz,elem) = usum*dynamics%ke_pre (:,nz,elem)/2.0_WP - dynamics%ke_adv_xVEL (:,nz,elem) = usum*dynamics%ke_adv (:,nz,elem)/2.0_WP - dynamics%ke_cor_xVEL (:,nz,elem) = usum*dynamics%ke_cor (:,nz,elem)/2.0_WP - dynamics%ke_hvis_xVEL(:,nz,elem) = usum*dynamics%ke_hvis(:,nz,elem)/2.0_WP - dynamics%ke_vvis_xVEL(:,nz,elem) = usum*dynamics%ke_vvis(:,nz,elem)/2.0_WP - dynamics%ke_umean(:,nz,elem) = usum/2.0_WP - dynamics%ke_u2mean(:,nz,elem) = (usum*usum)/4.0_WP - - if (nz==nzmin) then - dynamics%ke_wind_xVEL(:,elem)=usum*dynamics%ke_wind(:,elem)/2.0_WP - end if - - if (nz==nzmax-1) then - dynamics%ke_drag_xVEL(:,elem)=usum*dynamics%ke_drag(:,elem)/2.0_WP - end if - END DO + DO nz=nzmin, nzmax-1 + dynamics%ke_pre(1,nz,elem) =dynamics%ke_pre(1,nz,elem) + Fx + dynamics%ke_pre(2,nz,elem) =dynamics%ke_pre(2,nz,elem) + Fy + + + ! U_(n+1) - U_n = Urhs_n |* (U_(n+1)+U_n) + ! U_(n+1)^2 - U_n^2 = Urhs_n * (U_(n+1)+U_n) + ! | + ! +-> U_(n+1) = U_n+Urhs_n + ! U_(n+1)^2 - U_n^2 = Urhs_n * (2*U_n + Urhs) + ! | | + ! v v + ! udiff usum + usum(1) = 2.0_WP*UV(1,nz,elem)+(UV_rhs(1,nz,elem) + Fx) + usum(2) = 2.0_WP*UV(2,nz,elem)+(UV_rhs(2,nz,elem) + Fy) + udiff(1) = UV_rhs(1,nz,elem) + Fx + udiff(2) = UV_rhs(2,nz,elem) + Fy + + ! (U_(n+1)^2 - U_n^2)/2 = usum*udiff/2 + dynamics%ke_du2 (:,nz,elem) = usum*udiff/2.0_WP + + dynamics%ke_pre_xVEL (:,nz,elem) = usum*dynamics%ke_pre (:,nz,elem)/2.0_WP + dynamics%ke_adv_xVEL (:,nz,elem) = usum*dynamics%ke_adv (:,nz,elem)/2.0_WP + dynamics%ke_cor_xVEL (:,nz,elem) = usum*dynamics%ke_cor (:,nz,elem)/2.0_WP + dynamics%ke_hvis_xVEL(:,nz,elem) = usum*dynamics%ke_hvis(:,nz,elem)/2.0_WP + dynamics%ke_vvis_xVEL(:,nz,elem) = usum*dynamics%ke_vvis(:,nz,elem)/2.0_WP + + ! U_(n+0.5) = U_n + 0.5*Urhs + dynamics%ke_umean( :,nz,elem) = usum/2.0_WP + ! U_(n+0.5)^2 + dynamics%ke_u2mean( :,nz,elem) = (usum*usum)/4.0_WP + + if (nz==nzmin) then + dynamics%ke_wind_xVEL(:,elem) = usum*dynamics%ke_wind(:,elem)/2.0_WP + end if + + if (nz==nzmax-1) then + dynamics%ke_drag_xVEL(:,elem) = usum*dynamics%ke_drag(:,elem)/2.0_WP + end if + END DO end if DO nz=nzmin, nzmax-1 From 2bcb58b74e23e932dad9611a9e05e71a854cd239 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 14 Sep 2023 16:29:13 +0200 Subject: [PATCH 29/50] make loops OMP conform, reduce allocation of bottomdrag and UVBT_harmvisc array --- src/MOD_DYN.F90 | 9 ++ src/oce_ale_ssh_splitexpl_subcycl.F90 | 191 ++++++++++++++++++-------- src/oce_setup_step.F90 | 8 ++ 3 files changed, 153 insertions(+), 55 deletions(-) diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index 1345527c3..218f097ae 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -75,6 +75,11 @@ MODULE MOD_DYN ! UBTmean_mean... Mean BT velocity to trim 3D velocity in tracers real(kind=WP), allocatable, dimension(:,:) :: se_uvBT, se_uvBT_theta, se_uvBT_mean, se_uvBT_12 + ! array that are needed for viscosity and bottomdrag stabilization of + ! split-expl subcycling method + real(kind=WP), allocatable, dimension(:,:) :: se_uvBT_stab_hvisc + real(kind=WP), allocatable, dimension(:) :: se_uvBT_stab_bdrag + !___________________________________________________________________________ ! summarizes solver input parameter type(t_solverinfo) :: solverinfo @@ -290,6 +295,8 @@ subroutine WRITE_T_DYN(dynamics, unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_12 , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT_stab_hvisc , unit, iostat, iomsg) + call write_bin_array(dynamics%se_uvBT_stab_bdrag , unit, iostat, iomsg) end if @@ -346,6 +353,8 @@ subroutine READ_T_DYN(dynamics, unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_theta, unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_mean , unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_12 , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT_stab_hvisc , unit, iostat, iomsg) + call read_bin_array(dynamics%se_uvBT_stab_bdrag , unit, iostat, iomsg) end if end subroutine READ_T_DYN diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 17d0e8a86..0c069ffae 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -90,7 +90,7 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) real(kind=WP) :: uv12, uv1, uv2, qc, qu, qd, num_ord=0.95_WP integer :: ednodes(2), edelem(2) real(kind=WP) :: wu(mesh%nl), wv(mesh%nl), un1(mesh%nl), un2(mesh%nl) -!PS real(kind=WP) :: un, uu, vv, x1, x2, y1, y2 + !___________________________________________________________________________ ! pointer on necessary derived types real(kind=WP), dimension(:,:,:), pointer :: UV, UV_rhsAB, UVnode_rhs, UVnode, UVh @@ -109,7 +109,7 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) ! 1st. compute vertical momentum advection component: w * du/dz, w*dv/dz !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(node, elem, ed, nz, nl1, ul1, nl2, ul2, nl12, ul12, & !$OMP uv1, uv2, uv12, qc, qu, qd, wu, wv, & -!$OMP ednodes, edelem, dx1, dy1, dx2, dy2, un, uu, vv) +!$OMP ednodes, edelem, un1, un2) !$OMP DO do node=1, myDim_nod2D @@ -522,9 +522,9 @@ subroutine impl_vert_visc_ale_vtransp(dynamics, partit, mesh) UVh =>dynamics%se_uvh(:,:,:) !___________________________________________________________________________ -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(a, b, c, ur, vr, cp, up, vp, elem, & -!$OMP nz, nzmin, nzmax, elnodes, & -!$OMP zinv, m, friction, wu, wv, uu, vv) +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(nz, nzmin, nzmax, elem, elnodes, & +!$OMP a, b, c, m, ur, vr, cp, up, vp, & +!$OMP zinv, friction, wu, wd, uu, vv) !___________________________________________________________________________ !$OMP DO @@ -704,7 +704,8 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ab2=1.0_WP-ab1 !___________________________________________________________________________ -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, Fx, Fy, & +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax, & +!$OMP Fx, Fy, hh, & !$OMP vert_sum_u, vert_sum_v, hh) !___________________________________________________________________________ @@ -791,16 +792,15 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(inout) , target :: mesh !___________________________________________________________________________ - real(kind=WP) :: dtBT, BT_inv, hh, f, rx, ry, a, b, d, c1, c2, ax, ay + real(kind=WP) :: dtBT, BT_inv, hh, ff, rx, ry, a, b, d, c1, c2, ax, ay real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, thetaBT integer :: step, elem, edge, node, elnodes(3), ednodes(2), edelem(2) real(kind=WP) :: update_ubt, update_vbt, vi, len, nzmax - real(kind=WP), allocatable :: bottomdrag(:), UVBT_harmvisc(:,:) !___________________________________________________________________________ ! pointer on necessary derived types - real(kind=WP), dimension(:) , pointer :: eta_n - real(kind=WP), dimension(:,:) , pointer :: UVBT_rhs, UVBT, UVBT_theta, UVBT_mean, UVBT_12 + real(kind=WP), dimension(:) , pointer :: eta_n, bottomdrag + real(kind=WP), dimension(:,:) , pointer :: UVBT_rhs, UVBT, UVBT_theta, UVBT_mean, UVBT_12, UVBT_harmvisc real(kind=WP), dimension(:,:,:), pointer :: UV #include "associate_part_def.h" #include "associate_mesh_def.h" @@ -813,13 +813,12 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) UVBT_mean =>dynamics%se_uvBT_mean(:,:) UVBT_12 =>dynamics%se_uvBT_12(:,:) if (dynamics%splitexpl_bottdrag) then - UV =>dynamics%uv(:,:,:) + UV =>dynamics%uv(:,:,:) + bottomdrag =>dynamics%se_uvBT_stab_bdrag(:) + end if + if (dynamics%splitexpl_visc) then + UVBT_harmvisc=>dynamics%se_uvBT_stab_hvisc(:,:) end if - - allocate(bottomdrag( myDim_elem2D+eDim_elem2D)) - allocate(UVBT_harmvisc(2, myDim_elem2D+eDim_elem2D)) - bottomdrag = 0.0_WP - UVBT_harmvisc = 0.0_WP !___________________________________________________________________________ ! Dissipation parameter of FB dissipative method 0.14 is the default value @@ -841,6 +840,9 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) if (dynamics%splitexpl_visc) then !_______________________________________________________________________ ! remove viscosity +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, nzmax, hh, len, & +!$OMP vi, update_ubt, update_vbt) +!$OMP DO do edge=1, myDim_edge2D+eDim_edge2D ! if ed is an outer boundary edge, skip it @@ -882,26 +884,37 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do ! --> do edge=1, myDim_edge2D+eDim_edge2D +!$OMP END DO end if ! --> if (dynamics%splitexpl_visc) then !___________________________________________________________________________ ! remove bottom drag if (dynamics%splitexpl_bottdrag) then +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nzmax, hh) +!$OMP DO do elem=1, myDim_elem2D elnodes= elem2D_nodes(:,elem) nzmax = nlevels(elem) -!PS hh = -zbar(nzmax)+sum(eta_n(elnodes))/3.0_WP + !PS hh = -zbar(nzmax)+sum(eta_n(elnodes))/3.0_WP hh = -zbar(nzmax) bottomdrag(elem) = dt*C_d*sqrt(UV(1, nzmax-1, elem)**2 + UV(2, nzmax-1, elem)**2) UVBT_rhs(1, elem)=UVBT_rhs(1, elem) + bottomdrag(elem)*UVBT(1, elem)/hh UVBT_rhs(2, elem)=UVBT_rhs(2, elem) + bottomdrag(elem)*UVBT(2, elem)/hh end do +!$OMP END PARALLEL DO end if ! --> if (dynamics%splitexpl_bottdrag) then + !___________________________________________________________________________ + ! initialise UVBT_mean with zeros --> OMP style +!$OMP PARALLEL DO + do elem=1, myDim_elem2D+eDim_elem2D + UVBT_mean(:, elem) = 0.0_WP + end do +!$OMP END PARALLEL DO + !___________________________________________________________________________ ! eta_n elevation used in BT stepping, it is just a copy of eta_n ! UBT and VBT are transport velocities - UVBT_mean= 0.0_WP do step=1, dynamics%splitexpl_BTsteps !####################################################################### !########## Dissipative forward--backward time stepping ########## @@ -910,7 +923,19 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !_______________________________________________________________________ ! compute harmonic viscosity for stability if (dynamics%splitexpl_visc) then - UVBT_harmvisc = 0.0_WP + + !___________________________________________________________________ + ! initialise UVBT_harmvisc with zeros --> OMP style +!$OMP PARALLEL DO + do elem=1, myDim_elem2D+eDim_elem2D + UVBT_harmvisc(:, elem) = 0.0_WP + end do +!$OMP END PARALLEL DO + + !___________________________________________________________________ +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, nzmax, hh, len, & +!$OMP vi, update_ubt, update_vbt) +!$OMP DO do edge=1, myDim_edge2D+eDim_edge2D ! if ed is an outer boundary edge, skip it @@ -953,10 +978,14 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do ! --> do edge=1, myDim_edge2D+eDim_edge2D +!$OMP END PARALLEL DO end if ! -> if (dynamics%splitexpl_visc) then !_______________________________________________________________________ ! Advance velocities. I use SI stepping for the Coriolis +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, hh, ff, rx, ry, a, b, & +!$OMP d, ax, ay) +!$OMP DO do elem=1, myDim_elem2D elnodes=elem2D_nodes(:,elem) !___________________________________________________________________ @@ -964,15 +993,15 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! AAA = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+(m)/M)) ! - h*H^m*grad_H*eta^((n+m)/M) ! - Rbt-->UVBT_rhs ] -!PS hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth + !PS hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth hh = -zbar(nlevels(elem)) ! Total fluid depth - f = mesh%coriolis(elem) + ff = mesh%coriolis(elem) - rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + f*UVBT(2, elem)) & + rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + ff*UVBT(2, elem)) & + BT_inv*UVBT_rhs(1, elem) & + BT_inv*(UVBT_harmvisc(1, elem) - bottomdrag(elem)*UVBT(1, elem)/hh) ! <-- stabilization terms - ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - f*UVBT(1, elem)) & + ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - ff*UVBT(1, elem)) & + BT_inv*UVBT_rhs(2, elem) & + BT_inv*(UVBT_harmvisc(2, elem) - bottomdrag(elem)*UVBT(2, elem)/hh) ! <-- stabilization terms @@ -988,26 +1017,35 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! Ubt^(n+(m+1)/M)-Ubt^(n+m/M) = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+m/M)) ! - h*H^m*grad_H*eta^(n+m/M) ! - Rbt-->UVBT_rhs ] + ! + dt/M*[grad_h*A_h*H^n*grad_h(Ubt^(n+m/M)/H^n)] + ! - dt/M*[cd*|Ubot|*Ubt^(n+(m+1)/M)/H^n] + ! + ! --> a = dt/(2*M)*ff + ! --> b = dt/M*cd*|Ubot|/H^n ! - ! deltaU - dt/(2*M)*f*deltaV = dt/M*f*Vbt^(n+m/M) - h*H^m*gradx_H*eta^(n+m/M) + Rbtx - ! deltaV + dt/(2*M)*f*deltaU = -dt/M*f*Ubt^(n+m/M) - h*H^m*grady_H*eta^(n+m/M) + Rbty + ! deltaU + b*deltaU - a*deltaV = dt/M*f*Vbt^(n+m/M) - h*H^m*gradx_H*eta^(n+m/M) + Rbtx + ! + dt/M*[grad_h*A_h*H^n*grad_h(Ubt^(n+m/M)/H^n)] + ! + dt/M*[cd*|Ubot|*Ubt^(n+m/M)/H^n] + ! deltaV + b*deltaV + a*deltaU = -dt/M*f*Ubt^(n+m/M) - h*H^m*grady_H*eta^(n+m/M) + Rbty + ! + dt/M*[grad_h*A_h*H^n*grad_h(Vbt^(n+m/M)/H^n)] + ! + dt/M*[cd*|Ubot|VUbt^(n+m/M)/H^n] ! \________________________v___________________________/ - ! --> a = dt/(2*M)*f Rx, Ry + ! Rx, Ry ! - ! | 1 -a | * | deltaU | = MAT* | deltaU | = | Rx | - ! | a 1 | | deltaV | = | deltaV | = | Ry | - ! - ! --> d = 1/(1 + a^2) ; inv(MAT) = d* | 1 a | - ! | -a 1 | + ! | 1+b -a | * | deltaU | = MAT* | deltaU | = | Rx | + ! | a 1+b | | deltaV | = | deltaV | = | Ry | + ! --> b' = b+1 + ! --> d = 1/(b'^2 + a^2) ; inv(MAT) = d* | b' a | + ! | -a b' | ! ! | deltaU | = inv(MAT) * | Rx | ! | deltaV | = | Ry | ! - ! Ubt^(n+(m+1)/M) = Ubt^(n+m/M) + d*( Rx + a*Ry) - ! Vbt^(n+(m+1)/M) = Vbt^(n+m/M) + d*(-a*Rx + Ry) + ! Ubt^(n+(m+1)/M) = Ubt^(n+m/M) + d*( b'*Rx + a *Ry) + ! Vbt^(n+(m+1)/M) = Vbt^(n+m/M) + d*(-a *Rx + b'*Ry) ! ! Semi-Implicit Coriolis - a = dtBT*f*0.5_WP + a = dtBT*ff*0.5_WP if (dynamics%splitexpl_bdrag_si) then b = 1.0_WP+BT_inv*bottomdrag(elem)/hh else @@ -1022,7 +1060,6 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! Ubt^(n+(m+1)/M) = Ubt^(n+(m)/M) + AAA ! equation (6) in T. Banerjee et al.,Split-Explicite external ! mode solver in FESOM2, - ! compute barotropic velocity at time step (n+(m+1)/M) UVBT( 1, elem) = UVBT(1, elem) + ax UVBT( 2, elem) = UVBT(2, elem) + ay @@ -1045,9 +1082,15 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) UVBT_mean( 2, elem) = UVBT_mean( 2, elem) + UVBT_theta(2, elem)*BT_inv end do +!$OMP END PARALLEL DO + + !_______________________________________________________________________ +!$OMP MASTER call exchange_elem_begin(UVBT, partit) call exchange_elem_end(partit) - +!$OMP END MASTER +!$OMP BARRIER + !_______________________________________________________________________ ! Store mid-step velocity (to trim 3D velocities in momentum) if(step==dynamics%splitexpl_BTsteps/2) then @@ -1058,6 +1101,9 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! Advance thickness ! compute: dt/M * div_H * [(1+theta)*Ubt^(n+(m+1)/M) - theta*Ubt^(n+(m)/M)] ! and advance ssh --> eta^(n+(m+1)/M) = eta^(n+(m)/M) - dt/M * div_H * [...] +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, ednodes, edelem, c1, c2, & +!$OMP deltaX1, deltaX2, deltaY1, deltaY2) +!$OMP DO do edge=1, myDim_edge2D ednodes = edges(:,edge) edelem = edge_tri(:,edge) @@ -1078,28 +1124,46 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! advance ssh --> eta^(n+(m+1)/M) = eta^(n+(m)/M) - dt/M * div_H * [...] ! equation (6) in T. Banerjee et al.,Split-Explicite external ! mode solver in FESOM2, +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock(partit%plock(ednodes(1))) +#else +!$OMP ORDERED +#endif eta_n(ednodes(1))=eta_n(ednodes(1)) + (c1+c2)*dtBT/areasvol(1,ednodes(1)) +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(1))) + call omp_set_lock (partit%plock(ednodes(2))) +#endif eta_n(ednodes(2))=eta_n(ednodes(2)) - (c1+c2)*dtBT/areasvol(1,ednodes(2)) - +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(2))) +#else +!$OMP END ORDERED +#endif end do - +!$OMP END PARALLEL DO + !_______________________________________________________________________ ! Apply freshwater boundary condition if ( .not. trim(which_ALE)=='linfs') then +!$OMP PARALLEL DO do node=1,myDim_nod2D eta_n(node)=eta_n(node) - dtBT*water_flux(node) end do +!$OMP END PARALLEL DO end if !_______________________________________________________________________ +!$OMP MASTER call exchange_nod(eta_n, partit) - +!$OMP END MASTER +!$OMP BARRIER + end do ! --> do step=1, dynamics%splitexpl_BTsteps - deallocate(bottomdrag, UVBT_harmvisc) + !___________________________________________________________________________ hbar_old = hbar hbar = eta_n - end subroutine compute_BT_step_SE_ale ! ! @@ -1141,7 +1205,9 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) !___________________________________________________________________________ ! Trim full velocity to ensure that its vertical sum equals to the mean BT velocity ! The trimmed Uh,Vh are consistent with new total height defined by eta_n - if (mode==1) then + if (mode==1) then +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) +!$OMP DO do elem=1, myDim_elem2D nzmin = ulevels(elem) nzmax = nlevels(elem)-1 @@ -1168,12 +1234,12 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ubar = ubar+UVh(1, nz, elem) + UV_rhs(1, nz, elem) vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) hh_inv= hh_inv+helem(nz,elem) - end do + end do ! --> do nz=nzmin, nzmax hh_inv=1.0_WP/hh_inv - + !___________________________________________________________________ if (dynamics%ldiag_ke) then - DO nz=nzmin, nzmax + do nz=nzmin, nzmax ! U_(n+1) - U_n = Urhs_n |* (U_(n+1)+U_n) ! U_(n+1)^2 - U_n^2 = Urhs_n * (U_(n+1)+U_n) ! | @@ -1215,8 +1281,8 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) if (nz==nzmax) then dynamics%ke_drag_xVEL(:,elem) = usum*dynamics%ke_drag(:,elem)/2.0_WP end if - END DO - end if + end do ! --> do nz=nzmin, nzmax + end if ! --> if (dynamics%ldiag_ke) then !___________________________________________________________________ ! finalize horizontal transport by making vertically integrated @@ -1226,24 +1292,30 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! mode solver in FESOM2, ! U_k^(n+1/2) = U^(n+1/2,**) ! + [<>^(n+1/2) - sum(k, U_k^(n+1/2,**)] * h_k^(n+1/2)/sum(k,h_k^(n+1/2)) - DO nz=nzmin, nzmax + do nz=nzmin, nzmax !PS UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv !PS UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) - end do - end do + end do ! --> do nz=nzmin, nzmax + end do ! --> do elem=1, myDim_elem2D +!$OMP END PARALLEL DO !_______________________________________________________________________ +!$OMP MASTER call exchange_elem(UVh, partit) ! This exchange can be avoided, but test first. call exchange_elem(UV, partit) +!$OMP END MASTER +!$OMP BARRIER !___________________________________________________________________________ ! Trim to make velocity consistent with BT velocity at n+1/2 ! Velocity will be used to advance momentum - else + else +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) +!$OMP DO do elem=1, myDim_elem2D nzmin = ulevels(elem) nzmax = nlevels(elem)-1 @@ -1267,10 +1339,14 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do end do - !___________________________________________________________________ +!$OMP END PARALLEL DO + + !_______________________________________________________________________ +!$OMP MASTER call exchange_elem(UVh, partit) ! call exchange_elem(UV , partit) ! Check if this is needed - +!$OMP END MASTER +!$OMP BARRIER end if ! --> if (mode==1) then end subroutine update_trim_vel_ale_vtransp ! @@ -1302,6 +1378,8 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) !___________________________________________________________________________ ! leave bottom layer again untouched +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax, hh_inv) +!$OMP DO do node=1, myDim_nod2D+eDim_nod2D nzmin = ulevels_nod2D(node) nzmax = nlevels_nod2D_min(node)-1 @@ -1309,10 +1387,13 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) do nz=nzmin, nzmax-1 hnode_new(nz,node)=(zbar(nz)-zbar(nz+1))*(1.0_WP+hh_inv*eta_n(node)) end do - end do + end do ! --> do node=1, myDim_nod2D+eDim_nod2D +!$OMP END PARALLEL DO !___________________________________________________________________________ ! --> update mean layer thinkness at element +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) +!$OMP DO do elem=1, myDim_elem2D nzmin = ulevels(elem) nzmax = nlevels(elem)-1 @@ -1321,7 +1402,7 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP end do end do - +!$OMP END PARALLEL DO end subroutine compute_thickness_zstar diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index d0fb21572..998f2c35d 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -509,6 +509,14 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) dynamics%se_uvBT_theta = 0.0_WP dynamics%se_uvBT_mean = 0.0_WP dynamics%se_uvBT_12 = 0.0_WP + if (dynamics%splitexpl_visc) then + allocate(dynamics%se_uvBT_stab_hvisc(2, elem_size)) + dynamics%se_uvBT_stab_hvisc = 0.0_WP + end if + if (dynamics%splitexpl_bottdrag) then + allocate(dynamics%se_uvBT_stab_bdrag(elem_size)) + dynamics%se_uvBT_stab_bdrag = 0.0_WP + end if else allocate(dynamics%d_eta( node_size)) allocate(dynamics%ssh_rhs( node_size)) From 1c71839db3443409ebab5f5bf187d00b06b06e4d Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 14 Sep 2023 16:35:38 +0200 Subject: [PATCH 30/50] fix OMP bug? --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 0c069ffae..03a360be0 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -884,7 +884,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do ! --> do edge=1, myDim_edge2D+eDim_edge2D -!$OMP END DO +!$OMP END PARALLEL DO end if ! --> if (dynamics%splitexpl_visc) then !___________________________________________________________________________ From 1dd2eaa20ae65e70e85a36faaa6f0948941511d4 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 14 Sep 2023 16:45:02 +0200 Subject: [PATCH 31/50] fix OMP bug? --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 29 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 03a360be0..1a7cdc36f 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -884,7 +884,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do ! --> do edge=1, myDim_edge2D+eDim_edge2D -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL end if ! --> if (dynamics%splitexpl_visc) then !___________________________________________________________________________ @@ -901,7 +902,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) UVBT_rhs(1, elem)=UVBT_rhs(1, elem) + bottomdrag(elem)*UVBT(1, elem)/hh UVBT_rhs(2, elem)=UVBT_rhs(2, elem) + bottomdrag(elem)*UVBT(2, elem)/hh end do -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL end if ! --> if (dynamics%splitexpl_bottdrag) then !___________________________________________________________________________ @@ -978,7 +980,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do ! --> do edge=1, myDim_edge2D+eDim_edge2D -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL end if ! -> if (dynamics%splitexpl_visc) then !_______________________________________________________________________ @@ -1082,7 +1085,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) UVBT_mean( 2, elem) = UVBT_mean( 2, elem) + UVBT_theta(2, elem)*BT_inv end do -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL !_______________________________________________________________________ !$OMP MASTER @@ -1141,7 +1145,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END ORDERED #endif end do -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL !_______________________________________________________________________ ! Apply freshwater boundary condition @@ -1236,7 +1241,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) hh_inv= hh_inv+helem(nz,elem) end do ! --> do nz=nzmin, nzmax hh_inv=1.0_WP/hh_inv - + !___________________________________________________________________ if (dynamics%ldiag_ke) then do nz=nzmin, nzmax @@ -1301,7 +1306,8 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) end do ! --> do nz=nzmin, nzmax end do ! --> do elem=1, myDim_elem2D -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL !_______________________________________________________________________ !$OMP MASTER @@ -1339,7 +1345,8 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do end do -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL !_______________________________________________________________________ !$OMP MASTER @@ -1388,7 +1395,8 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) hnode_new(nz,node)=(zbar(nz)-zbar(nz+1))*(1.0_WP+hh_inv*eta_n(node)) end do end do ! --> do node=1, myDim_nod2D+eDim_nod2D -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL !___________________________________________________________________________ ! --> update mean layer thinkness at element @@ -1402,7 +1410,8 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP end do end do -!$OMP END PARALLEL DO +!$OMP END DO +!$OMP END PARALLEL end subroutine compute_thickness_zstar From b025d82cb56b3eaba925b210af18d59e29846876 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 14 Sep 2023 17:00:19 +0200 Subject: [PATCH 32/50] fix OMP bug? --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 1a7cdc36f..e4542ab7d 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -705,10 +705,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) !___________________________________________________________________________ !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax, & -!$OMP Fx, Fy, hh, & -!$OMP vert_sum_u, vert_sum_v, hh) - - !___________________________________________________________________________ +!$OMP Fx, Fy, hh, vert_sum_u, vert_sum_v) !$OMP DO do elem=1, myDim_elem2D elnodes= elem2D_nodes(:,elem) From 1b14331f6adee4d13c6a8e64eae598bbb4ba27aa Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 14 Sep 2023 17:11:24 +0200 Subject: [PATCH 33/50] fix OMP bug? --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index e4542ab7d..7caed87b9 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -791,8 +791,8 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !___________________________________________________________________________ real(kind=WP) :: dtBT, BT_inv, hh, ff, rx, ry, a, b, d, c1, c2, ax, ay real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, thetaBT - integer :: step, elem, edge, node, elnodes(3), ednodes(2), edelem(2) - real(kind=WP) :: update_ubt, update_vbt, vi, len, nzmax + integer :: step, elem, edge, node, elnodes(3), ednodes(2), edelem(2), nzmax + real(kind=WP) :: update_ubt, update_vbt, vi, len !___________________________________________________________________________ ! pointer on necessary derived types From 32af1988cb44c3a4f83f5f5798e9164544d76554 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 14 Sep 2023 17:37:14 +0200 Subject: [PATCH 34/50] fix OMP bug? --- src/oce_ale.F90 | 133 +++++++++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 52 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index dc72d7c16..e71a891b6 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2638,9 +2638,7 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) type(t_mesh) , intent(inout), target :: mesh integer :: node, elem, nz, ed, nzmin, nzmax, ednodes(2), edelem(2) real(kind=WP) :: hh_inv, deltaX1, deltaX2, deltaY1, deltaY2 - real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1) - real(kind=WP) :: e2c1(mesh%nl-1), e2c2(mesh%nl-1) - + real(kind=WP) :: c1(mesh%nl-1), c2(mesh%nl-1) !___________________________________________________________________________ ! pointer on necessary derived types @@ -2659,11 +2657,19 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) end if !___________________________________________________________________________ +!$OMP PARALLEL DO do node=1, myDim_nod2D+eDim_nod2D Wvel(:, node)=0.0_WP + if (Fer_GM) then + fer_Wvel(:, node)=0.0_WP + end if end do ! --> do node=1, myDim_nod2D+eDim_nod2D - +!$OMP END PARALLEL DO + !___________________________________________________________________________ +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(ed, ednodes, edelem, nz, nzmin, nzmax, & +!$OMP deltaX1, deltaY1, deltaX2, deltaY2, c1, c2) +!$OMP DO do ed=1, myDim_edge2D ! local indice of nodes that span up edge ed ednodes=edges(:,ed) @@ -2686,87 +2692,102 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) ! --> h * u_vec * n_vec ! --> e_vec = (dx,dy), n_vec = (-dy,dx); ! --> h * u*(-dy) + v*dx - e1c1(nz)=( UVh(2, nz, edelem(1))*deltaX1 - UVh(1, nz, edelem(1))*deltaY1 ) + c1(nz)=( UVh(2, nz, edelem(1))*deltaX1 - UVh(1, nz, edelem(1))*deltaY1 ) ! inflow(outflow) "flux" to control volume of node enodes1 ! is equal to outflow(inflow) "flux" to control volume of node enodes2 end do ! --> do nz=nzmax,nzmin,-1 - Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e1c1(nzmin:nzmax) - Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e1c1(nzmin:nzmax) - if (Fer_GM) then do nz = nzmax, nzmin, -1 - e1c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1 - fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) - end do ! --> do nz=nzmax,nzmin,-1 - fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e1c2(nzmin:nzmax) - fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e1c2(nzmin:nzmax) + c2(nz)=(fer_UV(2, nz, edelem(1))*deltaX1 - fer_UV(1, nz, edelem(1))*deltaY1)*helem(nz, edelem(1)) + end do + end if +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(ednodes(1))) +#else +!$OMP ORDERED +#endif + Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+c1(nzmin:nzmax) + if (Fer_GM) then + fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+c2(nzmin:nzmax) + end if +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(1))) + call omp_set_lock (partit%plock(ednodes(2))) +#endif + Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-c1(nzmin:nzmax) + if (Fer_GM) then + fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-c2(nzmin:nzmax) end if +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(2))) +#else +!$OMP END ORDERED +#endif !_______________________________________________________________________ ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer ! for edelem(2) - e2c1 = 0.0_WP - e2c2 = 0.0_WP + c1 = 0.0_WP + c2 = 0.0_WP if(edelem(2)>0)then deltaX2=edge_cross_dxdy(3,ed) deltaY2=edge_cross_dxdy(4,ed) nzmin = ulevels(edelem(2)) nzmax = nlevels(edelem(2))-1 do nz = nzmax, nzmin, -1 - e2c1(nz)=-(UVh(2, nz, edelem(2))*deltaX2 - UVh(1, nz, edelem(2))*deltaY2) + c1(nz)=-(UVh(2, nz, edelem(2))*deltaX2 - UVh(1, nz, edelem(2))*deltaY2) end do ! --> do nz=nzmax,nzmin,-1 - Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+e2c1(nzmin:nzmax) - Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-e2c1(nzmin:nzmax) - if (Fer_GM) then do nz = nzmax, nzmin, -1 - e2c2(nz)=-(fer_UV(2, nz, edelem(2))*deltaX2-fer_UV(1, nz, edelem(2))*deltaY2)*helem(nz, edelem(2)) + c2(nz)=-(fer_UV(2, nz, edelem(2))*deltaX2-fer_UV(1, nz, edelem(2))*deltaY2)*helem(nz, edelem(2)) end do ! --> do nz=nzmax,nzmin,-1 - fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+e2c2(nzmin:nzmax) - fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-e2c2(nzmin:nzmax) end if - end if +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_set_lock (partit%plock(ednodes(1))) +#else +!$OMP ORDERED +#endif + Wvel(nzmin:nzmax, ednodes(1))= Wvel(nzmin:nzmax, ednodes(1))+c1(nzmin:nzmax) + if (Fer_GM) then + fer_Wvel(nzmin:nzmax, ednodes(1))= fer_Wvel(nzmin:nzmax, ednodes(1))+c2(nzmin:nzmax) + end if +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(1))) + call omp_set_lock (partit%plock(ednodes(2))) +#endif + Wvel(nzmin:nzmax, ednodes(2))= Wvel(nzmin:nzmax, ednodes(2))-c1(nzmin:nzmax) + if (Fer_GM) then + fer_Wvel(nzmin:nzmax, ednodes(2))= fer_Wvel(nzmin:nzmax, ednodes(2))-c2(nzmin:nzmax) + end if +#if defined(_OPENMP) && !defined(__openmp_reproducible) + call omp_unset_lock(partit%plock(ednodes(2))) +#else +!$OMP END ORDERED +#endif + end if !--> if(edelem(2)>0)then - !_______________________________________________________________________ -!PS if ( any(Wvel(nzmin:nzmax, ednodes(1))/=Wvel(nzmin:nzmax, ednodes(1))) .or. & -!PS any(Wvel(nzmin:nzmax, ednodes(2))/=Wvel(nzmin:nzmax, ednodes(2)))) then -!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' edge =', ed -!PS write(*,*) ' enodes =', enodes -!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, ednodes(1)) -!PS write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, ednodes(2)) -!PS write(*,*) ' e1c1', e1c1(nzmin:nzmax) -!PS write(*,*) ' e1c2', e1c2(nzmin:nzmax) -!PS write(*,*) ' e2c1', e2c1(nzmin:nzmax) -!PS write(*,*) ' e2c2', e2c2(nzmin:nzmax) -!PS end if end do ! --> do ed=1, myDim_edge2D - +!$OMP END DO +!$OMP END PARALLEL + !___________________________________________________________________________ ! add the contribution from -dh/dt * area +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax) do node=1, myDim_nod2D nzmin = ulevels_nod2D(node) nzmax = nlevels_nod2d(node)-1 do nz=nzmax,nzmin,-1 Wvel(nz, node)=Wvel(nz, node)-(hnode_new(nz, node)-hnode(nz, node))*area(nz, node)/dt end do ! --> do nz=nzmax,nzmin,-1 - - !_______________________________________________________________________ -!PS if ( any(Wvel(nzmin:nzmax, node)/=Wvel(nzmin:nzmax, node))) then -!PS write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after +dhnode/dt=W' -!PS write(*,*) ' mype =', mype -!PS write(*,*) ' node =', node -!PS write(*,*) ' Wvel( nzmin:nzmax, node)=', Wvel(nzmin:nzmax, node) -!PS write(*,*) ' hnode_new(nzmin:nzmax, node)=', hnode_new(nzmin:nzmax, node) -!PS write(*,*) ' hnode( nzmin:nzmax, node)=', hnode(nzmin:nzmax, node) -!PS end if end do ! --> do node=1, myDim_nod2D - +!$OMP END PARALLEL DO + !___________________________________________________________________________ ! Sum up to get W*area ! cumulative summation of div(u_vec*h) vertically ! W_k = W_k+1 - div(h_k*u_k) ! W_k ... vertical flux troughdo node=1, myDim_nod2D +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax) do node=1, myDim_nod2D nzmin = ulevels_nod2D(node) nzmax = nlevels_nod2d(node)-1 @@ -2777,10 +2798,12 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) end if end do ! --> do nz=nzmax,nzmin,-1 end do ! --> do node=1, myDim_nod2D - +!$OMP END PARALLEL DO + !___________________________________________________________________________ ! divide with depth dependent cell area to convert from Vertical flux to ! physical vertical velocities in units m/s +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax) do node=1, myDim_nod2D nzmin = ulevels_nod2D(node) nzmax = nlevels_nod2d(node)-1 @@ -2791,21 +2814,27 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) end if end do ! --> do nz=nzmax,nzmin,-1 end do ! --> do node=1, myDim_nod2D - +!$OMP END PARALLEL DO + !___________________________________________________________________________ ! Add surface fresh water flux as upper boundary condition for ! continutity if (.not. (trim(which_ale)=='linfs' )) then +!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nzmin) do node=1, myDim_nod2D nzmin = ulevels_nod2D(node) if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) end do ! --> do node=1, myDim_nod2D end if - +!$OMP END PARALLEL DO + !___________________________________________________________________________ +!$OMP MASTER call exchange_nod(Wvel, partit) if (Fer_GM) call exchange_nod(fer_Wvel, partit) - +!$OMP END MASTER +!$OMP BARRIER + !___________________________________________________________________________ ! compute vertical CFL_z criteria call compute_CFLz(dynamics, partit, mesh) From 11549ae9cdf22dac388126387a917987c427c865 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 15 Sep 2023 10:39:19 +0200 Subject: [PATCH 35/50] return to c1 and c2 varaibles --- src/oce_ale.F90 | 63 +++++++++++++------------------------------------ 1 file changed, 16 insertions(+), 47 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index e71a891b6..778ae8134 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2038,7 +2038,7 @@ subroutine vert_vel_ale(dynamics, partit, mesh) integer :: el(2), enodes(2), n, nz, ed, nzmin, nzmax, uln1, uln2, nln1, nln2 real(kind=WP) :: deltaX1, deltaY1, deltaX2, deltaY2, dd, dd1, dddt, cflmax ! still to be understood but if you allocate these arrays statically the results will be different: - real(kind=WP) :: e1c1(mesh%nl-1), e1c2(mesh%nl-1), e2c1(mesh%nl-1), e2c2(mesh%nl-1) + real(kind=WP) :: c1(mesh%nl-1), c2(mesh%nl-1) ! --> zlevel with local zstar real(kind=WP) :: dhbar_total, dhbar_rest, distrib_dhbar_int real(kind=WP), dimension(:), allocatable :: max_dhbar2distr, cumsum_maxdhbar, distrib_dhbar @@ -2107,11 +2107,11 @@ subroutine vert_vel_ale(dynamics, partit, mesh) ! --> h * u_vec * n_vec ! --> e_vec = (dx,dy), n_vec = (-dy,dx); ! --> h * u*(-dy) + v*dx - e1c1(nz)=( UV(2,nz,el(1))*deltaX1 - UV(1,nz,el(1))*deltaY1 )*helem(nz,el(1)) + c1(nz)=( UV(2,nz,el(1))*deltaX1 - UV(1,nz,el(1))*deltaY1 )*helem(nz,el(1)) ! inflow(outflow) "flux" to control volume of node enodes1 ! is equal to outflow(inflow) "flux" to control volume of node enodes2 if (Fer_GM) then - e1c2(nz)=(fer_UV(2,nz,el(1))*deltaX1- fer_UV(1,nz,el(1))*deltaY1)*helem(nz,el(1)) + c2(nz)=(fer_UV(2,nz,el(1))*deltaX1- fer_UV(1,nz,el(1))*deltaY1)*helem(nz,el(1)) end if end do #if defined(_OPENMP) && !defined(__openmp_reproducible) @@ -2119,17 +2119,17 @@ subroutine vert_vel_ale(dynamics, partit, mesh) #else !$OMP ORDERED #endif - Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+e1c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+e1c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(1))) call omp_set_lock (partit%plock(enodes(2))) #endif - Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-e1c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-e1c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(2))) @@ -2139,73 +2139,42 @@ subroutine vert_vel_ale(dynamics, partit, mesh) !_______________________________________________________________________ ! if ed is not a boundary edge --> calc div(u_vec*h) for every layer ! for el(2) - e2c1 = 0.0_WP - e2c2 = 0.0_WP + c1 = 0.0_WP + c2 = 0.0_WP if(el(2)>0)then deltaX2=edge_cross_dxdy(3,ed) deltaY2=edge_cross_dxdy(4,ed) nzmin = ulevels(el(2)) nzmax = nlevels(el(2))-1 do nz = nzmax, nzmin, -1 - e2c1(nz)=-(UV(2,nz,el(2))*deltaX2 - UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) + c1(nz)=-(UV(2,nz,el(2))*deltaX2 - UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) if (Fer_GM) then - e2c2(nz)=-(fer_UV(2,nz,el(2))*deltaX2-fer_UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) + c2(nz)=-(fer_UV(2,nz,el(2))*deltaX2-fer_UV(1,nz,el(2))*deltaY2)*helem(nz,el(2)) end if - - !_______________________________________________________________ - if ( e2c1(nz)/=e2c1(nz) ) then - write(*,*) ' --> subroutine vert_vel_ale --> found Nan in e2c1(nz)' - write(*,*) ' mype =', mype - write(*,*) ' edge =', ed - write(*,*) ' el =', el - write(*,*) ' nz =', nz - write(*,*) ' e2c1(nz) =', e2c1(nz) - write(*,*) ' UV(1,nz,el(2)) =', UV(1,nz,el(2)) - write(*,*) ' UV(2,nz,el(2)) =', UV(2,nz,el(2)) - write(*,*) ' helem(nz,el(2))=', helem(nz,el(2)) - - end if end do #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_set_lock (partit%plock(enodes(1))) #else !$OMP ORDERED #endif - Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+e2c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(1))= Wvel (nzmin:nzmax, enodes(1))+c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+e2c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(1))= fer_Wvel(nzmin:nzmax, enodes(1))+c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(1))) call omp_set_lock (partit%plock(enodes(2))) #endif - Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-e2c1(nzmin:nzmax) + Wvel (nzmin:nzmax, enodes(2))= Wvel (nzmin:nzmax, enodes(2))-c1(nzmin:nzmax) if (Fer_GM) then - fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-e2c2(nzmin:nzmax) + fer_Wvel(nzmin:nzmax, enodes(2))= fer_Wvel(nzmin:nzmax, enodes(2))-c2(nzmin:nzmax) end if #if defined(_OPENMP) && !defined(__openmp_reproducible) call omp_unset_lock(partit%plock(enodes(2))) #else !$OMP END ORDERED #endif - end if - - !_______________________________________________________________________ - if ( any(Wvel(nzmin:nzmax, enodes(1))/=Wvel(nzmin:nzmax, enodes(1))) .or. & - any(Wvel(nzmin:nzmax, enodes(2))/=Wvel(nzmin:nzmax, enodes(2)))) then - write(*,*) ' --> subroutine vert_vel_ale --> found Nan in Wvel after div_H(...)' - write(*,*) ' mype =', mype - write(*,*) ' edge =', ed - write(*,*) ' enodes =', enodes - write(*,*) ' Wvel(nzmin:nzmax, enodes(1))=', Wvel(nzmin:nzmax, enodes(1)) - write(*,*) ' Wvel(nzmin:nzmax, enodes(2))=', Wvel(nzmin:nzmax, enodes(2)) - write(*,*) ' e1c1', e1c1(nzmin:nzmax) - write(*,*) ' e1c2', e1c2(nzmin:nzmax) - write(*,*) ' e2c1', e2c1(nzmin:nzmax) - write(*,*) ' e2c2', e2c2(nzmin:nzmax) - - end if - + end if !~-> if(el(2)>0)then end do ! --> do ed=1, myDim_edge2D !$OMP END DO !$OMP END PARALLEL From cb078f75f77057c3e27d418464b9c2c4a41a270a Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 15 Sep 2023 10:45:54 +0200 Subject: [PATCH 36/50] fix another OMP issue --- src/oce_ale.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 778ae8134..09ad5cd05 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2794,8 +2794,9 @@ subroutine compute_vert_vel_transpv(dynamics, partit, mesh) nzmin = ulevels_nod2D(node) if (nzmin==1) Wvel(nzmin, node)=Wvel(nzmin, node)-water_flux(node) end do ! --> do node=1, myDim_nod2D +!$OMP END PARALLEL DO end if -!$OMP END PARALLEL DO + !___________________________________________________________________________ !$OMP MASTER From 57256cfea1d0e0ee2a735f23ef014b9ecb33b222 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 15 Sep 2023 10:52:24 +0200 Subject: [PATCH 37/50] fix OMP issue --- src/write_step_info.F90 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/write_step_info.F90 b/src/write_step_info.F90 index 94421ac0f..e1c4d0393 100644 --- a/src/write_step_info.F90 +++ b/src/write_step_info.F90 @@ -109,7 +109,8 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) loc_deta =0. loc_dhbar =0. loc_wflux =0. - loc =0. + loc =0. + !_______________________________________________________________________ #if !defined(__openmp_reproducible) !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n) REDUCTION(+:loc_eta, loc_hbar, loc_deta, loc_dhbar, loc_wflux) @@ -123,10 +124,13 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) end if loc_wflux = loc_wflux + areasvol(ulevels_nod2D(n), n)*water_flux(n) end do - if (dynamics%use_ssh_splitexpl_subcycl) loc_deta=loc_dhbar #if !defined(__openmp_reproducible) !$OMP END PARALLEL DO #endif + if (dynamics%use_ssh_splitexpl_subcycl) then + loc_deta=loc_dhbar + end if + !_______________________________________________________________________ call MPI_AllREDUCE(loc_eta , int_eta , 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) call MPI_AllREDUCE(loc_hbar , int_hbar , 1, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_FESOM, MPIerr) From 2898ca212b12dad868a0629fe18a9f7fddc08ef7 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 15 Sep 2023 11:40:32 +0200 Subject: [PATCH 38/50] rename splitexpl_* flag to shorter se_* --- config/namelist.dyn | 10 +++++++ src/MOD_DYN.F90 | 30 +++++++++---------- src/io_blowup.F90 | 2 +- src/io_meandata.F90 | 2 +- src/oce_ale.F90 | 20 ++++++------- src/oce_ale_ssh_splitexpl_subcycl.F90 | 42 +++++++++++++-------------- src/oce_ale_vel_rhs.F90 | 6 ++-- src/oce_dyn.F90 | 6 ++-- src/oce_setup_step.F90 | 42 +++++++++++++-------------- src/write_step_info.F90 | 22 +++++++------- 10 files changed, 96 insertions(+), 86 deletions(-) diff --git a/config/namelist.dyn b/config/namelist.dyn index 91816a5e4..4a598ee22 100644 --- a/config/namelist.dyn +++ b/config/namelist.dyn @@ -21,5 +21,15 @@ use_wsplit = .false. ! Switch for implicite/explicte splitting of vert. veloci wsplit_maxcfl= 1.0 ! maximum allowed CFL criteria in vertical (0.5 < w_max_cfl < 1.) ! in older FESOM it used to be w_exp_max=1.e-3 ldiag_KE=.false. ! activates energy diagnostics + +use_ssh_se_subcycl = .false. +se_BTsteps = 50 !50 +se_BTtheta = 0.14 ! default: 0.14, core2 stability plateu@ 0.37 +se_bottdrag = .true. +se_bdrag_si = .true. ! bottomdrag semi-implicite/explicite +se_visc = .true. +se_visc_gamma0 = 10 ! 150 +se_visc_gamma1 = 19500 ! 150 +se_visc_gamma2 = 0 / diff --git a/src/MOD_DYN.F90 b/src/MOD_DYN.F90 index 218f097ae..b7998f1aa 100644 --- a/src/MOD_DYN.F90 +++ b/src/MOD_DYN.F90 @@ -122,19 +122,19 @@ MODULE MOD_DYN real(kind=WP) :: wsplit_maxcfl = 1.0 ! switch between ssh computation, by solver or split explicite subcycling - ! use_ssh_splitexpl_subcycl = .false. --> solver - ! use_ssh_splitexpl_subcycl = .true. --> split explicite subcycling - logical :: use_ssh_splitexpl_subcycl = .false. + ! use_ssh_se_subcycl = .false. --> solver + ! use_ssh_se_subcycl = .true. --> split explicite subcycling + logical :: use_ssh_se_subcycl = .false. ! barotropic subcycling time-steps and dissipation parameter - integer :: splitexpl_BTsteps = 50 - real(kind=WP) :: splitexpl_BTtheta = 0.14_WP - logical :: splitexpl_bottdrag = .true. - logical :: splitexpl_bdrag_si = .true. - logical :: splitexpl_visc = .true. - real(kind=WP) :: splitexpl_visc_gamma0 = 10 - real(kind=WP) :: splitexpl_visc_gamma1 = 2750 - real(kind=WP) :: splitexpl_visc_gamma2 = 0 + integer :: se_BTsteps = 50 + real(kind=WP) :: se_BTtheta = 0.14_WP + logical :: se_bottdrag = .true. + logical :: se_bdrag_si = .true. + logical :: se_visc = .true. + real(kind=WP) :: se_visc_gamma0 = 10 + real(kind=WP) :: se_visc_gamma1 = 2750 + real(kind=WP) :: se_visc_gamma2 = 0 !___________________________________________________________________________ ! energy diagnostic part: will be computed inside the model ("hard integration"): @@ -266,7 +266,7 @@ subroutine WRITE_T_DYN(dynamics, unit, iostat, iomsg) write(unit, iostat=iostat, iomsg=iomsg) dynamics%use_freeslip write(unit, iostat=iostat, iomsg=iomsg) dynamics%use_wsplit write(unit, iostat=iostat, iomsg=iomsg) dynamics%wsplit_maxcfl - write(unit, iostat=iostat, iomsg=iomsg) dynamics%use_ssh_splitexpl_subcycl + write(unit, iostat=iostat, iomsg=iomsg) dynamics%use_ssh_se_subcycl !___________________________________________________________________________ call dynamics%solverinfo%WRITE_T_SOLVERINFO(unit) @@ -287,7 +287,7 @@ subroutine WRITE_T_DYN(dynamics, unit, iostat, iomsg) call write_bin_array(dynamics%fer_w , unit, iostat, iomsg) call write_bin_array(dynamics%fer_uv, unit, iostat, iomsg) end if - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then call write_bin_array(dynamics%se_uvh , unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_rhs , unit, iostat, iomsg) call write_bin_array(dynamics%se_uvBT_4AB , unit, iostat, iomsg) @@ -324,7 +324,7 @@ subroutine READ_T_DYN(dynamics, unit, iostat, iomsg) read(unit, iostat=iostat, iomsg=iomsg) dynamics%use_freeslip read(unit, iostat=iostat, iomsg=iomsg) dynamics%use_wsplit read(unit, iostat=iostat, iomsg=iomsg) dynamics%wsplit_maxcfl - read(unit, iostat=iostat, iomsg=iomsg) dynamics%use_ssh_splitexpl_subcycl + read(unit, iostat=iostat, iomsg=iomsg) dynamics%use_ssh_se_subcycl !___________________________________________________________________________ call dynamics%solverinfo%READ_T_SOLVERINFO(unit) @@ -345,7 +345,7 @@ subroutine READ_T_DYN(dynamics, unit, iostat, iomsg) call read_bin_array(dynamics%fer_w , unit, iostat, iomsg) call read_bin_array(dynamics%fer_uv , unit, iostat, iomsg) end if - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then call read_bin_array(dynamics%se_uvh , unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_rhs , unit, iostat, iomsg) call read_bin_array(dynamics%se_uvBT_4AB , unit, iostat, iomsg) diff --git a/src/io_blowup.F90 b/src/io_blowup.F90 index ec42e1067..4bd5245b0 100644 --- a/src/io_blowup.F90 +++ b/src/io_blowup.F90 @@ -105,7 +105,7 @@ subroutine ini_blowup_io(year, ice, dynamics, tracers, partit, mesh) !___ALE related fields______________________________________________________ call def_variable(bid, 'hbar' , (/nod2D/) , 'ALE surface elevation hbar_n+0.5', 'm', hbar); !!PS call def_variable(bid, 'hbar_old' , (/nod2D/) , 'ALE surface elevation hbar_n-0.5', 'm', hbar_old); - if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. dynamics%use_ssh_se_subcycl) then call def_variable(bid, 'd_eta' , (/nod2D/) , 'change in ssh from solver', 'm', dynamics%d_eta); call def_variable(bid, 'ssh_rhs' , (/nod2D/) , 'RHS for the elevation', '?', dynamics%ssh_rhs); call def_variable(bid, 'ssh_rhs_old', (/nod2D/) , 'RHS for the elevation', '?', dynamics%ssh_rhs_old); diff --git a/src/io_meandata.F90 b/src/io_meandata.F90 index 03d6906c1..2ba3c87f0 100644 --- a/src/io_meandata.F90 +++ b/src/io_meandata.F90 @@ -621,7 +621,7 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh) !_______________________________________________________________________________ ! Split-Explicite subcycling varaibles CASE ('SPLIT-EXPL') - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then call def_stream(elem2D, myDim_elem2D , 'ubt' , 'zonal barotrop. transport' , '?' , dynamics%se_uvBT(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) call def_stream(elem2D, myDim_elem2D , 'vbt' , 'merid. barotrop. transport', '?' , dynamics%se_uvBT(2,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) call def_stream(elem2D, myDim_elem2D , 'ubt_rhs', 'zonal barotrop. transport RHS' , '?' , dynamics%se_uvBT_rhs(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 09ad5cd05..7d83f28dd 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2064,7 +2064,7 @@ subroutine vert_vel_ale(dynamics, partit, mesh) fer_Wvel=> dynamics%fer_w(:,:) end if - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then d_eta => dynamics%d_eta(:) ssh_rhs => dynamics%ssh_rhs(:) ssh_rhs_old => dynamics%ssh_rhs_old(:) @@ -2535,7 +2535,7 @@ subroutine vert_vel_ale(dynamics, partit, mesh) write(*,*) 'water_flux = ', water_flux(n) write(*,*) write(*,*) "eta_n = ", eta_n(n) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then write(*,*) "d_eta = ", d_eta(n) write(*,*) "ssh_rhs = ", ssh_rhs(n) write(*,*) "ssh_rhs_old = ", ssh_rhs_old(n) @@ -3467,7 +3467,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) ! Energy diagnostic contribution if (dynamics%ldiag_ke) then ! if use solver - if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. dynamics%use_ssh_se_subcycl) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) do elem=1, myDim_elem2D nzmax = nlevels(elem) @@ -3503,7 +3503,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) ! Energy diagnostic contribution if (dynamics%ldiag_ke) then ! if use solver - if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. dynamics%use_ssh_se_subcycl) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) do elem=1, myDim_elem2D nzmax = nlevels(elem) @@ -3550,7 +3550,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call impl_vert_visc_ale'//achar(27)//'[0m' if(dynamics%use_ivertvisc) then - if ( .not. dynamics%use_ssh_splitexpl_subcycl ) then + if ( .not. dynamics%use_ssh_se_subcycl ) then call impl_vert_visc_ale(dynamics,partit, mesh) else call impl_vert_visc_ale_vtransp(dynamics, partit, mesh) @@ -3560,7 +3560,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ if (dynamics%ldiag_ke) then ! if use solver - if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. dynamics%use_ssh_se_subcycl) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax) do elem=1, myDim_elem2D nzmax = nlevels(elem) @@ -3592,7 +3592,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) ! Compute SSH via solver ! Update stiffness matrix by dhe=hbar(n+1/2)-hbar(n-1/2) on elements, only ! needed for zlevel and zstar - if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. dynamics%use_ssh_se_subcycl) then if (.not. trim(which_ale)=='linfs') call update_stiff_mat_ale(partit, mesh) if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call compute_ssh_rhs_ale'//achar(27)//'[0m' ! ssh_rhs=-alpha*\nabla\int(U_n+U_rhs)dz-(1-alpha)*... @@ -3658,7 +3658,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) call update_trim_vel_ale_vtransp(1, dynamics, partit, mesh) t4=MPI_Wtime() t5=t4 - end if ! --> if (.not. dynamics%use_ssh_splitexpl_subcycl) then + end if ! --> if (.not. dynamics%use_ssh_se_subcycl) then !___________________________________________________________________________ ! Do horizontal and vertical scaling of GM/Redi diffusivity @@ -3689,7 +3689,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) ! The main step of ALE procedure --> this is were the magic happens --> here ! is decided how change in hbar is distributed over the vertical layers if (flag_debug .and. mype==0) print *, achar(27)//'[36m'//' --> call vert_vel_ale'//achar(27)//'[0m' - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then call vert_vel_ale(dynamics, partit, mesh) else if (trim(which_ale)=='zstar' ) then @@ -3723,7 +3723,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ ! Trim to make velocity consistent with BT velocity at n+1/2 ! Velocity will be used to advance momentum - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then call update_trim_vel_ale_vtransp(2, dynamics, partit, mesh) end if diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 7caed87b9..9f5a92b57 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -809,22 +809,22 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) UVBT_theta=>dynamics%se_uvBT_theta(:,:) UVBT_mean =>dynamics%se_uvBT_mean(:,:) UVBT_12 =>dynamics%se_uvBT_12(:,:) - if (dynamics%splitexpl_bottdrag) then + if (dynamics%se_bottdrag) then UV =>dynamics%uv(:,:,:) bottomdrag =>dynamics%se_uvBT_stab_bdrag(:) end if - if (dynamics%splitexpl_visc) then + if (dynamics%se_visc) then UVBT_harmvisc=>dynamics%se_uvBT_stab_hvisc(:,:) end if !___________________________________________________________________________ ! Dissipation parameter of FB dissipative method 0.14 is the default value ! from Demange et al. - thetaBT= dynamics%splitexpl_BTtheta + thetaBT= dynamics%se_BTtheta ! BTsteps should be 30 or 40. - dtBT = dt/dynamics%splitexpl_BTsteps - BT_inv = 1.0_WP/(1.0_WP*dynamics%splitexpl_BTsteps) + dtBT = dt/dynamics%se_BTsteps + BT_inv = 1.0_WP/(1.0_WP*dynamics%se_BTsteps) !___SPLIT-EXPLICITE STABILIZATION___________________________________________ ! trim R (UVBT_rhs): @@ -834,7 +834,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! compute_BT_step_SE_ale) ! --> use only harmonmic viscosity operator applied to the barotropic ! velocity - if (dynamics%splitexpl_visc) then + if (dynamics%se_visc) then !_______________________________________________________________________ ! remove viscosity !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, nzmax, hh, len, & @@ -854,9 +854,9 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh vi=update_ubt*update_ubt + update_vbt*update_vbt - vi=-dt*sqrt(max(dynamics%splitexpl_visc_gamma0, & - max(dynamics%splitexpl_visc_gamma1*sqrt(vi), & - dynamics%splitexpl_visc_gamma2*vi) & + vi=-dt*sqrt(max(dynamics%se_visc_gamma0, & + max(dynamics%se_visc_gamma1*sqrt(vi), & + dynamics%se_visc_gamma2*vi) & )*len) update_ubt=update_ubt*vi update_vbt=update_vbt*vi @@ -883,11 +883,11 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) end do ! --> do edge=1, myDim_edge2D+eDim_edge2D !$OMP END DO !$OMP END PARALLEL - end if ! --> if (dynamics%splitexpl_visc) then + end if ! --> if (dynamics%se_visc) then !___________________________________________________________________________ ! remove bottom drag - if (dynamics%splitexpl_bottdrag) then + if (dynamics%se_bottdrag) then !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nzmax, hh) !$OMP DO do elem=1, myDim_elem2D @@ -901,7 +901,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) end do !$OMP END DO !$OMP END PARALLEL - end if ! --> if (dynamics%splitexpl_bottdrag) then + end if ! --> if (dynamics%se_bottdrag) then !___________________________________________________________________________ ! initialise UVBT_mean with zeros --> OMP style @@ -914,14 +914,14 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !___________________________________________________________________________ ! eta_n elevation used in BT stepping, it is just a copy of eta_n ! UBT and VBT are transport velocities - do step=1, dynamics%splitexpl_BTsteps + do step=1, dynamics%se_BTsteps !####################################################################### !########## Dissipative forward--backward time stepping ########## !####################################################################### !_______________________________________________________________________ ! compute harmonic viscosity for stability - if (dynamics%splitexpl_visc) then + if (dynamics%se_visc) then !___________________________________________________________________ ! initialise UVBT_harmvisc with zeros --> OMP style @@ -950,9 +950,9 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh vi=update_ubt*update_ubt + update_vbt*update_vbt - vi=dt*sqrt(max(dynamics%splitexpl_visc_gamma0, & - max(dynamics%splitexpl_visc_gamma1*sqrt(vi), & - dynamics%splitexpl_visc_gamma2*vi) & + vi=dt*sqrt(max(dynamics%se_visc_gamma0, & + max(dynamics%se_visc_gamma1*sqrt(vi), & + dynamics%se_visc_gamma2*vi) & )*len) update_ubt=update_ubt*vi update_vbt=update_vbt*vi @@ -979,7 +979,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) end do ! --> do edge=1, myDim_edge2D+eDim_edge2D !$OMP END DO !$OMP END PARALLEL - end if ! -> if (dynamics%splitexpl_visc) then + end if ! -> if (dynamics%se_visc) then !_______________________________________________________________________ ! Advance velocities. I use SI stepping for the Coriolis @@ -1046,7 +1046,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! ! Semi-Implicit Coriolis a = dtBT*ff*0.5_WP - if (dynamics%splitexpl_bdrag_si) then + if (dynamics%se_bdrag_si) then b = 1.0_WP+BT_inv*bottomdrag(elem)/hh else b = 1.0_WP @@ -1094,7 +1094,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !_______________________________________________________________________ ! Store mid-step velocity (to trim 3D velocities in momentum) - if(step==dynamics%splitexpl_BTsteps/2) then + if(step==dynamics%se_BTsteps/2) then UVBT_12=UVBT end if @@ -1161,7 +1161,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END MASTER !$OMP BARRIER - end do ! --> do step=1, dynamics%splitexpl_BTsteps + end do ! --> do step=1, dynamics%se_BTsteps !___________________________________________________________________________ hbar_old = hbar diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index 261d82a4f..7efade82c 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -82,7 +82,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) inv_rhowat=> ice%thermo%inv_rhowat ! if split-explicite ssh subcycling is used - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then UVh => dynamics%se_uvh UVBT_4AB => dynamics%se_uvBT_4AB end if @@ -160,7 +160,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) !_______________________________________________________________________ ! when ssh split-explicite subcycling method is setted use transport velocities ! u*h, v*h instead of u,v - if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. dynamics%use_ssh_se_subcycl) then do nz=nzmin, nzmax-1 ! add pressure gradient terms UV_rhs( 1, nz, elem) = UV_rhs(1, nz, elem) + (Fx-pgf_x(nz, elem))*elem_area(elem) @@ -216,7 +216,7 @@ subroutine compute_vel_rhs(ice, dynamics, partit, mesh) if (mype==0) write(*,*) 'in moment not adapted mom_adv advection typ for ALE, check your namelist' call par_ex(partit%MPI_COMM_FESOM, partit%mype, 1) elseif (dynamics%momadv_opt==2) then - if (.not. dynamics%use_ssh_splitexpl_subcycl) then + if (.not. dynamics%use_ssh_se_subcycl) then call momentum_adv_scalar(dynamics, partit, mesh) else call momentum_adv_scalar_transpv(dynamics, partit, mesh) diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index db6bf24d8..92b3b099b 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -406,7 +406,7 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) nzmin = ulevels(ed) nzmax = nlevels(ed) !_______________________________________________________________________ - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then !SD Approximate update for transports. We do not care about accuracy !SD here. --> of course, helem will be better. Do nz=nzmin, nzmax-1 @@ -539,7 +539,7 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) nzmin = maxval(ulevels(el)) nzmax = minval(nlevels(el)) !_______________________________________________________________________ - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then !SD Approximate update for transports. We do not care about accuracy !SD here. --> of course, helem will be better. do nz=nzmin,nzmax-1 @@ -680,7 +680,7 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) nzmin = maxval(ulevels(el)) nzmax = minval(nlevels(el)) !_______________________________________________________________________ - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then !SD Approximate update for transports. We do not care about accuracy !SD here. --> of course, helem will be better. do nz=nzmin,nzmax-1 diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index 998f2c35d..0b67b03d9 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -393,18 +393,18 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) logical :: ldiag_KE =.false. logical :: check_opt_visc=.true. real(kind=WP) :: wsplit_maxcfl - logical :: use_ssh_splitexpl_subcycl=.false. - integer :: splitexpl_BTsteps - real(kind=WP) :: splitexpl_BTtheta - logical :: splitexpl_visc, splitexpl_bottdrag, splitexpl_bdrag_si - real(kind=WP) :: splitexpl_visc_gamma0, splitexpl_visc_gamma1, splitexpl_visc_gamma2 + logical :: use_ssh_se_subcycl=.false. + integer :: se_BTsteps + real(kind=WP) :: se_BTtheta + logical :: se_visc, se_bottdrag, se_bdrag_si + real(kind=WP) :: se_visc_gamma0, se_visc_gamma1, se_visc_gamma2 namelist /dynamics_visc / opt_visc, check_opt_visc, visc_gamma0, visc_gamma1, visc_gamma2, & use_ivertvisc, visc_easybsreturn namelist /dynamics_general/ momadv_opt, use_freeslip, use_wsplit, wsplit_maxcfl, & - ldiag_KE, use_ssh_splitexpl_subcycl, splitexpl_BTsteps, & - splitexpl_BTtheta, splitexpl_bottdrag, splitexpl_bdrag_si, & - splitexpl_visc, splitexpl_visc_gamma0, splitexpl_visc_gamma1, splitexpl_visc_gamma2 + ldiag_KE, use_ssh_se_subcycl, se_BTsteps, & + se_BTtheta, se_bottdrag, se_bdrag_si, & + se_visc, se_visc_gamma0, se_visc_gamma1, se_visc_gamma2 !___________________________________________________________________________ ! pointer on necessary derived types #include "associate_part_def.h" @@ -440,16 +440,16 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) dynamics%use_wsplit = use_wsplit dynamics%wsplit_maxcfl = wsplit_maxcfl dynamics%ldiag_KE = ldiag_KE - dynamics%use_ssh_splitexpl_subcycl = use_ssh_splitexpl_subcycl - if (dynamics%use_ssh_splitexpl_subcycl) then - dynamics%splitexpl_BTsteps = splitexpl_BTsteps - dynamics%splitexpl_BTtheta = splitexpl_BTtheta - dynamics%splitexpl_bottdrag = splitexpl_bottdrag - dynamics%splitexpl_bdrag_si = splitexpl_bdrag_si - dynamics%splitexpl_visc = splitexpl_visc - dynamics%splitexpl_visc_gamma0 = splitexpl_visc_gamma0 - dynamics%splitexpl_visc_gamma1 = splitexpl_visc_gamma1 - dynamics%splitexpl_visc_gamma2 = splitexpl_visc_gamma2 + dynamics%use_ssh_se_subcycl = use_ssh_se_subcycl + if (dynamics%use_ssh_se_subcycl) then + dynamics%se_BTsteps = se_BTsteps + dynamics%se_BTtheta = se_BTtheta + dynamics%se_bottdrag = se_bottdrag + dynamics%se_bdrag_si = se_bdrag_si + dynamics%se_visc = se_visc + dynamics%se_visc_gamma0 = se_visc_gamma0 + dynamics%se_visc_gamma1 = se_visc_gamma1 + dynamics%se_visc_gamma2 = se_visc_gamma2 end if !___________________________________________________________________________ @@ -494,7 +494,7 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) ! allocate/initialise ssh arrays in derived type allocate(dynamics%eta_n( node_size)) dynamics%eta_n = 0.0_WP - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then allocate(dynamics%se_uvh( 2, nl-1, elem_size)) allocate(dynamics%se_uvBT_rhs( 2, elem_size)) allocate(dynamics%se_uvBT_4AB( 4, elem_size)) @@ -509,11 +509,11 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) dynamics%se_uvBT_theta = 0.0_WP dynamics%se_uvBT_mean = 0.0_WP dynamics%se_uvBT_12 = 0.0_WP - if (dynamics%splitexpl_visc) then + if (dynamics%se_visc) then allocate(dynamics%se_uvBT_stab_hvisc(2, elem_size)) dynamics%se_uvBT_stab_hvisc = 0.0_WP end if - if (dynamics%splitexpl_bottdrag) then + if (dynamics%se_bottdrag) then allocate(dynamics%se_uvBT_stab_bdrag(elem_size)) dynamics%se_uvBT_stab_bdrag = 0.0_WP end if diff --git a/src/write_step_info.F90 b/src/write_step_info.F90 index e1c4d0393..85bcf9b81 100644 --- a/src/write_step_info.F90 +++ b/src/write_step_info.F90 @@ -91,7 +91,7 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) Wvel => dynamics%w(:,:) CFL_z => dynamics%cfl_z(:,:) eta_n => dynamics%eta_n(:) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) d_eta => dynamics%d_eta(:) + if ( .not. dynamics%use_ssh_se_subcycl) d_eta => dynamics%d_eta(:) m_ice => ice%data(2)%values(:) if (mod(istep,outfreq)==0) then @@ -119,7 +119,7 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) loc_eta = loc_eta + areasvol(ulevels_nod2D(n), n)*eta_n(n) loc_hbar = loc_hbar + areasvol(ulevels_nod2D(n), n)*hbar(n) loc_dhbar = loc_dhbar + areasvol(ulevels_nod2D(n), n)*(hbar(n)-hbar_old(n)) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then loc_deta = loc_deta + areasvol(ulevels_nod2D(n), n)*d_eta(n) end if loc_wflux = loc_wflux + areasvol(ulevels_nod2D(n), n)*water_flux(n) @@ -127,7 +127,7 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) #if !defined(__openmp_reproducible) !$OMP END PARALLEL DO #endif - if (dynamics%use_ssh_splitexpl_subcycl) then + if (dynamics%use_ssh_se_subcycl) then loc_deta=loc_dhbar end if @@ -168,7 +168,7 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) call MPI_AllREDUCE(loc , min_vvel , 1, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_FESOM, MPIerr) loc=omp_min_max_sum1(UVnode(2,2,:), 1, myDim_nod2D, 'min', partit) call MPI_AllREDUCE(loc , min_vvel2 , 1, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_FESOM, MPIerr) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then loc=omp_min_max_sum1(d_eta, 1, myDim_nod2D, 'min', partit) else loc=omp_min_max_sum1(hbar-hbar_old, 1, myDim_nod2D, 'min', partit) @@ -204,7 +204,7 @@ subroutine write_step_info(istep, outfreq, ice, dynamics, tracers, partit, mesh) call MPI_AllREDUCE(loc , max_vvel , 1, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_FESOM, MPIerr) loc=omp_min_max_sum1(UVnode(2,2,:), 1, myDim_nod2D, 'max', partit) call MPI_AllREDUCE(loc , max_vvel2 , 1, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_FESOM, MPIerr) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then loc=omp_min_max_sum1(d_eta, 1, myDim_nod2D, 'max', partit) else loc=omp_min_max_sum1(hbar-hbar_old, 1, myDim_nod2D, 'max', partit) @@ -329,7 +329,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) a_ice_old => ice%data(1)%values_old(:) m_ice_old => ice%data(2)%values_old(:) m_snow_old => ice%data(3)%values_old(:) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then d_eta => dynamics%d_eta(:) ssh_rhs => dynamics%ssh_rhs(:) ssh_rhs_old => dynamics%ssh_rhs_old(:) @@ -361,7 +361,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'zbar_3d_n = ',zbar_3d_n(:,n) write(*,*) 'Z_3d_n = ',Z_3d_n(:,n) write(*,*) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then write(*,*) 'ssh_rhs = ',ssh_rhs(n),', ssh_rhs_old = ',ssh_rhs_old(n) end if write(*,*) @@ -414,7 +414,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'd_eta(n) = ',d_eta(n) write(*,*) 'hbar = ',hbar(n) write(*,*) 'hbar_old = ',hbar_old(n) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then write(*,*) 'ssh_rhs = ',ssh_rhs(n) write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) end if @@ -442,7 +442,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'd_eta(n) = ',d_eta(n) write(*,*) 'hbar = ',hbar(n) write(*,*) 'hbar_old = ',hbar_old(n) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then write(*,*) 'ssh_rhs = ',ssh_rhs(n) write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) end if @@ -480,7 +480,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'd_eta(n) = ',d_eta(n) write(*,*) 'hbar = ',hbar(n) write(*,*) 'hbar_old = ',hbar_old(n) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then write(*,*) 'ssh_rhs = ',ssh_rhs(n) write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) end if @@ -530,7 +530,7 @@ subroutine check_blowup(istep, ice, dynamics, tracers, partit, mesh) write(*,*) 'd_eta(n) = ',d_eta(n) write(*,*) 'hbar = ',hbar(n) write(*,*) 'hbar_old = ',hbar_old(n) - if ( .not. dynamics%use_ssh_splitexpl_subcycl) then + if ( .not. dynamics%use_ssh_se_subcycl) then write(*,*) 'ssh_rhs = ',ssh_rhs(n) write(*,*) 'ssh_rhs_old = ',ssh_rhs_old(n) end if From 8aa79ee57e6553fecbb35ea1abd00b960306c906 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 15 Sep 2023 16:28:02 +0200 Subject: [PATCH 39/50] try to make sure that subcycling is consistent with partial cells --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 75 +++++++++++++++++---------- src/oce_ale_vel_rhs.F90 | 2 +- src/oce_dyn.F90 | 18 ++++--- 3 files changed, 61 insertions(+), 34 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 9f5a92b57..8c2773269 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -68,7 +68,6 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) end interface end module - ! ! !_______________________________________________________________________________ @@ -110,7 +109,6 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(node, elem, ed, nz, nl1, ul1, nl2, ul2, nl12, ul12, & !$OMP uv1, uv2, uv12, qc, qu, qd, wu, wv, & !$OMP ednodes, edelem, un1, un2) - !$OMP DO do node=1, myDim_nod2D ul1 = ulevels_nod2D(node) @@ -382,14 +380,18 @@ subroutine momentum_adv_scalar_transpv(dynamics, partit, mesh) ! for energz diagnostic if (dynamics%ldiag_ke) then !we repeat the computation here and there are multiple ways to speed it up !$OMP DO - do elem=1, myDim_elem2D - ul1 = ulevels(elem) - nl1 = nlevels(elem)-1 - dynamics%ke_adv_AB(1:2, ul1:nl1, elem) = dynamics%ke_adv_AB(1:2, ul1:nl1, elem) + elem_area(elem)* & - ( UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(1, elem)) & - + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(2, elem)) & - + UVnode_rhs(1:2, ul1:nl1, elem2D_nodes(3, elem))) / 3.0_WP - end do + do elem=1, myDim_elem2D + ul1 = ulevels(elem) + nl1 = nlevels(elem)-1 + dynamics%ke_adv_AB(1, ul1:nl1, elem) = dynamics%ke_adv_AB(1, ul1:nl1, elem) + elem_area(elem)* & + ( UVnode_rhs(1, ul1:nl1, elem2D_nodes(1, elem)) & + + UVnode_rhs(1, ul1:nl1, elem2D_nodes(2, elem)) & + + UVnode_rhs(1, ul1:nl1, elem2D_nodes(3, elem))) / 3.0_WP / helem(ul1:nl1, elem) + dynamics%ke_adv_AB(2, ul1:nl1, elem) = dynamics%ke_adv_AB(2, ul1:nl1, elem) + elem_area(elem)* & + ( UVnode_rhs(2, ul1:nl1, elem2D_nodes(1, elem)) & + + UVnode_rhs(2, ul1:nl1, elem2D_nodes(2, elem)) & + + UVnode_rhs(2, ul1:nl1, elem2D_nodes(3, elem))) / 3.0_WP / helem(ul1:nl1, elem) + end do !$OMP END DO end if !$OMP END PARALLEL @@ -726,7 +728,11 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ! for in the barotropic equation Fx = g*dt*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) Fy = g*dt*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - hh = sum(helem(nzmin:nzmax, elem)) + + ! total ocean depth H + !PS hh = sum(helem(nzmin:nzmax, elem)) + hh = -zbar_e_bot(elem) + sum(eta_n(elnodes))/3.0_WP + vert_sum_u=vert_sum_u + Fx*hh vert_sum_v=vert_sum_v + Fy*hh @@ -837,7 +843,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) if (dynamics%se_visc) then !_______________________________________________________________________ ! remove viscosity -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, nzmax, hh, len, & +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, ednodes, nzmax, hh, len, & !$OMP vi, update_ubt, update_vbt) !$OMP DO do edge=1, myDim_edge2D+eDim_edge2D @@ -846,11 +852,15 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) if(myList_edge2D(edge)>edge2D_in) cycle ! elem indices that participate in edge - edelem= edge_tri(:,edge) - nzmax = minval(nlevels(edelem)) - hh = -zbar(nzmax) - len = sqrt(sum(elem_area(edelem))) - + edelem = edge_tri(:,edge) + ednodes = edges(:,edge) + nzmax = minval(nlevels(edelem)) + + ! total ocean depth H + !PS hh = -zbar(nzmax) + hh = -sum(zbar_e_bot(edelem))*0.5_WP + sum(hbar(ednodes))*0.5_WP + + len = sqrt(sum(elem_area(edelem))) update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh vi=update_ubt*update_ubt + update_vbt*update_vbt @@ -893,8 +903,12 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) do elem=1, myDim_elem2D elnodes= elem2D_nodes(:,elem) nzmax = nlevels(elem) + + ! total ocean depth H !PS hh = -zbar(nzmax)+sum(eta_n(elnodes))/3.0_WP - hh = -zbar(nzmax) + !PS hh = -zbar(nzmax) + hh = -zbar_e_bot(elem) + sum(hbar(elnodes))/3.0_WP + bottomdrag(elem) = dt*C_d*sqrt(UV(1, nzmax-1, elem)**2 + UV(2, nzmax-1, elem)**2) UVBT_rhs(1, elem)=UVBT_rhs(1, elem) + bottomdrag(elem)*UVBT(1, elem)/hh UVBT_rhs(2, elem)=UVBT_rhs(2, elem) + bottomdrag(elem)*UVBT(2, elem)/hh @@ -932,7 +946,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END PARALLEL DO !___________________________________________________________________ -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, nzmax, hh, len, & +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, ednodes, nzmax, hh, len, & !$OMP vi, update_ubt, update_vbt) !$OMP DO do edge=1, myDim_edge2D+eDim_edge2D @@ -941,12 +955,15 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) if(myList_edge2D(edge)>edge2D_in) cycle ! elem indices that participate in edge - edelem = edge_tri(:,edge) + edelem = edge_tri(:, edge) + ednodes = edges(:, edge) + nzmax = minval(nlevels(edelem)) - nzmax = minval(nlevels(edelem)) - hh = -zbar(nzmax) - len = sqrt(sum(elem_area(edelem))) - + ! total ocean depth H + !PS hh = -zbar(nzmax) + hh = -sum(zbar_e_bot(edelem))*0.5_WP + sum(hbar(ednodes))*0.5_WP + + len = sqrt(sum(elem_area(edelem))) update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh vi=update_ubt*update_ubt + update_vbt*update_vbt @@ -993,9 +1010,12 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! AAA = - dt/M*[ + 0.5*f*e_z x (Ubt^(n+(m+1)/M) + Ubt^(n+(m)/M)) ! - h*H^m*grad_H*eta^((n+m)/M) ! - Rbt-->UVBT_rhs ] + ! total ocean depth H !PS hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth - hh = -zbar(nlevels(elem)) ! Total fluid depth - ff = mesh%coriolis(elem) + !PS hh = -zbar(nlevels(elem)) ! Total fluid depth + hh = -zbar_e_bot(elem) + sum(hbar(elnodes))/3.0_WP + + ff = mesh%coriolis(elem) rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + ff*UVBT(2, elem)) & + BT_inv*UVBT_rhs(1, elem) & @@ -1387,7 +1407,8 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) do node=1, myDim_nod2D+eDim_nod2D nzmin = ulevels_nod2D(node) nzmax = nlevels_nod2D_min(node)-1 - hh_inv=-1.0_WP/zbar(nzmax) + !PS hh_inv=-1.0_WP/zbar(nzmax) + hh_inv=-1.0_WP/zbar_3d_n(nzmax, node) do nz=nzmin, nzmax-1 hnode_new(nz,node)=(zbar(nz)-zbar(nz+1))*(1.0_WP+hh_inv*eta_n(node)) end do diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index 7efade82c..7c8a74cb7 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -400,7 +400,7 @@ subroutine momentum_adv_scalar(dynamics, partit, mesh) ul2 = ulevels(el2) un2(ul2:nl2) = - UV(2,ul2:nl2,el2)*edge_cross_dxdy(3,ed) & - + UV(1,ul2:nl2,el2)*edge_cross_dxdy(4,ed) + + UV(1,ul2:nl2,el2)*edge_cross_dxdy(4,ed) ! fill with zeros to combine the loops ! Usually, no or only a very few levels have to be filled. In this case, diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index 92b3b099b..9746dcce7 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -410,8 +410,10 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) !SD Approximate update for transports. We do not care about accuracy !SD here. --> of course, helem will be better. Do nz=nzmin, nzmax-1 - UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+(U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) - UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+(V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) + !PS UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+(U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) + !PS UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+(V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) + UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+(U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP)*helem(nz, ed) +!PS UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+(V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP)*helem(nz, ed) END DO else Do nz=nzmin, nzmax-1 @@ -543,8 +545,10 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) !SD Approximate update for transports. We do not care about accuracy !SD here. --> of course, helem will be better. do nz=nzmin,nzmax-1 - update_u(nz)=(U_c(nz,el(1))-U_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) - update_v(nz)=(V_c(nz,el(1))-V_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + !PS update_u(nz)=(U_c(nz,el(1))-U_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + !PS update_v(nz)=(V_c(nz,el(1))-V_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + update_u(nz)=(U_c(nz,el(1))-U_c(nz,el(2))) * (helem(nz, el(1))+helem(nz, el(2)))*0.5_WP + update_v(nz)=(V_c(nz,el(1))-V_c(nz,el(2))) * (helem(nz, el(1))+helem(nz, el(2)))*0.5_WP end do else do nz=nzmin,nzmax-1 @@ -692,8 +696,10 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) dynamics%visc_gamma2*vi) & )*len) ! vi=-dt*sqrt(max(dynamics%visc_gamma0, dynamics%visc_gamma1*max(sqrt(vi), dynamics%visc_gamma2*vi))*len) - update_u(nz)=vi*(U_c(nz,el(1))-U_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) - update_v(nz)=vi*(V_c(nz,el(1))-V_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + !PS update_u(nz)=vi*(U_c(nz,el(1))-U_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + !PS update_v(nz)=vi*(V_c(nz,el(1))-V_c(nz,el(2)))*(zbar(nz)-zbar(nz+1)) + update_u(nz)=vi*(U_c(nz,el(1))-U_c(nz,el(2)))*(helem(nz, el(1))+helem(nz, el(2)))*0.5_WP + update_v(nz)=vi*(V_c(nz,el(1))-V_c(nz,el(2)))*(helem(nz, el(1))+helem(nz, el(2)))*0.5_WP end do else do nz=nzmin,nzmax-1 From c5b5c6d2c4af3cdadb3d8480e3529928529ee7ac Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 15 Sep 2023 17:14:22 +0200 Subject: [PATCH 40/50] make subcycling to work with partial cells --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 19 ++++++++++++------- src/oce_dyn.F90 | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 8c2773269..e714e9670 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -731,6 +731,7 @@ subroutine compute_BT_rhs_SE_vtransp(dynamics, partit, mesh) ! total ocean depth H !PS hh = sum(helem(nzmin:nzmax, elem)) + !PS hh = -zbar_e_bot(elem) + sum(eta_n(elnodes))/3.0_WP hh = -zbar_e_bot(elem) + sum(eta_n(elnodes))/3.0_WP vert_sum_u=vert_sum_u + Fx*hh @@ -843,7 +844,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) if (dynamics%se_visc) then !_______________________________________________________________________ ! remove viscosity -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, ednodes, nzmax, hh, len, & +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, ednodes, hh, len, & !$OMP vi, update_ubt, update_vbt) !$OMP DO do edge=1, myDim_edge2D+eDim_edge2D @@ -854,11 +855,12 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! elem indices that participate in edge edelem = edge_tri(:,edge) ednodes = edges(:,edge) - nzmax = minval(nlevels(edelem)) ! total ocean depth H + !PS nzmax = minval(nlevels(edelem)) !PS hh = -zbar(nzmax) - hh = -sum(zbar_e_bot(edelem))*0.5_WP + sum(hbar(ednodes))*0.5_WP + !PS hh = minval(-zbar_e_bot(edelem)) + hh = -sum(zbar_e_bot(edelem))*0.5_WP + sum(hbar(ednodes))*0.5_WP len = sqrt(sum(elem_area(edelem))) update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh @@ -907,6 +909,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! total ocean depth H !PS hh = -zbar(nzmax)+sum(eta_n(elnodes))/3.0_WP !PS hh = -zbar(nzmax) + !PS hh = -zbar_e_bot(elem) hh = -zbar_e_bot(elem) + sum(hbar(elnodes))/3.0_WP bottomdrag(elem) = dt*C_d*sqrt(UV(1, nzmax-1, elem)**2 + UV(2, nzmax-1, elem)**2) @@ -946,7 +949,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) !$OMP END PARALLEL DO !___________________________________________________________________ -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, ednodes, nzmax, hh, len, & +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(edge, edelem, ednodes, hh, len, & !$OMP vi, update_ubt, update_vbt) !$OMP DO do edge=1, myDim_edge2D+eDim_edge2D @@ -957,12 +960,13 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! elem indices that participate in edge edelem = edge_tri(:, edge) ednodes = edges(:, edge) - nzmax = minval(nlevels(edelem)) - ! total ocean depth H + ! total ocean depth H + !PS nzmax = minval(nlevels(edelem)) !PS hh = -zbar(nzmax) + !PS hh = minval(-zbar_e_bot(edelem)) hh = -sum(zbar_e_bot(edelem))*0.5_WP + sum(hbar(ednodes))*0.5_WP - + len = sqrt(sum(elem_area(edelem))) update_ubt=(UVBT(1, edelem(1))-UVBT(1, edelem(2)))/hh update_vbt=(UVBT(2, edelem(1))-UVBT(2, edelem(2)))/hh @@ -1013,6 +1017,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! total ocean depth H !PS hh = -zbar(nlevels(elem))+sum(eta_n(elnodes))/3.0_WP ! Total fluid depth !PS hh = -zbar(nlevels(elem)) ! Total fluid depth + !PS hh = -zbar_e_bot(elem) hh = -zbar_e_bot(elem) + sum(hbar(elnodes))/3.0_WP ff = mesh%coriolis(elem) diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index 9746dcce7..d2a27243d 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -413,7 +413,7 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) !PS UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+(U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) !PS UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+(V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP)*(zbar(nz)-zbar(nz+1)) UV_rhs(1,nz,ed)=UV_rhs(1,nz,ed)+(U_b(nz,ed) -dynamics%visc_easybsreturn*sum(U_c(nz,nelem))/3.0_WP)*helem(nz, ed) -!PS UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+(V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP)*helem(nz, ed) + UV_rhs(2,nz,ed)=UV_rhs(2,nz,ed)+(V_b(nz,ed) -dynamics%visc_easybsreturn*sum(V_c(nz,nelem))/3.0_WP)*helem(nz, ed) END DO else Do nz=nzmin, nzmax-1 From cad7f36aa7cb6ef07b5923573b65c123d4e949b3 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 29 Sep 2023 14:44:43 +0200 Subject: [PATCH 41/50] add halo to helem array, so far was not needed but for split-explicit we need the helem halo in the viscostiy --- src/associate_mesh_ass.h | 8 ++++---- src/oce_ale.F90 | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/associate_mesh_ass.h b/src/associate_mesh_ass.h index e21a32faa..777aa5b49 100644 --- a/src/associate_mesh_ass.h +++ b/src/associate_mesh_ass.h @@ -54,14 +54,14 @@ hnode(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D) => mesh%hnode(:,:) hnode_new(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D) => mesh%hnode_new(:,:) zbar_3d_n(1:mesh%nl, 1:myDim_nod2D+eDim_nod2D) => mesh%zbar_3d_n(:,:) Z_3d_n(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D) => mesh%Z_3d_n(:,:) -helem(1:mesh%nl-1, 1:myDim_elem2D) => mesh%helem(:,:) -bottom_elem_thickness(1:myDim_elem2D) => mesh%bottom_elem_thickness(:) +helem(1:mesh%nl-1, 1:myDim_elem2D+eDim_elem2D) => mesh%helem(:,:) +bottom_elem_thickness(1:myDim_elem2D+eDim_elem2D) => mesh%bottom_elem_thickness(:) bottom_node_thickness(1:myDim_nod2D+eDim_nod2D) => mesh%bottom_node_thickness(:) dhe(1:myDim_elem2D) => mesh%dhe(:) hbar(1:myDim_nod2D+eDim_nod2D) => mesh%hbar(:) hbar_old(1:myDim_nod2D+eDim_nod2D) => mesh%hbar_old(:) -!zbar_n(1:mesh%nl) => mesh%zbar_n -!Z_n(1:mesh%nl-1) => mesh%Z_n +!zbar_n(1:mesh%nl) => mesh%zbar_n +!Z_n(1:mesh%nl-1) => mesh%Z_n zbar_n_bot(1:myDim_nod2D+eDim_nod2D) => mesh%zbar_n_bot(:) zbar_e_bot(1:myDim_elem2D+eDim_elem2D) => mesh%zbar_e_bot(:) zbar_n_srf(1:myDim_nod2D+eDim_nod2D) => mesh%zbar_n_srf(:) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 7d83f28dd..76a6f9951 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -249,7 +249,7 @@ subroutine init_ale(dynamics, partit, mesh) allocate(mesh%hbar_old(myDim_nod2D+eDim_nod2D)) ! helem: layer thickness at elements. It is interpolated from hnode. - allocate(mesh%helem(1:nl-1, myDim_elem2D)) + allocate(mesh%helem(1:nl-1, myDim_elem2D+eDim_nod2D)) ! dhe: The increment of total fluid depth on elements. It is used to update the matrix ! of the ssh operator. @@ -261,7 +261,7 @@ subroutine init_ale(dynamics, partit, mesh) allocate(mesh%Z_3d_n(nl-1,myDim_nod2D+eDim_nod2D)) ! bottom_elem_tickness: changed bottom layer thinkness due to partial cells - allocate(mesh%bottom_elem_thickness(myDim_elem2D)) + allocate(mesh%bottom_elem_thickness(myDim_elem2D+eDim_nod2D)) allocate(mesh%zbar_e_bot(myDim_elem2D+eDim_elem2D)) allocate(mesh%zbar_e_srf(myDim_elem2D+eDim_elem2D)) @@ -277,8 +277,8 @@ subroutine init_ale(dynamics, partit, mesh) hnode_new(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D) => mesh%hnode_new(:,:) zbar_3d_n(1:mesh%nl, 1:myDim_nod2D+eDim_nod2D) => mesh%zbar_3d_n(:,:) Z_3d_n(1:mesh%nl-1, 1:myDim_nod2D+eDim_nod2D) => mesh%Z_3d_n(:,:) - helem(1:mesh%nl-1, 1:myDim_elem2D) => mesh%helem(:,:) - bottom_elem_thickness(1:myDim_elem2D) => mesh%bottom_elem_thickness(:) + helem(1:mesh%nl-1, 1:myDim_elem2D+eDim_nod2D) => mesh%helem(:,:) + bottom_elem_thickness(1:myDim_elem2D+eDim_nod2D) => mesh%bottom_elem_thickness(:) bottom_node_thickness(1:myDim_nod2D+eDim_nod2D) => mesh%bottom_node_thickness(:) dhe(1:myDim_elem2D) => mesh%dhe(:) hbar(1:myDim_nod2D+eDim_nod2D) => mesh%hbar(:) @@ -476,6 +476,7 @@ subroutine init_bottom_elem_thickness(partit, mesh) !___________________________________________________________________________ call exchange_elem(zbar_e_bot, partit) + call exchange_elem(bottom_elem_thickness, partit) end subroutine init_bottom_elem_thickness ! @@ -752,6 +753,7 @@ subroutine init_thickness_ale(dynamics, partit, mesh) USE MOD_PARTIT USE MOD_PARSUP USE MOD_DYN + use g_comm_auto implicit none type(t_dyn) , intent(inout), target :: dynamics type(t_partit), intent(inout), target :: partit @@ -995,7 +997,7 @@ subroutine init_thickness_ale(dynamics, partit, mesh) !___________________________________________________________________________ hnode_new=hnode ! Should be initialized, because only variable part is updated. - + call exchange_elem(helem, partit) !!PS call check_total_volume(partit, mesh) end subroutine init_thickness_ale @@ -1010,6 +1012,7 @@ subroutine update_thickness_ale(partit, mesh) USE MOD_PARSUP use o_ARRAYS use g_config,only: which_ale,lzstar_lev,min_hnode + use g_comm_auto implicit none type(t_partit), intent(inout), target :: partit type(t_mesh) , intent(inout), target :: mesh @@ -1165,6 +1168,7 @@ subroutine update_thickness_ale(partit, mesh) end do !$OMP END PARALLEL DO endif + call exchange_elem(helem, partit) end subroutine update_thickness_ale ! ! From 832804fc4d4be53c9f28fb0e75b90e8afa76a8ba Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 29 Sep 2023 14:45:53 +0200 Subject: [PATCH 42/50] add output message for the split-explicit parameters --- src/oce_setup_step.F90 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 index 0b67b03d9..9eeeeb0b7 100755 --- a/src/oce_setup_step.F90 +++ b/src/oce_setup_step.F90 @@ -450,6 +450,17 @@ SUBROUTINE dynamics_init(dynamics, partit, mesh) dynamics%se_visc_gamma0 = se_visc_gamma0 dynamics%se_visc_gamma1 = se_visc_gamma1 dynamics%se_visc_gamma2 = se_visc_gamma2 + if (mype==0) then + write(*,*) " ___Split-Explicit barotropic subcycling_________", dynamics%se_BTsteps + write(*,*) " se_BTsteps = ", dynamics%se_BTsteps + write(*,*) " se_BTtheta = ", dynamics%se_BTtheta + write(*,*) " se_bottdrag = ", dynamics%se_bottdrag + write(*,*) " se_bdrag_si = ", dynamics%se_bdrag_si + write(*,*) " se_visc = ", dynamics%se_visc + write(*,*) " se_visc_gamma0 = ", dynamics%se_visc_gamma0 + write(*,*) " se_visc_gamma1 = ", dynamics%se_visc_gamma1 + write(*,*) " se_visc_gamma2 = ", dynamics%se_visc_gamma2 + end if end if !___________________________________________________________________________ From ba95af6ab681344c3704cc4958f26c7a12db63b1 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 29 Sep 2023 14:49:52 +0200 Subject: [PATCH 43/50] solve compiler error when split_explicit flag is false --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index e714e9670..f36c9271e 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1023,12 +1023,20 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ff = mesh%coriolis(elem) rx = dtBT*(-g*hh*sum(gradient_sca(1:3,elem)*eta_n(elnodes)) + ff*UVBT(2, elem)) & - + BT_inv*UVBT_rhs(1, elem) & - + BT_inv*(UVBT_harmvisc(1, elem) - bottomdrag(elem)*UVBT(1, elem)/hh) ! <-- stabilization terms + + BT_inv*UVBT_rhs(1, elem) !& +!PS + BT_inv*(UVBT_harmvisc(1, elem) - bottomdrag(elem)*UVBT(1, elem)/hh) ! <-- stabilization terms ry = dtBT*(-g*hh*sum(gradient_sca(4:6,elem)*eta_n(elnodes)) - ff*UVBT(1, elem)) & - + BT_inv*UVBT_rhs(2, elem) & - + BT_inv*(UVBT_harmvisc(2, elem) - bottomdrag(elem)*UVBT(2, elem)/hh) ! <-- stabilization terms + + BT_inv*UVBT_rhs(2, elem) !& +!PS + BT_inv*(UVBT_harmvisc(2, elem) - bottomdrag(elem)*UVBT(2, elem)/hh) ! <-- stabilization terms + if (dynamics%se_visc) then + rx = rx + BT_inv*UVBT_harmvisc(1, elem) + ry = ry + BT_inv*UVBT_harmvisc(2, elem) + end if + if (dynamics%se_bottdrag) then + rx = rx - BT_inv*bottomdrag(elem)*UVBT(1, elem)/hh + ry = ry - BT_inv*bottomdrag(elem)*UVBT(2, elem)/hh + end if ! compute new velocity Ubt^(n+(m+1)/M), Vbt^(n+(m+1)/M) considering ! in terms of Increments (deltaU) and semi-Implicit Coriolis @@ -1071,7 +1079,7 @@ subroutine compute_BT_step_SE_ale(dynamics, partit, mesh) ! ! Semi-Implicit Coriolis a = dtBT*ff*0.5_WP - if (dynamics%se_bdrag_si) then + if ( (dynamics%se_bdrag_si) .and. (dynamics%se_bottdrag) ) then b = 1.0_WP+BT_inv*bottomdrag(elem)/hh else b = 1.0_WP @@ -1433,6 +1441,7 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP end do end do + call exchange_elem(helem, partit) !$OMP END DO !$OMP END PARALLEL end subroutine compute_thickness_zstar From ce0bb641aaa8cb501efe98f6f52d21defdb9f7fc Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 29 Sep 2023 16:06:10 +0200 Subject: [PATCH 44/50] fix openmp error message --- src/oce_ale.F90 | 6 ++++++ src/oce_ale_ssh_splitexpl_subcycl.F90 | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 76a6f9951..0e0a379c5 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -1168,7 +1168,13 @@ subroutine update_thickness_ale(partit, mesh) end do !$OMP END PARALLEL DO endif + + !___________________________________________________________________________ +!$OMP MASTER call exchange_elem(helem, partit) +!$OMP END MASTER +!$OMP BARRIER + end subroutine update_thickness_ale ! ! diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index f36c9271e..5fff2f09b 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1441,9 +1441,15 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP end do end do - call exchange_elem(helem, partit) !$OMP END DO !$OMP END PARALLEL + + !___________________________________________________________________________ +!$OMP MASTER + call exchange_elem(helem, partit) +!$OMP END MASTER +!$OMP BARRIER + end subroutine compute_thickness_zstar From e366c12fe94ba8c96c49d1967a14bcb8797937a0 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 17 Nov 2023 15:55:33 +0100 Subject: [PATCH 45/50] add outpur variable ubt_hvisc and vbt_hvisc --- src/io_meandata.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/io_meandata.F90 b/src/io_meandata.F90 index 2ba3c87f0..11b9c5c22 100644 --- a/src/io_meandata.F90 +++ b/src/io_meandata.F90 @@ -636,6 +636,10 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh) call def_stream((/nl-1,elem2D/), (/nl-1,myDim_elem2D/) , 'v_rhs' , 'merid. transport rhs', '?' , dynamics%uv_rhs(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) call def_stream((/nl-1,elem2D/), (/nl-1,myDim_elem2D/) , 'uh' , 'zonal transport' , '?' , dynamics%se_uvh(1,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) call def_stream((/nl-1,elem2D/), (/nl-1,myDim_elem2D/) , 'vh' , 'merid. transport', '?' , dynamics%se_uvh(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + if (dynamics%se_visc) then + call def_stream(elem2D, myDim_elem2D , 'ubt_hvisc' , 'zonal hvisc. stabil.' , '?' , dynamics%se_uvBT_stab_hvisc(1,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + call def_stream(elem2D, myDim_elem2D , 'vbt_hvisc' , 'merid. hvisc. stabil.' , '?' , dynamics%se_uvBT_stab_hvisc(2,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh) + end if end if From ab2272b25829e4a9d1ce845d13deeed70ca72d6a Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 17 Nov 2023 15:57:26 +0100 Subject: [PATCH 46/50] kick out second triming, in moment does not seem to have an advantage --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 112 +++++++++++++++++++------- 1 file changed, 85 insertions(+), 27 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 5fff2f09b..285b48ac3 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1251,7 +1251,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 !PS !___________________________________________________________ !PS ! Update transport velocities: U^(n+1/2,**) = U^(n+1/2,*) + U_rhs !PS UVh(1, nz, elem)=UVh(1, nz, elem)+UV_rhs(1, nz, elem) @@ -1270,8 +1270,16 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) hh_inv= hh_inv+helem(nz,elem) end do ! --> do nz=nzmin, nzmax + ! inverse total heigh over which the trimming part is distributed --> + ! here exclude bottom cell hh_inv=1.0_WP/hh_inv + ! sum transport over entire water column including bottom cell need + ! the trimming with the barotropic transport + nz = nzmax + ubar = ubar+UVh(1, nz, elem) + UV_rhs(1, nz, elem) + vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) + !___________________________________________________________________ if (dynamics%ldiag_ke) then do nz=nzmin, nzmax @@ -1283,8 +1291,13 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! | | ! v v ! udiff usum - usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv - usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + if (nz==nzmax) then + usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + else + usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + end if ! transform: transport vel --> velocity usum(1) = usum(1)/helem(nz,elem) usum(2) = usum(2)/helem(nz,elem) @@ -1327,7 +1340,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! mode solver in FESOM2, ! U_k^(n+1/2) = U^(n+1/2,**) ! + [<>^(n+1/2) - sum(k, U_k^(n+1/2,**)] * h_k^(n+1/2)/sum(k,h_k^(n+1/2)) - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 !PS UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv !PS UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv @@ -1335,6 +1348,14 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) end do ! --> do nz=nzmin, nzmax + + ! apply no trimming in the bottom cell + nz = nzmax + UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) + end do ! --> do elem=1, myDim_elem2D !$OMP END DO !$OMP END PARALLEL @@ -1349,7 +1370,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) !___________________________________________________________________________ ! Trim to make velocity consistent with BT velocity at n+1/2 ! Velocity will be used to advance momentum - else + elseif (mode==2) then !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) !$OMP DO do elem=1, myDim_elem2D @@ -1360,20 +1381,28 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 ubar=ubar+UVh(1, nz, elem) vbar=vbar+UVh(2, nz, elem) hh_inv=hh_inv+helem(nz,elem) end do + nz=nzmax + ubar=ubar+UVh(1, nz, elem) + vbar=vbar+UVh(2, nz, elem) !___________________________________________________________________ hh_inv=1.0_WP/hh_inv ! Postpone 2nd order, just use available thickness - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_12(1, elem)-ubar)*helem(nz,elem)*hh_inv UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_12(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do + nz=nzmax + UVh(1, nz, elem)= UVh(1, nz, elem) + UVh(2, nz, elem)= UVh(2, nz, elem) + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do !$OMP END DO !$OMP END PARALLEL @@ -1384,6 +1413,35 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) call exchange_elem(UV , partit) ! Check if this is needed !$OMP END MASTER !$OMP BARRIER + + !___________________________________________________________________________ + ! skip trimming only adnvance velocity + ! Velocity will be used to advance momentum + elseif (mode==3) then +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) +!$OMP DO + do elem=1, myDim_elem2D + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + + !___________________________________________________________________ + do nz=nzmin, nzmax + UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection + end do + end do +!$OMP END DO +!$OMP END PARALLEL + + !_______________________________________________________________________ +!$OMP MASTER + call exchange_elem(UVh, partit) ! + call exchange_elem(UV , partit) ! Check if this is needed +!$OMP END MASTER +!$OMP BARRIER + end if ! --> if (mode==1) then end subroutine update_trim_vel_ale_vtransp ! @@ -1429,26 +1487,26 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) !$OMP END DO !$OMP END PARALLEL - !___________________________________________________________________________ - ! --> update mean layer thinkness at element -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) -!$OMP DO - do elem=1, myDim_elem2D - nzmin = ulevels(elem) - nzmax = nlevels(elem)-1 - elnodes=elem2D_nodes(:,elem) - do nz=nzmin, nzmax-1 - helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP - end do - end do -!$OMP END DO -!$OMP END PARALLEL - - !___________________________________________________________________________ -!$OMP MASTER - call exchange_elem(helem, partit) -!$OMP END MASTER -!$OMP BARRIER +!PS !___________________________________________________________________________ +!PS ! --> update mean layer thinkness at element +!PS !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) +!PS !$OMP DO +!PS do elem=1, myDim_elem2D +!PS nzmin = ulevels(elem) +!PS nzmax = nlevels(elem)-1 +!PS elnodes=elem2D_nodes(:,elem) +!PS do nz=nzmin, nzmax-1 +!PS helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP +!PS end do +!PS end do +!PS !$OMP END DO +!PS !$OMP END PARALLEL +!PS +!PS !___________________________________________________________________________ +!PS !$OMP MASTER +!PS call exchange_elem(helem, partit) +!PS !$OMP END MASTER +!PS !$OMP BARRIER end subroutine compute_thickness_zstar From 12bb0c64c0587bfd78430ec606eace57487f27fa Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 17 Nov 2023 16:03:13 +0100 Subject: [PATCH 47/50] Revert " kick out second triming, in moment does not seem to have an advantage" This reverts commit ab2272b25829e4a9d1ce845d13deeed70ca72d6a. --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 112 +++++++------------------- 1 file changed, 27 insertions(+), 85 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 285b48ac3..5fff2f09b 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1251,7 +1251,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - do nz=nzmin, nzmax-1 + do nz=nzmin, nzmax !PS !___________________________________________________________ !PS ! Update transport velocities: U^(n+1/2,**) = U^(n+1/2,*) + U_rhs !PS UVh(1, nz, elem)=UVh(1, nz, elem)+UV_rhs(1, nz, elem) @@ -1270,16 +1270,8 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) hh_inv= hh_inv+helem(nz,elem) end do ! --> do nz=nzmin, nzmax - ! inverse total heigh over which the trimming part is distributed --> - ! here exclude bottom cell hh_inv=1.0_WP/hh_inv - ! sum transport over entire water column including bottom cell need - ! the trimming with the barotropic transport - nz = nzmax - ubar = ubar+UVh(1, nz, elem) + UV_rhs(1, nz, elem) - vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) - !___________________________________________________________________ if (dynamics%ldiag_ke) then do nz=nzmin, nzmax @@ -1291,13 +1283,8 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! | | ! v v ! udiff usum - if (nz==nzmax) then - usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) - usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) - else - usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv - usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv - end if + usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv ! transform: transport vel --> velocity usum(1) = usum(1)/helem(nz,elem) usum(2) = usum(2)/helem(nz,elem) @@ -1340,7 +1327,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! mode solver in FESOM2, ! U_k^(n+1/2) = U^(n+1/2,**) ! + [<>^(n+1/2) - sum(k, U_k^(n+1/2,**)] * h_k^(n+1/2)/sum(k,h_k^(n+1/2)) - do nz=nzmin, nzmax-1 + do nz=nzmin, nzmax !PS UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv !PS UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv @@ -1348,14 +1335,6 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) end do ! --> do nz=nzmin, nzmax - - ! apply no trimming in the bottom cell - nz = nzmax - UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) - UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) - UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed - UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) - end do ! --> do elem=1, myDim_elem2D !$OMP END DO !$OMP END PARALLEL @@ -1370,7 +1349,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) !___________________________________________________________________________ ! Trim to make velocity consistent with BT velocity at n+1/2 ! Velocity will be used to advance momentum - elseif (mode==2) then + else !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) !$OMP DO do elem=1, myDim_elem2D @@ -1381,28 +1360,20 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - do nz=nzmin, nzmax-1 + do nz=nzmin, nzmax ubar=ubar+UVh(1, nz, elem) vbar=vbar+UVh(2, nz, elem) hh_inv=hh_inv+helem(nz,elem) end do - nz=nzmax - ubar=ubar+UVh(1, nz, elem) - vbar=vbar+UVh(2, nz, elem) !___________________________________________________________________ hh_inv=1.0_WP/hh_inv ! Postpone 2nd order, just use available thickness - do nz=nzmin, nzmax-1 + do nz=nzmin, nzmax UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_12(1, elem)-ubar)*helem(nz,elem)*hh_inv UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_12(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do - nz=nzmax - UVh(1, nz, elem)= UVh(1, nz, elem) - UVh(2, nz, elem)= UVh(2, nz, elem) - UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed - UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do !$OMP END DO !$OMP END PARALLEL @@ -1413,35 +1384,6 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) call exchange_elem(UV , partit) ! Check if this is needed !$OMP END MASTER !$OMP BARRIER - - !___________________________________________________________________________ - ! skip trimming only adnvance velocity - ! Velocity will be used to advance momentum - elseif (mode==3) then -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) -!$OMP DO - do elem=1, myDim_elem2D - nzmin = ulevels(elem) - nzmax = nlevels(elem)-1 - - !___________________________________________________________________ - do nz=nzmin, nzmax - UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) - UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) - UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed - UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection - end do - end do -!$OMP END DO -!$OMP END PARALLEL - - !_______________________________________________________________________ -!$OMP MASTER - call exchange_elem(UVh, partit) ! - call exchange_elem(UV , partit) ! Check if this is needed -!$OMP END MASTER -!$OMP BARRIER - end if ! --> if (mode==1) then end subroutine update_trim_vel_ale_vtransp ! @@ -1487,26 +1429,26 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) !$OMP END DO !$OMP END PARALLEL -!PS !___________________________________________________________________________ -!PS ! --> update mean layer thinkness at element -!PS !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) -!PS !$OMP DO -!PS do elem=1, myDim_elem2D -!PS nzmin = ulevels(elem) -!PS nzmax = nlevels(elem)-1 -!PS elnodes=elem2D_nodes(:,elem) -!PS do nz=nzmin, nzmax-1 -!PS helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP -!PS end do -!PS end do -!PS !$OMP END DO -!PS !$OMP END PARALLEL -!PS -!PS !___________________________________________________________________________ -!PS !$OMP MASTER -!PS call exchange_elem(helem, partit) -!PS !$OMP END MASTER -!PS !$OMP BARRIER + !___________________________________________________________________________ + ! --> update mean layer thinkness at element +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) +!$OMP DO + do elem=1, myDim_elem2D + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + elnodes=elem2D_nodes(:,elem) + do nz=nzmin, nzmax-1 + helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP + end do + end do +!$OMP END DO +!$OMP END PARALLEL + + !___________________________________________________________________________ +!$OMP MASTER + call exchange_elem(helem, partit) +!$OMP END MASTER +!$OMP BARRIER end subroutine compute_thickness_zstar From 05dad380cd7c2bc9d7db22614d92255ebc940870 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 17 Nov 2023 16:04:45 +0100 Subject: [PATCH 48/50] kick out second triming, in moment does not seem to have an advantage --- src/oce_ale.F90 | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 0e0a379c5..bcc5e3297 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -2898,13 +2898,13 @@ subroutine compute_CFLz(dynamics, partit, mesh) nzmax = nlevels_nod2D(node)-1 do nz=nzmin, nzmax if (abs(CFL_z(nz,node)-cflmax) < 1.e-12 .and. CFL_z(nz,node) > 1.75_WP .and. CFL_z(nz,node)<=2.5_WP ) then - print '(A, A, F4.2, A, I6, A, F7.2,A,F6.2, A, I3,I3)', achar(27)//'[33m'//' --> WARNING CFLz>1.75:'//achar(27)//'[0m',& + print '(A, A, F4.2, A, I6, A, F7.2,A,F6.2, A, I3,I3,I3)', achar(27)//'[33m'//' --> WARNING CFLz>1.75:'//achar(27)//'[0m',& 'CFLz_max=',cflmax,',mstep=',mstep,',glon/glat=',geo_coord_nod2D(1,node)/rad,'/',geo_coord_nod2D(2,node)/rad,& - ',nz/nzmin=',nz,nzmin + ',nz/nzmin/nzmax=',nz,nzmin,nzmax elseif (abs(CFL_z(nz,node)-cflmax) < 1.e-12 .and. CFL_z(nz,node) > 2.5_WP) then - print '(A, A, F4.2, A, I6, A, F7.2,A,F6.2, A, I3,I3)', achar(27)//'[31m'//' --> WARNING CFLz>2.5:'//achar(27)//'[0m',& + print '(A, A, F4.2, A, I6, A, F7.2,A,F6.2, A, I3,I3,I3)', achar(27)//'[31m'//' --> WARNING CFLz>2.5:'//achar(27)//'[0m',& 'CFLz_max=',cflmax,',mstep=',mstep,',glon/glat=',geo_coord_nod2D(1,node)/rad,'/',geo_coord_nod2D(2,node)/rad,& - ',nz/nzmin=',nz,nzmin + ',nz/nzmin/nzmax=',nz,nzmin,nzmax !!PS write(*,*) '***********************************************************' !!PS write(*,*) 'max. CFL_z = ', cflmax, ' mype = ', mype !!PS write(*,*) 'mstep = ', mstep @@ -3665,7 +3665,7 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) t3=MPI_Wtime() ! Trim U to be consistent with BT transport - call update_trim_vel_ale_vtransp(1, dynamics, partit, mesh) + call update_trim_vel_ale_vtransp(1, dynamics, partit, mesh) t4=MPI_Wtime() t5=t4 end if ! --> if (.not. dynamics%use_ssh_se_subcycl) then @@ -3730,12 +3730,12 @@ subroutine oce_timestep_ale(n, ice, dynamics, tracers, partit, mesh) call update_thickness_ale(partit, mesh) t9=MPI_Wtime() - !___________________________________________________________________________ - ! Trim to make velocity consistent with BT velocity at n+1/2 - ! Velocity will be used to advance momentum - if (dynamics%use_ssh_se_subcycl) then - call update_trim_vel_ale_vtransp(2, dynamics, partit, mesh) - end if +!PS !___________________________________________________________________________ +!PS ! Trim to make velocity consistent with BT velocity at n+1/2 +!PS ! Velocity will be used to advance momentum +!PS if (dynamics%use_ssh_se_subcycl) then +!PS call update_trim_vel_ale_vtransp(2, dynamics, partit, mesh) +!PS end if !___________________________________________________________________________ ! write out global fields for debugging From 2b5d59b94144b8c437bd5f02202e6696c99df902 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 17 Nov 2023 16:14:34 +0100 Subject: [PATCH 49/50] make sure that trimming is not done in the bottom cell, do not update helem to early since it still needed for tracer and divergence computation --- src/oce_ale_ssh_splitexpl_subcycl.F90 | 112 +++++++++++++++++++------- 1 file changed, 85 insertions(+), 27 deletions(-) diff --git a/src/oce_ale_ssh_splitexpl_subcycl.F90 b/src/oce_ale_ssh_splitexpl_subcycl.F90 index 5fff2f09b..285b48ac3 100644 --- a/src/oce_ale_ssh_splitexpl_subcycl.F90 +++ b/src/oce_ale_ssh_splitexpl_subcycl.F90 @@ -1251,7 +1251,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 !PS !___________________________________________________________ !PS ! Update transport velocities: U^(n+1/2,**) = U^(n+1/2,*) + U_rhs !PS UVh(1, nz, elem)=UVh(1, nz, elem)+UV_rhs(1, nz, elem) @@ -1270,8 +1270,16 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) hh_inv= hh_inv+helem(nz,elem) end do ! --> do nz=nzmin, nzmax + ! inverse total heigh over which the trimming part is distributed --> + ! here exclude bottom cell hh_inv=1.0_WP/hh_inv + ! sum transport over entire water column including bottom cell need + ! the trimming with the barotropic transport + nz = nzmax + ubar = ubar+UVh(1, nz, elem) + UV_rhs(1, nz, elem) + vbar = vbar+UVh(2, nz, elem) + UV_rhs(2, nz, elem) + !___________________________________________________________________ if (dynamics%ldiag_ke) then do nz=nzmin, nzmax @@ -1283,8 +1291,13 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! | | ! v v ! udiff usum - usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv - usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + if (nz==nzmax) then + usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + else + usum(1) = 2.0_WP*UVh(1,nz,elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv + usum(2) = 2.0_WP*UVh(2,nz,elem) + UV_rhs(2, nz, elem) + (UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv + end if ! transform: transport vel --> velocity usum(1) = usum(1)/helem(nz,elem) usum(2) = usum(2)/helem(nz,elem) @@ -1327,7 +1340,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ! mode solver in FESOM2, ! U_k^(n+1/2) = U^(n+1/2,**) ! + [<>^(n+1/2) - sum(k, U_k^(n+1/2,**)] * h_k^(n+1/2)/sum(k,h_k^(n+1/2)) - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 !PS UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv !PS UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_mean(2, elem)-vbar)*helem(nz,elem)*hh_inv UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + (UVBT_mean(1, elem)-ubar)*helem(nz,elem)*hh_inv @@ -1335,6 +1348,14 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) end do ! --> do nz=nzmin, nzmax + + ! apply no trimming in the bottom cell + nz = nzmax + UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) + end do ! --> do elem=1, myDim_elem2D !$OMP END DO !$OMP END PARALLEL @@ -1349,7 +1370,7 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) !___________________________________________________________________________ ! Trim to make velocity consistent with BT velocity at n+1/2 ! Velocity will be used to advance momentum - else + elseif (mode==2) then !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) !$OMP DO do elem=1, myDim_elem2D @@ -1360,20 +1381,28 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) ubar = 0.0_WP vbar = 0.0_WP hh_inv = 0.0_WP - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 ubar=ubar+UVh(1, nz, elem) vbar=vbar+UVh(2, nz, elem) hh_inv=hh_inv+helem(nz,elem) end do + nz=nzmax + ubar=ubar+UVh(1, nz, elem) + vbar=vbar+UVh(2, nz, elem) !___________________________________________________________________ hh_inv=1.0_WP/hh_inv ! Postpone 2nd order, just use available thickness - do nz=nzmin, nzmax + do nz=nzmin, nzmax-1 UVh(1, nz, elem)= UVh(1, nz, elem)+(UVBT_12(1, elem)-ubar)*helem(nz,elem)*hh_inv UVh(2, nz, elem)= UVh(2, nz, elem)+(UVBT_12(2, elem)-vbar)*helem(nz,elem)*hh_inv UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do + nz=nzmax + UVh(1, nz, elem)= UVh(1, nz, elem) + UVh(2, nz, elem)= UVh(2, nz, elem) + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection end do !$OMP END DO !$OMP END PARALLEL @@ -1384,6 +1413,35 @@ subroutine update_trim_vel_ale_vtransp(mode, dynamics, partit, mesh) call exchange_elem(UV , partit) ! Check if this is needed !$OMP END MASTER !$OMP BARRIER + + !___________________________________________________________________________ + ! skip trimming only adnvance velocity + ! Velocity will be used to advance momentum + elseif (mode==3) then +!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, nz, nzmin, nzmax, ubar, vbar, hh_inv) +!$OMP DO + do elem=1, myDim_elem2D + nzmin = ulevels(elem) + nzmax = nlevels(elem)-1 + + !___________________________________________________________________ + do nz=nzmin, nzmax + UVh(1, nz, elem)= UVh(1, nz, elem) + UV_rhs(1, nz, elem) + UVh(2, nz, elem)= UVh(2, nz, elem) + UV_rhs(2, nz, elem) + UV( 1, nz, elem)= UVh(1, nz, elem)/helem(nz,elem) ! velocities are still needed + UV( 2, nz, elem)= UVh(2, nz, elem)/helem(nz,elem) ! to compute momentum advection + end do + end do +!$OMP END DO +!$OMP END PARALLEL + + !_______________________________________________________________________ +!$OMP MASTER + call exchange_elem(UVh, partit) ! + call exchange_elem(UV , partit) ! Check if this is needed +!$OMP END MASTER +!$OMP BARRIER + end if ! --> if (mode==1) then end subroutine update_trim_vel_ale_vtransp ! @@ -1429,26 +1487,26 @@ subroutine compute_thickness_zstar(dynamics, partit, mesh) !$OMP END DO !$OMP END PARALLEL - !___________________________________________________________________________ - ! --> update mean layer thinkness at element -!$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) -!$OMP DO - do elem=1, myDim_elem2D - nzmin = ulevels(elem) - nzmax = nlevels(elem)-1 - elnodes=elem2D_nodes(:,elem) - do nz=nzmin, nzmax-1 - helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP - end do - end do -!$OMP END DO -!$OMP END PARALLEL - - !___________________________________________________________________________ -!$OMP MASTER - call exchange_elem(helem, partit) -!$OMP END MASTER -!$OMP BARRIER +!PS !___________________________________________________________________________ +!PS ! --> update mean layer thinkness at element +!PS !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) +!PS !$OMP DO +!PS do elem=1, myDim_elem2D +!PS nzmin = ulevels(elem) +!PS nzmax = nlevels(elem)-1 +!PS elnodes=elem2D_nodes(:,elem) +!PS do nz=nzmin, nzmax-1 +!PS helem(nz,elem)=sum(hnode_new(nz,elnodes))/3.0_WP +!PS end do +!PS end do +!PS !$OMP END DO +!PS !$OMP END PARALLEL +!PS +!PS !___________________________________________________________________________ +!PS !$OMP MASTER +!PS call exchange_elem(helem, partit) +!PS !$OMP END MASTER +!PS !$OMP BARRIER end subroutine compute_thickness_zstar From 56486f0629143d0f339a1864eb8dc1898fd87d08 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 20 Nov 2023 11:59:29 +0100 Subject: [PATCH 50/50] actualise namelist.dyn --- config/namelist.dyn | 8 ++++---- work/job_albedo_chain | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config/namelist.dyn b/config/namelist.dyn index 4a598ee22..35a8b35f0 100644 --- a/config/namelist.dyn +++ b/config/namelist.dyn @@ -23,13 +23,13 @@ wsplit_maxcfl= 1.0 ! maximum allowed CFL criteria in vertical (0.5 < w_max_c ldiag_KE=.false. ! activates energy diagnostics use_ssh_se_subcycl = .false. -se_BTsteps = 50 !50 -se_BTtheta = 0.14 ! default: 0.14, core2 stability plateu@ 0.37 +se_BTsteps = 50 +se_BTtheta = 0.14 ! default: 0.14, se_bottdrag = .true. se_bdrag_si = .true. ! bottomdrag semi-implicite/explicite se_visc = .true. -se_visc_gamma0 = 10 ! 150 -se_visc_gamma1 = 19500 ! 150 +se_visc_gamma0 = 10 +se_visc_gamma1 = 19500 !19500 (core2@32spd), 2750 (core2@72spd) se_visc_gamma2 = 0 / diff --git a/work/job_albedo_chain b/work/job_albedo_chain index 1bcf0eefc..81f55c259 100755 --- a/work/job_albedo_chain +++ b/work/job_albedo_chain @@ -27,11 +27,11 @@ chain_s=1 # starting chain id # time frame of model simulation # ___COREv2___ -year_s=1948 -year_e=2009 +# year_s=1948 +# year_e=2009 # ___JRA55____ -#year_s=1958 -#year_e=2018 +year_s=1958 +year_e=2018 prescribe_rlen=0 # run length in namelist.config --> if 0 value from namelist.config is taken fedit=1 @@ -226,7 +226,7 @@ if [ $is_newsimul -eq 1 ] ; then #___BACKUP NAMELIST.* FILES INTO RESULT DIRECTORY_______________________ cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \ - namelist.cvmix ${dname_result}/. + namelist.dyn namelist.tra namelist.cvmix ${dname_result}/. cp fesom.x ${dname_result}/. #___BACKUP SRC FILES INTO RESULT DIRECTORY______________________________