sync threads before and after MPI exchanges

src/gen_support.F90

@@ -85,11 +85,7 @@ subroutine smooth_nod2D(arr, N, partit, mesh)
        arr(node)=work_array(node)
     ENDDO
 !$OMP END PARALLEL DO
-!$OMP BARRIER
-!$OMP MASTER
     call exchange_nod(arr, partit)
-!$OMP END MASTER
-!$OMP BARRIER
   END DO
   deallocate(work_array)
 end subroutine smooth_nod2D

src/ice_EVP.F90

@@ -819,9 +819,6 @@ subroutine EVPdynamics(ice, partit, mesh)
         !_______________________________________________________________________
         call exchange_nod(U_ice,V_ice,partit, luse_g2g = .true.)
 
-!ifndef ENABLE_OPENACC
-!$OMP BARRIER
-!endif
     END DO !--> do shortstep=1, ice%evp_rheol_steps
 
 end subroutine EVPdynamics

src/ice_fct.F90

@@ -337,9 +337,6 @@ subroutine ice_solve_low_order(ice, partit, mesh)
     call exchange_nod(m_templ, partit, luse_g2g = .true.)
 #endif
 
-#ifndef ENABLE_OPENACC
-!$OMP BARRIER
-#endif
 end subroutine ice_solve_low_order
 !
 !
@@ -417,9 +414,6 @@ subroutine ice_solve_high_order(ice, partit, mesh)
 #if defined (__oifs) || defined (__ifsinterface)
     call exchange_nod(dm_temp, partit, luse_g2g = .true.)
 #endif /* (__oifs) */
-#ifndef ENABLE_OPENACC
-!$OMP BARRIER
-#endif
     !___________________________________________________________________________
     !iterate
 
@@ -482,9 +476,6 @@ subroutine ice_solve_high_order(ice, partit, mesh)
 #if defined (__oifs) || defined (__ifsinterface)
         call exchange_nod(dm_temp, partit, luse_g2g = .true.)
 #endif /* (__oifs) */
-#ifndef ENABLE_OPENACC
-!$OMP BARRIER
-#endif
     end do
 end subroutine ice_solve_high_order
 !
@@ -838,6 +829,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh)
 !$ACC wait
 
 #if defined(_OPENMP)
+!$OMP BARRIER
 !$OMP MASTER
 #endif
     call exchange_nod(icepminus, icepplus, partit, luse_g2g = .true.)
@@ -1124,7 +1116,6 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh)
 
 !$ACC END DATA
 
-!$OMP BARRIER
 end subroutine ice_fem_fct
 !
 !
@@ -1517,9 +1508,6 @@ subroutine ice_update_for_div(ice, partit, mesh)
 #if defined (__oifs) || defined (__ifsinterface)
     call exchange_nod(dm_temp, partit, luse_g2g = .true.)
 #endif /* (__oifs) */
-#ifndef ENABLE_OPENACC
-!$OMP BARRIER
-#endif
     !___________________________________________________________________________
     !iterate
     do n=1,num_iter_solve-1
@@ -1586,9 +1574,6 @@ subroutine ice_update_for_div(ice, partit, mesh)
 #if defined (__oifs) || defined (__ifsinterface)
         call exchange_nod(dm_temp, partit, luse_g2g = .true.)
 #endif /* (__oifs) */
-#ifndef ENABLE_OPENACC
-!$OMP BARRIER
-#endif
     end do
 
 #ifndef ENABLE_OPENACC

src/ice_maEVP.F90

@@ -856,16 +856,20 @@ subroutine EVPdynamics_m(ice, partit, mesh)
         end do ! --> do ed=1,myDim_edge2D
 !$OMP END DO
         !_______________________________________________________________________
+!$OMP BARRIER
 !$OMP MASTER
         call exchange_nod_begin(u_ice_aux, v_ice_aux, partit)
 !$OMP END MASTER
 !$OMP BARRIER
+! Some thing wierd about why init this during communication 
+! can't it be moved after exchange_end?
 !$OMP DO
         do row=1, myDim_nod2d
            u_rhs_ice(row)=0.0_WP
            v_rhs_ice(row)=0.0_WP
         end do
 !$OMP END DO
+!$OMP BARRIER
 !$OMP MASTER
         call exchange_nod_end(partit)
 !$OMP END MASTER

src/ice_oce_coupling.F90

@@ -350,7 +350,6 @@ subroutine oce_fluxes(ice, dynamics, tracers, partit, mesh)
     if (use_cavity) then
         call cavity_heat_water_fluxes_3eq(ice, dynamics, tracers, partit, mesh)   
         call exchange_nod(heat_flux, water_flux, partit)
-!$OMP BARRIER
     end if 
 
     !___________________________________________________________________________

src/ice_thermo_oce.F90

@@ -226,6 +226,7 @@ subroutine thermodynamics(ice, partit, mesh)
         ustar_aux(i)=sqrt(ustar*ice%cd_oce_ice)
     end do
 !$OMP END DO
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_nod(ustar_aux, partit)
 !$OMP END MASTER

src/oce_adv_tra_driver.F90

@@ -246,7 +246,6 @@ subroutine do_oce_adv_tra(dt, vel, w, wi, we, tr_num, dynamics, tracers, partit,
             call adv_tra_ver_upw1(w, ttf, partit, mesh, adv_flux_ver, o_init_zero=.true.)
         end if
         call exchange_nod(fct_LO, partit, luse_g2g = .true.)
-!$OMP BARRIER
     end if
     do_zero_flux=.true.
     if (trim(tracers%data(tr_num)%tra_adv_lim)=='FCT') do_zero_flux=.false.

src/oce_adv_tra_fct.F90

@@ -397,6 +397,7 @@ subroutine oce_tra_adv_fct(dt, ttf, lo, adf_h, adf_v, fct_ttf_min, fct_ttf_max,
     !$ACC END PARALLEL LOOP
 #endif
     ! fct_minus and fct_plus must be known to neighbouring PE
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_nod(fct_plus, fct_minus, partit, luse_g2g = .true.)
 !$OMP END MASTER

src/oce_ale.F90

@@ -1803,7 +1803,6 @@ subroutine compute_ssh_rhs_ale(dynamics, partit, mesh)
     end if
 !$OMP END PARALLEL
     call exchange_nod(ssh_rhs, partit)
-!$OMP BARRIER
 end subroutine compute_ssh_rhs_ale
 !
 !
@@ -1926,7 +1925,6 @@ subroutine compute_hbar_ale(dynamics, partit, mesh)
         end do
 !$OMP END PARALLEL DO
         call exchange_nod(ssh_rhs_old, partit)
-!$OMP BARRIER
     end if 
     !___________________________________________________________________________
     ! update the thickness
@@ -1943,7 +1941,6 @@ subroutine compute_hbar_ale(dynamics, partit, mesh)
     end do
 !$OMP END PARALLEL DO
     call exchange_nod(hbar, partit)
-!$OMP BARRIER
     !___________________________________________________________________________
     ! fill the array for updating the stiffness matrix
 !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, elnodes)
@@ -2486,7 +2483,6 @@ subroutine vert_vel_ale(dynamics, partit, mesh)
     call exchange_nod(Wvel, partit)
     call exchange_nod(hnode_new, partit)   ! Or extend cycles above  
     if (Fer_GM) call exchange_nod(fer_Wvel, partit)
-!$OMP BARRIER    
     !___________________________________________________________________________
     ! calc vertical CFL criteria for debugging purpose and vertical Wvel splitting
 !$OMP PARALLEL DO

src/oce_ale_mixing_kpp.F90

@@ -374,7 +374,6 @@ SUBROUTINE oce_mixing_KPP(viscAE, diffK, dynamics, tracers, partit, mesh)
          call smooth_nod(blmc(:,:,j), 3, partit, mesh)
         end do
      end if
-!$OMP BARRIER
 ! then combine blmc and viscA/diffK
 !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax)
      DO node=1, myDim_nod2D
@@ -400,7 +399,6 @@ SUBROUTINE oce_mixing_KPP(viscAE, diffK, dynamics, tracers, partit, mesh)
 
      ! OVER ELEMENTS 
      call exchange_nod(viscA, partit) !Warning: don't forget to communicate before averaging on elements!!!
-!$OMP BARRIER
      minmix=3.0e-3_WP
 !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax)
      DO elem    = 1, myDim_elem2D
@@ -597,7 +595,6 @@ SUBROUTINE bldepth(partit, mesh)
     call exchange_nod(hbl, partit)
     call smooth_nod(hbl, 3, partit, mesh)
   end if
-!$OMP BARRIER
 
 !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax, dzup, coeff_sw)
   DO node=1, myDim_nod2D
@@ -784,7 +781,6 @@ subroutine ri_iwmix(viscA, diffK, dynamics, tracers, partit, mesh)
      IF (smooth_Ri_hor) then
        call smooth_nod(diffK(:,:,1), 3, partit, mesh)
      END IF
-!$OMP BARRIER
     !___________________________________________________________________________
     ! compute viscA and diffK
 !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax, Rigg, ratio, frit, Kv0_b)

src/oce_ale_pressure_bv.F90

@@ -491,7 +491,6 @@ subroutine pressure_bv(tracers, partit, mesh)
         end if
     end do
 !$OMP END DO
-!$OMP BARRIER
 !$OMP END PARALLEL
 call smooth_nod (bvfreq, 1, partit, mesh)
 end subroutine pressure_bv
@@ -2835,7 +2834,6 @@ subroutine sw_alpha_beta(TF1,SF1, partit, mesh)
    end do
  end do
 !$OMP END DO
-!$OMP BARRIER
 !$OMP END PARALLEL
 call exchange_nod(sw_alpha, partit)
 call exchange_nod(sw_beta, partit)
@@ -2925,7 +2923,6 @@ subroutine compute_sigma_xy(TF1,SF1, partit, mesh)
         sigma_xy(2,uln:nln,n) = (-sw_alpha(uln:nln,n)*ty(uln:nln)+sw_beta(uln:nln,n)*sy(uln:nln))/vol(uln:nln)*density_0
   END DO
 !$OMP END DO
-!$OMP BARRIER
 !$OMP END PARALLEL
 ! call exchange_nod(sigma_xy, partit)
 CALL MPI_BARRIER(MPI_COMM_FESOM,MPIerr)
@@ -2985,7 +2982,6 @@ subroutine compute_neutral_slope(partit, mesh)
         enddo
     enddo
 !$OMP END DO
-!$OMP BARRIER
 !$OMP END PARALLEL
     call exchange_nod(neutral_slope, partit)
     call exchange_nod(slope_tapered, partit)

src/oce_ale_tracer.F90

@@ -243,7 +243,6 @@ subroutine solve_tracers_ale(ice, dynamics, tracers, partit, mesh)
             call relax_to_clim(tr_num, tracers, partit, mesh)
         end if
         call exchange_nod(tracers%data(tr_num)%values(:,:), partit)
-!$OMP BARRIER
     end do
 !!!        !$ACC UPDATE HOST (tracers%work%fct_ttf_min, tracers%work%fct_ttf_max, tracers%work%fct_plus, tracers%work%fct_minus) &
 !!!        !$ACC HOST  (tracers%work%edge_up_dn_grad)
@@ -1236,6 +1235,7 @@ SUBROUTINE diff_part_bh(tr_num, dynamics, tracers, partit, mesh)
 #endif
     END DO
 !$OMP END DO
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_nod(temporary_ttf, partit)
 !$OMP END MASTER
@@ -1284,7 +1284,6 @@ SUBROUTINE diff_part_bh(tr_num, dynamics, tracers, partit, mesh)
 !$OMP END DO
 !$OMP END PARALLEL
 call exchange_nod(ttf, partit)
-!$OMP BARRIER
 end subroutine diff_part_bh
 !
 !

src/oce_ale_vel_rhs.F90

@@ -442,6 +442,7 @@ subroutine momentum_adv_scalar(dynamics, partit, mesh)
     end do !-->do n=1,myDim_nod2d
 !$OMP END DO
     !___________________________________________________________________________
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_nod(UVnode_rhs, partit)
 !$OMP END MASTER

src/oce_dyn.F90

@@ -145,7 +145,6 @@ SUBROUTINE update_vel(dynamics, partit, mesh)
         END DO
     END DO
 !$OMP END PARALLEL DO
-!$OMP BARRIER
 !!PS Why we do this here eta_n is anyway overwriten through ...
 !!PS do node=1, myDim_nod2D+eDim_nod2D
 !!PS     if (ulevels_nod2D(node)==1) eta_n(node)=alpha*hbar(node)+(1.0_WP-alpha)*hbar_old(node)
@@ -207,7 +206,6 @@ subroutine compute_vel_nodes(dynamics, partit, mesh)
         END DO
     END DO
 !$OMP END PARALLEL DO
-!$OMP BARRIER
     call exchange_nod(UVnode, partit)
 end subroutine compute_vel_nodes
 !
@@ -345,6 +343,7 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh)
 #endif
     END DO
 !$OMP END DO
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_elem(U_b, partit)
     call exchange_elem(V_b, partit)
@@ -372,6 +371,7 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh)
         END DO
     END DO
 !$OMP END DO
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_nod(U_c, V_c, partit)
 !$OMP END MASTER
@@ -485,6 +485,7 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh)
         END DO
     END DO
 !$OMP END DO
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_elem(U_c, partit)
     call exchange_elem(V_c, partit)
@@ -608,6 +609,7 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh)
 
     END DO
 !$OMP END DO
+!$OMP BARRIER
 !$OMP MASTER
     call exchange_elem(U_c, partit)
     call exchange_elem(V_c, partit)
@@ -859,7 +861,6 @@ subroutine check_validviscopt_5(partit, mesh)
     end do
 !$OMP END DO
 !$OMP END PARALLEL
-!$OMP BARRIER
 
     !___________________________________________________________________________
     ! compute global mean ratio --> core2 Ratio=4.26 (eddy parameterizted), 

src/oce_fer_gm.F90

@@ -158,7 +158,6 @@ subroutine fer_solve_Gamma(partit, mesh)
     END DO   !!! cycle over nodes
 !$OMP END DO
 !$OMP END PARALLEL
-!$OMP BARRIER
     call exchange_nod(fer_gamma, partit)
 END subroutine fer_solve_Gamma
 !
@@ -206,7 +205,6 @@ subroutine fer_gamma2vel(dynamics, partit, mesh)
     END DO
 !$OMP END PARALLEL DO
     call exchange_elem(fer_uv, partit)
-!$OMP BARRIER
 end subroutine fer_gamma2vel
 !
 !
@@ -412,7 +410,6 @@ subroutine init_Redi_GM(partit, mesh) !fer_compute_C_K_Redi
    end do
 !$OMP END DO
 !$OMP END PARALLEL
-!$OMP BARRIER
    if (Fer_GM) call exchange_nod(fer_c, partit)
    if (Fer_GM) call exchange_nod(fer_k, partit)
    if (Redi)   call exchange_nod(Ki, partit)

src/oce_tracer_mod.F90

@@ -49,7 +49,6 @@ SUBROUTINE init_tracers_AB(tr_num, tracers, partit, mesh)
     if (flag_debug .and. partit%mype==0)  print *, achar(27)//'[38m'//'             --> call tracer_gradient_z'//achar(27)//'[0m'
     call tracer_gradient_z(tracers%data(tr_num)%values, partit, mesh)    !WHY NOT AB HERE? DSIDOREN!
     call exchange_elem_end(partit)      ! tr_xy used in fill_up_dn_grad
-!$OMP BARRIER
 
     call exchange_nod_begin(tr_z, partit) ! not used in fill_up_dn_grad 
 

src/solver.F90

@@ -162,7 +162,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh)
   END DO
 !$OMP END PARALLEL DO 
   call exchange_nod(rr, partit)
-!$OMP BARRIER
   ! =============
   ! z_0=M^{-1} r_0  (M^{-1} is the precondit. matrix)
   ! pp is the search direction
@@ -233,7 +232,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh)
      ! New z
      ! ===========
   call exchange_nod(rr, partit)     ! Update before matrix-vector multiplications
-!$OMP BARRIER
 !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(row)
   DO row=1, myDim_nod2D
      zz(row)= sum(pr_values(rptr(row):rptr(row+1)-1)*rr(cind(rptr(row):rptr(row+1)-1)))
@@ -257,7 +255,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh)
 
   call MPI_Allreduce(MPI_IN_PLACE, sprod, 2, MPI_DOUBLE, MPI_SUM, partit%MPI_COMM_FESOM, MPIerr)
 
-!$OMP BARRIER
      ! ===========
      ! Exit if tolerance is achieved
      ! ===========
@@ -277,7 +274,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh)
   END DO
  ! At the end: The result is in X, but it needs a halo exchange.
   call exchange_nod(x, partit)
-!$OMP BARRIER
 end subroutine ssh_solve_cg 
 
 ! ===================================================================