Skip to content
This repository was archived by the owner on Oct 23, 2020. It is now read-only.

Modify ocean core barotropic subcycle to reuse halo data structure #1521

Open
wants to merge 1 commit into
base: ocean/develop
Choose a base branch
from
Open

Modify ocean core barotropic subcycle to reuse halo data structure #1521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/core_ocean/mode_forward/mpas_ocn_forward_mode.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,11 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{

endif

call mpas_dmpar_exch_group_create(domain, 'subcycleFields')
call mpas_dmpar_exch_group_add_field(domain, 'subcycleFields', 'sshSubcycle')
call mpas_dmpar_exch_group_add_field(domain, 'subcycleFields', 'normalBarotropicVelocitySubcycle')
call mpas_dmpar_exch_group_build_reusable_buffers(domain, 'subcycleFields')

end function ocn_forward_mode_init!}}}

!***********************************************************************
Expand Down Expand Up @@ -694,6 +699,8 @@ function ocn_forward_mode_finalize(domain) result(iErr)!{{{

integer :: ierr

call mpas_dmpar_exch_group_destroy_reusable_buffers(domain, 'subcycleFields')

call ocn_analysis_finalize(domain, ierr)

call mpas_destroy_clock(domain % clock, ierr)
Expand Down
51 changes: 21 additions & 30 deletions src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
logical :: activeTracersOnly ! if true only compute tendencies for active tracers
integer :: tsIter
integer :: edgeHaloComputeCounter, cellHaloComputeCounter
integer :: neededHalos

! Config options
character (len=StrKIND), pointer :: config_time_integrator
Expand Down Expand Up @@ -744,12 +745,26 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
call mpas_pool_get_field(scratchPool, 'btrvel_temp', btrvel_tempField)
call mpas_allocate_scratch_field(btrvel_tempField, .false.)

cellHaloComputeCounter = 0
edgeHaloComputeCounter = 0
neededHalos = 1 + config_n_btr_cor_iter

call mpas_threading_barrier()

call mpas_timer_start('btr se subcycle loop')
do j = 1, nBtrSubcycles * config_btr_subcycle_loop_factor
cellHaloComputeCounter = config_num_halos
edgeHaloComputeCounter = config_num_halos + 1
if(cellHaloComputeCounter < neededHalos) then

call mpas_threading_barrier()
call mpas_timer_start('se halo subcycle')
call mpas_dmpar_exch_group_reuse_halo_exch(domain, subcycleGroupName, timeLevel=oldBtrSubcycleTime)
call mpas_threading_barrier()
call mpas_timer_stop('se halo subcycle')

cellHaloComputeCounter = config_num_halos - mod( config_num_halos, neededHalos )
edgeHaloComputeCounter = config_num_halos + 1 - mod( config_num_halos, neededHalos )
call mpas_threading_barrier()
end if

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! Barotropic subcycle: VELOCITY PREDICTOR STEP
Expand Down Expand Up @@ -1036,17 +1051,9 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
block => block % next
end do ! block

! mrp 170503 This is the original line. Go back to this once I
! verify that halo regions are truly three cells wide.
! if ( edgeHaloComputeCounter == 1 .and. BtrCorIter /= config_n_btr_cor_iter ) then
! replaced with this, to force a halo update on the first iteration:
if ( edgeHaloComputeCounter == 1 .or. BtrCorIter == 1 ) then
edgeHaloComputeCounter = config_num_halos + 1
call mpas_timer_start("se halo btr vel corr")
call mpas_dmpar_field_halo_exch(domain, 'normalBarotropicVelocitySubcycle', timeLevel=newBtrSubcycleTime)
call mpas_timer_stop("se halo btr vel corr")
else
edgeHaloComputeCounter = edgeHaloComputeCounter - 1
edgeHaloComputeCounter = edgeHaloComputeCounter - 1
if ( BtrCorIter >= 1 .or. config_btr_solve_SSH2 == .false.) then
cellHaloComputeCounter = cellHaloComputeCounter - 1
end if

end do !do BtrCorIter=1,config_n_btr_cor_iter
Expand Down Expand Up @@ -1173,25 +1180,9 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{

block => block % next
end do ! block
edgeHaloComputeCounter = config_num_halos + 1
endif ! config_btr_solve_SSH2

! boundary update on SSHnew
call mpas_timer_start("se halo subcycle")
call mpas_dmpar_exch_group_create(domain, subcycleGroupName)
call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'sshSubcycle', timeLevel=newBtrSubcycleTime)
call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'normalBarotropicVelocitySubcycle', &
timeLevel=newBtrSubcycleTime)

call mpas_threading_barrier()
call mpas_dmpar_exch_group_full_halo_exch(domain, subcycleGroupName)

call mpas_dmpar_exch_group_destroy(domain, subcycleGroupName)
call mpas_timer_stop("se halo subcycle")

! Reset the halo counters
edgeHaloComputeCounter = config_num_halos + 1
cellHaloComputeCounter = config_num_halos

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! Barotropic subcycle: Accumulate running sums, advance timestep pointers
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Expand Down