Skip to content
This repository has been archived by the owner on Jun 1, 2019. It is now read-only.

Jedwards/flexable pelayout #52

Closed
wants to merge 8 commits into from
Closed

Jedwards/flexable pelayout #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9799ffa
Merge branch 'master' into nuopc-cmeps
jedwards4b Sep 27, 2018
889b862
Merge branch 'nuopc-cmeps' of https://github.com/ESMCI/cmeps-cime int…
jedwards4b Oct 23, 2018
2752937
not ready for primetime
jedwards4b Oct 9, 2018
dd773fc
Fixes for redistributing meshes in the mediator
rsdunlapiv Oct 9, 2018
b4f59ae
Updates to mesh redistribution in the mediator and mesh diagnostic ou…
rsdunlapiv Oct 16, 2018
32b51be
fix some tests
jedwards4b Oct 25, 2018
c872286
Create distgrid with balanceFlag = true for transferred meshes
rsdunlapiv Oct 25, 2018
c4a8c8d
tempory change to esmf master build
jedwards4b Oct 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/cesm/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ This allows using a different mpirun command to launch unit tests
<env name="MPI_USE_ARRAY"/>
</environment_variables>
<environment_variables comp_interface="nuopc">
<env name="ESMFMKFILE">/glade/u/home/dunlap/ESMF-INSTALL/8.0.0bs16/lib/libg/Linux.intel.64.mpich2.default/esmf.mk</env>
<env name="ESMFMKFILE">/glade/u/home/dunlap/ESMF-INSTALL/master/lib/libg/Linux.intel.64.mpich2.default/esmf.mk</env>
</environment_variables>
<environment_variables unit_testing="true">
<env name="MPI_USE_ARRAY">false</env>
Expand Down
1 change: 1 addition & 0 deletions src/components/data_comps/dwav/nuopc/wav_comp_nuopc.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,7 @@ end subroutine InitializeRealize
!===============================================================================

subroutine ModelAdvance(gcomp, rc)
use shr_nuopc_utils_mod, only : shr_nuopc_memcheck
type(ESMF_GridComp) :: gcomp
integer, intent(out) :: rc

Expand Down
131 changes: 111 additions & 20 deletions src/drivers/nuopc/mediator/med.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -797,8 +797,9 @@ subroutine realizeConnectedGrid(State,string,rc)
use ESMF , only : ESMF_FieldGet, ESMF_DistGridGet, ESMF_GridCompGet
use ESMF , only : ESMF_GeomType_Grid, ESMF_AttributeGet, ESMF_DistGridCreate, ESMF_FieldEmptySet
use ESMF , only : ESMF_GridCreate, ESMF_LogWrite, ESMF_LogMsg_Info, ESMF_GridGet, ESMF_Failure
use ESMF , only : ESMF_LogMsg_Warning
use ESMF , only : ESMF_FieldStatus_Empty, ESMF_FieldStatus_Complete, ESMF_FieldStatus_GridSet
use ESMF , only : ESMF_GeomType_Mesh
use ESMF , only : ESMF_GeomType_Mesh, ESMF_MeshGet, ESMF_Mesh, ESMF_MeshEmptyCreate
use shr_nuopc_methods_mod , only: shr_nuopc_methods_Field_GeomPrint

type(ESMF_State) , intent(inout) :: State
Expand All @@ -808,9 +809,11 @@ subroutine realizeConnectedGrid(State,string,rc)
! local variables
type(ESMF_Field) :: field
type(ESMF_Grid) :: grid
type(ESMF_Mesh) :: mesh, newmesh
integer :: localDeCount

type(ESMF_DistGrid) :: distgrid
type(ESMF_DistGrid) :: elemdistgrid, newelemdistgrid
type(ESMF_DistGridConnection), allocatable :: connectionList(:)
integer :: arbDimCount
integer :: dimCount, tileCount, petCount
Expand All @@ -822,9 +825,10 @@ subroutine realizeConnectedGrid(State,string,rc)
type(ESMF_GeomType_Flag) :: geomtype
character(ESMF_MAXSTR),allocatable :: fieldNameList(:)
type(ESMF_FieldStatus_Flag) :: fieldStatus
integer :: dbrc
character(len=CX) :: msgString
character(len=*),parameter :: subname='(module_MEDIATOR:realizeConnectedGrid)'
integer :: dbrc
character(len=CX):: msgString


!NOTE: All of the Fields that set their TransferOfferGeomObject Attribute
!NOTE: to "cannot provide" should now have the accepted Grid available.
Expand All @@ -847,8 +851,14 @@ subroutine realizeConnectedGrid(State,string,rc)
call ESMF_StateGet(State, itemNameList=fieldNameList, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! do n=1, fieldCount
do n=1, min(fieldCount,1)
call ESMF_GridCompGet(gcomp, petCount=petCount, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! do not loop here, assuming that all fields share the
! same grid/mesh and because it is more efficient - if
! a component has fields on multiple grids/meshes, this
! would need to be revisited
do n=1, min(fieldCount, 1)

call ESMF_StateGet(State, field=field, itemName=fieldNameList(n), rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
Expand All @@ -858,7 +868,10 @@ subroutine realizeConnectedGrid(State,string,rc)

if (fieldStatus==ESMF_FIELDSTATUS_GRIDSET) then

! while this is still an empty field, it does now hold a Grid with DistGrid
! The Mediator is accepting a Grid/Mesh passed to it
! through the Connector

! While this is still an empty field, it does now hold a Grid/Mesh with DistGrid
call ESMF_FieldGet(field, geomtype=geomtype, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

Expand Down Expand Up @@ -989,8 +1002,6 @@ subroutine realizeConnectedGrid(State,string,rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! construct a default regDecompPTile -> TODO: move this into ESMF as default
call ESMF_GridCompGet(gcomp, petCount=petCount, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

allocate(regDecompPTile(dimCount, tileCount))
deCountPTile = petCount/tileCount
Expand Down Expand Up @@ -1073,23 +1084,28 @@ subroutine realizeConnectedGrid(State,string,rc)
call ESMF_FieldGet(field, status=fieldStatus, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

if (fieldStatus==ESMF_FIELDSTATUS_EMPTY) then
if (fieldStatus==ESMF_FIELDSTATUS_EMPTY .or. fieldStatus==ESMF_FIELDSTATUS_GRIDSET) then
call ESMF_FieldEmptySet(field, grid=grid, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
if (dbug_flag > 1) then
call ESMF_LogWrite(trim(subname)//trim(string)//": attach grid for "//trim(fieldNameList(n1)), &
ESMF_LOGMSG_INFO, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
endif
if (dbug_flag > 1) then
call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
end if
else
if (dbug_flag > 1) then
call ESMF_LogWrite(trim(subname)//trim(string)//": NOT replacing grid for field: "//trim(fieldNameList(n1)), &
ESMF_LOGMSG_WARNING, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
endif
endif

if (dbug_flag > 1) then
call ESMF_LogWrite(trim(subname)//trim(string)//": attach grid for "//trim(fieldNameList(n1)), &
ESMF_LOGMSG_INFO, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
endif

if (dbug_flag > 1) then
call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
end if
enddo


elseif (geomtype == ESMF_GEOMTYPE_MESH) then

if (dbug_flag > 1) then
Expand All @@ -1102,6 +1118,81 @@ subroutine realizeConnectedGrid(State,string,rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
end if

call ESMF_FieldGet(field, mesh=mesh, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

call ESMF_MeshGet(mesh, elementDistGrid=elemDistGrid, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

!call ESMF_DistGridGet(elemDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc)
!if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

newelemdistgrid = ESMF_DistGridCreate(elemDistGrid, balanceFlag=.true., rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount
! allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount))
! ! get minIndex and maxIndex arrays
! call ESMF_DistGridGet(elemDistGrid, minIndexPTile=minIndexPTile, &
! maxIndexPTile=maxIndexPTile, rc=rc)
! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! ! use default regular decomposition
! newelemdistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, &
! maxIndexPTile=maxIndexPTile, rc=rc)
! deallocate(minIndexPTile, maxIndexPTile)

! call ESMF_DistGridGet(nodalDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc)
! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount
! allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount))
! ! get minIndex and maxIndex arrays
! call ESMF_DistGridGet(nodalDistGrid, minIndexPTile=minIndexPTile, &
! maxIndexPTile=maxIndexPTile, rc=rc)
! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! ! use default regular decomposition
! newnodaldistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, &
! maxIndexPTile=maxIndexPTile, rc=rc)
! deallocate(minIndexPTile, maxIndexPTile)

! Create a new Grid on the new DistGrid and swap it in the Field
newmesh = ESMF_MeshEmptyCreate(elementDistGrid=newelemdistgrid, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

! Swap all the Meshes in the State

! do n1=n,n
do n1=1, fieldCount
! access a field in the State and set the Mesh
call ESMF_StateGet(State, field=field, itemName=fieldNameList(n1), rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

call ESMF_FieldGet(field, status=fieldStatus, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return

if (fieldStatus==ESMF_FIELDSTATUS_EMPTY .or. fieldStatus==ESMF_FIELDSTATUS_GRIDSET) then
call ESMF_FieldEmptySet(field, mesh=newmesh, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
if (dbug_flag > 1) then
call ESMF_LogWrite(trim(subname)//trim(string)//": attach mesh for "//trim(fieldNameList(n1)), &
ESMF_LOGMSG_INFO, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
endif
if (dbug_flag > 1) then
call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
end if
else
if (dbug_flag > 1) then
call ESMF_LogWrite(trim(subname)//trim(string)//": NOT replacing mesh for field: "//trim(fieldNameList(n1)), &
ESMF_LOGMSG_WARNING, rc=rc)
if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return
endif
endif
enddo

else ! geomtype

call ESMF_LogWrite(trim(subname)//": ERROR geomtype not supported ", ESMF_LOGMSG_INFO, rc=rc)
Expand Down
2 changes: 1 addition & 1 deletion src/drivers/nuopc/shr/seq_comm_mct.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -973,7 +973,7 @@ subroutine seq_comm_setptrs(ID,mpicom,mpigrp,npes,nthreads,iam,iamroot,gloroot,
end subroutine seq_comm_setptrs
!---------------------------------------------------------
subroutine seq_comm_setnthreads(nthreads)

use shr_sys_mod, only : shr_sys_abort
implicit none
integer,intent(in) :: nthreads
character(*),parameter :: subName = '(seq_comm_setnthreads) '
Expand Down
Loading