From 2752937984502d2dda93601b9db09ef4d2ffaae9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 9 Oct 2018 14:59:14 -0600 Subject: [PATCH 1/6] not ready for primetime --- src/drivers/nuopc/mediator/med.F90 | 65 ++++++++++++++++++--- src/drivers/nuopc/shr/med_constants_mod.F90 | 2 +- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/drivers/nuopc/mediator/med.F90 b/src/drivers/nuopc/mediator/med.F90 index 878a515bf..7ecda3f01 100644 --- a/src/drivers/nuopc/mediator/med.F90 +++ b/src/drivers/nuopc/mediator/med.F90 @@ -798,7 +798,7 @@ subroutine realizeConnectedGrid(State,string,rc) use ESMF , only : ESMF_GeomType_Grid, ESMF_AttributeGet, ESMF_DistGridCreate, ESMF_FieldEmptySet use ESMF , only : ESMF_GridCreate, ESMF_LogWrite, ESMF_LogMsg_Info, ESMF_GridGet, ESMF_Failure use ESMF , only : ESMF_FieldStatus_Empty, ESMF_FieldStatus_Complete, ESMF_FieldStatus_GridSet - use ESMF , only : ESMF_GeomType_Mesh + use ESMF , only : ESMF_GeomType_Mesh, ESMF_MeshGet, ESMF_Mesh, ESMF_MeshCreate use shr_nuopc_methods_mod , only: shr_nuopc_methods_Field_GeomPrint type(ESMF_State) , intent(inout) :: State @@ -808,6 +808,7 @@ subroutine realizeConnectedGrid(State,string,rc) ! local variables type(ESMF_Field) :: field type(ESMF_Grid) :: grid + type(ESMF_Mesh) :: mesh, newmesh integer :: localDeCount type(ESMF_DistGrid) :: distgrid @@ -847,6 +848,9 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_StateGet(State, itemNameList=fieldNameList, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + call ESMF_GridCompGet(gcomp, petCount=petCount, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + ! do n=1, fieldCount do n=1, min(fieldCount,1) @@ -989,8 +993,6 @@ subroutine realizeConnectedGrid(State,string,rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return ! construct a default regDecompPTile -> TODO: move this into ESMF as default - call ESMF_GridCompGet(gcomp, petCount=petCount, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return allocate(regDecompPTile(dimCount, tileCount)) deCountPTile = petCount/tileCount @@ -1097,10 +1099,59 @@ subroutine realizeConnectedGrid(State,string,rc) ESMF_LOGMSG_INFO, rc=dbrc) end if - if (dbug_flag > 1) then - call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - end if +! if (dbug_flag > 1) then +! call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) +! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return +! end if + + call ESMF_FieldGet(field, mesh=mesh, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_MeshGet(mesh, elementDistGrid=distgrid, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_DistGridGet(distgrid, dimCount=dimCount, tileCount=tileCount, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount + allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount)) + ! get minIndex and maxIndex arrays + call ESMF_DistGridGet(distgrid, minIndexPTile=minIndexPTile, & + maxIndexPTile=maxIndexPTile, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! construct a default regDecompPTile -> TODO: move this into ESMF as default + + allocate(regDecompPTile(dimCount, tileCount)) + deCountPTile = petCount/tileCount + extraDEs = max(0, petCount-deCountPTile) + do i=1, tileCount + if (i<=extraDEs) then + regDecompPTile(1, i) = deCountPTile + 1 + else + regDecompPTile(1, i) = deCountPTile + endif + do j=2, dimCount + regDecompPTile(j, i) = 1 + enddo + enddo + ! create the new DistGrid with the same minIndexPTile and maxIndexPTile, + ! but with a default regDecompPTile + distgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & + maxIndexPTile=maxIndexPTile, regDecompPTile=regDecompPTile, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! Create a new Grid on the new DistGrid and swap it in the Field + newmesh = ESMF_MeshCreate(mesh, nodalDistGrid=distgrid, & + elementDistGrid=distgrid, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_FieldEmptySet(field, mesh=newmesh, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! local clean-up + deallocate(minIndexPTile, maxIndexPTile, regDecompPTile) + else ! geomtype diff --git a/src/drivers/nuopc/shr/med_constants_mod.F90 b/src/drivers/nuopc/shr/med_constants_mod.F90 index 4307aa70c..95557316b 100644 --- a/src/drivers/nuopc/shr/med_constants_mod.F90 +++ b/src/drivers/nuopc/shr/med_constants_mod.F90 @@ -37,6 +37,6 @@ module med_constants_mod !----------------------------------------------------------------------------- ! TODO: This is not a constant and should be moved elsewhere - integer :: med_constants_dbug_flag = 0 + integer :: med_constants_dbug_flag = 2 end module med_constants_mod From dd773fc77c1bb95928c6fc3ad8ce4987034550fc Mon Sep 17 00:00:00 2001 From: Rocky Dunlap Date: Tue, 9 Oct 2018 16:49:07 -0600 Subject: [PATCH 2/6] Fixes for redistributing meshes in the mediator --- src/drivers/nuopc/mediator/med.F90 | 73 ++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/src/drivers/nuopc/mediator/med.F90 b/src/drivers/nuopc/mediator/med.F90 index 7ecda3f01..808e7f381 100644 --- a/src/drivers/nuopc/mediator/med.F90 +++ b/src/drivers/nuopc/mediator/med.F90 @@ -823,9 +823,10 @@ subroutine realizeConnectedGrid(State,string,rc) type(ESMF_GeomType_Flag) :: geomtype character(ESMF_MAXSTR),allocatable :: fieldNameList(:) type(ESMF_FieldStatus_Flag) :: fieldStatus + integer :: dbrc + character(len=CX) :: msgString character(len=*),parameter :: subname='(module_MEDIATOR:realizeConnectedGrid)' - integer :: dbrc - character(len=CX):: msgString + !NOTE: All of the Fields that set their TransferOfferGeomObject Attribute !NOTE: to "cannot provide" should now have the accepted Grid available. @@ -851,8 +852,11 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_GridCompGet(gcomp, petCount=petCount, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - ! do n=1, fieldCount - do n=1, min(fieldCount,1) + ! do not loop here, assuming that all fields share the + ! same grid/mesh and because it is more efficient - if + ! a component has fields on multiple grids/meshes, this + ! would need to be revisited + do n=1, min(fieldCount, 1) call ESMF_StateGet(State, field=field, itemName=fieldNameList(n), rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return @@ -860,18 +864,25 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_FieldGet(field, status=fieldStatus, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + !call NUOPC_GetAttribute(field, name="TransferActionGeomObject", & + ! value=transferAction, rc=rc) + !if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + if (fieldStatus==ESMF_FIELDSTATUS_GRIDSET) then - ! while this is still an empty field, it does now hold a Grid with DistGrid + ! The Mediator is accepting a Grid/Mesh passed to it + ! through the Connector + + ! While this is still an empty field, it does now hold a Grid/Mesh with DistGrid call ESMF_FieldGet(field, geomtype=geomtype, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return if (geomtype == ESMF_GEOMTYPE_GRID) then - if (dbug_flag > 1) then - call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - end if + !if (dbug_flag > 1) then + ! call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) + ! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + !end if call ESMF_AttributeGet(field, name="ArbDimCount", value=arbDimCount, & convention="NUOPC", purpose="Instance", rc=rc) @@ -1099,10 +1110,10 @@ subroutine realizeConnectedGrid(State,string,rc) ESMF_LOGMSG_INFO, rc=dbrc) end if -! if (dbug_flag > 1) then -! call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) -! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return -! end if + if (dbug_flag > 1) then + call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + end if call ESMF_FieldGet(field, mesh=mesh, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return @@ -1142,16 +1153,42 @@ subroutine realizeConnectedGrid(State,string,rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return ! Create a new Grid on the new DistGrid and swap it in the Field - newmesh = ESMF_MeshCreate(mesh, nodalDistGrid=distgrid, & - elementDistGrid=distgrid, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - - call ESMF_FieldEmptySet(field, mesh=newmesh, rc=rc) + newmesh = ESMF_MeshCreate(distgrid, distgrid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return ! local clean-up deallocate(minIndexPTile, maxIndexPTile, regDecompPTile) + ! need to check whether to destroy the original mesh? + ! call ESMF_MeshDestroy(mesh, rc=rc) + + ! Swap all the Meshes in the State + + ! do n1=n,n + do n1=1, fieldCount + ! access a field in the State and set the Mesh + call ESMF_StateGet(State, field=field, itemName=fieldNameList(n1), rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_FieldGet(field, status=fieldStatus, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + if (fieldStatus==ESMF_FIELDSTATUS_EMPTY) then + call ESMF_FieldEmptySet(field, mesh=newmesh, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + endif + + if (dbug_flag > 1) then + call ESMF_LogWrite(trim(subname)//trim(string)//": attach mesh for "//trim(fieldNameList(n1)), & + ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + endif + + if (dbug_flag > 1) then + call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + end if + enddo else ! geomtype From b4f59aeb90f50500af61c2ca49592546a56d6b11 Mon Sep 17 00:00:00 2001 From: Rocky Dunlap Date: Tue, 16 Oct 2018 13:10:56 -0600 Subject: [PATCH 3/6] Updates to mesh redistribution in the mediator and mesh diagnostic output to the log. There is a known issue with the land mesh failing in the redistribution. --- src/drivers/nuopc/mediator/med.F90 | 113 ++++++------ .../nuopc/shr/shr_nuopc_methods_mod.F90 | 167 ++++++++++++++---- 2 files changed, 189 insertions(+), 91 deletions(-) diff --git a/src/drivers/nuopc/mediator/med.F90 b/src/drivers/nuopc/mediator/med.F90 index 808e7f381..254412d94 100644 --- a/src/drivers/nuopc/mediator/med.F90 +++ b/src/drivers/nuopc/mediator/med.F90 @@ -797,8 +797,9 @@ subroutine realizeConnectedGrid(State,string,rc) use ESMF , only : ESMF_FieldGet, ESMF_DistGridGet, ESMF_GridCompGet use ESMF , only : ESMF_GeomType_Grid, ESMF_AttributeGet, ESMF_DistGridCreate, ESMF_FieldEmptySet use ESMF , only : ESMF_GridCreate, ESMF_LogWrite, ESMF_LogMsg_Info, ESMF_GridGet, ESMF_Failure + use ESMF , only : ESMF_LogMsg_Warning use ESMF , only : ESMF_FieldStatus_Empty, ESMF_FieldStatus_Complete, ESMF_FieldStatus_GridSet - use ESMF , only : ESMF_GeomType_Mesh, ESMF_MeshGet, ESMF_Mesh, ESMF_MeshCreate + use ESMF , only : ESMF_GeomType_Mesh, ESMF_MeshGet, ESMF_Mesh, ESMF_MeshEmptyCreate use shr_nuopc_methods_mod , only: shr_nuopc_methods_Field_GeomPrint type(ESMF_State) , intent(inout) :: State @@ -812,6 +813,8 @@ subroutine realizeConnectedGrid(State,string,rc) integer :: localDeCount type(ESMF_DistGrid) :: distgrid + type(ESMF_DistGrid) :: nodaldistgrid, newnodaldistgrid + type(ESMF_DistGrid) :: elemdistgrid, newelemdistgrid type(ESMF_DistGridConnection), allocatable :: connectionList(:) integer :: arbDimCount integer :: dimCount, tileCount, petCount @@ -1086,23 +1089,28 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_FieldGet(field, status=fieldStatus, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - if (fieldStatus==ESMF_FIELDSTATUS_EMPTY) then + if (fieldStatus==ESMF_FIELDSTATUS_EMPTY .or. fieldStatus==ESMF_FIELDSTATUS_GRIDSET) then call ESMF_FieldEmptySet(field, grid=grid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + if (dbug_flag > 1) then + call ESMF_LogWrite(trim(subname)//trim(string)//": attach grid for "//trim(fieldNameList(n1)), & + ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + endif + if (dbug_flag > 1) then + call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + end if + else + if (dbug_flag > 1) then + call ESMF_LogWrite(trim(subname)//trim(string)//": NOT replacing grid for field: "//trim(fieldNameList(n1)), & + ESMF_LOGMSG_WARNING, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + endif endif - - if (dbug_flag > 1) then - call ESMF_LogWrite(trim(subname)//trim(string)//": attach grid for "//trim(fieldNameList(n1)), & - ESMF_LOGMSG_INFO, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - endif - - if (dbug_flag > 1) then - call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - end if enddo + elseif (geomtype == ESMF_GEOMTYPE_MESH) then if (dbug_flag > 1) then @@ -1118,49 +1126,44 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_FieldGet(field, mesh=mesh, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_MeshGet(mesh, elementDistGrid=distgrid, rc=rc) + call ESMF_MeshGet(mesh, elementDistGrid=elemDistGrid, & + nodalDistGrid=nodalDistGrid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_DistGridGet(distgrid, dimCount=dimCount, tileCount=tileCount, rc=rc) + call ESMF_DistGridGet(elemDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount)) ! get minIndex and maxIndex arrays - call ESMF_DistGridGet(distgrid, minIndexPTile=minIndexPTile, & + call ESMF_DistGridGet(elemDistGrid, minIndexPTile=minIndexPTile, & maxIndexPTile=maxIndexPTile, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - ! construct a default regDecompPTile -> TODO: move this into ESMF as default + ! use default regular decomposition + newelemdistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & + maxIndexPTile=maxIndexPTile, rc=rc) + deallocate(minIndexPTile, maxIndexPTile) - allocate(regDecompPTile(dimCount, tileCount)) - deCountPTile = petCount/tileCount - extraDEs = max(0, petCount-deCountPTile) - do i=1, tileCount - if (i<=extraDEs) then - regDecompPTile(1, i) = deCountPTile + 1 - else - regDecompPTile(1, i) = deCountPTile - endif - do j=2, dimCount - regDecompPTile(j, i) = 1 - enddo - enddo - ! create the new DistGrid with the same minIndexPTile and maxIndexPTile, - ! but with a default regDecompPTile - distgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & - maxIndexPTile=maxIndexPTile, regDecompPTile=regDecompPTile, rc=rc) + call ESMF_DistGridGet(nodalDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - ! Create a new Grid on the new DistGrid and swap it in the Field - newmesh = ESMF_MeshCreate(distgrid, distgrid, rc=rc) + ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount + allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount)) + ! get minIndex and maxIndex arrays + call ESMF_DistGridGet(nodalDistGrid, minIndexPTile=minIndexPTile, & + maxIndexPTile=maxIndexPTile, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - ! local clean-up - deallocate(minIndexPTile, maxIndexPTile, regDecompPTile) + ! use default regular decomposition + newnodaldistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & + maxIndexPTile=maxIndexPTile, rc=rc) + deallocate(minIndexPTile, maxIndexPTile) - ! need to check whether to destroy the original mesh? - ! call ESMF_MeshDestroy(mesh, rc=rc) + ! Create a new Grid on the new DistGrid and swap it in the Field + newmesh = ESMF_MeshEmptyCreate(elementDistGrid=newelemdistgrid, & + nodalDistGrid=newnodalDistGrid, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return ! Swap all the Meshes in the State @@ -1173,21 +1176,25 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_FieldGet(field, status=fieldStatus, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - if (fieldStatus==ESMF_FIELDSTATUS_EMPTY) then + if (fieldStatus==ESMF_FIELDSTATUS_EMPTY .or. fieldStatus==ESMF_FIELDSTATUS_GRIDSET) then call ESMF_FieldEmptySet(field, mesh=newmesh, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - endif - - if (dbug_flag > 1) then - call ESMF_LogWrite(trim(subname)//trim(string)//": attach mesh for "//trim(fieldNameList(n1)), & - ESMF_LOGMSG_INFO, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - endif - - if (dbug_flag > 1) then - call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - end if + if (dbug_flag > 1) then + call ESMF_LogWrite(trim(subname)//trim(string)//": attach mesh for "//trim(fieldNameList(n1)), & + ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + endif + if (dbug_flag > 1) then + call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + end if + else + if (dbug_flag > 1) then + call ESMF_LogWrite(trim(subname)//trim(string)//": NOT replacing mesh for field: "//trim(fieldNameList(n1)), & + ESMF_LOGMSG_WARNING, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + endif + endif enddo else ! geomtype diff --git a/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 b/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 index 332914f5f..515401c1c 100644 --- a/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 +++ b/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 @@ -2994,17 +2994,22 @@ end subroutine shr_nuopc_methods_Field_GeomPrint !----------------------------------------------------------------------------- subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) - use ESMF, only : ESMF_Mesh, ESMF_DistGrid, ESMF_MeshGet, ESMF_DistGridGet + use ESMF, only: ESMF_Mesh, ESMF_DistGrid, ESMF_MeshGet, ESMF_DistGridGet + use ESMF, only: ESMF_DELayoutGet, ESMF_DELayout + use ESMF, only: ESMF_MeshStatus_Flag, ESMF_MeshStatus_Complete type(ESMF_Mesh) , intent(in) :: mesh character(len=*), intent(in) :: string integer , intent(out) :: rc type(ESMF_Distgrid) :: distgrid + type(ESMF_DELayout) :: delayout integer :: pdim, sdim, nnodes, nelements integer :: localDeCount integer :: DeCount integer :: dimCount, tileCount integer, allocatable :: minIndexPTile(:,:), maxIndexPTile(:,:) + type(ESMF_MeshStatus_Flag) :: meshStatus + logical :: elemDGPresent, nodeDGPresent integer :: dbrc character(len=*),parameter :: subname='(shr_nuopc_methods_Mesh_Print)' @@ -3013,56 +3018,142 @@ subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) endif rc = ESMF_SUCCESS - ! access localDeCount to show this is a real Grid - call ESMF_MeshGet(mesh, parametricDim=pdim, spatialDim=sdim, elementDistgrid=distgrid, & - numOwnedNodes=nnodes, numOwnedElements=nelements, rc=rc) + call ESMF_MeshGet(mesh, elementDistGridIsPresent=elemDGPresent, & + nodalDistgridIsPresent=nodeDGPresent, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - write (msgString,*) trim(subname)//":"//trim(string)//": parametricDim=", pdim - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - write (msgString,*) trim(subname)//":"//trim(string)//": spatialDim=", sdim - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - write (msgString,*) trim(subname)//":"//trim(string)//": numOwnedNodes=", nnodes - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - write (msgString,*) trim(subname)//":"//trim(string)//": numOwnedElements=", nelements - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_MeshGet(mesh, status=meshStatus, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - ! get dimCount and tileCount - call ESMF_DistGridGet(distgrid, dimCount=dimCount, tileCount=tileCount, deCount=deCount, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + ! first get the distgrid, which should be available + if (elemDGPresent) then + call ESMF_MeshGet(mesh, elementDistgrid=distgrid, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": distGrid=element" + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - write (msgString,*) trim(subname)//":"//trim(string)//": dimCount=", dimCount - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + call ESMF_DistGridGet(distgrid, deLayout=deLayout, dimCount=dimCount, & + tileCount=tileCount, deCount=deCount, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": dimCount=", dimCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": tileCount=", tileCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": deCount=", deCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - write (msgString,*) trim(subname)//":"//trim(string)//": tileCount=", tileCount - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + call ESMF_DELayoutGet(deLayout, localDeCount=localDeCount, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - write (msgString,*) trim(subname)//":"//trim(string)//": deCount=", deCount - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + write (msgString,*) trim(subname)//":"//trim(string)//": localDeCount=", localDeCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount + allocate(minIndexPTile(dimCount, tileCount), & + maxIndexPTile(dimCount, tileCount)) + + ! get minIndex and maxIndex arrays + call ESMF_DistGridGet(distgrid, minIndexPTile=minIndexPTile, & + maxIndexPTile=maxIndexPTile, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": minIndexPTile=", minIndexPTile + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": maxIndexPTile=", maxIndexPTile + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + deallocate(minIndexPTile, maxIndexPTile) - ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount - allocate(minIndexPTile(dimCount, tileCount), & - maxIndexPTile(dimCount, tileCount)) + endif - ! get minIndex and maxIndex arrays - call ESMF_DistGridGet(distgrid, minIndexPTile=minIndexPTile, & - maxIndexPTile=maxIndexPTile, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + if (nodeDGPresent) then + call ESMF_MeshGet(mesh, nodalDistgrid=distgrid, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - write (msgString,*) trim(subname)//":"//trim(string)//": minIndexPTile=", minIndexPTile - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + write (msgString,*) trim(subname)//":"//trim(string)//": distGrid=nodal" + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - write (msgString,*) trim(subname)//":"//trim(string)//": maxIndexPTile=", maxIndexPTile - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + call ESMF_DistGridGet(distgrid, deLayout=deLayout, dimCount=dimCount, & + tileCount=tileCount, deCount=deCount, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": dimCount=", dimCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": tileCount=", tileCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": deCount=", deCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - deallocate(minIndexPTile, maxIndexPTile) + call ESMF_DELayoutGet(deLayout, localDeCount=localDeCount, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": localDeCount=", localDeCount + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount + allocate(minIndexPTile(dimCount, tileCount), & + maxIndexPTile(dimCount, tileCount)) + + ! get minIndex and maxIndex arrays + call ESMF_DistGridGet(distgrid, minIndexPTile=minIndexPTile, & + maxIndexPTile=maxIndexPTile, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": minIndexPTile=", minIndexPTile + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": maxIndexPTile=", maxIndexPTile + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + deallocate(minIndexPTile, maxIndexPTile) + endif + + if (.not. elemDGPresent .and. .not. nodeDGPresent) then + call ESMF_LogWrite(trim(subname)//": cannot print distgrid from mesh", & + ESMF_LOGMSG_WARNING, rc=rc) + return + endif + + ! if mesh is complete, also get additional parameters + if (meshStatus==ESMF_MESHSTATUS_COMPLETE) then + ! access localDeCount to show this is a real Grid + call ESMF_MeshGet(mesh, parametricDim=pdim, spatialDim=sdim, & + numOwnedNodes=nnodes, numOwnedElements=nelements, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + write (msgString,*) trim(subname)//":"//trim(string)//": parametricDim=", pdim + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + write (msgString,*) trim(subname)//":"//trim(string)//": spatialDim=", sdim + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + write (msgString,*) trim(subname)//":"//trim(string)//": numOwnedNodes=", nnodes + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + write (msgString,*) trim(subname)//":"//trim(string)//": numOwnedElements=", nelements + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + endif + if (dbug_flag > 10) then call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO, rc=dbrc) endif From 32b51be79621ed144b993f8bdd68426953c04b40 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 25 Oct 2018 09:22:55 -0600 Subject: [PATCH 4/6] fix some tests --- config/cesm/machines/config_machines.xml | 2 +- .../data_comps/dwav/nuopc/wav_comp_nuopc.F90 | 1 + src/drivers/nuopc/mediator/med.F90 | 20 ++++---- src/drivers/nuopc/shr/med_constants_mod.F90 | 2 +- src/drivers/nuopc/shr/seq_comm_mct.F90 | 2 +- .../nuopc/shr/shr_nuopc_methods_mod.F90 | 46 +++++++++---------- 6 files changed, 35 insertions(+), 38 deletions(-) diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index a79a8d995..ae3719b2d 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -332,7 +332,7 @@ This allows using a different mpirun command to launch unit tests - /glade/u/home/dunlap/ESMF-INSTALL/8.0.0bs16/lib/libg/Linux.intel.64.mpich2.default/esmf.mk + /glade/u/home/dunlap/ESMF-INSTALL/8.0.0bs21/lib/libg/Linux.intel.64.mpich2.default/esmf.mk false diff --git a/src/components/data_comps/dwav/nuopc/wav_comp_nuopc.F90 b/src/components/data_comps/dwav/nuopc/wav_comp_nuopc.F90 index 1d4d30147..c34c0b3aa 100644 --- a/src/components/data_comps/dwav/nuopc/wav_comp_nuopc.F90 +++ b/src/components/data_comps/dwav/nuopc/wav_comp_nuopc.F90 @@ -444,6 +444,7 @@ end subroutine InitializeRealize !=============================================================================== subroutine ModelAdvance(gcomp, rc) + use shr_nuopc_utils_mod, only : shr_nuopc_memcheck type(ESMF_GridComp) :: gcomp integer, intent(out) :: rc diff --git a/src/drivers/nuopc/mediator/med.F90 b/src/drivers/nuopc/mediator/med.F90 index 254412d94..2276141c5 100644 --- a/src/drivers/nuopc/mediator/med.F90 +++ b/src/drivers/nuopc/mediator/med.F90 @@ -867,10 +867,6 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_FieldGet(field, status=fieldStatus, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - !call NUOPC_GetAttribute(field, name="TransferActionGeomObject", & - ! value=transferAction, rc=rc) - !if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - if (fieldStatus==ESMF_FIELDSTATUS_GRIDSET) then ! The Mediator is accepting a Grid/Mesh passed to it @@ -882,10 +878,10 @@ subroutine realizeConnectedGrid(State,string,rc) if (geomtype == ESMF_GEOMTYPE_GRID) then - !if (dbug_flag > 1) then - ! call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) - ! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - !end if + if (dbug_flag > 1) then + call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n))//'_orig',rc) + if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + end if call ESMF_AttributeGet(field, name="ArbDimCount", value=arbDimCount, & convention="NUOPC", purpose="Instance", rc=rc) @@ -1126,7 +1122,7 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_FieldGet(field, mesh=mesh, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_MeshGet(mesh, elementDistGrid=elemDistGrid, & + call ESMF_MeshGet(mesh, elementDistGrid=elemDistGrid, & nodalDistGrid=nodalDistGrid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return @@ -1161,7 +1157,7 @@ subroutine realizeConnectedGrid(State,string,rc) deallocate(minIndexPTile, maxIndexPTile) ! Create a new Grid on the new DistGrid and swap it in the Field - newmesh = ESMF_MeshEmptyCreate(elementDistGrid=newelemdistgrid, & + newmesh = ESMF_MeshEmptyCreate(elementDistGrid=newelemdistgrid, & nodalDistGrid=newnodalDistGrid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return @@ -1188,13 +1184,13 @@ subroutine realizeConnectedGrid(State,string,rc) call shr_nuopc_methods_Field_GeomPrint(field,trim(fieldNameList(n1))//'_new',rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return end if - else + else if (dbug_flag > 1) then call ESMF_LogWrite(trim(subname)//trim(string)//": NOT replacing mesh for field: "//trim(fieldNameList(n1)), & ESMF_LOGMSG_WARNING, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return endif - endif + endif enddo else ! geomtype diff --git a/src/drivers/nuopc/shr/med_constants_mod.F90 b/src/drivers/nuopc/shr/med_constants_mod.F90 index 95557316b..4307aa70c 100644 --- a/src/drivers/nuopc/shr/med_constants_mod.F90 +++ b/src/drivers/nuopc/shr/med_constants_mod.F90 @@ -37,6 +37,6 @@ module med_constants_mod !----------------------------------------------------------------------------- ! TODO: This is not a constant and should be moved elsewhere - integer :: med_constants_dbug_flag = 2 + integer :: med_constants_dbug_flag = 0 end module med_constants_mod diff --git a/src/drivers/nuopc/shr/seq_comm_mct.F90 b/src/drivers/nuopc/shr/seq_comm_mct.F90 index d2208171b..18127088a 100644 --- a/src/drivers/nuopc/shr/seq_comm_mct.F90 +++ b/src/drivers/nuopc/shr/seq_comm_mct.F90 @@ -973,7 +973,7 @@ subroutine seq_comm_setptrs(ID,mpicom,mpigrp,npes,nthreads,iam,iamroot,gloroot, end subroutine seq_comm_setptrs !--------------------------------------------------------- subroutine seq_comm_setnthreads(nthreads) - + use shr_sys_mod, only : shr_sys_abort implicit none integer,intent(in) :: nthreads character(*),parameter :: subName = '(seq_comm_setnthreads) ' diff --git a/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 b/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 index 515401c1c..d5571e919 100644 --- a/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 +++ b/src/drivers/nuopc/shr/shr_nuopc_methods_mod.F90 @@ -3029,7 +3029,7 @@ subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) if (elemDGPresent) then call ESMF_MeshGet(mesh, elementDistgrid=distgrid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": distGrid=element" call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return @@ -3037,15 +3037,15 @@ subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) call ESMF_DistGridGet(distgrid, deLayout=deLayout, dimCount=dimCount, & tileCount=tileCount, deCount=deCount, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": dimCount=", dimCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": tileCount=", tileCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": deCount=", deCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return @@ -3056,25 +3056,25 @@ subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) write (msgString,*) trim(subname)//":"//trim(string)//": localDeCount=", localDeCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount allocate(minIndexPTile(dimCount, tileCount), & maxIndexPTile(dimCount, tileCount)) - + ! get minIndex and maxIndex arrays call ESMF_DistGridGet(distgrid, minIndexPTile=minIndexPTile, & maxIndexPTile=maxIndexPTile, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": minIndexPTile=", minIndexPTile call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": maxIndexPTile=", maxIndexPTile call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - - deallocate(minIndexPTile, maxIndexPTile) + + deallocate(minIndexPTile, maxIndexPTile) endif @@ -3089,15 +3089,15 @@ subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) call ESMF_DistGridGet(distgrid, deLayout=deLayout, dimCount=dimCount, & tileCount=tileCount, deCount=deCount, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": dimCount=", dimCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": tileCount=", tileCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": deCount=", deCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return @@ -3108,41 +3108,41 @@ subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) write (msgString,*) trim(subname)//":"//trim(string)//": localDeCount=", localDeCount call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount allocate(minIndexPTile(dimCount, tileCount), & maxIndexPTile(dimCount, tileCount)) - + ! get minIndex and maxIndex arrays call ESMF_DistGridGet(distgrid, minIndexPTile=minIndexPTile, & maxIndexPTile=maxIndexPTile, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": minIndexPTile=", minIndexPTile call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": maxIndexPTile=", maxIndexPTile call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - - deallocate(minIndexPTile, maxIndexPTile) + + deallocate(minIndexPTile, maxIndexPTile) endif if (.not. elemDGPresent .and. .not. nodeDGPresent) then - call ESMF_LogWrite(trim(subname)//": cannot print distgrid from mesh", & + call ESMF_LogWrite(trim(subname)//": cannot print distgrid from mesh", & ESMF_LOGMSG_WARNING, rc=rc) return endif ! if mesh is complete, also get additional parameters - if (meshStatus==ESMF_MESHSTATUS_COMPLETE) then + if (meshStatus==ESMF_MESHSTATUS_COMPLETE) then ! access localDeCount to show this is a real Grid call ESMF_MeshGet(mesh, parametricDim=pdim, spatialDim=sdim, & numOwnedNodes=nnodes, numOwnedElements=nelements, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - + write (msgString,*) trim(subname)//":"//trim(string)//": parametricDim=", pdim call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) write (msgString,*) trim(subname)//":"//trim(string)//": spatialDim=", sdim @@ -3153,7 +3153,7 @@ subroutine shr_nuopc_methods_Mesh_Print(mesh, string, rc) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return endif - + if (dbug_flag > 10) then call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO, rc=dbrc) endif From c8722860cf7c64d08506c871d0c01e375b0a6cdf Mon Sep 17 00:00:00 2001 From: Rocky Dunlap Date: Thu, 25 Oct 2018 15:41:51 -0600 Subject: [PATCH 5/6] Create distgrid with balanceFlag = true for transferred meshes --- src/drivers/nuopc/mediator/med.F90 | 64 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/drivers/nuopc/mediator/med.F90 b/src/drivers/nuopc/mediator/med.F90 index 2276141c5..6a92edb33 100644 --- a/src/drivers/nuopc/mediator/med.F90 +++ b/src/drivers/nuopc/mediator/med.F90 @@ -813,7 +813,6 @@ subroutine realizeConnectedGrid(State,string,rc) integer :: localDeCount type(ESMF_DistGrid) :: distgrid - type(ESMF_DistGrid) :: nodaldistgrid, newnodaldistgrid type(ESMF_DistGrid) :: elemdistgrid, newelemdistgrid type(ESMF_DistGridConnection), allocatable :: connectionList(:) integer :: arbDimCount @@ -1122,43 +1121,44 @@ subroutine realizeConnectedGrid(State,string,rc) call ESMF_FieldGet(field, mesh=mesh, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_MeshGet(mesh, elementDistGrid=elemDistGrid, & - nodalDistGrid=nodalDistGrid, rc=rc) + call ESMF_MeshGet(mesh, elementDistGrid=elemDistGrid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - call ESMF_DistGridGet(elemDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc) + !call ESMF_DistGridGet(elemDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc) + !if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + newelemdistgrid = ESMF_DistGridCreate(elemDistGrid, balanceFlag=.true., rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount - allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount)) - ! get minIndex and maxIndex arrays - call ESMF_DistGridGet(elemDistGrid, minIndexPTile=minIndexPTile, & - maxIndexPTile=maxIndexPTile, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - - ! use default regular decomposition - newelemdistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & - maxIndexPTile=maxIndexPTile, rc=rc) - deallocate(minIndexPTile, maxIndexPTile) - - call ESMF_DistGridGet(nodalDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - - ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount - allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount)) - ! get minIndex and maxIndex arrays - call ESMF_DistGridGet(nodalDistGrid, minIndexPTile=minIndexPTile, & - maxIndexPTile=maxIndexPTile, rc=rc) - if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return - - ! use default regular decomposition - newnodaldistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & - maxIndexPTile=maxIndexPTile, rc=rc) - deallocate(minIndexPTile, maxIndexPTile) + ! ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount + ! allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount)) + ! ! get minIndex and maxIndex arrays + ! call ESMF_DistGridGet(elemDistGrid, minIndexPTile=minIndexPTile, & + ! maxIndexPTile=maxIndexPTile, rc=rc) + ! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! ! use default regular decomposition + ! newelemdistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & + ! maxIndexPTile=maxIndexPTile, rc=rc) + ! deallocate(minIndexPTile, maxIndexPTile) + + ! call ESMF_DistGridGet(nodalDistGrid, dimCount=dimCount, tileCount=tileCount, rc=rc) + ! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! ! allocate minIndexPTile and maxIndexPTile accord. to dimCount and tileCount + ! allocate(minIndexPTile(dimCount, tileCount),maxIndexPTile(dimCount, tileCount)) + ! ! get minIndex and maxIndex arrays + ! call ESMF_DistGridGet(nodalDistGrid, minIndexPTile=minIndexPTile, & + ! maxIndexPTile=maxIndexPTile, rc=rc) + ! if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return + + ! ! use default regular decomposition + ! newnodaldistgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & + ! maxIndexPTile=maxIndexPTile, rc=rc) + ! deallocate(minIndexPTile, maxIndexPTile) ! Create a new Grid on the new DistGrid and swap it in the Field - newmesh = ESMF_MeshEmptyCreate(elementDistGrid=newelemdistgrid, & - nodalDistGrid=newnodalDistGrid, rc=rc) + newmesh = ESMF_MeshEmptyCreate(elementDistGrid=newelemdistgrid, rc=rc) if (shr_nuopc_methods_ChkErr(rc,__LINE__,u_FILE_u)) return ! Swap all the Meshes in the State From c4a8c8daeb0355b129290232246d0cd9c26cc593 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 26 Oct 2018 10:42:05 -0600 Subject: [PATCH 6/6] tempory change to esmf master build --- config/cesm/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index ae3719b2d..b711dab60 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -332,7 +332,7 @@ This allows using a different mpirun command to launch unit tests - /glade/u/home/dunlap/ESMF-INSTALL/8.0.0bs21/lib/libg/Linux.intel.64.mpich2.default/esmf.mk + /glade/u/home/dunlap/ESMF-INSTALL/master/lib/libg/Linux.intel.64.mpich2.default/esmf.mk false