From 8edf8ae369b0f9f990d106c32f867485e3963d71 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Mon, 2 Dec 2024 21:27:37 -0500 Subject: [PATCH 1/3] Parallel IO enhancements from GDIT --- .gitmodules | 12 +++++++---- CMakeLists.txt | 8 ++++++++ atmos_cubed_sphere | 2 +- io/fv3atm_restart_io.F90 | 43 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index 22c723ac1..79ea2f936 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,15 +1,19 @@ [submodule "atmos_cubed_sphere"] path = atmos_cubed_sphere - url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere - branch = dev/emc + #url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere + #branch = dev/emc + url = https://github.com/grantfirl/GFDL_atmos_cubed_sphere + branch = rrfsv1-to-ufs/dev6 [submodule "ccpp/framework"] path = ccpp/framework url = https://github.com/NCAR/ccpp-framework branch = main [submodule "ccpp/physics"] path = ccpp/physics - url = https://github.com/ufs-community/ccpp-physics - branch = ufs/dev + #url = https://github.com/ufs-community/ccpp-physics + #branch = ufs/dev + url = https://github.com/grantfirl/ccpp-physics + branch = rrfsv1-to-ufs/dev6 [submodule "upp"] path = upp url = https://github.com/NOAA-EMC/UPP diff --git a/CMakeLists.txt b/CMakeLists.txt index cdf597df2..e0ae99d95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,14 @@ if(NOT PARALLEL_NETCDF) list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF) endif() +if(ENABLE_PARALLELRESTART) + list(APPEND _fv3atm_defs_private ENABLE_PARALLELRESTART) +endif() + +if(ENABLE_RRFS_WAR) + list(APPEND _fv3atm_defs_private ENABLE_RRFS_WAR) +endif() + if(MOVING_NEST) list(APPEND _fv3atm_defs_private MOVING_NEST) if(NOT HYDRO) diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index e36e8572e..6e5411da4 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit e36e8572e7643d7e59a3979a61bdd83743ff7b00 +Subproject commit 6e5411da42eef1ec9070169e63ffeeb7034080ce diff --git a/io/fv3atm_restart_io.F90 b/io/fv3atm_restart_io.F90 index e7f60262b..979c2a000 100644 --- a/io/fv3atm_restart_io.F90 +++ b/io/fv3atm_restart_io.F90 @@ -19,7 +19,13 @@ module fv3atm_restart_io_mod read_restart, write_restart, write_data, & get_global_io_domain_indices, get_dimension_size, & global_att_exists, get_global_attribute - use mpp_domains_mod, only: domain2d +#ifdef ENABLE_PARALLELRESTART + use mpp_domains_mod, only: domain2d, mpp_get_domain_tile_commid, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#else + use mpp_domains_mod, only: domain2d, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#endif use fv3atm_common_io, only: create_2d_field_and_add_to_bundle, & create_3d_field_and_add_to_bundle, copy_from_gfs_data, axis_type use fv3atm_sfc_io @@ -532,6 +538,7 @@ end subroutine fv3atm_checksum !! Also calculates sncovr if it is not present in the restart file. subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum) use fv3atm_rrfs_sd_io + use atmosphere_mod, only: Atm,mygrid implicit none !--- interface variable definitions type(GFS_sfcprop_type), intent(inout) :: Sfcprop @@ -558,12 +565,25 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta type(FmsNetcdfDomainFile_t) :: Oro_restart, Sfc_restart, dust12m_restart, emi_restart, rrfssd_restart type(FmsNetcdfDomainFile_t) :: Oro_ls_restart, Oro_ss_restart + type(domain2D) :: domain_for_read + integer :: read_layout(2) !--- OROGRAPHY FILE !--- open file +#ifdef ENABLE_PARALLELRESTART + Oro_restart%use_collective = .true. + call mpp_get_layout(Atm(mygrid)%domain, read_layout) + call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + Oro_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + infile=trim(indir)//'/'//trim(fn_oro) + amiopen=open_file(Oro_restart, trim(infile), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else infile=trim(indir)//'/'//trim(fn_oro) amiopen=open_file(Oro_restart, trim(infile), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if (.not.amiopen) call mpp_error( FATAL, 'Error with opening file '//trim(infile) ) call oro%register(Model,Oro_restart,Atm_block) @@ -669,8 +689,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta !--- SURFACE FILE !--- open file +#ifdef ENABLE_PARALLELRESTART + Sfc_restart%use_collective = .true. + Sfc_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + infile=trim(indir)//'/'//trim(fn_srf) + amiopen=open_file(Sfc_restart, trim(infile), "read", domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else infile=trim(indir)//'/'//trim(fn_srf) amiopen=open_file(Sfc_restart, trim(infile), "read", domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if( .not.amiopen ) call mpp_error(FATAL, 'Error opening file'//trim(infile)) if (global_att_exists(Sfc_restart, "file_version")) then @@ -840,6 +868,7 @@ end subroutine sfc_prop_restart_write !! restart variables with the GFDL FMS restart subsystem. !! Calls a GFDL FMS routine to restore the data from a restart file. subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum) + use atmosphere_mod, only: Atm,mygrid implicit none !--- interface variable definitions type(GFS_restart_type), intent(in) :: GFS_Restart @@ -859,6 +888,8 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r type(phy_data_type) :: phy type(FmsNetcdfDomainFile_t) :: Phy_restart + type(domain2D) :: domain_for_read + integer :: read_layout(2) isc = Atm_block%isc iec = Atm_block%iec @@ -871,7 +902,17 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r !--- open restart file and register axes fname = trim(indir)//'/'//trim(fn_phy) +#ifdef ENABLE_PARALLELRESTART + Phy_restart%use_collective = .true. + call mpp_get_layout(Atm(mygrid)%domain, read_layout) + call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + Phy_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + amiopen=open_file(Phy_restart, trim(fname), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else amiopen=open_file(Phy_restart, trim(fname), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if( amiopen ) then call register_axis(Phy_restart, 'xaxis_1', 'X') call register_axis(Phy_restart, 'yaxis_1', 'Y') From fa65f967116804c8830b6f087ebe86debcb78631 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 23 Apr 2024 20:29:15 +0000 Subject: [PATCH 2/3] Add SW clear-sky downward flux at surface to available diagnostics --- ccpp/data/GFS_typedefs.F90 | 6 ++++++ ccpp/data/GFS_typedefs.meta | 14 ++++++++++++++ ccpp/driver/GFS_diagnostics.F90 | 12 ++++++++++++ ccpp/physics | 2 +- 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index f904b2f24..11e95c1b3 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -523,6 +523,7 @@ module GFS_typedefs !--- In (physics only) real (kind=kind_phys), pointer :: sfcdsw(:) => null() !< total sky sfc downward sw flux ( w/m**2 ) !< GFS_radtend_type%sfcfsw%dnfxc + real (kind=kind_phys), pointer :: sfcdswc(:) => null() !< total sky sfc downward sw flux assuming clear sky conditions( w/m**2 ) real (kind=kind_phys), pointer :: sfcnsw(:) => null() !< total sky sfc netsw flx into ground(w/m**2) !< difference of dnfxc & upfxc from GFS_radtend_type%sfcfsw real (kind=kind_phys), pointer :: sfcdlw(:) => null() !< total sky sfc downward lw flux ( w/m**2 ) @@ -2022,6 +2023,7 @@ module GFS_typedefs real (kind=kind_phys), pointer :: dlwsfci(:) => null() !< instantaneous sfc dnwd lw flux ( w/m**2 ) real (kind=kind_phys), pointer :: ulwsfci(:) => null() !< instantaneous sfc upwd lw flux ( w/m**2 ) real (kind=kind_phys), pointer :: dswsfci(:) => null() !< instantaneous sfc dnwd sw flux ( w/m**2 ) + real (kind=kind_phys), pointer :: dswsfcci(:) => null() !< instantaneous sfc dnwd sw flux ( w/m**2 ) (clear-sky) real (kind=kind_phys), pointer :: nswsfci(:) => null() !< instantaneous sfc net dnwd sw flux ( w/m**2 ) real (kind=kind_phys), pointer :: uswsfci(:) => null() !< instantaneous sfc upwd sw flux ( w/m**2 ) real (kind=kind_phys), pointer :: dusfci (:) => null() !< instantaneous u component of surface stress @@ -2938,11 +2940,13 @@ subroutine coupling_create (Coupling, Model) Coupling%visbmui = clear_val Coupling%visdfui = clear_val + allocate (Coupling%sfcdswc (IM)) allocate (Coupling%sfcdsw (IM)) allocate (Coupling%sfcnsw (IM)) allocate (Coupling%sfcdlw (IM)) allocate (Coupling%sfculw (IM)) + Coupling%sfcdswc = clear_val Coupling%sfcdsw = clear_val Coupling%sfcnsw = clear_val Coupling%sfcdlw = clear_val @@ -7800,6 +7804,7 @@ subroutine diag_create (Diag, Model) allocate (Diag%dlwsfci (IM)) allocate (Diag%ulwsfci (IM)) allocate (Diag%dswsfci (IM)) + allocate (Diag%dswsfcci(IM)) allocate (Diag%nswsfci (IM)) allocate (Diag%uswsfci (IM)) allocate (Diag%dusfci (IM)) @@ -8114,6 +8119,7 @@ subroutine diag_phys_zero (Diag, Model, linit, iauwindow_center) Diag%dlwsfci = zero Diag%ulwsfci = zero Diag%dswsfci = zero + Diag%dswsfcci = zero Diag%nswsfci = zero Diag%uswsfci = zero Diag%dusfci = zero diff --git a/ccpp/data/GFS_typedefs.meta b/ccpp/data/GFS_typedefs.meta index 567d0e37e..3c7a002a0 100644 --- a/ccpp/data/GFS_typedefs.meta +++ b/ccpp/data/GFS_typedefs.meta @@ -2480,6 +2480,13 @@ dimensions = (horizontal_dimension) type = real kind = kind_phys +[sfcdswc] + standard_name = surface_downwelling_shortwave_flux_on_radiation_timestep_assuming_clear_sky + long_name = total sky sfc downward sw flux assuming clear sky conditions + units = W m-2 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys [sfcnsw] standard_name = surface_net_downwelling_shortwave_flux_on_radiation_timestep long_name = total sky sfc netsw flx into ground @@ -9139,6 +9146,13 @@ dimensions = (horizontal_dimension) type = real kind = kind_phys +[dswsfcci] + standard_name = surface_downwelling_shortwave_flux_assuming_clear_sky + long_name = surface downwelling shortwave flux at current time assuming clear sky + units = W m-2 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys [nswsfci] standard_name = surface_net_downwelling_shortwave_flux long_name = surface net downwelling shortwave flux at current time diff --git a/ccpp/driver/GFS_diagnostics.F90 b/ccpp/driver/GFS_diagnostics.F90 index ea70c93de..607af364b 100644 --- a/ccpp/driver/GFS_diagnostics.F90 +++ b/ccpp/driver/GFS_diagnostics.F90 @@ -364,6 +364,18 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfci(Model%chunk_begin(nb):Model%chunk_end(nb)) enddo + idx = idx + 1 + ExtDiag(idx)%axes = 2 + ExtDiag(idx)%name = 'DSWRFCI' + ExtDiag(idx)%desc = 'instantaneous surface downward shortwave flux assuming clear sky' + ExtDiag(idx)%unit = 'w/m**2' + ExtDiag(idx)%mod_name = 'gfs_phys' + ExtDiag(idx)%intpl_method = 'bilinear' + allocate (ExtDiag(idx)%data(nblks)) + do nb = 1,nblks + ExtDiag(idx)%data(nb)%var2 => IntDiag(nb)%dswsfcci(:) + enddo + idx = idx + 1 ExtDiag(idx)%axes = 2 ExtDiag(idx)%name = 'USWRF' diff --git a/ccpp/physics b/ccpp/physics index 6e0467ad0..f295084c6 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 6e0467ad07a2231d298dc3be91a923c934354a7f +Subproject commit f295084c658811a0a6071df22f74dc8d1800faa6 From 4ee640f8bc6da0e6008595b7359a4170f1af6cee Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Wed, 4 Dec 2024 20:26:45 -0500 Subject: [PATCH 3/3] fix compilation error in GFS_diagnostics.F90 --- ccpp/driver/GFS_diagnostics.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/driver/GFS_diagnostics.F90 b/ccpp/driver/GFS_diagnostics.F90 index 607af364b..a74d14b8b 100644 --- a/ccpp/driver/GFS_diagnostics.F90 +++ b/ccpp/driver/GFS_diagnostics.F90 @@ -373,7 +373,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%intpl_method = 'bilinear' allocate (ExtDiag(idx)%data(nblks)) do nb = 1,nblks - ExtDiag(idx)%data(nb)%var2 => IntDiag(nb)%dswsfcci(:) + ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfcci(Model%chunk_begin(nb):Model%chunk_end(nb)) enddo idx = idx + 1