diff --git a/.gitmodules b/.gitmodules index 22c723ac1..79ea2f936 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,15 +1,19 @@ [submodule "atmos_cubed_sphere"] path = atmos_cubed_sphere - url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere - branch = dev/emc + #url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere + #branch = dev/emc + url = https://github.com/grantfirl/GFDL_atmos_cubed_sphere + branch = rrfsv1-to-ufs/dev6 [submodule "ccpp/framework"] path = ccpp/framework url = https://github.com/NCAR/ccpp-framework branch = main [submodule "ccpp/physics"] path = ccpp/physics - url = https://github.com/ufs-community/ccpp-physics - branch = ufs/dev + #url = https://github.com/ufs-community/ccpp-physics + #branch = ufs/dev + url = https://github.com/grantfirl/ccpp-physics + branch = rrfsv1-to-ufs/dev6 [submodule "upp"] path = upp url = https://github.com/NOAA-EMC/UPP diff --git a/CMakeLists.txt b/CMakeLists.txt index cdf597df2..e0ae99d95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,14 @@ if(NOT PARALLEL_NETCDF) list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF) endif() +if(ENABLE_PARALLELRESTART) + list(APPEND _fv3atm_defs_private ENABLE_PARALLELRESTART) +endif() + +if(ENABLE_RRFS_WAR) + list(APPEND _fv3atm_defs_private ENABLE_RRFS_WAR) +endif() + if(MOVING_NEST) list(APPEND _fv3atm_defs_private MOVING_NEST) if(NOT HYDRO) diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 9490871a6..da13577f5 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 9490871a616604efe9dd75608214081eef557ad6 +Subproject commit da13577f5356ba7939c52f37bb67980421e5d4eb diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index a7cb640c8..9c3d6551d 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -523,6 +523,7 @@ module GFS_typedefs !--- In (physics only) real (kind=kind_phys), pointer :: sfcdsw(:) => null() !< total sky sfc downward sw flux ( w/m**2 ) !< GFS_radtend_type%sfcfsw%dnfxc + real (kind=kind_phys), pointer :: sfcdswc(:) => null() !< total sky sfc downward sw flux assuming clear sky conditions( w/m**2 ) real (kind=kind_phys), pointer :: sfcnsw(:) => null() !< total sky sfc netsw flx into ground(w/m**2) !< difference of dnfxc & upfxc from GFS_radtend_type%sfcfsw real (kind=kind_phys), pointer :: sfcdlw(:) => null() !< total sky sfc downward lw flux ( w/m**2 ) @@ -2029,6 +2030,7 @@ module GFS_typedefs real (kind=kind_phys), pointer :: dlwsfci(:) => null() !< instantaneous sfc dnwd lw flux ( w/m**2 ) real (kind=kind_phys), pointer :: ulwsfci(:) => null() !< instantaneous sfc upwd lw flux ( w/m**2 ) real (kind=kind_phys), pointer :: dswsfci(:) => null() !< instantaneous sfc dnwd sw flux ( w/m**2 ) + real (kind=kind_phys), pointer :: dswsfcci(:) => null() !< instantaneous sfc dnwd sw flux ( w/m**2 ) (clear-sky) real (kind=kind_phys), pointer :: nswsfci(:) => null() !< instantaneous sfc net dnwd sw flux ( w/m**2 ) real (kind=kind_phys), pointer :: uswsfci(:) => null() !< instantaneous sfc upwd sw flux ( w/m**2 ) real (kind=kind_phys), pointer :: dusfci (:) => null() !< instantaneous u component of surface stress @@ -2945,11 +2947,13 @@ subroutine coupling_create (Coupling, Model) Coupling%visbmui = clear_val Coupling%visdfui = clear_val + allocate (Coupling%sfcdswc (IM)) allocate (Coupling%sfcdsw (IM)) allocate (Coupling%sfcnsw (IM)) allocate (Coupling%sfcdlw (IM)) allocate (Coupling%sfculw (IM)) + Coupling%sfcdswc = clear_val Coupling%sfcdsw = clear_val Coupling%sfcnsw = clear_val Coupling%sfcdlw = clear_val @@ -7822,6 +7826,7 @@ subroutine diag_create (Diag, Model) allocate (Diag%dlwsfci (IM)) allocate (Diag%ulwsfci (IM)) allocate (Diag%dswsfci (IM)) + allocate (Diag%dswsfcci(IM)) allocate (Diag%nswsfci (IM)) allocate (Diag%uswsfci (IM)) allocate (Diag%dusfci (IM)) @@ -8136,6 +8141,7 @@ subroutine diag_phys_zero (Diag, Model, linit, iauwindow_center) Diag%dlwsfci = zero Diag%ulwsfci = zero Diag%dswsfci = zero + Diag%dswsfcci = zero Diag%nswsfci = zero Diag%uswsfci = zero Diag%dusfci = zero diff --git a/ccpp/data/GFS_typedefs.meta b/ccpp/data/GFS_typedefs.meta index 5c9f73d70..fa20a47e7 100644 --- a/ccpp/data/GFS_typedefs.meta +++ b/ccpp/data/GFS_typedefs.meta @@ -2480,6 +2480,13 @@ dimensions = (horizontal_dimension) type = real kind = kind_phys +[sfcdswc] + standard_name = surface_downwelling_shortwave_flux_on_radiation_timestep_assuming_clear_sky + long_name = total sky sfc downward sw flux assuming clear sky conditions + units = W m-2 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys [sfcnsw] standard_name = surface_net_downwelling_shortwave_flux_on_radiation_timestep long_name = total sky sfc netsw flx into ground @@ -9173,6 +9180,13 @@ dimensions = (horizontal_dimension) type = real kind = kind_phys +[dswsfcci] + standard_name = surface_downwelling_shortwave_flux_assuming_clear_sky + long_name = surface downwelling shortwave flux at current time assuming clear sky + units = W m-2 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys [nswsfci] standard_name = surface_net_downwelling_shortwave_flux long_name = surface net downwelling shortwave flux at current time diff --git a/ccpp/driver/GFS_diagnostics.F90 b/ccpp/driver/GFS_diagnostics.F90 index d2a8bdf5a..38c3ef7d2 100644 --- a/ccpp/driver/GFS_diagnostics.F90 +++ b/ccpp/driver/GFS_diagnostics.F90 @@ -364,6 +364,18 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfci(Model%chunk_begin(nb):Model%chunk_end(nb)) enddo + idx = idx + 1 + ExtDiag(idx)%axes = 2 + ExtDiag(idx)%name = 'DSWRFCI' + ExtDiag(idx)%desc = 'instantaneous surface downward shortwave flux assuming clear sky' + ExtDiag(idx)%unit = 'w/m**2' + ExtDiag(idx)%mod_name = 'gfs_phys' + ExtDiag(idx)%intpl_method = 'bilinear' + allocate (ExtDiag(idx)%data(nblks)) + do nb = 1,nblks + ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfcci(Model%chunk_begin(nb):Model%chunk_end(nb)) + enddo + idx = idx + 1 ExtDiag(idx)%axes = 2 ExtDiag(idx)%name = 'USWRF' diff --git a/ccpp/physics b/ccpp/physics index 24643b0e0..061629b5c 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 24643b0e0f9754d76176d60ce155bf8956bfb44d +Subproject commit 061629b5c7d96875d8db2d106243be863fa68496 diff --git a/io/fv3atm_restart_io.F90 b/io/fv3atm_restart_io.F90 index e7f60262b..979c2a000 100644 --- a/io/fv3atm_restart_io.F90 +++ b/io/fv3atm_restart_io.F90 @@ -19,7 +19,13 @@ module fv3atm_restart_io_mod read_restart, write_restart, write_data, & get_global_io_domain_indices, get_dimension_size, & global_att_exists, get_global_attribute - use mpp_domains_mod, only: domain2d +#ifdef ENABLE_PARALLELRESTART + use mpp_domains_mod, only: domain2d, mpp_get_domain_tile_commid, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#else + use mpp_domains_mod, only: domain2d, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#endif use fv3atm_common_io, only: create_2d_field_and_add_to_bundle, & create_3d_field_and_add_to_bundle, copy_from_gfs_data, axis_type use fv3atm_sfc_io @@ -532,6 +538,7 @@ end subroutine fv3atm_checksum !! Also calculates sncovr if it is not present in the restart file. subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum) use fv3atm_rrfs_sd_io + use atmosphere_mod, only: Atm,mygrid implicit none !--- interface variable definitions type(GFS_sfcprop_type), intent(inout) :: Sfcprop @@ -558,12 +565,25 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta type(FmsNetcdfDomainFile_t) :: Oro_restart, Sfc_restart, dust12m_restart, emi_restart, rrfssd_restart type(FmsNetcdfDomainFile_t) :: Oro_ls_restart, Oro_ss_restart + type(domain2D) :: domain_for_read + integer :: read_layout(2) !--- OROGRAPHY FILE !--- open file +#ifdef ENABLE_PARALLELRESTART + Oro_restart%use_collective = .true. + call mpp_get_layout(Atm(mygrid)%domain, read_layout) + call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + Oro_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + infile=trim(indir)//'/'//trim(fn_oro) + amiopen=open_file(Oro_restart, trim(infile), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else infile=trim(indir)//'/'//trim(fn_oro) amiopen=open_file(Oro_restart, trim(infile), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if (.not.amiopen) call mpp_error( FATAL, 'Error with opening file '//trim(infile) ) call oro%register(Model,Oro_restart,Atm_block) @@ -669,8 +689,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta !--- SURFACE FILE !--- open file +#ifdef ENABLE_PARALLELRESTART + Sfc_restart%use_collective = .true. + Sfc_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + infile=trim(indir)//'/'//trim(fn_srf) + amiopen=open_file(Sfc_restart, trim(infile), "read", domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else infile=trim(indir)//'/'//trim(fn_srf) amiopen=open_file(Sfc_restart, trim(infile), "read", domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if( .not.amiopen ) call mpp_error(FATAL, 'Error opening file'//trim(infile)) if (global_att_exists(Sfc_restart, "file_version")) then @@ -840,6 +868,7 @@ end subroutine sfc_prop_restart_write !! restart variables with the GFDL FMS restart subsystem. !! Calls a GFDL FMS routine to restore the data from a restart file. subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum) + use atmosphere_mod, only: Atm,mygrid implicit none !--- interface variable definitions type(GFS_restart_type), intent(in) :: GFS_Restart @@ -859,6 +888,8 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r type(phy_data_type) :: phy type(FmsNetcdfDomainFile_t) :: Phy_restart + type(domain2D) :: domain_for_read + integer :: read_layout(2) isc = Atm_block%isc iec = Atm_block%iec @@ -871,7 +902,17 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r !--- open restart file and register axes fname = trim(indir)//'/'//trim(fn_phy) +#ifdef ENABLE_PARALLELRESTART + Phy_restart%use_collective = .true. + call mpp_get_layout(Atm(mygrid)%domain, read_layout) + call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + Phy_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + amiopen=open_file(Phy_restart, trim(fname), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else amiopen=open_file(Phy_restart, trim(fname), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if( amiopen ) then call register_axis(Phy_restart, 'xaxis_1', 'X') call register_axis(Phy_restart, 'yaxis_1', 'Y')