Sixth reconciliation PR from production/RRFS.v1

.gitmodules

@@ -1,15 +1,19 @@
 [submodule "atmos_cubed_sphere"]
   path = atmos_cubed_sphere
-  url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere
-  branch = dev/emc
+  #url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere
+  #branch = dev/emc
+  url = https://github.com/grantfirl/GFDL_atmos_cubed_sphere
+  branch = rrfsv1-to-ufs/dev6
 [submodule "ccpp/framework"]
   path = ccpp/framework
   url = https://github.com/NCAR/ccpp-framework
   branch = main
 [submodule "ccpp/physics"]
   path = ccpp/physics
-  url = https://github.com/ufs-community/ccpp-physics
-  branch = ufs/dev
+  #url = https://github.com/ufs-community/ccpp-physics
+  #branch = ufs/dev
+  url = https://github.com/grantfirl/ccpp-physics
+  branch = rrfsv1-to-ufs/dev6
 [submodule "upp"]
   path = upp
   url = https://github.com/NOAA-EMC/UPP

CMakeLists.txt

@@ -63,6 +63,14 @@ if(NOT PARALLEL_NETCDF)
   list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF)
 endif()
 
+if(ENABLE_PARALLELRESTART)
+  list(APPEND _fv3atm_defs_private ENABLE_PARALLELRESTART)
+endif()
+
+if(ENABLE_RRFS_WAR)
+  list(APPEND _fv3atm_defs_private ENABLE_RRFS_WAR)
+endif()
+
 if(MOVING_NEST)
   list(APPEND _fv3atm_defs_private MOVING_NEST)
   if(NOT HYDRO)

ccpp/data/GFS_typedefs.F90

@@ -523,6 +523,7 @@ module GFS_typedefs
     !--- In (physics only)
     real (kind=kind_phys), pointer :: sfcdsw(:)      => null()   !< total sky sfc downward sw flux ( w/m**2 )
                                                                  !< GFS_radtend_type%sfcfsw%dnfxc
+    real (kind=kind_phys), pointer :: sfcdswc(:)     => null()   !< total sky sfc downward sw flux assuming clear sky conditions( w/m**2 )
     real (kind=kind_phys), pointer :: sfcnsw(:)      => null()   !< total sky sfc netsw flx into ground(w/m**2)
                                                                  !< difference of dnfxc & upfxc from GFS_radtend_type%sfcfsw
     real (kind=kind_phys), pointer :: sfcdlw(:)      => null()   !< total sky sfc downward lw flux ( w/m**2 )
@@ -2028,6 +2029,7 @@ module GFS_typedefs
     real (kind=kind_phys), pointer :: dlwsfci(:)     => null()   !< instantaneous sfc dnwd lw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: ulwsfci(:)     => null()   !< instantaneous sfc upwd lw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: dswsfci(:)     => null()   !< instantaneous sfc dnwd sw flux ( w/m**2 )
+    real (kind=kind_phys), pointer :: dswsfcci(:)    => null()   !< instantaneous sfc dnwd sw flux ( w/m**2 ) (clear-sky)
     real (kind=kind_phys), pointer :: nswsfci(:)     => null()   !< instantaneous sfc net dnwd sw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: uswsfci(:)     => null()   !< instantaneous sfc upwd sw flux ( w/m**2 )
     real (kind=kind_phys), pointer :: dusfci (:)     => null()   !< instantaneous u component of surface stress
@@ -2944,11 +2946,13 @@ subroutine coupling_create (Coupling, Model)
     Coupling%visbmui = clear_val
     Coupling%visdfui = clear_val
 
+    allocate (Coupling%sfcdswc (IM))
     allocate (Coupling%sfcdsw (IM))
     allocate (Coupling%sfcnsw (IM))
     allocate (Coupling%sfcdlw (IM))
     allocate (Coupling%sfculw (IM))
 
+    Coupling%sfcdswc = clear_val
     Coupling%sfcdsw = clear_val
     Coupling%sfcnsw = clear_val
     Coupling%sfcdlw = clear_val
@@ -7818,6 +7822,7 @@ subroutine diag_create (Diag, Model)
     allocate (Diag%dlwsfci (IM))
     allocate (Diag%ulwsfci (IM))
     allocate (Diag%dswsfci (IM))
+    allocate (Diag%dswsfcci(IM))
     allocate (Diag%nswsfci (IM))
     allocate (Diag%uswsfci (IM))
     allocate (Diag%dusfci  (IM))
@@ -8132,6 +8137,7 @@ subroutine diag_phys_zero (Diag, Model, linit, iauwindow_center)
     Diag%dlwsfci    = zero
     Diag%ulwsfci    = zero
     Diag%dswsfci    = zero
+    Diag%dswsfcci   = zero
     Diag%nswsfci    = zero
     Diag%uswsfci    = zero
     Diag%dusfci     = zero

ccpp/data/GFS_typedefs.meta

@@ -2480,6 +2480,13 @@
   dimensions = (horizontal_dimension)
   type = real
   kind = kind_phys
+[sfcdswc]
+  standard_name = surface_downwelling_shortwave_flux_on_radiation_timestep_assuming_clear_sky
+  long_name = total sky sfc downward sw flux assuming clear sky conditions
+  units = W m-2
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
 [sfcnsw]
   standard_name = surface_net_downwelling_shortwave_flux_on_radiation_timestep
   long_name = total sky sfc netsw flx into ground
@@ -9166,6 +9173,13 @@
   dimensions = (horizontal_dimension)
   type = real
   kind = kind_phys
+[dswsfcci]
+  standard_name = surface_downwelling_shortwave_flux_assuming_clear_sky
+  long_name = surface downwelling shortwave flux at current time assuming clear sky
+  units = W m-2
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
 [nswsfci]
   standard_name = surface_net_downwelling_shortwave_flux
   long_name = surface net downwelling shortwave flux at current time

ccpp/driver/GFS_diagnostics.F90

@@ -364,6 +364,18 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfci(Model%chunk_begin(nb):Model%chunk_end(nb))
     enddo
 
+    idx = idx + 1
+    ExtDiag(idx)%axes = 2
+    ExtDiag(idx)%name = 'DSWRFCI'
+    ExtDiag(idx)%desc = 'instantaneous surface downward shortwave flux assuming clear sky'
+    ExtDiag(idx)%unit = 'w/m**2'
+    ExtDiag(idx)%mod_name = 'gfs_phys'
+    ExtDiag(idx)%intpl_method = 'bilinear'
+    allocate (ExtDiag(idx)%data(nblks))
+    do nb = 1,nblks
+      ExtDiag(idx)%data(nb)%var2 => IntDiag%dswsfcci(Model%chunk_begin(nb):Model%chunk_end(nb))
+    enddo    
+
     idx = idx + 1
     ExtDiag(idx)%axes = 2
     ExtDiag(idx)%name = 'USWRF'

io/fv3atm_restart_io.F90

@@ -19,7 +19,13 @@ module fv3atm_restart_io_mod
                                 read_restart, write_restart, write_data,     &
                                 get_global_io_domain_indices, get_dimension_size, &
                                 global_att_exists, get_global_attribute
-  use mpp_domains_mod,    only: domain2d
+#ifdef ENABLE_PARALLELRESTART
+  use mpp_domains_mod,    only: domain2d, mpp_get_domain_tile_commid, mpp_copy_domain, &
+                                  mpp_define_io_domain, mpp_get_layout
+#else
+  use mpp_domains_mod,    only: domain2d, mpp_copy_domain, &
+                                  mpp_define_io_domain, mpp_get_layout
+#endif
   use fv3atm_common_io,   only: create_2d_field_and_add_to_bundle, &
        create_3d_field_and_add_to_bundle, copy_from_gfs_data, axis_type
   use fv3atm_sfc_io
@@ -532,6 +538,7 @@ end subroutine fv3atm_checksum
   !!  Also calculates sncovr if it is not present in the restart file.
   subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum)
     use fv3atm_rrfs_sd_io
+    use atmosphere_mod,     only: Atm,mygrid
     implicit none
     !--- interface variable definitions
     type(GFS_sfcprop_type),    intent(inout) :: Sfcprop
@@ -558,12 +565,25 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta
 
     type(FmsNetcdfDomainFile_t) :: Oro_restart, Sfc_restart, dust12m_restart, emi_restart, rrfssd_restart
     type(FmsNetcdfDomainFile_t) :: Oro_ls_restart, Oro_ss_restart
+    type(domain2D) :: domain_for_read
+    integer :: read_layout(2)
 
     !--- OROGRAPHY FILE
 
     !--- open file
+#ifdef ENABLE_PARALLELRESTART
+    Oro_restart%use_collective = .true.
+    call mpp_get_layout(Atm(mygrid)%domain, read_layout)
+    call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read)
+    call mpp_define_io_domain(domain_for_read, read_layout)
+    Oro_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    infile=trim(indir)//'/'//trim(fn_oro)
+    amiopen=open_file(Oro_restart, trim(infile), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     infile=trim(indir)//'/'//trim(fn_oro)
     amiopen=open_file(Oro_restart, trim(infile), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if (.not.amiopen) call mpp_error( FATAL, 'Error with opening file '//trim(infile) )
 
     call oro%register(Model,Oro_restart,Atm_block)
@@ -669,8 +689,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta
     !--- SURFACE FILE
 
     !--- open file
+#ifdef ENABLE_PARALLELRESTART
+    Sfc_restart%use_collective = .true.
+    Sfc_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    infile=trim(indir)//'/'//trim(fn_srf)
+    amiopen=open_file(Sfc_restart, trim(infile), "read", domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     infile=trim(indir)//'/'//trim(fn_srf)
     amiopen=open_file(Sfc_restart, trim(infile), "read", domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if( .not.amiopen ) call mpp_error(FATAL, 'Error opening file'//trim(infile))
 
     if (global_att_exists(Sfc_restart, "file_version")) then
@@ -840,6 +868,7 @@ end subroutine sfc_prop_restart_write
   !! restart variables with the GFDL FMS restart subsystem.
   !! Calls a GFDL FMS routine to restore the data from a restart file.
   subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum)
+    use atmosphere_mod,     only: Atm,mygrid
     implicit none
     !--- interface variable definitions
     type(GFS_restart_type),      intent(in) :: GFS_Restart
@@ -859,6 +888,8 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r
 
     type(phy_data_type) :: phy
     type(FmsNetcdfDomainFile_t) :: Phy_restart
+    type(domain2D) :: domain_for_read
+    integer :: read_layout(2)
 
     isc = Atm_block%isc
     iec = Atm_block%iec
@@ -871,7 +902,17 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r
 
     !--- open restart file and register axes
     fname = trim(indir)//'/'//trim(fn_phy)
+#ifdef ENABLE_PARALLELRESTART
+    Phy_restart%use_collective = .true.
+    call mpp_get_layout(Atm(mygrid)%domain, read_layout)
+    call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read)
+    call mpp_define_io_domain(domain_for_read, read_layout)
+    Phy_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    amiopen=open_file(Phy_restart, trim(fname), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     amiopen=open_file(Phy_restart, trim(fname), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if( amiopen ) then
       call register_axis(Phy_restart, 'xaxis_1', 'X')
       call register_axis(Phy_restart, 'yaxis_1', 'Y')