Update TEMPO for parallel CCPP implementation with Thompson and add 2…

…-moment graupel support (#5)

* Clean up for CCPP driver and module files

* Add more backwards compatability for MP (#2)

* Add TEMPO as a parallel option to Thompson

* Add hail-aware functionality

* Add option to call driver with hail-aware microphysics

* Move drivers to driver directory

drivers/ccpp/mp_tempo_post.F90

@@ -0,0 +1,150 @@
+module mp_tempo_post
+
+   use mpi_f08
+   use machine, only : kind_phys
+
+   implicit none
+
+   public :: mp_tempo_post_init, mp_tempo_post_run, mp_tempo_post_finalize
+
+   private
+
+   logical :: is_initialized = .false.
+
+   logical :: apply_limiter
+
+contains
+
+!! \section arg_table_mp_tempo_post_init Argument Table
+!! \htmlinclude mp_tempo_post_init.html
+!!
+   subroutine mp_tempo_post_init(ttendlim, errmsg, errflg)
+
+      implicit none
+
+      ! Interface variables
+      real(kind_phys), intent(in) :: ttendlim
+
+      ! CCPP error handling
+      character(len=*), intent(  out) :: errmsg
+      integer,          intent(  out) :: errflg
+
+      ! Local variables
+      integer :: i
+
+      ! Initialize the CCPP error handling variables
+      errmsg = ''
+      errflg = 0
+
+      ! Check initialization state
+      if (is_initialized) return
+
+      if (ttendlim < 0) then
+          apply_limiter = .false.
+      else
+          apply_limiter = .true.
+      end if
+
+      is_initialized = .true.
+
+   end subroutine mp_tempo_post_init
+
+!> \section arg_table_mp_tempo_post_run Argument Table
+!! \htmlinclude mp_tempo_post_run.html
+!!
+   subroutine mp_tempo_post_run(ncol, nlev, tgrs_save, tgrs, prslk, dtp, ttendlim, &
+                                   kdt, mpicomm, mpirank, mpiroot, errmsg, errflg)
+
+      implicit none
+
+      ! Interface variables
+      integer,                         intent(in)    :: ncol
+      integer,                         intent(in)    :: nlev
+      real(kind_phys), dimension(:,:), intent(in)    :: tgrs_save
+      real(kind_phys), dimension(:,:), intent(inout) :: tgrs
+      real(kind_phys), dimension(:,:), intent(in)    :: prslk
+      real(kind_phys),                 intent(in)    :: dtp
+      real(kind_phys),                 intent(in)    :: ttendlim
+      integer,                         intent(in)    :: kdt
+      ! MPI information
+      type(MPI_Comm),   intent(in   ) :: mpicomm
+      integer,          intent(in   ) :: mpirank
+      integer,          intent(in   ) :: mpiroot
+      ! CCPP error handling
+      character(len=*), intent(  out) :: errmsg
+      integer,          intent(  out) :: errflg
+
+      ! Local variables
+      real(kind_phys), dimension(1:ncol,1:nlev) :: mp_tend
+      integer :: i, k
+#ifdef DEBUG
+      integer :: events
+#endif
+
+      ! Initialize the CCPP error handling variables
+      errmsg = ''
+      errflg = 0
+
+      ! Check initialization state
+      if (.not.is_initialized) then
+         write(errmsg, fmt='((a))') 'mp_tempo_post_run called before mp_tempo_post_init'
+         errflg = 1
+         return
+      end if
+
+      ! If limiter is deactivated, return immediately
+      if (.not.apply_limiter) return
+
+      ! mp_tend and ttendlim are expressed in potential temperature
+      mp_tend = (tgrs - tgrs_save)/prslk
+
+#ifdef DEBUG
+      events = 0
+#endif
+      do k=1,nlev
+         do i=1,ncol
+            mp_tend(i,k) = max( -ttendlim*dtp, min( ttendlim*dtp, mp_tend(i,k) ) )
+
+#ifdef DEBUG
+            if (tgrs_save(i,k) + mp_tend(i,k)*prslk(i,k) .ne. tgrs(i,k)) then
+              write(0,'(a,3i6,3e16.7)') "mp_tempo_post_run mp_tend limiter: kdt, i, k, t_old, t_new, t_lim:", &
+                                      & kdt, i, k, tgrs_save(i,k), tgrs(i,k), tgrs_save(i,k) + mp_tend(i,k)*prslk(i,k)
+              events = events + 1
+            end if
+#endif
+            tgrs(i,k) = tgrs_save(i,k) + mp_tend(i,k)*prslk(i,k)
+         end do
+      end do
+
+#ifdef DEBUG
+      if (events > 0) then
+        write(0,'(a,i0,a,i0,a,i0)') "mp_tempo_post_run: ttendlim applied ", events, "/", nlev*ncol, &
+                                  & " times at timestep ", kdt
+      end if
+#endif
+
+   end subroutine mp_tempo_post_run
+
+!! \section arg_table_mp_tempo_post_finalize Argument Table
+!! \htmlinclude mp_tempo_post_finalize.html
+!!
+   subroutine mp_tempo_post_finalize(errmsg, errflg)
+
+      implicit none
+
+      ! CCPP error handling
+      character(len=*),          intent(  out) :: errmsg
+      integer,                   intent(  out) :: errflg
+
+      ! initialize ccpp error handling variables
+      errmsg = ''
+      errflg = 0
+
+      ! Check initialization state
+      if (.not. is_initialized) return
+
+      is_initialized = .false.
+
+   end subroutine mp_tempo_post_finalize
+
+end module mp_tempo_post

drivers/ccpp/mp_tempo_post.meta

@@ -0,0 +1,154 @@
+[ccpp-table-properties]
+  name = mp_tempo_post
+  type = scheme
+  dependencies = ../../../../hooks/machine.F
+
+########################################################################
+[ccpp-arg-table]
+  name = mp_tempo_post_init
+  type = scheme
+[ttendlim]
+  standard_name = max_tendency_of_air_potential_temperature_due_to_large_scale_precipitation
+  long_name = temperature tendency limiter per physics time step
+  units = K s-1
+  dimensions = ()
+  type = real
+  kind = kind_phys
+  intent = in
+[errmsg]
+  standard_name = ccpp_error_message
+  long_name = error message for error handling in CCPP
+  units = none
+  dimensions = ()
+  type = character
+  kind = len=*
+  intent = out
+[errflg]
+  standard_name = ccpp_error_code
+  long_name = error code for error handling in CCPP
+  units = 1
+  dimensions = ()
+  type = integer
+  intent = out
+
+########################################################################
+[ccpp-arg-table]
+  name = mp_tempo_post_run
+  type = scheme
+[ncol]
+  standard_name = horizontal_loop_extent
+  long_name = horizontal loop extent
+  units = count
+  dimensions = ()
+  type = integer
+  intent = in
+[nlev]
+  standard_name = vertical_layer_dimension
+  long_name = number of vertical levels
+  units = count
+  dimensions = ()
+  type = integer
+  intent = in
+[tgrs_save]
+  standard_name = air_temperature_save
+  long_name = air temperature before entering a physics scheme
+  units = K
+  dimensions = (horizontal_loop_extent,vertical_layer_dimension)
+  type = real
+  kind = kind_phys
+  intent = in
+[tgrs]
+  standard_name = air_temperature_of_new_state
+  long_name = model layer mean temperature
+  units = K
+  dimensions = (horizontal_loop_extent,vertical_layer_dimension)
+  type = real
+  kind = kind_phys
+  intent = inout
+[prslk]
+  standard_name = dimensionless_exner_function
+  long_name = dimensionless Exner function at model layer centers
+  units = none
+  dimensions = (horizontal_loop_extent,vertical_layer_dimension)
+  type = real
+  kind = kind_phys
+  intent = in
+[dtp]
+  standard_name = timestep_for_physics
+  long_name = physics timestep
+  units = s
+  dimensions = ()
+  type = real
+  kind = kind_phys
+  intent = in
+[ttendlim]
+  standard_name = max_tendency_of_air_potential_temperature_due_to_large_scale_precipitation
+  long_name = temperature tendency limiter per physics time step
+  units = K s-1
+  dimensions = ()
+  type = real
+  kind = kind_phys
+  intent = in
+[kdt]
+  standard_name = index_of_timestep
+  long_name = current forecast iteration
+  units = index
+  dimensions = ()
+  type = integer
+  intent = in
+[mpicomm]
+  standard_name = mpi_communicator
+  long_name = MPI communicator
+  units = index
+  dimensions = ()
+  type = MPI_Comm
+  intent = in
+[mpirank]
+  standard_name = mpi_rank
+  long_name = current MPI-rank
+  units = index
+  dimensions = ()
+  type = integer
+  intent = in
+[mpiroot]
+  standard_name = mpi_root
+  long_name = master MPI-rank
+  units = index
+  dimensions = ()
+  type = integer
+  intent = in
+[errmsg]
+  standard_name = ccpp_error_message
+  long_name = error message for error handling in CCPP
+  units = none
+  dimensions = ()
+  type = character
+  kind = len=*
+  intent = out
+[errflg]
+  standard_name = ccpp_error_code
+  long_name = error code for error handling in CCPP
+  units = 1
+  dimensions = ()
+  type = integer
+  intent = out
+
+########################################################################
+[ccpp-arg-table]
+  name = mp_tempo_post_finalize
+  type = scheme
+[errmsg]
+  standard_name = ccpp_error_message
+  long_name = error message for error handling in CCPP
+  units = none
+  dimensions = ()
+  type = character
+  kind = len=*
+  intent = out
+[errflg]
+  standard_name = ccpp_error_code
+  long_name = error code for error handling in CCPP
+  units = 1
+  dimensions = ()
+  type = integer
+  intent = out

drivers/ccpp/mp_tempo_pre.F90

@@ -0,0 +1,44 @@
+!>\file mp_tempo_pre.F90
+!!
+
+! CCPP license goes here, as well as further documentation
+!>\ingroup aatempo
+module mp_tempo_pre
+
+      use machine, only : kind_phys
+
+      implicit none
+
+      public :: mp_tempo_pre_run
+
+      private
+
+   contains
+
+!> \section arg_table_mp_tempo_pre_run Argument Table
+!! \htmlinclude mp_tempo_pre_run.html
+!!
+      subroutine mp_tempo_pre_run(ncol, nlev, tgrs, tgrs_save, errmsg, errflg)
+
+         implicit none
+
+         ! Interface variables
+         integer,                   intent(in   ) :: ncol
+         integer,                   intent(in   ) :: nlev
+         real(kind_phys),           intent(in   ) :: tgrs(:,:)
+         real(kind_phys),           intent(  out) :: tgrs_save(:,:)
+
+         ! CCPP error handling
+         character(len=*),          intent(  out) :: errmsg
+         integer,                   intent(  out) :: errflg
+
+         ! Initialize the CCPP error handling variables
+         errmsg = ''
+         errflg = 0
+
+         ! Save current air temperature for tendency limiters in mp_tempo_post
+         tgrs_save = tgrs
+
+      end subroutine mp_tempo_pre_run
+
+end module mp_tempo_pre