Add diag_manager testing for openmp

diag_manager/fms_diag_field_object.F90

@@ -388,37 +388,49 @@ subroutine set_data_buffer (this, input_data, diag_axis, is, js, ks, ie, je, ke)
   class (fmsDiagField_type) , intent(inout):: this !< The field object
   class(*), dimension(:,:,:,:), intent(in) :: input_data !< The input array
   class(fmsDiagAxisContainer_type),intent(in)   :: diag_axis(:)          !< Array of diag_axis
-  integer :: is, js, ks !< Starting indicies of the field_data
-  integer :: ie, je, ke !< Ending indicied of the field_data
-!> Allocate the buffer if it is not allocated
+  integer :: is, js, ks !< Starting indicies of the field_data relative to the global domain
+  integer :: ie, je, ke !< Ending indicied of the field_data relative to the global domain
+  integer :: isc, jsc, ksc !< Starting indicies of the field_data relative to the compute domain
+  integer :: iec, jec, kec !< Ending indicied of the field_data relative to the compute domain
+  integer :: cds(4) !< Compute domain starting indices
+
+  !> Allocate the buffer if it is not allocated
   if (.not.allocated(this%data_buffer_allocated)) this%data_buffer_allocated = .false.
   if (.not.this%data_buffer_allocated) &
     this%data_buffer_allocated =  allocate_data_buffer(this, input_data, diag_axis)
   if (.not.this%data_buffer_allocated) &
     call mpp_error ("set_data_buffer", "The data buffer for the field "//trim(this%varname)//" was unable to be "//&
       "allocated.", FATAL)
 
+  cds = get_starting_compute_domain(this%axis_ids, diag_axis)
+  isc = is - cds(1) + 1
+  jsc = js - cds(2) + 1
+  ksc = ks - cds(3) + 1
+  iec = isc + size(input_data, 1) - 1
+  jec = jsc + size(input_data, 2) - 1
+  kec = ksc + size(input_data, 3) - 1
+
 !> Buffer a copy of the data
   select type (input_data)
     type is (real(kind=r4_kind))
       select type (db => this%data_buffer)
         type is (real(kind=r4_kind))
-          db(is:ie, js:je, ks:ke, :) = input_data
+          db(isc:iec, jsc:jec, ksc:kec, :) = input_data
       end select
     type is (real(kind=r8_kind))
       select type (db => this%data_buffer)
         type is (real(kind=r8_kind))
-          db(is:ie, js:je, ks:ke, :) = input_data
+          db(isc:iec, jsc:jec, ksc:kec, :) = input_data
       end select
     type is (integer(kind=i4_kind))
       select type (db => this%data_buffer)
         type is (integer(kind=i4_kind))
-          db(is:ie, js:je, ks:ke, :) = input_data
+          db(isc:iec, jsc:jec, ksc:kec, :) = input_data
       end select
     type is (integer(kind=i8_kind))
       select type (db => this%data_buffer)
         type is (integer(kind=i8_kind))
-          db(is:ie, js:je, ks:ke, :) = input_data
+          db(isc:iec, jsc:jec, ksc:kec, :) = input_data
       end select
     class default
         call mpp_error ("set_data_buffer", "The data input to set_data_buffer for "//&
@@ -1640,5 +1652,27 @@ subroutine dump_field_obj (this, unit_num)
 
 end subroutine
 
+!< @brief Get the starting compute domain indices for a set of axis
+!! @return compute domain starting indices
+function get_starting_compute_domain(axis_ids, diag_axis) &
+result(compute_domain)
+  integer,                         intent(in) :: axis_ids(:)  !< Array of axis ids
+  class(fmsDiagAxisContainer_type),intent(in) :: diag_axis(:) !< Array of axis object
+
+  integer :: compute_domain(4)
+  integer :: a              !< For looping through axes
+  integer :: compute_idx(2) !< Compute domain indices (starting, ending)
+  logical :: dummy          !< Dummy variable for the `get_compute_domain` subroutine
+
+  compute_domain = 1
+  axis_loop: do a = 1,size(axis_ids)
+    select type (axis => diag_axis(axis_ids(a))%axis)
+      type is (fmsDiagFullAxis_type)
+        call axis%get_compute_domain(compute_idx, dummy)
+        if ( compute_idx(1) .ne. diag_null) compute_domain(a) = compute_idx(1)
+    end select
+  enddo axis_loop
+end function get_starting_compute_domain
+
 #endif
 end module fms_diag_field_object_mod

test_fms/diag_manager/Makefile.am

@@ -30,7 +30,7 @@ LDADD = $(top_builddir)/libFMS/libFMS.la
 # Build this test program.
 check_PROGRAMS = test_diag_manager test_diag_manager_time \
  test_diag_dlinked_list test_diag_yaml test_diag_ocean test_modern_diag test_diag_buffer \
-	test_flexible_time test_diag_update_buffer
+	test_flexible_time test_diag_update_buffer test_dm_openmp
 
 # This is the source code for the test.
 test_diag_manager_SOURCES = test_diag_manager.F90
@@ -42,6 +42,7 @@ test_diag_ocean_SOURCES = test_diag_ocean.F90
 test_modern_diag_SOURCES = test_modern_diag.F90
 test_diag_buffer_SOURCES= test_diag_buffer.F90
 test_flexible_time_SOURCES = test_flexible_time.F90
+test_dm_openmp_SOURCES = test_dm_openmp.F90
 
 TEST_EXTENSIONS = .sh
 SH_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \

test_fms/diag_manager/test_diag_manager2.sh

@@ -891,6 +891,36 @@ printf "&diag_manager_nml \n use_modern_diag = .false. \n use_clock_average = .t
     mpirun -n 1 ../test_flexible_time
   '
 
+printf "&diag_manager_nml \n use_modern_diag = .true. \n /" | cat > input.nml
+cat <<_EOF > diag_table.yaml
+title: test_diag_manager
+base_date: 2 1 1 0 0 0
+diag_files:
+- file_name: file_openmp_test
+  freq: 1
+  freq_units: hours
+  time_units: hours
+  unlimdim: time
+  varlist:
+  - module: ocn_mod
+    var_name: var1
+    reduction: none
+    kind: r4
+  - module: ocn_mod
+    var_name: var2
+    reduction: none
+    kind: r4
+  - module: ocn_mod
+    var_name: var3
+    reduction: none
+    kind: r4
+_EOF
+
+export OMP_NUM_THREADS=2
+my_test_count=`expr $my_test_count + 1`
+  test_expect_success "Test the modern diag manager end to end but it uses the openmp stuff(test $my_test_count)" '
+    mpirun -n 6 ../test_dm_openmp
+  '
 else
   my_test_count=`expr $my_test_count + 1`
   test_expect_failure "test modern diag manager failure when compiled without -Duse-yaml flag (test $my_test_count)" '

test_fms/diag_manager/test_dm_openmp.F90

@@ -0,0 +1,139 @@
+!***********************************************************************
+!*                   GNU Lesser General Public License
+!*
+!* This file is part of the GFDL Flexible Modeling System (FMS).
+!*
+!* FMS is free software: you can redistribute it and/or modify it under
+!* the terms of the GNU Lesser General Public License as published by
+!* the Free Software Foundation, either version 3 of the License, or (at
+!* your option) any later version.
+!*
+!* FMS is distributed in the hope that it will be useful, but WITHOUT
+!* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+!* FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+!* for more details.
+!*
+!* You should have received a copy of the GNU Lesser General Public
+!* License along with FMS.  If not, see <http://www.gnu.org/licenses/>.
+!***********************************************************************
+
+!> @brief  This programs tests the modern diag_manager
+
+program test_diag_openmp
+  use omp_lib
+  use mpp_mod,           only: mpp_npes, mpp_pe, mpp_sync
+  use platform_mod,      only: r8_kind
+  use mpp_domains_mod,   only: domain2d, mpp_define_domains, mpp_define_io_domain, mpp_get_compute_domain
+  use block_control_mod, only: block_control_type, define_blocks
+  use fms_mod,           only: fms_init, fms_end
+  use diag_manager_mod,  only: diag_manager_init, diag_manager_end, diag_axis_init, register_diag_field, &
+                               diag_send_complete, diag_manager_set_time_end, send_data, register_static_field
+  use time_manager_mod,  only: time_type, set_calendar_type, set_date, JULIAN, set_time
+
+
+  implicit none
+
+  integer                            :: nx           !< Number of points in the x direction
+  integer                            :: ny           !< Number of points in the y direction
+  integer                            :: nz           !< Number of points in the z direction
+  integer                            :: layout(2)    !< Layout
+  integer                            :: io_layout(2) !< Io layout
+  type(domain2d)                     :: Domain       !< 2D domain
+  integer                            :: is           !< Starting x compute index
+  integer                            :: ie           !< Ending x compute index
+  integer                            :: js           !< Starting y compute index
+  integer                            :: je           !< Ending y compute index
+  type(time_type)                    :: Time         !< Time of the simulation
+  type(time_type)                    :: Time_step    !< Time of the simulation
+  real,    dimension(:), allocatable :: x            !< X axis data
+  integer                            :: id_x         !< axis id for the x dimension
+  real,    dimension(:), allocatable :: y            !< Y axis_data
+  integer                            :: id_y         !< axis id for the y dimension
+  real,    dimension(:), allocatable :: z            !< Z axis data
+  integer                            :: id_z         !< axis id for the z dimension
+  real(kind=r8_kind),    allocatable :: var(:,:,:)   !< Dummy variable data
+  integer                            :: i, j         !< For do loops
+  type(block_control_type)           :: my_block     !< Returns instantiated @ref block_control_type
+  logical                            :: message      !< Flag for outputting debug message
+  integer                            :: isw          !< Starting index for each thread in the x direction
+  integer                            :: iew          !< Ending index for each thread in the x direction
+  integer                            :: jsw          !< Starting index for each thread in the y direction
+  integer                            :: jew          !< Ending index for each thread in the y direction
+  integer                            :: id_var1      !< diag_field id for var in 1d
+  integer                            :: id_var2      !< diag_field id for var in lon/lat grid
+  integer                            :: id_var3      !< diag_field id for var in lon/lat/z grid
+  logical                            :: used         !< .true. if the send_data call was sucessful
+
+  call fms_init
+  call set_calendar_type(JULIAN)
+  call diag_manager_init
+
+  nx = 96
+  ny = 96
+  nz = 5
+  layout = (/1, mpp_npes()/)
+  io_layout = (/1, 1/)
+
+  ! Set up the intial time
+  Time = set_date(2,1,1,0,0,0)
+
+  !< Create a lat/lon domain
+  call mpp_define_domains( (/1,nx,1,ny/), layout, Domain, name='2D domain')
+  call mpp_define_io_domain(Domain, io_layout)
+  call mpp_get_compute_domain(Domain, is, ie, js, je)
+
+  ! Set up the data
+  allocate(x(nx), y(ny), z(nz))
+  allocate(var(is:ie, js:je, nz))
+  do i=1,nx
+    x(i) = i
+  enddo
+
+  do i=1,ny
+    y(i) = i
+  enddo
+
+  do i=1,nz
+    z(i) = i
+  enddo
+
+  !< Register the axis:
+  id_x  = diag_axis_init('x',  x,  'point_E', 'x', long_name='point_E', Domain2=Domain)
+  id_y  = diag_axis_init('y',  y,  'point_N', 'y', long_name='point_N', Domain2=Domain)
+  id_z  = diag_axis_init('z',  z,  'pressure', 'z', long_name='too much pressure')
+
+  !< Register the variables
+  id_var1 = register_diag_field  ('ocn_mod', 'var1', (/id_x/), Time, 'Var in a lon domain', 'mullions')
+  id_var2 = register_diag_field  ('ocn_mod', 'var2', (/id_x, id_y/), Time, 'Var in a lon/lat domain', 'mullions')
+  id_var3 = register_diag_field  ('ocn_mod', 'var3', (/id_x, id_y, id_z/), Time, &
+    'Var in a lon/lat/z domain', 'mullions')
+
+  call diag_manager_set_time_end(set_date(2,1,2,0,0,0))
+
+  !< Divide the domain further into blocks
+  call define_blocks ('testing_model', my_block, is, ie, js, je, kpts=0, &
+                      nx_block=1, ny_block=4, message=message)
+
+  Time_step = set_time (3600,0) !< 1 hour
+  do j = 1, 23 !simulated time
+    Time = set_date(2,1,1,j,0,0)
+    var = real(j, kind=r8_kind) !< Set the data
+!$OMP parallel do default(shared) private(i, isw, iew, jsw, jew) schedule (dynamic,1)
+    do i = 1, 4
+      isw = my_block%ibs(i)
+      jsw = my_block%jbs(i)
+      iew = my_block%ibe(i)
+      jew = my_block%jbe(i)
+
+      used=send_data(id_var1, var(isw:iew, 1, 1), time, is_in=isw, ie_in=iew)
+      used=send_data(id_var2, var(isw:iew, jsw:jew, 1), time, is_in=isw, js_in=jsw, &
+                     ie_in=iew, je_in=jew)
+      used=send_data(id_var3, var(isw:iew, jsw:jew, :), time, is_in=isw, js_in=jsw, &
+                     ie_in=iew, je_in=jew, ks_in=1, ke_in=nz)
+    enddo
+    call diag_send_complete(Time_step)
+  enddo
+
+  call diag_manager_end(Time)
+  call fms_end
+end program test_diag_openmp