Merge remote-tracking branch 'upstream/main' into history_domain_update

.github/actions/build_run_model/action.yml

@@ -23,10 +23,10 @@ inputs:
     required: false
     default: ''
   use-mpi:
-    description: 'specify whether to run program with MPI or not'
-    type: boolean
+    description: 'specify whether to run program without MPI, with MPI, or with MPIF08'
+    type: string
     required: true
-    default: true
+    default: mpi
   mpi-n-tasks:
     description: 'specify number of mpi tasks to run with program if mpi is used'
     type: number
@@ -44,21 +44,40 @@ runs:
         echo 'FFLAGS = -g -Wuninitialized -Wunused -ffree-line-length-none -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow $(INCS)' >> mkmf.template
       shell: bash
     # Steps to compile and build model
-    - name: Building ${{ inputs.model }} model
+    - name: Building ${{ inputs.model }} model (use-mpi=nompi)
+      if: ${{ inputs.use-mpi == 'nompi' }}
+      run: |
+        cd ${{ inputs.models-directory }}/${{ inputs.model }}/work
+        ./quickbuild.sh nompi
+      shell: bash
+    - name: Building ${{ inputs.model }} model (use-mpi=mpi)
+      if: ${{ inputs.use-mpi == 'mpi' }}
       run: |
         cd ${{ inputs.models-directory }}/${{ inputs.model }}/work
         ./quickbuild.sh
       shell: bash
+    - name: Building ${{ inputs.model }} model (use-mpi=mpif08)
+      if: ${{ inputs.use-mpi == 'mpif08' }}
+      run: |
+        cd ${{ inputs.models-directory }}/${{ inputs.model }}/work
+        ./quickbuild.sh mpif08
+      shell: bash
     # Steps to run the specified run-program with mpi options
-    - name: Running ${{ inputs.model }} ${{ inputs.run-program }} program (use-mpi=true)
-      if: ${{ inputs.use-mpi }} 
+    - name: Running ${{ inputs.model }} ${{ inputs.run-program }} program (use-mpi=nompi)
+      if: ${{ inputs.use-mpi == 'nompi' }}
+      run: |
+        cd ${{ inputs.models-directory }}/${{ inputs.model }}/work/
+        ./${{ inputs.run-program }} ${{ inputs.run-program-args }}
+      shell: bash
+    - name: Running ${{ inputs.model }} ${{ inputs.run-program }} program (use-mpi=mpi)
+      if: ${{ inputs.use-mpi == 'mpi' }}
       run: |
         cd ${{ inputs.models-directory }}/${{ inputs.model }}/work/
         mpirun -n ${{ inputs.mpi-n-tasks }} --allow-run-as-root ./${{ inputs.run-program }} ${{ inputs.run-program-args }}
       shell: bash
-    - name: Running ${{ inputs.model }} ${{ inputs.run-program }} program (use-mpi=false)
-      if: ${{ !inputs.use-mpi }}
+    - name: Running ${{ inputs.model }} ${{ inputs.run-program }} program (use-mpi=mpif08)
+      if: ${{ inputs.use-mpi == 'mpif08' }}
       run: |
         cd ${{ inputs.models-directory }}/${{ inputs.model }}/work/
-        ./${{ inputs.run-program }} ${{ inputs.run-program-args }}
+        mpirun -n ${{ inputs.mpi-n-tasks }} --allow-run-as-root ./${{ inputs.run-program }} ${{ inputs.run-program-args }}
       shell: bash

.github/workflows/action_on_pull_request.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   # Job 1
-  build-run-lorenz_96:
+  build-run-lorenz_96-mpi:
     # Runner instance OS
     runs-on: ubuntu-latest
     # Deploy container on top of runner instance
@@ -19,16 +19,16 @@ jobs:
         uses: actions/checkout@v3
       - name: Set checked out repo as a safe git directory
         run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
-      - name: Build and run lorenz_96
+      - name: Build and run lorenz_96 with mpi
         uses: ./.github/actions/build_run_model 
         with: 
           model: lorenz_96
           run-program: ./filter
-          use-mpi: true
+          use-mpi: mpi
           mpi-n-tasks: 2
 
   # Job 2
-  build-run-lorenz_63:
+  build-run-lorenz_63-nompi:
     runs-on: ubuntu-latest
     container:
       image: hkershaw/dart-dep:1.0
@@ -43,4 +43,23 @@ jobs:
         with:
           model: lorenz_63
           run-program: ./filter
-          use-mpi: false
+          use-mpi: nompi
+
+  # Job 3
+  build-run-lorenz_96-mpif08:
+    runs-on: ubuntu-latest
+    container:
+      image: hkershaw/dart-dep:1.0
+      options: '--cap-add=SYS_PTRACE'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Set checked out repo as a safe git directory
+        run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
+      - name: Build and run lorenz_96 with mpif08
+        uses: ./.github/actions/build_run_model
+        with:
+          model: lorenz_96
+          run-program: ./filter
+          use-mpi: mpif08
+          mpi-n-tasks: 2

CHANGELOG.rst

@@ -22,6 +22,28 @@ individual files.
 
 The changes are now listed with the most recent at the top.
 
+**November 9 2023 :: Github Actions MPIf08 Check. Tag v10.9.1**
+
+- Adds a new check to the Github Actions workflow that uses the
+  mpif08 module (compiles with ./quickbuild mpif08 and runs
+  filter on 2 mpi tasks with the lorenz_96 model).
+
+**November 7 2023 :: MPI f08 quickbuild option. Tag v10.9.0**
+
+- quickbuild.sh mpif08 option to build using the mpi_f08 module
+- nvhpc mkmf.template for use on Derecho
+
+bug-fixes:
+
+- filter_mod.dopperlerfold in sync with filter_mod
+- unnecessary loop removed from Mersenne twister developer test
+
+doc-fixes:
+
+- rename assim_model_mod.rst to match the module
+- fix various Sphinx warnings and broken link
+
+
 **October 5 2023 :: WRF-DART tutorial diagnostic section. Tag v10.8.5**
 
 - Improvements:

assimilation_code/modules/assimilation/filter_mod.dopplerfold.f90

@@ -385,7 +385,6 @@ subroutine filter_main()
 write(msgstring, '(A,I5)') 'running with an ensemble size of ', ens_size
 call error_handler(E_MSG,'filter_main:', msgstring, source)
 
-
 call set_missing_ok_status(allow_missing_clm)
 allow_missing = get_missing_ok_status()
 
@@ -982,6 +981,7 @@ subroutine filter_main()
       call timestamp_message('After  computing posterior observation values')
       call     trace_message('After  computing posterior observation values')
 
+
       call trace_message('Before posterior obs space diagnostics')
 
       ! Write posterior observation space diagnostics
@@ -995,6 +995,10 @@ subroutine filter_main()
 
       call trace_message('After  posterior obs space diagnostics')
    else
+      ! call this alternate routine to collect any updated QC values that may
+      ! have been set in the assimilation loop and copy them to the outgoing obs seq
+      call obs_space_sync_QCs(obs_fwd_op_ens_handle, seq, keys, num_obs_in_set, &
+                              OBS_GLOBAL_QC_COPY, DART_qc_index)
       call deallocate_single_copy(obs_fwd_op_ens_handle, prior_qc_copy)
    endif
 
@@ -1593,7 +1597,7 @@ subroutine obs_space_diagnostics(obs_fwd_op_ens_handle, qc_ens_handle, ens_size,
    OBS_MEAN_START, OBS_VAR_START, OBS_GLOBAL_QC_COPY, OBS_VAL_COPY, &
    OBS_ERR_VAR_COPY, DART_qc_index, do_post)
 
-! Do prior observation space diagnostics on the set of obs corresponding to keys
+! Do observation space diagnostics on the set of obs corresponding to keys
 
 type(ensemble_type),     intent(inout) :: obs_fwd_op_ens_handle, qc_ens_handle
 integer,                 intent(in)    :: ens_size
@@ -1701,6 +1705,52 @@ end subroutine obs_space_diagnostics
 
 !-------------------------------------------------------------------------
 
+subroutine obs_space_sync_QCs(obs_fwd_op_ens_handle,  &
+   seq, keys, num_obs_in_set, OBS_GLOBAL_QC_COPY, DART_qc_index)
+
+
+type(ensemble_type),     intent(inout) :: obs_fwd_op_ens_handle
+integer,                 intent(in)    :: num_obs_in_set
+integer,                 intent(in)    :: keys(num_obs_in_set)
+type(obs_sequence_type), intent(inout) :: seq
+integer,                 intent(in)    :: OBS_GLOBAL_QC_COPY
+integer,                 intent(in)    :: DART_qc_index
+
+integer               :: j
+integer               :: io_task, my_task
+real(r8), allocatable :: obs_temp(:)
+real(r8)              :: rvalue(1)
+
+! this is a query routine to return which task has 
+! logical processing element 0 in this ensemble.
+io_task = map_pe_to_task(obs_fwd_op_ens_handle, 0)
+my_task = my_task_id()
+
+! create temp space for QC values
+if (my_task == io_task) then
+   allocate(obs_temp(num_obs_in_set))
+else 
+   allocate(obs_temp(1))
+endif
+
+! Optimize: Could we use a gather instead of a transpose and get copy?
+call all_copies_to_all_vars(obs_fwd_op_ens_handle)
+
+! Update the qc global value
+call get_copy(io_task, obs_fwd_op_ens_handle, OBS_GLOBAL_QC_COPY, obs_temp)
+if(my_task == io_task) then
+   do j = 1, obs_fwd_op_ens_handle%num_vars
+      rvalue(1) = obs_temp(j)
+      call replace_qc(seq, keys(j), rvalue, DART_qc_index)
+   end do
+endif
+
+deallocate(obs_temp)
+
+end subroutine obs_space_sync_QCs
+
+!-------------------------------------------------------------------------
+
 subroutine filter_sync_keys_time(ens_handle, key_bounds, num_obs_in_set, time1, time2)
 
 integer,             intent(inout)  :: key_bounds(2), num_obs_in_set

assimilation_code/modules/utilities/fixsystem

@@ -39,7 +39,7 @@
 #
 
 # default file list, and both marker strings must exist in these files
-export FLIST="mpi_utilities_mod.f90 null_mpi_utilities_mod.f90"
+export FLIST="mpi_utilities_mod.f90 null_mpi_utilities_mod.f90 mpif08_utilities_mod.f90 "
 export STRINGS_REQUIRED=1
 
 # compiler name required.  additional filenames optional.