Restructure the bufr sounding job

env/WCOSS2.env

@@ -223,7 +223,7 @@ elif [[ "${step}" = "postsnd" ]]; then
     export OMP_NUM_THREADS=1
 
     export NTHREADS_POSTSND=${NTHREADS1}
-    export APRUN_POSTSND="${APRUN} --depth=${NTHREADS_POSTSND} --cpu-bind depth"
+    export APRUN_POSTSND="${APRUN} -n ${ntasks} -ppn ${tasks_per_node} ${mpmd_opt}"
 
     export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
     [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}

parm/config/gfs/config.resources

@@ -1094,9 +1094,9 @@ case ${step} in
 
   "postsnd")
     walltime="02:00:00"
-    ntasks=40
-    threads_per_task=8
-    tasks_per_node=10
+    export ntasks=141
+    threads_per_task=6
+    export tasks_per_node=21 
     export ntasks_postsndcfp=9
     export tasks_per_node_postsndcfp=1
     postsnd_req_cores=$(( tasks_per_node * threads_per_task ))

scripts/exgfs_atmos_postsnd.sh

@@ -18,11 +18,16 @@
 #   7) 2018-07-18       Guang Ping Lou Generalize this version to other platforms
 #   8) 2019-10-18       Guang Ping Lou Transition to reading in NetCDF model data
 #   9) 2019-12-18       Guang Ping Lou generalizing to reading in NetCDF or nemsio
+#  10) 2024-08-08       Bo Cui Update to handle one forecast at a time
+#                          For GFSv17 bufr, total number of forecast hours is 141(num_hours=141) 
+#                          it requires 7 nodes & allocate 21 processes per node(num_ppn=21)
 ################################################################
 
 source "${USHgfs}/preamble.sh"
 
-cd $DATA
+runscript=${USHgfs}/gfs_bufr.sh 
+
+cd "${DATA}" || exit 2
 
 ########################################
 
@@ -44,47 +49,114 @@ export NINT3=${FHOUT_GFS:-3}
 
 rm -f -r "${COM_ATMOS_BUFR}"
 mkdir -p "${COM_ATMOS_BUFR}"
+
 GETDIM="${USHgfs}/getncdimlen"
 LEVS=$(${GETDIM} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf000.${atmfm}" pfull)
 declare -x LEVS
 
-### Loop for the hour and wait for the sigma and surface flux file:
-export FSTART=$STARTHOUR
-sleep_interval=10
-max_tries=360
-#
-while [ $FSTART -lt $ENDHOUR ]
-do
-export FINT=$NINT1
-   # Define the end hour for the input
-   export FEND=$(expr $FSTART + $INCREMENT) 
-   if test $FEND -lt 100; then FEND=0$FEND; fi 
-   if [ $FSTART -eq 00 ]
-   then 
-       export F00FLAG=YES
-   else
-       export F00FLAG=NO
-   fi
-
-   if [ $FEND -eq $ENDHOUR ]
-   then
-       export MAKEBUFR=YES
-   fi
-
-   filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}"
-   if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then
-     err_exit "FATAL ERROR: logf${FEND} not found after waiting $((sleep_interval * ( max_tries - 1) )) secs"
-   fi
+# Initialize an empty list to store the hours
+hour_list=()
 
-## 1-hourly output before $NEND1, 3-hourly output after
-   if [[ $((10#$FEND)) -gt $((10#$NEND1)) ]]; then
-     export FINT=$NINT3
-   fi
-   ${USHgfs}/gfs_bufr.sh
-
-   export FSTART="${FEND}"
+# Generate hours from 0 to NEND1 with interval NINT1
+for (( hour=0; hour<=NEND1 && hour<=ENDHOUR; hour+=NINT1 )); do
+  hour_list+=("$(printf "%03d" "$hour")")
+done
+
+# Generate hours from NEND1 + NINT3 to ENDHOUR with interval NINT3
+for (( hour=NEND1+NINT3; hour<=ENDHOUR; hour+=NINT3 )); do
+  hour_list+=("$(printf "%03d" "$hour")")
+done
+
+# Print the hour list
+echo "Hour List:" "${hour_list[@]}"
+
+# Count the number of elements in the hour_list
+export ntasks="${#hour_list[@]}"
+
+# Print the total number of hours
+echo "Total number of hours: $ntasks"
+
+# allocate 21 processes per node
+# don't allocate more processes, or it might have memory issue
+#export tasks_per_node=21
+export APRUN="mpiexec -np ${ntasks} -ppn ${tasks_per_node} --cpu-bind core cfp "
+
+if [ -s "${DATA}/poescript_bufr" ]; then
+  rm ${DATA}/poescript_bufr
+fi
+
+for fhr in "${hour_list[@]}"; do
+
+  if [ ! -s "${DATA}/${fhr}" ]; then mkdir -p ${DATA}/${fhr}; fi
+  export FINT=${NINT1}
+  ## 1-hourly output before $NEND1, 3-hourly output after
+  if [[ $((10#${fhr})) -gt $((10#${NEND1})) ]]; then
+    export FINT=${NINT3}
+  fi
+  if [[ $((10#${fhr})) -eq 0 ]]; then 
+     export F00FLAG="YES"
+  else
+     export F00FLAG="NO"
+  fi
+
+  # Convert fhr to integer
+  fhr_int=$((10#$fhr))
+
+  # Get previous hour
+  if (( fhr_int == STARTHOUR )); then
+    fhr_p=${fhr_int}
+  else
+    fhr_p=$(( fhr_int - FINT ))
+  fi
+
+  # Format fhr_p with leading zeros
+  fhr_p="$(printf "%03d" "$fhr_p")" 
+
+  filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${fhr}.${logfm}"
+  if [[ -z ${filename} ]]; then
+    echo "File ${filename} is required but not found."
+    err_exit "FATAL ERROR: logf${fhr} not found."
+  else
+    echo "${runscript} \"${fhr}\" \"${fhr_p}\" \"${FINT}\" \"${F00FLAG}\" \"${DATA}/${fhr}\"" >> "${DATA}/poescript_bufr"
+  fi
 done
 
+# Run with MPMD 
+chmod +x "${DATA}/poescript_bufr"
+$APRUN "${DATA}/poescript_bufr"
+export err=$?; err_chk
+
+# Run with MPMD 
+#chmod +x "${DATA}/poescript_bufr"
+#bash +x "${DATA}/poescript_bufr" > mpmd.out 2>&1
+#export err=$?; err_chk
+
+cd "${DATA}" || exit 2
+
+# Initialize fortnum
+fortnum=20
+
+# Loop through each element in the array
+for fhr in "${hour_list[@]}"; do
+    # Increment fortnum
+    fortnum=$((fortnum + 1))
+    ${NLN} "${DATA}/${fhr}/fort.${fortnum}" "fort.${fortnum}"
+done
+
+export MAKEBUFR=YES
+export fhr=${ENDHOUR}
+export FINT=${NINT1}
+## 1-hourly output before $NEND1, 3-hourly output after
+if [[ $((10#${fhr})) -gt $((10#${NEND1})) ]]; then
+  export FINT=${NINT3}
+fi
+if [[ $((10#${fhr})) -eq 0 ]]; then
+  export F00FLAG="YES"
+else
+  export F00FLAG="NO"
+fi
+${runscript} "${fhr}" "${fhr_p}" "${FINT}" "${F00FLAG}" "${DATA}" 
+
 ##############################################################
 # Tar and gzip the individual bufr files and send them to /com
 ##############################################################
@@ -105,7 +177,7 @@ fi
 # add appropriate WMO Headers.
 ########################################
 rm -rf poe_col
-for (( m = 1; m <= NUM_SND_COLLECTIVES ; m++ )); do
+for (( m = 1; m <= NUM_SND_COLLECTIVES; m++ )); do
     echo "sh ${USHgfs}/gfs_sndp.sh ${m} " >> poe_col
 done
 
@@ -123,4 +195,5 @@ ${APRUN_POSTSNDCFP} cmdfile
 sh "${USHgfs}/gfs_bfr2gpk.sh"
 
 
+
 ############## END OF SCRIPT #######################

ush/gfs_bufr.sh

@@ -18,8 +18,17 @@
 # 2018-05-30  Guang Ping Lou: Make sure all files are available.
 # 2019-10-10  Guang Ping Lou: Read in NetCDF files
 # 2024-03-03 Bo Cui: Add options to use different bufr table for different resolution NetCDF files
+# 2024-08-08 Bo Cui: Update to handle one forecast at a time
 # echo "History: February 2003 - First implementation of this utility script"
 #
+fhr="$1"
+fhr_p="$2"
+FINT="$3"
+F00FLAG="$4"
+workdir="$5"
+
+cd "${workdir}" || exit 2
+
 source "${USHgfs}/preamble.sh"
 
 if [[ "${F00FLAG}" == "YES" ]]; then
@@ -45,30 +54,34 @@ cat << EOF > gfsparm
  &NAMMET
   levs=${LEVS},makebufr=${bufrflag},
   dird="${COM_ATMOS_BUFR}/bufr",
-  nstart=${FSTART},nend=${FEND},nint=${FINT},
+  nstart=${fhr},nend=${fhr},nint=${FINT},
   nend1=${NEND1},nint1=${NINT1},nint3=${NINT3},
-  nsfc=80,f00=${f00flag},fformat=${fformat},np1=0
+  nsfc=80,f00=${f00flag},fformat=${fformat},np1=0,
+  fnsig="sigf${fhr}",
+  fngrib="flxf${fhr}", 
+  fngrib2="flxf${fhr_p}" 
 /
 EOF
 
-sleep_interval=10
-max_tries=1000
-for (( hr = 10#${FSTART}; hr <= 10#${FEND}; hr = hr + 10#${FINT} )); do
-   hh2=$(printf %02i "${hr}")
-   hh3=$(printf %03i "${hr}")
-
-   #---------------------------------------------------------
-   # Make sure all files are available:
-   filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${hh3}.${logfm}"
-   if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then
-     echo "FATAL ERROR: COULD NOT LOCATE logf${hh3} file"
-     exit 2
-   fi
-
-   #------------------------------------------------------------------
-   ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh3}.${atmfm}" "sigf${hh2}"
-   ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh3}.${atmfm}" "flxf${hh2}"
-done
+#---------------------------------------------------------
+# Make sure all files are available:
+
+filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${fhr}.${logfm}"
+if [[ -z ${filename} ]]; then
+  echo "FATAL ERROR: COULD NOT LOCATE logf${fhr} file"
+  exit 2
+fi
+
+filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${fhr_p}.${logfm}"
+if [[ -z ${filename} ]]; then
+  echo "FATAL ERROR: COULD NOT LOCATE logf${fhr_p} file"
+  exit 2
+fi
+
+#------------------------------------------------------------------
+${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${fhr}.${atmfm}" "sigf${fhr}"
+${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${fhr}.${atmfm}" "flxf${fhr}"
+${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${fhr_p}.${atmfm}" "flxf${fhr_p}"
 
 #  define input BUFR table file.
 ${NLN} "${PARMgfs}/product/bufr_gfs_${CLASS}.tbl" fort.1
@@ -82,18 +95,19 @@ case "${CASE}" in
         ${NLN} "${PARMgfs}/product/bufr_ij9km.txt"  fort.7
         ;;
     *)
-        echo "WARNING: No bufr table for this resolution, using the one for C768"
-        ${NLN} "${PARMgfs}/product/bufr_ij13km.txt" fort.7
+        echo "FATAL ERROR: Unrecognized bufr_ij*km.txt For CASE ${CASE}, ABORT!"
+        exit 1
         ;;
 esac
 
-${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}"
+"${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${fhr}"
+
 export err=$?
 
 if [[ "${err}" -ne 0 ]]; then
    echo "GFS postsnd job error, Please check files "
-   echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh2}.${atmfm}"
-   echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh2}.${atmfm}"
+   echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${fhr}.${atmfm}"
+   echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${fhr}.${atmfm}"
    err_chk
 fi