Close BTD Snapshot when last slice is filled in. (#2516)

* Add logic to close snapshot when the last slice, with index 0, is filled

* fix eol

* call flush as soon as last valid Zslice is full

* clean up unwanted print statements

* add comments in code and doxygen comments

* remove commented line

* fix eol

* add doxygen comments and describe the new parameter introduced in the function

* fix typo from Axel

Co-authored-by: Axel Huebl <[email protected]>

* Update Source/Diagnostics/BTDiagnostics.cpp

* Update Source/Diagnostics/BTDiagnostics.cpp

* snake_style

* fix eol

* modified code so we dont use level info when setting snapshot full info

Co-authored-by: Axel Huebl <[email protected]>

Source/Diagnostics/BTDiagnostics.H

@@ -181,6 +181,15 @@ private:
      *  will be equal to the predicted m_max_buffer_multifabs for each snapshot.
      */
     amrex::Vector<int> m_max_buffer_multifabs;
+    /** Vector of integers to indicate if the snapshot is full (0 not full, 1 for full).
+     *  If the snapshot is full, then the snapshot files are closed.
+     */
+    amrex::Vector<int> m_snapshot_full;
+    /** Vector of integers to store if the last valid slice in the lab-frame
+     *  is being filled. When the value is 1, then the buffer is flushed, and
+     *  m_snapshot_full is set to 1 for that snapshot index.
+     */
+    amrex::Vector<int> m_lastValidZSlice;
     /** Vector of counters tracking number of times the buffer of multifab is
      *  flushed out and emptied before being refilled again for each snapshot */
     amrex::Vector<int> m_buffer_flush_counter;
@@ -276,6 +285,12 @@ private:
     void IncrementBufferFlushCounter(int i_buffer) {
         m_buffer_flush_counter[i_buffer]++;
     }
+    /** Set Snapshot full status to 1 if the last valid zslice, m_lastValidZSlice,
+     *  for the ith snapshot index, given by, i_buffer, is filled.
+     *
+     *  \param[in] i_buffer snapshot index
+     */
+    void SetSnapshotFullStatus (const int i_buffer);
     /** Vector of field-data stored in the cell-centered multifab, m_cell_centered_data.
      *  All the fields are stored regardless of the specific fields to plot selected
      *  by the user.

Source/Diagnostics/BTDiagnostics.cpp

@@ -83,8 +83,13 @@ void BTDiagnostics::DerivedInitData ()
     m_buffer_flush_counter.resize(m_num_buffers);
     // allocate vector of geometry objects corresponding to each snapshot
     m_geom_snapshot.resize( m_num_buffers );
+    m_snapshot_full.resize( m_num_buffers );
+    m_lastValidZSlice.resize( m_num_buffers );
     for (int i = 0; i < m_num_buffers; ++i) {
         m_geom_snapshot[i].resize(nmax_lev);
+        // initialize snapshot full boolean to false
+        m_snapshot_full[i] = 0;
+        m_lastValidZSlice[i] = 0;
     }
 
     for (int lev = 0; lev < nmax_lev; ++lev) {
@@ -156,9 +161,14 @@ BTDiagnostics::DoDump (int step, int i_buffer, bool force_flush)
     // timestep < 0, i.e., at initialization time when step == -1
     if (step < 0 )
         return false;
-    // buffer for this lab snapshot is full, time to dump it and continue
-    // to collect more slices afterwards
-    else if (buffer_full(i_buffer))
+    // Do not call dump if the snapshot is already full and the files are closed.
+    else if (m_snapshot_full[i_buffer] == 1)
+        return false;
+    // If buffer for this lab snapshot is full then dump it and continue to collect
+    // slices afterwards; or
+    // If last z-slice in the lab-frame snapshot is filled, call dump to
+    // write the buffer and close the file.
+    else if (buffer_full(i_buffer) || m_lastValidZSlice[i_buffer] == 1)
         return true;
     // forced: at the end of the simulation
     // empty: either lab snapshot was already fully written and buffer was reset
@@ -442,9 +452,12 @@ BTDiagnostics::PrepareFieldDataForOutput ()
                                              i_buffer, ZSliceInDomain,
                                              m_current_z_boost[i_buffer],
                                              m_buffer_box[i_buffer],
-                                             k_index_zlab(i_buffer, lev), m_max_box_size );
+                                             k_index_zlab(i_buffer, lev), m_max_box_size,
+                                             m_snapshot_full[i_buffer] );
 
                 if (ZSliceInDomain) ++m_buffer_counter[i_buffer];
+                // when the 0th z-index is filled, then set lastValidZSlice to 1
+                if (k_index_zlab(i_buffer, lev) == 0) m_lastValidZSlice[i_buffer] = 1;
             }
         }
     }
@@ -465,15 +478,23 @@ BTDiagnostics::k_index_zlab (int i_buffer, int lev)
     amrex::Real prob_domain_zmin_lab = m_prob_domain_lab[i_buffer].lo( m_moving_window_dir );
     amrex::IntVect ref_ratio = amrex::IntVect(1);
     if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1);
-    int k_lab = static_cast<unsigned>( (
+    int k_lab = static_cast<int>(floor (
                           ( m_current_z_lab[i_buffer]
                             - (prob_domain_zmin_lab + 0.5*dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]) ) )
                           / dz_lab( warpx.getdt(lev), ref_ratio[m_moving_window_dir] )
                       ) );
     return k_lab;
 }
 
+void
+BTDiagnostics::SetSnapshotFullStatus (const int i_buffer)
+{
+   if (m_snapshot_full[i_buffer] == 1) return;
+   // if the last valid z-index of the snapshot, which is 0, is filled, then
+   // set the snapshot full integer to 1
+   if (m_lastValidZSlice[i_buffer] == 1) m_snapshot_full[i_buffer] = 1;
 
+}
 
 void
 BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev)
@@ -610,8 +631,8 @@ BTDiagnostics::Flush (int i_buffer)
         file_name = amrex::Concatenate(m_file_prefix,i_buffer,5);
         file_name = file_name+"/buffer";
     }
-    bool isLastBTDFlush = ( ( m_max_buffer_multifabs[i_buffer]
-                               - m_buffer_flush_counter[i_buffer]) == 1) ? true : false;
+    SetSnapshotFullStatus(i_buffer);
+    bool isLastBTDFlush = ( m_snapshot_full[i_buffer] == 1 ) ? true : false;
     bool const isBTD = true;
     double const labtime = m_t_lab[i_buffer];
     m_flush_format->WriteToFile(

Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H

@@ -60,21 +60,22 @@ public:
 
     /** \brief Prepare data required to back-transform fields for lab-frame snapshot, i_buffer
      *
-     * \param[in] i_buffer        index of the snapshot
-     * \param[in] ZSliceInDomain  if the z-slice at current_z_boost is within
-     *                            the boosted-frame and lab-frame domain.
-     *                            The fields are sliced and back-transformed only
-     *                            if this value is true.
-     * \param[in] current_z_boost current z-coordinate of the slice in boosted-frame
-     * \param[in] buffer_box      Box with index-space in lab-frame for the ith buffer
-     * \param[in] k_index_zlab    k-index in the lab-frame corresponding to the
-     *                            current z co-ordinate in the lab-frame for the ith buffer.
-     * \param[in] max_box_size    maximum box size for the multifab to generate box arrays
+     * \param[in] i_buffer          index of the snapshot
+     * \param[in] z_slice_in_domain if the z-slice at current_z_boost is within
+     *            the boosted-frame and lab-frame domain.
+     *            The fields are sliced and back-transformed only if this value is true.
+     * \param[in] buffer_box        Box with index-space in lab-frame for the ith buffer
+     * \param[in] k_index_zlab      k-index in the lab-frame corresponding to the
+     *            current z co-ordinate in the lab-frame for the ith buffer.
+     * \param[in] max_box_size      maximum box size for the multifab to generate box arrays
+     * \param[in] snapshot_full     if the current snapshot, with index, i_buffer, is full (1)
+                  or not (0). If it is full, then Lorentz-transform is not performed
+                  by setting m_perform_backtransform to 0.
      */
-    void PrepareFunctorData ( int i_buffer, bool ZSliceInDomain,
+    void PrepareFunctorData ( int i_buffer, bool z_slice_in_domain,
                               amrex::Real current_z_boost,
                               amrex::Box buffer_box, const int k_index_zlab,
-                              const int max_box_size ) override;
+                              const int max_box_size, const int snapshot_full ) override;
     /** Allocate and initialize member variables and arrays required to back-transform
      *  field-data from boosted-frame to lab-frame.
      */
@@ -103,7 +104,7 @@ private:
     amrex::Vector<amrex::Real> m_current_z_boost;
     /** Vector of 0s and 1s stored to check if back-transformation is to be performed
      *  for the ith buffer. The value is set 0 (false) or 1 (true) using the boolean
-     *  ZSliceInDomain in PrepareFunctorData().
+     *  z_slice_in_domain in PrepareFunctorData().
      */
     amrex::Vector<int> m_perform_backtransform;
     /** Vector of k-index correspoding to the current lab-frame z co-ordinate for each buffer */

Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp

@@ -119,15 +119,15 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const
 
 void
 BackTransformFunctor::PrepareFunctorData (int i_buffer,
-                          bool ZSliceInDomain, amrex::Real current_z_boost,
+                          bool z_slice_in_domain, amrex::Real current_z_boost,
                           amrex::Box buffer_box, const int k_index_zlab,
-                          const int max_box_size )
+                          const int max_box_size, const int snapshot_full)
 {
     m_buffer_box[i_buffer] = buffer_box;
     m_current_z_boost[i_buffer] = current_z_boost;
     m_k_index_zlab[i_buffer] = k_index_zlab;
     m_perform_backtransform[i_buffer] = 0;
-    if (ZSliceInDomain) m_perform_backtransform[i_buffer] = 1;
+    if (z_slice_in_domain == true and snapshot_full == 0) m_perform_backtransform[i_buffer] = 1;
     m_max_box_size = max_box_size;
 }
 

Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H

@@ -30,14 +30,30 @@ public:
      * multifab */
     int nComp () const { return m_ncomp; }
 
-    virtual void PrepareFunctorData ( int i_buffer, bool ZSliceInDomain,
+    /** \brief Prepare data required to process fields in the operator()
+     *         Note that this function has parameters that are specific to
+     *         back-transformed diagnostics, that are unused for regular diagnostics.
+     *
+     * \param[in] i_buffer       index of the back-transform snapshot
+     * \param[in] z_slice_in_domain if the z-slice at current_z_boost is within
+     *            the boosted-frame and lab-frame domain.
+     *            The fields are sliced and back-transformed only if this value is true.
+     * \param[in] buffer_box     Box with index-space in lab-frame for the ith buffer
+     * \param[in] k_index_zlab   k-index in the lab-frame corresponding to the
+     *                           current z co-ordinate in the lab-frame for the ith buffer.
+     * \param[in] max_box_size   maximum box size for the multifab to generate box arrays
+     * \param[in] snapshot_full   if the current snapshot, with index, i_buffer, is full (1)
+                                 or not (0). If it is full, then Lorentz-transform
+                                 is not performed by setting m_perform_backtransform to 0;
+     */
+    virtual void PrepareFunctorData ( int i_buffer, bool z_slice_in_domain,
                                       amrex::Real current_z_boost,
                                       amrex::Box buffer_box, const int k_index_zlab,
-                                      const int max_box_size) {
+                                      const int max_box_size, const int snapshot_full) {
                                           amrex::ignore_unused(i_buffer,
-                                          ZSliceInDomain,
+                                          z_slice_in_domain,
                                           current_z_boost, buffer_box,
-                                          k_index_zlab, max_box_size);
+                                          k_index_zlab, max_box_size, snapshot_full);
                                       }
     virtual void InitData() {}
 private: