cleanup procedure change: flush persistent data on leave

include/derecho/core/detail/group_impl.hpp

@@ -276,10 +276,11 @@ Group<ReplicatedTypes...>::Group(const SubgroupInfo& subgroup_info, Factory<Repl
 
 template <typename... ReplicatedTypes>
 Group<ReplicatedTypes...>::~Group() {
-    // shutdown the persistence manager
-    // TODO-discussion:
-    // Will a node be able to come back once it leaves? if not, maybe we should
-    // shut it down on leave().
+    /**
+        We shut down the persistence_manager on leave, assuming a process will not come back once it leaves.
+        Here we shut it down again, in case the process is just leaving by itself instead of a system-level
+        shutdown. Please note that the shutdown opeartion is idempotent.
+     */
     persistence_manager.shutdown(true);
 }
 
@@ -531,6 +532,9 @@ void Group<ReplicatedTypes...>::leave(bool group_shutdown) {
     if(group_shutdown) {
         view_manager.silence();
         view_manager.barrier_sync();
+        /* Here we flush the data to persistent store and wait until it finished. */
+        persistence_manager.shutdown(true);
+        view_manager.barrier_sync();
     }
     view_manager.leave();
 }

src/core/persistence_manager.cpp

@@ -35,6 +35,9 @@ PersistenceManager::PersistenceManager(
 }
 
 PersistenceManager::~PersistenceManager() {
+    if (this->persist_thread.joinable()) {
+        this->persist_thread.join();
+    }
     sem_destroy(&persistence_request_sem);
 }
 
@@ -233,7 +236,9 @@ void PersistenceManager::shutdown(bool wait) {
     sem_post(&persistence_request_sem);  // kick the persistence thread in case it is sleeping
 
     if(wait) {
-        this->persist_thread.join();
+        if (this->persist_thread.joinable()) {
+            this->persist_thread.join();
+        }
     }
 }
 }  // namespace derecho