feat: add --track-peak-rss to daemon (#2696)

CHANGELOG.md

@@ -23,6 +23,8 @@ Notable updates:
   [#2579](https://github.com/ChainSafe/forest/pull/2579)
 - [forest daemon] Automatic database garbage collection.
   [#2638](https://github.com/ChainSafe/forest/pull/2638)
+- [forest daemon] Add `--track-peak-rss` to forest daemon
+  [#2696](https://github.com/ChainSafe/forest/pull/2696)
 
 ### Changed
 

forest/daemon/src/daemon.rs

@@ -31,7 +31,7 @@ use forest_rpc::start_rpc;
 use forest_rpc_api::data_types::RPCState;
 use forest_shim::version::NetworkVersion;
 use forest_state_manager::StateManager;
-use forest_utils::{io::write_to_file, retry};
+use forest_utils::{io::write_to_file, monitoring::MemStatsTracker, retry};
 use futures::{select, FutureExt};
 use fvm_ipld_blockstore::Blockstore;
 use log::{debug, error, info, warn};
@@ -74,6 +74,7 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result<Rolli
     }
 
     set_sigint_handler();
+
     let (shutdown_send, mut shutdown_recv) = tokio::sync::mpsc::channel(1);
     let mut terminate = signal(SignalKind::terminate())?;
 
@@ -124,6 +125,14 @@ pub(super) async fn start(opts: CliOpts, config: Config) -> anyhow::Result<Rolli
 
     let mut services = JoinSet::new();
 
+    if opts.track_peak_rss {
+        let mem_stats_tracker = MemStatsTracker::default();
+        services.spawn(async move {
+            mem_stats_tracker.run_loop().await;
+            Ok(())
+        });
+    }
+
     {
         // Start Prometheus server port
         let prometheus_listener = TcpListener::bind(config.client.metrics_address).context(

forest/shared/src/cli/mod.rs

@@ -138,6 +138,9 @@ pub struct CliOpts {
     /// If provided, indicates the file to which to save the admin token.
     #[arg(long)]
     pub save_token: Option<PathBuf>,
+    /// Track peak physical memory usage and print on exit
+    #[arg(long)]
+    pub track_peak_rss: bool,
 }
 
 impl CliOpts {

scripts/calibnet_health_check.sh

@@ -38,7 +38,7 @@ $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --hei
 echo "Checking DB stats"
 $FOREST_CLI_PATH --chain calibnet db stats
 echo "Running forest in detached mode"
-$FOREST_PATH --chain calibnet --encrypt-keystore false --log-dir "$LOG_DIRECTORY" --detach --save-token ./admin_token
+$FOREST_PATH --chain calibnet --encrypt-keystore false --log-dir "$LOG_DIRECTORY" --detach --save-token ./admin_token --track-peak-rss
 
 echo "Validating checkpoint tipset hashes"
 $FOREST_CLI_PATH chain validate-tipset-checkpoints

scripts/gen_coverage_report.sh

@@ -31,7 +31,7 @@ cargo llvm-cov --workspace --no-report --features slow_tests
 cov forest-cli --chain calibnet db clean --force
 cov forest-cli --chain calibnet snapshot fetch --aria2 -s "$TMP_DIR"
 SNAPSHOT_PATH=$(find "$TMP_DIR" -name \*.car | head -n 1)
-cov forest --chain calibnet --encrypt-keystore false --import-snapshot "$SNAPSHOT_PATH" --height=-200 --detach
+cov forest --chain calibnet --encrypt-keystore false --import-snapshot "$SNAPSHOT_PATH" --height=-200 --detach --track-peak-rss
 cov forest-cli sync wait
 cov forest-cli sync status
 cov forest-cli chain validate-tipset-checkpoints

utils/forest_utils/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
-name = "forest_utils"
+authors.workspace = true
+edition.workspace = true
 license.workspace = true
+name = "forest_utils"
 repository.workspace = true
 version.workspace = true
-authors.workspace = true
-edition.workspace = true
 
 [dependencies]
 ahash.workspace = true
@@ -26,6 +26,7 @@ hyper-rustls.workspace = true
 hyper.workspace = true
 libc = "0.2"
 log.workspace = true
+memory-stats = "1.1"
 parking_lot.workspace = true
 pbr = "1.1"
 pin-project-lite.workspace = true

utils/forest_utils/src/lib.rs

@@ -8,4 +8,5 @@ pub mod encoding;
 pub mod io;
 pub mod json;
 pub mod macros;
+pub mod monitoring;
 pub mod net;

utils/forest_utils/src/monitoring/mem_tracker.rs

@@ -0,0 +1,58 @@
+// Copyright 2019-2023 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use std::{
+    sync::atomic::{self, AtomicBool, AtomicUsize},
+    time::Duration,
+};
+
+use human_repr::HumanCount;
+use log::info;
+use memory_stats::memory_stats;
+
+pub struct MemStatsTracker {
+    check_interval: Duration,
+    peak_physical_mem: AtomicUsize,
+    cancelled: AtomicBool,
+}
+
+impl MemStatsTracker {
+    pub fn new(check_interval: Duration) -> Self {
+        assert!(check_interval > Duration::default());
+
+        Self {
+            check_interval,
+            peak_physical_mem: Default::default(),
+            cancelled: Default::default(),
+        }
+    }
+
+    /// A blocking loop that records peak RRS periodically
+    pub async fn run_loop(&self) {
+        while !self.cancelled.load(atomic::Ordering::Relaxed) {
+            if let Some(usage) = memory_stats() {
+                self.peak_physical_mem
+                    .fetch_max(usage.physical_mem, atomic::Ordering::Relaxed);
+            }
+            tokio::time::sleep(self.check_interval).await;
+        }
+    }
+}
+
+impl Default for MemStatsTracker {
+    fn default() -> Self {
+        Self::new(Duration::from_millis(1000))
+    }
+}
+
+impl Drop for MemStatsTracker {
+    fn drop(&mut self) {
+        self.cancelled.store(true, atomic::Ordering::Relaxed);
+        info!(
+            "Peak physical memory usage: {}",
+            self.peak_physical_mem
+                .load(atomic::Ordering::Relaxed)
+                .human_count_bytes()
+        );
+    }
+}

utils/forest_utils/src/monitoring/mod.rs

@@ -0,0 +1,5 @@
+// Copyright 2019-2023 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+mod mem_tracker;
+pub use mem_tracker::*;