Fall back in proposal checking

fendermint/app/config/default.toml

@@ -5,6 +5,8 @@
 
 # Database files.
 data_dir = "data"
+# State snapshots.
+snapshots_dir = "snapshots"
 # Solidity contracts.
 contracts_dir = "contracts"
 # Builtin actor bundle CAR file.
@@ -37,6 +39,24 @@ port = 26658
 # Keep unlimited history by default.
 state_hist_size = 0
 
+[snapshots]
+# Enable the export and import of snapshots.
+enabled = false
+# How often to attempt to export snapshots in terms of block height.
+# With 1-2s block time, we can aim for 8h snapshots and cover the last 24h keeping the last 3.
+# The Filecoin Calibration net does not allow longer lookups than 16h, so we should have a snapshot within that time.
+block_interval = 30000
+# Number of snapshots to keep before purging old ones.
+# Keep the last 2-3 snapshots around as recommended by CometBFT docs.
+hist_size = 3
+# Target chunk size, in bytes.
+# It has to be less than 16MB and the FVM has max 1MB blocks, so 10MB as recommended by CometBFT docs is a good start.
+chunk_size_bytes = 10485760
+# How long to keep a snapshot from being purged after it has been requested by a peer.
+last_access_hold = 300
+# Ask CometBFT every now and then whether it's syncing; snapshot production is skipped
+sync_poll_interval = 60
+
 [broadcast]
 # Maximum number of times to retry broadcasting a transaction after failure.
 max_retries = 5

fendermint/app/settings/src/lib.rs

@@ -106,7 +106,7 @@ pub struct BroadcastSettings {
 
 #[serde_as]
 #[derive(Debug, Deserialize, Clone)]
-pub struct TopDownConfig {
+pub struct TopDownSettings {
     /// The number of blocks to delay before reporting a height as final on the parent chain.
     /// To propose a certain number of epochs delayed from the latest height, we see to be
     /// conservative and avoid other from rejecting the proposal because they don't see the
@@ -143,31 +143,53 @@ pub struct IpcSettings {
     pub subnet_id: SubnetID,
     /// The config for top down checkpoint. It's None if subnet id is root or not activating
     /// any top down checkpoint related operations
-    pub topdown: Option<TopDownConfig>,
+    pub topdown: Option<TopDownSettings>,
 }
 
 impl IpcSettings {
     pub fn is_topdown_enabled(&self) -> bool {
         !self.subnet_id.is_root() && self.topdown.is_some()
     }
 
-    pub fn topdown_config(&self) -> anyhow::Result<&TopDownConfig> {
+    pub fn topdown_config(&self) -> anyhow::Result<&TopDownSettings> {
         self.topdown
             .as_ref()
             .ok_or_else(|| anyhow!("top down config missing"))
     }
 }
 
+#[serde_as]
+#[derive(Debug, Deserialize, Clone)]
+pub struct SnapshotSettings {
+    /// Enable the export and import of snapshots.
+    pub enabled: bool,
+    /// How often to attempt to export snapshots in terms of block height.
+    pub block_interval: BlockHeight,
+    /// Number of snapshots to keep before purging old ones.
+    pub hist_size: usize,
+    /// Target chunk size, in bytes.
+    pub chunk_size_bytes: usize,
+    /// How long to keep a snapshot from being purged after it has been requested by a peer.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub last_access_hold: Duration,
+    /// How often to poll CometBFT to see whether it has caught up with the chain.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub sync_poll_interval: Duration,
+}
+
 #[derive(Debug, Deserialize, Clone)]
 pub struct Settings {
     /// Home directory configured on the CLI, to which all paths in settings can be set relative.
     home_dir: PathBuf,
     /// Database files.
     data_dir: PathBuf,
+    /// State snapshots.
+    snapshots_dir: PathBuf,
     /// Solidity contracts.
     contracts_dir: PathBuf,
     /// Builtin-actors CAR file.
     builtin_actors_bundle: PathBuf,
+
     /// Where to reach CometBFT for queries or broadcasting transactions.
     tendermint_rpc_url: Url,
 
@@ -176,6 +198,7 @@ pub struct Settings {
 
     pub abci: AbciSettings,
     pub db: DbSettings,
+    pub snapshots: SnapshotSettings,
     pub eth: EthSettings,
     pub fvm: FvmSettings,
     pub resolver: ResolverSettings,
@@ -207,7 +230,12 @@ macro_rules! home_relative {
 }
 
 impl Settings {
-    home_relative!(data_dir, contracts_dir, builtin_actors_bundle);
+    home_relative!(
+        data_dir,
+        snapshots_dir,
+        contracts_dir,
+        builtin_actors_bundle
+    );
 
     /// Load the default configuration from a directory,
     /// then potential overrides specific to the run mode,

fendermint/app/src/cmd/run.rs

@@ -15,6 +15,7 @@ use fendermint_vm_interpreter::{
     signed::SignedMessageInterpreter,
 };
 use fendermint_vm_resolver::ipld::IpldResolver;
+use fendermint_vm_snapshot::SnapshotManager;
 use fendermint_vm_topdown::proxy::IPCProviderProxy;
 use fendermint_vm_topdown::sync::launch_polling_syncer;
 use fendermint_vm_topdown::{CachedFinalityProvider, Toggle};
@@ -67,8 +68,9 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
     let tendermint_rpc_url = settings.tendermint_rpc_url()?;
     tracing::info!("Connecting to Tendermint at {tendermint_rpc_url}");
 
-    let client: tendermint_rpc::HttpClient = tendermint_rpc::HttpClient::new(tendermint_rpc_url)
-        .context("failed to create Tendermint client")?;
+    let tendermint_client: tendermint_rpc::HttpClient =
+        tendermint_rpc::HttpClient::new(tendermint_rpc_url)
+            .context("failed to create Tendermint client")?;
 
     let validator = match settings.validator_key {
         Some(ref key) => {
@@ -92,7 +94,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
         // For now we are using the validator key for submitting transactions.
         // This allows us to identify transactions coming from bonded validators, to give priority to protocol related transactions.
         let broadcaster = Broadcaster::new(
-            client.clone(),
+            tendermint_client.clone(),
             addr,
             sk.clone(),
             settings.fvm.gas_fee_cap.clone(),
@@ -106,7 +108,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
     });
 
     let interpreter = FvmMessageInterpreter::<NamespaceBlockstore, _>::new(
-        client.clone(),
+        tendermint_client.clone(),
         validator_ctx,
         settings.contracts_dir(),
         settings.fvm.gas_overestimation_rate,
@@ -127,6 +129,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
 
     let resolve_pool = CheckpointPool::new();
 
+    // If enabled, start a resolver that communicates with the application through the resolve pool.
     if settings.resolver.enabled() {
         let service =
             make_resolver_service(&settings, db.clone(), state_store.clone(), ns.bit_store)?;
@@ -180,6 +183,29 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
         (Arc::new(Toggle::disabled()), None)
     };
 
+    // Start a snapshot manager in the background.
+    let snapshots = if settings.snapshots.enabled {
+        let (manager, client) = SnapshotManager::new(
+            state_store.clone(),
+            settings.snapshots_dir(),
+            settings.snapshots.block_interval,
+            settings.snapshots.chunk_size_bytes,
+            settings.snapshots.hist_size,
+            settings.snapshots.last_access_hold,
+            settings.snapshots.sync_poll_interval,
+        )
+        .context("failed to create snapshot manager")?;
+
+        tracing::info!("starting the SnapshotManager...");
+        let tendermint_client = tendermint_client.clone();
+        tokio::spawn(async move { manager.run(tendermint_client).await });
+
+        Some(client)
+    } else {
+        info!("snapshots disabled");
+        None
+    };
+
     let app: App<_, _, AppStore, _> = App::new(
         AppConfig {
             app_namespace: ns.app,
@@ -192,7 +218,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
         interpreter,
         resolve_pool,
         parent_finality_provider.clone(),
-        None,
+        snapshots,
     )?;
 
     if let Some((agent_proxy, config)) = ipc_tuple {
@@ -203,7 +229,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> {
                 config,
                 parent_finality_provider,
                 agent_proxy,
-                client,
+                tendermint_client,
             )
             .await
             {

fendermint/testing/Makefile/ci.env

@@ -0,0 +1,3 @@
+CMT_WAIT_MILLIS = 20000
+# Help debug any issues with simplecoin by logging requests and responses.
+VERBOSITY = "--verbose"

fendermint/testing/Makefile/cometbft.toml

@@ -0,0 +1,3 @@
+extend = [
+  { path = "../../../infra/scripts/cometbft.toml" },
+]

fendermint/testing/Makefile/common.env

@@ -0,0 +1,22 @@
+CMT_DOCKER_IMAGE="cometbft/cometbft:v0.37.x"
+FM_DOCKER_IMAGE="fendermint:latest"
+CMT_CONTAINER_NAME="${NETWORK_NAME}-cometbft"
+FM_CONTAINER_NAME="${NETWORK_NAME}-fendermint"
+ETHAPI_CONTAINER_NAME="${NETWORK_NAME}-ethapi"
+CMT_P2P_HOST_PORT=26656
+CMT_RPC_HOST_PORT=26657
+ETHAPI_HOST_PORT=8545
+FM_NETWORK=main
+FM_LOG_LEVEL=info
+ETHAPI_LOG_LEVEL=debug
+# If this wasn't present, any wait task is skipped.
+CARGO_MAKE_WAIT_MILLISECONDS=5000
+# This wait time seems to work locally.
+CMT_WAIT_MILLIS=10000
+# Keep example logs to a minimum.
+VERBOSITY=""
+TEST_SCRIPTS_DIR="${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/fendermint/testing/${TEST_DIR}/scripts"
+TEST_DATA_DIR="${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/fendermint/testing/${TEST_DIR}/test-data"
+NODE_NAME=node-0
+CMT_DIR=${TEST_DATA_DIR}/${NODE_NAME}/cometbft
+BASE_DIR=${TEST_DATA_DIR}

fendermint/testing/Makefile/common.toml

@@ -0,0 +1,82 @@
+# smoke-test infrastructure:
+# cargo install cargo-make
+#
+# cd fendermint/testing/smoke-test
+# - then -
+# cargo make --profile ci
+# - or -
+# cargo make setup
+# cargo make test
+# docker logs smoke-ethapi
+# cargo make teardown
+
+extend = [
+  { path = "./docker.toml" },
+  { path = "./cometbft.toml" },
+  { path = "./fendermint.toml" },
+  { path = "./ethapi.toml" },
+]
+
+
+# Define the following in test specific `Makefile.toml` files,
+# where `env.env` defines `NETWORK_NAME` and `TEST_DIR`, expected by `common.env`:
+# env_files = [
+#   { path = "./env.env" },
+#   { path = "../Makefile/common.env" },
+#   { path = "../Makefile/ci.env", profile = "ci" },
+# ]
+
+
+[tasks.default]
+clear = true
+run_task = { name = [
+  "setup",
+  "test",
+], fork = true, cleanup_task = "teardown" }
+
+# Waiting before starting the Eth API for the CometBFT websocket to start listening.
+[tasks.setup]
+dependencies = [
+  "test-data-dir",
+  "docker-network-create",
+  "cometbft-init",
+  "fendermint-init",
+  "fendermint-start",
+  "cometbft-start",
+  "ethapi-start",
+  "cometbft-wait",
+  "fendermint-logs",
+  "cometbft-logs",
+  "ethapi-logs",
+]
+
+[tasks.test]
+clear = true
+dependencies = []
+
+[tasks.teardown]
+# `dependencies` doesn't seem to work with `cleanup_task`.
+run_task = { name = [
+  "cometbft-stop",
+  "cometbft-rm",
+  "fendermint-stop",
+  "fendermint-rm",
+  "ethapi-stop",
+  "ethapi-rm",
+  "docker-network-rm",
+  "test-data-dir-rm",
+] }
+
+
+[tasks.test-data-dir]
+script = """
+mkdir -p ${TEST_DATA_DIR}/keys;
+mkdir -p ${TEST_DATA_DIR}/${NODE_NAME}/fendermint/data/logs;
+mkdir -p ${TEST_DATA_DIR}/${NODE_NAME}/cometbft;
+cp -r ${TEST_SCRIPTS_DIR} ${TEST_DATA_DIR}/scripts
+"""
+
+[tasks.test-data-dir-rm]
+script = """
+rm -rf ${TEST_DATA_DIR}
+"""

fendermint/testing/Makefile/docker.toml

@@ -0,0 +1,3 @@
+extend = [
+  { path = "../../../infra/scripts/docker.toml" },
+]

fendermint/testing/Makefile/ethapi.toml

@@ -0,0 +1,3 @@
+extend = [
+  { path = "../../../infra/scripts/ethapi.toml" },
+]

fendermint/testing/Makefile/fendermint.toml

@@ -0,0 +1,7 @@
+extend = [
+  { path = "../../../infra/scripts/fendermint.toml" },
+]
+
+[tasks.fendermint-init]
+extend = "fendermint-run"
+env = { "ENTRY" = "/data/scripts/init.sh", "FLAGS" = "-a STDOUT -a STDERR --rm" }

fendermint/testing/contract-test/tests/staking/machine.rs

@@ -116,6 +116,7 @@ impl StateMachine for StakingMachine {
                 .unwrap(),
             min_validators: 1,
             min_cross_msg_fee: et::U256::zero(),
+            permissioned: false,
         };
 
         eprintln!("\n> PARENT IPC: {parent_ipc:?}");