[Feature] Add new configuration to specify the snapshot directory

.github/workflows/integration-test.yml

@@ -17,26 +17,21 @@ jobs:
   integration-tests:
     name: Integration Test (${{ matrix.test.name }})
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     strategy:
       # other jobs should run even if one integration test fails
-      fail-fast: false
+      fail-fast: true
       matrix:
         test: [
           {
-            name: "Dex Module",
+            name: "Dex & Wasm Module",
             scripts: [
               "docker exec sei-node-0 integration_test/contracts/deploy_dex_contract.sh mars",
               "python3 integration_test/scripts/runner.py integration_test/dex_module/place_order_test.yaml",
-              "python3 integration_test/scripts/runner.py integration_test/dex_module/cancel_order_test.yaml"
-            ]
-          },
-          {
-            name: "Wasm Module",
-            scripts: [
+              "python3 integration_test/scripts/runner.py integration_test/dex_module/cancel_order_test.yaml",
               "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh",
               "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_delegation_test.yaml",
               "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_admin_test.yaml",
@@ -46,44 +41,35 @@ jobs:
             ]
           },
           {
-            name: "Mint Module",
+            name: "Mint & Staking & Bank Module",
             scripts: [
+              "python3 integration_test/scripts/runner.py integration_test/staking_module/staking_test.yaml",
+              "python3 integration_test/scripts/runner.py integration_test/bank_module/send_funds_test.yaml",
               "python3 integration_test/scripts/runner.py integration_test/mint_module/mint_test.yaml"
             ]
           },
           {
-            name: "Staking Module",
-            scripts: [
-              "python3 integration_test/scripts/runner.py integration_test/staking_module/staking_test.yaml"
-            ]
-          },
-          {
-            name: "Gov Module",
+            name: "Gov & Oracle & Authz Module",
             scripts: [
               "python3 integration_test/scripts/runner.py integration_test/gov_module/gov_proposal_test.yaml",
-              "python3 integration_test/scripts/runner.py integration_test/gov_module/staking_proposal_test.yaml"
-            ]
-          },
-          {
-            name: "Oracle Module",
-            scripts: [
+              "python3 integration_test/scripts/runner.py integration_test/gov_module/staking_proposal_test.yaml",
               "python3 integration_test/scripts/runner.py integration_test/oracle_module/verify_penalty_counts.yaml",
-              "python3 integration_test/scripts/runner.py integration_test/oracle_module/set_feeder_test.yaml"
-            ]
-          },
-          {
-            name: "Authz Module",
-            scripts: [
+              "python3 integration_test/scripts/runner.py integration_test/oracle_module/set_feeder_test.yaml",
               "python3 integration_test/scripts/runner.py integration_test/authz_module/send_authorization_test.yaml",
               "python3 integration_test/scripts/runner.py integration_test/authz_module/staking_authorization_test.yaml",
               "python3 integration_test/scripts/runner.py integration_test/authz_module/generic_authorization_test.yaml"
             ]
           },
           {
-            name: "Bank Module",
+            name: "Chain Operation Test",
             scripts: [
-              "python3 integration_test/scripts/runner.py integration_test/bank_module/send_funds_test.yaml",
+              "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done",
+              "until [[ $(docker exec sei-rpc-node build/seid status |jq -M -r .SyncInfo.latest_block_height) -gt 10 ]]; do sleep 10; done",
+              "echo rpc node started",
+              "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml",
+              "python3 integration_test/scripts/runner.py integration_test/chain_operation/statesync_operation.yaml"
             ]
+
           },
           {
             name: "Accesscontrol Module",
@@ -121,6 +107,9 @@ jobs:
           done
           sleep 10
 
+      - name: Start rpc node
+        run: make run-rpc-node-skipbuild &
+
       - name: Verify Sei Chain is running
         run: python3 integration_test/scripts/runner.py integration_test/startup/startup_test.yaml
 

Makefile

@@ -140,7 +140,7 @@ run-local-node: kill-sei-node build-docker-node
 .PHONY: run-local-node
 
 # Run a single rpc state sync node docker container
-run-rpc-node: kill-rpc-node build-rpc-node
+run-rpc-node: build-rpc-node
 	docker run --rm \
 	--name sei-rpc-node \
 	--network docker_localnet \
@@ -153,6 +153,20 @@ run-rpc-node: kill-rpc-node build-rpc-node
 	sei-chain/rpcnode
 .PHONY: run-rpc-node
 
+run-rpc-node-skipbuild: build-rpc-node
+	docker run --rm \
+	--name sei-rpc-node \
+	--network docker_localnet \
+	-v $(PROJECT_HOME):/sei-protocol/sei-chain:Z \
+	-v $(PROJECT_HOME)/../sei-tendermint:/sei-protocol/sei-tendermint:Z \
+    -v $(PROJECT_HOME)/../sei-cosmos:/sei-protocol/sei-cosmos:Z \
+	-v $(GO_PKG_PATH)/mod:/root/go/pkg/mod:Z \
+	-p 26668-26670:26656-26658 \
+	--platform linux/x86_64 \
+	--env SKIP_BUILD=true \
+	sei-chain/rpcnode
+.PHONY: run-rpc-node
+
 kill-sei-node:
 	docker ps --filter name=sei-node --filter status=running -aq | xargs docker kill
 

cmd/seid/cmd/root.go

@@ -243,12 +243,16 @@ func newApp(
 		panic(err)
 	}
 
-	snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots")
-	snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir)
+	snapshotDirectory := cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))
+	if snapshotDirectory == "" {
+		snapshotDirectory = filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots")
+	}
+
+	snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDirectory)
 	if err != nil {
 		panic(err)
 	}
-	snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir)
+	snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDirectory)
 	if err != nil {
 		panic(err)
 	}
@@ -290,6 +294,7 @@ func newApp(
 		baseapp.SetSnapshotStore(snapshotStore),
 		baseapp.SetSnapshotInterval(cast.ToUint64(appOpts.Get(server.FlagStateSyncSnapshotInterval))),
 		baseapp.SetSnapshotKeepRecent(cast.ToUint32(appOpts.Get(server.FlagStateSyncSnapshotKeepRecent))),
+		baseapp.SetSnapshotDirectory(cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))),
 		baseapp.SetOrphanConfig(&iavl.Options{
 			SeparateOrphanStorage:       cast.ToBool(appOpts.Get(server.FlagSeparateOrphanStorage)),
 			SeparateOphanVersionsToKeep: cast.ToInt64(appOpts.Get(server.FlagSeparateOrphanVersionsToKeep)),

docker/localnode/Dockerfile

@@ -21,7 +21,7 @@ COPY scripts/step0_build.sh /usr/bin/build.sh
 COPY scripts/step1_configure_init.sh /usr/bin/configure_init.sh
 COPY scripts/step2_genesis.sh /usr/bin/genesis.sh
 COPY scripts/step3_add_validator_to_genesis.sh /usr/bin/add_validator_to_gensis.sh
-COPY scripts/step4_persistent_peers.sh /usr/bin/persistent_peers.sh
+COPY scripts/step4_config_override.sh /usr/bin/config_override.sh
 COPY scripts/step5_start_sei.sh /usr/bin/start_sei.sh
 COPY scripts/step6_start_price_feeder.sh /usr/bin/start_price_feeder.sh
 ENV PATH "$PATH:$HOME/go/bin"

docker/localnode/config/app.toml

@@ -190,10 +190,13 @@ enable-unsafe-cors = true
 
 # snapshot-interval specifies the block interval at which local state sync snapshots are
 # taken (0 to disable). Must be a multiple of pruning-keep-every.
-snapshot-interval = 500
+snapshot-interval = 100
 
 # snapshot-keep-recent specifies the number of recent snapshots to keep and serve (0 to keep all).
-snapshot-keep-recent = 2
+snapshot-keep-recent = 3
+
+# snapshot-directory specifies and overrides the directory for where to store the snapshots
+snapshot-directory = ""
 
 [wasm]
 # This is the maximum sdk gas (wasm and storage) that we allow for any x/wasm "smart" queries

docker/localnode/scripts/deploy.sh

@@ -52,8 +52,8 @@ do
      sleep 1
 done
 
-# Step 4: Configure persistent peers
-/usr/bin/persistent_peers.sh
+# Step 4: Config overrides
+/usr/bin/config_override.sh
 
 # Step 5: Start the chain
 /usr/bin/start_sei.sh

docker/localnode/scripts/step1_configure_init.sh

@@ -24,10 +24,13 @@ seid init "$MONIKER" --chain-id sei >/dev/null 2>&1
 
 # Copy configs
 ORACLE_CONFIG_FILE="build/generated/node_$NODE_ID/price_feeder_config.toml"
-cp docker/localnode/config/app.toml ~/.sei/config/app.toml
-cp docker/localnode/config/config.toml ~/.sei/config/config.toml
+APP_CONFIG_FILE="build/generated/node_$NODE_ID/app.toml"
+TENDERMINT_CONFIG_FILE="build/generated/node_$NODE_ID/config.toml"
+cp docker/localnode/config/app.toml "$APP_CONFIG_FILE"
+cp docker/localnode/config/config.toml "$TENDERMINT_CONFIG_FILE"
 cp docker/localnode/config/price_feeder_config.toml "$ORACLE_CONFIG_FILE"
 
+
 # Set up persistent peers
 SEI_NODE_ID=$(seid tendermint show-node-id)
 NODE_IP=$(hostname -i | awk '{print $1}')

docker/localnode/scripts/step4_config_override.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+
+NODE_ID=${ID:-0}
+
+APP_CONFIG_FILE="build/generated/node_$NODE_ID/app.toml"
+TENDERMINT_CONFIG_FILE="build/generated/node_$NODE_ID/config.toml"
+cp build/generated/genesis.json ~/.sei/config/genesis.json
+cp "$APP_CONFIG_FILE" ~/.sei/config/app.toml
+cp "$TENDERMINT_CONFIG_FILE" ~/.sei/config/config.toml
+
+# Override up persistent peers
+NODE_IP=$(hostname -i | awk '{print $1}')
+PEERS=$(cat build/generated/persistent_peers.txt |grep -v "$NODE_IP" | paste -sd "," -)
+sed -i'' -e 's/persistent-peers = ""/persistent-peers = "'$PEERS'"/g' ~/.sei/config/config.toml
+
+# Override snapshot directory
+sed -i.bak -e "s|^snapshot-directory *=.*|snapshot-directory = \"./build/generated/node_$NODE_ID/snapshots\"|" ~/.sei/config/app.toml

docker/localnode/scripts/step5_start_sei.sh

@@ -6,9 +6,8 @@ INVARIANT_CHECK_INTERVAL=${INVARIANT_CHECK_INTERVAL:-0}
 LOG_DIR="build/generated/logs"
 mkdir -p $LOG_DIR
 
-# Starting sei chain
 echo "Starting the seid process for node $NODE_ID with invariant check interval=$INVARIANT_CHECK_INTERVAL..."
-cp build/generated/genesis.json ~/.sei/config/genesis.json
+
 seid start --chain-id sei --inv-check-period ${INVARIANT_CHECK_INTERVAL} > "$LOG_DIR/seid-$NODE_ID.log" 2>&1 &
 echo "Node $NODE_ID seid is started now"
 echo "Done" >> build/generated/launch.complete

docker/rpcnode/Dockerfile

@@ -2,6 +2,8 @@ FROM ubuntu:latest
 RUN apt-get update && \
     apt-get install -y make git golang jq python3 curl vim
 
+SHELL ["/bin/bash", "-c"]
+
 VOLUME [ "/sei-protocol" ]
 VOLUME [ "/root/go/pkg/mod" ]
 WORKDIR /sei-protocol/sei-chain
@@ -16,3 +18,4 @@ COPY scripts/deploy.sh /usr/bin/deploy.sh
 COPY scripts/step0_build.sh /usr/bin/build.sh
 COPY scripts/step1_configure_init.sh /usr/bin/configure_init.sh
 COPY scripts/step2_start_sei.sh /usr/bin/start_sei.sh
+ENV PATH "$PATH:$HOME/go/bin"

docker/rpcnode/scripts/deploy.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env sh
 
-NODE_ID=${ID:-0}
+SKIP_BUILD=${SKIP_BUILD:-""}
+
 # Set up env
 export GOPATH=$HOME/go
 export GOBIN=$GOPATH/bin
@@ -11,8 +12,13 @@ echo "GOBIN=$GOPATH/bin" >> /root/.bashrc
 echo "export PATH=$GOBIN:$PATH:/usr/local/go/bin:$BUILD_PATH" >> /root/.bashrc
 /bin/bash -c "source /root/.bashrc"
 mkdir -p $GOBIN
+
 # Step 1 build seid
-/usr/bin/build.sh
+if [ -z "$SKIP_BUILD" ]
+then
+  /usr/bin/build.sh
+fi
+cp build/seid "$GOBIN"/
 
 # Run init to set up state sync configurations
 /usr/bin/configure_init.sh

docker/rpcnode/scripts/step2_start_sei.sh

@@ -4,6 +4,7 @@ LOG_DIR="build/generated/logs"
 mkdir -p $LOG_DIR
 
 # Starting sei chain
-echo "Starting the sei chain rpc node"
+echo "RPC Node is starting now, check logs under $LOG_DIR"
+
 seid start --chain-id sei > "$LOG_DIR/rpc-node.log" 2>&1 &
-echo "RPC Node is started now, check logs under $LOG_DIR"
+echo "Done" >> build/generated/rpc-launch.complete

go.mod

@@ -273,7 +273,7 @@ require (
 replace (
 	github.com/CosmWasm/wasmd => github.com/sei-protocol/sei-wasmd v0.0.2
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
-	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.2.53
+	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.2.56
 	github.com/cosmos/iavl => github.com/sei-protocol/sei-iavl v0.1.7
 	github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.1.0
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1

go.sum

@@ -1072,10 +1072,10 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod
 github.com/securego/gosec/v2 v2.11.0 h1:+PDkpzR41OI2jrw1q6AdXZCbsNGNGT7pQjal0H0cArI=
 github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+fscA+Pulbpo=
 github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
+github.com/sei-protocol/sei-cosmos v0.2.56 h1:ypZVriOpRAxcqZrDJCLlmXCuYLIzY/iYAY+PWftKwKI=
+github.com/sei-protocol/sei-cosmos v0.2.56/go.mod h1:XSmrSNlBQ7OQrk6VZMgARWsnD+1JSnQeDEZ7LEmexbo=
 github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQpIbXDA=
 github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
-github.com/sei-protocol/sei-cosmos v0.2.53 h1:MSH00lb3ahioaOnJNC4yvMNNPuC9rC0kBZcY30ay5QI=
-github.com/sei-protocol/sei-cosmos v0.2.53/go.mod h1:XSmrSNlBQ7OQrk6VZMgARWsnD+1JSnQeDEZ7LEmexbo=
 github.com/sei-protocol/sei-iavl v0.1.7 h1:cUdHDBkxs0FF/kOt1qCVLm0K+Bqaw92/dbZSgn4kxiA=
 github.com/sei-protocol/sei-iavl v0.1.7/go.mod h1:7PfkEVT5dcoQE+s/9KWdoXJ8VVVP1QpYYPLdxlkSXFk=
 github.com/sei-protocol/sei-ibc-go/v3 v3.1.0 h1:rFHk2R/3vbG9iNr7cYtVclW/UyEDSRFjNVPz60zZswU=

integration_test/chain_operation/snapshot_operation.yaml

@@ -0,0 +1,9 @@
+- name: Test validators should be able to create snapshot with custom location
+  inputs:
+    # Check if snapshotd are created
+    - cmd: if [ -d "./build/generated/node_0/snapshots" ]; then echo "true"; else echo "false"; fi
+      env: FOUND
+      node: sei-node-0
+  verifiers:
+  - type: eval
+    expr: FOUND == "true"

integration_test/chain_operation/statesync_operation.yaml

@@ -0,0 +1,9 @@
+- name: Test rpc node should be able to state sync from the snapshots
+  inputs:
+    # Check if rpc node is up and running
+    - cmd: seid status |jq -M -r .SyncInfo.latest_block_height
+      env: HEIGHT
+      node: sei-rpc-node
+  verifiers:
+  - type: eval
+    expr: HEIGHT > 0