From 6aea6348ea28d71405756be1a28ac2cb68e6743a Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Thu, 29 Jun 2023 07:58:22 -0700 Subject: [PATCH 01/13] Add a new config to customize the snapshot directory --- .github/workflows/integration-test.yml | 45 ++++++++------------------ cmd/seid/cmd/root.go | 11 +++++-- docker/localnode/config/app.toml | 6 ++-- go.mod | 2 +- go.sum | 4 +-- 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 529bd35cc..1be68e566 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -17,26 +17,21 @@ jobs: integration-tests: name: Integration Test (${{ matrix.test.name }}) runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} strategy: # other jobs should run even if one integration test fails - fail-fast: false + fail-fast: true matrix: test: [ { - name: "Dex Module", + name: "Dex & Wasm Module", scripts: [ "docker exec sei-node-0 integration_test/contracts/deploy_dex_contract.sh mars", "python3 integration_test/scripts/runner.py integration_test/dex_module/place_order_test.yaml", - "python3 integration_test/scripts/runner.py integration_test/dex_module/cancel_order_test.yaml" - ] - }, - { - name: "Wasm Module", - scripts: [ + "python3 integration_test/scripts/runner.py integration_test/dex_module/cancel_order_test.yaml", "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh", "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_delegation_test.yaml", "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_admin_test.yaml", @@ -46,45 +41,31 @@ jobs: ] }, { - name: "Mint Module", + name: "Mint & Staking & Bank Module", scripts: [ + "python3 integration_test/scripts/runner.py integration_test/staking_module/staking_test.yaml", + "python3 integration_test/scripts/runner.py integration_test/bank_module/send_funds_test.yaml", "python3 integration_test/scripts/runner.py integration_test/mint_module/mint_test.yaml" ] }, { - name: "Staking Module", - scripts: [ - "python3 integration_test/scripts/runner.py integration_test/staking_module/staking_test.yaml" - ] - }, - { - name: "Gov Module", + name: "Gov & Oracle & Authz Module", scripts: [ "python3 integration_test/scripts/runner.py integration_test/gov_module/gov_proposal_test.yaml", - "python3 integration_test/scripts/runner.py integration_test/gov_module/staking_proposal_test.yaml" - ] - }, - { - name: "Oracle Module", - scripts: [ + "python3 integration_test/scripts/runner.py integration_test/gov_module/staking_proposal_test.yaml", "python3 integration_test/scripts/runner.py integration_test/oracle_module/verify_penalty_counts.yaml", - "python3 integration_test/scripts/runner.py integration_test/oracle_module/set_feeder_test.yaml" - ] - }, - { - name: "Authz Module", - scripts: [ + "python3 integration_test/scripts/runner.py integration_test/oracle_module/set_feeder_test.yaml", "python3 integration_test/scripts/runner.py integration_test/authz_module/send_authorization_test.yaml", "python3 integration_test/scripts/runner.py integration_test/authz_module/staking_authorization_test.yaml", "python3 integration_test/scripts/runner.py integration_test/authz_module/generic_authorization_test.yaml" ] }, { - name: "Bank Module", + name: "Chain Operation Tests", scripts: [ - "python3 integration_test/scripts/runner.py integration_test/bank_module/send_funds_test.yaml", + ] - }, + } ] steps: - uses: actions/checkout@v3 diff --git a/cmd/seid/cmd/root.go b/cmd/seid/cmd/root.go index 65f00f18d..76eeda337 100644 --- a/cmd/seid/cmd/root.go +++ b/cmd/seid/cmd/root.go @@ -243,12 +243,16 @@ func newApp( panic(err) } - snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots") - snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir) + snapshotDirectory := cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir)) + if snapshotDirectory == "" { + snapshotDirectory = filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots") + } + + snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDirectory) if err != nil { panic(err) } - snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir) + snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDirectory) if err != nil { panic(err) } @@ -290,6 +294,7 @@ func newApp( baseapp.SetSnapshotStore(snapshotStore), baseapp.SetSnapshotInterval(cast.ToUint64(appOpts.Get(server.FlagStateSyncSnapshotInterval))), baseapp.SetSnapshotKeepRecent(cast.ToUint32(appOpts.Get(server.FlagStateSyncSnapshotKeepRecent))), + baseapp.SetSnapshotDirectory(cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))), baseapp.SetOrphanConfig(&iavl.Options{ SeparateOrphanStorage: cast.ToBool(appOpts.Get(server.FlagSeparateOrphanStorage)), SeparateOphanVersionsToKeep: cast.ToInt64(appOpts.Get(server.FlagSeparateOrphanVersionsToKeep)), diff --git a/docker/localnode/config/app.toml b/docker/localnode/config/app.toml index a137e81a4..41e65b756 100644 --- a/docker/localnode/config/app.toml +++ b/docker/localnode/config/app.toml @@ -190,10 +190,12 @@ enable-unsafe-cors = true # snapshot-interval specifies the block interval at which local state sync snapshots are # taken (0 to disable). Must be a multiple of pruning-keep-every. -snapshot-interval = 500 +snapshot-interval = 100 # snapshot-keep-recent specifies the number of recent snapshots to keep and serve (0 to keep all). -snapshot-keep-recent = 2 +snapshot-keep-recent = 3 + +snapshot-directory = "./build/generated/snapshots" [wasm] # This is the maximum sdk gas (wasm and storage) that we allow for any x/wasm "smart" queries diff --git a/go.mod b/go.mod index b1fcc126e..943e89a03 100644 --- a/go.mod +++ b/go.mod @@ -272,7 +272,7 @@ require ( replace ( github.com/CosmWasm/wasmd => github.com/sei-protocol/sei-wasmd v0.0.2 github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 - github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.2.53 + github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.2.56 github.com/cosmos/iavl => github.com/sei-protocol/sei-iavl v0.1.7 github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.1.0 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 diff --git a/go.sum b/go.sum index 2c29205fc..ef158dced 100644 --- a/go.sum +++ b/go.sum @@ -1072,8 +1072,8 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod github.com/securego/gosec/v2 v2.11.0 h1:+PDkpzR41OI2jrw1q6AdXZCbsNGNGT7pQjal0H0cArI= github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+fscA+Pulbpo= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= -github.com/sei-protocol/sei-cosmos v0.2.53 h1:MSH00lb3ahioaOnJNC4yvMNNPuC9rC0kBZcY30ay5QI= -github.com/sei-protocol/sei-cosmos v0.2.53/go.mod h1:XSmrSNlBQ7OQrk6VZMgARWsnD+1JSnQeDEZ7LEmexbo= +github.com/sei-protocol/sei-cosmos v0.2.56 h1:ypZVriOpRAxcqZrDJCLlmXCuYLIzY/iYAY+PWftKwKI= +github.com/sei-protocol/sei-cosmos v0.2.56/go.mod h1:XSmrSNlBQ7OQrk6VZMgARWsnD+1JSnQeDEZ7LEmexbo= github.com/sei-protocol/sei-iavl v0.1.7 h1:cUdHDBkxs0FF/kOt1qCVLm0K+Bqaw92/dbZSgn4kxiA= github.com/sei-protocol/sei-iavl v0.1.7/go.mod h1:7PfkEVT5dcoQE+s/9KWdoXJ8VVVP1QpYYPLdxlkSXFk= github.com/sei-protocol/sei-ibc-go/v3 v3.1.0 h1:rFHk2R/3vbG9iNr7cYtVclW/UyEDSRFjNVPz60zZswU= From a8b6a8fbb1ac3ae65d3a710312956b9655e0e28f Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Thu, 29 Jun 2023 16:57:56 -0700 Subject: [PATCH 02/13] Add integration test for snapshotting and state sync --- .github/workflows/integration-test.yml | 6 +++++- docker/localnode/Dockerfile | 2 +- docker/localnode/config/app.toml | 3 ++- docker/localnode/scripts/deploy.sh | 4 ++-- .../localnode/scripts/step1_configure_init.sh | 7 +++++-- .../localnode/scripts/step4_config_override.sh | 17 +++++++++++++++++ .../scripts/step4_persistent_peers.sh | 9 --------- docker/localnode/scripts/step5_start_sei.sh | 3 +-- docker/rpcnode/scripts/step2_start_sei.sh | 5 +++-- .../chain_operation/snapshot_operation.yaml | 11 +++++++++++ .../chain_operation/statesync_operation.yaml | 9 +++++++++ snapshots/metadata.db/000001.log | 0 snapshots/metadata.db/CURRENT | 1 + snapshots/metadata.db/LOCK | 0 snapshots/metadata.db/LOG | 6 ++++++ snapshots/metadata.db/MANIFEST-000000 | Bin 0 -> 54 bytes 16 files changed, 63 insertions(+), 20 deletions(-) create mode 100755 docker/localnode/scripts/step4_config_override.sh delete mode 100755 docker/localnode/scripts/step4_persistent_peers.sh create mode 100644 integration_test/chain_operation/snapshot_operation.yaml create mode 100644 integration_test/chain_operation/statesync_operation.yaml create mode 100644 snapshots/metadata.db/000001.log create mode 100644 snapshots/metadata.db/CURRENT create mode 100644 snapshots/metadata.db/LOCK create mode 100644 snapshots/metadata.db/LOG create mode 100644 snapshots/metadata.db/MANIFEST-000000 diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1be68e566..d80b8ae03 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -61,8 +61,12 @@ jobs: ] }, { - name: "Chain Operation Tests", + name: "Chain Operation Test", scripts: [ + "make run-rpc-node &", + "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done" + "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", + "python3 integration_test/scripts/runner.py integration_test/gov_module/statesync_operation.yaml" ] } diff --git a/docker/localnode/Dockerfile b/docker/localnode/Dockerfile index c35df8fc1..1d0747903 100644 --- a/docker/localnode/Dockerfile +++ b/docker/localnode/Dockerfile @@ -21,7 +21,7 @@ COPY scripts/step0_build.sh /usr/bin/build.sh COPY scripts/step1_configure_init.sh /usr/bin/configure_init.sh COPY scripts/step2_genesis.sh /usr/bin/genesis.sh COPY scripts/step3_add_validator_to_genesis.sh /usr/bin/add_validator_to_gensis.sh -COPY scripts/step4_persistent_peers.sh /usr/bin/persistent_peers.sh +COPY scripts/step4_config_override.sh /usr/bin/config_override.sh COPY scripts/step5_start_sei.sh /usr/bin/start_sei.sh COPY scripts/step6_start_price_feeder.sh /usr/bin/start_price_feeder.sh ENV PATH "$PATH:$HOME/go/bin" diff --git a/docker/localnode/config/app.toml b/docker/localnode/config/app.toml index 41e65b756..d41087026 100644 --- a/docker/localnode/config/app.toml +++ b/docker/localnode/config/app.toml @@ -195,7 +195,8 @@ snapshot-interval = 100 # snapshot-keep-recent specifies the number of recent snapshots to keep and serve (0 to keep all). snapshot-keep-recent = 3 -snapshot-directory = "./build/generated/snapshots" +# snapshot-directory specifies and overrides the directory for where to store the snapshots +snapshot-directory = "" [wasm] # This is the maximum sdk gas (wasm and storage) that we allow for any x/wasm "smart" queries diff --git a/docker/localnode/scripts/deploy.sh b/docker/localnode/scripts/deploy.sh index 17abd5682..4b971a39a 100755 --- a/docker/localnode/scripts/deploy.sh +++ b/docker/localnode/scripts/deploy.sh @@ -52,8 +52,8 @@ do sleep 1 done -# Step 4: Configure persistent peers -/usr/bin/persistent_peers.sh +# Step 4: Config overrides +/usr/bin/config_override.sh # Step 5: Start the chain /usr/bin/start_sei.sh diff --git a/docker/localnode/scripts/step1_configure_init.sh b/docker/localnode/scripts/step1_configure_init.sh index 3c799bf3e..b006d3060 100755 --- a/docker/localnode/scripts/step1_configure_init.sh +++ b/docker/localnode/scripts/step1_configure_init.sh @@ -24,10 +24,13 @@ seid init "$MONIKER" --chain-id sei >/dev/null 2>&1 # Copy configs ORACLE_CONFIG_FILE="build/generated/node_$NODE_ID/price_feeder_config.toml" -cp docker/localnode/config/app.toml ~/.sei/config/app.toml -cp docker/localnode/config/config.toml ~/.sei/config/config.toml +APP_CONFIG_FILE="build/generated/node_$NODE_ID/app.toml" +TENDERMINT_CONFIG_FILE="build/generated/node_$NODE_ID/config.toml" +cp docker/localnode/config/app.toml "$APP_CONFIG_FILE" +cp docker/localnode/config/config.toml "$TENDERMINT_CONFIG_FILE" cp docker/localnode/config/price_feeder_config.toml "$ORACLE_CONFIG_FILE" + # Set up persistent peers SEI_NODE_ID=$(seid tendermint show-node-id) NODE_IP=$(hostname -i | awk '{print $1}') diff --git a/docker/localnode/scripts/step4_config_override.sh b/docker/localnode/scripts/step4_config_override.sh new file mode 100755 index 000000000..e0ec4b422 --- /dev/null +++ b/docker/localnode/scripts/step4_config_override.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +NODE_ID=${ID:-0} + +APP_CONFIG_FILE="build/generated/node_$NODE_ID/app.toml" +TENDERMINT_CONFIG_FILE="build/generated/node_$NODE_ID/config.toml" +cp build/generated/genesis.json ~/.sei/config/genesis.json +cp "$APP_CONFIG_FILE" ~/.sei/config/app.toml +cp "$TENDERMINT_CONFIG_FILE" ~/.sei/config/config.toml + +# Override up persistent peers +NODE_IP=$(hostname -i | awk '{print $1}') +PEERS=$(cat build/generated/persistent_peers.txt |grep -v "$NODE_IP" | paste -sd "," -) +sed -i'' -e 's/persistent-peers = ""/persistent-peers = "'$PEERS'"/g' ~/.sei/config/config.toml + +# Override snapshot directory +sed -i.bak -e "s|^snapshot-directory *=.*|snapshot-directory = \"./build/generated/node_$NODE_ID/snapshots\"|" ~/.sei/config/app.toml diff --git a/docker/localnode/scripts/step4_persistent_peers.sh b/docker/localnode/scripts/step4_persistent_peers.sh deleted file mode 100755 index c43ccc617..000000000 --- a/docker/localnode/scripts/step4_persistent_peers.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh - -# Greeting output -echo "Loading persistent peers" - -# Set up persistent peers -NODE_IP=$(hostname -i | awk '{print $1}') -PEERS=$(cat build/generated/persistent_peers.txt |grep -v $NODE_IP | paste -sd "," -) -sed -i'' -e 's/persistent-peers = ""/persistent-peers = "'$PEERS'"/g' ~/.sei/config/config.toml \ No newline at end of file diff --git a/docker/localnode/scripts/step5_start_sei.sh b/docker/localnode/scripts/step5_start_sei.sh index 0be95475d..bb02f0835 100755 --- a/docker/localnode/scripts/step5_start_sei.sh +++ b/docker/localnode/scripts/step5_start_sei.sh @@ -6,9 +6,8 @@ INVARIANT_CHECK_INTERVAL=${INVARIANT_CHECK_INTERVAL:-0} LOG_DIR="build/generated/logs" mkdir -p $LOG_DIR -# Starting sei chain echo "Starting the seid process for node $NODE_ID with invariant check interval=$INVARIANT_CHECK_INTERVAL..." -cp build/generated/genesis.json ~/.sei/config/genesis.json + seid start --chain-id sei --inv-check-period ${INVARIANT_CHECK_INTERVAL} > "$LOG_DIR/seid-$NODE_ID.log" 2>&1 & echo "Node $NODE_ID seid is started now" echo "Done" >> build/generated/launch.complete diff --git a/docker/rpcnode/scripts/step2_start_sei.sh b/docker/rpcnode/scripts/step2_start_sei.sh index 26401e808..b67c45571 100755 --- a/docker/rpcnode/scripts/step2_start_sei.sh +++ b/docker/rpcnode/scripts/step2_start_sei.sh @@ -4,6 +4,7 @@ LOG_DIR="build/generated/logs" mkdir -p $LOG_DIR # Starting sei chain -echo "Starting the sei chain rpc node" +echo "RPC Node is starting now, check logs under $LOG_DIR" + seid start --chain-id sei > "$LOG_DIR/rpc-node.log" 2>&1 & -echo "RPC Node is started now, check logs under $LOG_DIR" \ No newline at end of file +echo "Done" >> build/generated/rpc-launch.complete \ No newline at end of file diff --git a/integration_test/chain_operation/snapshot_operation.yaml b/integration_test/chain_operation/snapshot_operation.yaml new file mode 100644 index 000000000..9a4c6cab9 --- /dev/null +++ b/integration_test/chain_operation/snapshot_operation.yaml @@ -0,0 +1,11 @@ +- name: Test validators should be able to create snapshot with custom location + inputs: + # Wait for sei node 0 to hit height 120 + - cmd: until [ $(seid status |jq -M -r .SyncInfo.latest_block_height) -gt 120 ]; do sleep 10; done + node: sei-node-0 + - cmd: if [ -d "./build/generated/node_0/snapshots" ]; then echo "true"; else echo "false"; fi + env: FOUND + node: sei-node-0 + verifiers: + - type: eval + expr: FOUND == "true" diff --git a/integration_test/chain_operation/statesync_operation.yaml b/integration_test/chain_operation/statesync_operation.yaml new file mode 100644 index 000000000..133f4911a --- /dev/null +++ b/integration_test/chain_operation/statesync_operation.yaml @@ -0,0 +1,9 @@ +- name: Test rpc node should be able to state sync from the snapshots + inputs: + # Check if rpc node is up and running + - cmd: seid status |jq -M -r .SyncInfo.latest_block_height + env: HEIGHT + node: sei-rpc-node + verifiers: + - type: eval + expr: HEIGHT > 0 diff --git a/snapshots/metadata.db/000001.log b/snapshots/metadata.db/000001.log new file mode 100644 index 000000000..e69de29bb diff --git a/snapshots/metadata.db/CURRENT b/snapshots/metadata.db/CURRENT new file mode 100644 index 000000000..feda7d6b2 --- /dev/null +++ b/snapshots/metadata.db/CURRENT @@ -0,0 +1 @@ +MANIFEST-000000 diff --git a/snapshots/metadata.db/LOCK b/snapshots/metadata.db/LOCK new file mode 100644 index 000000000..e69de29bb diff --git a/snapshots/metadata.db/LOG b/snapshots/metadata.db/LOG new file mode 100644 index 000000000..888016b86 --- /dev/null +++ b/snapshots/metadata.db/LOG @@ -0,0 +1,6 @@ +=============== Jun 29, 2023 (UTC) =============== +21:50:28.323026 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +21:50:28.328504 db@open opening +21:50:28.330399 version@stat F·[] S·0B[] Sc·[] +21:50:28.332252 db@janitor F·2 G·0 +21:50:28.332694 db@open done T·3.952446ms diff --git a/snapshots/metadata.db/MANIFEST-000000 b/snapshots/metadata.db/MANIFEST-000000 new file mode 100644 index 0000000000000000000000000000000000000000..9d54f6733b1364dc8d53dd15ca59a6ec36a1c29d GIT binary patch literal 54 zcmdmC5aOo9z{n_-lUkOVlai$8R9TW*o>`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD literal 0 HcmV?d00001 From da68ed045e37d935f9c0c9d4ef746d3804c122f9 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Thu, 29 Jun 2023 17:02:07 -0700 Subject: [PATCH 03/13] Fix --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index d80b8ae03..de562cf41 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -64,7 +64,7 @@ jobs: name: "Chain Operation Test", scripts: [ "make run-rpc-node &", - "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done" + "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", "python3 integration_test/scripts/runner.py integration_test/gov_module/statesync_operation.yaml" From 70dd6b4fdfb2ec64a4612c48ea0e0fa419cd8489 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Thu, 29 Jun 2023 17:03:24 -0700 Subject: [PATCH 04/13] Remove unused files --- snapshots/metadata.db/000001.log | 0 snapshots/metadata.db/CURRENT | 1 - snapshots/metadata.db/LOCK | 0 snapshots/metadata.db/LOG | 6 ------ snapshots/metadata.db/MANIFEST-000000 | Bin 54 -> 0 bytes 5 files changed, 7 deletions(-) delete mode 100644 snapshots/metadata.db/000001.log delete mode 100644 snapshots/metadata.db/CURRENT delete mode 100644 snapshots/metadata.db/LOCK delete mode 100644 snapshots/metadata.db/LOG delete mode 100644 snapshots/metadata.db/MANIFEST-000000 diff --git a/snapshots/metadata.db/000001.log b/snapshots/metadata.db/000001.log deleted file mode 100644 index e69de29bb..000000000 diff --git a/snapshots/metadata.db/CURRENT b/snapshots/metadata.db/CURRENT deleted file mode 100644 index feda7d6b2..000000000 --- a/snapshots/metadata.db/CURRENT +++ /dev/null @@ -1 +0,0 @@ -MANIFEST-000000 diff --git a/snapshots/metadata.db/LOCK b/snapshots/metadata.db/LOCK deleted file mode 100644 index e69de29bb..000000000 diff --git a/snapshots/metadata.db/LOG b/snapshots/metadata.db/LOG deleted file mode 100644 index 888016b86..000000000 --- a/snapshots/metadata.db/LOG +++ /dev/null @@ -1,6 +0,0 @@ -=============== Jun 29, 2023 (UTC) =============== -21:50:28.323026 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed -21:50:28.328504 db@open opening -21:50:28.330399 version@stat F·[] S·0B[] Sc·[] -21:50:28.332252 db@janitor F·2 G·0 -21:50:28.332694 db@open done T·3.952446ms diff --git a/snapshots/metadata.db/MANIFEST-000000 b/snapshots/metadata.db/MANIFEST-000000 deleted file mode 100644 index 9d54f6733b1364dc8d53dd15ca59a6ec36a1c29d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 54 zcmdmC5aOo9z{n_-lUkOVlai$8R9TW*o>`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD From 76fec00b09b3955692e819ce2d0e0a09b6d40782 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Thu, 29 Jun 2023 17:47:13 -0700 Subject: [PATCH 05/13] Add skipbuild support for rpc node --- .github/workflows/integration-test.yml | 2 +- Makefile | 14 ++++++++++++++ docker/rpcnode/scripts/deploy.sh | 9 +++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index de562cf41..0abf66452 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -63,7 +63,7 @@ jobs: { name: "Chain Operation Test", scripts: [ - "make run-rpc-node &", + "make run-rpc-node-skipbuild &", "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", "python3 integration_test/scripts/runner.py integration_test/gov_module/statesync_operation.yaml" diff --git a/Makefile b/Makefile index b0192ec81..d04b33cba 100644 --- a/Makefile +++ b/Makefile @@ -153,6 +153,20 @@ run-rpc-node: kill-rpc-node build-rpc-node sei-chain/rpcnode .PHONY: run-rpc-node +run-rpc-node-skipbuild: kill-rpc-node build-rpc-node + docker run --rm \ + --name sei-rpc-node \ + --network docker_localnet \ + -v $(PROJECT_HOME):/sei-protocol/sei-chain:Z \ + -v $(PROJECT_HOME)/../sei-tendermint:/sei-protocol/sei-tendermint:Z \ + -v $(PROJECT_HOME)/../sei-cosmos:/sei-protocol/sei-cosmos:Z \ + -v $(GO_PKG_PATH)/mod:/root/go/pkg/mod:Z \ + -p 26668-26670:26656-26658 \ + --platform linux/x86_64 \ + --env SKIP_BUILD=true \ + sei-chain/rpcnode +.PHONY: run-rpc-node + kill-sei-node: docker ps --filter name=sei-node --filter status=running -aq | xargs docker kill diff --git a/docker/rpcnode/scripts/deploy.sh b/docker/rpcnode/scripts/deploy.sh index 2b9d28e77..f0cb14a8c 100755 --- a/docker/rpcnode/scripts/deploy.sh +++ b/docker/rpcnode/scripts/deploy.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh -NODE_ID=${ID:-0} +SKIP_BUILD=${SKIP_BUILD:-""} + # Set up env export GOPATH=$HOME/go export GOBIN=$GOPATH/bin @@ -11,8 +12,12 @@ echo "GOBIN=$GOPATH/bin" >> /root/.bashrc echo "export PATH=$GOBIN:$PATH:/usr/local/go/bin:$BUILD_PATH" >> /root/.bashrc /bin/bash -c "source /root/.bashrc" mkdir -p $GOBIN + # Step 1 build seid -/usr/bin/build.sh +if [ -z "$SKIP_BUILD" ] +then + /usr/bin/build.sh +fi # Run init to set up state sync configurations /usr/bin/configure_init.sh From edb7cfc85048c3284dc9df86852f71c1074016be Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Thu, 29 Jun 2023 18:14:09 -0700 Subject: [PATCH 06/13] Use separate task for rpc node --- .github/workflows/integration-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 0abf66452..0aa27fad9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -63,7 +63,6 @@ jobs: { name: "Chain Operation Test", scripts: [ - "make run-rpc-node-skipbuild &", "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", "python3 integration_test/scripts/runner.py integration_test/gov_module/statesync_operation.yaml" @@ -100,6 +99,9 @@ jobs: done sleep 10 + - name: Start rpc node + run: make run-rpc-node-skipbuild & + - name: Verify Sei Chain is running run: python3 integration_test/scripts/runner.py integration_test/startup/startup_test.yaml From 0561c1915f5091faefb54ee7890dd7902702a670 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Fri, 30 Jun 2023 02:55:23 -0700 Subject: [PATCH 07/13] Fix docker command --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d04b33cba..284e3b45f 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,7 @@ run-local-node: kill-sei-node build-docker-node .PHONY: run-local-node # Run a single rpc state sync node docker container -run-rpc-node: kill-rpc-node build-rpc-node +run-rpc-node: build-rpc-node docker run --rm \ --name sei-rpc-node \ --network docker_localnet \ @@ -153,7 +153,7 @@ run-rpc-node: kill-rpc-node build-rpc-node sei-chain/rpcnode .PHONY: run-rpc-node -run-rpc-node-skipbuild: kill-rpc-node build-rpc-node +run-rpc-node-skipbuild: build-rpc-node docker run --rm \ --name sei-rpc-node \ --network docker_localnet \ From bbb680a82646363247c00612f9435257c9dedd13 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Fri, 30 Jun 2023 03:07:47 -0700 Subject: [PATCH 08/13] Fix test --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 0aa27fad9..ac931bba3 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -65,7 +65,7 @@ jobs: scripts: [ "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", - "python3 integration_test/scripts/runner.py integration_test/gov_module/statesync_operation.yaml" + "python3 integration_test/scripts/runner.py integration_test/chain_operation/statesync_operation.yaml" ] } From 71e64d3d845c0c0d7527d8d8baf8afa0cc091b41 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Fri, 30 Jun 2023 03:26:27 -0700 Subject: [PATCH 09/13] Add env --- .github/workflows/integration-test.yml | 1 + docker/rpcnode/Dockerfile | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ac931bba3..7e1786ca1 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -64,6 +64,7 @@ jobs: name: "Chain Operation Test", scripts: [ "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", + "until [ $(docker exec sei-rpc-node seid status |jq -M -r .SyncInfo.latest_block_height) = 1 ]; do sleep 10; done", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", "python3 integration_test/scripts/runner.py integration_test/chain_operation/statesync_operation.yaml" diff --git a/docker/rpcnode/Dockerfile b/docker/rpcnode/Dockerfile index 952079ba9..55f33e826 100644 --- a/docker/rpcnode/Dockerfile +++ b/docker/rpcnode/Dockerfile @@ -2,6 +2,8 @@ FROM ubuntu:latest RUN apt-get update && \ apt-get install -y make git golang jq python3 curl vim +SHELL ["/bin/bash", "-c"] + VOLUME [ "/sei-protocol" ] VOLUME [ "/root/go/pkg/mod" ] WORKDIR /sei-protocol/sei-chain @@ -16,3 +18,4 @@ COPY scripts/deploy.sh /usr/bin/deploy.sh COPY scripts/step0_build.sh /usr/bin/build.sh COPY scripts/step1_configure_init.sh /usr/bin/configure_init.sh COPY scripts/step2_start_sei.sh /usr/bin/start_sei.sh +ENV PATH "$PATH:$HOME/go/bin" From e4909cd8bf38cb859555900c7fb66bc3f1cc86e7 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Fri, 30 Jun 2023 03:34:11 -0700 Subject: [PATCH 10/13] Fix --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 7e1786ca1..198c52b00 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -64,7 +64,7 @@ jobs: name: "Chain Operation Test", scripts: [ "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", - "until [ $(docker exec sei-rpc-node seid status |jq -M -r .SyncInfo.latest_block_height) = 1 ]; do sleep 10; done", + "until [[ $(docker exec sei-rpc-node seid status |jq -M -r .SyncInfo.latest_block_height) -gt 1 ]]; do sleep 10; done", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", "python3 integration_test/scripts/runner.py integration_test/chain_operation/statesync_operation.yaml" From a6550a9ad1dca8b15906ea78a9ba3f97fe20f946 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Fri, 30 Jun 2023 03:45:42 -0700 Subject: [PATCH 11/13] Fix --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 198c52b00..e5ec6c5e5 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -64,7 +64,7 @@ jobs: name: "Chain Operation Test", scripts: [ "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", - "until [[ $(docker exec sei-rpc-node seid status |jq -M -r .SyncInfo.latest_block_height) -gt 1 ]]; do sleep 10; done", + "until [[ $(docker exec sei-rpc-node build/seid status |jq -M -r .SyncInfo.latest_block_height) -gt 1 ]]; do sleep 10; done", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", "python3 integration_test/scripts/runner.py integration_test/chain_operation/statesync_operation.yaml" From e97a78fd2171a137a752d43ad6eb4218888a44f3 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Fri, 30 Jun 2023 03:57:08 -0700 Subject: [PATCH 12/13] Fix --- .github/workflows/integration-test.yml | 4 ++-- integration_test/chain_operation/snapshot_operation.yaml | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index e5ec6c5e5..0113973af 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -64,10 +64,10 @@ jobs: name: "Chain Operation Test", scripts: [ "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", - "until [[ $(docker exec sei-rpc-node build/seid status |jq -M -r .SyncInfo.latest_block_height) -gt 1 ]]; do sleep 10; done", + "until [[ $(docker exec sei-rpc-node build/seid status |jq -M -r .SyncInfo.latest_block_height) -gt 10 ]]; do sleep 10; done", + "echo rpc node started", "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml", "python3 integration_test/scripts/runner.py integration_test/chain_operation/statesync_operation.yaml" - ] } ] diff --git a/integration_test/chain_operation/snapshot_operation.yaml b/integration_test/chain_operation/snapshot_operation.yaml index 9a4c6cab9..81c05eb6b 100644 --- a/integration_test/chain_operation/snapshot_operation.yaml +++ b/integration_test/chain_operation/snapshot_operation.yaml @@ -1,8 +1,6 @@ - name: Test validators should be able to create snapshot with custom location inputs: - # Wait for sei node 0 to hit height 120 - - cmd: until [ $(seid status |jq -M -r .SyncInfo.latest_block_height) -gt 120 ]; do sleep 10; done - node: sei-node-0 + # Check if snapshotd are created - cmd: if [ -d "./build/generated/node_0/snapshots" ]; then echo "true"; else echo "false"; fi env: FOUND node: sei-node-0 From f94b95e4f8d6190192909e5922dcc12dc61031e4 Mon Sep 17 00:00:00 2001 From: Yiming Zang Date: Fri, 30 Jun 2023 04:15:53 -0700 Subject: [PATCH 13/13] Fix --- docker/rpcnode/scripts/deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/rpcnode/scripts/deploy.sh b/docker/rpcnode/scripts/deploy.sh index f0cb14a8c..e51b09a20 100755 --- a/docker/rpcnode/scripts/deploy.sh +++ b/docker/rpcnode/scripts/deploy.sh @@ -18,6 +18,7 @@ if [ -z "$SKIP_BUILD" ] then /usr/bin/build.sh fi +cp build/seid "$GOBIN"/ # Run init to set up state sync configurations /usr/bin/configure_init.sh