feat: implement reconciliation onchain

.github/workflows/integration.yml

@@ -27,7 +27,7 @@ jobs:
       # used to debug workflow
 #      - name: Setup tmate session
 #        uses: mxschmitt/action-tmate@v3
-      - run: make bep159_integration_test integration_test
+      - run: make bep159_integration_test recon_integration_test integration_test
   coverage-test:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -26,3 +26,6 @@ app/apptest/data
 app_test/data
 plugins/param/data
 plugins/tokens/data
+
+# e2e temp files
+e2e/priv_validator_key.json

Makefile

@@ -205,9 +205,14 @@ integration_test: build
 bep159_integration_test: build
 	@echo "-->BEP159 Integration Test"
 	@bash ./scripts/bep159_integration_test.sh
+
+recon_integration_test: build
+	@echo "-->Recon Integration Test"
+	@bash ./scripts/recon_integration_test.sh
+
 ########################################
 ### Pre Commit
-pre_commit: build test_unit bep159_integration_test integration_test format lint multi-nodes-test
+pre_commit: build test_unit recon_integration_test bep159_integration_test integration_test format lint multi-nodes-test
 
 ########################################
 ### Local validator nodes using docker and docker-compose

app/app.go

@@ -269,6 +269,18 @@ func NewBinanceChain(logger log.Logger, db dbm.DB, traceStore io.Writer, baseApp
 		cmn.Exit(err.Error())
 	}
 
+	// enable diff for reconciliation
+	accountIavl, ok := app.GetCommitMultiStore().GetCommitStore(common.AccountStoreKey).(*store.IavlStore)
+	if !ok {
+		cmn.Exit("cannot convert account store to ival store")
+	}
+	accountIavl.EnableDiff()
+	tokenIavl, ok := app.GetCommitMultiStore().GetCommitStore(common.TokenStoreKey).(*store.IavlStore)
+	if !ok {
+		cmn.Exit("cannot convert token store to ival store")
+	}
+	tokenIavl.EnableDiff()
+
 	// init app cache
 	accountStore := app.BaseApp.GetCommitMultiStore().GetKVStore(common.AccountStoreKey)
 	app.SetAccountStoreCache(cdc, accountStore, app.baseConfig.AccountCacheSize)
@@ -345,6 +357,8 @@ func SetUpgradeConfig(upgradeConfig *config.UpgradeConfig) {
 	upgrade.Mgr.AddUpgradeHeight(upgrade.BEP173, upgradeConfig.BEP173Height)
 	upgrade.Mgr.AddUpgradeHeight(upgrade.FixDoubleSignChainId, upgradeConfig.FixDoubleSignChainIdHeight)
 	upgrade.Mgr.AddUpgradeHeight(upgrade.BEP126, upgradeConfig.BEP126Height)
+	upgrade.Mgr.AddUpgradeHeight(upgrade.EnableReconciliation, upgradeConfig.EnableReconciliationHeight)
+	upgrade.Mgr.AddUpgradeHeight(upgrade.DisableMessagesPhase1, upgradeConfig.DisableMessagesPhase1Height)
 
 	// register store keys of upgrade
 	upgrade.Mgr.RegisterStoreKeys(upgrade.BEP9, common.TimeLockStoreKey.Name())
@@ -953,6 +967,11 @@ func (app *BinanceChain) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) a
 	pub.Pool.Clean()
 	// match may end with transaction failure, which is better to save into
 	// the EndBlock response. However, current cosmos doesn't support this.
+
+	if sdk.IsUpgrade(upgrade.EnableReconciliation) {
+		app.reconBalance(ctx)
+	}
+
 	return abci.ResponseEndBlock{
 		ValidatorUpdates: validatorUpdates,
 		Events:           ctx.EventManager().ABCIEvents(),

app/config/config.go

@@ -101,6 +101,10 @@ BEP173Height = {{ .UpgradeConfig.BEP173Height }}
 FixDoubleSignChainIdHeight = {{ .UpgradeConfig.FixDoubleSignChainIdHeight }}
 # Block height of BEP126 upgrade
 BEP126Height = {{ .UpgradeConfig.BEP126Height }}
+# Block height of reconciliation upgrade
+EnableReconciliationHeight = {{ .UpgradeConfig.EnableReconciliationHeight }}
+# Block height of disable messages phase1 upgrade
+DisableMessagesPhase1Height = {{ .UpgradeConfig.DisableMessagesPhase1Height }}
 
 [query]
 # ABCI query interface black list, suggested value: ["custom/gov/proposals", "custom/timelock/timelocks", "custom/atomicSwap/swapcreator", "custom/atomicSwap/swaprecipient"]
@@ -549,6 +553,8 @@ type UpgradeConfig struct {
 	BEP173Height                                    int64 `mapstructure:"BEP173Height"`
 	FixDoubleSignChainIdHeight                      int64 `mapstructure:"FixDoubleSignChainIdHeight"`
 	BEP126Height                                    int64 `mapstructure:"BEP126Height"`
+	EnableReconciliationHeight                      int64 `mapstructure:"EnableReconciliationHeight"`
+	DisableMessagesPhase1Height                     int64 `mapstructure:"DisableMessagesPhase1Height"`
 }
 
 func defaultUpgradeConfig() *UpgradeConfig {
@@ -569,7 +575,7 @@ func defaultUpgradeConfig() *UpgradeConfig {
 		BEP70Height:                       1,
 		LaunchBscUpgradeHeight:            1,
 		LimitConsAddrUpdateIntervalHeight: math.MaxInt64,
-		BEP126Height:               	   math.MaxInt64,
+		BEP126Height:                      math.MaxInt64,
 		BEP128Height:                      math.MaxInt64,
 		BEP151Height:                      math.MaxInt64,
 		BEP153Height:                      math.MaxInt64,
@@ -583,6 +589,8 @@ func defaultUpgradeConfig() *UpgradeConfig {
 		BEP171Height:                      math.MaxInt64,
 		FixFailAckPackageHeight:           math.MaxInt64,
 		EnableAccountScriptsForCrossChainTransferHeight: math.MaxInt64,
+		EnableReconciliationHeight:                      math.MaxInt64,
+		DisableMessagesPhase1Height:                     math.MaxInt64,
 	}
 }
 

app/reconciliation.go

@@ -0,0 +1,116 @@
+package app
+
+import (
+	"fmt"
+
+	"github.com/cosmos/cosmos-sdk/store"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+
+	"github.com/bnb-chain/node/common"
+	"github.com/bnb-chain/node/common/types"
+)
+
+const globalAccountNumber = "globalAccountNumber"
+
+// reconBalance will do reconciliation for accounts balances.
+func (app *BinanceChain) reconBalance(ctx sdk.Context) {
+	currentHeight := ctx.BlockHeight()
+
+	accountStore, ok := app.GetCommitMultiStore().GetCommitStore(common.AccountStoreKey).(*store.IavlStore)
+	if !ok {
+		panic("cannot convert account store to ival store")
+	}
+	accPre, accCurrent := app.getAccountChanges(ctx, accountStore)
+	accountStore.ResetDiff()
+
+	tokenStore, ok := app.GetCommitMultiStore().GetCommitStore(common.TokenStoreKey).(*store.IavlStore)
+	if !ok {
+		panic("cannot convert token store to ival store")
+	}
+	tokenPre, tokenCurrent := app.getTokenChanges(ctx, tokenStore)
+	tokenStore.ResetDiff()
+
+	left := tokenPre.Plus(accCurrent)
+	right := tokenCurrent.Plus(accPre)
+
+	if !left.IsEqual(right) {
+		err := fmt.Sprintf("unbalanced at block %d, pre: %s, current: %s \n",
+			currentHeight, left.String(), right.String())
+		ctx.Logger().Error(err)
+		panic(err)
+	}
+}
+
+func (app *BinanceChain) getAccountChanges(ctx sdk.Context, accountStore *store.IavlStore) (sdk.Coins, sdk.Coins) {
+	preCoins := sdk.Coins{}
+	currentCoins := sdk.Coins{}
+
+	diff := accountStore.GetDiff()
+	version := accountStore.GetTree().Version() - 1
+	for k, v := range diff {
+		if k == globalAccountNumber {
+			continue
+		}
+		var acc1 sdk.Account
+		err := app.Codec.UnmarshalBinaryBare(v, &acc1)
+		if err != nil {
+			panic("failed to unmarshal diff value " + err.Error())
+		}
+		nacc1 := acc1.(types.NamedAccount)
+		ctx.Logger().Debug("diff account", "address", nacc1.GetAddress(), "coins", nacc1.GetCoins().String())
+		currentCoins = currentCoins.Plus(nacc1.GetCoins())
+		currentCoins = currentCoins.Plus(nacc1.GetFrozenCoins())
+		currentCoins = currentCoins.Plus(nacc1.GetLockedCoins())
+
+		var acc2 sdk.Account
+		_, v = accountStore.GetTree().GetVersioned([]byte(k), version)
+		if v != nil { // it is not a new account
+			err = app.Codec.UnmarshalBinaryBare(v, &acc2)
+			if err != nil {
+				panic("failed to unmarshal previous value " + err.Error())
+			}
+			nacc2 := acc2.(types.NamedAccount)
+
+			ctx.Logger().Debug("pre account", "address", nacc2.GetAddress(), "coins", nacc2.GetCoins().String())
+			preCoins = preCoins.Plus(nacc2.GetCoins())
+			preCoins = preCoins.Plus(nacc2.GetFrozenCoins())
+			preCoins = preCoins.Plus(nacc2.GetLockedCoins())
+		}
+	}
+	ctx.Logger().Debug("account changes", "diff", currentCoins.String(), "previous", preCoins.String(), "height", ctx.BlockHeight())
+
+	return preCoins, currentCoins
+}
+
+func (app *BinanceChain) getTokenChanges(ctx sdk.Context, tokenStore *store.IavlStore) (sdk.Coins, sdk.Coins) {
+	preCoins := sdk.Coins{}
+	currentCoins := sdk.Coins{}
+
+	diff := tokenStore.GetDiff()
+	version := tokenStore.GetTree().Version() - 1
+	for k, v := range diff {
+		var token1 types.IToken
+		err := app.Codec.UnmarshalBinaryBare(v, &token1)
+		if err != nil {
+			panic("failed to unmarshal diff value " + err.Error())
+		}
+		currentCoins = currentCoins.Plus(sdk.Coins{
+			sdk.NewCoin(token1.GetSymbol(), token1.GetTotalSupply().ToInt64()),
+		})
+
+		var token2 types.IToken
+		_, v = tokenStore.GetTree().GetVersioned([]byte(k), version)
+		if v != nil { // it is not a new token
+			err = app.Codec.UnmarshalBinaryBare(v, &token2)
+			if err != nil {
+				panic("failed to unmarshal previous value " + err.Error())
+			}
+			preCoins = preCoins.Plus(sdk.Coins{
+				sdk.NewCoin(token2.GetSymbol(), token2.GetTotalSupply().ToInt64()),
+			})
+		}
+	}
+	ctx.Logger().Debug("token changes", "diff", currentCoins.String(), "previous", preCoins.String(), "height", ctx.BlockHeight())
+
+	return preCoins, currentCoins
+}

common/upgrade/upgrade.go

@@ -49,6 +49,8 @@ const (
 	BEP171                      = sdk.BEP171 // https://github.com/bnb-chain/BEPs/pull/171 Security Enhancement for Cross-Chain Module
 	BEP173                      = sdk.BEP173 // https://github.com/bnb-chain/BEPs/pull/173 Text Proposal
 	FixDoubleSignChainId        = sdk.FixDoubleSignChainId
+	EnableReconciliation        = sdk.EnableReconciliation
+	DisableMessagesPhase1       = sdk.DisableMessagesPhase1
 )
 
 func UpgradeBEP10(before func(), after func()) {

go.mod

@@ -88,7 +88,7 @@ require (
 )
 
 replace (
-	github.com/cosmos/cosmos-sdk => github.com/bnb-chain/bnc-cosmos-sdk v0.26.5
+	github.com/cosmos/cosmos-sdk => github.com/forcodedancing/bnc-cosmos-sdk v0.25.8-0.20230531023724-2e22c6326b67
 	github.com/tendermint/go-amino => github.com/bnb-chain/bnc-go-amino v0.14.1-binance.2
 	github.com/tendermint/iavl => github.com/bnb-chain/bnc-tendermint-iavl v0.12.0-binance.5
 	github.com/tendermint/tendermint => github.com/bnb-chain/bnc-tendermint v0.32.3-bc.10

go.sum

@@ -58,8 +58,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bnb-chain/bnc-cosmos-sdk v0.26.5 h1:6NFIkrAURJdXP1w+cdVnyIxrtuDev4sg3kM86kteVvA=
-github.com/bnb-chain/bnc-cosmos-sdk v0.26.5/go.mod h1:XiDYVT+XqECR+AyCBO4KBsrbL/d1x2UTsVU36SvHxI8=
 github.com/bnb-chain/bnc-go-amino v0.14.1-binance.2 h1:iAlp9gqG0f2LGAauf3ZiijWlT6NI+W2r9y70HH9LI3k=
 github.com/bnb-chain/bnc-go-amino v0.14.1-binance.2/go.mod h1:LiCO7jev+3HwLGAiN9gpD0z+jTz95RqgSavbse55XOY=
 github.com/bnb-chain/bnc-tendermint v0.32.3-bc.10 h1:E4iSwEbJCLYchHiHE1gnOM3jjmJXLBxARhy/RCl8CpI=
@@ -143,6 +141,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
+github.com/forcodedancing/bnc-cosmos-sdk v0.25.8-0.20230531023724-2e22c6326b67 h1:yysn9/l3K9jEcdJ4Nd25tvPLHQxw1J9PkS4a3RyNRNM=
+github.com/forcodedancing/bnc-cosmos-sdk v0.25.8-0.20230531023724-2e22c6326b67/go.mod h1:XiDYVT+XqECR+AyCBO4KBsrbL/d1x2UTsVU36SvHxI8=
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=

networks/demo/bind.exp

@@ -0,0 +1,19 @@
+#!/usr/bin/expect
+
+set symbol [lindex $argv 0]
+set contract_address [lindex $argv 1]
+set amount [lindex $argv 2]
+set expire_time [lindex $argv 3]
+set from [lindex $argv 4]
+set chain_id [lindex $argv 5]
+set home [lindex $argv 6]
+
+set timeout 30
+	if {"${home}" == ""} {
+		spawn ./bnbcli bridge bind --symbol $symbol --contract-address $contract_address --amount $amount --contract-decimals 18 --expire-time $expire_time --from $from --chain-id $chain_id
+	} else {
+		spawn ./bnbcli bridge bind --home $home --symbol $symbol --contract-address $contract_address --amount $amount --contract-decimals 18 --expire-time $expire_time --from $from --chain-id $chain_id
+	}
+	expect "Password*"
+	send "12345678\r"
+expect eof

networks/demo/burn.exp

@@ -0,0 +1,17 @@
+#!/usr/bin/expect
+
+set symbol [lindex $argv 0]
+set amount [lindex $argv 1]
+set from [lindex $argv 2]
+set chain_id [lindex $argv 3]
+set home [lindex $argv 4]
+
+set timeout 30
+	if {"${home}" == ""} {
+		spawn ./bnbcli token burn -s $symbol  -n $amount --from $from --chain-id $chain_id
+	} else {
+		spawn ./bnbcli token burn --home $home -s $symbol -n $amount --from $from --chain-id $chain_id
+	}
+	expect "Password*"
+	send "12345678\r"
+expect eof

networks/demo/freeze.exp

@@ -0,0 +1,17 @@
+#!/usr/bin/expect
+
+set symbol [lindex $argv 0]
+set amount [lindex $argv 1]
+set from [lindex $argv 2]
+set chain_id [lindex $argv 3]
+set home [lindex $argv 4]
+
+set timeout 30
+	if {"${home}" == ""} {
+		spawn ./bnbcli token freeze -s $symbol  -n $amount --from $from --chain-id $chain_id
+	} else {
+		spawn ./bnbcli token freeze --home $home -s $symbol -n $amount --from $from --chain-id $chain_id
+	}
+	expect "Password*"
+	send "12345678\r"
+expect eof

networks/demo/transfer-out.exp

@@ -0,0 +1,14 @@
+#!/usr/bin/expect
+
+set home [lindex $argv 0]
+set from [lindex $argv 1]
+set chain_id [lindex $argv 2]
+set amount [lindex $argv 3]
+set to [lindex $argv 4]
+set expire_time [lindex $argv 5]
+
+set timeout 30
+    spawn ./bnbcli bridge transfer-out --home $home --from $from --chain-id=$chain_id --amount $amount --to $to --expire-time $expire_time
+	expect "Password*"
+	send "12345678\r"
+expect eof

networks/demo/unbind.exp

@@ -0,0 +1,16 @@
+#!/usr/bin/expect
+
+set symbol [lindex $argv 0]
+set from [lindex $argv 1]
+set chain_id [lindex $argv 2]
+set home [lindex $argv 3]
+
+set timeout 30
+	if {"${home}" == ""} {
+		spawn ./bnbcli bridge unbind --symbol $symbol --from $from --chain-id $chain_id
+	} else {
+		spawn ./bnbcli bridge unbind --home $home --symbol $symbol --from $from --chain-id $chain_id
+	}
+	expect "Password*"
+	send "12345678\r"
+expect eof

networks/demo/unfreeze.exp

@@ -0,0 +1,17 @@
+#!/usr/bin/expect
+
+set symbol [lindex $argv 0]
+set amount [lindex $argv 1]
+set from [lindex $argv 2]
+set chain_id [lindex $argv 3]
+set home [lindex $argv 4]
+
+set timeout 30
+	if {"${home}" == ""} {
+		spawn ./bnbcli token unfreeze -s $symbol  -n $amount --from $from --chain-id $chain_id
+	} else {
+		spawn ./bnbcli token unfreeze --home $home -s $symbol -n $amount --from $from --chain-id $chain_id
+	}
+	expect "Password*"
+	send "12345678\r"
+expect eof

plugins/bridge/handler.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/bnb-chain/node/common/log"
 	cmmtypes "github.com/bnb-chain/node/common/types"
+	"github.com/bnb-chain/node/common/upgrade"
 	"github.com/bnb-chain/node/plugins/bridge/types"
 )
 
@@ -22,6 +23,9 @@ func NewHandler(keeper Keeper) sdk.Handler {
 		case TransferOutMsg:
 			return handleTransferOutMsg(ctx, keeper, msg)
 		case BindMsg:
+			if sdk.IsUpgrade(upgrade.DisableMessagesPhase1) {
+				return sdk.ErrMsgNotSupported("BindMsg disabled in DisableMessagesPhase1 upgrade").Result()
+			}
 			return handleBindMsg(ctx, keeper, msg)
 		case UnbindMsg:
 			return handleUnbindMsg(ctx, keeper, msg)