Skip to content

Commit

Permalink
feat: implement reconciliation onchain (#956)
Browse files Browse the repository at this point in the history
  • Loading branch information
forcodedancing authored Jul 6, 2023
1 parent 600ede1 commit a2be24e
Show file tree
Hide file tree
Showing 18 changed files with 765 additions and 13 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
# used to debug workflow
# - name: Setup tmate session
# uses: mxschmitt/action-tmate@v3
- run: make bep159_integration_test integration_test
- run: make bep159_integration_test recon_integration_test integration_test
coverage-test:
runs-on: ubuntu-latest
steps:
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,6 @@ app/apptest/data
app_test/data
plugins/param/data
plugins/tokens/data

# e2e temp files
e2e/priv_validator_key.json
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,14 @@ integration_test: build
bep159_integration_test: build
@echo "-->BEP159 Integration Test"
@bash ./scripts/bep159_integration_test.sh

recon_integration_test: build
@echo "-->Recon Integration Test"
@bash ./scripts/recon_integration_test.sh

########################################
### Pre Commit
pre_commit: build test_unit bep159_integration_test integration_test format lint multi-nodes-test
pre_commit: build test_unit recon_integration_test bep159_integration_test integration_test format lint multi-nodes-test

########################################
### Local validator nodes using docker and docker-compose
Expand Down
24 changes: 24 additions & 0 deletions app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ func NewBinanceChain(logger log.Logger, db dbm.DB, traceStore io.Writer, baseApp
common.BridgeStoreKey,
common.OracleStoreKey,
common.IbcStoreKey,
common.ReconStoreKey,
)
app.SetAnteHandler(tx.NewAnteHandler(app.AccountKeeper))
app.SetPreChecker(tx.NewTxPreChecker())
Expand All @@ -269,6 +270,18 @@ func NewBinanceChain(logger log.Logger, db dbm.DB, traceStore io.Writer, baseApp
cmn.Exit(err.Error())
}

// enable diff for reconciliation
accountIavl, ok := app.GetCommitMultiStore().GetCommitStore(common.AccountStoreKey).(*store.IavlStore)
if !ok {
cmn.Exit("cannot convert account store to ival store")
}
accountIavl.EnableDiff()
tokenIavl, ok := app.GetCommitMultiStore().GetCommitStore(common.TokenStoreKey).(*store.IavlStore)
if !ok {
cmn.Exit("cannot convert token store to ival store")
}
tokenIavl.EnableDiff()

// init app cache
accountStore := app.BaseApp.GetCommitMultiStore().GetKVStore(common.AccountStoreKey)
app.SetAccountStoreCache(cdc, accountStore, app.baseConfig.AccountCacheSize)
Expand Down Expand Up @@ -345,13 +358,15 @@ func SetUpgradeConfig(upgradeConfig *config.UpgradeConfig) {
upgrade.Mgr.AddUpgradeHeight(upgrade.BEP173, upgradeConfig.BEP173Height)
upgrade.Mgr.AddUpgradeHeight(upgrade.FixDoubleSignChainId, upgradeConfig.FixDoubleSignChainIdHeight)
upgrade.Mgr.AddUpgradeHeight(upgrade.BEP126, upgradeConfig.BEP126Height)
upgrade.Mgr.AddUpgradeHeight(upgrade.BEP255, upgradeConfig.BEP255Height)

// register store keys of upgrade
upgrade.Mgr.RegisterStoreKeys(upgrade.BEP9, common.TimeLockStoreKey.Name())
upgrade.Mgr.RegisterStoreKeys(upgrade.BEP3, common.AtomicSwapStoreKey.Name())
upgrade.Mgr.RegisterStoreKeys(upgrade.LaunchBscUpgrade, common.IbcStoreKey.Name(), common.SideChainStoreKey.Name(),
common.SlashingStoreKey.Name(), common.BridgeStoreKey.Name(), common.OracleStoreKey.Name())
upgrade.Mgr.RegisterStoreKeys(upgrade.BEP128, common.StakeRewardStoreKey.Name())
upgrade.Mgr.RegisterStoreKeys(upgrade.BEP255, common.ReconStoreKey.Name())

// register msg types of upgrade
upgrade.Mgr.RegisterMsgTypes(upgrade.BEP9,
Expand Down Expand Up @@ -953,6 +968,15 @@ func (app *BinanceChain) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) a
pub.Pool.Clean()
// match may end with transaction failure, which is better to save into
// the EndBlock response. However, current cosmos doesn't support this.

accountIavl, _ := app.GetCommitMultiStore().GetCommitStore(common.AccountStoreKey).(*store.IavlStore)
tokenIavl, _ := app.GetCommitMultiStore().GetCommitStore(common.TokenStoreKey).(*store.IavlStore)
if sdk.IsUpgrade(upgrade.BEP255) {
app.reconBalance(ctx, accountIavl, tokenIavl)
}
accountIavl.ResetDiff()
tokenIavl.ResetDiff()

return abci.ResponseEndBlock{
ValidatorUpdates: validatorUpdates,
Events: ctx.EventManager().ABCIEvents(),
Expand Down
6 changes: 5 additions & 1 deletion app/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ BEP173Height = {{ .UpgradeConfig.BEP173Height }}
FixDoubleSignChainIdHeight = {{ .UpgradeConfig.FixDoubleSignChainIdHeight }}
# Block height of BEP126 upgrade
BEP126Height = {{ .UpgradeConfig.BEP126Height }}
# Block height of BEP255 upgrade
BEP255Height = {{ .UpgradeConfig.BEP255Height }}
[query]
# ABCI query interface black list, suggested value: ["custom/gov/proposals", "custom/timelock/timelocks", "custom/atomicSwap/swapcreator", "custom/atomicSwap/swaprecipient"]
Expand Down Expand Up @@ -549,6 +551,7 @@ type UpgradeConfig struct {
BEP173Height int64 `mapstructure:"BEP173Height"`
FixDoubleSignChainIdHeight int64 `mapstructure:"FixDoubleSignChainIdHeight"`
BEP126Height int64 `mapstructure:"BEP126Height"`
BEP255Height int64 `mapstructure:"BEP255Height"`
}

func defaultUpgradeConfig() *UpgradeConfig {
Expand All @@ -569,7 +572,7 @@ func defaultUpgradeConfig() *UpgradeConfig {
BEP70Height: 1,
LaunchBscUpgradeHeight: 1,
LimitConsAddrUpdateIntervalHeight: math.MaxInt64,
BEP126Height: math.MaxInt64,
BEP126Height: math.MaxInt64,
BEP128Height: math.MaxInt64,
BEP151Height: math.MaxInt64,
BEP153Height: math.MaxInt64,
Expand All @@ -583,6 +586,7 @@ func defaultUpgradeConfig() *UpgradeConfig {
BEP171Height: math.MaxInt64,
FixFailAckPackageHeight: math.MaxInt64,
EnableAccountScriptsForCrossChainTransferHeight: math.MaxInt64,
BEP255Height: math.MaxInt64,
}
}

Expand Down
143 changes: 143 additions & 0 deletions app/reconciliation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package app

import (
"encoding/binary"
"fmt"
"strings"

"github.com/cosmos/cosmos-sdk/store"
sdk "github.com/cosmos/cosmos-sdk/types"

"github.com/bnb-chain/node/common"
"github.com/bnb-chain/node/common/types"
)

const globalAccountNumber = "globalAccountNumber"

// unbalancedBlockHeightKey for saving unbalanced block height for reconciliation
var unbalancedBlockHeightKey = []byte("0x01")

// reconBalance will do reconciliation for accounts balances.
func (app *BinanceChain) reconBalance(ctx sdk.Context, accountIavl *store.IavlStore, tokenIavl *store.IavlStore) {
height, exists := app.getUnbalancedBlockHeight(ctx)
if exists {
panic(fmt.Sprintf("unbalanced state at block height %d, please use hardfork to bypass it", height))
}

accPre, accCurrent := app.getAccountChanges(ctx, accountIavl)
tokenPre, tokenCurrent := app.getTokenChanges(ctx, tokenIavl)

// accPre and tokenPre are positive, there will be no overflow
accountDiff := accCurrent.Plus(accPre.Negative())
tokenDiff := tokenCurrent.Plus(tokenPre.Negative())

if !accountDiff.IsEqual(tokenDiff) {
ctx.Logger().Error(fmt.Sprintf("unbalanced at block %d, account diff: %s, token diff: %s \n",
ctx.BlockHeight(), accountDiff.String(), tokenDiff.String()))
app.saveUnbalancedBlockHeight(ctx)
}
}

func (app *BinanceChain) getAccountChanges(ctx sdk.Context, accountStore *store.IavlStore) (sdk.Coins, sdk.Coins) {
preCoins := sdk.Coins{}
currentCoins := sdk.Coins{}

diff := accountStore.GetDiff()
version := accountStore.GetTree().Version() - 1
for k := range diff {
if strings.Contains(k, globalAccountNumber) {
continue
}
v := accountStore.Get([]byte(k))
if v != nil {
var acc1 sdk.Account
err := app.Codec.UnmarshalBinaryBare(v, &acc1)
if err != nil {
ctx.Logger().Error("failed to unmarshal current account value", "err", err.Error())
} else {
nacc1 := acc1.(types.NamedAccount)
ctx.Logger().Debug("current account", "address", nacc1.GetAddress(), "coins", nacc1.GetCoins().String())
currentCoins = currentCoins.Plus(nacc1.GetCoins())
currentCoins = currentCoins.Plus(nacc1.GetFrozenCoins())
currentCoins = currentCoins.Plus(nacc1.GetLockedCoins())
}
}

_, v = accountStore.GetTree().GetVersioned([]byte(k), version)
if v != nil { // it is not a new account
var acc2 sdk.Account
err := app.Codec.UnmarshalBinaryBare(v, &acc2)
if err != nil {
ctx.Logger().Error("failed to unmarshal previous account value", "err", err.Error())
} else {
nacc2 := acc2.(types.NamedAccount)
ctx.Logger().Debug("previous account", "address", nacc2.GetAddress(), "coins", nacc2.GetCoins().String())
preCoins = preCoins.Plus(nacc2.GetCoins())
preCoins = preCoins.Plus(nacc2.GetFrozenCoins())
preCoins = preCoins.Plus(nacc2.GetLockedCoins())
}
}
}
ctx.Logger().Debug("account changes", "current", currentCoins.String(), "previous", preCoins.String(),
"version", version, "height", ctx.BlockHeight())

return preCoins, currentCoins
}

func (app *BinanceChain) getTokenChanges(ctx sdk.Context, tokenStore *store.IavlStore) (sdk.Coins, sdk.Coins) {
preCoins := sdk.Coins{}
currentCoins := sdk.Coins{}

diff := tokenStore.GetDiff()
version := tokenStore.GetTree().Version() - 1
for k := range diff {
v := tokenStore.Get([]byte(k))
if v != nil {
var token1 types.IToken
err := app.Codec.UnmarshalBinaryBare(v, &token1)
if err != nil {
ctx.Logger().Error("failed to unmarshal current token value", "err", err.Error())
} else {
ctx.Logger().Debug("current token", "symbol", token1.GetSymbol(), "supply", token1.GetTotalSupply().ToInt64())
currentCoins = currentCoins.Plus(sdk.Coins{
sdk.NewCoin(token1.GetSymbol(), token1.GetTotalSupply().ToInt64()),
})
}
}

_, v = tokenStore.GetTree().GetVersioned([]byte(k), version)
if v != nil { // it is not a new token
var token2 types.IToken
err := app.Codec.UnmarshalBinaryBare(v, &token2)
if err != nil {
ctx.Logger().Error("failed to unmarshal previous token value", "err", err.Error())
} else {
ctx.Logger().Debug("previous token", "symbol", token2.GetSymbol(), "supply", token2.GetTotalSupply().ToInt64())
preCoins = preCoins.Plus(sdk.Coins{
sdk.NewCoin(token2.GetSymbol(), token2.GetTotalSupply().ToInt64()),
})
}
}
}
ctx.Logger().Debug("token changes", "current", currentCoins.String(), "previous", preCoins.String(),
"version", version, "height", ctx.BlockHeight())

return preCoins, currentCoins
}

func (app *BinanceChain) saveUnbalancedBlockHeight(ctx sdk.Context) {
reconStore := app.GetCommitMultiStore().GetCommitStore(common.ReconStoreKey).(*store.IavlStore)
bz := make([]byte, 8)
binary.BigEndian.PutUint64(bz[:], uint64(ctx.BlockHeight()))
reconStore.Set(unbalancedBlockHeightKey, bz)
}

func (app *BinanceChain) getUnbalancedBlockHeight(ctx sdk.Context) (uint64, bool) {
reconStore := app.GetCommitMultiStore().GetCommitStore(common.ReconStoreKey).(*store.IavlStore)

bz := reconStore.Get(unbalancedBlockHeightKey)
if bz == nil {
return 0, false
}
return binary.BigEndian.Uint64(bz), true
}
66 changes: 66 additions & 0 deletions app/reconciliation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package app

import (
"math"
"testing"

"github.com/cosmos/cosmos-sdk/types"
"github.com/stretchr/testify/require"
)

func Test_Reconciliation_Overflow(t *testing.T) {
defer func() {
if r := recover(); r == nil {
t.Errorf("should panic for overflow")
}
}()

accountPre := types.Coins{
types.NewCoin("BNB", 10),
}
accountCurrent := types.Coins{
types.NewCoin("BNB", math.MaxInt64),
}
tokenPre := types.Coins{
types.NewCoin("BNB", 10),
}
tokenCurrent := types.Coins{
types.NewCoin("BNB", math.MaxInt64),
}

_ = accountPre.Plus(tokenCurrent).IsEqual(accountCurrent.Plus(tokenPre))
}

func Test_Reconciliation_NoOverflow(t *testing.T) {
accountPre := types.Coins{
types.NewCoin("BNB", 10),
}
accountCurrent := types.Coins{
types.NewCoin("BNB", math.MaxInt64),
}
tokenPre := types.Coins{
types.NewCoin("BNB", 10),
}
tokenCurrent := types.Coins{
types.NewCoin("BNB", math.MaxInt64),
}

equal := accountCurrent.Plus(accountPre.Negative()).IsEqual(tokenCurrent.Plus(tokenPre.Negative()))
require.True(t, equal)

accountPre = types.Coins{
types.NewCoin("BNB", math.MaxInt64),
}
accountCurrent = types.Coins{
types.NewCoin("BNB", 10),
}
tokenPre = types.Coins{
types.NewCoin("BNB", math.MaxInt64),
}
tokenCurrent = types.Coins{
types.NewCoin("BNB", 10),
}

equal = accountCurrent.Plus(accountPre.Negative()).IsEqual(tokenCurrent.Plus(tokenPre.Negative()))
require.True(t, equal)
}
4 changes: 4 additions & 0 deletions common/stores.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const (
OracleStoreName = "oracle"
IbcStoreName = "ibc"
SideChainStoreName = "sc"
ReconStoreName = "recon"

StakeTransientStoreName = "transient_stake"
ParamsTransientStoreName = "transient_params"
Expand All @@ -44,6 +45,7 @@ var (
OracleStoreKey = sdk.NewKVStoreKey(OracleStoreName)
IbcStoreKey = sdk.NewKVStoreKey(IbcStoreName)
SideChainStoreKey = sdk.NewKVStoreKey(SideChainStoreName)
ReconStoreKey = sdk.NewKVStoreKey(ReconStoreName)

TStakeStoreKey = sdk.NewTransientStoreKey(StakeTransientStoreName)
TParamsStoreKey = sdk.NewTransientStoreKey(ParamsTransientStoreName)
Expand All @@ -66,6 +68,7 @@ var (
SideChainStoreName: SideChainStoreKey,
BridgeStoreName: BridgeStoreKey,
OracleStoreName: OracleStoreKey,
ReconStoreName: ReconStoreKey,
StakeTransientStoreName: TStakeStoreKey,
ParamsTransientStoreName: TParamsStoreKey,
}
Expand All @@ -88,6 +91,7 @@ var (
SideChainStoreName,
BridgeStoreName,
OracleStoreName,
ReconStoreName,
}
)

Expand Down
1 change: 1 addition & 0 deletions common/upgrade/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ const (
BEP171 = sdk.BEP171 // https://github.com/bnb-chain/BEPs/pull/171 Security Enhancement for Cross-Chain Module
BEP173 = sdk.BEP173 // https://github.com/bnb-chain/BEPs/pull/173 Text Proposal
FixDoubleSignChainId = sdk.FixDoubleSignChainId
BEP255 = sdk.BEP255 // https://github.com/bnb-chain/BEPs/pull/255 Asset Reconciliation for Security Enhancement
)

func UpgradeBEP10(before func(), after func()) {
Expand Down
Loading

0 comments on commit a2be24e

Please sign in to comment.