From f3f7da406ee1e5a1d1bdfab7f9d02f2bf65920fe Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Wed, 27 Dec 2023 21:46:05 -0500 Subject: [PATCH 01/13] test(simulation): fix non-determinism sim test --- app/app.go | 2 +- app/keepers.go | 35 ++++---------------- contrib/make/simulation.mk | 2 +- simapp/sim_test.go | 47 ++++++++------------------- x/common/testutil/testapp/testapp.go | 4 ++- x/sudo/module.go | 25 +++++++++++++-- x/sudo/simulation/genesis.go | 48 ++++++++++++++++++++++++++++ 7 files changed, 97 insertions(+), 66 deletions(-) create mode 100644 x/sudo/simulation/genesis.go diff --git a/app/app.go b/app/app.go index 3e28de638..346c8eda7 100644 --- a/app/app.go +++ b/app/app.go @@ -179,7 +179,7 @@ func NewNibiruApp( // add test gRPC service for testing gRPC queries in isolation testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{}) - app.InitSimulationManager(app.appCodec) + app.initSimulationManager(app.appCodec) // initialize stores app.MountKVStores(keys) diff --git a/app/keepers.go b/app/keepers.go index a60a8914e..40361da70 100644 --- a/app/keepers.go +++ b/app/keepers.go @@ -847,34 +847,13 @@ func initParamsKeeper( return paramsKeeper } -// TODO: Simulation manager -func (app *NibiruApp) InitSimulationManager( +func (app *NibiruApp) initSimulationManager( appCodec codec.Codec, ) { - // // create the simulation manager and define the order of the modules for deterministic simulations - // // - // // NOTE: this is not required apps that don't use the simulator for fuzz testing - // // transactions - // epochsModule := epochs.NewAppModule(appCodec, app.EpochsKeeper) - // app.sm = module.NewSimulationManager( - // auth.NewAppModule(appCodec, app.AccountKeeper, authsims.RandomGenesisAccounts), - // bank.NewAppModule(appCodec, app.BankKeeper, app.AccountKeeper), - // feegrantmodule.NewAppModule(appCodec, app.AccountKeeper, app.BankKeeper, app.FeeGrantKeeper, app.interfaceRegistry), - // gov.NewAppModule(appCodec, app.GovKeeper, app.AccountKeeper, app.BankKeeper), - // staking.NewAppModule(appCodec, app.stakingKeeper, app.AccountKeeper, app.BankKeeper), - // distr.NewAppModule(appCodec, app.DistrKeeper, app.AccountKeeper, app.BankKeeper, app.stakingKeeper), - // slashing.NewAppModule(appCodec, app.slashingKeeper, app.AccountKeeper, app.BankKeeper, app.stakingKeeper), - // params.NewAppModule(app.paramsKeeper), - // authzmodule.NewAppModule(appCodec, app.authzKeeper, app.AccountKeeper, app.BankKeeper, app.interfaceRegistry), - // // native x/ - // epochsModule, - // // ibc - // capability.NewAppModule(appCodec, *app.capabilityKeeper), - // evidence.NewAppModule(app.evidenceKeeper), - // ibc.NewAppModule(app.ibcKeeper), - // ibctransfer.NewAppModule(app.transferKeeper), - // ibcfee.NewAppModule(app.ibcFeeKeeper), - // ) - // - // app.sm.RegisterStoreDecoders() + overrideModules := map[string]module.AppModuleSimulation{ + authtypes.ModuleName: auth.NewAppModule(app.appCodec, app.AccountKeeper, authsims.RandomGenesisAccounts, app.GetSubspace(authtypes.ModuleName)), + } + app.sm = module.NewSimulationManagerFromAppModules(app.mm.Modules, overrideModules) + + app.sm.RegisterStoreDecoders() } diff --git a/contrib/make/simulation.mk b/contrib/make/simulation.mk index 31836c6bf..7461c93b9 100644 --- a/contrib/make/simulation.mk +++ b/contrib/make/simulation.mk @@ -11,7 +11,7 @@ $(RUNSIM): .PHONY: test-sim-nondeterminism test-sim-nondeterminism: @echo "Running non-determinism test..." - @go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true \ + go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true \ -NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h .PHONY: test-sim-default-genesis-fast diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 5454d709d..09fcf9dcf 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -7,52 +7,33 @@ import ( "os" "testing" - "github.com/cosmos/ibc-go/v7/testing/simapp" - dbm "github.com/cometbft/cometbft-db" + "github.com/cometbft/cometbft/libs/log" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/testutil/sims" helpers "github.com/cosmos/cosmos-sdk/testutil/sims" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" simulationtypes "github.com/cosmos/cosmos-sdk/types/simulation" "github.com/cosmos/cosmos-sdk/x/simulation" simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" + "github.com/cosmos/ibc-go/v7/testing/simapp" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "github.com/NibiruChain/nibiru/app" appsim "github.com/NibiruChain/nibiru/app/sim" - "github.com/NibiruChain/nibiru/x/common/testutil" "github.com/NibiruChain/nibiru/x/common/testutil/testapp" ) // SimAppChainID hardcoded chainID for simulation const SimAppChainID = "simulation-app" -type SimulationTestSuite struct { - suite.Suite -} - -func TestSimulationTestSuite(t *testing.T) { - suite.Run(t, new(SimulationTestSuite)) -} - -var _ suite.SetupTestSuite = (*SimulationTestSuite)(nil) - func init() { // We call GetSimulatorFlags here in order to set the value for // 'simapp.FlagEnabledValue', which enables simulations appsim.GetSimulatorFlags() } -// SetupTest: Runs before every test in the suite. -func (s *SimulationTestSuite) SetupTest() { - testutil.BeforeIntegrationSuite(s.T()) - if !simapp.FlagEnabledValue { - s.T().Skip("skipping application simulation") - } -} - -func (s *SimulationTestSuite) TestFullAppSimulation() { - t := s.T() +func TestFullAppSimulation(t *testing.T) { config := simcli.NewConfigFromFlags() config.ChainID = SimAppChainID @@ -105,11 +86,7 @@ func (s *SimulationTestSuite) TestFullAppSimulation() { } } -func (s *SimulationTestSuite) TestAppStateDeterminism() { - t := s.T() - - encoding := app.MakeEncodingConfig() - +func TestAppStateDeterminism(t *testing.T) { config := simapp.NewConfigFromFlags() config.InitialBlockHeight = 1 config.ExportParamsPath = "" @@ -126,7 +103,11 @@ func (s *SimulationTestSuite) TestAppStateDeterminism() { for j := 0; j < numTimesToRunPerSeed; j++ { db := dbm.NewMemDB() - app := testapp.NewNibiruTestApp(app.NewDefaultGenesisState(encoding.Marshaler)) + logger := log.NewNopLogger() + encoding := app.MakeEncodingConfig() + + app := app.NewNibiruApp(logger, db, nil, true, encoding, sims.EmptyAppOptions{}, baseapp.SetChainID(SimAppChainID)) + appCodec := app.AppCodec() fmt.Printf( "running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n", @@ -137,12 +118,12 @@ func (s *SimulationTestSuite) TestAppStateDeterminism() { t, os.Stdout, app.BaseApp, - AppStateFn(app.AppCodec(), app.SimulationManager()), + AppStateFn(appCodec, app.SimulationManager()), simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - helpers.SimulationOperations(app, app.AppCodec(), config), + helpers.SimulationOperations(app, appCodec, config), app.ModuleAccountAddrs(), config, - app.AppCodec(), + appCodec, ) require.NoError(t, err) diff --git a/x/common/testutil/testapp/testapp.go b/x/common/testutil/testapp/testapp.go index 20425a570..3d5f1f989 100644 --- a/x/common/testutil/testapp/testapp.go +++ b/x/common/testutil/testapp/testapp.go @@ -8,6 +8,7 @@ import ( abci "github.com/cometbft/cometbft/abci/types" "github.com/cometbft/cometbft/libs/log" tmproto "github.com/cometbft/cometbft/proto/tendermint/types" + "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/testutil/sims" sdk "github.com/cosmos/cosmos-sdk/types" bankkeeper "github.com/cosmos/cosmos-sdk/x/bank/keeper" @@ -103,7 +104,7 @@ func NewNibiruTestAppAndContextAtTime(startTime time.Time) (*app.NibiruApp, sdk. // NewNibiruTestApp initializes a chain with the given genesis state to // creates an application instance ('app.NibiruApp'). This app uses an // in-memory database ('tmdb.MemDB') and has logging disabled. -func NewNibiruTestApp(gen app.GenesisState) *app.NibiruApp { +func NewNibiruTestApp(gen app.GenesisState, baseAppOptions ...func(*baseapp.BaseApp)) *app.NibiruApp { db := tmdb.NewMemDB() logger := log.NewNopLogger() @@ -117,6 +118,7 @@ func NewNibiruTestApp(gen app.GenesisState) *app.NibiruApp { /*loadLatest=*/ true, encoding, /*appOpts=*/ sims.EmptyAppOptions{}, + baseAppOptions..., ) gen, err := GenesisStateWithSingleValidator(encoding.Marshaler, gen) diff --git a/x/sudo/module.go b/x/sudo/module.go index 72f35fbae..1e0031ad1 100644 --- a/x/sudo/module.go +++ b/x/sudo/module.go @@ -11,18 +11,21 @@ import ( codectypes "github.com/cosmos/cosmos-sdk/codec/types" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/module" + simtypes "github.com/cosmos/cosmos-sdk/types/simulation" "github.com/grpc-ecosystem/grpc-gateway/runtime" "github.com/spf13/cobra" "github.com/NibiruChain/nibiru/x/sudo/cli" sudokeeper "github.com/NibiruChain/nibiru/x/sudo/keeper" + simulation "github.com/NibiruChain/nibiru/x/sudo/simulation" "github.com/NibiruChain/nibiru/x/sudo/types" ) // Ensure the interface is properly implemented at compile time var ( - _ module.AppModule = AppModule{} - _ module.AppModuleBasic = AppModuleBasic{} + _ module.AppModule = AppModule{} + _ module.AppModuleBasic = AppModuleBasic{} + _ module.AppModuleSimulation = AppModule{} ) // ---------------------------------------------------------------------------- @@ -151,3 +154,21 @@ func (am AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} func (am AppModule) EndBlock(_ sdk.Context, _ abci.RequestEndBlock) []abci.ValidatorUpdate { return []abci.ValidatorUpdate{} } + +//---------------------------------------------------------------------------- +// AppModuleSimulation functions +//---------------------------------------------------------------------------- + +// GenerateGenesisState implements module.AppModuleSimulation. +func (AppModule) GenerateGenesisState(simState *module.SimulationState) { + simulation.RandomizedGenState(simState) +} + +// RegisterStoreDecoder implements module.AppModuleSimulation. +func (AppModule) RegisterStoreDecoder(sdk.StoreDecoderRegistry) { +} + +// WeightedOperations implements module.AppModuleSimulation. +func (AppModule) WeightedOperations(simState module.SimulationState) []simtypes.WeightedOperation { + return nil +} diff --git a/x/sudo/simulation/genesis.go b/x/sudo/simulation/genesis.go new file mode 100644 index 000000000..89a863582 --- /dev/null +++ b/x/sudo/simulation/genesis.go @@ -0,0 +1,48 @@ +package simulation + +// DONTCOVER + +import ( + "encoding/json" + "fmt" + "math/rand" + + "cosmossdk.io/math" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/module" + + "github.com/NibiruChain/nibiru/x/sudo/types" +) + +// Simulation parameter constants +const ( + CommunityTax = "community_tax" + WithdrawEnabled = "withdraw_enabled" +) + +// GenCommunityTax randomized CommunityTax +func GenCommunityTax(r *rand.Rand) math.LegacyDec { + return sdk.NewDecWithPrec(1, 2).Add(sdk.NewDecWithPrec(int64(r.Intn(30)), 2)) +} + +// GenWithdrawEnabled returns a randomized WithdrawEnabled parameter. +func GenWithdrawEnabled(r *rand.Rand) bool { + return r.Int63n(101) <= 95 // 95% chance of withdraws being enabled +} + +// RandomizedGenState generates a random GenesisState for distribution +func RandomizedGenState(simState *module.SimulationState) { + genState := types.GenesisState{ + Sudoers: types.Sudoers{ + Root: simState.Accounts[0].Address.String(), + Contracts: []string{}, + }, + } + + bz, err := json.MarshalIndent(&genState, "", " ") + if err != nil { + panic(err) + } + fmt.Printf("Selected randomly generated x/sudo parameters:\n%s\n", bz) + simState.GenState[types.ModuleName] = simState.Cdc.MustMarshalJSON(&genState) +} From b4db887e6aafb2b7d53266bd9661327ae883e780 Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Wed, 27 Dec 2023 23:10:01 -0500 Subject: [PATCH 02/13] fix: CLI flags and app params --- contrib/make/simulation.mk | 4 ++-- simapp/params.json | 4 ++++ simapp/sim_test.go | 31 ++++++++++++++++--------------- simapp/state_test.go | 1 + x/sudo/module.go | 13 +++++++++++++ 5 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 simapp/params.json diff --git a/contrib/make/simulation.mk b/contrib/make/simulation.mk index 7461c93b9..ff43b7104 100644 --- a/contrib/make/simulation.mk +++ b/contrib/make/simulation.mk @@ -11,8 +11,8 @@ $(RUNSIM): .PHONY: test-sim-nondeterminism test-sim-nondeterminism: @echo "Running non-determinism test..." - go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true \ - -NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h + go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true -Params=params.json \ + -NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -Verbose=true -timeout 24h .PHONY: test-sim-default-genesis-fast test-sim-default-genesis-fast: diff --git a/simapp/params.json b/simapp/params.json new file mode 100644 index 000000000..67f8d113c --- /dev/null +++ b/simapp/params.json @@ -0,0 +1,4 @@ +{ + "op_weight_msg_create_validator": 0, + "op_weight_msg_edit_validator": 0 +} \ No newline at end of file diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 09fcf9dcf..909773ac9 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -10,17 +10,14 @@ import ( dbm "github.com/cometbft/cometbft-db" "github.com/cometbft/cometbft/libs/log" "github.com/cosmos/cosmos-sdk/baseapp" - "github.com/cosmos/cosmos-sdk/testutil/sims" - helpers "github.com/cosmos/cosmos-sdk/testutil/sims" + simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" simulationtypes "github.com/cosmos/cosmos-sdk/types/simulation" "github.com/cosmos/cosmos-sdk/x/simulation" simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" - "github.com/cosmos/ibc-go/v7/testing/simapp" "github.com/stretchr/testify/require" "github.com/NibiruChain/nibiru/app" - appsim "github.com/NibiruChain/nibiru/app/sim" "github.com/NibiruChain/nibiru/x/common/testutil/testapp" ) @@ -29,15 +26,15 @@ const SimAppChainID = "simulation-app" func init() { // We call GetSimulatorFlags here in order to set the value for - // 'simapp.FlagEnabledValue', which enables simulations - appsim.GetSimulatorFlags() + // 'simcli.FlagEnabledValue', which enables simulations + simcli.GetSimulatorFlags() } func TestFullAppSimulation(t *testing.T) { config := simcli.NewConfigFromFlags() config.ChainID = SimAppChainID - db, dir, _, skip, err := helpers.SetupSimulation( + db, dir, _, skip, err := simtestutil.SetupSimulation( config, "goleveldb-app-sim", "Simulation", @@ -66,14 +63,14 @@ func TestFullAppSimulation(t *testing.T) { /* app */ app.BaseApp, /* appStateFn */ AppStateFn(app.AppCodec(), app.SimulationManager()), /* randAccFn */ simulationtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - /* ops */ helpers.SimulationOperations(app, app.AppCodec(), config), // Run all registered operations + /* ops */ simtestutil.SimulationOperations(app, app.AppCodec(), config), // Run all registered operations /* blockedAddrs */ app.ModuleAccountAddrs(), /* config */ config, /* cdc */ app.AppCodec(), ) // export state and simParams before the simulation error is checked - if err = helpers.CheckExportSimulation(app, config, simParams); err != nil { + if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil { t.Fatal(err) } @@ -82,18 +79,22 @@ func TestFullAppSimulation(t *testing.T) { } if config.Commit { - simapp.PrintStats(db) + simtestutil.PrintStats(db) } } func TestAppStateDeterminism(t *testing.T) { - config := simapp.NewConfigFromFlags() + // if !simcli.FlagEnabledValue { + // t.Skip("skipping application simulation") + // } + + config := simcli.NewConfigFromFlags() config.InitialBlockHeight = 1 config.ExportParamsPath = "" config.OnOperation = false config.AllInvariants = false config.ChainID = SimAppChainID - + fmt.Println(config) numSeeds := 3 numTimesToRunPerSeed := 5 appHashList := make([]json.RawMessage, numTimesToRunPerSeed) @@ -106,7 +107,7 @@ func TestAppStateDeterminism(t *testing.T) { logger := log.NewNopLogger() encoding := app.MakeEncodingConfig() - app := app.NewNibiruApp(logger, db, nil, true, encoding, sims.EmptyAppOptions{}, baseapp.SetChainID(SimAppChainID)) + app := app.NewNibiruApp(logger, db, nil, true, encoding, simtestutil.EmptyAppOptions{}, baseapp.SetChainID(SimAppChainID)) appCodec := app.AppCodec() fmt.Printf( @@ -120,7 +121,7 @@ func TestAppStateDeterminism(t *testing.T) { app.BaseApp, AppStateFn(appCodec, app.SimulationManager()), simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - helpers.SimulationOperations(app, appCodec, config), + simtestutil.SimulationOperations(app, appCodec, config), app.ModuleAccountAddrs(), config, appCodec, @@ -128,7 +129,7 @@ func TestAppStateDeterminism(t *testing.T) { require.NoError(t, err) if config.Commit { - simapp.PrintStats(db) + simtestutil.PrintStats(db) } appHash := app.LastCommitID().Hash diff --git a/simapp/state_test.go b/simapp/state_test.go index 869e400c0..948dbc68e 100644 --- a/simapp/state_test.go +++ b/simapp/state_test.go @@ -64,6 +64,7 @@ func AppStateFn(cdc codec.JSONCodec, simManager *module.SimulationManager) simty if err != nil { panic(err) } + appState, simAccs = AppStateRandomizedFn(simManager, r, cdc, accs, genesisTimestamp, appParams) default: diff --git a/x/sudo/module.go b/x/sudo/module.go index 1e0031ad1..fd398c115 100644 --- a/x/sudo/module.go +++ b/x/sudo/module.go @@ -170,5 +170,18 @@ func (AppModule) RegisterStoreDecoder(sdk.StoreDecoderRegistry) { // WeightedOperations implements module.AppModuleSimulation. func (AppModule) WeightedOperations(simState module.SimulationState) []simtypes.WeightedOperation { + appParams := simState.AppParams + + fmt.Println("Map:", fmt.Sprintf("%v", appParams)) + for k, v := range appParams { + var val int + fmt.Printf("Key: %s, Value: %d\n", k, v) + err := json.Unmarshal(v, &val) + if err != nil { + panic(err) + } + fmt.Printf("Key: %s, Value: %d\n", k, val) + } + return nil } From 0d3c85b91c8f878a2d81882a31ce2211eb845d8c Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 07:51:27 -0500 Subject: [PATCH 03/13] refactor: clean up TestAppStateDeterminism --- simapp/sim_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 909773ac9..0f3bab5dd 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -10,6 +10,8 @@ import ( dbm "github.com/cometbft/cometbft-db" "github.com/cometbft/cometbft/libs/log" "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" simulationtypes "github.com/cosmos/cosmos-sdk/types/simulation" @@ -83,10 +85,11 @@ func TestFullAppSimulation(t *testing.T) { } } +// Tests that the app state hash is deterministic when the operations are run func TestAppStateDeterminism(t *testing.T) { - // if !simcli.FlagEnabledValue { - // t.Skip("skipping application simulation") - // } + if !simcli.FlagEnabledValue { + t.Skip("skipping application simulation") + } config := simcli.NewConfigFromFlags() config.InitialBlockHeight = 1 @@ -94,10 +97,13 @@ func TestAppStateDeterminism(t *testing.T) { config.OnOperation = false config.AllInvariants = false config.ChainID = SimAppChainID - fmt.Println(config) numSeeds := 3 numTimesToRunPerSeed := 5 + appHashList := make([]json.RawMessage, numTimesToRunPerSeed) + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = app.DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue for i := 0; i < numSeeds; i++ { config.Seed = rand.Int63() @@ -107,7 +113,7 @@ func TestAppStateDeterminism(t *testing.T) { logger := log.NewNopLogger() encoding := app.MakeEncodingConfig() - app := app.NewNibiruApp(logger, db, nil, true, encoding, simtestutil.EmptyAppOptions{}, baseapp.SetChainID(SimAppChainID)) + app := app.NewNibiruApp(logger, db, nil, true, encoding, appOptions, baseapp.SetChainID(SimAppChainID)) appCodec := app.AppCodec() fmt.Printf( @@ -120,7 +126,7 @@ func TestAppStateDeterminism(t *testing.T) { os.Stdout, app.BaseApp, AppStateFn(appCodec, app.SimulationManager()), - simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtypes.RandomAccounts, simtestutil.SimulationOperations(app, appCodec, config), app.ModuleAccountAddrs(), config, From 5bfbeedcd0f5c0870554fe1e9e51632552006af1 Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 08:12:15 -0500 Subject: [PATCH 04/13] test(sim): fix TestFullAppSimulation --- contrib/make/simulation.mk | 22 +++++++++++++--- simapp/sim_test.go | 54 ++++++++++++++++---------------------- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/contrib/make/simulation.mk b/contrib/make/simulation.mk index ff43b7104..de4d141f0 100644 --- a/contrib/make/simulation.mk +++ b/contrib/make/simulation.mk @@ -11,14 +11,28 @@ $(RUNSIM): .PHONY: test-sim-nondeterminism test-sim-nondeterminism: @echo "Running non-determinism test..." - go test -mod=readonly $(SIMAPP) -run TestAppStateDeterminism -Enabled=true -Params=params.json \ - -NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -Verbose=true -timeout 24h + @go test -mod=readonly -v $(SIMAPP) \ + -run TestAppStateDeterminism \ + -Enabled=true \ + -Params=params.json \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Period=0 \ + -Verbose=true .PHONY: test-sim-default-genesis-fast test-sim-default-genesis-fast: @echo "Running default genesis simulation..." - @go test -mod=readonly $(SIMAPP) -run TestFullAppSimulation \ - -Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v + @go test -mod=readonly -v $(SIMAPP) \ + -run TestFullAppSimulation \ + -Params=params.json \ + -Enabled=true \ + -NumBlocks=100 \ + -BlockSize=200 \ + -Commit=true \ + -Seed=99 \ + -Period=0 .PHONY: test-sim-custom-genesis-multi-seed test-sim-custom-genesis-multi-seed: runsim diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 0f3bab5dd..3d4113635 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -14,13 +14,11 @@ import ( "github.com/cosmos/cosmos-sdk/server" simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" - simulationtypes "github.com/cosmos/cosmos-sdk/types/simulation" "github.com/cosmos/cosmos-sdk/x/simulation" simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" "github.com/stretchr/testify/require" "github.com/NibiruChain/nibiru/app" - "github.com/NibiruChain/nibiru/x/common/testutil/testapp" ) // SimAppChainID hardcoded chainID for simulation @@ -36,49 +34,43 @@ func TestFullAppSimulation(t *testing.T) { config := simcli.NewConfigFromFlags() config.ChainID = SimAppChainID - db, dir, _, skip, err := simtestutil.SetupSimulation( - config, - "goleveldb-app-sim", - "Simulation", - simcli.FlagVerboseValue, simcli.FlagEnabledValue, - ) + db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) if skip { t.Skip("skipping application simulation") } require.NoError(t, err, "simulation setup failed") defer func() { - db.Close() - err = os.RemoveAll(dir) - if err != nil { - t.Fatal(err) - } + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) }() + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = app.DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + encoding := app.MakeEncodingConfig() - app := testapp.NewNibiruTestApp(app.NewDefaultGenesisState(encoding.Marshaler)) + app := app.NewNibiruApp(logger, db, nil, true, encoding, appOptions, baseapp.SetChainID(SimAppChainID)) + require.Equal(t, "Nibiru", app.Name()) + appCodec := app.AppCodec() - // Run randomized simulation: + // run randomized simulation _, simParams, simErr := simulation.SimulateFromSeed( - /* tb */ t, - /* w */ os.Stdout, - /* app */ app.BaseApp, - /* appStateFn */ AppStateFn(app.AppCodec(), app.SimulationManager()), - /* randAccFn */ simulationtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 - /* ops */ simtestutil.SimulationOperations(app, app.AppCodec(), config), // Run all registered operations - /* blockedAddrs */ app.ModuleAccountAddrs(), - /* config */ config, - /* cdc */ app.AppCodec(), + t, + os.Stdout, + app.BaseApp, + AppStateFn(appCodec, app.SimulationManager()), + simtypes.RandomAccounts, + simtestutil.SimulationOperations(app, appCodec, config), + app.ModuleAccountAddrs(), + config, + appCodec, ) // export state and simParams before the simulation error is checked - if err = simtestutil.CheckExportSimulation(app, config, simParams); err != nil { - t.Fatal(err) - } - - if simErr != nil { - t.Fatal(simErr) - } + err = simtestutil.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) if config.Commit { simtestutil.PrintStats(db) From d2b1adcc7b0d90d64680c45e1494fa381f7e99a7 Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 08:59:31 -0500 Subject: [PATCH 05/13] refactor: make ModuleManager public --- app/app.go | 10 +++++----- app/export.go | 2 +- app/keepers.go | 16 ++++++++-------- app/upgrades.go | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/app.go b/app/app.go index 346c8eda7..943ecc4d8 100644 --- a/app/app.go +++ b/app/app.go @@ -90,7 +90,7 @@ type NibiruApp struct { AppKeepers // embed all module keepers // the module manager - mm *module.Manager + ModuleManager *module.Manager // simulation manager sm *module.SimulationManager @@ -251,12 +251,12 @@ func (app *NibiruApp) Name() string { return app.BaseApp.Name() } // BeginBlocker application updates every begin block func (app *NibiruApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock { - return app.mm.BeginBlock(ctx, req) + return app.ModuleManager.BeginBlock(ctx, req) } // EndBlocker application updates every end block func (app *NibiruApp) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock { - return app.mm.EndBlock(ctx, req) + return app.ModuleManager.EndBlock(ctx, req) } // InitChainer application update at chain initialization @@ -265,8 +265,8 @@ func (app *NibiruApp) InitChainer(ctx sdk.Context, req abci.RequestInitChain) ab if err := json.Unmarshal(req.AppStateBytes, &genesisState); err != nil { panic(err) } - app.upgradeKeeper.SetModuleVersionMap(ctx, app.mm.GetVersionMap()) - return app.mm.InitGenesis(ctx, app.appCodec, genesisState) + app.upgradeKeeper.SetModuleVersionMap(ctx, app.ModuleManager.GetVersionMap()) + return app.ModuleManager.InitGenesis(ctx, app.appCodec, genesisState) } // LoadHeight loads a particular height diff --git a/app/export.go b/app/export.go index 37fe4efd5..7a9a5ed6a 100644 --- a/app/export.go +++ b/app/export.go @@ -29,7 +29,7 @@ func (app *NibiruApp) ExportAppStateAndValidators( app.prepForZeroHeightGenesis(ctx, jailAllowedAddrs) } - genState := app.mm.ExportGenesisForModules(ctx, app.appCodec, modulesToExport) + genState := app.ModuleManager.ExportGenesisForModules(ctx, app.appCodec, modulesToExport) appState, err := json.MarshalIndent(genState, "", " ") if err != nil { return servertypes.ExportedApp{}, err diff --git a/app/keepers.go b/app/keepers.go index 40361da70..e252bc554 100644 --- a/app/keepers.go +++ b/app/keepers.go @@ -705,23 +705,23 @@ func (app *NibiruApp) initModuleManager( encodingConfig EncodingConfig, skipGenesisInvariants bool, ) { - app.mm = module.NewManager( + app.ModuleManager = module.NewManager( app.initAppModules(encodingConfig, skipGenesisInvariants)..., ) orderedModules := orderedModuleNames() - app.mm.SetOrderBeginBlockers(orderedModules...) - app.mm.SetOrderEndBlockers(orderedModules...) - app.mm.SetOrderInitGenesis(orderedModules...) - app.mm.SetOrderExportGenesis(orderedModules...) + app.ModuleManager.SetOrderBeginBlockers(orderedModules...) + app.ModuleManager.SetOrderEndBlockers(orderedModules...) + app.ModuleManager.SetOrderInitGenesis(orderedModules...) + app.ModuleManager.SetOrderExportGenesis(orderedModules...) // Uncomment if you want to set a custom migration order here. // app.mm.SetOrderMigrations(custom order) - app.mm.RegisterInvariants(&app.crisisKeeper) + app.ModuleManager.RegisterInvariants(&app.crisisKeeper) app.configurator = module.NewConfigurator( app.appCodec, app.MsgServiceRouter(), app.GRPCQueryRouter()) - app.mm.RegisterServices(app.configurator) + app.ModuleManager.RegisterServices(app.configurator) // see https://github.com/cosmos/cosmos-sdk/blob/666c345ad23ddda9523cc5cd1b71187d91c26f34/simapp/upgrades.go#L35-L57 for _, subspace := range app.paramsKeeper.GetSubspaces() { @@ -853,7 +853,7 @@ func (app *NibiruApp) initSimulationManager( overrideModules := map[string]module.AppModuleSimulation{ authtypes.ModuleName: auth.NewAppModule(app.appCodec, app.AccountKeeper, authsims.RandomGenesisAccounts, app.GetSubspace(authtypes.ModuleName)), } - app.sm = module.NewSimulationManagerFromAppModules(app.mm.Modules, overrideModules) + app.sm = module.NewSimulationManagerFromAppModules(app.ModuleManager.Modules, overrideModules) app.sm.RegisterStoreDecoders() } diff --git a/app/upgrades.go b/app/upgrades.go index 8487b5f22..26093c521 100644 --- a/app/upgrades.go +++ b/app/upgrades.go @@ -20,7 +20,7 @@ func (app *NibiruApp) setupUpgrades() { func (app *NibiruApp) setUpgradeHandlers() { for _, u := range Upgrades { - app.upgradeKeeper.SetUpgradeHandler(u.UpgradeName, u.CreateUpgradeHandler(app.mm, app.configurator)) + app.upgradeKeeper.SetUpgradeHandler(u.UpgradeName, u.CreateUpgradeHandler(app.ModuleManager, app.configurator)) } } From 65198cbe0655e4564abc1cbee85eb1d92483cdd1 Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:00:52 -0500 Subject: [PATCH 06/13] test(sim): add TestAppImportExport --- contrib/make/simulation.mk | 33 +++----- simapp/sim_test.go | 156 +++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 22 deletions(-) diff --git a/contrib/make/simulation.mk b/contrib/make/simulation.mk index de4d141f0..1eb0d78ad 100644 --- a/contrib/make/simulation.mk +++ b/contrib/make/simulation.mk @@ -1,13 +1,5 @@ -BINDIR = $(GOPATH)/bin -RUNSIM = $(BINDIR)/runsim SIMAPP = ./simapp -.PHONY: runsim -runsim: $(RUNSIM) -$(RUNSIM): - @echo "Installing runsim..." - @(cd /tmp && go install github.com/cosmos/tools/cmd/runsim@v1.0.0) - .PHONY: test-sim-nondeterminism test-sim-nondeterminism: @echo "Running non-determinism test..." @@ -34,20 +26,17 @@ test-sim-default-genesis-fast: -Seed=99 \ -Period=0 -.PHONY: test-sim-custom-genesis-multi-seed -test-sim-custom-genesis-multi-seed: runsim - @echo "Running multi-seed custom genesis simulation..." - @$(RUNSIM) -SimAppPkg=$(SIMAPP) -ExitOnFail 400 5 TestFullAppSimulation - -.PHONY: test-sim-multi-seed-long -test-sim-multi-seed-long: runsim - @echo "Running long multi-seed application simulation. This may take awhile!" - @$(RUNSIM) -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 500 50 TestFullAppSimulation - -.PHONY: test-sim-multi-seed-short -test-sim-multi-seed-short: runsim - @echo "Running short multi-seed application simulation. This may take awhile!" - @$(RUNSIM) -Jobs=4 -SimAppPkg=$(SIMAPP) -ExitOnFail 50 10 TestFullAppSimulation +.PHONY: test-sim-import-export +test-sim-import-export: + @echo "Running application import/export simulation. This may take several minutes..." + @go test -mod=readonly -v $(SIMAPP) \ + -run TestAppImportExport \ + -Params=params.json \ + -Enabled=true \ + -NumBlocks=100 \ + -Commit=true \ + -Seed=99 \ + -Period=0 .PHONY: test-sim-benchmark-invariants test-sim-benchmark-invariants: diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 3d4113635..147075f9e 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -5,20 +5,44 @@ import ( "fmt" "math/rand" "os" + "runtime/debug" + "strings" "testing" dbm "github.com/cometbft/cometbft-db" "github.com/cometbft/cometbft/libs/log" + tmproto "github.com/cometbft/cometbft/proto/tendermint/types" "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/server" + storetypes "github.com/cosmos/cosmos-sdk/store/types" simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" + sdk "github.com/cosmos/cosmos-sdk/types" simtypes "github.com/cosmos/cosmos-sdk/types/simulation" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + authzkeeper "github.com/cosmos/cosmos-sdk/x/authz/keeper" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" + capabilitytypes "github.com/cosmos/cosmos-sdk/x/capability/types" + distrtypes "github.com/cosmos/cosmos-sdk/x/distribution/types" + evidencetypes "github.com/cosmos/cosmos-sdk/x/evidence/types" + govtypes "github.com/cosmos/cosmos-sdk/x/gov/types" + minttypes "github.com/cosmos/cosmos-sdk/x/mint/types" + paramtypes "github.com/cosmos/cosmos-sdk/x/params/types" "github.com/cosmos/cosmos-sdk/x/simulation" simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli" + slashingtypes "github.com/cosmos/cosmos-sdk/x/slashing/types" + stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" "github.com/stretchr/testify/require" "github.com/NibiruChain/nibiru/app" + devgastypes "github.com/NibiruChain/nibiru/x/devgas/v1/types" + epochstypes "github.com/NibiruChain/nibiru/x/epochs/types" + inflationtypes "github.com/NibiruChain/nibiru/x/inflation/types" + oracletypes "github.com/NibiruChain/nibiru/x/oracle/types" + perptypes "github.com/NibiruChain/nibiru/x/perp/v2/types" + spottypes "github.com/NibiruChain/nibiru/x/spot/types" + sudotypes "github.com/NibiruChain/nibiru/x/sudo/types" + tokenfactorytypes "github.com/NibiruChain/nibiru/x/tokenfactory/types" ) // SimAppChainID hardcoded chainID for simulation @@ -30,6 +54,12 @@ func init() { simcli.GetSimulatorFlags() } +type StoreKeysPrefixes struct { + A storetypes.StoreKey + B storetypes.StoreKey + Prefixes [][]byte +} + func TestFullAppSimulation(t *testing.T) { config := simcli.NewConfigFromFlags() config.ChainID = SimAppChainID @@ -142,3 +172,129 @@ func TestAppStateDeterminism(t *testing.T) { } } } + +func TestAppImportExport(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = SimAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + if skip { + t.Skip("skipping application import/export simulation") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = app.DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + encoding := app.MakeEncodingConfig() + oldApp := app.NewNibiruApp(logger, db, nil, true, encoding, appOptions, baseapp.SetChainID(SimAppChainID)) + require.Equal(t, "Nibiru", oldApp.Name()) + appCodec := oldApp.AppCodec() + + // Run randomized simulation + _, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + oldApp.BaseApp, + AppStateFn(appCodec, oldApp.SimulationManager()), + simtypes.RandomAccounts, + simtestutil.SimulationOperations(oldApp, oldApp.AppCodec(), config), + oldApp.ModuleAccountAddrs(), + config, + oldApp.AppCodec(), + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(oldApp, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } + + fmt.Printf("exporting genesis...\n") + + exported, err := oldApp.ExportAppStateAndValidators(false, []string{}, []string{}) + require.NoError(t, err) + + fmt.Printf("importing genesis...\n") + + newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, newDB.Close()) + require.NoError(t, os.RemoveAll(newDir)) + }() + + newApp := app.NewNibiruApp(log.NewNopLogger(), newDB, nil, true, encoding, appOptions, baseapp.SetChainID(SimAppChainID)) + require.Equal(t, "Nibiru", newApp.Name()) + + var genesisState app.GenesisState + err = json.Unmarshal(exported.AppState, &genesisState) + require.NoError(t, err) + + defer func() { + if r := recover(); r != nil { + err := fmt.Sprintf("%v", r) + if !strings.Contains(err, "validator set is empty after InitGenesis") { + panic(r) + } + logger.Info("Skipping simulation as all validators have been unbonded") + logger.Info("err", err, "stacktrace", string(debug.Stack())) + } + }() + + ctxA := oldApp.NewContext(true, tmproto.Header{Height: oldApp.LastBlockHeight()}) + ctxB := newApp.NewContext(true, tmproto.Header{Height: oldApp.LastBlockHeight()}) + newApp.ModuleManager.InitGenesis(ctxB, oldApp.AppCodec(), genesisState) + newApp.StoreConsensusParams(ctxB, exported.ConsensusParams) + + fmt.Printf("comparing stores...\n") + + storeKeysPrefixes := []StoreKeysPrefixes{ + {oldApp.GetKey(authtypes.StoreKey), newApp.GetKey(authtypes.StoreKey), [][]byte{}}, + { + oldApp.GetKey(stakingtypes.StoreKey), newApp.GetKey(stakingtypes.StoreKey), + [][]byte{ + stakingtypes.UnbondingQueueKey, stakingtypes.RedelegationQueueKey, stakingtypes.ValidatorQueueKey, + stakingtypes.HistoricalInfoKey, stakingtypes.UnbondingIDKey, stakingtypes.UnbondingIndexKey, stakingtypes.UnbondingTypeKey, stakingtypes.ValidatorUpdatesKey, + }, + }, // ordering may change but it doesn't matter + {oldApp.GetKey(slashingtypes.StoreKey), newApp.GetKey(slashingtypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(minttypes.StoreKey), newApp.GetKey(minttypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(distrtypes.StoreKey), newApp.GetKey(distrtypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(banktypes.StoreKey), newApp.GetKey(banktypes.StoreKey), [][]byte{banktypes.BalancesPrefix}}, + {oldApp.GetKey(paramtypes.StoreKey), newApp.GetKey(paramtypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(govtypes.StoreKey), newApp.GetKey(govtypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(evidencetypes.StoreKey), newApp.GetKey(evidencetypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(capabilitytypes.StoreKey), newApp.GetKey(capabilitytypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(authzkeeper.StoreKey), newApp.GetKey(authzkeeper.StoreKey), [][]byte{authzkeeper.GrantKey, authzkeeper.GrantQueuePrefix}}, + {oldApp.GetKey(devgastypes.StoreKey), newApp.GetKey(devgastypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(epochstypes.StoreKey), newApp.GetKey(epochstypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(inflationtypes.StoreKey), newApp.GetKey(inflationtypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(oracletypes.StoreKey), newApp.GetKey(oracletypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(perptypes.StoreKey), newApp.GetKey(perptypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(spottypes.StoreKey), newApp.GetKey(spottypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(sudotypes.StoreKey), newApp.GetKey(sudotypes.StoreKey), [][]byte{}}, + {oldApp.GetKey(tokenfactorytypes.StoreKey), newApp.GetKey(tokenfactorytypes.StoreKey), [][]byte{}}, + } + + for _, skp := range storeKeysPrefixes { + storeA := ctxA.KVStore(skp.A) + storeB := ctxB.KVStore(skp.B) + + failedKVAs, failedKVBs := sdk.DiffKVStores(storeA, storeB, skp.Prefixes) + require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare") + + fmt.Printf("compared %d different key/value pairs between %s and %s\n", len(failedKVAs), skp.A, skp.B) + require.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(skp.A.Name(), oldApp.SimulationManager().StoreDecoders, failedKVAs, failedKVBs)) + } +} From 2e6581f926c6bbbf4a46cf66b47e2c0d6e8fc5d3 Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:17:54 -0500 Subject: [PATCH 07/13] test(sim): add TestAppSimulationAfterImport --- contrib/make/simulation.mk | 19 +++++--- simapp/sim_test.go | 89 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 7 deletions(-) diff --git a/contrib/make/simulation.mk b/contrib/make/simulation.mk index 1eb0d78ad..20eb2151c 100644 --- a/contrib/make/simulation.mk +++ b/contrib/make/simulation.mk @@ -36,11 +36,16 @@ test-sim-import-export: -NumBlocks=100 \ -Commit=true \ -Seed=99 \ - -Period=0 + -Period=5 -.PHONY: test-sim-benchmark-invariants -test-sim-benchmark-invariants: - @echo "Running simulation invariant benchmarks..." - @go test -mod=readonly $(SIMAPP) -benchmem -bench=BenchmarkInvariants -run=^$ \ - -Enabled=true -NumBlocks=1000 -BlockSize=200 \ - -Period=1 -Commit=true -Seed=57 -v -timeout 24h \ No newline at end of file +.PHONY: test-sim-after-import +test-sim-after-import: + @echo "Running application simulation-after-import. This may take several minutes..." + @go test -mod=readonly -v $(SIMAPP) \ + -run TestAppSimulationAfterImport \ + -Params=params.json \ + -Enabled=true \ + -NumBlocks=50 \ + -Commit=true \ + -Seed=99 \ + -Period=5 diff --git a/simapp/sim_test.go b/simapp/sim_test.go index 147075f9e..a1e1bc4ce 100644 --- a/simapp/sim_test.go +++ b/simapp/sim_test.go @@ -10,6 +10,7 @@ import ( "testing" dbm "github.com/cometbft/cometbft-db" + abci "github.com/cometbft/cometbft/abci/types" "github.com/cometbft/cometbft/libs/log" tmproto "github.com/cometbft/cometbft/proto/tendermint/types" "github.com/cosmos/cosmos-sdk/baseapp" @@ -298,3 +299,91 @@ func TestAppImportExport(t *testing.T) { require.Equal(t, 0, len(failedKVAs), simtestutil.GetSimulationLog(skp.A.Name(), oldApp.SimulationManager().StoreDecoders, failedKVAs, failedKVBs)) } } + +func TestAppSimulationAfterImport(t *testing.T) { + config := simcli.NewConfigFromFlags() + config.ChainID = SimAppChainID + + db, dir, logger, skip, err := simtestutil.SetupSimulation(config, "goleveldb-app-sim", "Simulation", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + if skip { + t.Skip("skipping application simulation after import") + } + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, db.Close()) + require.NoError(t, os.RemoveAll(dir)) + }() + + appOptions := make(simtestutil.AppOptionsMap, 0) + appOptions[flags.FlagHome] = app.DefaultNodeHome + appOptions[server.FlagInvCheckPeriod] = simcli.FlagPeriodValue + + encoding := app.MakeEncodingConfig() + oldApp := app.NewNibiruApp(logger, db, nil, true, encoding, appOptions, baseapp.SetChainID(SimAppChainID)) + require.Equal(t, "Nibiru", oldApp.Name()) + appCodec := oldApp.AppCodec() + + // Run randomized simulation + stopEarly, simParams, simErr := simulation.SimulateFromSeed( + t, + os.Stdout, + oldApp.BaseApp, + AppStateFn(appCodec, oldApp.SimulationManager()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(oldApp, appCodec, config), + oldApp.ModuleAccountAddrs(), + config, + appCodec, + ) + + // export state and simParams before the simulation error is checked + err = simtestutil.CheckExportSimulation(oldApp, config, simParams) + require.NoError(t, err) + require.NoError(t, simErr) + + if config.Commit { + simtestutil.PrintStats(db) + } + + if stopEarly { + fmt.Println("can't export or import a zero-validator genesis, exiting test...") + return + } + + fmt.Printf("exporting genesis...\n") + + exported, err := oldApp.ExportAppStateAndValidators(true, []string{}, []string{}) + require.NoError(t, err) + + fmt.Printf("importing genesis...\n") + + newDB, newDir, _, _, err := simtestutil.SetupSimulation(config, "leveldb-app-sim-2", "Simulation-2", simcli.FlagVerboseValue, simcli.FlagEnabledValue) + require.NoError(t, err, "simulation setup failed") + + defer func() { + require.NoError(t, newDB.Close()) + require.NoError(t, os.RemoveAll(newDir)) + }() + + newApp := app.NewNibiruApp(log.NewNopLogger(), newDB, nil, true, encoding, appOptions, baseapp.SetChainID(SimAppChainID)) + require.Equal(t, "Nibiru", newApp.Name()) + + newApp.InitChain(abci.RequestInitChain{ + ChainId: SimAppChainID, + AppStateBytes: exported.AppState, + }) + + _, _, err = simulation.SimulateFromSeed( + t, + os.Stdout, + newApp.BaseApp, + AppStateFn(appCodec, newApp.SimulationManager()), + simtypes.RandomAccounts, // Replace with own random account function if using keys other than secp256k1 + simtestutil.SimulationOperations(newApp, newApp.AppCodec(), config), + newApp.ModuleAccountAddrs(), + config, + oldApp.AppCodec(), + ) + require.NoError(t, err) +} From af636d810f66469af907ef04477b12c09618fe41 Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:21:14 -0500 Subject: [PATCH 08/13] chore: update changelog --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00870819d..8055ef8a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,15 +75,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * [#1695](https://github.com/NibiruChain/nibiru/pull/1695) - feat(inflation): add events for inflation distribution * [#1695](https://github.com/NibiruChain/nibiru/pull/1695) - fix(sudo): Make blank sudoers root invalid at genesis time. * [#1710](https://github.com/NibiruChain/nibiru/pull/1710) - refactor(perp): Clean and organize module errors for x/perp -* [#1714](https://github.com/NibiruChain/nibiru/pull/1714) - ci(localnet.sh): Fix script, simplify, and test in CI. +* [#1714](https://github.com/NibiruChain/nibiru/pull/1714) - ci(localnet.sh): Fix script, simplify, and test in CI. * [#1719](https://github.com/NibiruChain/nibiru/pull/1719) - refactor(test): add is not mandatory interface to action * [#1723](https://github.com/NibiruChain/nibiru/pull/1723) - ci(e2e-wasm.yml): rm unused workflow +* [#1735](https://github.com/NibiruChain/nibiru/pull/1735) - test(sim): fix simulation tests ### Dependencies + - Bump `google.golang.org/grpc` from 1.59.0 to 1.60.0 ([#1720](https://github.com/NibiruChain/nibiru/pull/1720)) -- Bump `golang.org/x/crypto` from 0.15.0 to 0.17.0 ([#1724](https://github.com/NibiruChain/nibiru/pull/1724)) -- Bump `github.com/holiman/uint256` from 1.2.3 to 1.2.4 ([#1730](https://github.com/NibiruChain/nibiru/pull/1730)) -- Bump `github.com/dvsekhvalnov/jose2go` from 1.5.0 to 1.6.0 ([#1733](https://github.com/NibiruChain/nibiru/pull/1733)) +* Bump `golang.org/x/crypto` from 0.15.0 to 0.17.0 ([#1724](https://github.com/NibiruChain/nibiru/pull/1724)) +* Bump `github.com/holiman/uint256` from 1.2.3 to 1.2.4 ([#1730](https://github.com/NibiruChain/nibiru/pull/1730)) +* Bump `github.com/dvsekhvalnov/jose2go` from 1.5.0 to 1.6.0 ([#1733](https://github.com/NibiruChain/nibiru/pull/1733)) * Bump `github.com/spf13/cast` from 1.5.1 to 1.6.0 ([#1689](https://github.com/NibiruChain/nibiru/pull/1689)) * Bump `cosmossdk.io/math` from 1.1.2 to 1.2.0 ([#1676](https://github.com/NibiruChain/nibiru/pull/1676)) @@ -92,7 +94,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Bump `golang` from 1.19 to 1.21 ([#1698](https://github.com/NibiruChain/nibiru/pull/1698)) * [#1678](https://github.com/NibiruChain/nibiru/pull/1678) - chore(deps): collections to v0.4.0 for math.Int value encoder - ## [v1.1.0] - 2023-11-20 * [[Release Link](https://github.com/NibiruChain/nibiru/releases/tag/v1.1.0)] From 08066afc074bec22ef02cd0a7fab0d69cb4d15c3 Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:29:46 -0500 Subject: [PATCH 09/13] ci: add GHA workflow to run sim tests --- .github/workflows/sims.yml | 47 ------------------- .github/workflows/simulation-tests.yml | 63 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/sims.yml create mode 100644 .github/workflows/simulation-tests.yml diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml deleted file mode 100644 index 38dfbbb3b..000000000 --- a/.github/workflows/sims.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Run short simulations - -on: - pull_request: - paths: ["**.go", "**.proto", "go.mod", "go.sum"] - -jobs: - install-runsim: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: 1.21 - cache: true - - uses: actions/cache@v3 - with: - path: ~/go/bin - key: ${{ runner.os }}-go-runsim-binary - - name: Install runsim - run: go install github.com/cosmos/tools/cmd/runsim@v1.0.0 - - test-sim-nondeterminism: - runs-on: ubuntu-latest - needs: [install-runsim] - steps: - - uses: actions/checkout@v4 - - uses: technote-space/get-diff-action@v6 - with: - SUFFIX_FILTER: | - **/**.go - go.mod - go.sum - - uses: actions/setup-go@v5 - with: - go-version: 1.21 - cache: true - if: env.GIT_DIFF - - uses: actions/cache@v3 - with: - path: ~/go/bin - key: ${{ runner.os }}-go-runsim-binary - if: env.GIT_DIFF - - name: test-sim-nondeterminism - run: | - make test-sim-nondeterminism - if: env.GIT_DIFF diff --git a/.github/workflows/simulation-tests.yml b/.github/workflows/simulation-tests.yml new file mode 100644 index 000000000..c446cddb1 --- /dev/null +++ b/.github/workflows/simulation-tests.yml @@ -0,0 +1,63 @@ +name: Run simulations + +on: + push: + branches: + # every push to default branch + - main + schedule: + # once per day + - cron: "0 0 * * *" + pull_request: + branches: + # every pull request to default branch + - main + +jobs: + test-sim-nondeterminism: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: 1.21 + cache: true + - name: TestAppStateDeterminism + run: | + make test-sim-nondeterminism + + test-sim-default-genesis-fast: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: 1.21 + cache: true + - name: TestFullAppSimulation + run: | + make test-sim-default-genesis-fast + + test-sim-import-export: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: 1.21 + cache: true + - name: TestAppImportExport + run: | + make test-sim-import-export + + test-sim-after-import: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: 1.21 + cache: true + - name: TestAppSimulationAfterImport + run: | + make test-sim-after-import \ No newline at end of file From faa6c8d11808cbe67a821c34b7de4108b132c20d Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:30:36 -0500 Subject: [PATCH 10/13] ci: update workflow name --- .github/workflows/simulation-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/simulation-tests.yml b/.github/workflows/simulation-tests.yml index c446cddb1..ccd345815 100644 --- a/.github/workflows/simulation-tests.yml +++ b/.github/workflows/simulation-tests.yml @@ -1,4 +1,4 @@ -name: Run simulations +name: Simulation tests on: push: From 0d37801c5cb10c62d55c145e72199baa499e866a Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:38:16 -0500 Subject: [PATCH 11/13] docs: add simulation tests readme --- simapp/README.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 simapp/README.md diff --git a/simapp/README.md b/simapp/README.md new file mode 100644 index 000000000..edc956ea5 --- /dev/null +++ b/simapp/README.md @@ -0,0 +1,59 @@ +# Simulation Tests + +This directory contains the simulation tests for the `simapp` module + +## Test Cases + +### Non-Determinism + +```sh +make test-sim-non-determinism +``` + +This test case checks that the simulation is deterministic. It does so by +running the simulation twice with the same seed and comparing the resulting +state. If the simulation is deterministic, the resulting state should be the +same. + +### Full App + +```sh +make test-sim-default-genesis-fast +``` + +This test case runs the simulation with the default genesis file. It checks that +the simulation does not panic and that the resulting state is valid. + +### Import/Export + +```sh +make test-sim-import-export +``` + +This test case runs the simulation with the default genesis file. It checks that +the simulation does not panic and that the resulting state is valid. It then +exports the state to a file and imports it back. It checks that the imported +state is the same as the exported state. + +### Simulation After Import + +```sh +make test-sim-after-import +``` + +This test case runs the simulation with the default genesis file. It checks that +the simulation does not panic and that the resulting state is valid. It then +exports the state to a file and imports it back. It checks that the imported +state is the same as the exported state. It then runs the simulation again with +the imported state. It checks that the simulation does not panic and that the +resulting state is valid. + +## Params + +A `params.json` file is included that sets the operation weights for +`CreateValidator` and `EditValidator` to zero. It's a hack to make the +simulation tests pass. The random commission rates sometimes halt the simulation +because the max commission rate is set to 0.25 in the AnteHandler, but sometimes +the random commission rate is higher than that. The random commission is set by +the cosmos-sdk x/staking module simulation operations, so we have no control +over injecting a manual value. From e6e8131368669b6be85c4ac55c88ad46f243c49c Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:39:11 -0500 Subject: [PATCH 12/13] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8055ef8a8..e2f30950c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * [#1686](https://github.com/NibiruChain/nibiru/pull/1686) - test(perp): add more tests for perp module msg server for DnR * [#1683](https://github.com/NibiruChain/nibiru/pull/1683) - feat(perp): Add `StartDnREpoch` to `AfterEpochEnd` hook * [#1680](https://github.com/NibiruChain/nibiru/pull/1680) - feat(perp): MsgShiftPegMultiplier, MsgShiftSwapInvariant. -* [#1680](https://github.com/NibiruChain/nibiru/pull/1680) - feat(perp): MsgShiftPegMultiplier, MsgShiftSwapInvariant. * [#1677](https://github.com/NibiruChain/nibiru/pull/1677) - fix(perp): make Gen_market set initial perp versions * [#1669](https://github.com/NibiruChain/nibiru/pull/1669) - feat(perp): add query to get collateral metadata * [#1663](https://github.com/NibiruChain/nibiru/pull/1663) - feat(perp): Add volume based rebates @@ -82,7 +81,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Dependencies -- Bump `google.golang.org/grpc` from 1.59.0 to 1.60.0 ([#1720](https://github.com/NibiruChain/nibiru/pull/1720)) +* Bump `google.golang.org/grpc` from 1.59.0 to 1.60.0 ([#1720](https://github.com/NibiruChain/nibiru/pull/1720)) + * Bump `golang.org/x/crypto` from 0.15.0 to 0.17.0 ([#1724](https://github.com/NibiruChain/nibiru/pull/1724)) * Bump `github.com/holiman/uint256` from 1.2.3 to 1.2.4 ([#1730](https://github.com/NibiruChain/nibiru/pull/1730)) * Bump `github.com/dvsekhvalnov/jose2go` from 1.5.0 to 1.6.0 ([#1733](https://github.com/NibiruChain/nibiru/pull/1733)) From 811a00f21167a3b67bc6f7f86d556529dc772d2e Mon Sep 17 00:00:00 2001 From: Kevin Yang <5478483+k-yang@users.noreply.github.com> Date: Thu, 28 Dec 2023 09:40:02 -0500 Subject: [PATCH 13/13] fix: remove println --- x/sudo/module.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/x/sudo/module.go b/x/sudo/module.go index fd398c115..1e0031ad1 100644 --- a/x/sudo/module.go +++ b/x/sudo/module.go @@ -170,18 +170,5 @@ func (AppModule) RegisterStoreDecoder(sdk.StoreDecoderRegistry) { // WeightedOperations implements module.AppModuleSimulation. func (AppModule) WeightedOperations(simState module.SimulationState) []simtypes.WeightedOperation { - appParams := simState.AppParams - - fmt.Println("Map:", fmt.Sprintf("%v", appParams)) - for k, v := range appParams { - var val int - fmt.Printf("Key: %s, Value: %d\n", k, v) - err := json.Unmarshal(v, &val) - if err != nil { - panic(err) - } - fmt.Printf("Key: %s, Value: %d\n", k, val) - } - return nil }