Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

memiavl changes #311

Open
wants to merge 40 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
5e3f93f
Changes for memiavl
Jul 26, 2023
84bdda0
Fix changes for config
Jul 26, 2023
5a6e3eb
Remove debug log
Jul 26, 2023
539054b
fix go mod fir iavl
Jul 26, 2023
9d2e9c1
Bump golang version to 1.20
Jul 26, 2023
b717c11
Bump to 1.20
Jul 26, 2023
82df860
Fix go version
Jul 26, 2023
a9730a9
Expose Snapshot function
Jul 27, 2023
73eee20
Bump version
yzang2019 Aug 30, 2023
929da20
Bump mmap-iavl version
yzang2019 Sep 1, 2023
b03c2d4
Fix go sum
yzang2019 Sep 1, 2023
518544a
Add method for snapshot
yzang2019 Sep 2, 2023
2d06820
Bump iavl version
yzang2019 Sep 2, 2023
39068d8
Remove condition check
yzang2019 Sep 2, 2023
0d8aa92
Fix snapshot error handling logic
yzang2019 Sep 3, 2023
cd731c9
Merge branch 'main' into yzang/memiavl
yzang2019 Sep 3, 2023
6431ea7
Bump iavl and tendermint version
yzang2019 Sep 4, 2023
20c9714
Adding close for cache multistore in abci query
yzang2019 Sep 6, 2023
ed5e785
Add logs for cache multistore
yzang2019 Sep 7, 2023
6880b18
Add logs for cache multistore
yzang2019 Sep 7, 2023
ec9638a
Switch to seidb
yzang2019 Oct 11, 2023
ab957f6
Add configs for seidb
yzang2019 Oct 24, 2023
7dcdd39
Bump seidb version
yzang2019 Oct 26, 2023
c51bdcb
Add query multi store for cosmos sdk
yzang2019 Nov 1, 2023
6302d64
Add qms
yzang2019 Nov 1, 2023
1d56ec8
Add snapshot manager for SC and SS
yzang2019 Nov 2, 2023
b96d79f
Fix snapshot manager
yzang2019 Nov 3, 2023
eab249d
Merge latest main
yzang2019 Nov 3, 2023
948d27a
Bump seidb version
yzang2019 Nov 6, 2023
2cc26a2
Add pruning settings for qms
yzang2019 Nov 7, 2023
9b57f8d
Revert change
yzang2019 Nov 7, 2023
cc31016
Add config for ss pruning manager
yzang2019 Nov 8, 2023
bf22e17
Bump seidb version
yzang2019 Nov 13, 2023
66ba9e6
Add config for num workers
yzang2019 Nov 13, 2023
06833bd
Add log for snapshot restore time
yzang2019 Nov 13, 2023
d15a1aa
Merge branch 'main' into yzang/memiavl
yzang2019 Nov 15, 2023
9dda5d7
Bump seidb version
yzang2019 Nov 15, 2023
4350536
Add closer for qms for propoer auto remediation
yzang2019 Nov 16, 2023
3c67c68
Fix abci for qms close
yzang2019 Nov 16, 2023
e5febd1
Remove some debug log
yzang2019 Nov 16, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19
go-version: '1.20'
- name: Create a file with all core Cosmos SDK pkgs
run: go list ./... > pkgs.txt
- name: Split pkgs into 10 files
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:
- uses: actions/setup-python@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19
go-version: '1.20'
- uses: technote-space/[email protected]
with:
PATTERNS: |
Expand Down Expand Up @@ -116,7 +116,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19
go-version: '1.20'

# Download all coverage reports from the 'tests' job
- name: Download coverage reports
Expand Down
42 changes: 36 additions & 6 deletions baseapp/abci.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sort"
"strings"
Expand Down Expand Up @@ -344,15 +345,19 @@ func (app *BaseApp) Commit(ctx context.Context) (res *abci.ResponseCommit, err e
app.halt()
}

if app.snapshotInterval > 0 && uint64(header.Height)%app.snapshotInterval == 0 {
go app.snapshot(header.Height)
}
app.SnapshotIfApplicable(uint64(header.Height))

return &abci.ResponseCommit{
RetainHeight: retainHeight,
}, nil
}

func (app *BaseApp) SnapshotIfApplicable(height uint64) {
if app.snapshotInterval > 0 && height%app.snapshotInterval == 0 {
go app.Snapshot(int64(height))
}
}

// halt attempts to gracefully shutdown the node via SIGINT and SIGTERM falling
// back on os.Exit if both fail.
func (app *BaseApp) halt() {
Expand All @@ -376,7 +381,7 @@ func (app *BaseApp) halt() {
}

// snapshot takes a snapshot of the current state and prunes any old snapshottypes.
func (app *BaseApp) snapshot(height int64) {
func (app *BaseApp) Snapshot(height int64) {
if app.snapshotManager == nil {
app.logger.Info("snapshot manager not configured")
return
Expand Down Expand Up @@ -589,6 +594,16 @@ func (app *BaseApp) ApplySnapshotChunk(context context.Context, req *abci.Reques

func (app *BaseApp) handleQueryGRPC(handler GRPCQueryHandler, req abci.RequestQuery) abci.ResponseQuery {
ctx, err := app.CreateQueryContext(req.Height, req.Prove)
defer func() {
// We should close the multistore to avoid leaking resources.
if closer, ok := ctx.MultiStore().(io.Closer); ok {
err := closer.Close()
if err != nil {
app.logger.Error("failed to close Cache MultiStore", "err", err)
}
}
}()

if err != nil {
return sdkerrors.QueryResultWithDebug(err, app.trace)
}
Expand Down Expand Up @@ -641,7 +656,11 @@ func (app *BaseApp) CreateQueryContext(height int64, prove bool) (sdk.Context, e
return sdk.Context{}, err
}

lastBlockHeight := app.LastBlockHeight()
var qms sdk.MultiStore = app.qms
if qms == nil || height == app.cms.LatestVersion() {
qms = app.cms
}
lastBlockHeight := qms.LatestVersion()
if height > lastBlockHeight {
return sdk.Context{},
sdkerrors.Wrap(
Expand All @@ -663,7 +682,7 @@ func (app *BaseApp) CreateQueryContext(height int64, prove bool) (sdk.Context, e
)
}

cacheMS, err := app.cms.CacheMultiStoreWithVersion(height)
cacheMS, err := qms.CacheMultiStoreWithVersion(height)
if err != nil {
return sdk.Context{},
sdkerrors.Wrapf(
Expand Down Expand Up @@ -872,6 +891,17 @@ func handleQueryCustom(app *BaseApp, path []string, req abci.RequestQuery) abci.
}

ctx, err := app.CreateQueryContext(req.Height, req.Prove)

defer func() {
// We should close the multistore to avoid leaking resources.
if closer, ok := ctx.MultiStore().(io.Closer); ok {
err := closer.Close()
if err != nil {
app.logger.Error("failed to close Cache MultiStore", "err", err)
}
}
}()

if err != nil {
return sdkerrors.QueryResultWithDebug(err, app.trace)
}
Expand Down
37 changes: 19 additions & 18 deletions baseapp/baseapp.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
servertypes "github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/snapshots"
"github.com/cosmos/cosmos-sdk/store"
"github.com/cosmos/cosmos-sdk/store/rootmulti"
"github.com/cosmos/cosmos-sdk/telemetry"
sdk "github.com/cosmos/cosmos-sdk/types"
acltypes "github.com/cosmos/cosmos-sdk/types/accesscontrol"
Expand Down Expand Up @@ -173,6 +172,7 @@ type BaseApp struct { //nolint: maligned
type appStore struct {
db dbm.DB // common DB backend
cms sdk.CommitMultiStore // Main (uncached) state
qms sdk.QueryMultiStore // Optional alternative multistore for querying only
storeLoader StoreLoader // function to handle store loading, may be overridden with SetStoreLoader()

// an inter-block write-through cache provided to the context during deliverState
Expand Down Expand Up @@ -290,9 +290,6 @@ func NewBaseApp(
panic("must pass --chain-id when calling 'seid start' or set in ~/.sei/config/client.toml")
}
app.startCompactionRoutine(db)
if app.orphanConfig != nil {
app.cms.(*rootmulti.Store).SetOrphanConfig(app.orphanConfig)
}

return app
}
Expand Down Expand Up @@ -407,6 +404,13 @@ func (app *BaseApp) CommitMultiStore() sdk.CommitMultiStore {
return app.cms
}

// QueryMultiStore returns the query multi-store.
// App constructor can use this to access the `qms`.
// UNSAFE: must not be used during the abci life cycle.
func (app *BaseApp) QueryMultiStore() sdk.QueryMultiStore {
return app.qms
}

// SnapshotManager returns the snapshot manager.
// application use this to register extra extension snapshotters.
func (app *BaseApp) SnapshotManager() *snapshots.Manager {
Expand Down Expand Up @@ -443,20 +447,12 @@ func (app *BaseApp) init() error {
app.setCheckState(tmproto.Header{})
app.Seal()

// make sure the snapshot interval is a multiple of the pruning KeepEvery interval
if app.snapshotManager != nil && app.snapshotInterval > 0 {
rms, ok := app.cms.(*rootmulti.Store)
if !ok {
return errors.New("state sync snapshots require a rootmulti store")
}
pruningOpts := rms.GetPruning()
if pruningOpts.KeepEvery > 0 && app.snapshotInterval%pruningOpts.KeepEvery != 0 {
return fmt.Errorf(
"state sync snapshot interval %v must be a multiple of pruning keep every interval %v",
app.snapshotInterval, pruningOpts.KeepEvery)
}
if app.cms == nil {
return errors.New("commit multi-store must not be nil")
}

return app.cms.GetPruning().Validate()

return nil
}

Expand Down Expand Up @@ -862,7 +858,7 @@ func (app *BaseApp) runTx(ctx sdk.Context, mode runTxMode, txBytes []byte) (gInf
}

// Wait for signals to complete before starting the transaction. This is needed before any of the
// resources are acceessed by the ante handlers and message handlers.
// resources are accessed by the ante handlers and message handlers.
defer acltypes.SendAllSignalsForTx(ctx.TxCompletionChannels())
acltypes.WaitForAllSignalsForTx(ctx.TxBlockingChannels())
// check for existing parent tracer, and if applicable, use it
Expand Down Expand Up @@ -1174,6 +1170,9 @@ func (app *BaseApp) Close() error {
if err := app.appStore.db.Close(); err != nil {
return err
}
if err := app.qms.Close(); err != nil {
return err
}
return app.snapshotManager.Close()
}

Expand All @@ -1188,11 +1187,13 @@ func (app *BaseApp) ReloadDB() error {
app.db = db
app.cms = store.NewCommitMultiStore(db)
if app.snapshotManager != nil {
app.snapshotManager.SetMultiStore(app.cms)
app.snapshotManager.SetStateCommitStore(app.cms)
}
return nil
}

func (app *BaseApp) GetCheckCtx() sdk.Context {
return app.checkState.ctx
}

//
9 changes: 8 additions & 1 deletion baseapp/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ func (app *BaseApp) SetSnapshotStore(snapshotStore *snapshots.Store) {
app.snapshotManager = nil
return
}
app.snapshotManager = snapshots.NewManager(snapshotStore, app.cms)
app.snapshotManager = snapshots.NewManager(snapshotStore, app.cms, app.qms, app.logger)
}

// SetSnapshotInterval sets the snapshot interval.
Expand Down Expand Up @@ -328,3 +328,10 @@ func (app *BaseApp) SetStreamingService(s StreamingService) {
// BaseApp will pass BeginBlock, DeliverTx, and EndBlock requests and responses to the streaming services to update their ABCI context
app.abciListeners = append(app.abciListeners, s)
}

// SetQueryMultiStore set an alternative MultiStore implementation to support historical queries.
//
// Ref: https://github.com/cosmos/cosmos-sdk/issues/13317
func (app *BaseApp) SetQueryMultiStore(ms sdk.QueryMultiStore) {
app.qms = ms
}
Loading
Loading