From 804049c08b5e8693ccf37f228d851080c3d21596 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 28 Jan 2025 11:53:13 -0800 Subject: [PATCH 1/4] remove db in config add pebble dir --- cmd/node_builder.go | 4 +++- cmd/scaffold.go | 7 +------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/node_builder.go b/cmd/node_builder.go index dc203f66ad4..78b365dffee 100644 --- a/cmd/node_builder.go +++ b/cmd/node_builder.go @@ -151,6 +151,7 @@ type BaseConfig struct { DynamicStartupEpoch string DynamicStartupSleepInterval time.Duration datadir string + pebbleDir string secretsdir string secretsDBEnabled bool InsecureSecretsDB bool @@ -164,7 +165,6 @@ type BaseConfig struct { MetricsEnabled bool guaranteesCacheSize uint receiptsCacheSize uint - db *badger.DB HeroCacheMetricsEnable bool SyncCoreConfig chainsync.Config CodecFactory func() network.Codec @@ -253,6 +253,7 @@ type StateExcerptAtBoot struct { func DefaultBaseConfig() *BaseConfig { datadir := "/data/protocol" + pebbleDir := "/data/protocol-pebble" // NOTE: if the codec used in the network component is ever changed any code relying on // the message format specific to the codec must be updated. i.e: the AuthorizedSenderValidator. @@ -269,6 +270,7 @@ func DefaultBaseConfig() *BaseConfig { ObserverMode: false, BootstrapDir: "bootstrap", datadir: datadir, + pebbleDir: pebbleDir, secretsdir: NotSet, secretsDBEnabled: true, level: "info", diff --git a/cmd/scaffold.go b/cmd/scaffold.go index ad94c641f3f..657b25a6c72 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -164,6 +164,7 @@ func (fnb *FlowNodeBuilder) BaseFlags() { fnb.flags.StringVar(&fnb.BaseConfig.BindAddr, "bind", defaultConfig.BindAddr, "address to bind on") fnb.flags.StringVarP(&fnb.BaseConfig.BootstrapDir, "bootstrapdir", "b", defaultConfig.BootstrapDir, "path to the bootstrap directory") fnb.flags.StringVarP(&fnb.BaseConfig.datadir, "datadir", "d", defaultConfig.datadir, "directory to store the public database (protocol state)") + fnb.flags.StringVar(&fnb.BaseConfig.pebbleDir, "pebble-dir", defaultConfig.pebbleDir, "directory to store the public pebble database (protocol state)") fnb.flags.StringVar(&fnb.BaseConfig.secretsdir, "secretsdir", defaultConfig.secretsdir, "directory to store private database (secrets)") fnb.flags.StringVarP(&fnb.BaseConfig.level, "loglevel", "l", defaultConfig.level, "level for logging output") fnb.flags.Uint32Var(&fnb.BaseConfig.debugLogLimit, "debug-log-limit", defaultConfig.debugLogLimit, "max number of debug/trace log events per second") @@ -1058,12 +1059,6 @@ func (fnb *FlowNodeBuilder) initProfiler() error { func (fnb *FlowNodeBuilder) initDB() error { - // if a db has been passed in, use that instead of creating one - if fnb.BaseConfig.db != nil { - fnb.DB = fnb.BaseConfig.db - return nil - } - // Pre-create DB path (Badger creates only one-level dirs) err := os.MkdirAll(fnb.BaseConfig.datadir, 0700) if err != nil { From 03270cd7330fb42b0a57b2fe4ce2493b3d5d6964 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 28 Jan 2025 11:58:22 -0800 Subject: [PATCH 2/4] remove the unused WithDB --- cmd/scaffold.go | 12 +----------- follower/consensus_follower.go | 12 ------------ 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 657b25a6c72..fd6b94f4cad 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -1884,9 +1884,7 @@ func WithBindAddress(bindAddress string) Option { func WithDataDir(dataDir string) Option { return func(config *BaseConfig) { - if config.db == nil { - config.datadir = dataDir - } + config.datadir = dataDir } } @@ -1920,14 +1918,6 @@ func WithLogLevel(level string) Option { } } -// WithDB takes precedence over WithDataDir and datadir will be set to empty if DB is set using this option -func WithDB(db *badger.DB) Option { - return func(config *BaseConfig) { - config.db = db - config.datadir = "" - } -} - // FlowNode creates a new Flow node builder with the given name. func FlowNode(role string, opts ...Option) *FlowNodeBuilder { config := DefaultBaseConfig() diff --git a/follower/consensus_follower.go b/follower/consensus_follower.go index 35cc77a3965..7c110d028c6 100644 --- a/follower/consensus_follower.go +++ b/follower/consensus_follower.go @@ -69,15 +69,6 @@ func WithLogLevel(level string) Option { } } -// WithDB sets the underlying database that will be used to store the chain state -// WithDB takes precedence over WithDataDir and datadir will be set to empty if DB is set using this option -func WithDB(db *badger.DB) Option { - return func(cf *Config) { - cf.db = db - cf.dataDir = "" - } -} - func WithExposeMetrics(expose bool) Option { return func(c *Config) { c.exposeMetrics = expose @@ -142,9 +133,6 @@ func getBaseOptions(config *Config) []cmd.Option { if config.logLevel != "" { options = append(options, cmd.WithLogLevel(config.logLevel)) } - if config.db != nil { - options = append(options, cmd.WithDB(config.db)) - } if config.exposeMetrics { options = append(options, cmd.WithMetricsEnabled(config.exposeMetrics)) } From 07039d54a91853c364a57632d214395e9da12cc8 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 28 Jan 2025 12:34:49 -0800 Subject: [PATCH 3/4] add open pebble --- cmd/node_builder.go | 2 ++ cmd/scaffold.go | 20 ++++++++++++++++++-- cmd/scaffold/pebble_db.go | 26 ++++++++++++++++++++++++++ cmd/scaffold/pebble_db_test.go | 17 +++++++++++++++++ storage/pebble/open.go | 2 ++ 5 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 cmd/scaffold/pebble_db.go create mode 100644 cmd/scaffold/pebble_db_test.go diff --git a/cmd/node_builder.go b/cmd/node_builder.go index 78b365dffee..4e0bf699680 100644 --- a/cmd/node_builder.go +++ b/cmd/node_builder.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/cockroachdb/pebble" "github.com/dgraph-io/badger/v2" madns "github.com/multiformats/go-multiaddr-dns" "github.com/onflow/crypto" @@ -198,6 +199,7 @@ type NodeConfig struct { MetricsRegisterer prometheus.Registerer Metrics Metrics DB *badger.DB + PebbleDB *pebble.DB SecretsDB *badger.DB Storage Storage ProtocolEvents *events.Distributor diff --git a/cmd/scaffold.go b/cmd/scaffold.go index fd6b94f4cad..ca70846c027 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -31,6 +31,7 @@ import ( "github.com/onflow/flow-go/admin/commands/common" storageCommands "github.com/onflow/flow-go/admin/commands/storage" "github.com/onflow/flow-go/cmd/build" + "github.com/onflow/flow-go/cmd/scaffold" "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/consensus/hotstuff/persister" "github.com/onflow/flow-go/fvm/initialize" @@ -1057,7 +1058,7 @@ func (fnb *FlowNodeBuilder) initProfiler() error { return nil } -func (fnb *FlowNodeBuilder) initDB() error { +func (fnb *FlowNodeBuilder) initBadgerDB() error { // Pre-create DB path (Badger creates only one-level dirs) err := os.MkdirAll(fnb.BaseConfig.datadir, 0700) @@ -1105,6 +1106,17 @@ func (fnb *FlowNodeBuilder) initDB() error { return nil } +func (fnb *FlowNodeBuilder) initPebbleDB() error { + db, closer, err := scaffold.InitPebbleDB(fnb.BaseConfig.pebbleDir) + if err != nil { + return err + } + + fnb.PebbleDB = db + fnb.ShutdownFunc(closer.Close) + return nil +} + func (fnb *FlowNodeBuilder) initSecretsDB() error { // if the secrets DB is disabled (only applicable for Consensus Follower, @@ -2023,7 +2035,11 @@ func (fnb *FlowNodeBuilder) onStart() error { return err } - if err := fnb.initDB(); err != nil { + if err := fnb.initBadgerDB(); err != nil { + return err + } + + if err := fnb.initPebbleDB(); err != nil { return err } diff --git a/cmd/scaffold/pebble_db.go b/cmd/scaffold/pebble_db.go new file mode 100644 index 00000000000..4e785ae5075 --- /dev/null +++ b/cmd/scaffold/pebble_db.go @@ -0,0 +1,26 @@ +package scaffold + +import ( + "fmt" + "io" + "os" + + "github.com/cockroachdb/pebble" + + pebblestorage "github.com/onflow/flow-go/storage/pebble" +) + +func InitPebbleDB(dir string) (*pebble.DB, io.Closer, error) { + // Pre-create DB path + err := os.MkdirAll(dir, 0700) + if err != nil { + return nil, nil, fmt.Errorf("could not create pebble db (path: %s): %w", dir, err) + } + + db, err := pebblestorage.OpenDefaultPebbleDB(dir) + if err != nil { + return nil, nil, err + } + + return db, db, nil +} diff --git a/cmd/scaffold/pebble_db_test.go b/cmd/scaffold/pebble_db_test.go new file mode 100644 index 00000000000..ca42d23e4d1 --- /dev/null +++ b/cmd/scaffold/pebble_db_test.go @@ -0,0 +1,17 @@ +package scaffold + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/utils/unittest" +) + +func TestInitPebbleDB(t *testing.T) { + unittest.RunWithTempDir(t, func(dir string) { + _, closer, err := InitPebbleDB(dir) + require.NoError(t, err) + require.NoError(t, closer.Close()) + }) +} diff --git a/storage/pebble/open.go b/storage/pebble/open.go index b8938abe685..ffefcffbb45 100644 --- a/storage/pebble/open.go +++ b/storage/pebble/open.go @@ -82,6 +82,8 @@ func MustOpenDefaultPebbleDB(dir string) (*pebble.DB, error) { } // IsPebbleInitialized checks if the given folder contains a valid Pebble DB. +// return error if the folder does not exist, is not a directory, or is missing required files +// return nil if the folder contains a valid Pebble DB func IsPebbleInitialized(folderPath string) error { // Check if the folder exists info, err := os.Stat(folderPath) From 46122fb5bf279ae1a481d8ecf9b295551a1a0355 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 30 Jan 2025 13:07:18 -0800 Subject: [PATCH 4/4] remove db in follower engine --- follower/consensus_follower.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/follower/consensus_follower.go b/follower/consensus_follower.go index 7c110d028c6..d42aac16575 100644 --- a/follower/consensus_follower.go +++ b/follower/consensus_follower.go @@ -5,7 +5,6 @@ import ( "fmt" "sync" - "github.com/dgraph-io/badger/v2" "github.com/onflow/crypto" "github.com/rs/zerolog" @@ -36,7 +35,6 @@ type Config struct { networkPrivKey crypto.PrivateKey // the network private key of this node bootstrapNodes []BootstrapNodeInfo // the bootstrap nodes to use bindAddr string // address to bind on - db *badger.DB // the badger DB storage to use for the protocol state dataDir string // directory to store the protocol state (if the badger storage is not provided) bootstrapDir string // path to the bootstrap directory logLevel string // log level @@ -51,9 +49,7 @@ type Option func(c *Config) // If a database is supplied, then data directory will be set to empty string func WithDataDir(dataDir string) Option { return func(cf *Config) { - if cf.db == nil { - cf.dataDir = dataDir - } + cf.dataDir = dataDir } }