Database update

README.md

@@ -26,27 +26,27 @@ You can also use Docker to install b7s. See the [Docker documentation](/docker/R
 
 For a more detailed overview of the configuration options, see the [b7s-node Readme](/cmd/node/README.md#usage).
 
-| Flag                      | Short Form | Default Value           | Description                                                                                   |
-| ------------------------- | ---------- | ----------------------- | --------------------------------------------------------------------------------------------- |
-| config                    | N/A        | N/A                     | Specifies the config file to load.
-| log-level                 | -l         | "info"                  | Specifies the level of logging to use.                                                        |
-| peer-db                   | N/A        | "peer-db"               | Specifies the path to database used for persisting peer data.                                 |
-| function-db               | N/A        | "function-db"           | Specifies the path to database used for persisting function data.                             |
-| role                      | -r         | "worker"                | Specifies the role this node will have in the Blockless protocol (head or worker).            |
-| address                   | -a         | "0.0.0.0"               | Specifies the address that the libp2p host will use.                                          |
-| port                      | -p         | 0                       | Specifies the port that the libp2p host will use.                                             |
-| websocket-port            | N/A        | 0                       | Specifies the port that the libp2p host will use for websocket connections.                   |
-| private-key               | N/A        | N/A                     | Specifies the private key that the libp2p host will use.                                      |
-| concurrency               | -c         | node.DefaultConcurrency | Specifies the maximum number of requests the node will process in parallel.                   |
-| rest-api                  | N/A        | N/A                     | Specifies the address where the head node REST API will listen on.                            |
-| boot-nodes                | N/A        | N/A                     | Specifies a list of addresses that this node will connect to on startup, in multiaddr format. |
-| workspace                 | N/A        | "./workspace"           | Specifies the directory that the node can use for file storage.                               |
-| runtime                   | N/A        | N/A                     | Specifies the runtime address used by the worker node.                                        |
-| dialback-address          | N/A        | N/A                     | Specifies the advertised dialback address of the Node.                                        |
-| dialback-port             | N/A        | N/A                     | Specifies the advertised dialback port of the Node.                                           |
-| websocket-dialback-port   | N/A        | 0                       | Specifies the advertised dialback port for Websocket connections.
+| Flag                      | Short Form | Default Value           | Description                                                                                           |
+| ------------------------- | ---------- | ----------------------- | ----------------------------------------------------------------------------------------------------- |
+| config                    | N/A        | N/A                     | Specifies the config file to load.                                                                    |
+| log-level                 | -l         | "info"                  | Specifies the level of logging to use.                                                                |
+| db                        | N/A        | "db"                    | Specifies the path to database used for persisting peer and function data.                            |
+| role                      | -r         | "worker"                | Specifies the role this node will have in the Blockless protocol (head or worker).                    |
+| address                   | -a         | "0.0.0.0"               | Specifies the address that the libp2p host will use.                                                  |
+| port                      | -p         | 0                       | Specifies the port that the libp2p host will use.                                                     |
+| websocket-port            | N/A        | 0                       | Specifies the port that the libp2p host will use for websocket connections.                           |
+| private-key               | N/A        | N/A                     | Specifies the private key that the libp2p host will use.                                              |
+| concurrency               | -c         | node.DefaultConcurrency | Specifies the maximum number of requests the node will process in parallel.                           |
+| rest-api                  | N/A        | N/A                     | Specifies the address where the head node REST API will listen on.                                    |
+| boot-nodes                | N/A        | N/A                     | Specifies a list of addresses that this node will connect to on startup, in multiaddr format.         |
+| workspace                 | N/A        | "./workspace"           | Specifies the directory that the node can use for file storage.                                       |
+| runtime                   | N/A        | N/A                     | Specifies the runtime address used by the worker node.                                                |
+| dialback-address          | N/A        | N/A                     | Specifies the advertised dialback address of the Node.                                                |
+| dialback-port             | N/A        | N/A                     | Specifies the advertised dialback port of the Node.                                                   |
+| websocket-dialback-port   | N/A        | 0                       | Specifies the advertised dialback port for Websocket connections.                                     |
 | cpu-percentage-limit      | N/A        | 1.0                     | Specifies the amount of CPU time allowed for Blockless Functions in the 0-1 range, 1 being unlimited. |
-| memory-limit              | N/A        | N/A                     | Specifies the memory limit for Blockless Functions, in kB. |
+| memory-limit              | N/A        | N/A                     | Specifies the memory limit for Blockless Functions, in kB.                                            |
+| no-dialback-peers         | N/A        | false                   | Specifies if the node should avoid dialing back peers known from past runs                            |
 
 ## Dependencies
 

cmd/node/README.md

@@ -26,29 +26,29 @@ List of supported CLI flags is listed below.
 
 ```console
 Usage of b7s-node:
-      --config string                  path to a config file
   -r, --role string                    role this node will have in the Blockless protocol (head or worker) (default "worker")
   -c, --concurrency uint               maximum number of requests node will process in parallel (default 10)
       --boot-nodes strings             list of addresses that this node will connect to on startup, in multiaddr format
       --workspace string               directory that the node can use for file storage
-      --attributes                     node should try to load its attribute data from IPFS
-      --peer-db string                 path to the database used for persisting peer data (default "peer-db")
-      --function-db string             path to the database used for persisting function data (default "function-db")
+      --load-attributes                node should try to load its attribute data from IPFS
       --topics strings                 topics node should subscribe to
+      --db string                      path to the database used for persisting peer and function data
   -l, --log-level string               log level to use (default "info")
   -a, --address string                 address that the b7s host will use (default "0.0.0.0")
   -p, --port uint                      port that the b7s host will use
       --private-key string             private key that the b7s host will use
+      --dialback-address string        external address that the b7s host will advertise
+      --dialback-port uint             external port that the b7s host will advertise
   -w, --websocket                      should the node use websocket protocol for communication
       --websocket-port uint            port to use for websocket connections
-      --dialback-address string        external address that the b7s host will advertise (default "0.0.0.0")
-      --dialback-port uint             external port that the b7s host will advertise
       --websocket-dialback-port uint   external port that the b7s host will advertise for websocket connections
+      --no-dialback-peers              start without dialing back peers from previous runs
+      --rest-api string                address where the head node REST API will listen on
       --runtime-path string            Blockless Runtime location (used by the worker node)
-      --runtime-cli string             runtime CLI name (used by the worker node) (default "bls-runtime")
-      --cpu-percentage-limit float     amount of CPU time allowed for Blockless Functions in the 0-1 range, 1 being unlimited (default 1)
+      --runtime-cli string             runtime CLI name (used by the worker node)
+      --cpu-percentage-limit float     amount of CPU time allowed for Blockless Functions in the 0-1 range, 1 being unlimited
       --memory-limit int               memory limit (kB) for Blockless Functions
-      --rest-api string                address where the head node REST API will listen on
+      --config string                  path to a config file
 ```
 
 Alternatively to the CLI flags, you can create a YAML file and specify the parameters there.
@@ -94,12 +94,11 @@ If a private key is not specified the node will start with a randomly generated
 ### Starting a Worker Node
 
 ```console
-$ ./node --peer-db peer-database --log-level debug --port 9000 --role worker --runtime ~/.local/bin --workspace workspace --private-key ./keys/priv.bin
+$ ./node --db /tmp/db --log-level debug --port 9000 --role worker --runtime ~/.local/bin --workspace workspace --private-key ./keys/priv.bin
 ```
 
 The created `node` will listen on all addresses on TCP port 9000.
-Database used to persist Node data between runs will be created in the `peer-database` subdirectory.
-On the other hand, Node will persist function data in the default database, in the `function-db` subdirectory.
+Database used to persist Node data between runs will be created in the `/tmp/db` subdirectory.
 
 Blockless Runtime path is given as `/home/user/.local/bin`.
 At startup, node will check if the Blockless Runtime is actually found there, namely the [bls-runtime](https://blockless.network/docs/protocol/runtime).
@@ -111,12 +110,11 @@ Any transient files needed for node operation will be created in the `workspace`
 ### Starting a Head Node
 
 ```console
-$ ./node --peer-db /var/tmp/b7s/peerdb --function-db /var/tmp/b7s/fdb --log-level debug --port 9002 -r head --workspace /var/tmp/b7s/workspace --private-key ~/keys/priv.bin --rest-api ':8080'
+$ ./node --db /var/tmp/b7s/db --log-level debug --port 9002 -r head --workspace /var/tmp/b7s/workspace --private-key ~/keys/priv.bin --rest-api ':8080'
 ```
 
 The created `node` will listen on all addresses on TCP port 9002.
-Database used to persist Node peer data between runs will be created at `/var/tmp/b7s/peerdb`.
-Database used to persist Node function data will be created at `/var/tmp/b7s/fdb`.
+Database used to persist Node peer and function data between runs will be created at `/var/tmp/b7s/db`.
 
 Any transient files needed for node operation will be created in the `/var/tmp/b7s/workspace` directory.
 

cmd/node/example.yaml

@@ -7,11 +7,8 @@
 # directory where node will keep files needed for operation
 # workspace: workspace
 
-# directory where node will maintain its peer database
-# peer-db: pdb
-
-# directory where node will maintain its function database
-# function-db: fdb
+# directory where node will maintain its database
+# db: db
 
 # multiaddresses of nodes this node will try to connect to on boot
 # boot-nodes: []

cmd/node/main.go

@@ -22,8 +22,8 @@ import (
 	"github.com/blocklessnetwork/b7s/host"
 	"github.com/blocklessnetwork/b7s/models/blockless"
 	"github.com/blocklessnetwork/b7s/node"
-	"github.com/blocklessnetwork/b7s/peerstore"
 	"github.com/blocklessnetwork/b7s/store"
+	"github.com/blocklessnetwork/b7s/store/codec"
 )
 
 const (
@@ -84,14 +84,13 @@ func run() int {
 		}
 	}
 
-	// Set relevant working paths for workspace, peerDB and functionDB.
+	// Set relevant working paths for workspace and DB.
 	// If paths were set using the CLI flags, use those. Else, use generated path, e.g. .b7s_<peer-id>/<default-option-for-directory>.
 	updateDirPaths(nodeDir, cfg)
 
 	log.Info().
 		Str("workspace", cfg.Workspace).
-		Str("peer_db", cfg.PeerDB).
-		Str("function_db", cfg.FunctionDB).
+		Str("db", cfg.DB).
 		Msg("filepaths used by the node")
 
 	// Convert workspace path to an absolute one.
@@ -103,23 +102,15 @@ func run() int {
 	cfg.Workspace = workspace
 
 	// Open the pebble peer database.
-	pdb, err := pebble.Open(cfg.PeerDB, &pebble.Options{Logger: &pebbleNoopLogger{}})
+	db, err := pebble.Open(cfg.DB, &pebble.Options{Logger: &pebbleNoopLogger{}})
 	if err != nil {
-		log.Error().Err(err).Str("db", cfg.PeerDB).Msg("could not open pebble peer database")
+		log.Error().Err(err).Str("db", cfg.DB).Msg("could not open pebble database")
 		return failure
 	}
-	defer pdb.Close()
+	defer db.Close()
 
 	// Create a new store.
-	pstore := store.New(pdb)
-	peerstore := peerstore.New(pstore)
-
-	// Get the list of dial back peers.
-	peers, err := peerstore.Peers()
-	if err != nil {
-		log.Error().Err(err).Msg("could not get list of dial-back peers")
-		return failure
-	}
+	store := store.New(db, codec.NewJSONCodec())
 
 	// Get the list of boot nodes addresses.
 	bootNodeAddrs, err := getBootNodeAddresses(cfg.BootNodes)
@@ -128,17 +119,29 @@ func run() int {
 		return failure
 	}
 
-	// Create libp2p host.
-	host, err := host.New(log, cfg.Connectivity.Address, cfg.Connectivity.Port,
+	hostOpts := []func(*host.Config){
 		host.WithPrivateKey(cfg.Connectivity.PrivateKey),
 		host.WithBootNodes(bootNodeAddrs),
-		host.WithDialBackPeers(peers),
 		host.WithDialBackAddress(cfg.Connectivity.DialbackAddress),
 		host.WithDialBackPort(cfg.Connectivity.DialbackPort),
 		host.WithDialBackWebsocketPort(cfg.Connectivity.WebsocketDialbackPort),
 		host.WithWebsocket(cfg.Connectivity.Websocket),
 		host.WithWebsocketPort(cfg.Connectivity.WebsocketPort),
-	)
+	}
+
+	if !cfg.Connectivity.NoDialbackPeers {
+		// Get the list of dial back peers.
+		peers, err := store.RetrievePeers()
+		if err != nil {
+			log.Error().Err(err).Msg("could not get list of dial-back peers")
+			return failure
+		}
+
+		hostOpts = append(hostOpts, host.WithDialBackPeers(peers))
+	}
+
+	// Create libp2p host.
+	host, err := host.New(log, cfg.Connectivity.Address, cfg.Connectivity.Port, hostOpts...)
 	if err != nil {
 		log.Error().Err(err).Str("key", cfg.Connectivity.PrivateKey).Msg("could not create host")
 		return failure
@@ -149,7 +152,6 @@ func run() int {
 		Str("id", host.ID().String()).
 		Strs("addresses", host.Addresses()).
 		Int("boot_nodes", len(bootNodeAddrs)).
-		Int("dial_back_peers", len(peers)).
 		Msg("created host")
 
 	// Set node options.
@@ -202,26 +204,16 @@ func run() int {
 		opts = append(opts, node.WithWorkspace(cfg.Workspace))
 	}
 
-	// Open the pebble function database.
-	fdb, err := pebble.Open(cfg.FunctionDB, &pebble.Options{Logger: &pebbleNoopLogger{}})
-	if err != nil {
-		log.Error().Err(err).Str("db", cfg.FunctionDB).Msg("could not open pebble function database")
-		return failure
-	}
-	defer fdb.Close()
-
-	functionStore := store.New(fdb)
-
 	// Create function store.
-	fstore := fstore.New(log, functionStore, cfg.Workspace)
+	fstore := fstore.New(log, store, cfg.Workspace)
 
 	// If we have topics specified, use those.
 	if len(cfg.Topics) > 0 {
 		opts = append(opts, node.WithTopics(cfg.Topics))
 	}
 
 	// Instantiate node.
-	node, err := node.New(log, host, peerstore, fstore, opts...)
+	node, err := node.New(log, host, store, fstore, opts...)
 	if err != nil {
 		log.Error().Err(err).Msg("could not create node")
 		return failure
@@ -327,17 +319,11 @@ func updateDirPaths(root string, cfg *config.Config) {
 	}
 	cfg.Workspace = workspace
 
-	peerDB := cfg.PeerDB
-	if peerDB == "" {
-		peerDB = filepath.Join(root, config.DefaultPeerDBName)
-	}
-	cfg.PeerDB = peerDB
-
-	functionDB := cfg.FunctionDB
-	if functionDB == "" {
-		functionDB = filepath.Join(root, config.DefaultFunctionDBName)
+	db := cfg.DB
+	if db == "" {
+		db = filepath.Join(root, config.DefaultDBName)
 	}
-	cfg.FunctionDB = functionDB
+	cfg.DB = db
 }
 
 func generateNodeDirName(id string) string {

config/config.go

@@ -16,9 +16,8 @@ const (
 
 // Default names for storage directories.
 const (
-	DefaultPeerDBName     = "peer-db"
-	DefaultFunctionDBName = "function-db"
-	DefaultWorkspaceName  = "workspace"
+	DefaultDBName        = "db"
+	DefaultWorkspaceName = "workspace"
 )
 
 var DefaultConfig = Config{
@@ -45,8 +44,7 @@ type Config struct {
 	LoadAttributes bool     `koanf:"load-attributes" flag:"load-attributes"` // TODO: Head node probably doesn't need attributes..?
 	Topics         []string `koanf:"topics"          flag:"topics"`
 
-	PeerDB     string `koanf:"peer-db"     flag:"peer-db"`
-	FunctionDB string `koanf:"function-db" flag:"function-db"` // TODO: Head node doesn't need a function database.
+	DB string `koanf:"db" flag:"db"`
 
 	Log          Log          `koanf:"log"`
 	Connectivity Connectivity `koanf:"connectivity"`
@@ -69,6 +67,7 @@ type Connectivity struct {
 	Websocket             bool   `koanf:"websocket"         flag:"websocket,w"`
 	WebsocketPort         uint   `koanf:"websocket-port"    flag:"websocket-port"`
 	WebsocketDialbackPort uint   `koanf:"websocket-dialback-port" flag:"websocket-dialback-port"`
+	NoDialbackPeers       bool   `koanf:"no-dialback-peers" flag:"no-dialback-peers"`
 }
 
 type Head struct {
@@ -108,10 +107,8 @@ func getFlagDescription(flag string) string {
 		return "node should try to load its attribute data from IPFS"
 	case "topics":
 		return "topics node should subscribe to"
-	case "peer-db":
-		return "path to the database used for persisting peer data"
-	case "function-db":
-		return "path to the database used for persisting function data"
+	case "db":
+		return "path to the database used for persisting peer and function data"
 	case "log-level":
 		return "log level to use"
 	case "address":
@@ -140,6 +137,8 @@ func getFlagDescription(flag string) string {
 		return "amount of CPU time allowed for Blockless Functions in the 0-1 range, 1 being unlimited"
 	case "memory-limit":
 		return "memory limit (kB) for Blockless Functions"
+	case "no-dialback-peers":
+		return "start without dialing back peers from previous runs"
 	default:
 		return ""
 	}