fix(gno.land): pre-load all standard libraries in vm.Initialize (gnol…

…ang#2504)

Fixes gnolang#2283 (with a different approach from gnolang#2319)

This PR loads all the standard libraries into the database when
vm.Initialize is called. It doesn't fully fix the problems that gnolang#2319 is
trying to address, but it does fix the most immediate bug of not being
able to publish certain packages on portal loop.

With these changes, we don't need a PackageGetter on-chain anymore. All
packages are already loaded into the store, thus solving the problem
@jaekwon was talking about
[here](gnolang#2319 (comment))
at its root.

This PR has a problem, which is that adding loading the entire stdlibs
in the VM initialization step brings a huge overhead computationally;
this is not a problem on the node, but it is when testing as it needs to
happen very often. This translates to 2x slower txtar tests, compared to
master. On my PC, it adds a 2-3 second overhead whenever running
Initialize.

I tried working out on a system which could save the data to quickly
recover it, at least for some cases where we need to Initialize often;
[see this
diff](https://gist.github.com/thehowl/cb1ee79e63cf77d3f323730580eb2d18).
But I didn't get it to work so far; after copying the DB, attempting
initialization crashes because [ParseMemPackage is being handed a nil
package, for some
reason](https://gist.github.com/thehowl/d1efa51858d865fb5beb9c3a9cb0eeef).
@gfanton, any tips?? :)) I'd want to avoid lazy loading in the node, as
that's what got us here in the first place.

<details><summary>Contributors' checklist...</summary>

- [ ] Added new tests, or not needed, or not feasible
- [x] Provided an example (e.g. screenshot) to aid review or the PR is
self-explanatory
- [x] Updated the official documentation or not needed
- [x] No breaking changes were made, or a `BREAKING CHANGE: xxx` message
was included in the description
- [x] Added references to related issues and PRs
- [x] Provided any useful hints for running manual tests
- [x] Added new benchmarks to [generated
graphs](https://gnoland.github.io/benchmarks), if any. More info
[here](https://github.com/gnolang/gno/blob/master/.benchmarks/README.md).
</details>

---------

Co-authored-by: Manfred Touron <[email protected]>

gno.land/cmd/gnoland/start.go

@@ -234,17 +234,17 @@ func execStart(ctx context.Context, c *startCfg, io commands.IO) error {
 		return fmt.Errorf("unable to initialize telemetry, %w", err)
 	}
 
+	// Print the starting graphic
+	if c.logFormat != string(log.JSONFormat) {
+		io.Println(startGraphic)
+	}
+
 	// Create application and node
 	cfg.LocalApp, err = gnoland.NewApp(nodeDir, c.skipFailingGenesisTxs, logger, c.genesisMaxVMCycles)
 	if err != nil {
 		return fmt.Errorf("unable to create the Gnoland app, %w", err)
 	}
 
-	// Print the starting graphic
-	if c.logFormat != string(log.JSONFormat) {
-		io.Println(startGraphic)
-	}
-
 	// Create a default node, with the given setup
 	gnoNode, err := node.DefaultNewNode(cfg, genesisPath, logger)
 	if err != nil {

gno.land/pkg/gnoland/app.go

@@ -33,6 +33,11 @@ type AppOptions struct {
 	GenesisTxHandler GenesisTxHandler
 	Logger           *slog.Logger
 	MaxCycles        int64
+	// Whether to cache the result of loading the standard libraries.
+	// This is useful if you have to start many nodes, like in testing.
+	// This disables loading existing packages; so it should only be used
+	// on a fresh database.
+	CacheStdlibLoad bool
 }
 
 func NewAppOptions() *AppOptions {
@@ -121,7 +126,7 @@ func NewAppWithOptions(cfg *AppOptions) (abci.Application, error) {
 
 	// Initialize the VMKeeper.
 	ms := baseApp.GetCacheMultiStore()
-	vmKpr.Initialize(ms)
+	vmKpr.Initialize(cfg.Logger, ms, cfg.CacheStdlibLoad)
 	ms.MultiWrite() // XXX why was't this needed?
 
 	return baseApp, nil
@@ -157,7 +162,12 @@ func PanicOnFailingTxHandler(ctx sdk.Context, tx std.Tx, res sdk.Result) {
 }
 
 // InitChainer returns a function that can initialize the chain with genesis.
-func InitChainer(baseApp *sdk.BaseApp, acctKpr auth.AccountKeeperI, bankKpr bank.BankKeeperI, resHandler GenesisTxHandler) func(sdk.Context, abci.RequestInitChain) abci.ResponseInitChain {
+func InitChainer(
+	baseApp *sdk.BaseApp,
+	acctKpr auth.AccountKeeperI,
+	bankKpr bank.BankKeeperI,
+	resHandler GenesisTxHandler,
+) func(sdk.Context, abci.RequestInitChain) abci.ResponseInitChain {
 	return func(ctx sdk.Context, req abci.RequestInitChain) abci.ResponseInitChain {
 		if req.AppState != nil {
 			// Get genesis state

gno.land/pkg/gnoland/node_inmemory.go

@@ -94,6 +94,7 @@ func NewInMemoryNode(logger *slog.Logger, cfg *InMemoryNodeConfig) (*node.Node,
 		GenesisTxHandler: cfg.GenesisTxHandler,
 		MaxCycles:        cfg.GenesisMaxVMCycles,
 		DB:               memdb.NewMemDB(),
+		CacheStdlibLoad:  true,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error initializing new app: %w", err)

gno.land/pkg/sdk/vm/builtins.go

@@ -1,47 +1,11 @@
 package vm
 
 import (
-	"os"
-	"path/filepath"
-
-	gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
 	"github.com/gnolang/gno/tm2/pkg/crypto"
-	osm "github.com/gnolang/gno/tm2/pkg/os"
 	"github.com/gnolang/gno/tm2/pkg/sdk"
 	"github.com/gnolang/gno/tm2/pkg/std"
 )
 
-// NOTE: this function may add loaded dependencies to store if they don't
-// already exist, including mem packages. If this happens during a transaction
-// with the tx context store, the transaction caller will pay for operations.
-// NOTE: native functions/methods added here must be quick operations, or
-// account for gas before operation.
-// TODO: define criteria for inclusion, and solve gas calculations(???).
-func (vm *VMKeeper) getPackage(pkgPath string, store gno.Store) (pn *gno.PackageNode, pv *gno.PackageValue) {
-	// otherwise, built-in package value.
-	// first, load from filepath.
-	stdlibPath := filepath.Join(vm.stdlibsDir, pkgPath)
-	if !osm.DirExists(stdlibPath) {
-		// does not exist.
-		return nil, nil
-	}
-	memPkg := gno.ReadMemPackage(stdlibPath, pkgPath)
-	if memPkg.IsEmpty() {
-		// no gno files are present, skip this package
-		return nil, nil
-	}
-
-	m2 := gno.NewMachineWithOptions(gno.MachineOptions{
-		PkgPath: "gno.land/r/stdlibs/" + pkgPath,
-		// PkgPath: pkgPath,
-		Output: os.Stdout,
-		Store:  store,
-	})
-	defer m2.Release()
-	pn, pv = m2.RunMemPackage(memPkg, true)
-	return
-}
-
 // ----------------------------------------
 // SDKBanker
 

gno.land/pkg/sdk/vm/common_test.go

@@ -25,6 +25,14 @@ type testEnv struct {
 }
 
 func setupTestEnv() testEnv {
+	return _setupTestEnv(true)
+}
+
+func setupTestEnvCold() testEnv {
+	return _setupTestEnv(false)
+}
+
+func _setupTestEnv(cacheStdlibs bool) testEnv {
 	db := memdb.NewMemDB()
 
 	baseCapKey := store.NewStoreKey("baseCapKey")
@@ -41,7 +49,9 @@ func setupTestEnv() testEnv {
 	stdlibsDir := filepath.Join("..", "..", "..", "..", "gnovm", "stdlibs")
 	vmk := NewVMKeeper(baseCapKey, iavlCapKey, acck, bank, stdlibsDir, 100_000_000)
 
-	vmk.Initialize(ms.MultiCacheWrap())
+	mcw := ms.MultiCacheWrap()
+	vmk.Initialize(log.NewNoopLogger(), mcw, cacheStdlibs)
+	mcw.MultiWrite()
 
 	return testEnv{ctx: ctx, vmk: vmk, bank: bank, acck: acck}
 }

gno.land/pkg/sdk/vm/gas_test.go

@@ -67,7 +67,7 @@ func TestAddPkgDeliverTx(t *testing.T) {
 	gasDeliver := gctx.GasMeter().GasConsumed()
 
 	assert.True(t, res.IsOK())
-	assert.Equal(t, int64(87929), gasDeliver)
+	assert.Equal(t, int64(87965), gasDeliver)
 }
 
 // Enough gas for a failed transaction.

gno.land/pkg/sdk/vm/keeper.go

@@ -6,17 +6,25 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"log/slog"
 	"os"
+	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
 	"github.com/gnolang/gno/gnovm/stdlibs"
+	"github.com/gnolang/gno/tm2/pkg/db/memdb"
 	"github.com/gnolang/gno/tm2/pkg/errors"
+	osm "github.com/gnolang/gno/tm2/pkg/os"
 	"github.com/gnolang/gno/tm2/pkg/sdk"
 	"github.com/gnolang/gno/tm2/pkg/sdk/auth"
 	"github.com/gnolang/gno/tm2/pkg/sdk/bank"
 	"github.com/gnolang/gno/tm2/pkg/std"
 	"github.com/gnolang/gno/tm2/pkg/store"
+	"github.com/gnolang/gno/tm2/pkg/store/dbadapter"
+	"github.com/gnolang/gno/tm2/pkg/store/types"
 	"github.com/gnolang/gno/tm2/pkg/telemetry"
 	"github.com/gnolang/gno/tm2/pkg/telemetry/metrics"
 	"go.opentelemetry.io/otel/attribute"
@@ -73,31 +81,133 @@ func NewVMKeeper(
 	return vmk
 }
 
-func (vm *VMKeeper) Initialize(ms store.MultiStore) {
+func (vm *VMKeeper) Initialize(
+	logger *slog.Logger,
+	ms store.MultiStore,
+	cacheStdlibLoad bool,
+) {
 	if vm.gnoStore != nil {
 		panic("should not happen")
 	}
-	alloc := gno.NewAllocator(maxAllocTx)
 	baseSDKStore := ms.GetStore(vm.baseKey)
 	iavlSDKStore := ms.GetStore(vm.iavlKey)
-	vm.gnoStore = gno.NewStore(alloc, baseSDKStore, iavlSDKStore)
-	vm.gnoStore.SetPackageGetter(vm.getPackage)
-	vm.gnoStore.SetNativeStore(stdlibs.NativeStore)
-	if vm.gnoStore.NumMemPackages() > 0 {
+
+	if cacheStdlibLoad {
+		// Testing case (using the cache speeds up starting many nodes)
+		vm.gnoStore = cachedStdlibLoad(vm.stdlibsDir, baseSDKStore, iavlSDKStore)
+	} else {
+		// On-chain case
+		vm.gnoStore = uncachedPackageLoad(logger, vm.stdlibsDir, baseSDKStore, iavlSDKStore)
+	}
+}
+
+func uncachedPackageLoad(
+	logger *slog.Logger,
+	stdlibsDir string,
+	baseStore, iavlStore store.Store,
+) gno.Store {
+	alloc := gno.NewAllocator(maxAllocTx)
+	gnoStore := gno.NewStore(alloc, baseStore, iavlStore)
+	gnoStore.SetNativeStore(stdlibs.NativeStore)
+	if gnoStore.NumMemPackages() == 0 {
+		// No packages in the store; set up the stdlibs.
+		start := time.Now()
+
+		loadStdlib(stdlibsDir, gnoStore)
+
+		logger.Debug("Standard libraries initialized",
+			"elapsed", time.Since(start))
+	} else {
 		// for now, all mem packages must be re-run after reboot.
 		// TODO remove this, and generally solve for in-mem garbage collection
 		// and memory management across many objects/types/nodes/packages.
+		start := time.Now()
+
 		m2 := gno.NewMachineWithOptions(
 			gno.MachineOptions{
 				PkgPath: "",
 				Output:  os.Stdout, // XXX
-				Store:   vm.gnoStore,
+				Store:   gnoStore,
 			})
 		defer m2.Release()
 		gno.DisableDebug()
 		m2.PreprocessAllFilesAndSaveBlockNodes()
 		gno.EnableDebug()
+
+		logger.Debug("GnoVM packages preprocessed",
+			"elapsed", time.Since(start))
 	}
+	return gnoStore
+}
+
+func cachedStdlibLoad(stdlibsDir string, baseStore, iavlStore store.Store) gno.Store {
+	cachedStdlibOnce.Do(func() {
+		cachedStdlibBase = memdb.NewMemDB()
+		cachedStdlibIavl = memdb.NewMemDB()
+
+		cachedGnoStore = gno.NewStore(nil,
+			dbadapter.StoreConstructor(cachedStdlibBase, types.StoreOptions{}),
+			dbadapter.StoreConstructor(cachedStdlibIavl, types.StoreOptions{}))
+		cachedGnoStore.SetNativeStore(stdlibs.NativeStore)
+		loadStdlib(stdlibsDir, cachedGnoStore)
+	})
+
+	itr := cachedStdlibBase.Iterator(nil, nil)
+	for ; itr.Valid(); itr.Next() {
+		baseStore.Set(itr.Key(), itr.Value())
+	}
+
+	itr = cachedStdlibIavl.Iterator(nil, nil)
+	for ; itr.Valid(); itr.Next() {
+		iavlStore.Set(itr.Key(), itr.Value())
+	}
+
+	alloc := gno.NewAllocator(maxAllocTx)
+	gs := gno.NewStore(alloc, baseStore, iavlStore)
+	gs.SetNativeStore(stdlibs.NativeStore)
+	gno.CopyCachesFromStore(gs, cachedGnoStore)
+	return gs
+}
+
+var (
+	cachedStdlibOnce sync.Once
+	cachedStdlibBase *memdb.MemDB
+	cachedStdlibIavl *memdb.MemDB
+	cachedGnoStore   gno.Store
+)
+
+func loadStdlib(stdlibsDir string, store gno.Store) {
+	stdlibInitList := stdlibs.InitOrder()
+	for _, lib := range stdlibInitList {
+		if lib == "testing" {
+			// XXX: testing is skipped for now while it uses testing-only packages
+			// like fmt and encoding/json
+			continue
+		}
+		loadStdlibPackage(lib, stdlibsDir, store)
+	}
+}
+
+func loadStdlibPackage(pkgPath, stdlibsDir string, store gno.Store) {
+	stdlibPath := filepath.Join(stdlibsDir, pkgPath)
+	if !osm.DirExists(stdlibPath) {
+		// does not exist.
+		panic(fmt.Sprintf("failed loading stdlib %q: does not exist", pkgPath))
+	}
+	memPkg := gno.ReadMemPackage(stdlibPath, pkgPath)
+	if memPkg.IsEmpty() {
+		// no gno files are present
+		panic(fmt.Sprintf("failed loading stdlib %q: not a valid MemPackage", pkgPath))
+	}
+
+	m := gno.NewMachineWithOptions(gno.MachineOptions{
+		PkgPath: "gno.land/r/stdlibs/" + pkgPath,
+		// PkgPath: pkgPath, XXX why?
+		Output: os.Stdout,
+		Store:  store,
+	})
+	defer m.Release()
+	m.RunMemPackage(memPkg, true)
 }
 
 func (vm *VMKeeper) getGnoStore(ctx sdk.Context) gno.Store {

gno.land/pkg/sdk/vm/keeper_test.go

@@ -366,6 +366,18 @@ func main() {
 // Call Run with stdlibs.
 func TestVMKeeperRunImportStdlibs(t *testing.T) {
 	env := setupTestEnv()
+	testVMKeeperRunImportStdlibs(t, env)
+}
+
+// Call Run with stdlibs, "cold" loading the standard libraries
+func TestVMKeeperRunImportStdlibsColdStdlibLoad(t *testing.T) {
+	env := setupTestEnvCold()
+	testVMKeeperRunImportStdlibs(t, env)
+}
+
+func testVMKeeperRunImportStdlibs(t *testing.T, env testEnv) {
+	t.Helper()
+
 	ctx := env.ctx
 
 	// Give "addr1" some gnots.

gnovm/pkg/gnolang/store.go

@@ -112,6 +112,16 @@ func NewStore(alloc *Allocator, baseStore, iavlStore store.Store) *defaultStore
 	return ds
 }
 
+// CopyCachesFromStore allows to copy a store's internal object, type and
+// BlockNode cache into the dst store.
+// This is mostly useful for testing, where many stores have to be initialized.
+func CopyCachesFromStore(dst, src Store) {
+	ds, ss := dst.(*defaultStore), src.(*defaultStore)
+	ds.cacheObjects = maps.Clone(ss.cacheObjects)
+	ds.cacheTypes = maps.Clone(ss.cacheTypes)
+	ds.cacheNodes = maps.Clone(ss.cacheNodes)
+}
+
 func (ds *defaultStore) GetAllocator() *Allocator {
 	return ds.alloc
 }
@@ -562,7 +572,7 @@ func (ds *defaultStore) getMemPackage(path string, isRetry bool) *std.MemPackage
 		// implementations works by running Machine.RunMemPackage with save = true,
 		// which would add the package to the store after running.
 		// Some packages may never be persisted, thus why we only attempt this twice.
-		if !isRetry {
+		if !isRetry && ds.pkgGetter != nil {
 			if pv := ds.GetPackage(path, false); pv != nil {
 				return ds.getMemPackage(path, true)
 			}