fix(gno.land): pre-load all standard libraries in vm.Initialize

gno.land/cmd/gnoland/start.go

@@ -234,17 +234,17 @@ func execStart(ctx context.Context, c *startCfg, io commands.IO) error {
 		return fmt.Errorf("unable to initialize telemetry, %w", err)
 	}
 
+	// Print the starting graphic
+	if c.logFormat != string(log.JSONFormat) {
+		io.Println(startGraphic)
+	}
+
 	// Create application and node
 	cfg.LocalApp, err = gnoland.NewApp(nodeDir, c.skipFailingGenesisTxs, logger, c.genesisMaxVMCycles)
 	if err != nil {
 		return fmt.Errorf("unable to create the Gnoland app, %w", err)
 	}
 
-	// Print the starting graphic
-	if c.logFormat != string(log.JSONFormat) {
-		io.Println(startGraphic)
-	}
-
 	// Create a default node, with the given setup
 	gnoNode, err := node.DefaultNewNode(cfg, genesisPath, logger)
 	if err != nil {

gno.land/pkg/gnoland/app.go

@@ -121,7 +121,7 @@
 
 	// Initialize the VMKeeper.
 	ms := baseApp.GetCacheMultiStore()
-	vmKpr.Initialize(ms)
+	vmKpr.Initialize(cfg.Logger, ms)
 	ms.MultiWrite() // XXX why was't this needed?
 
 	return baseApp, nil

gno.land/pkg/sdk/vm/builtins.go

@@ -1,47 +1,11 @@
 package vm
 
 import (
-	"os"
-	"path/filepath"
-
-	gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
 	"github.com/gnolang/gno/tm2/pkg/crypto"
-	osm "github.com/gnolang/gno/tm2/pkg/os"
 	"github.com/gnolang/gno/tm2/pkg/sdk"
 	"github.com/gnolang/gno/tm2/pkg/std"
 )
 
-// NOTE: this function may add loaded dependencies to store if they don't
-// already exist, including mem packages. If this happens during a transaction
-// with the tx context store, the transaction caller will pay for operations.
-// NOTE: native functions/methods added here must be quick operations, or
-// account for gas before operation.
-// TODO: define criteria for inclusion, and solve gas calculations(???).
-func (vm *VMKeeper) getPackage(pkgPath string, store gno.Store) (pn *gno.PackageNode, pv *gno.PackageValue) {
-	// otherwise, built-in package value.
-	// first, load from filepath.
-	stdlibPath := filepath.Join(vm.stdlibsDir, pkgPath)
-	if !osm.DirExists(stdlibPath) {
-		// does not exist.
-		return nil, nil
-	}
-	memPkg := gno.ReadMemPackage(stdlibPath, pkgPath)
-	if memPkg.IsEmpty() {
-		// no gno files are present, skip this package
-		return nil, nil
-	}
-
-	m2 := gno.NewMachineWithOptions(gno.MachineOptions{
-		PkgPath: "gno.land/r/stdlibs/" + pkgPath,
-		// PkgPath: pkgPath,
-		Output: os.Stdout,
-		Store:  store,
-	})
-	defer m2.Release()
-	pn, pv = m2.RunMemPackage(memPkg, true)
-	return
-}
-
 // ----------------------------------------
 // SDKBanker
 

gno.land/pkg/sdk/vm/common_test.go

@@ -41,7 +41,9 @@ func setupTestEnv() testEnv {
 	stdlibsDir := filepath.Join("..", "..", "..", "..", "gnovm", "stdlibs")
 	vmk := NewVMKeeper(baseCapKey, iavlCapKey, acck, bank, stdlibsDir, 100_000_000)
 
-	vmk.Initialize(ms.MultiCacheWrap())
+	mcw := ms.MultiCacheWrap()
+	vmk.Initialize(log.NewNoopLogger(), mcw)
+	mcw.MultiWrite()
 
 	return testEnv{ctx: ctx, vmk: vmk, bank: bank, acck: acck}
 }

gno.land/pkg/sdk/vm/gas_test.go

@@ -67,7 +67,7 @@ func TestAddPkgDeliverTx(t *testing.T) {
 	gasDeliver := gctx.GasMeter().GasConsumed()
 
 	assert.True(t, res.IsOK())
-	assert.Equal(t, int64(87929), gasDeliver)
+	assert.Equal(t, int64(87965), gasDeliver)
 }
 
 // Enough gas for a failed transaction.

gno.land/pkg/sdk/vm/keeper.go

@@ -6,12 +6,16 @@
 	"bytes"
 	"context"
 	"fmt"
+	"log/slog"
 	"os"
+	"path/filepath"
 	"strings"
+	"time"
 
 	gno "github.com/gnolang/gno/gnovm/pkg/gnolang"
 	"github.com/gnolang/gno/gnovm/stdlibs"
 	"github.com/gnolang/gno/tm2/pkg/errors"
+	osm "github.com/gnolang/gno/tm2/pkg/os"
 	"github.com/gnolang/gno/tm2/pkg/sdk"
 	"github.com/gnolang/gno/tm2/pkg/sdk/auth"
 	"github.com/gnolang/gno/tm2/pkg/sdk/bank"
@@ -73,20 +77,36 @@
 	return vmk
 }
 
-func (vm *VMKeeper) Initialize(ms store.MultiStore) {
+func (vm *VMKeeper) Initialize(logger *slog.Logger, ms store.MultiStore) {
 	if vm.gnoStore != nil {
 		panic("should not happen")
 	}
 	alloc := gno.NewAllocator(maxAllocTx)
 	baseSDKStore := ms.GetStore(vm.baseKey)
 	iavlSDKStore := ms.GetStore(vm.iavlKey)
 	vm.gnoStore = gno.NewStore(alloc, baseSDKStore, iavlSDKStore)
-	vm.gnoStore.SetPackageGetter(vm.getPackage)
 	vm.gnoStore.SetNativeStore(stdlibs.NativeStore)
-	if vm.gnoStore.NumMemPackages() > 0 {
+	if vm.gnoStore.NumMemPackages() == 0 {
+		// No packages in the store; set up the stdlibs.
+		start := time.Now()
+
+		stdlibInitList := stdlibs.InitOrder()
+		for _, lib := range stdlibInitList {
+			if lib == "testing" {
+				// XXX: testing is skipped for now while it uses testing-only packages
+				// like fmt and encoding/json
+				continue
+			}
+			vm.loadPackage(lib, vm.gnoStore)
+		}
+
+		logger.Debug("Standard libraries initialized",
+			"elapsed", time.Since(start))
+	} else {
 		// for now, all mem packages must be re-run after reboot.
 		// TODO remove this, and generally solve for in-mem garbage collection
 		// and memory management across many objects/types/nodes/packages.
+		start := time.Now()
 		m2 := gno.NewMachineWithOptions(
 			gno.MachineOptions{
 				PkgPath: "",
@@ -97,7 +117,31 @@
 		gno.DisableDebug()
 		m2.PreprocessAllFilesAndSaveBlockNodes()
 		gno.EnableDebug()
+		logger.Debug("GnoVM packages preprocessed",
+			"elapsed", time.Since(start))
+	}
+}
+
+func (vm *VMKeeper) loadPackage(pkgPath string, store gno.Store) {
+	stdlibPath := filepath.Join(vm.stdlibsDir, pkgPath)
+	if !osm.DirExists(stdlibPath) {
+		// does not exist.
+		return
+	}
+	memPkg := gno.ReadMemPackage(stdlibPath, pkgPath)
+	if memPkg.IsEmpty() {
+		// no gno files are present, skip this package
+		return
 	}
+
+	m := gno.NewMachineWithOptions(gno.MachineOptions{
+		PkgPath: "gno.land/r/stdlibs/" + pkgPath,
+		// PkgPath: pkgPath, XXX why?
+		Output: os.Stdout,
+		Store:  store,
+	})
+	defer m.Release()
+	m.RunMemPackage(memPkg, true)
 }
 
 func (vm *VMKeeper) getGnoStore(ctx sdk.Context) gno.Store {

gnovm/pkg/gnolang/store.go

@@ -562,7 +562,7 @@
 		// implementations works by running Machine.RunMemPackage with save = true,
 		// which would add the package to the store after running.
 		// Some packages may never be persisted, thus why we only attempt this twice.
-		if !isRetry {
+		if !isRetry && ds.pkgGetter != nil {
 			if pv := ds.GetPackage(path, false); pv != nil {
 				return ds.getMemPackage(path, true)
 			}

gnovm/stdlibs/native.go → gnovm/stdlibs/generated.go

@@ -1018,3 +1018,51 @@
 		},
 	},
 }
+
+var initOrder = [...]string{
+	"errors",
+	"internal/bytealg",
+	"io",
+	"unicode",
+	"unicode/utf8",
+	"bytes",
+	"strings",
+	"bufio",
+	"encoding/binary",
+	"math/bits",
+	"math",
+	"crypto/chacha20/chacha",
+	"crypto/cipher",
+	"crypto/chacha20",
+	"strconv",
+	"crypto/chacha20/rand",
+	"crypto/ed25519",
+	"crypto/sha256",
+	"encoding",
+	"encoding/base64",
+	"encoding/hex",
+	"hash",
+	"hash/adler32",
+	"math/overflow",
+	"math/rand",
+	"path",
+	"sort",
+	"net/url",
+	"regexp/syntax",
+	"regexp",
+	"std",
+	"testing",
+	"time",
+	"unicode/utf16",
+}
+
+// InitOrder returns the initialization order of the standard libraries.
+// This is calculated starting from the list of all standard libraries and
+// iterating through each: if a package depends on an unitialized package, that
+// is processed first, and so on recursively; matching the behaviour of Go's
+// [program initialization].
+//
+// [program initialization]: https://go.dev/ref/spec#Program_initialization
+func InitOrder() []string {
+	return initOrder[:]
+}

gnovm/tests/stdlibs/native.go → gnovm/tests/stdlibs/generated.go

@@ -315,3 +315,19 @@
 		},
 	},
 }
+
+var initOrder = [...]string{
+	"std",
+	"testing",
+}
+
+// InitOrder returns the initialization order of the standard libraries.
+// This is calculated starting from the list of all standard libraries and
+// iterating through each: if a package depends on an unitialized package, that
+// is processed first, and so on recursively; matching the behaviour of Go's
+// [program initialization].
+//
+// [program initialization]: https://go.dev/ref/spec#Program_initialization
+func InitOrder() []string {
+	return initOrder[:]
+}