eval: add benchmarks

These changes add four benchmarks for the evaluator: one for the
evalutator itself, one focused on `fn::open`, one focused on
environment loading, and one for the entire stack. Each
benchmark uses the same set of environments with different
parameters for the time taken by an `fn::open` call or an
environment load.

Local results are below.

goos: darwin
goarch: arm64
pkg: github.com/pulumi/esc/eval
cpu: Apple M1 Max
BenchmarkEval-10           	    196	  6029274 ns/op	4626576 B/op	  23578 allocs/op
BenchmarkEval-10           	    195	  6051825 ns/op	4625788 B/op	  23576 allocs/op
BenchmarkEval-10           	    195	  6041508 ns/op	4625803 B/op	  23576 allocs/op
BenchmarkEval-10           	    195	  6070728 ns/op	4625819 B/op	  23577 allocs/op
BenchmarkEval-10           	    194	  6065964 ns/op	4625696 B/op	  23577 allocs/op
BenchmarkEval-10           	    196	  6075660 ns/op	4626653 B/op	  23578 allocs/op
BenchmarkEval-10           	    195	  6161176 ns/op	4626057 B/op	  23578 allocs/op
BenchmarkEval-10           	    194	  6123274 ns/op	4625977 B/op	  23577 allocs/op
BenchmarkEval-10           	    194	  6098131 ns/op	4626670 B/op	  23578 allocs/op
BenchmarkEval-10           	    196	  6064363 ns/op	4626654 B/op	  23579 allocs/op
BenchmarkEvalOpen-10       	      9	124478532 ns/op	4629023 B/op	  23614 allocs/op
BenchmarkEvalOpen-10       	      9	124931037 ns/op	4630025 B/op	  23609 allocs/op
BenchmarkEvalOpen-10       	      9	123642606 ns/op	4634537 B/op	  23618 allocs/op
BenchmarkEvalOpen-10       	      9	124292352 ns/op	4631007 B/op	  23611 allocs/op
BenchmarkEvalOpen-10       	      9	122027546 ns/op	4633108 B/op	  23617 allocs/op
BenchmarkEvalOpen-10       	      9	123727792 ns/op	4630472 B/op	  23613 allocs/op
BenchmarkEvalOpen-10       	      9	121888051 ns/op	4630566 B/op	  23617 allocs/op
BenchmarkEvalOpen-10       	      9	123239685 ns/op	4630572 B/op	  23619 allocs/op
BenchmarkEvalOpen-10       	      9	124360343 ns/op	4631780 B/op	  23616 allocs/op
BenchmarkEvalOpen-10       	      9	122856963 ns/op	4631718 B/op	  23613 allocs/op
BenchmarkEvalEnvLoad-10    	      4	305906031 ns/op	4634742 B/op	  23643 allocs/op
BenchmarkEvalEnvLoad-10    	      4	304883292 ns/op	4636056 B/op	  23644 allocs/op
BenchmarkEvalEnvLoad-10    	      4	309884938 ns/op	4637584 B/op	  23643 allocs/op
BenchmarkEvalEnvLoad-10    	      4	301879375 ns/op	4634852 B/op	  23632 allocs/op
BenchmarkEvalEnvLoad-10    	      4	301303094 ns/op	4632002 B/op	  23639 allocs/op
BenchmarkEvalEnvLoad-10    	      4	303479062 ns/op	4638610 B/op	  23647 allocs/op
BenchmarkEvalEnvLoad-10    	      4	303368458 ns/op	4634654 B/op	  23635 allocs/op
BenchmarkEvalEnvLoad-10    	      4	304764969 ns/op	4637982 B/op	  23655 allocs/op
BenchmarkEvalEnvLoad-10    	      4	308650562 ns/op	4636046 B/op	  23640 allocs/op
BenchmarkEvalEnvLoad-10    	      4	305385281 ns/op	4629074 B/op	  23628 allocs/op
BenchmarkEvalAll-10        	      3	421889528 ns/op	4647314 B/op	  23686 allocs/op
BenchmarkEvalAll-10        	      3	412783556 ns/op	4639096 B/op	  23683 allocs/op
BenchmarkEvalAll-10        	      3	418679208 ns/op	4642520 B/op	  23682 allocs/op
BenchmarkEvalAll-10        	      3	413973958 ns/op	4632093 B/op	  23650 allocs/op
BenchmarkEvalAll-10        	      3	415022083 ns/op	4645818 B/op	  23679 allocs/op
BenchmarkEvalAll-10        	      3	421364694 ns/op	4633874 B/op	  23662 allocs/op
BenchmarkEvalAll-10        	      3	413825070 ns/op	4634840 B/op	  23661 allocs/op
BenchmarkEvalAll-10        	      3	424170639 ns/op	4630058 B/op	  23666 allocs/op
BenchmarkEvalAll-10        	      3	421530069 ns/op	4638128 B/op	  23684 allocs/op
BenchmarkEvalAll-10        	      3	416247500 ns/op	4638744 B/op	  23670 allocs/op

eval/eval_test.go

@@ -23,8 +23,11 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"testing"
+	"time"
 
+	"github.com/pgavlin/fx"
 	"github.com/pulumi/esc"
 	"github.com/pulumi/esc/schema"
 	"github.com/pulumi/esc/syntax"
@@ -110,6 +113,19 @@ func (testSchemaProvider) Open(ctx context.Context, inputs map[string]esc.Value,
 	return esc.NewValue(inputs), nil
 }
 
+type benchProvider struct {
+	delay time.Duration
+}
+
+func (benchProvider) Schema() (*schema.Schema, *schema.Schema) {
+	return schema.Always(), schema.Always()
+}
+
+func (p benchProvider) Open(ctx context.Context, inputs map[string]esc.Value, context esc.EnvExecContext) (esc.Value, error) {
+	time.Sleep(p.delay)
+	return esc.NewValue(p.delay.String()), nil
+}
+
 type testProvider struct{}
 
 func (testProvider) Schema() (*schema.Schema, *schema.Schema) {
@@ -120,16 +136,20 @@ func (testProvider) Open(ctx context.Context, inputs map[string]esc.Value, conte
 	return esc.NewValue(inputs), nil
 }
 
-type testProviders struct{}
+type testProviders struct {
+	benchDelay time.Duration
+}
 
-func (testProviders) LoadProvider(ctx context.Context, name string) (esc.Provider, error) {
+func (tp testProviders) LoadProvider(ctx context.Context, name string) (esc.Provider, error) {
 	switch name {
 	case "error":
 		return errorProvider{}, nil
 	case "schema":
 		return testSchemaProvider{}, nil
 	case "test":
 		return testProvider{}, nil
+	case "bench":
+		return benchProvider{delay: tp.benchDelay}, nil
 	}
 	return nil, fmt.Errorf("unknown provider %q", name)
 }
@@ -146,6 +166,48 @@ func (e *testEnvironments) LoadEnvironment(ctx context.Context, name string) ([]
 	return bytes, rot128{}, nil
 }
 
+type benchEnvironments struct {
+	defs  map[string][]byte
+	delay time.Duration
+}
+
+func newBenchEnvironments(root string, delay time.Duration) (*benchEnvironments, error) {
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+
+	defs, err := fx.TryMap(fx.Map(
+		fx.FMap(fx.IterSlice(entries), func(e os.DirEntry) (fx.Pair[string, string], bool) {
+			name, ok := strings.CutSuffix(e.Name(), ".yaml")
+			return fx.NewPair(name, filepath.Join(root, e.Name())), ok
+		}),
+		func(namepath fx.Pair[string, string]) fx.Result[fx.Pair[string, []byte]] {
+			name, path := namepath.Unpack()
+			bytes, err := os.ReadFile(path)
+			if err != nil {
+				return fx.Err[fx.Pair[string, []byte]](err)
+			}
+			return fx.OK(fx.NewPair(name, bytes))
+		}),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &benchEnvironments{defs, delay}, nil
+}
+
+func (e *benchEnvironments) LoadEnvironment(ctx context.Context, name string) ([]byte, Decrypter, error) {
+	time.Sleep(e.delay)
+
+	bytes, ok := e.defs[name]
+	if !ok {
+		return nil, nil, os.ErrNotExist
+	}
+	return bytes, rot128{}, nil
+}
+
 func sortEnvironmentDiagnostics(diags syntax.Diagnostics) {
 	sort.Slice(diags, func(i, j int) bool {
 		di, dj := diags[i], diags[j]
@@ -200,6 +262,10 @@ func TestEval(t *testing.T) {
 	entries, err := os.ReadDir(path)
 	require.NoError(t, err)
 	for _, e := range entries {
+		if e.Name() == "bench" {
+			continue
+		}
+
 		t.Run(e.Name(), func(t *testing.T) {
 			basePath := filepath.Join(path, e.Name())
 			envPath := filepath.Join(basePath, "env.yaml")
@@ -328,3 +394,51 @@ func TestEval(t *testing.T) {
 		})
 	}
 }
+
+func benchmarkEval(b *testing.B, openDelay, loadDelay time.Duration) {
+	basePath := filepath.Join("testdata", "eval", "bench")
+	envPath := filepath.Join(basePath, "env.yaml")
+
+	envBytes, err := os.ReadFile(envPath)
+	require.NoError(b, err)
+
+	envs, err := newBenchEnvironments(basePath, loadDelay)
+	require.NoError(b, err)
+
+	for i := 0; i < b.N; i++ {
+		execContext, err := esc.NewExecContext(map[string]esc.Value{
+			"pulumi": esc.NewValue(map[string]esc.Value{
+				"user": esc.NewValue(map[string]esc.Value{
+					"id": esc.NewValue("USER_123"),
+				}),
+			}),
+		})
+		require.NoError(b, err)
+
+		environmentName := "bench"
+
+		env, loadDiags, err := LoadYAMLBytes(environmentName, envBytes)
+		require.NoError(b, err)
+		require.Empty(b, loadDiags)
+
+		_, evalDiags := EvalEnvironment(context.Background(), environmentName, env, rot128{}, testProviders{benchDelay: openDelay},
+			envs, execContext)
+		require.Empty(b, evalDiags)
+	}
+}
+
+func BenchmarkEval(b *testing.B) {
+	benchmarkEval(b, 0, 0)
+}
+
+func BenchmarkEvalOpen(b *testing.B) {
+	benchmarkEval(b, 10*time.Millisecond, 0)
+}
+
+func BenchmarkEvalEnvLoad(b *testing.B) {
+	benchmarkEval(b, 0, 10*time.Millisecond)
+}
+
+func BenchmarkEvalAll(b *testing.B) {
+	benchmarkEval(b, 10*time.Millisecond, 10*time.Millisecond)
+}

eval/testdata/eval/bench/a.yaml

@@ -0,0 +1,7 @@
+values:
+  env_a:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_a: ${env_a.token}
+  pulumiConfig:
+    env_a::token: ${env_a.token}

eval/testdata/eval/bench/b.yaml

@@ -0,0 +1,7 @@
+values:
+  env_b:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_b: ${env_b.token}
+  pulumiConfig:
+    env_b::token: ${env_b.token}

eval/testdata/eval/bench/c.yaml

@@ -0,0 +1,7 @@
+values:
+  env_c:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_c: ${env_c.token}
+  pulumiConfig:
+    env_c::token: ${env_c.token}

eval/testdata/eval/bench/d.yaml

@@ -0,0 +1,7 @@
+values:
+  env_d:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_d: ${env_d.token}
+  pulumiConfig:
+    env_d::token: ${env_d.token}

eval/testdata/eval/bench/e.yaml

@@ -0,0 +1,7 @@
+values:
+  env_e:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_e: ${env_e.token}
+  pulumiConfig:
+    env_e::token: ${env_e.token}

eval/testdata/eval/bench/env.yaml

@@ -0,0 +1,39 @@
+imports:
+  - a
+  - b
+  - c
+  - d
+  - e
+  - f
+  - g
+  - h
+  - i
+  - j
+  - k
+  - l
+  - m
+  - n
+  - o
+  - p
+  - q
+  - r
+  - s
+  - t
+  - u
+  - v
+  - w
+  - x
+  - y
+  - z
+values:
+  delays:
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}
+    - fn::open::bench: {}

eval/testdata/eval/bench/f.yaml

@@ -0,0 +1,7 @@
+values:
+  env_f:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_f: ${env_f.token}
+  pulumiConfig:
+    env_f::token: ${env_f.token}

eval/testdata/eval/bench/g.yaml

@@ -0,0 +1,7 @@
+values:
+  env_g:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_g: ${env_g.token}
+  pulumiConfig:
+    env_g::token: ${env_g.token}

eval/testdata/eval/bench/h.yaml

@@ -0,0 +1,7 @@
+values:
+  env_h:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_h: ${env_h.token}
+  pulumiConfig:
+    env_h::token: ${env_h.token}

eval/testdata/eval/bench/i.yaml

@@ -0,0 +1,7 @@
+values:
+  env_i:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_i: ${env_i.token}
+  pulumiConfig:
+    env_i::token: ${env_i.token}

eval/testdata/eval/bench/j.yaml

@@ -0,0 +1,7 @@
+values:
+  env_j:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_j: ${env_j.token}
+  pulumiConfig:
+    env_j::token: ${env_j.token}

eval/testdata/eval/bench/k.yaml

@@ -0,0 +1,7 @@
+values:
+  env_k:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_k: ${env_k.token}
+  pulumiConfig:
+    env_k::token: ${env_k.token}

eval/testdata/eval/bench/l.yaml

@@ -0,0 +1,7 @@
+values:
+  env_l:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_l: ${env_l.token}
+  pulumiConfig:
+    env_l::token: ${env_l.token}

eval/testdata/eval/bench/m.yaml

@@ -0,0 +1,7 @@
+values:
+  env_m:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_m: ${env_m.token}
+  pulumiConfig:
+    env_m::token: ${env_m.token}

eval/testdata/eval/bench/n.yaml

@@ -0,0 +1,7 @@
+values:
+  env_n:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_n: ${env_n.token}
+  pulumiConfig:
+    env_n::token: ${env_n.token}

eval/testdata/eval/bench/o.yaml

@@ -0,0 +1,7 @@
+values:
+  env_o:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_o: ${env_o.token}
+  pulumiConfig:
+    env_o::token: ${env_o.token}

eval/testdata/eval/bench/p.yaml

@@ -0,0 +1,7 @@
+values:
+  env_p:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_p: ${env_p.token}
+  pulumiConfig:
+    env_p::token: ${env_p.token}

eval/testdata/eval/bench/q.yaml

@@ -0,0 +1,7 @@
+values:
+  env_q:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_q: ${env_q.token}
+  pulumiConfig:
+    env_q::token: ${env_q.token}

eval/testdata/eval/bench/r.yaml

@@ -0,0 +1,7 @@
+values:
+  env_r:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_r: ${env_r.token}
+  pulumiConfig:
+    env_r::token: ${env_r.token}

eval/testdata/eval/bench/s.yaml

@@ -0,0 +1,7 @@
+values:
+  env_s:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_s: ${env_s.token}
+  pulumiConfig:
+    env_s::token: ${env_s.token}

eval/testdata/eval/bench/t.yaml

@@ -0,0 +1,7 @@
+values:
+  env_t:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_t: ${env_t.token}
+  pulumiConfig:
+    env_t::token: ${env_t.token}

eval/testdata/eval/bench/u.yaml

@@ -0,0 +1,7 @@
+values:
+  env_u:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_u: ${env_u.token}
+  pulumiConfig:
+    env_u::token: ${env_u.token}

eval/testdata/eval/bench/v.yaml

@@ -0,0 +1,7 @@
+values:
+  env_v:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_v: ${env_v.token}
+  pulumiConfig:
+    env_v::token: ${env_v.token}

eval/testdata/eval/bench/w.yaml

@@ -0,0 +1,7 @@
+values:
+  env_w:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_w: ${env_w.token}
+  pulumiConfig:
+    env_w::token: ${env_w.token}

eval/testdata/eval/bench/x.yaml

@@ -0,0 +1,7 @@
+values:
+  env_x:
+    token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}}
+  environmentVariables:
+    env_x: ${env_x.token}
+  pulumiConfig:
+    env_x::token: ${env_x.token}