diff --git a/eval/eval_test.go b/eval/eval_test.go index 3f5f1e12..62444b2f 100644 --- a/eval/eval_test.go +++ b/eval/eval_test.go @@ -23,8 +23,11 @@ import ( "os" "path/filepath" "sort" + "strings" "testing" + "time" + "github.com/pgavlin/fx" "github.com/pulumi/esc" "github.com/pulumi/esc/schema" "github.com/pulumi/esc/syntax" @@ -110,6 +113,19 @@ func (testSchemaProvider) Open(ctx context.Context, inputs map[string]esc.Value, return esc.NewValue(inputs), nil } +type benchProvider struct { + delay time.Duration +} + +func (benchProvider) Schema() (*schema.Schema, *schema.Schema) { + return schema.Always(), schema.Always() +} + +func (p benchProvider) Open(ctx context.Context, inputs map[string]esc.Value, context esc.EnvExecContext) (esc.Value, error) { + time.Sleep(p.delay) + return esc.NewValue(p.delay.String()), nil +} + type testProvider struct{} func (testProvider) Schema() (*schema.Schema, *schema.Schema) { @@ -120,9 +136,11 @@ func (testProvider) Open(ctx context.Context, inputs map[string]esc.Value, conte return esc.NewValue(inputs), nil } -type testProviders struct{} +type testProviders struct { + benchDelay time.Duration +} -func (testProviders) LoadProvider(ctx context.Context, name string) (esc.Provider, error) { +func (tp testProviders) LoadProvider(ctx context.Context, name string) (esc.Provider, error) { switch name { case "error": return errorProvider{}, nil @@ -130,6 +148,8 @@ func (testProviders) LoadProvider(ctx context.Context, name string) (esc.Provide return testSchemaProvider{}, nil case "test": return testProvider{}, nil + case "bench": + return benchProvider{delay: tp.benchDelay}, nil } return nil, fmt.Errorf("unknown provider %q", name) } @@ -146,6 +166,48 @@ func (e *testEnvironments) LoadEnvironment(ctx context.Context, name string) ([] return bytes, rot128{}, nil } +type benchEnvironments struct { + defs map[string][]byte + delay time.Duration +} + +func newBenchEnvironments(root string, delay time.Duration) (*benchEnvironments, error) { + entries, err := os.ReadDir(root) + if err != nil { + return nil, err + } + + defs, err := fx.TryMap(fx.Map( + fx.FMap(fx.IterSlice(entries), func(e os.DirEntry) (fx.Pair[string, string], bool) { + name, ok := strings.CutSuffix(e.Name(), ".yaml") + return fx.NewPair(name, filepath.Join(root, e.Name())), ok + }), + func(namepath fx.Pair[string, string]) fx.Result[fx.Pair[string, []byte]] { + name, path := namepath.Unpack() + bytes, err := os.ReadFile(path) + if err != nil { + return fx.Err[fx.Pair[string, []byte]](err) + } + return fx.OK(fx.NewPair(name, bytes)) + }), + ) + if err != nil { + return nil, err + } + + return &benchEnvironments{defs, delay}, nil +} + +func (e *benchEnvironments) LoadEnvironment(ctx context.Context, name string) ([]byte, Decrypter, error) { + time.Sleep(e.delay) + + bytes, ok := e.defs[name] + if !ok { + return nil, nil, os.ErrNotExist + } + return bytes, rot128{}, nil +} + func sortEnvironmentDiagnostics(diags syntax.Diagnostics) { sort.Slice(diags, func(i, j int) bool { di, dj := diags[i], diags[j] @@ -200,6 +262,10 @@ func TestEval(t *testing.T) { entries, err := os.ReadDir(path) require.NoError(t, err) for _, e := range entries { + if e.Name() == "bench" { + continue + } + t.Run(e.Name(), func(t *testing.T) { basePath := filepath.Join(path, e.Name()) envPath := filepath.Join(basePath, "env.yaml") @@ -328,3 +394,51 @@ func TestEval(t *testing.T) { }) } } + +func benchmarkEval(b *testing.B, openDelay, loadDelay time.Duration) { + basePath := filepath.Join("testdata", "eval", "bench") + envPath := filepath.Join(basePath, "env.yaml") + + envBytes, err := os.ReadFile(envPath) + require.NoError(b, err) + + envs, err := newBenchEnvironments(basePath, loadDelay) + require.NoError(b, err) + + for i := 0; i < b.N; i++ { + execContext, err := esc.NewExecContext(map[string]esc.Value{ + "pulumi": esc.NewValue(map[string]esc.Value{ + "user": esc.NewValue(map[string]esc.Value{ + "id": esc.NewValue("USER_123"), + }), + }), + }) + require.NoError(b, err) + + environmentName := "bench" + + env, loadDiags, err := LoadYAMLBytes(environmentName, envBytes) + require.NoError(b, err) + require.Empty(b, loadDiags) + + _, evalDiags := EvalEnvironment(context.Background(), environmentName, env, rot128{}, testProviders{benchDelay: openDelay}, + envs, execContext) + require.Empty(b, evalDiags) + } +} + +func BenchmarkEval(b *testing.B) { + benchmarkEval(b, 0, 0) +} + +func BenchmarkEvalOpen(b *testing.B) { + benchmarkEval(b, 10*time.Millisecond, 0) +} + +func BenchmarkEvalEnvLoad(b *testing.B) { + benchmarkEval(b, 0, 10*time.Millisecond) +} + +func BenchmarkEvalAll(b *testing.B) { + benchmarkEval(b, 10*time.Millisecond, 10*time.Millisecond) +} diff --git a/eval/testdata/eval/bench/a.yaml b/eval/testdata/eval/bench/a.yaml new file mode 100644 index 00000000..72295e00 --- /dev/null +++ b/eval/testdata/eval/bench/a.yaml @@ -0,0 +1,7 @@ +values: + env_a: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_a: ${env_a.token} + pulumiConfig: + env_a::token: ${env_a.token} diff --git a/eval/testdata/eval/bench/b.yaml b/eval/testdata/eval/bench/b.yaml new file mode 100644 index 00000000..41cbfbc5 --- /dev/null +++ b/eval/testdata/eval/bench/b.yaml @@ -0,0 +1,7 @@ +values: + env_b: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_b: ${env_b.token} + pulumiConfig: + env_b::token: ${env_b.token} diff --git a/eval/testdata/eval/bench/c.yaml b/eval/testdata/eval/bench/c.yaml new file mode 100644 index 00000000..f769cea9 --- /dev/null +++ b/eval/testdata/eval/bench/c.yaml @@ -0,0 +1,7 @@ +values: + env_c: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_c: ${env_c.token} + pulumiConfig: + env_c::token: ${env_c.token} diff --git a/eval/testdata/eval/bench/d.yaml b/eval/testdata/eval/bench/d.yaml new file mode 100644 index 00000000..00dae595 --- /dev/null +++ b/eval/testdata/eval/bench/d.yaml @@ -0,0 +1,7 @@ +values: + env_d: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_d: ${env_d.token} + pulumiConfig: + env_d::token: ${env_d.token} diff --git a/eval/testdata/eval/bench/e.yaml b/eval/testdata/eval/bench/e.yaml new file mode 100644 index 00000000..713363a7 --- /dev/null +++ b/eval/testdata/eval/bench/e.yaml @@ -0,0 +1,7 @@ +values: + env_e: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_e: ${env_e.token} + pulumiConfig: + env_e::token: ${env_e.token} diff --git a/eval/testdata/eval/bench/env.yaml b/eval/testdata/eval/bench/env.yaml new file mode 100644 index 00000000..b9ea57a7 --- /dev/null +++ b/eval/testdata/eval/bench/env.yaml @@ -0,0 +1,39 @@ +imports: + - a + - b + - c + - d + - e + - f + - g + - h + - i + - j + - k + - l + - m + - n + - o + - p + - q + - r + - s + - t + - u + - v + - w + - x + - y + - z +values: + delays: + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} + - fn::open::bench: {} diff --git a/eval/testdata/eval/bench/f.yaml b/eval/testdata/eval/bench/f.yaml new file mode 100644 index 00000000..0e5325de --- /dev/null +++ b/eval/testdata/eval/bench/f.yaml @@ -0,0 +1,7 @@ +values: + env_f: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_f: ${env_f.token} + pulumiConfig: + env_f::token: ${env_f.token} diff --git a/eval/testdata/eval/bench/g.yaml b/eval/testdata/eval/bench/g.yaml new file mode 100644 index 00000000..1261d0de --- /dev/null +++ b/eval/testdata/eval/bench/g.yaml @@ -0,0 +1,7 @@ +values: + env_g: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_g: ${env_g.token} + pulumiConfig: + env_g::token: ${env_g.token} diff --git a/eval/testdata/eval/bench/h.yaml b/eval/testdata/eval/bench/h.yaml new file mode 100644 index 00000000..e0b29cd9 --- /dev/null +++ b/eval/testdata/eval/bench/h.yaml @@ -0,0 +1,7 @@ +values: + env_h: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_h: ${env_h.token} + pulumiConfig: + env_h::token: ${env_h.token} diff --git a/eval/testdata/eval/bench/i.yaml b/eval/testdata/eval/bench/i.yaml new file mode 100644 index 00000000..321b8a0d --- /dev/null +++ b/eval/testdata/eval/bench/i.yaml @@ -0,0 +1,7 @@ +values: + env_i: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_i: ${env_i.token} + pulumiConfig: + env_i::token: ${env_i.token} diff --git a/eval/testdata/eval/bench/j.yaml b/eval/testdata/eval/bench/j.yaml new file mode 100644 index 00000000..7c87692c --- /dev/null +++ b/eval/testdata/eval/bench/j.yaml @@ -0,0 +1,7 @@ +values: + env_j: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_j: ${env_j.token} + pulumiConfig: + env_j::token: ${env_j.token} diff --git a/eval/testdata/eval/bench/k.yaml b/eval/testdata/eval/bench/k.yaml new file mode 100644 index 00000000..874c0683 --- /dev/null +++ b/eval/testdata/eval/bench/k.yaml @@ -0,0 +1,7 @@ +values: + env_k: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_k: ${env_k.token} + pulumiConfig: + env_k::token: ${env_k.token} diff --git a/eval/testdata/eval/bench/l.yaml b/eval/testdata/eval/bench/l.yaml new file mode 100644 index 00000000..07a041b0 --- /dev/null +++ b/eval/testdata/eval/bench/l.yaml @@ -0,0 +1,7 @@ +values: + env_l: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_l: ${env_l.token} + pulumiConfig: + env_l::token: ${env_l.token} diff --git a/eval/testdata/eval/bench/m.yaml b/eval/testdata/eval/bench/m.yaml new file mode 100644 index 00000000..e94ac6ed --- /dev/null +++ b/eval/testdata/eval/bench/m.yaml @@ -0,0 +1,7 @@ +values: + env_m: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_m: ${env_m.token} + pulumiConfig: + env_m::token: ${env_m.token} diff --git a/eval/testdata/eval/bench/n.yaml b/eval/testdata/eval/bench/n.yaml new file mode 100644 index 00000000..3c66b2d0 --- /dev/null +++ b/eval/testdata/eval/bench/n.yaml @@ -0,0 +1,7 @@ +values: + env_n: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_n: ${env_n.token} + pulumiConfig: + env_n::token: ${env_n.token} diff --git a/eval/testdata/eval/bench/o.yaml b/eval/testdata/eval/bench/o.yaml new file mode 100644 index 00000000..0e4fd23f --- /dev/null +++ b/eval/testdata/eval/bench/o.yaml @@ -0,0 +1,7 @@ +values: + env_o: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_o: ${env_o.token} + pulumiConfig: + env_o::token: ${env_o.token} diff --git a/eval/testdata/eval/bench/p.yaml b/eval/testdata/eval/bench/p.yaml new file mode 100644 index 00000000..e35b42e0 --- /dev/null +++ b/eval/testdata/eval/bench/p.yaml @@ -0,0 +1,7 @@ +values: + env_p: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_p: ${env_p.token} + pulumiConfig: + env_p::token: ${env_p.token} diff --git a/eval/testdata/eval/bench/q.yaml b/eval/testdata/eval/bench/q.yaml new file mode 100644 index 00000000..09ea9625 --- /dev/null +++ b/eval/testdata/eval/bench/q.yaml @@ -0,0 +1,7 @@ +values: + env_q: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_q: ${env_q.token} + pulumiConfig: + env_q::token: ${env_q.token} diff --git a/eval/testdata/eval/bench/r.yaml b/eval/testdata/eval/bench/r.yaml new file mode 100644 index 00000000..f8b8339e --- /dev/null +++ b/eval/testdata/eval/bench/r.yaml @@ -0,0 +1,7 @@ +values: + env_r: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_r: ${env_r.token} + pulumiConfig: + env_r::token: ${env_r.token} diff --git a/eval/testdata/eval/bench/s.yaml b/eval/testdata/eval/bench/s.yaml new file mode 100644 index 00000000..14ceaddf --- /dev/null +++ b/eval/testdata/eval/bench/s.yaml @@ -0,0 +1,7 @@ +values: + env_s: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_s: ${env_s.token} + pulumiConfig: + env_s::token: ${env_s.token} diff --git a/eval/testdata/eval/bench/t.yaml b/eval/testdata/eval/bench/t.yaml new file mode 100644 index 00000000..dd7caefb --- /dev/null +++ b/eval/testdata/eval/bench/t.yaml @@ -0,0 +1,7 @@ +values: + env_t: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_t: ${env_t.token} + pulumiConfig: + env_t::token: ${env_t.token} diff --git a/eval/testdata/eval/bench/u.yaml b/eval/testdata/eval/bench/u.yaml new file mode 100644 index 00000000..2a112de4 --- /dev/null +++ b/eval/testdata/eval/bench/u.yaml @@ -0,0 +1,7 @@ +values: + env_u: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_u: ${env_u.token} + pulumiConfig: + env_u::token: ${env_u.token} diff --git a/eval/testdata/eval/bench/v.yaml b/eval/testdata/eval/bench/v.yaml new file mode 100644 index 00000000..b43d796c --- /dev/null +++ b/eval/testdata/eval/bench/v.yaml @@ -0,0 +1,7 @@ +values: + env_v: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_v: ${env_v.token} + pulumiConfig: + env_v::token: ${env_v.token} diff --git a/eval/testdata/eval/bench/w.yaml b/eval/testdata/eval/bench/w.yaml new file mode 100644 index 00000000..3b2b2b14 --- /dev/null +++ b/eval/testdata/eval/bench/w.yaml @@ -0,0 +1,7 @@ +values: + env_w: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_w: ${env_w.token} + pulumiConfig: + env_w::token: ${env_w.token} diff --git a/eval/testdata/eval/bench/x.yaml b/eval/testdata/eval/bench/x.yaml new file mode 100644 index 00000000..b634385d --- /dev/null +++ b/eval/testdata/eval/bench/x.yaml @@ -0,0 +1,7 @@ +values: + env_x: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_x: ${env_x.token} + pulumiConfig: + env_x::token: ${env_x.token} diff --git a/eval/testdata/eval/bench/y.yaml b/eval/testdata/eval/bench/y.yaml new file mode 100644 index 00000000..a04f7dcb --- /dev/null +++ b/eval/testdata/eval/bench/y.yaml @@ -0,0 +1,7 @@ +values: + env_y: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_y: ${env_y.token} + pulumiConfig: + env_y::token: ${env_y.token} diff --git a/eval/testdata/eval/bench/z.yaml b/eval/testdata/eval/bench/z.yaml new file mode 100644 index 00000000..9013a2cc --- /dev/null +++ b/eval/testdata/eval/bench/z.yaml @@ -0,0 +1,7 @@ +values: + env_z: + token: {fn::secret: {ciphertext: ZXNjeAAAAAHo9e705fKyKo30VQ==}} + environmentVariables: + env_z: ${env_z.token} + pulumiConfig: + env_z::token: ${env_z.token}