more efficient js evaluation

cmd/sst/main.go

@@ -12,7 +12,6 @@ import (
 )
 
 func main() {
-
 	app := &cli.App{
 		Name:  "sst",
 		Usage: "deploy anything",
@@ -141,10 +140,6 @@ func initProject() (*project.Project, error) {
 		return nil, err
 	}
 
-	if err := p.Stack.Login(); err != nil {
-		return nil, err
-	}
-
 	if err := p.GenerateTypes(); err != nil {
 		return nil, err
 	}

pkg/js/js.go

@@ -2,14 +2,17 @@ package js
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
+	"io"
 	"log/slog"
 	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
 
 	esbuild "github.com/evanw/esbuild/pkg/api"
+	"github.com/sst/ion/pkg/global"
 )
 
 type EvalOptions struct {
@@ -26,6 +29,174 @@ type EvalResult struct {
 	file string
 }
 
+type Process struct {
+	cmd *exec.Cmd
+	in  io.WriteCloser
+	Out *bufio.Scanner
+}
+
+var Command = struct {
+	Done string
+}{
+	Done: "~d",
+}
+
+const LOOP = `
+  import readline from "readline"
+  import fs from "fs/promises"
+
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  })
+
+  rl.on("line", async (line) => {
+    const msg = JSON.parse(line)
+    if (msg.type === "eval") {
+      try {
+        const result = await import(msg.module)
+        console.log("~d")
+      } finally {
+        // await fs.rm(msg.module)
+      }
+    }
+  })
+
+  rl.on("close", () => {
+    process.exit(0)
+  })
+`
+
+func Start(dir string) (*Process, error) {
+	loopPath :=
+		filepath.Join(
+			global.ConfigDir(),
+			"loop.mjs",
+		)
+	err := os.WriteFile(
+		loopPath,
+		[]byte(LOOP),
+		0644,
+	)
+	if err != nil {
+		return nil, err
+	}
+	cmd := exec.Command("node", "--input-type=module", "-e", LOOP)
+	cmd.Dir = dir
+
+	stdIn, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	stdOut, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	stdErr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	scanner := bufio.NewScanner(io.MultiReader(
+		stdOut,
+		stdErr,
+	))
+
+	err = cmd.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Process{
+		cmd: cmd,
+		in:  stdIn,
+		Out: scanner,
+	}, nil
+}
+
+func (p *Process) Scan() (bool, string) {
+	for p.Out.Scan() {
+		line := p.Out.Text()
+		if line == Command.Done {
+			break
+		}
+		return false, line
+	}
+	return true, ""
+}
+
+type Message struct {
+}
+
+type EvalMessage struct {
+	Type   string `json:"type"`
+	Module string `json:"module"`
+}
+
+func (p *Process) Eval(input EvalOptions) error {
+	outfile := filepath.Join(input.Dir,
+		"eval",
+		fmt.Sprintf("eval-%x.mjs", rand.Int()),
+	)
+	slog.Info("esbuild building")
+	result := esbuild.Build(esbuild.BuildOptions{
+		Banner: map[string]string{
+			"js": `
+        import { createRequire as topLevelCreateRequire } from 'module';
+        const require = topLevelCreateRequire(import.meta.url);
+        import { fileURLToPath as topLevelFileUrlToPath, URL as topLevelURL } from "url"
+        const __dirname = topLevelFileUrlToPath(new topLevelURL(".", import.meta.url))
+        import * as aws from "@pulumi/aws";
+        import * as util from "@pulumi/pulumi";
+        globalThis.aws = aws
+        globalThis.util = util
+      `,
+		},
+		External: []string{
+			"@pulumi/pulumi",
+			"@pulumi/aws",
+		},
+		Format:   esbuild.FormatESModule,
+		Platform: esbuild.PlatformNode,
+		Stdin: &esbuild.StdinOptions{
+			Contents:   input.Code,
+			ResolveDir: input.Dir,
+			Sourcefile: "eval.ts",
+			Loader:     esbuild.LoaderTS,
+		},
+		Outfile: outfile,
+		Write:   true,
+		Bundle:  true,
+	})
+	if len(result.Errors) > 0 {
+		slog.Error("esbuild errors", "errors", result.Errors)
+		return fmt.Errorf("esbuild errors: %v", result.Errors)
+	}
+	slog.Info("esbuild built")
+
+	slog.Info("sending eval message", "module", outfile)
+	bytes, err := json.Marshal(EvalMessage{
+		Type:   "eval",
+		Module: outfile,
+	})
+	if err != nil {
+		return err
+	}
+
+	_, err = p.in.Write(bytes)
+	if err != nil {
+		return err
+	}
+	_, err = p.in.Write([]byte("\n"))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func Eval(input EvalOptions) (*EvalResult, error) {
 	outfile := filepath.Join(input.Dir,
 		"eval",
@@ -39,6 +210,10 @@ func Eval(input EvalOptions) (*EvalResult, error) {
         const require = topLevelCreateRequire(import.meta.url);
         import { fileURLToPath as topLevelFileUrlToPath, URL as topLevelURL } from "url"
         const __dirname = topLevelFileUrlToPath(new topLevelURL(".", import.meta.url))
+        import * as aws from "@pulumi/aws";
+        import * as util from "@pulumi/pulumi";
+        globalThis.aws = aws
+        globalThis.util = util
       `,
 		},
 		External: []string{

pkg/project/project.go

@@ -16,6 +16,7 @@ type Project struct {
 	name    string
 	profile string
 	stage   string
+	process *js.Process
 
 	AWS       *projectAws
 	Bootstrap *bootstrap
@@ -32,12 +33,19 @@ func New() (*Project, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	rootPath := filepath.Dir(cfgPath)
 
+	process, err := js.Start(
+		rootPath,
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	proj := &Project{
-		root:   rootPath,
-		config: cfgPath,
+		root:    rootPath,
+		config:  cfgPath,
+		process: process,
 	}
 	proj.AWS = &projectAws{
 		project: proj,
@@ -61,7 +69,7 @@ func New() (*Project, error) {
 		}
 	}
 
-	eval, err := js.Eval(
+	err = process.Eval(
 		js.EvalOptions{
 			Dir: tmp,
 			Code: fmt.Sprintf(`
@@ -74,16 +82,17 @@ func New() (*Project, error) {
 		return nil, err
 	}
 
-	eval.Start()
-
-	for eval.Out.Scan() {
-		line := eval.Out.Bytes()
+	for {
+		done, line := process.Scan()
+		if done {
+			break
+		}
 		parsed := struct {
 			Name    string `json:"name"`
 			Profile string `json:"profile"`
 			Stage   string `json:"stage"`
 		}{}
-		err = json.Unmarshal(line, &parsed)
+		err = json.Unmarshal([]byte(line), &parsed)
 		if err != nil {
 			return nil, err
 		}
@@ -93,12 +102,6 @@ func New() (*Project, error) {
 		}
 		proj.profile = parsed.Profile
 		proj.stage = parsed.Stage
-		break
-	}
-
-	err = eval.Wait()
-	if err != nil {
-		return nil, err
 	}
 
 	return proj, nil