enable typesense (#23)

Implements typesense indexing

cli/Makefile

@@ -4,7 +4,7 @@ DOCKER_IMAGE := builder
 DOCKER_TARGET := cli-base-alpine
 BUILD_PATH := /go/src/app/dist
 
-${BIN_PATH}/${TARGET}: cli.go lib/*.go go.mod .vendor_modified build.sh init/* build-php.sh
+${BIN_PATH}/${TARGET}: cli.go lib/* go.mod .vendor_modified build.sh init/* build-php.sh
 	mkdir -p ${BIN_PATH}
 	cd .. && docker build --pull --target ${DOCKER_TARGET} -t ${DOCKER_IMAGE} .
 	docker create --name builder builder || ( docker rm -f builder && false )

cli/cli.go

@@ -39,7 +39,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"path/filepath"
+	"runtime"
 	"strings"
 	"syscall"
 	"time"
@@ -60,39 +60,14 @@ func getLogger(options map[string]string) *zap.Logger {
 	return zap.New(core)
 }
 
-func findBootstrap(options map[string]string, logger *zap.Logger) string {
-	bootstrap := options["bootstrap"]
-
-	if options["bootstrap"] == "" {
-		bootstrap = "src/bootstrap.php"
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		logger.Error("Could not get the current working directory", zap.Error(err))
-	} else {
-		bootstrap = filepath.Join(cwd, bootstrap)
-	}
-
-	if _, err := os.Stat(bootstrap); err != nil {
-		logger.Warn("Bootstrap file does not exist; default DI implementation used", zap.String("filename", bootstrap))
-		return ""
-	}
-
-	return bootstrap
-}
-
 func execute(args []string, options map[string]string) int {
 	logger := getLogger(options)
 
-	nurl := ""
-	if options["nats-server"] == "" {
-		nurl = options["nats-server"]
-	}
+	config := lib.ApplyOptions(lib.GetProjectConfig(), options)
 
 	boostrapNats := false
 
-	if nurl == "" {
+	if config.Nat.Internal {
 		logger.Warn("Running in dev mode, all data will be deleted at the end of this")
 		data, err := os.MkdirTemp("", "nats-state-*")
 		if err != nil {
@@ -121,7 +96,6 @@ func execute(args []string, options map[string]string) int {
 			StoreDir:       data,
 		})
 		defer s.Shutdown()
-		nurl = nats.DefaultURL
 		boostrapNats = true
 	}
 
@@ -130,41 +104,44 @@ func execute(args []string, options map[string]string) int {
 		nats.RetryOnFailedConnect(true),
 	}
 
-	if options["jwt"] != "" && options["nkey"] != "" {
-		nopts = append(nopts, nats.UserCredentials(options["jwt"], strings.Split(options["nkey"], ",")...))
+	if config.Nat.Jwt != "" && config.Nat.Nkey != "" {
+		nopts = append(nopts, nats.UserCredentials(config.Nat.Jwt, config.Nat.Nkey))
+	}
+
+	if config.Nat.Tls.Ca != "" {
+		nopts = append(nopts, nats.RootCAs(strings.Split(config.Nat.Tls.Ca, ",")...))
 	}
 
-	ns, err := nats.Connect(nurl, nopts...)
+	if config.Nat.Tls.KeyFile != "" {
+		nopts = append(nopts, nats.ClientCert(config.Nat.Tls.ClientCert, config.Nat.Tls.KeyFile))
+	}
+
+	ns, err := nats.Connect(config.Nat.Url, nopts...)
 	if err != nil {
 		panic(err)
 	}
 	js, err := jetstream.New(ns)
 	if err != nil {
 		panic(err)
 	}
-	ctx := context.WithValue(context.Background(), "bootstrap", findBootstrap(options, logger))
-
-	streamName := options["stream"]
-	if streamName == "" {
-		streamName = "test"
-	}
+	ctx := context.WithValue(context.Background(), "bootstrap", config.Bootstrap)
 
 	if boostrapNats {
 		stream, _ := js.CreateStream(ctx, jetstream.StreamConfig{
-			Name:        streamName,
+			Name:        config.Stream,
 			Description: "Handles durable-php events",
-			Subjects:    []string{streamName + ".>"},
+			Subjects:    []string{config.Stream + ".>"},
 			Retention:   jetstream.WorkQueuePolicy,
 			Storage:     jetstream.FileStorage,
 			AllowRollup: false,
 			DenyDelete:  true,
 			DenyPurge:   true,
 		})
-		history, _ := js.CreateStream(ctx, jetstream.StreamConfig{
-			Name:        streamName + "_history",
+		_, _ = js.CreateStream(ctx, jetstream.StreamConfig{
+			Name:        config.Stream + "_history",
 			Description: "The history of the stream",
 			Mirror: &jetstream.StreamSource{
-				Name: streamName,
+				Name: config.Stream,
 			},
 			Retention:   jetstream.LimitsPolicy,
 			AllowRollup: true,
@@ -180,28 +157,20 @@ func execute(args []string, options map[string]string) int {
 
 		for _, kind := range consumers {
 			_, _ = stream.CreateConsumer(ctx, jetstream.ConsumerConfig{
-				Durable:       streamName + "-" + kind,
-				FilterSubject: streamName + "." + kind + ".>",
+				Durable:       config.Stream + "-" + kind,
+				FilterSubject: config.Stream + "." + kind + ".>",
 				AckPolicy:     jetstream.AckExplicitPolicy,
 				AckWait:       5 * time.Minute,
 			})
 		}
-
-		_, _ = history.CreateConsumer(ctx, jetstream.ConsumerConfig{
-			Durable: "billing",
-		})
-
-		_, _ = history.CreateConsumer(ctx, jetstream.ConsumerConfig{
-			Durable: "monitoring",
-		})
 	}
 
-	stream, err := js.Stream(ctx, streamName)
+	stream, err := js.Stream(ctx, config.Stream)
 	if err != nil {
 		panic(err)
 	}
 
-	opts := []frankenphp.Option{frankenphp.WithNumThreads(32), frankenphp.WithLogger(logger)}
+	opts := []frankenphp.Option{frankenphp.WithNumThreads(runtime.NumCPU() * 2), frankenphp.WithLogger(logger)}
 
 	if err := frankenphp.Init(opts...); err != nil {
 		panic(err)
@@ -210,17 +179,34 @@ func execute(args []string, options map[string]string) int {
 
 	if options["no-activities"] != "true" {
 		logger.Info("Starting activity consumer")
-		go lib.BuildConsumer(stream, ctx, streamName, lib.Activity, logger, js)
+		go lib.BuildConsumer(stream, ctx, config, lib.Activity, logger, js)
 	}
 
 	if options["no-entities"] != "true" {
 		logger.Info("Starting entity consumer")
-		go lib.BuildConsumer(stream, ctx, streamName, lib.Entity, logger, js)
+		go lib.BuildConsumer(stream, ctx, config, lib.Entity, logger, js)
 	}
 
 	if options["no-orchestrations"] != "true" {
 		logger.Info("Starting orchestration consumer")
-		go lib.BuildConsumer(stream, ctx, streamName, lib.Orchestration, logger, js)
+		go lib.BuildConsumer(stream, ctx, config, lib.Orchestration, logger, js)
+	}
+
+	if len(config.Extensions.Search.Collections) > 0 {
+		for _, collection := range config.Extensions.Search.Collections {
+			switch collection {
+			case "entities":
+				err := lib.IndexerListen(ctx, config, lib.Entity, js, logger)
+				if err != nil {
+					panic(err)
+				}
+			case "orchestrations":
+				err := lib.IndexerListen(ctx, config, lib.Orchestration, js, logger)
+				if err != nil {
+					panic(err)
+				}
+			}
+		}
 	}
 
 	port := options["port"]
@@ -229,7 +215,7 @@ func execute(args []string, options map[string]string) int {
 	}
 
 	if options["no-api"] != "true" {
-		lib.Startup(ctx, js, logger, port, streamName)
+		lib.Startup(ctx, js, logger, port, config)
 	} else {
 		block := make(chan struct{})
 		<-block
@@ -363,8 +349,6 @@ func main() {
 		WithOption(cli.NewOption("no-api", "Disable the api server").WithType(cli.TypeBool)).
 		WithOption(cli.NewOption("verbose", "Enable info level logging").WithType(cli.TypeBool)).
 		WithOption(cli.NewOption("debug", "Enable debug logging").WithType(cli.TypeBool)).
-		WithOption(cli.NewOption("jwt", "Use a jwt file for connecting").WithType(cli.TypeString).WithChar('j')).
-		WithOption(cli.NewOption("nkey", "Use a nkey seed file for connecting").WithType(cli.TypeString).WithChar('n')).
 		WithCommand(run).
 		WithCommand(initCmd).
 		WithCommand(version).

cli/go.mod

@@ -14,15 +14,21 @@ require go.uber.org/zap v1.27.0
 
 require github.com/gorilla/mux v1.8.1
 
+require github.com/typesense/typesense-go v1.0.0
+
 require (
+	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
+	github.com/deepmap/oapi-codegen v1.12.3 // indirect
 	github.com/dolthub/maphash v0.1.0 // indirect
 	github.com/gammazero/deque v0.2.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/maypok86/otter v1.2.0 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/nats-io/jwt/v2 v2.5.5 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect

cli/go.sum

@@ -1,13 +1,27 @@
+github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deepmap/oapi-codegen v1.12.3 h1:+DDYKeIwlKChzHjhVtlISegatFevDDazBhtk/dnp4V4=
+github.com/deepmap/oapi-codegen v1.12.3/go.mod h1:ao2aFwsl/muMHbez870+KelJ1yusV01RznwAFFrVjDc=
 github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
 github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
 github.com/dunglas/frankenphp v1.1.1-0.20240313210106-6a3db9429dca h1:n3ESB4u5gJTMzIgw60SbN9KhcRZCbezirOukQYbFJ80=
 github.com/dunglas/frankenphp v1.1.1-0.20240313210106-6a3db9429dca/go.mod h1:u4UasfhFvcNwFoLR2InkcXWeWA2oK71USghSo8F4d0g=
 github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
 github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI=
+github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
 github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
 github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/maypok86/otter v1.2.0 h1:djwBBNpp9+dyzBTY0zscIG+pyAQVXRRRMbzztf8iJ4U=
@@ -26,10 +40,17 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
+github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/teris-io/cli v1.0.1 h1:J6jnVHC552uqx7zT+Ux0++tIvLmJQULqxVhCid2u/Gk=
 github.com/teris-io/cli v1.0.1/go.mod h1:V9nVD5aZ873RU/tQXLSXO8FieVPQhQvuNohsdsKXsGw=
+github.com/typesense/typesense-go v1.0.0 h1:/8Lr1yf9YjmUKdn/xbTNy+OhwOvBd0noBTRkcB22Uhw=
+github.com/typesense/typesense-go v1.0.0/go.mod h1:4mq4FYHzU7csU/KHaZoyG2bCSKl7GrCeyAr2YhXT1/0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

cli/init/execute.go

@@ -20,10 +20,6 @@ func createProjectFolder(name string) {
 	}
 }
 
-func installComposer() {
-	// todo
-}
-
 type projectConfig struct {
 	Name string
 }

cli/init/template/dphp.json

@@ -0,0 +1,8 @@
+{
+  "project": "{{.Name}}",
+  "bootstrap": "src/bootstrap.php",
+  "nats": {
+    "embeddedServer": true
+  },
+  "historyRetentionDays": 1
+}