add new resource secret, fix bugs, add tests and misc performance imp…

README.md

@@ -9,7 +9,7 @@
 ![GitHub Release](https://img.shields.io/github/v/release/dgate-io/dgate)
 
 
-DGate is a distributed API Gateway built for developers. DGate allows you to use JavaScript/TypeScript to modify request/response data(L7). Inpired by [k6](https://github.com/grafana/k6) and [kong](https://github.com/Kong/kong).
+DGate is a distributed API Gateway built for developers. DGate allows you to use JavaScript/TypeScript to modify request/response data(L7). Inspired by [k6](https://github.com/grafana/k6) and [kong](https://github.com/Kong/kong).
 
 > DGate is currently in development and is not ready for production use. Please use at your own discretion.
 
@@ -30,6 +30,10 @@ go install github.com/dgate-io/dgate/cmd/dgate-server@latest
 
 DGate Server is proxy and admin server bundled into one. the admin server is responsible for managing the state of the proxy server. The proxy server is responsible for routing requests to upstream servers. The admin server can also be used to manage the state of the cluster using the Raft Consensus Algorithm.
 
+### DGate CLI (dgate-cli)
+
+DGate CLI is a command-line interface that can be used to interact with the DGate Server. It can be used to deploy modules, manage the state of the cluster, and more.
+
 #### Proxy Modules
 
 - Fetch Upstream Module (`fetchUpstream`) - executed before the request is sent to the upstream server. This module is used to decided which upstream server to send the current request to. (Essentially a custom load balancer module)

TODO.md

@@ -44,6 +44,10 @@ Also to execute code on specific events, like when a new route is added, or when
   - resource CRUD operations (namespace/domain/service/module/route/collection/document)
 - execute cron jobs: @every 1m, @cron 0 0 * * *, @daily, @weekly, @monthly, @yearly
 
+At a higher level, background jobs can be used to enable features like health checks, which can periodically check the health of the upstream servers and disable/enable them if they are not healthy.
+
+Other features include: automatic service discovery, ping-based load balancing, 
+
 # Metrics
 
 -- add support for prometheus, datadog, sentry, etc.
@@ -149,3 +153,27 @@ time based tags
 ## Module Permissions
 
 - Allow users to define permissions for modules to access certain dgate resources/apis and/or OS resources.
+  - resource:document:read
+  - resource:document:write
+  - os:net:(http/tcp/udp)
+  - os:file:read
+  - os:env:read
+
+# Bundles
+
+- Add support for bundles that can be used to extend the functionality of DGate. Bundles are a grouping of resources that can be used to extend the functionality of DGate. Bundles can be used to add new modules, resources, and more.
+A good example of a bundle would be a bundle that adds support for OAuth2 authentication. It would need to setup the necessary routes, modules, and configurations to enable OAuth2 authentication. 
+
+## Module/Plugin Variables
+
+- Allow users to define variables that can be used in modules/plugins. These variables can be set by the user, eventually the Admin Console should allow these variables to be set, and the variables can be used in the modules/plugins.
+
+## Mutual TLS Support (low priority)
+
+## Versioning Modules
+
+Differing from common resource versioning, modules can have multiple versions that can be used at the same time. This can be used to test new versions of modules before deploying them to the cluster.
+
+## Secrets
+
+- Add support for secrets that can be used in modules. Secrets can be used to store sensitive information like API keys, passwords, etc. Secrets can only be used in modules and cannot be accessed by the API. Explicit permissions can be set to allow certain modules to access certain secrets. Secrets are also versioned and can be rolled back if necessary. This also allows different modules to use different versions of the same secret.

functional-tests/admin_tests/performance_test_prep.sh

@@ -56,4 +56,6 @@ curl -s --fail-with-body ${PROXY_URL}/svctest -H Host:dgate.dev
 
 curl -s --fail-with-body ${PROXY_URL}/modtest -H Host:dgate.dev
 
-curl -s ${PROXY_URL}/blank -H Host:dgate.dev
+curl -s ${PROXY_URL}/blank -H Host:dgate.dev
+
+echo "Performance Test Prep Done"

internal/admin/routes/secret_routes.go

@@ -0,0 +1,105 @@
+package routes
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/dgate-io/chi-router"
+	"github.com/dgate-io/dgate/internal/config"
+	"github.com/dgate-io/dgate/internal/proxy"
+	"github.com/dgate-io/dgate/pkg/spec"
+	"github.com/dgate-io/dgate/pkg/util"
+)
+
+func ConfigureSecretAPI(server chi.Router, proxyState *proxy.ProxyState, appConfig *config.DGateConfig) {
+	rm := proxyState.ResourceManager()
+	server.Put("/secret", func(w http.ResponseWriter, r *http.Request) {
+		eb, err := io.ReadAll(r.Body)
+		defer r.Body.Close()
+		if err != nil {
+			util.JsonError(w, http.StatusBadRequest, "error reading body")
+			return
+		}
+		scrt := spec.Secret{}
+		err = json.Unmarshal(eb, &scrt)
+		if err != nil {
+			util.JsonError(w, http.StatusBadRequest, "error unmarshalling body")
+			return
+		}
+		if scrt.Data == "" {
+			util.JsonError(w, http.StatusBadRequest, "payload is required")
+			return
+		}
+		if scrt.NamespaceName == "" {
+			if appConfig.DisableDefaultNamespace {
+				util.JsonError(w, http.StatusBadRequest, "namespace is required")
+				return
+			}
+			scrt.NamespaceName = spec.DefaultNamespace.Name
+		}
+		cl := spec.NewChangeLog(&scrt, scrt.NamespaceName, spec.AddSecretCommand)
+		if err = proxyState.ApplyChangeLog(cl); err != nil {
+			util.JsonError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		if repl := proxyState.Raft(); repl != nil {
+			future := repl.Barrier(time.Second * 5)
+			if err := future.Error(); err != nil {
+				util.JsonError(w, http.StatusInternalServerError, err.Error())
+				return
+			}
+		}
+		util.JsonResponse(w, http.StatusCreated,
+			spec.TransformDGateSecrets(true,
+				rm.GetSecretsByNamespace(scrt.NamespaceName)...))
+	})
+
+	server.Delete("/secret", func(w http.ResponseWriter, r *http.Request) {
+		eb, err := io.ReadAll(r.Body)
+		defer r.Body.Close()
+		if err != nil {
+			util.JsonError(w, http.StatusBadRequest, "error reading body")
+			return
+		}
+		scrt := spec.Secret{}
+		err = json.Unmarshal(eb, &scrt)
+		if err != nil {
+			util.JsonError(w, http.StatusBadRequest, "error unmarshalling body")
+			return
+		}
+		if scrt.NamespaceName == "" {
+			if appConfig.DisableDefaultNamespace {
+				util.JsonError(w, http.StatusBadRequest, "namespace is required")
+				return
+			}
+			scrt.NamespaceName = spec.DefaultNamespace.Name
+		}
+		cl := spec.NewChangeLog(&scrt, scrt.NamespaceName, spec.DeleteSecretCommand)
+		if err = proxyState.ApplyChangeLog(cl); err != nil {
+			util.JsonError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		w.WriteHeader(http.StatusAccepted)
+	})
+
+	server.Get("/secret", func(w http.ResponseWriter, r *http.Request) {
+		nsName := r.URL.Query().Get("namespace")
+		if nsName == "" {
+			if appConfig.DisableDefaultNamespace {
+				util.JsonError(w, http.StatusBadRequest, "namespace is required")
+				return
+			}
+			nsName = spec.DefaultNamespace.Name
+		} else {
+			if _, ok := rm.GetNamespace(nsName); !ok {
+				util.JsonError(w, http.StatusBadRequest, "namespace not found: "+nsName)
+				return
+			}
+		}
+		util.JsonResponse(w, http.StatusCreated,
+			spec.TransformDGateSecrets(true,
+				rm.GetSecretsByNamespace(nsName)...))
+	})
+}

internal/config/config.go

@@ -173,8 +173,9 @@ type (
 		Routes         []spec.Route      `koanf:"routes"`
 		Modules        []ModuleSpec      `koanf:"modules"`
 		Domains        []DomainSpec      `koanf:"domains"`
-		Collections    []spec.Collection `koanf:"-"`
+		Collections    []spec.Collection `koanf:"collections"`
 		Documents      []spec.Document   `koanf:"documents"`
+		Secrets        []spec.Secret     `koanf:"secrets"`
 	}
 
 	DomainSpec struct {