feat: change logging to zap and change how change logs applied (#9)

* feat: change logging to slog and change how change logs are applied, also added more tests for admin routes

* fix local tests

* replace slog with zap

* restartState's callback changed to sync
uncommend collectiontype
add lock to reconfigure state
remove ip from logs

* improve tests, logger logic, add cli file eq, modify url shortener mod, etc.

* fix bug where uing the wrong err

* add to todo list

Dockerfile

@@ -9,8 +9,8 @@ RUN go build -o /usr/bin/dgate-cli ./cmd/dgate-cli
 
 FROM alpine:3.19 as runner
 WORKDIR /app
-COPY --from=builder /usr/bin/dgate-server ./
-COPY --from=builder /usr/bin/dgate-cli ./
+COPY --from=builder /usr/bin/dgate-server /usr/bin/
+COPY --from=builder /usr/bin/dgate-cli /usr/bin/
 COPY --from=builder /app/config.dgate.yaml ./
 EXPOSE 80 443 9080 9443
-CMD [ "./dgate-server" ]
+CMD [ "dgate-server" ]

TODO.md

@@ -9,8 +9,6 @@
   - other commands (backup, restore, etc.) (low priority)
   - replace k6 with wrk for performance tests
 
-## Replace zerolog with slog
-
 ## Add Module Tests
 
 - Test multiple modules being used at the same time
@@ -113,7 +111,7 @@ time based tags
 - ?*name:data1,data2
   - means that the server must have *one (and only one)* of these tags, for the object to be applied
 
-## Module Permissions
+## Module Permissions (using tags?)
 
 - Allow users to define permissions for modules to access certain dgate resources/apis and/or OS resources.
   - resource:document:read
@@ -146,8 +144,19 @@ dgate-cli ns mk my-ns nmae=my-ns
 Variable 'nmae' is not recognized. Did you mean 'name'?
 ```
 
+## DGate CLI - help command show required variables
+
+When the user runs the help command, the CLI should show the required variables for the command. For example, if the user runs `dgate-cli ns mk --help`, the CLI should show the required variables for the `ns mk` command. `name` is a required variable for the `ns mk` command. Also, the CLI should show non-required variables.
+
 ## Improve Module Debugability
 
 Make it easier to debug modules by adding more logging and error handling. This can be done by adding more logging to the modules and making it easier to see the logs in the Admin Console.
 
-Add stack tracing for typescript modules.
+Add stack tracing for typescript modules.
+
+
+## Decouple Admin API from Raft Implementation
+
+Currently, Raft Implementation is tightly coupled with the Admin API. This makes it difficult to change the Raft Implementation without changing the Admin API. Decouple the Raft Implementation from the Admin API to make it easier to change the Raft Implementation.
+
+## Add Telemetry (sentry, datadog, etc.)

cmd/dgate-cli/commands/collection_cmd.go

@@ -12,7 +12,7 @@ func CollectionCommand(client dgclient.DGateClient) *cli.Command {
 		Aliases:   []string{"col"},
 		Args:      true,
 		ArgsUsage: "<command> <name>",
-		Usage:     "collection commands",
+		Usage:     "collection <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/document_cmd.go

@@ -12,7 +12,7 @@ func DocumentCommand(client dgclient.DGateClient) *cli.Command {
 		Aliases:   []string{"doc"},
 		Args:      true,
 		ArgsUsage: "<command> <name>",
-		Usage:     "document commands",
+		Usage:     "document <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/domain_cmd.go

@@ -12,7 +12,7 @@ func DomainCommand(client dgclient.DGateClient) *cli.Command {
 		Aliases:   []string{"dom"},
 		Args:      true,
 		ArgsUsage: "<command> <name>",
-		Usage:     "domain commands",
+		Usage:     "domain <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/module_cmd.go

@@ -12,7 +12,7 @@ func ModuleCommand(client dgclient.DGateClient) *cli.Command {
 		Aliases:   []string{"mod"},
 		Args:      true,
 		ArgsUsage: "<command> <name>",
-		Usage:     "module commands",
+		Usage:     "module <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/namespace_cmd.go

@@ -8,9 +8,11 @@ import (
 
 func NamespaceCommand(client dgclient.DGateClient) *cli.Command {
 	return &cli.Command{
-		Name:    "namespace",
-		Aliases: []string{"ns"},
-		Args:    true,
+		Name:      "namespace",
+		Args:      true,
+		Aliases:   []string{"ns"},
+		ArgsUsage: "<command> <name>",
+		Usage:     "namespace <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/route_cmd.go

@@ -12,7 +12,7 @@ func RouteCommand(client dgclient.DGateClient) *cli.Command {
 		Aliases:   []string{"rt"},
 		Args:      true,
 		ArgsUsage: "<command> <name>",
-		Usage:     "route commands",
+		Usage:     "route <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/run_cmd_test.go

@@ -61,7 +61,7 @@ func TestGenericCommands(t *testing.T) {
 				case "service":
 					os.Args = append(os.Args, "urls=http://localhost.net")
 				case "module":
-					os.Args = append(os.Args, "payload=QUJD")
+					os.Args = append(os.Args, "payload@=testdata/test.ts")
 				case "domain":
 					os.Args = append(os.Args, "patterns=*")
 				case "secret":
@@ -133,7 +133,7 @@ func TestCommands_ClientError(t *testing.T) {
 				case "service":
 					os.Args = append(os.Args, "urls=http://localhost.net")
 				case "module":
-					os.Args = append(os.Args, "payload=QUJD")
+					os.Args = append(os.Args, "payload@=testdata/test.ts")
 				case "domain":
 					os.Args = append(os.Args, "patterns=*")
 				case "secret":

cmd/dgate-cli/commands/secret_cmd.go

@@ -12,7 +12,7 @@ func SecretCommand(client dgclient.DGateClient) *cli.Command {
 		Aliases:   []string{"sec"},
 		Args:      true,
 		ArgsUsage: "<command> <name>",
-		Usage:     "secret commands",
+		Usage:     "secret <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/service_cmd.go

@@ -12,7 +12,7 @@ func ServiceCommand(client dgclient.DGateClient) *cli.Command {
 		Aliases:   []string{"svc"},
 		Args:      true,
 		ArgsUsage: "<command> <name>",
-		Usage:     "service commands",
+		Usage:     "service <action> <args>",
 		Subcommands: []*cli.Command{
 			{
 				Name:    "create",

cmd/dgate-cli/commands/testdata/test.ts

@@ -0,0 +1 @@
+export const requestHandler = async () => {};

cmd/dgate-cli/commands/util.go

@@ -1,9 +1,11 @@
 package commands
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/dgate-io/dgate/internal/config"
@@ -16,6 +18,30 @@ func createMapFromArgs[N any](
 	required ...string,
 ) (*N, error) {
 	m := make(map[string]any)
+
+	// parse file string
+	for i, arg := range args {
+		if !strings.Contains(arg, "@=") {
+			continue
+		}
+		pair := strings.SplitN(arg, "@=", 2)
+		if len(pair) != 2 || pair[0] == "" {
+			return nil, fmt.Errorf("invalid key-value pair: %s", arg)
+		}
+		var v any
+		if pair[1] != "" {
+			file, err := os.ReadFile(pair[1])
+			if err != nil {
+				return nil, fmt.Errorf("error reading file: %s", err.Error())
+			}
+			v = base64.StdEncoding.EncodeToString(file)
+		} else {
+			v = ""
+		}
+		m[pair[0]] = v
+		args[i] = ""
+	}
+
 	// parse json strings
 	for i, arg := range args {
 		if !strings.Contains(arg, ":=") {

cmd/dgate-server/main.go

@@ -60,8 +60,14 @@ func main() {
 			fmt.Printf("Error loading config: %s\n", err)
 			os.Exit(1)
 		} else {
-			proxyState := proxy.StartProxyGateway(version, dgateConfig)
-			admin.StartAdminAPI(dgateConfig, proxyState)
+			logger, err := dgateConfig.GetLogger()
+			if err != nil {
+				fmt.Printf("Error setting up logger: %s\n", err)
+				os.Exit(1)
+			}
+			defer logger.Sync()
+			proxyState := proxy.NewProxyState(logger.Named("proxy"), dgateConfig)
+			admin.StartAdminAPI(version, dgateConfig, logger.Named("admin"), proxyState)
 			if err := proxyState.Start(); err != nil {
 				fmt.Printf("Error loading config: %s\n", err)
 				os.Exit(1)
@@ -75,7 +81,6 @@ func main() {
 			)
 			<-sigchan
 			proxyState.Stop()
-			os.Exit(1)
 		}
 	}
 }

config.dgate.yaml

@@ -11,11 +11,13 @@ test_server:
   port: 8888
   host: 0.0.0.0
   global_headers:
-    X-Node-Id: ${HOSTNAME:-N/A}
+    X-Test-Header: ${TEST_HEADER:-test}
 proxy:
   port: ${PORT:-80}
   host: 0.0.0.0
   enable_console_logger: true
+  transport:
+    dns_prefer_go: true
   init_resources:
     namespaces:
       - name: "admin"
@@ -43,8 +45,6 @@ admin:
   port: 9080
   host: 0.0.0.0
   allow_list:
-    - "127.0.0.1"
     - "127.0.0.1/8"
     - "::1"
-  auth_method: null
 

functional-tests/admin_tests/url_shortener.ts

@@ -1,17 +1,19 @@
-// @ts-nocheck
+// @ts-ignore
 import { createHash } from "dgate/crypto";
+// @ts-ignore
 import { addDocument, getDocument } from "dgate/state";
 
 export const requestHandler = (ctx: any) => {
     const req = ctx.request();
     const res = ctx.response();
     if (req.method == "GET") {
-        if (!req.query.get("id")) {
+        const pathId = ctx.pathParam("id")
+        if (!pathId) {
             res.status(400).json({ error: "id is required" })
             return;
         }
         // get the document with the ID from the collection
-        return getDocument("short_link", req.query.get("id"))
+        return getDocument("short_link", pathId)
             .then((doc: any) => {
                 // check if the document contains the URL
                 if (!doc?.data?.url) {
@@ -26,27 +28,26 @@ export const requestHandler = (ctx: any) => {
             });
     } else if (req.method == "POST") {
         const link = req.query.get("url");
-        console.log("link", link);
         if (!link) {
-            res.status(400).json({ error: "link is required" });
+          res.status(400).json({ error: "url is required" });
+          return;
         }
-        // create a hash of the link and use it as the ID
-        let hash = createHash("sha1")
-            .update(link)
-            .digest("base64rawurl")
-            .slice(-8);
+        // hash the url
+        const hash = hashURL(link);
+
         // create a new document with the hash as the ID, and the link as the data
         return addDocument({
             id: hash,
             collection: "short_link",
             // the collection schema is defined in url_shortener_test.sh
             data: { url: link },
-        }).then(() => {
-            res.status(201).json({ id: hash });
-        }).catch((e: any) => {
-            res.status(500).json({ error: e?.message });
-        });
+        })
+            .then(() => res.status(201).json({ id: hash }))
+            .catch((e: any) => res.status(500).json({ error: e?.message }));
     } else {
         return res.status(405).json({ error: "method not allowed" });
     }
-};
+};
+
+const hashURL = (url: string) => createHash("sha1").
+  update(url).digest("base64rawurl").slice(-8);

functional-tests/admin_tests/url_shortener_test.sh

@@ -23,24 +23,24 @@ dgate-cli collection create \
     type=document \
     namespace=url_shortener-ns
 
-MOD_B64="$(base64 < $DIR/url_shortener.ts)"
-dgate-cli module create \
-    name=printer \
-    payload="$MOD_B64" \
+dgate-cli module create name=url_shortener-mod \
+    payload@=$DIR/url_shortener.ts \
     namespace=url_shortener-ns
 
 dgate-cli route create \
-    name=base_rt \
-    paths:='["/test","/hello"]' \
+    name=base_rt paths:='["/", "/{id}"]' \
+    modules:='["url_shortener-mod"]' \
     methods:='["GET","POST"]' \
-    modules:='["printer"]' \
     stripPath:=true \
     preserveHost:=true \
-    namespace=url_shortener-ns #\ service='base_svc'
+    namespace=url_shortener-ns
 
-JSON_RESP=$(curl -sG -X POST -H Host:url_shortener.com ${PROXY_URL}/test --data-urlencode "url=${PROXY_URL}/hello")
-echo $JSON_RESP
+JSON_RESP=$(curl -sG -X POST \
+    -H Host:url_shortener.com ${PROXY_URL}/ \
+    --data-urlencode 'url=https://dgate.io')
 
 URL_ID=$(echo $JSON_RESP | jq -r '.id')
 
-curl -s --fail-with-body ${PROXY_URL}/test\?id=$URL_ID -H Host:url_shortener.com
+curl -s --fail-with-body \
+    ${PROXY_URL}/$URL_ID \
+    -H Host:url_shortener.com

go.mod

@@ -80,6 +80,8 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.26.0 // indirect
 	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect

go.sum

@@ -278,6 +278,10 @@ go.opentelemetry.io/otel/sdk/metric v1.26.0 h1:cWSks5tfriHPdWFnl+qpX3P681aAYqlZH
 go.opentelemetry.io/otel/sdk/metric v1.26.0/go.mod h1:ClMFFknnThJCksebJwz7KIyEDHO+nTB6gK8obLy8RyE=
 go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
 go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=