improve tests, logger logic, add cli file eq, modify url shortener mo…

…d, etc.

cmd/dgate-cli/commands/run_cmd_test.go

@@ -61,7 +61,7 @@ func TestGenericCommands(t *testing.T) {
 				case "service":
 					os.Args = append(os.Args, "urls=http://localhost.net")
 				case "module":
-					os.Args = append(os.Args, "payload=QUJD")
+					os.Args = append(os.Args, "payload@=testdata/test.ts")
 				case "domain":
 					os.Args = append(os.Args, "patterns=*")
 				case "secret":
@@ -133,7 +133,7 @@ func TestCommands_ClientError(t *testing.T) {
 				case "service":
 					os.Args = append(os.Args, "urls=http://localhost.net")
 				case "module":
-					os.Args = append(os.Args, "payload=QUJD")
+					os.Args = append(os.Args, "payload@=testdata/test.ts")
 				case "domain":
 					os.Args = append(os.Args, "patterns=*")
 				case "secret":

cmd/dgate-cli/commands/testdata/test.ts

@@ -0,0 +1 @@
+export const requestHandler = async () => {};

cmd/dgate-cli/commands/util.go

@@ -1,9 +1,11 @@
 package commands
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/dgate-io/dgate/internal/config"
@@ -16,6 +18,30 @@ func createMapFromArgs[N any](
 	required ...string,
 ) (*N, error) {
 	m := make(map[string]any)
+
+	// parse file string
+	for i, arg := range args {
+		if !strings.Contains(arg, "@=") {
+			continue
+		}
+		pair := strings.SplitN(arg, "@=", 2)
+		if len(pair) != 2 || pair[0] == "" {
+			return nil, fmt.Errorf("invalid key-value pair: %s", arg)
+		}
+		var v any
+		if pair[1] != "" {
+			file, err := os.ReadFile(pair[1])
+			if err != nil {
+				return nil, fmt.Errorf("error reading file: %s", err.Error())
+			}
+			v = base64.StdEncoding.EncodeToString(file)
+		} else {
+			v = ""
+		}
+		m[pair[0]] = v
+		args[i] = ""
+	}
+
 	// parse json strings
 	for i, arg := range args {
 		if !strings.Contains(arg, ":=") {

config.dgate.yaml

@@ -16,6 +16,8 @@ proxy:
   port: ${PORT:-80}
   host: 0.0.0.0
   enable_console_logger: true
+  transport:
+    dns_prefer_go: true
   init_resources:
     namespaces:
       - name: "admin"

functional-tests/admin_tests/url_shortener.ts

@@ -1,17 +1,19 @@
-// @ts-nocheck
+// @ts-ignore
 import { createHash } from "dgate/crypto";
+// @ts-ignore
 import { addDocument, getDocument } from "dgate/state";
 
 export const requestHandler = (ctx: any) => {
     const req = ctx.request();
     const res = ctx.response();
     if (req.method == "GET") {
-        if (!req.query.get("id")) {
+        const pathId = ctx.pathParam("id")
+        if (!pathId) {
             res.status(400).json({ error: "id is required" })
             return;
         }
         // get the document with the ID from the collection
-        return getDocument("short_link", req.query.get("id"))
+        return getDocument("short_link", pathId)
             .then((doc: any) => {
                 // check if the document contains the URL
                 if (!doc?.data?.url) {
@@ -26,27 +28,26 @@ export const requestHandler = (ctx: any) => {
             });
     } else if (req.method == "POST") {
         const link = req.query.get("url");
-        console.log("link", link);
         if (!link) {
-            res.status(400).json({ error: "link is required" });
+          res.status(400).json({ error: "url is required" });
+          return;
         }
-        // create a hash of the link and use it as the ID
-        let hash = createHash("sha1")
-            .update(link)
-            .digest("base64rawurl")
-            .slice(-8);
+        // hash the url
+        const hash = hashURL(link);
+
         // create a new document with the hash as the ID, and the link as the data
         return addDocument({
             id: hash,
             collection: "short_link",
             // the collection schema is defined in url_shortener_test.sh
             data: { url: link },
-        }).then(() => {
-            res.status(201).json({ id: hash });
-        }).catch((e: any) => {
-            res.status(500).json({ error: e?.message });
-        });
+        })
+            .then(() => res.status(201).json({ id: hash }))
+            .catch((e: any) => res.status(500).json({ error: e?.message }));
     } else {
         return res.status(405).json({ error: "method not allowed" });
     }
-};
+};
+
+const hashURL = (url: string) => createHash("sha1").
+  update(url).digest("base64rawurl").slice(-8);

functional-tests/admin_tests/url_shortener_test.sh

@@ -23,24 +23,24 @@ dgate-cli collection create \
     type=document \
     namespace=url_shortener-ns
 
-MOD_B64="$(base64 < $DIR/url_shortener.ts)"
-dgate-cli module create \
-    name=printer \
-    payload="$MOD_B64" \
+dgate-cli module create name=url_shortener-mod \
+    payload@=$DIR/url_shortener.ts \
     namespace=url_shortener-ns
 
 dgate-cli route create \
-    name=base_rt \
-    paths:='["/test","/hello"]' \
+    name=base_rt paths:='["/", "/{id}"]' \
+    modules:='["url_shortener-mod"]' \
     methods:='["GET","POST"]' \
-    modules:='["printer"]' \
     stripPath:=true \
     preserveHost:=true \
-    namespace=url_shortener-ns #\ service='base_svc'
+    namespace=url_shortener-ns
 
-JSON_RESP=$(curl -sG -X POST -H Host:url_shortener.com ${PROXY_URL}/test --data-urlencode "url=${PROXY_URL}/hello")
-echo $JSON_RESP
+JSON_RESP=$(curl -sG -X POST \
+    -H Host:url_shortener.com ${PROXY_URL}/ \
+    --data-urlencode 'url=https://dgate.io')
 
 URL_ID=$(echo $JSON_RESP | jq -r '.id')
 
-curl -s --fail-with-body ${PROXY_URL}/test\?id=$URL_ID -H Host:url_shortener.com
+curl -s --fail-with-body \
+    ${PROXY_URL}/$URL_ID \
+    -H Host:url_shortener.com

internal/admin/routes/collection_routes.go

@@ -125,11 +125,12 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 		if collection, ok := rm.GetCollection(collectionName, namespaceName); !ok {
 			util.JsonError(w, http.StatusNotFound, "collection not found")
 			return
-		} else {
-			if collection.Type != spec.CollectionTypeDocument {
-				util.JsonError(w, http.StatusBadRequest, "collection is not a document collection")
-				return
-			}
+		} else if collection.Type != "" && collection.Type != spec.CollectionTypeDocument {
+			util.JsonError(w, http.StatusBadRequest, "collection is not a document collection")
+			return
+		} else if collection.Visibility == spec.CollectionVisibilityPrivate {
+			util.JsonError(w, http.StatusForbidden, "collection is private")
+			return
 		}
 		limit, err := util.ParseInt(r.URL.Query().Get("limit"), 100)
 		if err != nil {
@@ -185,12 +186,14 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 		if collection, ok := rm.GetCollection(collectionName, namespaceName); !ok {
 			util.JsonError(w, http.StatusNotFound, "collection not found")
 			return
-		} else {
-			if collection.Type != spec.CollectionTypeDocument {
-				util.JsonError(w, http.StatusBadRequest, "collection is not a document collection")
-				return
-			}
+		} else if collection.Type != spec.CollectionTypeDocument {
+			util.JsonError(w, http.StatusBadRequest, "collection is not a document collection")
+			return
+		} else if collection.Visibility == spec.CollectionVisibilityPrivate {
+			util.JsonError(w, http.StatusForbidden, "collection is private")
+			return
 		}
+
 		document, err := dm.GetDocumentByID(namespaceName, collectionName, documentId)
 		if err != nil {
 			util.JsonError(w, http.StatusNotFound, err.Error())

internal/config/config.go

@@ -248,7 +248,6 @@ func (conf *DGateConfig) GetLogger() (*zap.Logger, error) {
 	if logger, err := conf.Logging.ZapConfig.Build(); err != nil {
 		return nil, err
 	} else {
-		logger.Sync()
 		return logger, nil
 	}
 }

internal/config/resources.go

@@ -153,10 +153,10 @@ func (resources *DGateResources) Validate() (int, error) {
 		if col.Visibility != spec.CollectionVisibilityPublic && col.Visibility != spec.CollectionVisibilityPrivate {
 			return 0, errors.New("collection (" + col.Name + ") must specify visibility")
 		}
+		if col.Type != spec.CollectionTypeDocument && col.Type != spec.CollectionTypeFetcher {
+			return 0, errors.New("collection (" + col.Name + ") must specify type")
+		}
 		// TODO: Uncomment when modules are supported for collections
-		// if col.Type != spec.CollectionTypeDocument && col.Type != spec.CollectionTypeFetcher {
-		// 	return 0, errors.New("collection (" + col.Name + ") must specify type")
-		// }
 		// for _, modName := range col.Modules {
 		// 	if _, ok := modules[modName+"-"+col.NamespaceName]; !ok {
 		// 		return 0, errors.New("collection (" + col.Name + ") references non-existent module (" + modName + ")")

internal/proxy/dynamic_proxy.go

@@ -299,24 +299,26 @@ func (ps *ProxyState) Start() (err error) {
 func (ps *ProxyState) Stop() {
 	go func() {
 		<-time.After(10 * time.Second)
+		defer os.Exit(1)
 		ps.logger.Error("Failed to stop proxy server")
-		os.Exit(1)
 	}()
+
 	ps.logger.Info("Stopping proxy server")
-	defer ps.proxyLock.Unlock()
 	ps.proxyLock.Lock()
-	ps.Logger().Sync()
+	defer ps.proxyLock.Unlock()
+	defer os.Exit(0)
+	defer ps.Logger().Sync()
+
 	if raftNode := ps.Raft(); raftNode != nil {
 		raftNode.Shutdown().Error()
 	}
-	os.Exit(0)
 }
 
 func (ps *ProxyState) HandleRoute(requestCtxProvider *RequestContextProvider, pattern string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// ctx, cancel := context.WithCancel(requestCtxPrdovider.ctx)
 		// defer cancel()
-		ps.ProxyHandlerFunc(ps, requestCtxProvider.
+		ps.ProxyHandler(ps, requestCtxProvider.
 			CreateRequestContext(requestCtxProvider.ctx, w, r, pattern))
 	}
 }