add new async sleep

add imple retry timeout
add disable metrics opt
refactor module logic

.gitignore

@@ -13,3 +13,4 @@ dist/
 .dgate*/
 cov.out
 go.work.sum
+.env

TODO.md

@@ -11,11 +11,6 @@
 
 ## Replace zerolog with slog
 
-## Improve async function performance
-
-There is a pretty significant difference in performance when using async function.
-
-
 ## Add Module Tests
 
 - Test multiple modules being used at the same time
@@ -93,38 +88,6 @@ DGate Runtime is a JavaScript/TypeScript runtime that can be used to test module
 
 RuntimePool is a pool of runtimes that can be used to execute modules. It can be used to manage the different modules and clean up resources when they are no longer needed or idle for a certain amount of time.
 
-## TCP Proxy/Gateway (L4LB) (low priority)
-
-Using the same architecture as the HTTP Proxy/Gateway, create a TCP Proxy/Gateway that can be used to proxy TCP connections to upstream servers.
-
-A 'Custom Protocols API'can  allow users to define custom protocols that can be used to proxy TCP connections to upstream servers or handle the connections themselves.
-
-The custom protocols can be defined using JavaScript/TypeScript function or using protocol definitions (API) which will allow these values to be passed to the JavaScript/TypeScript code.
-
-```
-{
-  "name": "custom_protocol",
-  "version": "1",
-  "description": "Custom Protocol",
-  "modules": ["module_x"]
-  "format_definitions": [
-    {
-      "name": "command",
-      "type": "uint8"
-    }
-    {
-      "name": "data_len",
-      "type": "int16"
-    }
-    {
-      "name": "data",
-      "type": "string",
-      "length": "variable.data_len.length"
-    }
-  ]
-}
-```
-
 ## Server Tags
 
 No special characters are allowed in the tag name or value
@@ -174,6 +137,17 @@ A good example of a bundle would be a bundle that adds support for OAuth2 authen
 
 Differing from common resource versioning, modules can have multiple versions that can be used at the same time. This can be used to test new versions of modules before deploying them to the cluster.
 
-## Secrets
 
-- Add support for secrets that can be used in modules. Secrets can be used to store sensitive information like API keys, passwords, etc. Secrets can only be used in modules and cannot be accessed by the API. Explicit permissions can be set to allow certain modules to access certain secrets. Secrets are also versioned and can be rolled back if necessary. This also allows different modules to use different versions of the same secret.
+## DGate CLI - argument variable suggestions
+
+For example, if the user types an argument that is not recognized, the CLI can suggest the correct argument by search the available arguments and finding the closest match.
+```
+dgate-cli ns mk my-ns nmae=my-ns
+Variable 'nmae' is not recognized. Did you mean 'name'?
+```
+
+## Improve Module Debugability
+
+Make it easier to debug modules by adding more logging and error handling. This can be done by adding more logging to the modules and making it easier to see the logs in the Admin Console.
+
+Add stack tracing for typescript modules.

functional-tests/admin_tests/iphash_load_balancer_test.sh

@@ -2,11 +2,12 @@
 
 set -eo xtrace
 
-ADMIN_URL=${ADMIN_URL:-"http://localhost:9080/api/v1"}
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
+export DGATE_ADMIN_API=$ADMIN_URL
 
 dgate-cli namespace create \
     name=test-lb-ns

functional-tests/admin_tests/merge_responses_test.sh

@@ -2,11 +2,13 @@
 
 set -eo xtrace
 
-ADMIN_URL=${ADMIN_URL:-"http://localhost:9080/api/v1"}
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
+export DGATE_ADMIN_API=$ADMIN_URL
+
 dgate-cli namespace create \
     name=test-ns
 

functional-tests/admin_tests/modify_request_test.sh

@@ -2,11 +2,13 @@
 
 set -eo xtrace
 
-ADMIN_URL=${ADMIN_URL:-"http://localhost:9080/api/v1"}
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
+export DGATE_ADMIN_API=$ADMIN_URL
+
 dgate-cli namespace create \
     name=modify_request_test-ns
 

functional-tests/admin_tests/modify_response_test.sh

@@ -2,11 +2,13 @@
 
 set -eo xtrace
 
-ADMIN_URL=${ADMIN_URL:-"http://localhost:9080/api/v1"}
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
+export DGATE_ADMIN_API=$ADMIN_URL
+
 dgate-cli namespace create \
     name=test-ns
 

functional-tests/admin_tests/performance_test_prep.sh

@@ -2,23 +2,24 @@
 
 set -eo xtrace
 
-ADMIN_URL=${ADMIN_URL:-"http://localhost:9080/api/v1"}
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
-# domain setup
 
-dgate-cli namespace create \
+export DGATE_ADMIN_API=$ADMIN_URL
+
+dgate-cli -V -f namespace create \
     name=test-ns1
 
 dgate-cli domain create \
     name=test-dm patterns:='["dgate.dev"]' \
-    namespace=test-ns1
+    namespace=test-ns1 priority:=100
 
 dgate-cli service create \
     name=test-svc urls:='["http://localhost:8888"]' \
-    namespace=test-ns1 retries=5
+    namespace=test-ns1 retries:=3 retryTimeout=50ms
 
 MOD_B64="$(base64 < $DIR/performance_test_prep.ts)"
 dgate-cli module create \

functional-tests/admin_tests/performance_test_prep.ts

@@ -1,15 +1,22 @@
+import { sleep } from "dgate";
 
-export const fetchUpstream = (ctx) =>
+export const fetchUpstream = async (ctx) =>
     console.debug("fetchUpstream:", JSON.stringify(ctx));
 
-export const requestModifier = (ctx) =>
+export const requestModifier = async (ctx) =>
     console.debug("requestModifier:", JSON.stringify(ctx));
 
-export const responseModifier = (ctx) =>
+export const responseModifier = async (ctx) =>
     console.debug("responseModifier:", JSON.stringify(ctx));
 
-export const errorHandler = (ctx, err) =>
+export const errorHandler = async (ctx, err) =>
     console.debug("errorHandler:", JSON.stringify(ctx), err);
 
-export const requestHandler = (ctx) =>
+export const requestHandler = async (ctx) => {
     console.debug("requestHandler:", JSON.stringify(ctx));
+    const req = ctx.request()
+    if (req.query.has("wait")) {
+        const wait = req.query.get("wait")
+        await sleep(+wait || wait);
+    }
+};

functional-tests/admin_tests/url_shortener_test.sh

@@ -2,11 +2,13 @@
 
 set -eo xtrace
 
-ADMIN_URL=${ADMIN_URL:-"http://localhost:9080/api/v1"}
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
+export DGATE_ADMIN_API=$ADMIN_URL
+
 dgate-cli namespace create \
     name=url_shortener-ns
 

functional-tests/raft_tests/raft_test.sh

@@ -45,21 +45,15 @@ dgate-cli -f route create \
     preserveHost:=false \
     stripPath:=true
 
+curl -f $ADMIN_URL1/readyz
+
 for i in {1..5}; do
-    curl -f $PROXY_URL1/$id/$i -H Host:$id.example.com
-    curl -f $PROXY_URL2/$id/$i -H Host:$id.example.com
-    curl -f $PROXY_URL3/$id/$i -H Host:$id.example.com
-    curl -f $PROXY_URL4/$id/$i -H Host:$id.example.com
-    curl -f $PROXY_URL5/$id/$i -H Host:$id.example.com
+    for j in {1..3}; do
+        proxy_url=PROXY_URL$i
+        curl -f ${!proxy_url}/$id/$j -H Host:$id.example.com
+    done
 done
 
-dgate-cli -V --admin $ADMIN_URL1 route get name=rt-$id namespace=ns-$id
-dgate-cli -V --admin $ADMIN_URL2 route get name=rt-$id namespace=ns-$id
-dgate-cli -V --admin $ADMIN_URL3 route get name=rt-$id namespace=ns-$id
-dgate-cli -V --admin $ADMIN_URL4 route get name=rt-$id namespace=ns-$id
-dgate-cli -V --admin $ADMIN_URL5 route get name=rt-$id namespace=ns-$id
-
-
 if dgate-cli --admin $ADMIN_URL4 namespace create name=0; then
     echo "Expected error when creating namespace"
     exit 1

functional-tests/raft_tests/test1.yaml

@@ -25,28 +25,9 @@ proxy:
   redirect_https: 
     - /^(.+\.)?example\.(yyy|net)$/
   allowed_domains:
-    - "bubba"
-    - "localhost"
+    - "*local*"
+    - "*dgate.dev"
     - /^(.+\.)?example\.(yyy|com|net)$/
-  client_transport:
-    dns_server: udp://1.1.1.1:53
-    dns_timeout: 10s
-    max_conns_per_host: 5
-    max_idle_conns: 100
-    max_idle_conns_per_host: 5
-    idle_conn_timeout: 60s
-    force_attempt_http2: false
-    disable_compression: false
-    tls_handshake_timeout: 10s
-    expect_continue_timeout: 5s
-    max_response_header_bytes: 262144
-    write_buffer_size: 4096
-    read_buffer_size: 4096
-    max_conns_per_client: 1000
-    max_body_bytes: 32768
-    disable_keep_alives: false
-    response_header_timeout: 10s
-    dial_timeout: 10s
 admin:
   port: 9081
   host: 0.0.0.0

functional-tests/raft_tests/test2.yaml

@@ -25,25 +25,6 @@ proxy:
     - "bubba"
     - "localhost"
     - /^(.+\.)?example\.(yyy|com|net)$/
-  client_transport:
-    dns_server: udp://1.1.1.1:53
-    dns_timeout: 10s
-    max_conns_per_host: 1000
-    max_idle_conns: 5000
-    max_idle_conns_per_host: 1000
-    idle_conn_timeout: 60s
-    force_attempt_http2: false
-    disable_compression: false
-    tls_handshake_timeout: 10s
-    expect_continue_timeout: 5s
-    max_response_header_bytes: 4096
-    write_buffer_size: 4096
-    read_buffer_size: 4096
-    max_conns_per_client: 1000
-    max_body_bytes: 32768
-    disable_keep_alives: false
-    response_header_timeout: 10s
-    dial_timeout: 10s
 admin:
   port: 9082
   host: 0.0.0.0

functional-tests/raft_tests/test3.yaml

@@ -26,25 +26,6 @@ proxy:
     - "bubba"
     - "localhost"
     - /^(.+\.)?example\.(yyy|com|net)$/
-  client_transport:
-    dns_server: udp://1.1.1.1:53
-    dns_timeout: 10s
-    max_conns_per_host: 1000
-    max_idle_conns: 5000
-    max_idle_conns_per_host: 1000
-    idle_conn_timeout: 60s
-    force_attempt_http2: false
-    disable_compression: false
-    tls_handshake_timeout: 10s
-    expect_continue_timeout: 5s
-    max_response_header_bytes: 4096
-    write_buffer_size: 4096
-    read_buffer_size: 4096
-    max_conns_per_client: 1000
-    max_body_bytes: 32768
-    disable_keep_alives: false
-    response_header_timeout: 10s
-    dial_timeout: 10s
 admin:
   port: 9083
   host: 0.0.0.0

functional-tests/raft_tests/test4_watch.yaml

@@ -25,25 +25,6 @@ proxy:
     - "bubba"
     - "localhost"
     - /^(.+\.)?example\.(yyy|com|net)$/
-  client_transport:
-    dns_server: udp://1.1.1.1:53
-    dns_timeout: 10s
-    max_conns_per_host: 1000
-    max_idle_conns: 5000
-    max_idle_conns_per_host: 1000
-    idle_conn_timeout: 60s
-    force_attempt_http2: false
-    disable_compression: false
-    tls_handshake_timeout: 10s
-    expect_continue_timeout: 5s
-    max_response_header_bytes: 4096
-    write_buffer_size: 4096
-    read_buffer_size: 4096
-    max_conns_per_client: 1000
-    max_body_bytes: 32768
-    disable_keep_alives: false
-    response_header_timeout: 10s
-    dial_timeout: 10s
 admin:
   port: 9084
   host: 0.0.0.0

functional-tests/raft_tests/test5_watch.yaml

@@ -25,25 +25,6 @@ proxy:
     - "bubba"
     - "localhost"
     - /^(.+\.)?example\.(yyy|com|net)$/
-  client_transport:
-    dns_server: udp://1.1.1.1:53
-    dns_timeout: 10s
-    max_conns_per_host: 1000
-    max_idle_conns: 5000
-    max_idle_conns_per_host: 1000
-    idle_conn_timeout: 60s
-    force_attempt_http2: false
-    disable_compression: false
-    tls_handshake_timeout: 10s
-    expect_continue_timeout: 5s
-    max_response_header_bytes: 4096
-    write_buffer_size: 4096
-    read_buffer_size: 4096
-    max_conns_per_client: 1000
-    max_body_bytes: 32768
-    disable_keep_alives: false
-    response_header_timeout: 10s
-    dial_timeout: 10s
 admin:
   port: 9085
   host: 0.0.0.0

internal/admin/admin_fsm.go

@@ -81,7 +81,11 @@ func (fsm *dgateAdminFSM) applyLog(log *raft.Log) (*spec.ChangeLog, error) {
 				Msgf("%d: config update - server: %s", i, server.Address)
 		}
 	case raft.LogBarrier:
-		fsm.ps.WaitForChanges()
+		err := fsm.ps.WaitForChanges()
+		if err != nil {
+			fsm.logger.Err(err).
+				Msg("Error waiting for changes")
+		}
 	default:
 		fsm.ps.Logger().Error().
 			Msg("Unknown log type in FSM Apply")