v0.11.0 (#13)

* improve extractor promise wait timeout
add  prevotereq to rafthttp
optimize promise waiting
change example domains so they work on raft instances
add admin test

* code cleaning and make uuid optional

* create directory if it doesnt exists

* change all commands to use redirect forwarding and version logging

* feat: Add TEST_URL for functional tests

* refactor: WaitForChanges, SetReady

* refactor: Update methods in MockChangeState for better error handling.

* Add tests, refactor storage, optimize start up time for raft

* update test to be inclusive for distributed nodes as well, service url validation, improve document loading and storing

* fix e2e pipeline

* refactor admin, raft, and tests: fix deadlocks, slow startup, and

* remove distributed tests

.github/workflows/e2e.yml

@@ -36,22 +36,29 @@ jobs:
           dgate-cli --version
           go install github.com/dgate-io/dgate/cmd/dgate-server
           dgate-server --version
-
-      - run: go run cmd/dgate-server/main.go &
-
-      - name: Wait for server to start
-        run: sleep 5
     
       - name: Install jq
         run: |
           sudo apt install -y jq
           jq --version
       
-      - name: Functional Tests
+      - name: Install goreman
+        run: |
+          go install github.com/mattn/goreman@latest
+          goreman version
+
+      - run: go run cmd/dgate-server/main.go &
+
+      - run: cd functional-tests/raft_tests && goreman start &
+
+      - name: Wait for server to start
+        run: sleep 10
+
+      - name: Functional Standalone Tests
         run: |
           for i in functional-tests/admin_tests/*.sh; \
             do bash -c $i; done
-  
+
       - name: Run local k6 test
         uses: grafana/[email protected]
         with:

README.md

@@ -15,7 +15,7 @@ DGate is a distributed API Gateway built for developers. DGate allows you to use
 
 ## Getting Started
 
-Coming soon @ http://dgate.io/docs/getting-started
+http://dgate.io/docs/getting-started
 
 ### Installing
 

TODO.md

@@ -8,6 +8,12 @@
   - cluster management (raft commands, replica commands, etc.) (low priority)
   - other commands (backup, restore, etc.) (low priority)
 
+# Raft Snapshots
+
+- Add support for Raft snapshots to reduce the size of the Raft log. This can be used to reduce the size of the Raft log and improve the performance of the cluster.
+  - [ ] - Snapshot documents
+  - [ ] - Snapshot resources (with correlactions)
+
 ## Add Module Tests
 
 - Test multiple modules being used at the same time
@@ -138,13 +144,12 @@ Make it easier to debug modules by adding more logging and error handling. This
 
 Add stack tracing for typescript modules.
 
-
-## Decouple Admin API from Raft Implementation
-
-Currently, Raft Implementation is tightly coupled with the Admin API. This makes it difficult to change the Raft Implementation without changing the Admin API. Decouple the Raft Implementation from the Admin API to make it easier to change the Raft Implementation.
-
 ## Add Telemetry (sentry, datadog, etc.)
 
 ## ResourceManager callback for resource changes
 
-Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more.
+Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more.
+
+## Enable WAF
+
+https://github.com/corazawaf/coraza

cmd/dgate-server/main.go

@@ -58,19 +58,23 @@ func main() {
 		}
 		if dgateConfig, err := config.LoadConfig(*configPath); err != nil {
 			fmt.Printf("Error loading config: %s\n", err)
-			os.Exit(1)
+			panic(err)
 		} else {
 			logger, err := dgateConfig.GetLogger()
 			if err != nil {
 				fmt.Printf("Error setting up logger: %s\n", err)
-				os.Exit(1)
+				panic(err)
 			}
 			defer logger.Sync()
 			proxyState := proxy.NewProxyState(logger.Named("proxy"), dgateConfig)
-			admin.StartAdminAPI(version, dgateConfig, logger.Named("admin"), proxyState)
+			err = admin.StartAdminAPI(version, dgateConfig, logger.Named("admin"), proxyState)
+			if err != nil {
+				fmt.Printf("Error starting admin api: %s\n", err)
+				panic(err)
+			}
 			if err := proxyState.Start(); err != nil {
 				fmt.Printf("Error loading config: %s\n", err)
-				os.Exit(1)
+				panic(err)
 			}
 
 			sigchan := make(chan os.Signal, 1)

config.dgate.yaml

@@ -1,9 +1,8 @@
 version: v1
 debug: true
-log_level: ${LOG_LEVEL:-info}
+log_level: ${LOG_LEVEL:-debug}
 disable_default_namespace: true
-tags:
-  - debug
+tags: [debug, local, test]
 storage:
   type: file
   dir: .dgate/data/
@@ -15,7 +14,7 @@ test_server:
 proxy:
   port: ${PORT:-80}
   host: 0.0.0.0
-  enable_console_logger: true
+  console_log_level: info
   transport:
     dns_prefer_go: true
   init_resources:

functional-tests/admin_tests/admin_test.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eo xtrace
+
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
+PROXY_URL=${PROXY_URL:-"http://localhost"}
+TEST_URL=${TEST_URL:-"http://localhost:8888"}
+
+DIR="$( cd "$( dirname "$0" )" && pwd )"
+
+export DGATE_ADMIN_API=$ADMIN_URL
+
+# check if uuid is available
+if ! command -v uuid > /dev/null; then
+    id=X$RANDOM-$RANDOM-$RANDOM
+else
+    id=$(uuid)
+fi
+
+dgate-cli -Vf namespace create name=ns-$id
+
+dgate-cli -Vf domain create name=dm-$id \
+    namespace=ns-$id priority:=$RANDOM patterns="$id.example.com"
+
+dgate-cli -Vf service create \
+    name=svc-$id namespace=ns-$id \
+    urls="$TEST_URL/$RANDOM"
+
+dgate-cli -Vf module create name=module1 \
+    payload@=$DIR/admin_test.ts \
+    namespace=ns-$id
+
+dgate-cli -Vf route create \
+    name=rt-$id \
+    service=svc-$id \
+    namespace=ns-$id \
+    paths="/,/{id},/$id,/$id/{id}" \
+    methods=GET,POST,PUT \
+    modules=module1 \
+    preserveHost:=false \
+    stripPath:=false
+
+curl -sf $ADMIN_URL/readyz > /dev/null
+
+curl -f ${PROXY_URL}/$id/$RANDOM -H Host:$id.example.com
+
+echo "Admin Test Succeeded"

functional-tests/admin_tests/admin_test.ts

@@ -0,0 +1,4 @@
+
+export const responseModifier = async (ctx: any) => {
+    console.log("responseModifier -> path params", ctx.pathParams());
+}

functional-tests/admin_tests/change_checker.sh

@@ -9,41 +9,41 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=change_checker-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=change_checker-dm \
-    patterns:='["change_checker.com"]' \
+    patterns:='["change_checker.example.com"]' \
     namespace=change_checker-ns
 
-dgate-cli module create name=change_checker-mod \
+dgate-cli -Vf module create name=change_checker-mod \
     payload@=$DIR/change_checker_1.ts \
     namespace=change_checker-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt paths:='["/", "/{id}"]' \
     modules:='["change_checker-mod"]' \
     methods:='["GET","POST"]' \
     stripPath:=true \
     preserveHost:=true \
     namespace=change_checker-ns
 
-MODID1=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+MODID1=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod')
 
 if [ "$MODID1" != "module1" ]; then
     echo "Initial assert failed"
     exit 1
 fi
 
 
-dgate-cli module create name=change_checker-mod \
+dgate-cli -Vf module create name=change_checker-mod \
     payload@=$DIR/change_checker_2.ts \
     namespace=change_checker-ns
 
 # dgate-cli r.ker-ns
 
-MODID2=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+MODID2=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod')
 
 if [ "$MODID2" != "module2" ]; then
     echo "module update failed"

functional-tests/admin_tests/iphash_load_balancer_test.sh

@@ -4,32 +4,33 @@ set -eo xtrace
 
 ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
+TEST_URL=${TEST_URL:-"http://localhost:8888"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=test-lb-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=test-lb-dm \
-    patterns:='["test-lb.com"]' \
+    patterns:='["test-lb.example.com"]' \
     namespace=test-lb-ns
 
 MOD_B64="$(base64 < $DIR/iphash_load_balancer.ts)"
-dgate-cli module create \
+dgate-cli -Vf module create \
     name=printer \
     payload="$MOD_B64" \
     namespace=test-lb-ns
 
 
-dgate-cli service create \
+dgate-cli -Vf service create \
     name=base_svc \
-    urls:='["http://localhost:8888/a","http://localhost:8888/b","http://localhost:8888/c"]' \
+    urls="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \
     namespace=test-lb-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt \
     paths:='["/test-lb","/hello"]' \
     methods:='["GET"]' \
@@ -39,9 +40,9 @@ dgate-cli route create \
     preserveHost:=true \
     namespace=test-lb-ns
 
-path1="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com | jq -r '.data.path')"
+path1="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')"
 
-path2="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
+path2="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
 
 if [ "$path1" != "$path2" ]; then
     echo "IP Hash Load Balancer Test Passed"

functional-tests/admin_tests/merge_responses_test.sh

@@ -9,21 +9,21 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=test-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=test-dm \
-    patterns:='["test.com"]' \
+    patterns:='["test.example.com"]' \
     namespace=test-ns
 
 MOD_B64="$(base64 < $DIR/merge_responses.ts)"
-dgate-cli module create \
+dgate-cli -Vf module create \
     name=printer \
     payload="$MOD_B64" \
     namespace=test-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt \
     paths:='["/test","/hello"]' \
     methods:='["GET"]' \
@@ -32,6 +32,6 @@ dgate-cli route create \
     preserveHost:=true \
     namespace=test-ns
 
-curl -s --fail-with-body ${PROXY_URL}/hello -H Host:test.com
+curl -sf ${PROXY_URL}/hello -H Host:test.example.com
 
 echo "Merge Responses Test Passed"

functional-tests/admin_tests/modify_request_test.sh

@@ -4,30 +4,31 @@ set -eo xtrace
 
 ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
 PROXY_URL=${PROXY_URL:-"http://localhost"}
+TEST_URL=${TEST_URL:-"http://localhost:8888"}
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 export DGATE_ADMIN_API=$ADMIN_URL
 
-dgate-cli namespace create \
+dgate-cli -Vf namespace create \
     name=modify_request_test-ns
 
-dgate-cli domain create \
+dgate-cli -Vf domain create \
     name=modify_request_test-dm \
-    patterns:='["modify_request_test.com"]' \
+    patterns:='["modify_request_test.example.com"]' \
     namespace=modify_request_test-ns
 
 MOD_B64="$(base64 < $DIR/modify_request.ts)"
-dgate-cli module create \
+dgate-cli -Vf module create \
     name=printer payload="$MOD_B64" \
     namespace=modify_request_test-ns
 
-dgate-cli service create \
+dgate-cli -Vf service create \
     name=base_svc \
-    urls:='["http://localhost:8888"]' \
+    urls="$TEST_URL" \
     namespace=modify_request_test-ns
 
-dgate-cli route create \
+dgate-cli -Vf route create \
     name=base_rt \
     paths:='["/modify_request_test"]' \
     methods:='["GET"]' \
@@ -37,8 +38,8 @@ dgate-cli route create \
     namespace=modify_request_test-ns \
     service='base_svc'
 
-curl -s --fail-with-body ${PROXY_URL}/modify_request_test \
-    -H Host:modify_request_test.com \
+curl -sf ${PROXY_URL}/modify_request_test \
+    -H Host:modify_request_test.example.com \
     -H X-Forwarded-For:1.1.1.1
 
 echo "Modify Request Test Passed"