From 4edf5ea920de5b3bb3c072607c1aca1adc400ee5 Mon Sep 17 00:00:00 2001 From: Joe Williams <7463219+bubbajoe@users.noreply.github.com> Date: Fri, 21 Jun 2024 02:46:38 +0900 Subject: [PATCH] v0.11.0 (#13) * improve extractor promise wait timeout add prevotereq to rafthttp optimize promise waiting change example domains so they work on raft instances add admin test * code cleaning and make uuid optional * create directory if it doesnt exists * change all commands to use redirect forwarding and version logging * feat: Add TEST_URL for functional tests * refactor: WaitForChanges, SetReady * refactor: Update methods in MockChangeState for better error handling. * Add tests, refactor storage, optimize start up time for raft * update test to be inclusive for distributed nodes as well, service url validation, improve document loading and storing * fix e2e pipeline * refactor admin, raft, and tests: fix deadlocks, slow startup, and * remove distributed tests --- .github/workflows/e2e.yml | 21 +- README.md | 2 +- TODO.md | 17 +- cmd/dgate-server/main.go | 12 +- config.dgate.yaml | 7 +- functional-tests/admin_tests/admin_test.sh | 47 +++ functional-tests/admin_tests/admin_test.ts | 4 + .../admin_tests/change_checker.sh | 16 +- .../admin_tests/iphash_load_balancer_test.sh | 19 +- .../admin_tests/merge_responses_test.sh | 12 +- .../admin_tests/modify_request_test.sh | 19 +- .../admin_tests/modify_response_test.sh | 17 +- .../admin_tests/multi_module_test.sh | 22 +- .../admin_tests/performance_test_prep.sh | 26 +- .../admin_tests/url_shortener_test.sh | 26 +- functional-tests/raft_tests/raft_test.sh | 48 +-- go.mod | 22 +- go.sum | 56 ++- internal/admin/admin_api.go | 11 +- internal/admin/admin_fsm.go | 133 +++++-- internal/admin/admin_raft.go | 143 ++++--- internal/admin/admin_routes.go | 75 ++-- internal/admin/admin_routes_test.go | 26 ++ internal/admin/changestate/change_state.go | 10 +- .../changestate/testutil/change_state.go | 28 +- internal/admin/routes/collection_routes.go | 13 +- internal/admin/routes/domain_routes.go | 5 +- internal/admin/routes/misc_routes.go | 66 ++-- internal/admin/routes/module_routes.go | 5 +- internal/admin/routes/module_routes_test.go | 54 ++- internal/admin/routes/namespace_routes.go | 8 +- .../admin/routes/namespace_routes_test.go | 52 ++- internal/admin/routes/route_routes.go | 8 +- internal/admin/routes/route_routes_test.go | 11 +- internal/admin/routes/secret_routes.go | 2 +- internal/admin/routes/service_routes.go | 31 +- internal/admin/routes/service_routes_test.go | 11 +- internal/config/config.go | 3 +- internal/config/configtest/dgate_configs.go | 20 +- internal/config/loader.go | 1 + internal/config/store_config.go | 5 +- internal/proxy/change_log.go | 361 +++++++++++------- internal/proxy/dynamic_proxy.go | 276 ++++++++----- internal/proxy/proxy_documents.go | 2 +- internal/proxy/proxy_handler.go | 4 +- internal/proxy/proxy_printer.go | 43 ++- internal/proxy/proxy_replication.go | 13 +- internal/proxy/proxy_state.go | 326 +++++++++------- internal/proxy/proxy_state_test.go | 15 +- internal/proxy/proxystore/proxy_store.go | 87 +++-- internal/proxy/proxystore/proxy_store_test.go | 121 ++++++ internal/proxy/runtime_context.go | 8 +- internal/proxy/util.go | 12 +- performance-tests/long-perf-test.js | 2 +- performance-tests/perf-test.js | 2 +- pkg/modules/dgate/state/state_mod.go | 8 +- pkg/modules/extractors/async_tracker.go | 66 ---- pkg/modules/extractors/extractors.go | 39 +- pkg/modules/extractors/extractors_test.go | 7 +- pkg/modules/extractors/runtime_test.go | 3 +- pkg/modules/testutil/testutil.go | 2 +- pkg/modules/types/generator.go | 47 --- .../{raftadmin_client.go => client.go} | 103 +++-- pkg/raftadmin/{raftadmin.go => server.go} | 55 ++- .../{raftadmin_test.go => server_test.go} | 7 +- pkg/rafthttp/rafthttp.go | 26 +- pkg/rafthttp/rafthttp_test.go | 4 +- pkg/storage/debug_storage.go | 93 ----- pkg/storage/file_storage.go | 225 +++++------ pkg/storage/mem_storage.go | 141 +++++++ pkg/storage/memory_storage.go | 27 -- pkg/storage/storage.go | 1 + pkg/typescript/typescript.go | 8 +- pkg/typescript/typescript_test.go | 3 +- pkg/util/http_test.go | 22 +- pkg/util/parse.go | 14 +- pkg/util/parse_test.go | 20 + pkg/util/queue/queue.go | 9 +- pkg/util/sliceutil/slice.go | 17 + pkg/util/sliceutil/slice_test.go | 114 ++++++ 80 files changed, 2042 insertions(+), 1405 deletions(-) create mode 100755 functional-tests/admin_tests/admin_test.sh create mode 100644 functional-tests/admin_tests/admin_test.ts create mode 100644 internal/admin/admin_routes_test.go create mode 100644 internal/proxy/proxystore/proxy_store_test.go delete mode 100644 pkg/modules/extractors/async_tracker.go delete mode 100644 pkg/modules/types/generator.go rename pkg/raftadmin/{raftadmin_client.go => client.go} (72%) rename pkg/raftadmin/{raftadmin.go => server.go} (82%) rename pkg/raftadmin/{raftadmin_test.go => server_test.go} (97%) delete mode 100644 pkg/storage/debug_storage.go create mode 100644 pkg/storage/mem_storage.go delete mode 100644 pkg/storage/memory_storage.go create mode 100644 pkg/util/parse_test.go create mode 100644 pkg/util/sliceutil/slice_test.go diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 55ffbc2..c9951bd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,22 +36,29 @@ jobs: dgate-cli --version go install github.com/dgate-io/dgate/cmd/dgate-server dgate-server --version - - - run: go run cmd/dgate-server/main.go & - - - name: Wait for server to start - run: sleep 5 - name: Install jq run: | sudo apt install -y jq jq --version - - name: Functional Tests + - name: Install goreman + run: | + go install github.com/mattn/goreman@latest + goreman version + + - run: go run cmd/dgate-server/main.go & + + - run: cd functional-tests/raft_tests && goreman start & + + - name: Wait for server to start + run: sleep 10 + + - name: Functional Standalone Tests run: | for i in functional-tests/admin_tests/*.sh; \ do bash -c $i; done - + - name: Run local k6 test uses: grafana/k6-action@v0.3.1 with: diff --git a/README.md b/README.md index de75246..2617067 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ DGate is a distributed API Gateway built for developers. DGate allows you to use ## Getting Started -Coming soon @ http://dgate.io/docs/getting-started +http://dgate.io/docs/getting-started ### Installing diff --git a/TODO.md b/TODO.md index 5bb5d29..9760e6d 100644 --- a/TODO.md +++ b/TODO.md @@ -8,6 +8,12 @@ - cluster management (raft commands, replica commands, etc.) (low priority) - other commands (backup, restore, etc.) (low priority) +# Raft Snapshots + +- Add support for Raft snapshots to reduce the size of the Raft log. This can be used to reduce the size of the Raft log and improve the performance of the cluster. + - [ ] - Snapshot documents + - [ ] - Snapshot resources (with correlactions) + ## Add Module Tests - Test multiple modules being used at the same time @@ -138,13 +144,12 @@ Make it easier to debug modules by adding more logging and error handling. This Add stack tracing for typescript modules. - -## Decouple Admin API from Raft Implementation - -Currently, Raft Implementation is tightly coupled with the Admin API. This makes it difficult to change the Raft Implementation without changing the Admin API. Decouple the Raft Implementation from the Admin API to make it easier to change the Raft Implementation. - ## Add Telemetry (sentry, datadog, etc.) ## ResourceManager callback for resource changes -Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more. \ No newline at end of file +Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more. + +## Enable WAF + +https://github.com/corazawaf/coraza diff --git a/cmd/dgate-server/main.go b/cmd/dgate-server/main.go index 0f1a8ed..95dc01e 100644 --- a/cmd/dgate-server/main.go +++ b/cmd/dgate-server/main.go @@ -58,19 +58,23 @@ func main() { } if dgateConfig, err := config.LoadConfig(*configPath); err != nil { fmt.Printf("Error loading config: %s\n", err) - os.Exit(1) + panic(err) } else { logger, err := dgateConfig.GetLogger() if err != nil { fmt.Printf("Error setting up logger: %s\n", err) - os.Exit(1) + panic(err) } defer logger.Sync() proxyState := proxy.NewProxyState(logger.Named("proxy"), dgateConfig) - admin.StartAdminAPI(version, dgateConfig, logger.Named("admin"), proxyState) + err = admin.StartAdminAPI(version, dgateConfig, logger.Named("admin"), proxyState) + if err != nil { + fmt.Printf("Error starting admin api: %s\n", err) + panic(err) + } if err := proxyState.Start(); err != nil { fmt.Printf("Error loading config: %s\n", err) - os.Exit(1) + panic(err) } sigchan := make(chan os.Signal, 1) diff --git a/config.dgate.yaml b/config.dgate.yaml index aeeca0d..d3fa32b 100644 --- a/config.dgate.yaml +++ b/config.dgate.yaml @@ -1,9 +1,8 @@ version: v1 debug: true -log_level: ${LOG_LEVEL:-info} +log_level: ${LOG_LEVEL:-debug} disable_default_namespace: true -tags: - - debug +tags: [debug, local, test] storage: type: file dir: .dgate/data/ @@ -15,7 +14,7 @@ test_server: proxy: port: ${PORT:-80} host: 0.0.0.0 - enable_console_logger: true + console_log_level: info transport: dns_prefer_go: true init_resources: diff --git a/functional-tests/admin_tests/admin_test.sh b/functional-tests/admin_tests/admin_test.sh new file mode 100755 index 0000000..e1c56a3 --- /dev/null +++ b/functional-tests/admin_tests/admin_test.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -eo xtrace + +ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"} +PROXY_URL=${PROXY_URL:-"http://localhost"} +TEST_URL=${TEST_URL:-"http://localhost:8888"} + +DIR="$( cd "$( dirname "$0" )" && pwd )" + +export DGATE_ADMIN_API=$ADMIN_URL + +# check if uuid is available +if ! command -v uuid > /dev/null; then + id=X$RANDOM-$RANDOM-$RANDOM +else + id=$(uuid) +fi + +dgate-cli -Vf namespace create name=ns-$id + +dgate-cli -Vf domain create name=dm-$id \ + namespace=ns-$id priority:=$RANDOM patterns="$id.example.com" + +dgate-cli -Vf service create \ + name=svc-$id namespace=ns-$id \ + urls="$TEST_URL/$RANDOM" + +dgate-cli -Vf module create name=module1 \ + payload@=$DIR/admin_test.ts \ + namespace=ns-$id + +dgate-cli -Vf route create \ + name=rt-$id \ + service=svc-$id \ + namespace=ns-$id \ + paths="/,/{id},/$id,/$id/{id}" \ + methods=GET,POST,PUT \ + modules=module1 \ + preserveHost:=false \ + stripPath:=false + +curl -sf $ADMIN_URL/readyz > /dev/null + +curl -f ${PROXY_URL}/$id/$RANDOM -H Host:$id.example.com + +echo "Admin Test Succeeded" diff --git a/functional-tests/admin_tests/admin_test.ts b/functional-tests/admin_tests/admin_test.ts new file mode 100644 index 0000000..eac6297 --- /dev/null +++ b/functional-tests/admin_tests/admin_test.ts @@ -0,0 +1,4 @@ + +export const responseModifier = async (ctx: any) => { + console.log("responseModifier -> path params", ctx.pathParams()); +} \ No newline at end of file diff --git a/functional-tests/admin_tests/change_checker.sh b/functional-tests/admin_tests/change_checker.sh index 84dd271..4371c74 100755 --- a/functional-tests/admin_tests/change_checker.sh +++ b/functional-tests/admin_tests/change_checker.sh @@ -9,19 +9,19 @@ DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli namespace create \ +dgate-cli -Vf namespace create \ name=change_checker-ns -dgate-cli domain create \ +dgate-cli -Vf domain create \ name=change_checker-dm \ - patterns:='["change_checker.com"]' \ + patterns:='["change_checker.example.com"]' \ namespace=change_checker-ns -dgate-cli module create name=change_checker-mod \ +dgate-cli -Vf module create name=change_checker-mod \ payload@=$DIR/change_checker_1.ts \ namespace=change_checker-ns -dgate-cli route create \ +dgate-cli -Vf route create \ name=base_rt paths:='["/", "/{id}"]' \ modules:='["change_checker-mod"]' \ methods:='["GET","POST"]' \ @@ -29,7 +29,7 @@ dgate-cli route create \ preserveHost:=true \ namespace=change_checker-ns -MODID1=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod') +MODID1=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod') if [ "$MODID1" != "module1" ]; then echo "Initial assert failed" @@ -37,13 +37,13 @@ if [ "$MODID1" != "module1" ]; then fi -dgate-cli module create name=change_checker-mod \ +dgate-cli -Vf module create name=change_checker-mod \ payload@=$DIR/change_checker_2.ts \ namespace=change_checker-ns # dgate-cli r.ker-ns -MODID2=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod') +MODID2=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod') if [ "$MODID2" != "module2" ]; then echo "module update failed" diff --git a/functional-tests/admin_tests/iphash_load_balancer_test.sh b/functional-tests/admin_tests/iphash_load_balancer_test.sh index 4a716e1..c6ae0b5 100755 --- a/functional-tests/admin_tests/iphash_load_balancer_test.sh +++ b/functional-tests/admin_tests/iphash_load_balancer_test.sh @@ -4,32 +4,33 @@ set -eo xtrace ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"} PROXY_URL=${PROXY_URL:-"http://localhost"} +TEST_URL=${TEST_URL:-"http://localhost:8888"} DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli namespace create \ +dgate-cli -Vf namespace create \ name=test-lb-ns -dgate-cli domain create \ +dgate-cli -Vf domain create \ name=test-lb-dm \ - patterns:='["test-lb.com"]' \ + patterns:='["test-lb.example.com"]' \ namespace=test-lb-ns MOD_B64="$(base64 < $DIR/iphash_load_balancer.ts)" -dgate-cli module create \ +dgate-cli -Vf module create \ name=printer \ payload="$MOD_B64" \ namespace=test-lb-ns -dgate-cli service create \ +dgate-cli -Vf service create \ name=base_svc \ - urls:='["http://localhost:8888/a","http://localhost:8888/b","http://localhost:8888/c"]' \ + urls="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \ namespace=test-lb-ns -dgate-cli route create \ +dgate-cli -Vf route create \ name=base_rt \ paths:='["/test-lb","/hello"]' \ methods:='["GET"]' \ @@ -39,9 +40,9 @@ dgate-cli route create \ preserveHost:=true \ namespace=test-lb-ns -path1="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com | jq -r '.data.path')" +path1="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')" -path2="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')" +path2="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')" if [ "$path1" != "$path2" ]; then echo "IP Hash Load Balancer Test Passed" diff --git a/functional-tests/admin_tests/merge_responses_test.sh b/functional-tests/admin_tests/merge_responses_test.sh index 83752c9..7774d91 100755 --- a/functional-tests/admin_tests/merge_responses_test.sh +++ b/functional-tests/admin_tests/merge_responses_test.sh @@ -9,21 +9,21 @@ DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli namespace create \ +dgate-cli -Vf namespace create \ name=test-ns -dgate-cli domain create \ +dgate-cli -Vf domain create \ name=test-dm \ - patterns:='["test.com"]' \ + patterns:='["test.example.com"]' \ namespace=test-ns MOD_B64="$(base64 < $DIR/merge_responses.ts)" -dgate-cli module create \ +dgate-cli -Vf module create \ name=printer \ payload="$MOD_B64" \ namespace=test-ns -dgate-cli route create \ +dgate-cli -Vf route create \ name=base_rt \ paths:='["/test","/hello"]' \ methods:='["GET"]' \ @@ -32,6 +32,6 @@ dgate-cli route create \ preserveHost:=true \ namespace=test-ns -curl -s --fail-with-body ${PROXY_URL}/hello -H Host:test.com +curl -sf ${PROXY_URL}/hello -H Host:test.example.com echo "Merge Responses Test Passed" diff --git a/functional-tests/admin_tests/modify_request_test.sh b/functional-tests/admin_tests/modify_request_test.sh index df42d71..c0e7977 100755 --- a/functional-tests/admin_tests/modify_request_test.sh +++ b/functional-tests/admin_tests/modify_request_test.sh @@ -4,30 +4,31 @@ set -eo xtrace ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"} PROXY_URL=${PROXY_URL:-"http://localhost"} +TEST_URL=${TEST_URL:-"http://localhost:8888"} DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli namespace create \ +dgate-cli -Vf namespace create \ name=modify_request_test-ns -dgate-cli domain create \ +dgate-cli -Vf domain create \ name=modify_request_test-dm \ - patterns:='["modify_request_test.com"]' \ + patterns:='["modify_request_test.example.com"]' \ namespace=modify_request_test-ns MOD_B64="$(base64 < $DIR/modify_request.ts)" -dgate-cli module create \ +dgate-cli -Vf module create \ name=printer payload="$MOD_B64" \ namespace=modify_request_test-ns -dgate-cli service create \ +dgate-cli -Vf service create \ name=base_svc \ - urls:='["http://localhost:8888"]' \ + urls="$TEST_URL" \ namespace=modify_request_test-ns -dgate-cli route create \ +dgate-cli -Vf route create \ name=base_rt \ paths:='["/modify_request_test"]' \ methods:='["GET"]' \ @@ -37,8 +38,8 @@ dgate-cli route create \ namespace=modify_request_test-ns \ service='base_svc' -curl -s --fail-with-body ${PROXY_URL}/modify_request_test \ - -H Host:modify_request_test.com \ +curl -sf ${PROXY_URL}/modify_request_test \ + -H Host:modify_request_test.example.com \ -H X-Forwarded-For:1.1.1.1 echo "Modify Request Test Passed" diff --git a/functional-tests/admin_tests/modify_response_test.sh b/functional-tests/admin_tests/modify_response_test.sh index 8d8196c..97390cc 100755 --- a/functional-tests/admin_tests/modify_response_test.sh +++ b/functional-tests/admin_tests/modify_response_test.sh @@ -4,30 +4,31 @@ set -eo xtrace ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"} PROXY_URL=${PROXY_URL:-"http://localhost"} +TEST_URL=${TEST_URL:-"http://localhost:8888"} DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli namespace create \ +dgate-cli -Vf namespace create \ name=test-ns -dgate-cli domain create \ +dgate-cli -Vf domain create \ name=test-dm \ - patterns:='["test.com"]' \ + patterns:='["test.example.com"]' \ namespace=test-ns MOD_B64="$(base64 < $DIR/modify_response.ts)" -dgate-cli module create \ +dgate-cli -Vf module create \ name=printer payload="$MOD_B64" \ namespace=test-ns -dgate-cli service create \ +dgate-cli -Vf service create \ name=base_svc \ - urls:='["http://localhost:8888"]' \ + urls="$TEST_URL"\ namespace=test-ns -dgate-cli route create \ +dgate-cli -Vf route create \ name=base_rt \ paths:='["/test","/hello"]' \ methods:='["GET"]' \ @@ -37,6 +38,6 @@ dgate-cli route create \ namespace=test-ns \ service='base_svc' -curl -s ${PROXY_URL}/test -H Host:test.com +curl -s ${PROXY_URL}/test -H Host:test.example.com echo "Modify Response Test Passed" diff --git a/functional-tests/admin_tests/multi_module_test.sh b/functional-tests/admin_tests/multi_module_test.sh index f5071f9..8b54c5a 100755 --- a/functional-tests/admin_tests/multi_module_test.sh +++ b/functional-tests/admin_tests/multi_module_test.sh @@ -4,17 +4,18 @@ set -eo xtrace ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"} PROXY_URL=${PROXY_URL:-"http://localhost"} +TEST_URL=${TEST_URL:-"http://localhost:8888"} DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli namespace create \ +dgate-cli -Vf namespace create \ name=multimod-test-ns -dgate-cli domain create \ +dgate-cli -Vf domain create \ name=multimod-test-dm \ - patterns:='["multimod-test.com"]' \ + patterns:='["multimod-test.example.com"]' \ namespace=multimod-test-ns MOD_B64=$(base64 <<-END @@ -32,7 +33,7 @@ END ) -dgate-cli module create \ +dgate-cli -Vf module create \ name=multimod1 \ payload="$MOD_B64" \ namespace=multimod-test-ns @@ -53,15 +54,14 @@ END ) -dgate-cli module create name=multimod2 \ +dgate-cli -Vf module create name=multimod2 \ payload="$MOD_B64" namespace=multimod-test-ns -URL='http://localhost:8888' -dgate-cli service create name=base_svc \ - urls="$URL/a","$URL/b","$URL/c" \ +dgate-cli -Vf service create name=base_svc \ + urls="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \ namespace=multimod-test-ns -dgate-cli route create name=base_rt \ +dgate-cli -Vf route create name=base_rt \ paths=/,/multimod-test \ methods:='["GET"]' \ modules=multimod1,multimod2 \ @@ -71,7 +71,7 @@ dgate-cli route create name=base_rt \ namespace=multimod-test-ns -curl -s --fail-with-body ${PROXY_URL}/ -H Host:multimod-test.com -curl -s --fail-with-body ${PROXY_URL}/multimod-test -H Host:multimod-test.com +curl -sf ${PROXY_URL}/ -H Host:multimod-test.example.com +curl -sf ${PROXY_URL}/multimod-test -H Host:multimod-test.example.com echo "Multi Module Test Passed" \ No newline at end of file diff --git a/functional-tests/admin_tests/performance_test_prep.sh b/functional-tests/admin_tests/performance_test_prep.sh index a9a8f3e..fcdd5fb 100755 --- a/functional-tests/admin_tests/performance_test_prep.sh +++ b/functional-tests/admin_tests/performance_test_prep.sh @@ -4,29 +4,29 @@ set -eo xtrace ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"} PROXY_URL=${PROXY_URL:-"http://localhost"} +TEST_URL=${TEST_URL:-"http://localhost:8888"} DIR="$( cd "$( dirname "$0" )" && pwd )" - export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli -V -f namespace create \ +dgate-cli -Vf namespace create \ name=test-ns1 -dgate-cli domain create \ - name=test-dm patterns:='["dgate.dev"]' \ +dgate-cli -Vf domain create \ + name=test-dm patterns:='["performance.example.com"]' \ namespace=test-ns1 priority:=100 -dgate-cli service create \ - name=test-svc urls:='["http://localhost:8888"]' \ +dgate-cli -Vf service create \ + name=test-svc urls="$TEST_URL" \ namespace=test-ns1 retries:=3 retryTimeout=50ms MOD_B64="$(base64 < $DIR/performance_test_prep.ts)" -dgate-cli module create \ +dgate-cli -Vf module create \ name=test-mod payload="$MOD_B64" \ namespace=test-ns1 -dgate-cli route create \ +dgate-cli -Vf route create \ name=base-rt1 \ service=test-svc \ methods:='["GET"]' \ @@ -35,7 +35,7 @@ dgate-cli route create \ stripPath:=true \ namespace=test-ns1 -dgate-cli route create \ +dgate-cli -Vf route create \ name=test-rt2 \ paths:='["/modtest","/modview"]' \ methods:='["GET"]' \ @@ -44,7 +44,7 @@ dgate-cli route create \ preserveHost:=false \ namespace=test-ns1 -dgate-cli route create \ +dgate-cli -Vf route create \ name=test-rt3 \ paths:='["/blank"]' \ methods:='["GET"]' \ @@ -53,10 +53,10 @@ dgate-cli route create \ namespace=test-ns1 -curl -s --fail-with-body ${PROXY_URL}/svctest -H Host:dgate.dev +curl -sf ${PROXY_URL}/svctest -H Host:performance.example.com -curl -s --fail-with-body ${PROXY_URL}/modtest -H Host:dgate.dev +curl -sf ${PROXY_URL}/modtest -H Host:performance.example.com -curl -s ${PROXY_URL}/blank -H Host:dgate.dev +curl -s ${PROXY_URL}/blank -H Host:performance.example.com echo "Performance Test Prep Done" \ No newline at end of file diff --git a/functional-tests/admin_tests/url_shortener_test.sh b/functional-tests/admin_tests/url_shortener_test.sh index fcbf135..5e92794 100755 --- a/functional-tests/admin_tests/url_shortener_test.sh +++ b/functional-tests/admin_tests/url_shortener_test.sh @@ -9,25 +9,23 @@ DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL -dgate-cli namespace create \ +dgate-cli -Vf namespace create \ name=url_shortener-ns -dgate-cli domain create \ +dgate-cli -Vf domain create \ name=url_shortener-dm \ - patterns:='["url_shortener.com"]' \ + patterns:='["url_shortener.example.com"]' \ namespace=url_shortener-ns -dgate-cli collection create \ +dgate-cli -Vf collection create \ schema:='{"type":"object","properties":{"url":{"type":"string"}}}' \ - name=short_link \ - type=document \ - namespace=url_shortener-ns + name=short_link type=document namespace=url_shortener-ns -dgate-cli module create name=url_shortener-mod \ +dgate-cli -Vf module create name=url_shortener-mod \ payload@=$DIR/url_shortener.ts \ namespace=url_shortener-ns -dgate-cli route create \ +dgate-cli -Vf route create \ name=base_rt paths:='["/", "/{id}"]' \ modules:='["url_shortener-mod"]' \ methods:='["GET","POST"]' \ @@ -35,12 +33,12 @@ dgate-cli route create \ preserveHost:=true \ namespace=url_shortener-ns -JSON_RESP=$(curl -sG -X POST \ - -H Host:url_shortener.com ${PROXY_URL}/ \ - --data-urlencode 'url=https://dgate.io') +JSON_RESP=$(curl -fsG -X POST \ + -H Host:url_shortener.example.com ${PROXY_URL}/ \ + --data-urlencode 'url=https://dgate.io/'$(uuid)) URL_ID=$(echo $JSON_RESP | jq -r '.id') -curl -s --fail-with-body \ +curl -sf \ ${PROXY_URL}/$URL_ID \ - -H Host:url_shortener.com + -H Host:url_shortener.example.com diff --git a/functional-tests/raft_tests/raft_test.sh b/functional-tests/raft_tests/raft_test.sh index 7829b1b..ae4f3bb 100755 --- a/functional-tests/raft_tests/raft_test.sh +++ b/functional-tests/raft_tests/raft_test.sh @@ -4,6 +4,7 @@ set -eo xtrace ADMIN_URL1=${ADMIN_URL1:-"http://localhost:9081"} PROXY_URL1=${PROXY_URL1:-"http://localhost:81"} +TEST_URL1=${TEST_URL1:-"http://localhost:8081"} ADMIN_URL2=${ADMIN_URL2:-"http://localhost:9082"} PROXY_URL2=${PROXY_URL2:-"http://localhost:82"} @@ -24,46 +25,49 @@ DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL1 +if ! command -v uuid > /dev/null; then + id=X$RANDOM-$RANDOM-$RANDOM +else + id=$(uuid) +fi -id=$(uuid) - -dgate-cli -f namespace create name=ns-$id +dgate-cli -Vf namespace create name=ns-$id -dgate-cli -f domain create name=dm-$id \ +dgate-cli -Vf domain create name=dm-$id \ namespace=ns-$id priority:=$RANDOM patterns="$id.example.com" -dgate-cli -f service create \ +dgate-cli -Vf service create \ name=svc-$id namespace=ns-$id \ - urls="http://localhost:8081/$RANDOM" + urls="$TEST_URL1/$RANDOM" -dgate-cli -f route create \ +dgate-cli -Vf route create \ name=rt-$id \ service=svc-$id \ namespace=ns-$id \ - paths="/$id/{id}" \ - methods:='["GET"]' \ + paths="/,/{},/$id,/$id/{id}" \ + methods=GET,POST,PUT \ preserveHost:=false \ - stripPath:=true + stripPath:=false -curl -f $ADMIN_URL1/readyz +curl -sf $ADMIN_URL1/readyz -for i in {1..5}; do +for i in {1..1}; do for j in {1..3}; do proxy_url=PROXY_URL$i - curl -f ${!proxy_url}/$id/$j -H Host:$id.example.com + curl -sf ${!proxy_url}/$id/$RANDOM-$j -H Host:$id.example.com done done -if dgate-cli --admin $ADMIN_URL4 namespace create name=0; then - echo "Expected error when creating namespace on non-voter" - exit 1 -fi +# if dgate-cli --admin $ADMIN_URL4 namespace create name=0; then +# echo "Expected error when creating namespace on non-voter" +# exit 1 +# fi -export DGATE_ADMIN_API=$ADMIN_URL5 +# export DGATE_ADMIN_API=$ADMIN_URL5 -if dgate-cli --admin $ADMIN_URL5 namespace create name=0; then - echo "Expected error when creating namespace on non-voter" - exit 1 -fi +# if dgate-cli --admin $ADMIN_URL5 namespace create name=0; then +# echo "Expected error when creating namespace on non-voter" +# exit 1 +# fi echo "Raft Test Succeeded" diff --git a/go.mod b/go.mod index 772cf23..a9cda3c 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.22.0 require ( github.com/clarkmcc/go-typescript v0.7.0 github.com/dgate-io/chi-router v0.0.0-20231217131951-d154152d5115 - github.com/dgate-io/raft-badger v0.0.0-20231217131807-c5eb3f9eafa5 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c github.com/google/uuid v1.3.1 - github.com/hashicorp/go-hclog v1.6.2 - github.com/hashicorp/raft v1.6.0 + github.com/hashicorp/go-hclog v1.6.3 + github.com/hashicorp/raft v1.7.0 + github.com/hashicorp/raft-boltdb/v2 v2.3.0 github.com/knadh/koanf/parsers/json v0.1.0 github.com/knadh/koanf/parsers/toml v0.1.0 github.com/knadh/koanf/parsers/yaml v0.1.0 @@ -26,26 +26,30 @@ require ( github.com/stoewer/go-strcase v1.3.0 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.1 + go.etcd.io/bbolt v1.3.10 go.opentelemetry.io/otel v1.26.0 go.opentelemetry.io/otel/exporters/prometheus v0.48.0 go.opentelemetry.io/otel/metric v1.26.0 go.opentelemetry.io/otel/sdk/metric v1.26.0 go.uber.org/zap v1.27.0 golang.org/x/net v0.21.0 + golang.org/x/sync v0.7.0 golang.org/x/term v0.19.0 ) require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/dop251/base64dec v0.0.0-20231022112746-c6c9f9a96217 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -58,12 +62,14 @@ require ( github.com/google/flatbuffers v1.12.1 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/go-msgpack v1.1.5 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/raft-boltdb v0.0.0-20231211162105-6c830fa4535e // indirect github.com/klauspost/compress v1.17.0 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -80,7 +86,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sys v0.19.0 // indirect + golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 7bb3c79..1a1907c 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,21 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -33,14 +38,13 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgate-io/chi-router v0.0.0-20231217131951-d154152d5115 h1:AVEnGd1UBqJU7MnbyAtPfp47mlI5GvMS4fFNZVMS0KA= github.com/dgate-io/chi-router v0.0.0-20231217131951-d154152d5115/go.mod h1:MyLj6L03q1t8GW/541pHuP6co58QfLppSYPS0PvLtC8= -github.com/dgate-io/raft-badger v0.0.0-20231217131807-c5eb3f9eafa5 h1:clmNs28JV+F8rV6cXhByKOvQIMWLNfIvOrBD9nxkUUE= -github.com/dgate-io/raft-badger v0.0.0-20231217131807-c5eb3f9eafa5/go.mod h1:vXdEq7albbhewiVglqyJ4+gTlGhRVVYKO4jhKv0qVuk= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b h1:SeiGBzKrEtuDddnBABHkp4kq9sBGE9nuYmk6FPTg0zg= +github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= @@ -61,8 +65,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -121,21 +125,30 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= -github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack/v2 v2.1.1 h1:xQEY9yB2wnHitoSzk/B9UjXWRQ67QKu5AOm8aFp8N3I= -github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= +github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= +github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0= +github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/raft v1.6.0 h1:tkIAORZy2GbJ2Trp5eUSggLXDPOJLXC+JJLNMMqtgtM= -github.com/hashicorp/raft v1.6.0/go.mod h1:Xil5pDgeGwRWuX4uPUmwa+7Vagg4N804dz6mhNi6S7o= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.7.0 h1:4u24Qn6lQ6uwziM++UgsyiT64Q8GyRn43CV41qPiz1o= +github.com/hashicorp/raft v1.7.0/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0= +github.com/hashicorp/raft-boltdb v0.0.0-20231211162105-6c830fa4535e h1:SK4y8oR4ZMHPvwVHryKI88kJPJda4UyWYvG5A6iEQxc= +github.com/hashicorp/raft-boltdb v0.0.0-20231211162105-6c830fa4535e/go.mod h1:EMz/UIuG93P0MBeHh6CbXQAEe8ckVJLZjhD17lBzK5Q= +github.com/hashicorp/raft-boltdb/v2 v2.3.0 h1:fPpQR1iGEVYjZ2OELvUHX600VAK5qmdnDEv3eXOwZUA= +github.com/hashicorp/raft-boltdb/v2 v2.3.0/go.mod h1:YHukhB04ChJsLHLJEUD6vjFyLX2L3dsX3wPBZcX4tmc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -178,8 +191,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -204,6 +217,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= @@ -214,11 +228,13 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= @@ -259,6 +275,8 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= @@ -294,6 +312,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -314,6 +333,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -335,8 +356,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= @@ -352,6 +373,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/internal/admin/admin_api.go b/internal/admin/admin_api.go index 5b4297e..3ab6963 100644 --- a/internal/admin/admin_api.go +++ b/internal/admin/admin_api.go @@ -22,15 +22,16 @@ import ( func StartAdminAPI( version string, conf *config.DGateConfig, logger *zap.Logger, cs changestate.ChangeState, -) { +) error { if conf.AdminConfig == nil { logger.Warn("Admin API is disabled") - return + return nil } mux := chi.NewRouter() - configureRoutes(mux, version, - logger.Named("routes"), cs, conf) + if err := configureRoutes(mux, version, logger.Named("routes"), cs, conf); err != nil { + return err + } // Start HTTP Server go func() { @@ -44,6 +45,7 @@ func StartAdminAPI( ErrorLog: zap.NewStdLog(adminHttpLogger), } if err := server.ListenAndServe(); err != nil { + logger.Error("Error starting admin api", zap.Error(err)) panic(err) } }() @@ -143,4 +145,5 @@ func StartAdminAPI( }() } } + return nil } diff --git a/internal/admin/admin_fsm.go b/internal/admin/admin_fsm.go index 781cd17..daf7480 100644 --- a/internal/admin/admin_fsm.go +++ b/internal/admin/admin_fsm.go @@ -2,6 +2,7 @@ package admin import ( "encoding/json" + "errors" "io" "github.com/dgate-io/dgate/internal/admin/changestate" @@ -10,83 +11,141 @@ import ( "go.uber.org/zap" ) -type dgateAdminFSM struct { - cs changestate.ChangeState - logger *zap.Logger - index uint64 +type AdminFSM struct { + cs changestate.ChangeState + storage raft.StableStore + logger *zap.Logger + + localState *saveState } -var _ raft.BatchingFSM = (*dgateAdminFSM)(nil) +var _ raft.BatchingFSM = (*AdminFSM)(nil) -func newDGateAdminFSM(logger *zap.Logger, cs changestate.ChangeState) *dgateAdminFSM { - return &dgateAdminFSM{cs, logger, 0} +type saveState struct { + AppliedIndex uint64 `json:"aindex"` } -func (fsm *dgateAdminFSM) SetIndex(index uint64) { - fsm.index = index +func newAdminFSM( + logger *zap.Logger, + storage raft.StableStore, + cs changestate.ChangeState, +) raft.FSM { + fsm := &AdminFSM{cs, storage, logger, &saveState{}} + stateBytes, err := storage.Get([]byte("prev_state")) + if err != nil { + logger.Error("error getting prev_state", zap.Error(err)) + } else if len(stateBytes) != 0 { + if err = json.Unmarshal(stateBytes, &fsm.localState); err != nil { + logger.Warn("corrupted state detected", zap.ByteString("prev_state", stateBytes)) + } else { + logger.Info("found state in store", zap.Any("prev_state", fsm.localState)) + return fsm + } + } + return fsm } -func (fsm *dgateAdminFSM) applyLog(log *raft.Log, replay bool) (*spec.ChangeLog, error) { - log.Index = fsm.index +func (fsm *AdminFSM) applyLog(log *raft.Log, reload bool) (*spec.ChangeLog, error) { switch log.Type { case raft.LogCommand: var cl spec.ChangeLog if err := json.Unmarshal(log.Data, &cl); err != nil { fsm.logger.Error("Error unmarshalling change log", zap.Error(err)) return nil, err + } + + if cl.ID == "" { + fsm.logger.Error("Change log ID is empty") + return nil, errors.New("change log ID is empty") } else if cl.Cmd.IsNoop() { return nil, nil - } else if cl.ID == "" { - fsm.logger.Error("Change log ID is empty", zap.Error(err)) - panic("change log ID is empty") } // find a way to only reload if latest index to save time - return &cl, fsm.cs.ProcessChangeLog(&cl, replay) + return &cl, fsm.cs.ProcessChangeLog(&cl, reload) case raft.LogConfiguration: servers := raft.DecodeConfiguration(log.Data).Servers - for i, server := range servers { - fsm.logger.Debug("configuration update server", - zap.Any("address", server.Address), - zap.Int("index", i), - ) - } + fsm.logger.Debug("configuration update server", + zap.Any("address", servers), + zap.Uint64("index", log.Index), + zap.Uint64("term", log.Term), + zap.Time("appended", log.AppendedAt), + ) default: fsm.logger.Error("Unknown log type in FSM Apply") } return nil, nil } -func (fsm *dgateAdminFSM) Apply(log *raft.Log) any { - _, err := fsm.applyLog(log, true) - return err +func (fsm *AdminFSM) Apply(log *raft.Log) any { + if resps := fsm.ApplyBatch([]*raft.Log{log}); len(resps) == 1 { + return resps[0] + } + panic("apply batch not returning the correct number of responses") } -func (fsm *dgateAdminFSM) ApplyBatch(logs []*raft.Log) []any { +func (fsm *AdminFSM) ApplyBatch(logs []*raft.Log) []any { rft := fsm.cs.Raft() - lastIndex := len(logs) - 1 + appliedIndex := rft.AppliedIndex() + lastLogIndex := logs[len(logs)-1].Index fsm.logger.Debug("apply log batch", - zap.Uint64("applied", rft.AppliedIndex()), + zap.Uint64("applied", appliedIndex), zap.Uint64("commit", rft.CommitIndex()), zap.Uint64("last", rft.LastIndex()), - zap.Uint64("fsmLastIndex", fsm.index), zap.Uint64("log[0]", logs[0].Index), - zap.Uint64("log[-1]", logs[lastIndex].Index), + zap.Uint64("log[-1]", lastLogIndex), zap.Int("logs", len(logs)), ) + + var err error results := make([]any, len(logs)) + for i, log := range logs { - // TODO: check to see if this can be optimized channels raft node provides - _, results[i] = fsm.applyLog( - log, lastIndex == i, - ) + isLast := len(logs)-1 == i + reload := fsm.shouldReload(log, isLast) + if _, err = fsm.applyLog(log, reload); err != nil { + fsm.logger.Error("Error applying log", zap.Error(err)) + results[i] = err + } } + + if appliedIndex != 0 && lastLogIndex >= appliedIndex { + fsm.localState.AppliedIndex = lastLogIndex + if err = fsm.saveFSMState(); err != nil { + fsm.logger.Warn("failed to save applied index state", + zap.Uint64("applied_index", lastLogIndex), + ) + } + // defer fsm.cs.SetReady(true) + } + return results } -func (fsm *dgateAdminFSM) Snapshot() (raft.FSMSnapshot, error) { - panic("snapshots not supported") +func (fsm *AdminFSM) saveFSMState() error { + fsm.logger.Debug("saving localState", + zap.Any("data", fsm.localState), + ) + stateBytes, err := json.Marshal(fsm.localState) + if err != nil { + return err + } + return fsm.storage.Set([]byte("prev_state"), stateBytes) +} + +func (fsm *AdminFSM) shouldReload(log *raft.Log, reload bool) bool { + if reload { + return log.Index >= fsm.localState.AppliedIndex + } + return false +} + +func (fsm *AdminFSM) Snapshot() (raft.FSMSnapshot, error) { + fsm.cs = nil + fsm.logger.Warn("snapshots not supported") + return nil, errors.New("snapshots not supported") } -func (fsm *dgateAdminFSM) Restore(rc io.ReadCloser) error { - panic("snapshots not supported") +func (fsm *AdminFSM) Restore(rc io.ReadCloser) error { + fsm.logger.Warn("snapshots not supported, cannot restore") + return nil } diff --git a/internal/admin/admin_raft.go b/internal/admin/admin_raft.go index f9d126f..8c90bf3 100644 --- a/internal/admin/admin_raft.go +++ b/internal/admin/admin_raft.go @@ -3,6 +3,7 @@ package admin import ( "context" "fmt" + "math" "net" "net/http" "path" @@ -13,12 +14,11 @@ import ( "github.com/dgate-io/dgate/internal/config" "github.com/dgate-io/dgate/pkg/raftadmin" "github.com/dgate-io/dgate/pkg/rafthttp" - "github.com/dgate-io/dgate/pkg/spec" "github.com/dgate-io/dgate/pkg/storage" + "github.com/dgate-io/dgate/pkg/util" "github.com/dgate-io/dgate/pkg/util/logadapter" - raftbadgerdb "github.com/dgate-io/raft-badger" - "github.com/dgraph-io/badger/v4" "github.com/hashicorp/raft" + boltdb "github.com/hashicorp/raft-boltdb/v2" "go.uber.org/zap" ) @@ -29,31 +29,44 @@ func setupRaft( cs changestate.ChangeState, ) { adminConfig := conf.AdminConfig - var sstore raft.StableStore - var lstore raft.LogStore + var logStore raft.LogStore + var configStore raft.StableStore + var snapStore raft.SnapshotStore switch conf.Storage.StorageType { case config.StorageTypeMemory: - sstore = raft.NewInmemStore() - lstore = raft.NewInmemStore() + logStore = raft.NewInmemStore() + configStore = raft.NewInmemStore() case config.StorageTypeFile: fileConfig, err := config.StoreConfig[storage.FileStoreConfig](conf.Storage.Config) if err != nil { panic(fmt.Errorf("invalid config: %s", err)) } - badgerLogger := logadapter.NewZap2BadgerAdapter(logger.Named("badger-file")) - raftDir := path.Join(fileConfig.Directory, "raft") - badgerStore, err := raftbadgerdb.New( - badger.DefaultOptions(raftDir). - WithLogger(badgerLogger), + raftDir := path.Join(fileConfig.Directory) + + snapStore, err = raft.NewFileSnapshotStore( + path.Join(raftDir), 5, + zap.NewStdLog(logger.Named("snap-file")).Writer(), ) if err != nil { panic(err) } - sstore = badgerStore - lstore = badgerStore + if boltStore, err := boltdb.NewBoltStore( + path.Join(raftDir, "raft.db"), + ); err != nil { + panic(err) + } else { + configStore = boltStore + logStore = boltStore + } default: panic(fmt.Errorf("invalid storage type: %s", conf.Storage.StorageType)) } + + logger.Info("raft store", + zap.Stringer("storage_type", conf.Storage.StorageType), + zap.Any("storage_config", conf.Storage.Config), + ) + raftConfig := adminConfig.Replication.LoadRaftConfig( &raft.Config{ ProtocolVersion: raft.ProtocolVersionMax, @@ -62,11 +75,12 @@ func setupRaft( ElectionTimeout: time.Second * 5, CommitTimeout: time.Second * 4, BatchApplyCh: false, - MaxAppendEntries: 512, + MaxAppendEntries: 1024, LeaderLeaseTimeout: time.Second * 4, + // TODO: Support snapshots - SnapshotInterval: time.Hour*2 ^ 32, - SnapshotThreshold: ^uint64(0), + SnapshotInterval: time.Duration(9999 * time.Hour), + SnapshotThreshold: math.MaxUint64, Logger: logadapter.NewZap2HCLogAdapter(logger), }, ) @@ -78,38 +92,30 @@ func setupRaft( address := raft.ServerAddress(advertAddr) raftHttpLogger := logger.Named("http") - if adminConfig.Replication.AdvertScheme != "http" && adminConfig.Replication.AdvertScheme != "https" { - panic(fmt.Errorf("invalid scheme: %s", adminConfig.Replication.AdvertScheme)) - } - transport := rafthttp.NewHTTPTransport( address, http.DefaultClient, raftHttpLogger, - adminConfig.Replication.AdvertScheme+"://(address)/raft", + adminConfig.Replication.AdvertScheme, ) fsmLogger := logger.Named("fsm") - snapstore := raft.NewInmemSnapshotStore() - fsm := newDGateAdminFSM(fsmLogger, cs) + adminFSM := newAdminFSM(fsmLogger, configStore, cs) raftNode, err := raft.NewRaft( - raftConfig, fsm, lstore, - sstore, snapstore, transport, + raftConfig, adminFSM, logStore, + configStore, snapStore, transport, ) if err != nil { panic(err) } - observerChan := make(chan raft.Observation, 10) - raftNode.RegisterObserver(raft.NewObserver(observerChan, false, nil)) - cs.SetupRaft(raftNode, observerChan) - // Setup raft handler server.Handle("/raft/*", transport) raftAdminLogger := logger.Named("admin") - raftAdmin := raftadmin.NewRaftAdminHTTPServer( - raftNode, raftAdminLogger, []raft.ServerAddress{address}, + raftAdmin := raftadmin.NewServer( + raftNode, raftAdminLogger, + []raft.ServerAddress{address}, ) - // Setup handler raft + // Setup handler for raft admin server.HandleFunc("/raftadmin/*", func(w http.ResponseWriter, r *http.Request) { if adminConfig.Replication.SharedKey != "" { sharedKey := r.Header.Get("X-DGate-Shared-Key") @@ -121,6 +127,45 @@ func setupRaft( raftAdmin.ServeHTTP(w, r) }) + // Setup handler for stats + server.Handle("/raftadmin/stats", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Raft-State", raftNode.State().String()) + util.JsonResponse(w, http.StatusOK, raftNode.Stats()) + })) + + // Setup handler for readys + server.Handle("/raftadmin/readyz", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Raft-State", raftNode.State().String()) + if err := cs.WaitForChanges(nil); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + leaderId, leaderAddr := raftNode.LeaderWithID() + util.JsonResponse(w, http.StatusOK, map[string]any{ + "status": "ok", + "proxy_ready": cs.Ready(), + "state": raftNode.State().String(), + "leader": leaderId, + "leader_addr": leaderAddr, + }) + })) + + doer := func(req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", "dgate") + if adminConfig.Replication.SharedKey != "" { + req.Header.Set("X-DGate-Shared-Key", adminConfig.Replication.SharedKey) + } + client := *http.DefaultClient + client.Timeout = time.Second * 10 + return client.Do(req) + } + adminClient := raftadmin.NewClient( + doer, logger.Named("raft-admin-client"), + adminConfig.Replication.AdvertScheme, + ) + + cs.SetupRaft(raftNode, adminClient) + configFuture := raftNode.GetConfiguration() if err = configFuture.Error(); err != nil { panic(err) @@ -136,8 +181,6 @@ func setupRaft( zap.Int("config_proto", int(raftConfig.ProtocolVersion)), ) - defer cs.ProcessChangeLog(spec.NewNoopChangeLog(), false) - if adminConfig.Replication.BootstrapCluster && len(serverConfig.Servers) == 0 { logger.Info("bootstrapping cluster", zap.String("id", raftId), @@ -188,33 +231,25 @@ func setupRaft( if len(addresses) > 0 { addresses = append(addresses, adminConfig.Replication.ClusterAddrs...) retries := 0 - doer := func(req *http.Request) (*http.Response, error) { - req.Header.Set("User-Agent", "dgate") - if adminConfig.Replication.SharedKey != "" { - req.Header.Set("X-DGate-Shared-Key", adminConfig.Replication.SharedKey) - } - return http.DefaultClient.Do(req) - } - adminClient := raftadmin.NewHTTPAdminClient(doer, - adminConfig.Replication.AdvertScheme+"://(address)/raftadmin", - logger.Named("raft-admin-client"), - ) RETRY: - for _, url := range addresses { - err = adminClient.VerifyLeader(context.Background(), raft.ServerAddress(url)) + for _, addr := range addresses { + err = adminClient.VerifyLeader( + context.Background(), + raft.ServerAddress(addr), + ) if err != nil { if err == raftadmin.ErrNotLeader { continue } if retries > 15 { logger.Error("Skipping verifying leader", - zap.String("url", url), zap.Error(err), + zap.String("url", addr), zap.Error(err), ) continue } retries += 1 logger.Debug("Retrying verifying leader", - zap.String("url", url), zap.Error(err)) + zap.String("url", addr), zap.Error(err)) <-time.After(3 * time.Second) goto RETRY } @@ -223,10 +258,10 @@ func setupRaft( logger.Info("Adding non-voter", zap.String("id", raftId), zap.String("leader", adminConfig.Replication.AdvertAddr), - zap.String("url", url), + zap.String("url", addr), ) resp, err := adminClient.AddNonvoter( - context.Background(), raft.ServerAddress(url), + context.Background(), raft.ServerAddress(addr), &raftadmin.AddNonvoterRequest{ ID: raftId, Address: adminConfig.Replication.AdvertAddr, @@ -242,9 +277,9 @@ func setupRaft( logger.Info("Adding voter: %s - leader: %s", zap.String("id", raftId), zap.String("leader", adminConfig.Replication.AdvertAddr), - zap.String("url", url), + zap.String("url", addr), ) - resp, err := adminClient.AddVoter(context.Background(), raft.ServerAddress(url), &raftadmin.AddVoterRequest{ + resp, err := adminClient.AddVoter(context.Background(), raft.ServerAddress(addr), &raftadmin.AddVoterRequest{ ID: raftId, Address: adminConfig.Replication.AdvertAddr, }) diff --git a/internal/admin/admin_routes.go b/internal/admin/admin_routes.go index 4539e60..e5f966d 100644 --- a/internal/admin/admin_routes.go +++ b/internal/admin/admin_routes.go @@ -1,6 +1,7 @@ package admin import ( + "errors" "fmt" "log" "net/http" @@ -28,50 +29,51 @@ func configureRoutes( logger *zap.Logger, cs changestate.ChangeState, conf *config.DGateConfig, -) { +) error { adminConfig := conf.AdminConfig - server.Use(func(next http.Handler) http.Handler { - ipList := iplist.NewIPList() - for _, address := range adminConfig.AllowList { - if strings.Contains(address, "/") { - if err := ipList.AddCIDRString(address); err != nil { - panic(fmt.Sprintf("invalid cidr address in admin.allow_list: %s", address)) - } - } else { - if err := ipList.AddIPString(address); err != nil { - panic(fmt.Sprintf("invalid ip address in admin.allow_list: %s", address)) - } + ipList := iplist.NewIPList() + for _, address := range adminConfig.AllowList { + if strings.Contains(address, "/") { + if err := ipList.AddCIDRString(address); err != nil { + return fmt.Errorf("invalid cidr address in admin.allow_list: %s", address) + } + } else { + if err := ipList.AddIPString(address); err != nil { + return fmt.Errorf("invalid ip address in admin.allow_list: %s", address) } } - // basic auth - var userMap map[string]string - // key auth - var keyMap map[string]struct{} + } + // basic auth + var userMap map[string]string + // key auth + var keyMap map[string]struct{} - switch adminConfig.AuthMethod { - case config.AuthMethodBasicAuth: - userMap = make(map[string]string) - if len(adminConfig.BasicAuth.Users) > 0 { - for i, user := range adminConfig.BasicAuth.Users { - if user.Username == "" || user.Password == "" { - panic(fmt.Sprintf("both username and password are required: admin.basic_auth.users[%d]", i)) - } - userMap[user.Username] = user.Password + switch adminConfig.AuthMethod { + case config.AuthMethodBasicAuth: + userMap = make(map[string]string) + if len(adminConfig.BasicAuth.Users) > 0 { + for i, user := range adminConfig.BasicAuth.Users { + if user.Username == "" || user.Password == "" { + return errors.New(fmt.Sprintf("both username and password are required: admin.basic_auth.users[%d]", i)) } + userMap[user.Username] = user.Password } - case config.AuthMethodKeyAuth: - keyMap = make(map[string]struct{}) - if adminConfig.KeyAuth != nil && len(adminConfig.KeyAuth.Keys) > 0 { - if adminConfig.KeyAuth.QueryParamName != "" && adminConfig.KeyAuth.HeaderName != "" { - panic("only one of admin.key_auth.query_param_name or admin.key_auth.header_name can be set") - } - for _, key := range adminConfig.KeyAuth.Keys { - keyMap[key] = struct{}{} - } + } + case config.AuthMethodKeyAuth: + keyMap = make(map[string]struct{}) + if adminConfig.KeyAuth != nil && len(adminConfig.KeyAuth.Keys) > 0 { + if adminConfig.KeyAuth.QueryParamName != "" && adminConfig.KeyAuth.HeaderName != "" { + return errors.New("only one of admin.key_auth.query_param_name or admin.key_auth.header_name can be set") + } + for _, key := range adminConfig.KeyAuth.Keys { + keyMap[key] = struct{}{} } - case config.AuthMethodJWTAuth: - panic("JWT Auth is not supported yet") } + case config.AuthMethodJWTAuth: + return errors.New("JWT Auth is not supported yet") + } + + server.Use(func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if ipList.Len() > 0 { remoteIp := util.GetTrustedIP(r, @@ -178,6 +180,7 @@ func configureRoutes( misc.Handle("/metrics", promhttp.Handler()) } }) + return nil } func setupMetricProvider( diff --git a/internal/admin/admin_routes_test.go b/internal/admin/admin_routes_test.go new file mode 100644 index 0000000..272f997 --- /dev/null +++ b/internal/admin/admin_routes_test.go @@ -0,0 +1,26 @@ +package admin + +import ( + "testing" + + "github.com/dgate-io/chi-router" + "github.com/dgate-io/dgate/internal/admin/changestate/testutil" + "github.com/dgate-io/dgate/internal/config/configtest" + "github.com/dgate-io/dgate/pkg/resources" + "go.uber.org/zap" +) + +func TestAdminRoutes_configureRoutes(t *testing.T) { + mux := chi.NewMux() + cs := testutil.NewMockChangeState() + cs.On("ResourceManager").Return(resources.NewManager()) + cs.On("DocumentManager").Return(nil) + conf := configtest.NewTestAdminConfig() + if err := configureRoutes( + mux, "test", + zap.NewNop(), + cs, conf, + ); err != nil { + t.Fatal(err) + } +} diff --git a/internal/admin/changestate/change_state.go b/internal/admin/changestate/change_state.go index 882288d..2e95c52 100644 --- a/internal/admin/changestate/change_state.go +++ b/internal/admin/changestate/change_state.go @@ -2,6 +2,7 @@ package changestate import ( "github.com/dgate-io/dgate/internal/proxy" + "github.com/dgate-io/dgate/pkg/raftadmin" "github.com/dgate-io/dgate/pkg/resources" "github.com/dgate-io/dgate/pkg/spec" "github.com/hashicorp/raft" @@ -10,17 +11,18 @@ import ( type ChangeState interface { // Change state ApplyChangeLog(cl *spec.ChangeLog) error - ProcessChangeLog(*spec.ChangeLog, bool) error - WaitForChanges() error + ProcessChangeLog(cl *spec.ChangeLog, reload bool) error + WaitForChanges(cl *spec.ChangeLog) error ReloadState(bool, ...*spec.ChangeLog) error - ChangeHash() uint32 + ChangeHash() uint64 ChangeLogs() []*spec.ChangeLog // Readiness Ready() bool + SetReady(bool) // Replication - SetupRaft(*raft.Raft, chan raft.Observation) + SetupRaft(*raft.Raft, *raftadmin.Client) Raft() *raft.Raft // Resources diff --git a/internal/admin/changestate/testutil/change_state.go b/internal/admin/changestate/testutil/change_state.go index d199958..f0e5a82 100644 --- a/internal/admin/changestate/testutil/change_state.go +++ b/internal/admin/changestate/testutil/change_state.go @@ -7,9 +7,9 @@ import ( "github.com/dgate-io/dgate/internal/admin/changestate" "github.com/dgate-io/dgate/pkg/resources" "github.com/dgate-io/dgate/pkg/spec" + "github.com/dgate-io/dgate/pkg/raftadmin" "github.com/hashicorp/raft" "github.com/stretchr/testify/mock" - "go.uber.org/zap" ) type MockChangeState struct { @@ -22,25 +22,26 @@ func (m *MockChangeState) ApplyChangeLog(cl *spec.ChangeLog) error { } // ChangeHash implements changestate.ChangeState. -func (m *MockChangeState) ChangeHash() uint32 { - return m.Called().Get(0).(uint32) +func (m *MockChangeState) ChangeHash() uint64 { + return m.Called().Get(0).(uint64) } // DocumentManager implements changestate.ChangeState. func (m *MockChangeState) DocumentManager() resources.DocumentManager { + if m.Called().Get(0) == nil { + return nil + } return m.Called().Get(0).(resources.DocumentManager) } // ResourceManager implements changestate.ChangeState. func (m *MockChangeState) ResourceManager() *resources.ResourceManager { + if m.Called().Get(0) == nil { + return nil + } return m.Called().Get(0).(*resources.ResourceManager) } -// Logger implements changestate.ChangeState. -func (m *MockChangeState) Logger() *zap.Logger { - return m.Called().Get(0).(*zap.Logger) -} - // ProcessChangeLog implements changestate.ChangeState. func (m *MockChangeState) ProcessChangeLog(cl *spec.ChangeLog, a bool) error { return m.Called(cl, a).Error(0) @@ -60,13 +61,18 @@ func (m *MockChangeState) Ready() bool { return m.Called().Get(0).(bool) } +// SetReady implements changestate.ChangeState. +func (m *MockChangeState) SetReady(ready bool) { + m.Called(ready) +} + // ReloadState implements changestate.ChangeState. func (m *MockChangeState) ReloadState(a bool, cls ...*spec.ChangeLog) error { return m.Called(a, cls).Error(0) } // SetupRaft implements changestate.ChangeState. -func (m *MockChangeState) SetupRaft(*raft.Raft, chan raft.Observation) { +func (m *MockChangeState) SetupRaft(*raft.Raft, *raftadmin.Client) { m.Called().Error(0) } @@ -76,8 +82,8 @@ func (m *MockChangeState) Version() string { } // WaitForChanges implements changestate.ChangeState. -func (m *MockChangeState) WaitForChanges() error { - return m.Called().Error(0) +func (m *MockChangeState) WaitForChanges(cl *spec.ChangeLog) error { + return m.Called(cl).Error(0) } // ChangeLogs implements changestate.ChangeState. diff --git a/internal/admin/routes/collection_routes.go b/internal/admin/routes/collection_routes.go index 1a34c31..52340f8 100644 --- a/internal/admin/routes/collection_routes.go +++ b/internal/admin/routes/collection_routes.go @@ -63,13 +63,12 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat } cl := spec.NewChangeLog(&collection, collection.NamespaceName, spec.AddCollectionCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } @@ -268,13 +267,12 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat } cl := spec.NewChangeLog(&doc, doc.NamespaceName, spec.AddDocumentCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } @@ -350,8 +348,7 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat return } cl := spec.NewChangeLog(document, namespaceName, spec.DeleteDocumentCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } diff --git a/internal/admin/routes/domain_routes.go b/internal/admin/routes/domain_routes.go index 58ca807..096fd77 100644 --- a/internal/admin/routes/domain_routes.go +++ b/internal/admin/routes/domain_routes.go @@ -41,13 +41,12 @@ func ConfigureDomainAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch domain.NamespaceName = spec.DefaultNamespace.Name } cl := spec.NewChangeLog(&domain, domain.NamespaceName, spec.AddDomainCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return diff --git a/internal/admin/routes/misc_routes.go b/internal/admin/routes/misc_routes.go index 636ffaa..4f169de 100644 --- a/internal/admin/routes/misc_routes.go +++ b/internal/admin/routes/misc_routes.go @@ -3,6 +3,7 @@ package routes import ( "encoding/json" "net/http" + "strconv" "github.com/dgate-io/chi-router" "github.com/dgate-io/dgate/internal/admin/changestate" @@ -11,35 +12,23 @@ import ( ) func ConfigureChangeLogAPI(server chi.Router, cs changestate.ChangeState, appConfig *config.DGateConfig) { - server.Get("/changelog/hash", func(w http.ResponseWriter, r *http.Request) { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return + server.Get("/changelog", func(w http.ResponseWriter, r *http.Request) { + hash := cs.ChangeHash() + logs := cs.ChangeLogs() + lastLogId := "" + if len(logs) > 0 { + lastLogId = logs[len(logs)-1].ID } - - if b, err := json.Marshal(map[string]any{ - "hash": cs.ChangeHash(), - }); err != nil { + b, err := json.Marshal(map[string]any{ + "count": len(logs), + "hash": strconv.FormatUint(hash, 36), + "latest": lastLogId, + }) + if err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) - } else { - w.Header().Set("Content-Type", "application/json") - w.Write([]byte(b)) - } - }) - server.Get("/changelog/count", func(w http.ResponseWriter, r *http.Request) { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } - - if b, err := json.Marshal(map[string]any{ - "count": len(cs.ChangeLogs()), - }); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - } else { - w.Header().Set("Content-Type", "application/json") - w.Write([]byte(b)) } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(b)) }) } @@ -54,18 +43,21 @@ func ConfigureHealthAPI(server chi.Router, version string, cs changestate.Change server.Get("/readyz", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - if r := cs.Raft(); r != nil { - w.Header().Set("X-Raft-State", r.State().String()) - if r.Leader() == "" { - w.WriteHeader(http.StatusServiceUnavailable) - w.Write([]byte(`{"status":"no leader"}`)) - return - } else if !cs.Ready() { - w.WriteHeader(http.StatusServiceUnavailable) - w.Write([]byte(`{"status":"not ready"}`)) - return + if cs.Ready() { + if r := cs.Raft(); r != nil { + w.Header().Set("X-Raft-State", r.State().String()) + if leaderAddr := r.Leader(); leaderAddr == "" { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"status":"no leader"}`)) + return + } else { + w.Header().Set("X-Raft-Leader", string(leaderAddr)) + } } + w.Write(healthlyResp) + } else { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"status":"not ready"}`)) } - w.Write(healthlyResp) }) } diff --git a/internal/admin/routes/module_routes.go b/internal/admin/routes/module_routes.go index 5aea17a..027da3d 100644 --- a/internal/admin/routes/module_routes.go +++ b/internal/admin/routes/module_routes.go @@ -48,7 +48,7 @@ func ConfigureModuleAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch return } - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } @@ -78,8 +78,7 @@ func ConfigureModuleAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch mod.NamespaceName = spec.DefaultNamespace.Name } cl := spec.NewChangeLog(&mod, mod.NamespaceName, spec.DeleteModuleCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } diff --git a/internal/admin/routes/module_routes_test.go b/internal/admin/routes/module_routes_test.go index 74db074..cfabb25 100644 --- a/internal/admin/routes/module_routes_test.go +++ b/internal/admin/routes/module_routes_test.go @@ -21,20 +21,21 @@ import ( func TestAdminRoutes_Module(t *testing.T) { namespaces := []string{"default", "test"} - for _, ns := range namespaces { - config := configtest.NewTest4DGateConfig() - ps := proxy.NewProxyState(zap.NewNop(), config) - mux := chi.NewMux() - mux.Route("/api/v1", func(r chi.Router) { - routes.ConfigureModuleAPI(r, zap.NewNop(), ps, config) - }) - server := httptest.NewServer(mux) - defer server.Close() + config := configtest.NewTest4DGateConfig() + ps := proxy.NewProxyState(zap.NewNop(), config) + if err := ps.Start(); err != nil { + t.Fatal(err) + } + mux := chi.NewMux() + mux.Route("/api/v1", func(r chi.Router) { + routes.ConfigureModuleAPI(r, zap.NewNop(), ps, config) + }) + server := httptest.NewServer(mux) + defer server.Close() + for _, ns := range namespaces { client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } @@ -77,25 +78,22 @@ func TestAdminRoutes_Module(t *testing.T) { } func TestAdminRoutes_ModuleError(t *testing.T) { + config := configtest.NewTest3DGateConfig() + cs := testutil.NewMockChangeState() + rm := resources.NewManager() + cs.On("ApplyChangeLog", mock.Anything). + Return(errors.New("test error")) + cs.On("ResourceManager").Return(rm) + mux := chi.NewMux() + mux.Route("/api/v1", func(r chi.Router) { + routes.ConfigureModuleAPI(r, zap.NewNop(), cs, config) + }) + server := httptest.NewServer(mux) + defer server.Close() namespaces := []string{"default", "test", ""} for _, ns := range namespaces { - config := configtest.NewTest3DGateConfig() - rm := resources.NewManager() - cs := testutil.NewMockChangeState() - cs.On("ApplyChangeLog", mock.Anything). - Return(errors.New("test error")) - cs.On("ResourceManager").Return(rm) - mux := chi.NewMux() - mux.Route("/api/v1", func(r chi.Router) { - routes.ConfigureModuleAPI(r, zap.NewNop(), cs, config) - }) - server := httptest.NewServer(mux) - defer server.Close() - client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } diff --git a/internal/admin/routes/namespace_routes.go b/internal/admin/routes/namespace_routes.go index 2cadce7..f6b175c 100644 --- a/internal/admin/routes/namespace_routes.go +++ b/internal/admin/routes/namespace_routes.go @@ -35,13 +35,12 @@ func ConfigureNamespaceAPI(server chi.Router, logger *zap.Logger, cs changestate } cl := spec.NewChangeLog(&namespace, namespace.Name, spec.AddNamespaceCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } @@ -69,8 +68,7 @@ func ConfigureNamespaceAPI(server chi.Router, logger *zap.Logger, cs changestate } cl := spec.NewChangeLog(&namespace, namespace.Name, spec.DeleteNamespaceCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } diff --git a/internal/admin/routes/namespace_routes_test.go b/internal/admin/routes/namespace_routes_test.go index 00cc227..a03ae77 100644 --- a/internal/admin/routes/namespace_routes_test.go +++ b/internal/admin/routes/namespace_routes_test.go @@ -19,21 +19,22 @@ import ( ) func TestAdminRoutes_Namespace(t *testing.T) { + config := configtest.NewTest3DGateConfig() + ps := proxy.NewProxyState(zap.NewNop(), config) + if err := ps.Start(); err != nil { + t.Fatal(err) + } + mux := chi.NewMux() + mux.Route("/api/v1", func(r chi.Router) { + routes.ConfigureNamespaceAPI(r, zap.NewNop(), ps, config) + }) + server := httptest.NewServer(mux) + defer server.Close() namespaces := []string{"_test", "default"} for _, ns := range namespaces { - config := configtest.NewTest3DGateConfig() - ps := proxy.NewProxyState(zap.NewNop(), config) - mux := chi.NewMux() - mux.Route("/api/v1", func(r chi.Router) { - routes.ConfigureNamespaceAPI(r, zap.NewNop(), ps, config) - }) - server := httptest.NewServer(mux) - defer server.Close() client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } @@ -71,25 +72,22 @@ func TestAdminRoutes_Namespace(t *testing.T) { } func TestAdminRoutes_NamespaceError(t *testing.T) { + config := configtest.NewTest3DGateConfig() + rm := resources.NewManager() + cs := testutil.NewMockChangeState() + cs.On("ApplyChangeLog", mock.Anything). + Return(errors.New("test error")) + cs.On("ResourceManager").Return(rm) + mux := chi.NewMux() + mux.Route("/api/v1", func(r chi.Router) { + routes.ConfigureNamespaceAPI(r, zap.NewNop(), cs, config) + }) + server := httptest.NewServer(mux) + defer server.Close() namespaces := []string{"default", "test", ""} for _, ns := range namespaces { - config := configtest.NewTest3DGateConfig() - rm := resources.NewManager() - cs := testutil.NewMockChangeState() - cs.On("ApplyChangeLog", mock.Anything). - Return(errors.New("test error")) - cs.On("ResourceManager").Return(rm) - mux := chi.NewMux() - mux.Route("/api/v1", func(r chi.Router) { - routes.ConfigureNamespaceAPI(r, zap.NewNop(), cs, config) - }) - server := httptest.NewServer(mux) - defer server.Close() - client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } diff --git a/internal/admin/routes/route_routes.go b/internal/admin/routes/route_routes.go index cf88c1c..23f064e 100644 --- a/internal/admin/routes/route_routes.go +++ b/internal/admin/routes/route_routes.go @@ -43,13 +43,12 @@ func ConfigureRouteAPI(server chi.Router, logger *zap.Logger, cs changestate.Cha } cl := spec.NewChangeLog(&route, route.NamespaceName, spec.AddRouteCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } @@ -86,8 +85,7 @@ func ConfigureRouteAPI(server chi.Router, logger *zap.Logger, cs changestate.Cha } cl := spec.NewChangeLog(&route, route.NamespaceName, spec.DeleteRouteCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } diff --git a/internal/admin/routes/route_routes_test.go b/internal/admin/routes/route_routes_test.go index e8efc07..42956bf 100644 --- a/internal/admin/routes/route_routes_test.go +++ b/internal/admin/routes/route_routes_test.go @@ -23,6 +23,9 @@ func TestAdminRoutes_Route(t *testing.T) { for _, ns := range namespaces { config := configtest.NewTest3DGateConfig() ps := proxy.NewProxyState(zap.NewNop(), config) + if err := ps.Start(); err != nil { + t.Fatal(err) + } mux := chi.NewMux() mux.Route("/api/v1", func(r chi.Router) { routes.ConfigureRouteAPI(r, zap.NewNop(), ps, config) @@ -31,9 +34,7 @@ func TestAdminRoutes_Route(t *testing.T) { defer server.Close() client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } @@ -90,9 +91,7 @@ func TestAdminRoutes_RouteError(t *testing.T) { defer server.Close() client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } diff --git a/internal/admin/routes/secret_routes.go b/internal/admin/routes/secret_routes.go index 1d555c2..fb2feb9 100644 --- a/internal/admin/routes/secret_routes.go +++ b/internal/admin/routes/secret_routes.go @@ -47,7 +47,7 @@ func ConfigureSecretAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch util.JsonError(w, http.StatusBadRequest, err.Error()) return } - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } diff --git a/internal/admin/routes/service_routes.go b/internal/admin/routes/service_routes.go index 4ea587f..fb2c631 100644 --- a/internal/admin/routes/service_routes.go +++ b/internal/admin/routes/service_routes.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + urllib "net/url" "github.com/dgate-io/chi-router" "github.com/dgate-io/dgate/internal/admin/changestate" @@ -50,6 +51,26 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C util.JsonError(w, http.StatusBadRequest, "retry timeout must be greater than 0") return } + if len(svc.URLs) == 0 { + util.JsonError(w, http.StatusBadRequest, "urls are required") + return + } else { + for i, url := range svc.URLs { + errPrefix := fmt.Sprintf(`error on urls["%d"]: `, i) + if url, err := urllib.Parse(url); err != nil { + util.JsonError(w, http.StatusBadRequest, errPrefix+err.Error()) + } else { + if url.Scheme == "" { + util.JsonError(w, http.StatusBadRequest, errPrefix+"url scheme cannot be empty") + return + } + if url.Host == "" { + util.JsonError(w, http.StatusBadRequest, errPrefix+"url host cannot be empty") + return + } + } + } + } if svc.NamespaceName == "" { if appConfig.DisableDefaultNamespace { util.JsonError(w, http.StatusBadRequest, "namespace is required") @@ -57,15 +78,14 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C } svc.NamespaceName = spec.DefaultNamespace.Name } + cl := spec.NewChangeLog(&svc, svc.NamespaceName, spec.AddServiceCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } - logger.Debug("Waiting for raft barrier") - if err := cs.WaitForChanges(); err != nil { + if err := cs.WaitForChanges(cl); err != nil { util.JsonError(w, http.StatusInternalServerError, err.Error()) return } @@ -96,8 +116,7 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C svc.NamespaceName = spec.DefaultNamespace.Name } cl := spec.NewChangeLog(&svc, svc.NamespaceName, spec.DeleteServiceCommand) - err = cs.ApplyChangeLog(cl) - if err != nil { + if err = cs.ApplyChangeLog(cl); err != nil { util.JsonError(w, http.StatusBadRequest, err.Error()) return } diff --git a/internal/admin/routes/service_routes_test.go b/internal/admin/routes/service_routes_test.go index 555959f..81be839 100644 --- a/internal/admin/routes/service_routes_test.go +++ b/internal/admin/routes/service_routes_test.go @@ -23,6 +23,9 @@ func TestAdminRoutes_Service(t *testing.T) { for _, ns := range namespaces { config := configtest.NewTest4DGateConfig() ps := proxy.NewProxyState(zap.NewNop(), config) + if err := ps.Start(); err != nil { + t.Fatal(err) + } mux := chi.NewMux() mux.Route("/api/v1", func(r chi.Router) { routes.ConfigureServiceAPI(r, zap.NewNop(), ps, config) @@ -31,9 +34,7 @@ func TestAdminRoutes_Service(t *testing.T) { defer server.Close() client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } @@ -90,9 +91,7 @@ func TestAdminRoutes_ServiceError(t *testing.T) { defer server.Close() client := dgclient.NewDGateClient() - if err := client.Init(server.URL, server.Client(), - dgclient.WithVerboseLogging(true), - ); err != nil { + if err := client.Init(server.URL, server.Client()); err != nil { t.Fatal(err) } diff --git a/internal/config/config.go b/internal/config/config.go index cf1c0c5..b95e427 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,7 +44,7 @@ type ( TLS *DGateTLSConfig `koanf:"tls"` EnableH2C bool `koanf:"enable_h2c"` EnableHTTP2 bool `koanf:"enable_http2"` - EnableConsoleLogger bool `koanf:"enable_console_logger"` + ConsoleLogLevel string `koanf:"console_log_level"` RedirectHttpsDomains []string `koanf:"redirect_https"` AllowedDomains []string `koanf:"allowed_domains"` GlobalHeaders map[string]string `koanf:"global_headers"` @@ -234,6 +234,7 @@ func (conf *DGateConfig) GetLogger() (*zap.Logger, error) { config.Development = conf.Debug config.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder config.OutputPaths = []string{"stdout"} + config.Sampling = nil if config.EncoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder; conf.LogColor { config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder diff --git a/internal/config/configtest/dgate_configs.go b/internal/config/configtest/dgate_configs.go index 36956f3..a45e8e6 100644 --- a/internal/config/configtest/dgate_configs.go +++ b/internal/config/configtest/dgate_configs.go @@ -18,12 +18,12 @@ func NewTestDGateConfig() *config.DGateConfig { Version: "v1", Tags: []string{"test"}, Storage: config.DGateStorageConfig{ - StorageType: config.StorageTypeDebug, + StorageType: config.StorageTypeMemory, }, ProxyConfig: config.DGateProxyConfig{ AllowedDomains: []string{"*test.com", "localhost"}, Host: "localhost", - Port: 8080, + Port: 0, InitResources: &config.DGateResources{ Namespaces: []spec.Namespace{ { @@ -69,7 +69,7 @@ func NewTest2DGateConfig() *config.DGateConfig { conf := NewTestDGateConfig() conf.ProxyConfig = config.DGateProxyConfig{ Host: "localhost", - Port: 16436, + Port: 0, InitResources: &config.DGateResources{ Namespaces: []spec.Namespace{ { @@ -112,7 +112,7 @@ func NewTest4DGateConfig() *config.DGateConfig { conf.DisableDefaultNamespace = false conf.ProxyConfig = config.DGateProxyConfig{ Host: "localhost", - Port: 16436, + Port: 0, InitResources: &config.DGateResources{ Namespaces: []spec.Namespace{ { @@ -168,3 +168,15 @@ func NewTestDGateConfig_DomainAndNamespaces2() *config.DGateConfig { conf.DisableDefaultNamespace = false return conf } + +func NewTestAdminConfig() *config.DGateConfig { + conf := NewTestDGateConfig() + conf.AdminConfig = &config.DGateAdminConfig{ + Host: "localhost", + Port: 0, + TLS: &config.DGateTLSConfig{ + Port: 0, + }, + } + return conf +} diff --git a/internal/config/loader.go b/internal/config/loader.go index 533f22c..9a2ab9a 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -156,6 +156,7 @@ func LoadConfig(dgateConfigPath string) (*DGateConfig, error) { kDefault(k, "proxy.port", 80) kDefault(k, "proxy.enable_h2c", false) kDefault(k, "proxy.enable_http2", false) + kDefault(k, "proxy.console_log_level", k.Get("log_level")) if k.Get("proxy.enable_h2c") == true && k.Get("proxy.enable_http2") == false { diff --git a/internal/config/store_config.go b/internal/config/store_config.go index 808170c..0b1705e 100644 --- a/internal/config/store_config.go +++ b/internal/config/store_config.go @@ -7,7 +7,6 @@ import ( type StorageType string const ( - StorageTypeDebug StorageType = "debug" StorageTypeMemory StorageType = "memory" StorageTypeFile StorageType = "file" ) @@ -26,3 +25,7 @@ func StoreConfig[T any, C any](config C) (T, error) { err = decoder.Decode(config) return output, err } + +func (st StorageType) String() string { + return string(st) +} diff --git a/internal/proxy/change_log.go b/internal/proxy/change_log.go index 1fab512..69ef075 100644 --- a/internal/proxy/change_log.go +++ b/internal/proxy/change_log.go @@ -2,125 +2,116 @@ package proxy import ( "fmt" - "strconv" "time" "errors" "github.com/dgate-io/dgate/pkg/spec" "github.com/dgate-io/dgate/pkg/util/sliceutil" - "github.com/dgraph-io/badger/v4" + "github.com/hashicorp/raft" "github.com/mitchellh/mapstructure" "go.uber.org/zap" ) // processChangeLog - processes a change log and applies the change to the proxy state func (ps *ProxyState) processChangeLog(cl *spec.ChangeLog, reload, store bool) (err error) { - defer func() { - if err != nil && !cl.Cmd.IsNoop() { - ps.ready.Store(true) - if changeHash, err := HashAny(ps.changeHash, cl); err != nil { - ps.logger.Error("error hashing change log", zap.Error(err)) - return - } else { - ps.changeHash = changeHash - } - } - }() - if !cl.Cmd.IsNoop() { - if len(ps.changeLogs) > 0 { - xcl := ps.changeLogs[len(ps.changeLogs)-1] - if xcl.ID == cl.ID { - ps.logger.Debug("duplicate change log", zap.String("id", cl.ID)) - return nil - } - } - strconv.FormatInt(time.Now().UnixNano(), 36) - ps.changeLogs = append(ps.changeLogs, cl) - switch cl.Cmd.Resource() { - case spec.Namespaces: - var item spec.Namespace - item, err = decode[spec.Namespace](cl.Item) - if err == nil { - ps.logger.Debug("Processing namespace", zap.String("name", item.Name)) - err = ps.processNamespace(&item, cl) - } - case spec.Services: - var item spec.Service - item, err = decode[spec.Service](cl.Item) - if err == nil { - ps.logger.Debug("Processing service", zap.String("name", item.Name)) - err = ps.processService(&item, cl) - } - case spec.Routes: - var item spec.Route - item, err = decode[spec.Route](cl.Item) - if err == nil { - ps.logger.Debug("Processing route", zap.String("name", item.Name)) - err = ps.processRoute(&item, cl) - } - case spec.Modules: - var item spec.Module - item, err = decode[spec.Module](cl.Item) + if reload { + defer func(start time.Time) { + ps.logger.Debug("processing change log", + zap.String("id", cl.ID), + zap.Duration("duration", time.Since(start)), + ) + }(time.Now()) + } + ps.proxyLock.Lock() + defer ps.proxyLock.Unlock() + + // store change log if there is no error + if store && !cl.Cmd.IsNoop() { + defer func() { if err == nil { - ps.logger.Debug("Processing module", zap.String("name", item.Name)) - err = ps.processModule(&item, cl) + if !ps.raftEnabled { + // dont store change logs + if err = ps.store.StoreChangeLog(cl); err != nil { + ps.logger.Error("Error storing change log, restarting state", zap.Error(err)) + return + } + } + if len(ps.changeLogs) > 0 { + xcl := ps.changeLogs[len(ps.changeLogs)-1] + if xcl.ID == cl.ID { + if r := ps.Raft(); r != nil && r.State() == raft.Leader { + return + } + ps.logger.Error("duplicate change log", + zap.String("id", cl.ID), + zap.Stringer("cmd", cl.Cmd), + ) + return + } + } + ps.changeLogs = append(ps.changeLogs, cl) } - case spec.Domains: - var item spec.Domain - item, err = decode[spec.Domain](cl.Item) + }() + } + + // apply change log to the state + if !cl.Cmd.IsNoop() { + defer func() { if err == nil { - ps.logger.Debug("Processing domain", zap.String("name", item.Name)) - err = ps.processDomain(&item, cl) + hash_retry: + oldHash := ps.changeHash.Load() + if newHash, err := HashAny(oldHash, cl.ID); err != nil { + ps.logger.Error("error hashing change log", zap.Error(err)) + } else if !ps.changeHash.CompareAndSwap(oldHash, newHash) { + goto hash_retry + } + } else { + go ps.restartState(func(err error) { + if err != nil { + ps.Stop() + } + }) } - case spec.Collections: - var item spec.Collection - item, err = decode[spec.Collection](cl.Item) - if err == nil { - ps.logger.Debug("Processing collection", zap.String("name", item.Name)) - err = ps.processCollection(&item, cl) + }() + if cl.Cmd.Resource() == spec.Documents { + var item *spec.Document + if item, err = decode[*spec.Document](cl.Item); err != nil { + return } - case spec.Documents: - var item spec.Document - item, err = decode[spec.Document](cl.Item) - if err == nil { - ps.logger.Debug("Processing document", zap.String("id", item.ID)) - err = ps.processDocument(&item, cl) + if err = ps.processDocument(item, cl, store); err != nil { + ps.logger.Error("error processing document change log", zap.Error(err)) + return } - case spec.Secrets: - var item spec.Secret - item, err = decode[spec.Secret](cl.Item) - if err == nil { - ps.logger.Debug("Processing secret", zap.String("name", item.Name)) - err = ps.processSecret(&item, cl) + } else { + if err = ps.processResource(cl); err != nil { + ps.logger.Error("error processing change log", + zap.String("id", cl.ID), + zap.Stringer("cmd", cl.Cmd), + zap.Error(err), + ) + return } - default: - err = fmt.Errorf("unknown command: %s", cl.Cmd) - } - if err != nil { - ps.logger.Error("decoding or processing change log", zap.Error(err)) - return } } + + // apply state changes to the proxy if reload { - if cl.Cmd.IsNoop() || cl.Cmd.Resource().IsRelatedTo(spec.Routes) { - ps.logger.Debug("Registering change log", zap.Stringer("cmd", cl.Cmd)) - if err = ps.reconfigureState(false); err != nil { + overrideReload := cl.Cmd.IsNoop() || ps.pendingChanges + if overrideReload || cl.Cmd.Resource().IsRelatedTo(spec.Routes) { + if err := ps.storeCachedDocuments(); err != nil { + ps.logger.Error("error storing cached documents", zap.Error(err)) + return err + } + ps.logger.Debug("Reloading change log", zap.String("id", cl.ID)) + if err = ps.reconfigureState(cl); err != nil { ps.logger.Error("Error registering change log", zap.Error(err)) return } + ps.pendingChanges = false } - } - if store { - if err = ps.store.StoreChangeLog(cl); err != nil { - ps.logger.Error("Error storing change log, restarting state", zap.Error(err)) - ps.restartState(func(err error) { - if err != nil { - go ps.Stop() - } - }) - return - } + } else if !cl.Cmd.IsNoop() { + ps.pendingChanges = true } return nil @@ -145,6 +136,49 @@ func decode[T any](input any) (T, error) { return output, nil } +func (ps *ProxyState) processResource(cl *spec.ChangeLog) (err error) { + switch cl.Cmd.Resource() { + case spec.Namespaces: + var item spec.Namespace + if item, err = decode[spec.Namespace](cl.Item); err == nil { + err = ps.processNamespace(&item, cl) + } + case spec.Services: + var item spec.Service + if item, err = decode[spec.Service](cl.Item); err == nil { + err = ps.processService(&item, cl) + } + case spec.Routes: + var item spec.Route + if item, err = decode[spec.Route](cl.Item); err == nil { + err = ps.processRoute(&item, cl) + } + case spec.Modules: + var item spec.Module + if item, err = decode[spec.Module](cl.Item); err == nil { + err = ps.processModule(&item, cl) + } + case spec.Domains: + var item spec.Domain + if item, err = decode[spec.Domain](cl.Item); err == nil { + err = ps.processDomain(&item, cl) + } + case spec.Collections: + var item spec.Collection + if item, err = decode[spec.Collection](cl.Item); err == nil { + err = ps.processCollection(&item, cl) + } + case spec.Secrets: + var item spec.Secret + if item, err = decode[spec.Secret](cl.Item); err == nil { + err = ps.processSecret(&item, cl) + } + default: + err = fmt.Errorf("unknown command: %s", cl.Cmd) + } + return err +} + func (ps *ProxyState) processNamespace(ns *spec.Namespace, cl *spec.ChangeLog) error { switch cl.Cmd.Action() { case spec.Add: @@ -232,17 +266,54 @@ func (ps *ProxyState) processCollection(col *spec.Collection, cl *spec.ChangeLog return err } -func (ps *ProxyState) processDocument(doc *spec.Document, cl *spec.ChangeLog) (err error) { +var docCache = []*spec.Document{} + +func (ps *ProxyState) storeCachedDocuments() error { + if len(docCache) == 0 { + return nil + } + ps.logger.Debug("Storing cached documents", zap.Int("count", len(docCache))) + err := ps.store.StoreDocuments(docCache) + if err != nil { + return err + } + docCache = []*spec.Document{} + return nil +} + +func (ps *ProxyState) processDocument(doc *spec.Document, cl *spec.ChangeLog, store bool) (err error) { if doc.NamespaceName == "" { doc.NamespaceName = cl.Namespace } - switch cl.Cmd.Action() { - case spec.Add: - err = ps.store.StoreDocument(doc) - case spec.Delete: - err = ps.store.DeleteDocument(doc.ID, doc.CollectionName, doc.NamespaceName) - default: - err = fmt.Errorf("unknown command: %s", cl.Cmd) + if store { + switch cl.Cmd.Action() { + case spec.Add: + err = ps.store.StoreDocument(doc) + case spec.Delete: + err = ps.store.DeleteDocument(doc.ID, doc.CollectionName, doc.NamespaceName) + default: + err = fmt.Errorf("unknown command: %s", cl.Cmd) + } + } else { + switch cl.Cmd.Action() { + case spec.Add: + docCache = append(docCache, doc) + case spec.Delete: + deletedIndex := sliceutil.BinarySearch(docCache, doc, func(doc1 *spec.Document, doc2 *spec.Document) int { + if doc1.ID == doc2.ID { + return 0 + } + if doc1.ID < doc2.ID { + return -1 + } + return 1 + }) + if deletedIndex >= 0 { + docCache = append(docCache[:deletedIndex], docCache[deletedIndex+1:]...) + } + default: + err = fmt.Errorf("unknown command: %s", cl.Cmd) + } } return err } @@ -262,67 +333,63 @@ func (ps *ProxyState) processSecret(scrt *spec.Secret, cl *spec.ChangeLog) (err return err } +// restoreFromChangeLogs - restores the proxy state from change logs; directApply is used to avoid locking the proxy state func (ps *ProxyState) restoreFromChangeLogs(directApply bool) error { - logs, err := ps.store.FetchChangeLogs() - if err != nil { - if err == badger.ErrKeyNotFound { - ps.logger.Debug("no state change logs found in storage") - } else { - return errors.New("failed to get state change logs from storage: " + err.Error()) + var logs []*spec.ChangeLog + var err error + if ps.raftEnabled { + if logs = ps.changeLogs; len(logs) == 0 { + return nil } - } else { - ps.logger.Info("restoring state change logs from storage", zap.Int("count", len(logs))) - // we might need to sort the change logs by timestamp - for i, cl := range logs { - ps.logger.Debug("restoring change log", - zap.Int("index", i), - zap.Stringer("changeLog", cl.Cmd), - ) - err = ps.processChangeLog(cl, false, false) - if err != nil { - if ps.config.Debug { - ps.logger.Error("error restorng from change logs", zap.Error(err)) - continue - } - return err - } + } else if logs, err = ps.store.FetchChangeLogs(); err != nil { + return errors.New("failed to get state change logs from storage: " + err.Error()) + } + ps.logger.Info("restoring state change logs from storage", zap.Int("count", len(logs))) + // we might need to sort the change logs by timestamp + for _, cl := range logs { + // skip documents as they are persisted in the store + if cl.Cmd.Resource() == spec.Documents { + continue } - if !directApply { - cl := spec.NewNoopChangeLog() - if err = ps.processChangeLog(cl, true, false); err != nil { - return err - } + if err = ps.processChangeLog(cl, false, false); err != nil { + return err } else { - if err = ps.reconfigureState(false); err != nil { - return nil - } + ps.changeLogs = append(ps.changeLogs, cl) + } + } + if cl := spec.NewNoopChangeLog(); !directApply { + if err = ps.reconfigureState(cl); err != nil { + return err } + } else if err = ps.processChangeLog(cl, true, false); err != nil { + return err + } - // TODO: optionally compact change logs through a flag in config? - if len(logs) > 1 { - removed, err := ps.compactChangeLogs(logs) - if err != nil { - ps.logger.Error("failed to compact state change logs", zap.Error(err)) - return err - } - if removed > 0 { - ps.logger.Info("compacted change logs", - zap.Int("removed", removed), - zap.Int("total", len(logs)), - ) - } + // DISABLED: compaction of change logs needs to have better testing + if len(logs) < 0 { + removed, err := ps.compactChangeLogs(logs) + if err != nil { + ps.logger.Error("failed to compact state change logs", zap.Error(err)) + return err + } + if removed > 0 { + ps.logger.Info("compacted change logs", + zap.Int("removed", removed), + zap.Int("total", len(logs)), + ) } } + return nil } func (ps *ProxyState) compactChangeLogs(logs []*spec.ChangeLog) (int, error) { removeList := compactChangeLogsRemoveList(ps.logger, sliceutil.SliceCopy(logs)) - removed, err := ps.store.DeleteChangeLogs(removeList) + err := ps.store.DeleteChangeLogs(removeList) if err != nil { - return removed, err + return 0, err } - return removed, nil + return len(logs), nil } /* diff --git a/internal/proxy/dynamic_proxy.go b/internal/proxy/dynamic_proxy.go index d635155..de74200 100644 --- a/internal/proxy/dynamic_proxy.go +++ b/internal/proxy/dynamic_proxy.go @@ -1,8 +1,10 @@ package proxy import ( + "context" "errors" "fmt" + "math" "net/http" "os" "time" @@ -11,119 +13,175 @@ import ( "github.com/dgate-io/dgate/pkg/modules/extractors" "github.com/dgate-io/dgate/pkg/spec" "github.com/dgate-io/dgate/pkg/typescript" + "github.com/dgate-io/dgate/pkg/util/tree/avl" "github.com/dop251/goja" "go.uber.org/zap" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" + "golang.org/x/sync/errgroup" ) -func (ps *ProxyState) reconfigureState(init bool) (err error) { +func (ps *ProxyState) reconfigureState(log *spec.ChangeLog) (err error) { defer func() { if err != nil { - ps.restartState(func(err error) { + ps.logger.Error("error occurred reloading state, restarting...", zap.Error(err)) + go ps.restartState(func(err error) { if err != nil { ps.logger.Error("Error restarting state", zap.Error(err)) - go ps.Stop() + ps.Stop() } }) } }() - ps.proxyLock.Lock() - defer ps.proxyLock.Unlock() + ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) + defer cancel() start := time.Now() - if err = ps.setupModules(); err != nil { + if err = ps.setupModules(ctx, log); err != nil { + ps.logger.Error("Error setting up modules", zap.Error(err)) return } - if err = ps.setupRoutes(); err != nil { + if err = ps.setupRoutes(ctx, log); err != nil { + ps.logger.Error("Error setting up routes", zap.Error(err)) return } elapsed := time.Since(start) - if !init { - ps.logger.Debug("State reloaded", - zap.Duration("elapsed", elapsed), - ) - } else { - ps.logger.Info("State initialized", - zap.Duration("elapsed", elapsed), - ) - } + ps.logger.Debug("State reloaded", + zap.Duration("elapsed", elapsed), + ) return nil } -func (ps *ProxyState) setupModules() error { - ps.logger.Debug("Setting up modules") - for _, route := range ps.rm.GetRoutes() { - if len(route.Modules) > 0 { - mod := route.Modules[0] - var ( - err error - program *goja.Program - modPayload string = mod.Payload - ) - start := time.Now() - if mod.Type == spec.ModuleTypeTypescript { - if modPayload, err = typescript.Transpile(modPayload); err != nil { - ps.logger.Error("Error transpiling module: " + mod.Name) - return err +func customErrGroup(ctx context.Context, count int) (*errgroup.Group, context.Context) { + grp, ctx := errgroup.WithContext(ctx) + limit := int(math.Log2(float64(count))) + limit = min(1, max(16, limit)) + grp.SetLimit(limit) + return grp, ctx +} + +func (ps *ProxyState) setupModules( + ctx context.Context, + log *spec.ChangeLog, +) error { + var routes = []*spec.DGateRoute{} + if log.Namespace == "" || ps.pendingChanges { + routes = ps.rm.GetRoutes() + } else { + routes = ps.rm.GetRoutesByNamespace(log.Namespace) + } + programs := avl.NewTree[string, *goja.Program]() + grp, ctx := customErrGroup(ctx, len(routes)) + start := time.Now() + for _, rt := range routes { + if len(rt.Modules) > 0 { + route := rt + grp.Go(func() error { + mod := route.Modules[0] + var ( + err error + program *goja.Program + modPayload string = mod.Payload + ) + if mod.Type == spec.ModuleTypeTypescript { + tsBucket := ps.sharedCache.Bucket("typescript") + // hash the typescript module payload + tsHash, err := HashString(1337, modPayload) + if err != nil { + ps.logger.Error("Error hashing module: " + mod.Name) + } else if cacheData, ok := tsBucket.Get(tsHash); ok { + if modPayload, ok = cacheData.(string); ok { + goto compile + } + } + if modPayload, err = typescript.Transpile(ctx, modPayload); err != nil { + ps.logger.Error("Error transpiling module: " + mod.Name) + return err + } else { + tsBucket.SetWithTTL(tsHash, modPayload, 5*time.Minute) + } } - } - if mod.Type == spec.ModuleTypeJavascript || mod.Type == spec.ModuleTypeTypescript { - if program, err = goja.Compile(mod.Name, modPayload, true); err != nil { - ps.logger.Error("Error compiling module: " + mod.Name) - return err + compile: + if mod.Type == spec.ModuleTypeJavascript || mod.Type == spec.ModuleTypeTypescript { + if program, err = goja.Compile(mod.Name, modPayload, true); err != nil { + ps.logger.Error("Error compiling module: " + mod.Name) + return err + } + } else { + return errors.New("invalid module type: " + mod.Type.String()) } - } else { - return errors.New("invalid module type: " + mod.Type.String()) - } - testRtCtx := NewRuntimeContext(ps, route, mod) - defer testRtCtx.Clean() - err = extractors.SetupModuleEventLoop(ps.printer, testRtCtx) - if err != nil { - ps.logger.Error("Error applying module changes", - zap.Error(err), zap.String("module", mod.Name), - ) - return err - } - ps.modPrograms.Insert(mod.Name+"/"+mod.Namespace.Name, program) - elapsed := time.Since(start) - ps.logger.Debug("Module changed applied", - zap.Duration("elapsed", elapsed), - zap.String("name", mod.Name), - zap.String("namespace", mod.Namespace.Name), - ) + tmpCtx := NewRuntimeContext(ps, route, mod) + defer tmpCtx.Clean() + if err = extractors.SetupModuleEventLoop(ps.printer, tmpCtx); err != nil { + ps.logger.Error("Error applying module changes", + zap.Error(err), zap.String("module", mod.Name), + ) + return err + } + programs.Insert(mod.Name+"/"+route.Namespace.Name, program) + return nil + }) } } + + if err := grp.Wait(); err != nil { + return err + } + programs.Each(func(s string, p *goja.Program) bool { + ps.modPrograms.Insert(s, p) + return true + }) + ps.logger.Debug("Modules setup", + zap.Duration("elapsed", time.Since(start)), + ) return nil } -func (ps *ProxyState) setupRoutes() (err error) { - ps.logger.Debug("Setting up routes") - // reqCtxProviders := avl.NewTree[string, *RequestContextProvider]() - for namespaceName, routes := range ps.rm.GetRouteNamespaceMap() { - mux := router.NewMux() - for _, rt := range routes { - reqCtxProvider := NewRequestContextProvider(rt, ps) - if len(rt.Modules) > 0 { - modExtFunc := ps.createModuleExtractorFunc(rt) - if modPool, err := NewModulePool( - 256, 1024, reqCtxProvider, modExtFunc, - ); err != nil { - ps.logger.Error("Error creating module buffer", zap.Error(err)) - return err - } else { - reqCtxProvider.SetModulePool(modPool) +func (ps *ProxyState) setupRoutes( + ctx context.Context, + log *spec.ChangeLog, +) error { + var rtMap map[string][]*spec.DGateRoute + if log.Namespace == "" || ps.pendingChanges { + rtMap = ps.rm.GetRouteNamespaceMap() + ps.providers.Clear() + } else { + rtMap = make(map[string][]*spec.DGateRoute) + routes := ps.rm.GetRoutesByNamespace(log.Namespace) + if len(routes) > 0 { + rtMap[log.Namespace] = routes + } else { + // if namespace has no routes, delete the router + ps.routers.Delete(log.Namespace) + } + } + start := time.Now() + grp, _ := customErrGroup(ctx, len(rtMap)) + for namespaceName, routes := range rtMap { + namespaceName, routes := namespaceName, routes + grp.Go(func() (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%v", r) } - } - ps.providers.Insert(rt.Namespace.Name+"/"+rt.Name, reqCtxProvider) - err = func(rt *spec.DGateRoute) (err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("%v", r) + }() + mux := router.NewMux() + for _, rt := range routes { + reqCtxProvider := NewRequestContextProvider(rt, ps) + if len(rt.Modules) > 0 { + modExtFunc := ps.createModuleExtractorFunc(rt) + if modPool, err := NewModulePool( + 256, 1024, reqCtxProvider, modExtFunc, + ); err != nil { + ps.logger.Error("Error creating module buffer", zap.Error(err)) + return err + } else { + reqCtxProvider.SetModulePool(modPool) } - }() + } + ps.providers.Insert(rt.Namespace.Name+"/"+rt.Name, reqCtxProvider) for _, path := range rt.Paths { if len(rt.Methods) > 0 && rt.Methods[0] == "*" { if len(rt.Methods) > 1 { @@ -141,19 +199,23 @@ func (ps *ProxyState) setupRoutes() (err error) { } } } - return nil - }(rt) - } - - ps.logger.Debug("Routes have changed, reloading") - if dr, ok := ps.routers.Find(namespaceName); ok { - dr.ReplaceMux(mux) - } else { - dr := router.NewRouterWithMux(mux) - ps.routers.Insert(namespaceName, dr) - } + } + if dr, ok := ps.routers.Find(namespaceName); ok { + dr.ReplaceMux(mux) + } else { + dr := router.NewRouterWithMux(mux) + ps.routers.Insert(namespaceName, dr) + } + return nil + }) } - return + if err := grp.Wait(); err != nil { + return err + } + ps.logger.Debug("Routes setup", + zap.Duration("elapsed", time.Since(start)), + ) + return nil } func (ps *ProxyState) createModuleExtractorFunc(rt *spec.DGateRoute) ModuleExtractorFunc { @@ -215,7 +277,7 @@ func (ps *ProxyState) startProxyServer() { cfg := ps.config.ProxyConfig hostPort := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) ps.logger.Info("Starting proxy server on " + hostPort) - proxyHttpLogger := ps.Logger() + proxyHttpLogger := ps.logger.Named("http") server := &http.Server{ Addr: hostPort, Handler: ps, @@ -232,8 +294,8 @@ func (ps *ProxyState) startProxyServer() { } } if err := server.ListenAndServe(); err != nil { - ps.logger.Error("Error starting proxy server", zap.Error(err)) - os.Exit(1) + ps.logger.Error("error starting proxy server", zap.Error(err)) + panic(err) } } @@ -244,11 +306,17 @@ func (ps *ProxyState) startProxyServerTLS() { } hostPort := fmt.Sprintf("%s:%d", cfg.Host, cfg.TLS.Port) ps.logger.Info("Starting secure proxy server on " + hostPort) - proxyHttpsLogger := ps.logger.Named("https") + goLogger, err := zap.NewStdLogAt( + ps.logger.Named("https"), + zap.DebugLevel, + ) + if err != nil { + panic(err) + } secureServer := &http.Server{ Addr: hostPort, Handler: ps, - ErrorLog: zap.NewStdLog(proxyHttpsLogger), + ErrorLog: goLogger, TLSConfig: ps.DynamicTLSConfig( cfg.TLS.CertFile, cfg.TLS.KeyFile, @@ -266,13 +334,14 @@ func (ps *ProxyState) startProxyServerTLS() { } if err := secureServer.ListenAndServeTLS("", ""); err != nil { ps.logger.Error("Error starting secure proxy server", zap.Error(err)) - os.Exit(1) + panic(err) } } func (ps *ProxyState) Start() (err error) { defer func() { if err != nil { + ps.logger.Error("error starting proxy", zap.Error(err)) ps.Stop() } }() @@ -285,33 +354,32 @@ func (ps *ProxyState) Start() (err error) { go ps.startProxyServer() go ps.startProxyServerTLS() - if !ps.replicationEnabled { + if !ps.raftEnabled { if err = ps.restoreFromChangeLogs(false); err != nil { return err } else { - ps.ready.Store(true) + ps.SetReady(true) } } - return nil } func (ps *ProxyState) Stop() { go func() { defer os.Exit(3) - <-time.After(5 * time.Second) + <-time.After(7 * time.Second) ps.logger.Error("Failed to stop proxy server") }() ps.logger.Info("Stopping proxy server") defer os.Exit(0) - defer ps.Logger().Sync() + defer ps.logger.Sync() ps.proxyLock.Lock() - raftNode := ps.Raft() - ps.proxyLock.Unlock() + defer ps.proxyLock.Unlock() + ps.logger.Info("Shutting down raft") - if raftNode != nil { + if raftNode := ps.Raft(); raftNode != nil { ps.logger.Info("Stopping Raft node") if err := raftNode.Shutdown().Error(); err != nil { ps.logger.Error("Error stopping Raft node", zap.Error(err)) diff --git a/internal/proxy/proxy_documents.go b/internal/proxy/proxy_documents.go index c75d031..4a88d86 100644 --- a/internal/proxy/proxy_documents.go +++ b/internal/proxy/proxy_documents.go @@ -18,7 +18,7 @@ func (ps *ProxyState) GetDocuments(collection, namespace string, limit, offset i if _, ok := ps.rm.GetCollection(collection, namespace); !ok { return nil, spec.ErrCollectionNotFound(collection) } - return ps.store.FetchDocuments(namespace, collection, limit, offset) + return ps.store.FetchDocuments(collection, namespace, limit, offset) } // GetDocumentByID is a function that returns a document in a collection by its ID. diff --git a/internal/proxy/proxy_handler.go b/internal/proxy/proxy_handler.go index 8c97626..547fe2e 100644 --- a/internal/proxy/proxy_handler.go +++ b/internal/proxy/proxy_handler.go @@ -38,7 +38,7 @@ func proxyHandler(ps *ProxyState, reqCtx *RequestContext) { if reqCtx.route.Service != nil { event = event.With(zap.String("service", reqCtx.route.Service.Name)) } - event.Info("Request log") + event.Debug("Request log") }() defer ps.metrics.MeasureProxyRequest(reqCtx, time.Now()) @@ -164,7 +164,7 @@ func handleServiceProxy(ps *ProxyState, reqCtx *RequestContext, modExt ModuleExt }). ErrorHandler(func(w http.ResponseWriter, r *http.Request, reqErr error) { upstreamErr = reqErr - ps.logger.Error("Error proxying request", + ps.logger.Debug("Error proxying request", zap.String("error", reqErr.Error()), zap.String("route", reqCtx.route.Name), zap.String("service", reqCtx.route.Service.Name), diff --git a/internal/proxy/proxy_printer.go b/internal/proxy/proxy_printer.go index e37278d..ccb0878 100644 --- a/internal/proxy/proxy_printer.go +++ b/internal/proxy/proxy_printer.go @@ -1,40 +1,49 @@ package proxy -import "go.uber.org/zap" +import ( + "go.uber.org/zap" +) type ( ProxyPrinter struct { logger *zap.Logger - // logs []*printerLog } - // printerLog struct { - // time time.Time - // level string - // msg string - // } ) -func NewProxyPrinter(logger *zap.Logger) *ProxyPrinter { - return &ProxyPrinter{ - logger: logger, - // logs: make([]*printerLog, 0), +// NewProxyPrinter creates a new ProxyPrinter. +func NewProxyPrinter(logger *zap.Logger, lvl zap.AtomicLevel) *ProxyPrinter { + newLogger := logger.WithOptions(zap.IncreaseLevel(lvl)) + if !logger.Core().Enabled(lvl.Level()) { + logger.Warn("the desired log level is lower than the global log level") } + return &ProxyPrinter{newLogger} } +// Error logs a message at error level. func (pp *ProxyPrinter) Error(s string) { - // pp.logs = append(pp.logs, &printerLog{ - // time.Now(), "error", s}) pp.logger.Error(s) } +// Warn logs a message at warn level. func (pp *ProxyPrinter) Warn(s string) { - // pp.logs = append(pp.logs, &printerLog{ - // time.Now(), "warn", s}) pp.logger.Warn(s) } +// Log logs a message at debug level. func (pp *ProxyPrinter) Log(s string) { - // pp.logs = append(pp.logs, &printerLog{ - // time.Now(), "info", s}) + pp.logger.Debug(s) +} + +/* + Note: The following methods are not used but are included for completeness. +*/ + +// Info logs a message at info level. +func (pp *ProxyPrinter) Info(s string) { + pp.logger.Info(s) +} + +// Debug logs a message at debug level. +func (pp *ProxyPrinter) Debug(s string) { pp.logger.Debug(s) } diff --git a/internal/proxy/proxy_replication.go b/internal/proxy/proxy_replication.go index 76f96e6..db1d46c 100644 --- a/internal/proxy/proxy_replication.go +++ b/internal/proxy/proxy_replication.go @@ -1,13 +1,18 @@ package proxy -import "github.com/hashicorp/raft" +import ( + "github.com/dgate-io/dgate/pkg/raftadmin" + "github.com/hashicorp/raft" +) type ProxyReplication struct { - raft *raft.Raft + raft *raft.Raft + client *raftadmin.Client } -func NewProxyReplication(raft *raft.Raft) *ProxyReplication { +func NewProxyReplication(raft *raft.Raft, client *raftadmin.Client) *ProxyReplication { return &ProxyReplication{ - raft: raft, + raft: raft, + client: client, } } diff --git a/internal/proxy/proxy_state.go b/internal/proxy/proxy_state.go index 0b0104b..834c6a4 100644 --- a/internal/proxy/proxy_state.go +++ b/internal/proxy/proxy_state.go @@ -1,6 +1,7 @@ package proxy import ( + "context" "crypto/tls" "encoding/base64" "encoding/json" @@ -21,6 +22,7 @@ import ( "github.com/dgate-io/dgate/internal/router" "github.com/dgate-io/dgate/pkg/cache" "github.com/dgate-io/dgate/pkg/modules/extractors" + "github.com/dgate-io/dgate/pkg/raftadmin" "github.com/dgate-io/dgate/pkg/resources" "github.com/dgate-io/dgate/pkg/scheduler" "github.com/dgate-io/dgate/pkg/spec" @@ -34,29 +36,29 @@ import ( ) type ProxyState struct { - version string - debugMode bool - changeHash uint32 - startTime time.Time - logger *zap.Logger - printer console.Printer - config *config.DGateConfig - store *proxystore.ProxyStore + debugMode bool + changeHash *atomic.Uint64 + startTime time.Time + logger *zap.Logger + printer console.Printer + config *config.DGateConfig + store *proxystore.ProxyStore + sharedCache cache.TCache + proxyLock *sync.RWMutex + ready *atomic.Bool + pendingChanges bool + metrics *ProxyMetrics + + rm *resources.ResourceManager + skdr scheduler.Scheduler changeLogs []*spec.ChangeLog - metrics *ProxyMetrics - sharedCache cache.TCache - proxyLock *sync.RWMutex - - rm *resources.ResourceManager - skdr scheduler.Scheduler - providers avl.Tree[string, *RequestContextProvider] modPrograms avl.Tree[string, *goja.Program] + routers avl.Tree[string, *router.DynamicRouter] - ready atomic.Bool - replicationSettings *ProxyReplication - replicationEnabled bool - routers avl.Tree[string, *router.DynamicRouter] + raft *raft.Raft + raftClient *raftadmin.Client + raftEnabled bool ReverseProxyBuilder reverse_proxy.Builder ProxyTransportBuilder proxy_transport.Builder @@ -66,20 +68,21 @@ type ProxyState struct { func NewProxyState(logger *zap.Logger, conf *config.DGateConfig) *ProxyState { var dataStore storage.Storage switch conf.Storage.StorageType { - case config.StorageTypeDebug: - dataStore = storage.NewDebugStore(&storage.DebugStoreConfig{ - Logger: logger, - }) case config.StorageTypeMemory: - dataStore = storage.NewMemoryStore(&storage.MemoryStoreConfig{ - Logger: logger, - }) + memConfig, err := config.StoreConfig[storage.MemStoreConfig](conf.Storage.Config) + if err != nil { + panic(fmt.Errorf("invalid config: %s", err)) + } else { + memConfig.Logger = logger + } + dataStore = storage.NewMemStore(&memConfig) case config.StorageTypeFile: fileConfig, err := config.StoreConfig[storage.FileStoreConfig](conf.Storage.Config) if err != nil { panic(fmt.Errorf("invalid config: %s", err)) + } else { + fileConfig.Logger = logger } - fileConfig.Logger = logger dataStore = storage.NewFileStore(&fileConfig) default: panic(fmt.Errorf("invalid storage type: %s", conf.Storage.StorageType)) @@ -91,36 +94,39 @@ func NewProxyState(logger *zap.Logger, conf *config.DGateConfig) *ProxyState { opt = resources.WithDefaultNamespace(spec.DefaultNamespace) } var printer console.Printer = &extractors.NoopPrinter{} - if conf.ProxyConfig.EnableConsoleLogger { - printer = NewProxyPrinter(logger) + consoleLevel, err := zap.ParseAtomicLevel(conf.ProxyConfig.ConsoleLogLevel) + if err != nil { + panic(fmt.Errorf("invalid console log level: %s", err)) } + printer = NewProxyPrinter(logger, consoleLevel) rpLogger := logger.Named("reverse-proxy") storeLogger := logger.Named("store") schedulerLogger := logger.Named("scheduler") - replicationEnabled := false + raftEnabled := false if conf.AdminConfig != nil && conf.AdminConfig.Replication != nil { - replicationEnabled = true + raftEnabled = true } state := &ProxyState{ - startTime: time.Now(), - ready: atomic.Bool{}, - logger: logger, - debugMode: conf.Debug, - config: conf, - metrics: NewProxyMetrics(), - printer: printer, - routers: avl.NewTree[string, *router.DynamicRouter](), - rm: resources.NewManager(opt), + startTime: time.Now(), + ready: new(atomic.Bool), + changeHash: new(atomic.Uint64), + logger: logger, + debugMode: conf.Debug, + config: conf, + metrics: NewProxyMetrics(), + printer: printer, + routers: avl.NewTree[string, *router.DynamicRouter](), + rm: resources.NewManager(opt), skdr: scheduler.New(scheduler.Options{ Logger: schedulerLogger, }), - providers: avl.NewTree[string, *RequestContextProvider](), - modPrograms: avl.NewTree[string, *goja.Program](), - proxyLock: new(sync.RWMutex), - sharedCache: cache.New(), - store: proxystore.New(dataStore, storeLogger), - replicationEnabled: replicationEnabled, + providers: avl.NewTree[string, *RequestContextProvider](), + modPrograms: avl.NewTree[string, *goja.Program](), + proxyLock: new(sync.RWMutex), + sharedCache: cache.New(), + store: proxystore.New(dataStore, storeLogger), + raftEnabled: raftEnabled, ReverseProxyBuilder: reverse_proxy.NewBuilder(). FlushInterval(-1). ErrorLogger(zap.NewStdLog(rpLogger)). @@ -150,110 +156,165 @@ func NewProxyState(logger *zap.Logger, conf *config.DGateConfig) *ProxyState { return state } -func (ps *ProxyState) Version() string { - return ps.version -} - func (ps *ProxyState) Store() *proxystore.ProxyStore { return ps.store } -func (ps *ProxyState) Logger() *zap.Logger { - return ps.logger -} - -func (ps *ProxyState) ChangeHash() uint32 { - return ps.changeHash +func (ps *ProxyState) ChangeHash() uint64 { + return ps.changeHash.Load() } func (ps *ProxyState) ChangeLogs() []*spec.ChangeLog { - return ps.changeLogs + // return a copy of the change logs + ps.proxyLock.RLock() + defer ps.proxyLock.RUnlock() + return append([]*spec.ChangeLog{}, ps.changeLogs...) } func (ps *ProxyState) Ready() bool { - if ps.replicationEnabled { - return ps.ready.Load() + return ps.ready.Load() +} + +func (ps *ProxyState) SetReady(ready bool) { + if !ps.Ready() && ready { + ps.logger.Info("Proxy state is ready", + zap.Duration("uptime", time.Since(ps.startTime)), + ) } - return true + ps.ready.Store(ready) } func (ps *ProxyState) Raft() *raft.Raft { - if ps.replicationEnabled { - return ps.replicationSettings.raft + if ps.raftEnabled { + return ps.raft } return nil } -func (ps *ProxyState) SetupRaft(r *raft.Raft, oc chan raft.Observation) { +func (ps *ProxyState) SetupRaft(r *raft.Raft, client *raftadmin.Client) { ps.proxyLock.Lock() defer ps.proxyLock.Unlock() + + ps.raft = r + ps.raftClient = client + + oc := make(chan raft.Observation, 32) + r.RegisterObserver(raft.NewObserver(oc, false, func(o *raft.Observation) bool { + switch o.Data.(type) { + case raft.LeaderObservation, raft.PeerObservation: + return true + } + return false + })) go func() { + logger := ps.logger.Named("raft-observer") for obs := range oc { - switch raftObs := obs.Data.(type) { + switch ro := obs.Data.(type) { case raft.PeerObservation: - ps.logger.Info("peer observation", - zap.Stringer("suffrage", raftObs.Peer.Suffrage), - zap.String("address", string(raftObs.Peer.Address)), - zap.String("id", string(raftObs.Peer.ID)), - ) + if ro.Removed { + logger.Info("peer removed", + zap.Stringer("suffrage", ro.Peer.Suffrage), + zap.String("address", string(ro.Peer.Address)), + zap.String("id", string(ro.Peer.ID)), + ) + } else { + logger.Info("peer added", + zap.Stringer("suffrage", ro.Peer.Suffrage), + zap.String("address", string(ro.Peer.Address)), + zap.String("id", string(ro.Peer.ID)), + ) + } case raft.LeaderObservation: - ps.logger.Info("leader observation", - zap.String("leader_addr", string(raftObs.LeaderAddr)), - zap.String("leader_id", string(raftObs.LeaderID)), - ) - case raft.RequestVoteRequest: - ps.logger.Info("request vote request", - zap.String("candidate_id", string(raftObs.GetRPCHeader().ID)), - zap.String("candidate_addr", string(raftObs.GetRPCHeader().Addr)), - zap.Uint64("term", raftObs.Term), - zap.Uint64("last-log-index", raftObs.LastLogIndex), - zap.Uint64("last-log-term", raftObs.LastLogTerm), + ps.SetReady(true) + logger.Info("leader observation", + zap.String("leader_addr", string(ro.LeaderAddr)), + zap.String("leader_id", string(ro.LeaderID)), ) } } - + panic("raft observer channel closed") }() - - ps.replicationSettings = NewProxyReplication(r) } -func (ps *ProxyState) WaitForChanges() error { - if rft := ps.Raft(); rft != nil { - return rft.Barrier(time.Second * 5).Error() - } else { - ps.proxyLock.RLock() - defer ps.proxyLock.RUnlock() +func (ps *ProxyState) WaitForChanges(log *spec.ChangeLog) error { + if r := ps.Raft(); r != nil { + waitTime := time.Second * 10 + if r.State() == raft.Leader { + err := r.Barrier(waitTime).Error() + if err != nil && log != nil { + ps.logger.Error("error waiting for changes", + zap.String("id", log.ID), + zap.Stringer("command", log.Cmd), + zap.Error(err), + ) + } + return err + } else { + if leaderAddr := r.Leader(); leaderAddr != "" { + ctx, cancel := context.WithTimeout( + context.Background(), waitTime) + defer cancel() + retries := 0 + RETRY: + await, err := ps.raftClient.Barrier(ctx, r.Leader()) + if err == nil && await.Error != "" { + err = errors.New(await.Error) + } + if err != nil && log != nil { + ps.logger.Error("error waiting for changes", + zap.String("id", log.ID), + zap.Stringer("command", log.Cmd), + zap.Error(err), + ) + } + if len(ps.changeLogs) > 0 && retries < 5 { + if log.ID >= ps.changeLogs[len(ps.changeLogs)-1].ID { + return nil + } + retries++ + goto RETRY + } + return err + } else { + return errors.New("no leader found") + } + } } return nil } +// ApplyChangeLog - apply change log to the proxy state func (ps *ProxyState) ApplyChangeLog(log *spec.ChangeLog) error { - if ps.replicationEnabled { - if log.Cmd.IsNoop() { - return ps.processChangeLog(log, true, false) - } - r := ps.replicationSettings.raft + if !ps.Ready() { + return errors.New("proxy state not ready") + } + if r := ps.Raft(); r != nil { if r.State() != raft.Leader { return raft.ErrNotLeader } - encodedCL, err := json.Marshal(log) - if err != nil { + if err := ps.processChangeLog(log, true, false); err != nil { return err } - raftLog := raft.Log{ - Data: encodedCL, - } - err = ps.ProcessChangeLog(log, true) + encodedCL, err := json.Marshal(log) if err != nil { return err } + raftLog := raft.Log{Data: encodedCL} + now := time.Now() future := r.ApplyLog(raftLog, time.Second*15) - ps.logger.With(). - Debug("waiting for reply from raft", - zap.String("id", log.ID), - zap.Stringer("command", log.Cmd), - ) - return future.Error() + err = future.Error() + if err != nil { + ps.logger.With(). + Error("error at ApplyLog", + zap.String("id", log.ID), + zap.Stringer("command", log.Cmd), + zap.Stringer("command", time.Since(now)), + zap.Uint64("index", future.Index()), + zap.Any("response", future.Response()), + zap.Error(err), + ) + } + return err } else { return ps.processChangeLog(log, true, true) } @@ -274,35 +335,27 @@ func (ps *ProxyState) SharedCache() cache.TCache { // restartState - restart state clears the state and reloads the configuration // this is useful for rollbacks when broken changes are made. func (ps *ProxyState) restartState(fn func(error)) { + ps.logger.Info("Attempting to restart state...") ps.proxyLock.Lock() defer ps.proxyLock.Unlock() - - ps.logger.Info("Attempting to restart state...") - + ps.changeHash.Store(0) + ps.pendingChanges = false ps.rm.Empty() ps.modPrograms.Clear() ps.providers.Clear() ps.routers.Clear() ps.sharedCache.Clear() - ps.Scheduler().Stop() + ps.skdr.Stop() if err := ps.initConfigResources(ps.config.ProxyConfig.InitResources); err != nil { - fn(err) + go fn(err) return } - if ps.replicationEnabled { - raft := ps.Raft() - err := raft.ReloadConfig(raft.ReloadableConfig()) - if err != nil { - fn(err) - return - } - } if err := ps.restoreFromChangeLogs(true); err != nil { - fn(err) + go fn(err) return } ps.logger.Info("State successfully restarted") - fn(nil) + go fn(nil) } // ReloadState - reload state checks the change logs to see if a reload is required, @@ -324,8 +377,7 @@ func (ps *ProxyState) ReloadState(check bool, logs ...*spec.ChangeLog) error { } func (ps *ProxyState) ProcessChangeLog(log *spec.ChangeLog, reload bool) error { - err := ps.processChangeLog(log, reload, !ps.replicationEnabled) - if err != nil { + if err := ps.processChangeLog(log, reload, true); err != nil { ps.logger.Error("processing error", zap.Error(err)) return err } @@ -421,6 +473,9 @@ func (ps *ProxyState) getDomainCertificate(domain string) (*tls.Certificate, err } func (ps *ProxyState) initConfigResources(resources *config.DGateResources) error { + processCL := func(cl *spec.ChangeLog) error { + return ps.processChangeLog(cl, false, false) + } if resources != nil { numChanges, err := resources.Validate() if err != nil { @@ -429,15 +484,14 @@ func (ps *ProxyState) initConfigResources(resources *config.DGateResources) erro if numChanges > 0 { defer func() { if err != nil { - err = ps.processChangeLog(nil, false, false) + err = processCL(nil) } }() } ps.logger.Info("Initializing resources") for _, ns := range resources.Namespaces { cl := spec.NewChangeLog(&ns, ns.Name, spec.AddNamespaceCommand) - err := ps.processChangeLog(cl, false, false) - if err != nil { + if err := processCL(cl); err != nil { return err } } @@ -455,22 +509,19 @@ func (ps *ProxyState) initConfigResources(resources *config.DGateResources) erro ) } cl := spec.NewChangeLog(&mod.Module, mod.NamespaceName, spec.AddModuleCommand) - err := ps.processChangeLog(cl, false, false) - if err != nil { + if err := processCL(cl); err != nil { return err } } for _, svc := range resources.Services { cl := spec.NewChangeLog(&svc, svc.NamespaceName, spec.AddServiceCommand) - err := ps.processChangeLog(cl, false, false) - if err != nil { + if err := processCL(cl); err != nil { return err } } for _, rt := range resources.Routes { cl := spec.NewChangeLog(&rt, rt.NamespaceName, spec.AddRouteCommand) - err := ps.processChangeLog(cl, false, false) - if err != nil { + if err := processCL(cl); err != nil { return err } } @@ -490,22 +541,19 @@ func (ps *ProxyState) initConfigResources(resources *config.DGateResources) erro dom.Key = string(key) } cl := spec.NewChangeLog(&dom.Domain, dom.NamespaceName, spec.AddDomainCommand) - err := ps.processChangeLog(cl, false, false) - if err != nil { + if err := processCL(cl); err != nil { return err } } for _, col := range resources.Collections { cl := spec.NewChangeLog(&col, col.NamespaceName, spec.AddCollectionCommand) - err := ps.processChangeLog(cl, false, false) - if err != nil { + if err := processCL(cl); err != nil { return err } } for _, doc := range resources.Documents { cl := spec.NewChangeLog(&doc, doc.NamespaceName, spec.AddDocumentCommand) - err := ps.processChangeLog(cl, false, false) - if err != nil { + if err := processCL(cl); err != nil { return err } } @@ -578,7 +626,7 @@ func (ps *ProxyState) ServeHTTP(w http.ResponseWriter, r *http.Request) { // if debug mode is enabled, return a 403 util.WriteStatusCodeError(w, http.StatusForbidden) if ps.debugMode { - w.Write([]byte(" - Domain not allowed")) + w.Write([]byte("domain not allowed")) } return } diff --git a/internal/proxy/proxy_state_test.go b/internal/proxy/proxy_state_test.go index 8b26709..f76cc2e 100644 --- a/internal/proxy/proxy_state_test.go +++ b/internal/proxy/proxy_state_test.go @@ -10,6 +10,7 @@ import ( "github.com/dgate-io/dgate/internal/proxy" "github.com/dgate-io/dgate/pkg/spec" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -38,7 +39,14 @@ func TestDynamicTLSConfig_DomainCert(t *testing.T) { func TestDynamicTLSConfig_DomainCertCache(t *testing.T) { conf := configtest.NewTestDGateConfig_DomainAndNamespaces() ps := proxy.NewProxyState(zap.NewNop(), conf) - d := ps.ResourceManager().GetDomainsByPriority()[0] + if err := ps.Start(); err != nil { + t.Fatal(err) + } + domains := ps.ResourceManager().GetDomainsByPriority() + if !assert.NotEqual(t, len(domains), 0) { + return + } + d := domains[0] key := fmt.Sprintf("cert:%s:%s:%d", d.Namespace.Name, d.Name, d.CreatedAt.UnixMilli()) tlsConfig := ps.DynamicTLSConfig("", "") @@ -409,7 +417,6 @@ func TestProcessChangeLog_Document(t *testing.T) { if err := ps.Store().InitStore(); err != nil { t.Fatal(err) } - c := &spec.Collection{ Name: "test123", NamespaceName: "test", @@ -432,7 +439,7 @@ func TestProcessChangeLog_Document(t *testing.T) { } cl = spec.NewChangeLog(d, d.NamespaceName, spec.AddDocumentCommand) - err = ps.ProcessChangeLog(cl, false) + err = ps.ProcessChangeLog(cl, true) if !assert.Nil(t, err, "error should be nil") { return } @@ -442,7 +449,7 @@ func TestProcessChangeLog_Document(t *testing.T) { if !assert.Nil(t, err, "error should be nil") { return } - assert.Equal(t, 1, len(documents), "should have 1 item") + require.Equal(t, 1, len(documents), "should have 1 item") assert.Equal(t, d.ID, documents[0].ID, "should have the same id") assert.Equal(t, d.NamespaceName, documents[0].NamespaceName, "should have the same namespace") assert.Equal(t, d.CollectionName, documents[0].CollectionName, "should have the same collection") diff --git a/internal/proxy/proxystore/proxy_store.go b/internal/proxy/proxystore/proxy_store.go index d55eb9a..58c877f 100644 --- a/internal/proxy/proxystore/proxy_store.go +++ b/internal/proxy/proxystore/proxy_store.go @@ -32,6 +32,14 @@ func (store *ProxyStore) InitStore() error { return nil } +func (store *ProxyStore) CloseStore() error { + err := store.storage.Close() + if err != nil { + return err + } + return nil +} + func (store *ProxyStore) FetchChangeLogs() ([]*spec.ChangeLog, error) { clBytes, err := store.storage.GetPrefix("changelog/", 0, -1) if err != nil { @@ -80,50 +88,56 @@ RETRY: return nil } -func (store *ProxyStore) DeleteChangeLogs(logs []*spec.ChangeLog) (int, error) { - removed := 0 - for _, cl := range logs { - err := store.storage.Delete("changelog/" + cl.ID) - if err != nil { - return removed, err +func (store *ProxyStore) DeleteChangeLogs(logs []*spec.ChangeLog) error { + err := store.storage.Txn(true, func(txn storage.StorageTxn) error { + for _, cl := range logs { + if err := txn.Delete("changelog/" + cl.ID); err != nil { + return err + } } - removed++ + return nil + }) + if err != nil { + return err } - return removed, nil + return nil } -func createDocumentKey(docId, colName, nsName string) string { +func docKey(docId, colName, nsName string) string { return "doc/" + nsName + "/" + colName + "/" + docId } func (store *ProxyStore) FetchDocument(docId, colName, nsName string) (*spec.Document, error) { - docBytes, err := store.storage.Get(createDocumentKey(docId, colName, nsName)) + docBytes, err := store.storage.Get(docKey(docId, colName, nsName)) if err != nil { - if err == storage.ErrStoreLocked { - return nil, err - } return nil, errors.New("failed to fetch document: " + err.Error()) + } else if docBytes == nil { + return nil, nil } doc := &spec.Document{} err = json.Unmarshal(docBytes, doc) if err != nil { - store.logger.Debug("failed to unmarshal document entry: %s, skipping %s", - zap.Error(err), zap.String("document_id", docId)) - return nil, errors.New("failed to unmarshal document entry" + err.Error()) + return nil, errors.New("failed to unmarshal document entry: " + err.Error()) } return doc, nil } func (store *ProxyStore) FetchDocuments( - namespaceName, collectionName string, + collectionName string, + namespaceName string, limit, offset int, ) ([]*spec.Document, error) { + if limit == 0 { + return nil, nil + } docs := make([]*spec.Document, 0) - docPrefix := createDocumentKey("", collectionName, namespaceName) + docPrefix := docKey("", collectionName, namespaceName) err := store.storage.IterateValuesPrefix(docPrefix, func(key string, val []byte) error { - if offset -= 1; offset > 0 { + if offset > 0 { + offset -= 1 return nil - } else if limit -= 1; limit != 0 { + } + if limit -= 1; limit != 0 { var newDoc spec.Document err := json.Unmarshal(val, &newDoc) if err != nil { @@ -144,32 +158,31 @@ func (store *ProxyStore) StoreDocument(doc *spec.Document) error { if err != nil { return err } - store.logger.Debug("storing document") - err = store.storage.Set(createDocumentKey(doc.ID, doc.CollectionName, doc.NamespaceName), docBytes) + key := docKey(doc.ID, doc.CollectionName, doc.NamespaceName) + err = store.storage.Set(key, docBytes) if err != nil { return err } return nil } -func (store *ProxyStore) DeleteDocument(id, colName, nsName string) error { - err := store.storage.Delete(createDocumentKey(id, colName, nsName)) - if err != nil { - if err == badger.ErrKeyNotFound { - return nil +func (store *ProxyStore) StoreDocuments(docs []*spec.Document) error { + for _, doc := range docs { + docBytes, err := json.Marshal(doc) + if err != nil { + return err + } + key := docKey(doc.ID, doc.CollectionName, doc.NamespaceName) + err = store.storage.Txn(true, func(txn storage.StorageTxn) error { + return txn.Set(key, docBytes) + }) + if err != nil { + return err } - return err } return nil } -func (store *ProxyStore) DeleteDocuments(doc *spec.Document) error { - err := store.storage.IterateTxnPrefix(createDocumentKey("", doc.CollectionName, doc.NamespaceName), - func(txn storage.StorageTxn, key string) error { - return txn.Delete(key) - }) - if err != nil { - return err - } - return nil +func (store *ProxyStore) DeleteDocument(id, colName, nsName string) error { + return store.storage.Delete(docKey(id, colName, nsName)) } diff --git a/internal/proxy/proxystore/proxy_store_test.go b/internal/proxy/proxystore/proxy_store_test.go new file mode 100644 index 0000000..78cb470 --- /dev/null +++ b/internal/proxy/proxystore/proxy_store_test.go @@ -0,0 +1,121 @@ +package proxystore_test + +import ( + "testing" + + "github.com/dgate-io/dgate/internal/proxy/proxystore" + "github.com/dgate-io/dgate/pkg/spec" + "github.com/dgate-io/dgate/pkg/storage" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func TestProxyStory_FileStorage_ChangeLogs(t *testing.T) { + fstore := storage.NewFileStore(&storage.FileStoreConfig{ + Directory: t.TempDir(), + }) + pstore := proxystore.New(fstore, zap.NewNop()) + assert.NoError(t, pstore.InitStore()) + defer func() { + assert.NoError(t, pstore.CloseStore()) + }() + + // Test SaveChangeLog + cl := &spec.ChangeLog{ + ID: "test", + Cmd: spec.AddNamespaceCommand, + Item: spec.Namespace{Name: "test"}, + } + assert.NoError(t, pstore.StoreChangeLog(cl)) + + // Test FetchChangeLogs + logs, err := pstore.FetchChangeLogs() + assert.NoError(t, err) + assert.Len(t, logs, 1) + + // Test FetchChangeLogs + logs, err = pstore.FetchChangeLogs() + assert.NoError(t, err) + assert.Len(t, logs, 1) + assert.Equal(t, cl.ID, logs[0].ID) + assert.Equal(t, cl.Cmd, logs[0].Cmd) + assert.NotNil(t, logs[0].Item) + + // Test StoreChangeLog + assert.NoError(t, pstore.StoreChangeLog(cl)) + + // Test DeleteChangeLogs + err = pstore.DeleteChangeLogs( + []*spec.ChangeLog{cl}, + ) + assert.NoError(t, err) + + // Test FetchChangeLogs + logs, err = pstore.FetchChangeLogs() + assert.NoError(t, err) + assert.Len(t, logs, 0) +} + +func TestProxyStory_FileStorage_Documents(t *testing.T) { + fstore := storage.NewFileStore(&storage.FileStoreConfig{ + Directory: t.TempDir(), + }) + pstore := proxystore.New(fstore, zap.NewNop()) + assert.NoError(t, pstore.InitStore()) + defer func() { + assert.NoError(t, pstore.CloseStore()) + }() + + // Test StoreDocument + doc := &spec.Document{ + ID: "test", + CollectionName: "col", + NamespaceName: "ns", + Data: "test", + } + assert.NoError(t, pstore.StoreDocument(doc)) + + // Test FetchDocument + doc, err := pstore.FetchDocument("test", "col", "ns") + assert.NoError(t, err) + if assert.NotNil(t, doc) { + assert.NotNil(t, doc.Data) + if dataString, ok := doc.Data.(string); !ok { + t.Fatal("failed to convert data to string") + } else { + assert.Equal(t, "test", dataString) + } + } + + // Test FetchDocuments + docs, err := pstore.FetchDocuments("col", "ns", 2, 0) + assert.NoError(t, err) + assert.Len(t, docs, 1) + if assert.NotNil(t, doc.Data) { + assert.Equal(t, "test", docs[0].ID) + assert.Equal(t, "test", docs[0].Data.(string)) + } + + docs, err = pstore.FetchDocuments("col", "ns", 0, 0) + assert.NoError(t, err) + assert.Len(t, docs, 0) + + docs, err = pstore.FetchDocuments("col", "ns", 2, 1) + assert.NoError(t, err) + assert.Len(t, docs, 0) + + // Test DeleteDocument + err = pstore.DeleteDocument("test", "col", "ns") + assert.NoError(t, err) + + // Test FetchDocument Error + doc, err = pstore.FetchDocument("test123", "col", "ns") + assert.NoError(t, err) + assert.Nil(t, doc) + + // Test FetchDocuments Error + docs, err = pstore.FetchDocuments("col", "ns", 2, 0) + assert.NoError(t, err) + assert.Len(t, docs, 0) + +} diff --git a/internal/proxy/runtime_context.go b/internal/proxy/runtime_context.go index 0c901b8..94c52db 100644 --- a/internal/proxy/runtime_context.go +++ b/internal/proxy/runtime_context.go @@ -25,6 +25,8 @@ type runtimeContext struct { modules []*spec.Module } +var _ modules.RuntimeContext = &runtimeContext{} + func NewRuntimeContext( proxyState *ProxyState, route *spec.DGateRoute, @@ -39,6 +41,7 @@ func NewRuntimeContext( reg := require.NewRegistryWithLoader(func(path string) ([]byte, error) { requireMod := strings.Replace(path, "node_modules/", "", 1) + // TODO: add support for other module types w/ permissions // 'https://' - requires network permissions and must be enabled in the config // 'file://' - requires file system permissions and must be enabled in the config // 'module://' - requires a module lookup and module permissions @@ -57,7 +60,8 @@ func NewRuntimeContext( return code.([]byte), nil } } - payload, err := typescript.Transpile(mod.Payload) + payload, err := typescript.Transpile( + context.TODO(), mod.Payload) if err != nil { return nil, err } @@ -71,8 +75,6 @@ func NewRuntimeContext( return rtCtx } -var _ modules.RuntimeContext = &runtimeContext{} - // UseRequestContext sets the request context func (rtCtx *runtimeContext) Use(reqCtx *RequestContext) (*runtimeContext, error) { if reqCtx != nil { diff --git a/internal/proxy/util.go b/internal/proxy/util.go index af8856e..2ffde8d 100644 --- a/internal/proxy/util.go +++ b/internal/proxy/util.go @@ -7,14 +7,14 @@ import ( "encoding/json" "errors" "hash" - "hash/crc32" + "hash/crc64" "net/http" "slices" "sort" ) -func saltHash[T any](salt uint32, objs ...T) (hash.Hash32, error) { - hash := crc32.NewIEEE() +func saltHash[T any](salt uint64, objs ...T) (hash.Hash64, error) { + hash := crc64.New(crc64.MakeTable(crc64.ECMA)) if salt != 0 { // uint32 to byte array b := make([]byte, 4) @@ -39,15 +39,15 @@ func saltHash[T any](salt uint32, objs ...T) (hash.Hash32, error) { return hash, nil } -func HashAny[T any](salt uint32, objs ...T) (uint32, error) { +func HashAny[T any](salt uint64, objs ...T) (uint64, error) { h, err := saltHash(salt, objs...) if err != nil { return 0, err } - return h.Sum32(), nil + return h.Sum64(), nil } -func HashString[T any](salt uint32, objs ...T) (string, error) { +func HashString[T any](salt uint64, objs ...T) (string, error) { h, err := saltHash(salt, objs...) if err != nil { return "", err diff --git a/performance-tests/long-perf-test.js b/performance-tests/long-perf-test.js index 78cf7ab..c835d21 100644 --- a/performance-tests/long-perf-test.js +++ b/performance-tests/long-perf-test.js @@ -91,7 +91,7 @@ export function dgatePath() { const dgatePath = __ENV._PROXY_URL || 'http://localhost' const path = __ENV.DGATE_PATH; let res = http.get(dgatePath + path, { - headers: { Host: 'dgate.dev' }, + headers: { Host: 'performance.example.com' }, }); let results = {}; results[path + ': status is ' + res.status] = diff --git a/performance-tests/perf-test.js b/performance-tests/perf-test.js index a906bc5..69b5c46 100644 --- a/performance-tests/perf-test.js +++ b/performance-tests/perf-test.js @@ -40,7 +40,7 @@ export function dgatePath() { const dgatePath = __ENV.PROXY_URL || 'http://localhost'; const path = __ENV.DGATE_PATH; let res = http.get(dgatePath + path, { - headers: { Host: 'dgate.dev' }, + headers: { Host: 'performance.example.com' }, }); let results = {}; results[path + ': status is ' + res.status] = (r) => r.status < 400; diff --git a/pkg/modules/dgate/state/state_mod.go b/pkg/modules/dgate/state/state_mod.go index 44ed32b..652fe5e 100644 --- a/pkg/modules/dgate/state/state_mod.go +++ b/pkg/modules/dgate/state/state_mod.go @@ -103,9 +103,9 @@ func (hp *ResourcesModule) getDocuments(payload FetchDocumentsPayload) (*goja.Pr } namespace := namespaceVal.(string) - docPromise, resolve, reject := rt.NewPromise() + prom, resolve, reject := rt.NewPromise() loop.RunOnLoop(func(rt *goja.Runtime) { - doc, err := state.DocumentManager(). + docs, err := state.DocumentManager(). GetDocuments( payload.Collection, namespace, @@ -116,9 +116,9 @@ func (hp *ResourcesModule) getDocuments(payload FetchDocumentsPayload) (*goja.Pr reject(rt.NewGoError(err)) return } - resolve(rt.ToValue(doc)) + resolve(rt.ToValue(docs)) }) - return docPromise, nil + return prom, nil } func writeFunc[T spec.Named](hp *ResourcesModule, cmd spec.Command) func(map[string]any) (*goja.Promise, error) { diff --git a/pkg/modules/extractors/async_tracker.go b/pkg/modules/extractors/async_tracker.go deleted file mode 100644 index c158f43..0000000 --- a/pkg/modules/extractors/async_tracker.go +++ /dev/null @@ -1,66 +0,0 @@ -package extractors - -import ( - "context" - "fmt" - "sync/atomic" - - "github.com/dop251/goja" -) - -var _ goja.AsyncContextTracker = &asyncTracker{} - -type asyncTracker struct { - count atomic.Int32 - exitChan chan int32 -} - -type TrackerEvent int - -const ( - Exited TrackerEvent = iota - Resumed -) - -func newAsyncTracker() *asyncTracker { - return &asyncTracker{ - count: atomic.Int32{}, - exitChan: make(chan int32, 128), - } -} - -// Exited is called when an async function is done -func (t *asyncTracker) Exited() { - t.exitChan <- t.count.Add(-1) -} - -// Grab is called when an async function is scheduled -func (t *asyncTracker) Grab() any { - t.exitChan <- t.count.Add(1) - return nil -} - -// Resumed is called when an async function is executed (ignore) -func (t *asyncTracker) Resumed(any) { - t.exitChan <- t.count.Load() -} - -func (t *asyncTracker) waitTimeout( - ctx context.Context, doneFn func() bool, -) error { - if doneFn() { - return nil - } else if t.count.Load() == 0 { - return nil - } - for { - select { - case <-ctx.Done(): - return fmt.Errorf("async tracker: %s", ctx.Err()) - case numLeft := <-t.exitChan: - if numLeft == 0 || doneFn() { - return nil - } - } - } -} diff --git a/pkg/modules/extractors/extractors.go b/pkg/modules/extractors/extractors.go index a52152d..cdfa54e 100644 --- a/pkg/modules/extractors/extractors.go +++ b/pkg/modules/extractors/extractors.go @@ -45,28 +45,20 @@ func RunAndWaitForResult( fn goja.Callable, args ...goja.Value, ) (res goja.Value, err error) { - tracker := newAsyncTracker() - rt.SetAsyncContextTracker(tracker) - defer func() { - rt.SetAsyncContextTracker(nil) - if err != nil { - rt.Interrupt(err.Error()) - } - }() - if res, err = fn(nil, args...); err != nil { return nil, err } else if prom, ok := res.Export().(*goja.Promise); ok { ctx, cancel := context.WithTimeout( - context.TODO(), 30*time.Second, - ) + context.TODO(), 30*time.Second) defer cancel() - if err := tracker.waitTimeout(ctx, func() bool { + if err = waitTimeout(ctx, func() bool { return prom.State() != goja.PromiseStatePending }); err != nil { + rt.Interrupt(err.Error()) return nil, errors.New("promise timed out: " + err.Error()) } if prom.State() == goja.PromiseStateRejected { + // no need to interrupt the runtime here return nil, errors.New(prom.Result().String()) } results := prom.Result() @@ -85,6 +77,29 @@ func nully(val goja.Value) bool { return val == nil || goja.IsUndefined(val) || goja.IsNull(val) } +func waitTimeout(ctx context.Context, doneFn func() bool) error { + if doneFn() { + return nil + } + maxTimeout := 100 * time.Millisecond + multiplier := 1.75 + backoffTimeout := 2 * time.Millisecond + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(backoffTimeout): + if !doneFn() { + backoffTimeout = min(time.Duration( + float64(backoffTimeout)*multiplier, + ), maxTimeout) + continue + } + return nil + } + } +} + func DefaultFetchUpstreamFunction() FetchUpstreamUrlFunc { roundRobinIndex := 0 return func(ctx *types.ModuleContext) (*url.URL, error) { diff --git a/pkg/modules/extractors/extractors_test.go b/pkg/modules/extractors/extractors_test.go index 71001ba..a58dde2 100644 --- a/pkg/modules/extractors/extractors_test.go +++ b/pkg/modules/extractors/extractors_test.go @@ -1,6 +1,7 @@ package extractors_test import ( + "context" "strconv" "testing" @@ -39,7 +40,7 @@ async function print() {console.log("log")} ` func Test_runAndWaitForResult(t *testing.T) { - src, err := typescript.Transpile(TS_PAYLOAD) + src, err := typescript.Transpile(context.Background(), TS_PAYLOAD) if err != nil { t.Fatal(err) } @@ -96,7 +97,7 @@ export async function named_func_async() { ` func TestExportedInformation(t *testing.T) { - src, err := typescript.Transpile(TS_PAYLOAD_EXPORTED) + src, err := typescript.Transpile(context.Background(), TS_PAYLOAD_EXPORTED) if err != nil { t.Fatal(err) } @@ -172,7 +173,7 @@ export async function test2() { ` func TestExportedPromiseErrors(t *testing.T) { - src, err := typescript.Transpile(TS_PAYLOAD_PROMISE) + src, err := typescript.Transpile(context.Background(), TS_PAYLOAD_PROMISE) if err != nil { t.Fatal(err) } diff --git a/pkg/modules/extractors/runtime_test.go b/pkg/modules/extractors/runtime_test.go index 068d812..324f537 100644 --- a/pkg/modules/extractors/runtime_test.go +++ b/pkg/modules/extractors/runtime_test.go @@ -1,6 +1,7 @@ package extractors_test import ( + "context" "testing" "github.com/dgate-io/dgate/internal/config/configtest" @@ -147,7 +148,7 @@ func BenchmarkNewModuleRuntime(b *testing.B) { b.Run("Transpile-TS", func(b *testing.B) { for i := 0; i < b.N; i++ { b.StartTimer() - _, err := typescript.Transpile(TS_PAYLOAD_CUSTOMFUNC) + _, err := typescript.Transpile(context.Background(), TS_PAYLOAD_CUSTOMFUNC) if err != nil { b.Fatal(err) } diff --git a/pkg/modules/testutil/testutil.go b/pkg/modules/testutil/testutil.go index 97a97cf..70ad006 100644 --- a/pkg/modules/testutil/testutil.go +++ b/pkg/modules/testutil/testutil.go @@ -127,7 +127,7 @@ type Crashable interface { } func CreateTSProgram(c Crashable, payload string) *goja.Program { - src, err := typescript.Transpile(payload) + src, err := typescript.Transpile(context.Background(), payload) if err != nil { c.Fatal(err) } diff --git a/pkg/modules/types/generator.go b/pkg/modules/types/generator.go deleted file mode 100644 index c7de7eb..0000000 --- a/pkg/modules/types/generator.go +++ /dev/null @@ -1,47 +0,0 @@ -package types - -import ( - "github.com/dgate-io/dgate/pkg/eventloop" - "github.com/dop251/goja" -) - -type generatorFunc[T any] func(rt *goja.Runtime) (T, bool, error) - -func asyncGenerator[T any](loop *eventloop.EventLoop, fn generatorFunc[T]) goja.Value { - rt := loop.Runtime() - return rt.ToValue(map[string]any{ - "next": func() (*goja.Promise, error) { - prom, resolve, reject := rt.NewPromise() - loop.RunOnLoop(func(rt *goja.Runtime) { - result, done, err := fn(rt) - if err != nil { - reject(rt.NewGoError(err)) - return - } - resultObject := map[string]any{"done": done} - if !done { - resultObject["value"] = result - } - resolve(rt.ToValue(resultObject)) - }) - return prom, nil - }, - }) -} - -func generator[T any](loop *eventloop.EventLoop, fn generatorFunc[T]) goja.Value { - rt := loop.Runtime() - return rt.ToValue(map[string]any{ - "next": func() (goja.Value, error) { - result, done, err := fn(rt) - if err != nil { - return nil, err - } - resultObject := map[string]any{"done": done} - if !done { - resultObject["value"] = result - } - return rt.ToValue(result), nil - }, - }) -} diff --git a/pkg/raftadmin/raftadmin_client.go b/pkg/raftadmin/client.go similarity index 72% rename from pkg/raftadmin/raftadmin_client.go rename to pkg/raftadmin/client.go index 905dfd2..e92041e 100644 --- a/pkg/raftadmin/raftadmin_client.go +++ b/pkg/raftadmin/client.go @@ -7,8 +7,6 @@ import ( "errors" "fmt" "net/http" - "strings" - "time" "github.com/hashicorp/raft" "go.uber.org/zap" @@ -16,39 +14,32 @@ import ( type Doer func(*http.Request) (*http.Response, error) -type HTTPAdminClient struct { +type Client struct { do Doer - urlFmt string + scheme string logger *zap.Logger } -func NewHTTPAdminClient(doer Doer, urlFmt string, logger *zap.Logger) *HTTPAdminClient { +func NewClient(doer Doer, logger *zap.Logger, scheme string) *Client { if doer == nil { doer = http.DefaultClient.Do } - if urlFmt == "" { - urlFmt = "http://(address)/raftadmin/" - } else { - if !strings.Contains(urlFmt, "(address)") { - panic("urlFmt must contain the string '(address)'") - } - if !strings.HasSuffix(urlFmt, "/") { - urlFmt += "/" - } + if scheme == "" { + scheme = "http" } - return &HTTPAdminClient{ + return &Client{ do: doer, - urlFmt: urlFmt, + scheme: scheme, logger: logger, } } -func (c *HTTPAdminClient) generateUrl(target raft.ServerAddress, action string) string { - return strings.ReplaceAll(c.urlFmt+action, - "(address)", string(target)) +func (c *Client) generateUrl(target raft.ServerAddress, action string) string { + uri := fmt.Sprintf("%s://%s/raftadmin/%s", c.scheme, target, action) + // c.logger.Debug("raftadmin: generated url", zap.String("url", uri)) + return uri } - -func (c *HTTPAdminClient) AddNonvoter(ctx context.Context, target raft.ServerAddress, req *AddNonvoterRequest) (*AwaitResponse, error) { +func (c *Client) AddNonvoter(ctx context.Context, target raft.ServerAddress, req *AddNonvoterRequest) (*AwaitResponse, error) { url := c.generateUrl(target, "AddNonvoter") buf, err := json.Marshal(req) if err != nil { @@ -58,7 +49,7 @@ func (c *HTTPAdminClient) AddNonvoter(ctx context.Context, target raft.ServerAdd if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -74,7 +65,7 @@ func (c *HTTPAdminClient) AddNonvoter(ctx context.Context, target raft.ServerAdd return &out, nil } -func (c *HTTPAdminClient) AddVoter(ctx context.Context, target raft.ServerAddress, req *AddVoterRequest) (*AwaitResponse, error) { +func (c *Client) AddVoter(ctx context.Context, target raft.ServerAddress, req *AddVoterRequest) (*AwaitResponse, error) { url := c.generateUrl(target, "AddVoter") buf, err := json.Marshal(req) if err != nil { @@ -84,7 +75,7 @@ func (c *HTTPAdminClient) AddVoter(ctx context.Context, target raft.ServerAddres if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -100,13 +91,13 @@ func (c *HTTPAdminClient) AddVoter(ctx context.Context, target raft.ServerAddres return &out, nil } -func (c *HTTPAdminClient) AppliedIndex(ctx context.Context, target raft.ServerAddress) (*AppliedIndexResponse, error) { +func (c *Client) AppliedIndex(ctx context.Context, target raft.ServerAddress) (*AppliedIndexResponse, error) { url := c.generateUrl(target, "AppliedIndex") r, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -122,7 +113,7 @@ func (c *HTTPAdminClient) AppliedIndex(ctx context.Context, target raft.ServerAd return &out, nil } -func (c *HTTPAdminClient) ApplyLog(ctx context.Context, target raft.ServerAddress, req *ApplyLogRequest) (*AwaitResponse, error) { +func (c *Client) ApplyLog(ctx context.Context, target raft.ServerAddress, req *ApplyLogRequest) (*AwaitResponse, error) { url := c.generateUrl(target, "ApplyLog") buf, err := json.Marshal(req) if err != nil { @@ -132,7 +123,7 @@ func (c *HTTPAdminClient) ApplyLog(ctx context.Context, target raft.ServerAddres if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -148,13 +139,13 @@ func (c *HTTPAdminClient) ApplyLog(ctx context.Context, target raft.ServerAddres return &out, nil } -func (c *HTTPAdminClient) Barrier(ctx context.Context, target raft.ServerAddress) (*AwaitResponse, error) { +func (c *Client) Barrier(ctx context.Context, target raft.ServerAddress) (*AwaitResponse, error) { url := c.generateUrl(target, "Barrier") r, err := http.NewRequest("POST", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -170,7 +161,7 @@ func (c *HTTPAdminClient) Barrier(ctx context.Context, target raft.ServerAddress return &out, nil } -func (c *HTTPAdminClient) DemoteVoter(ctx context.Context, target raft.ServerAddress, req *DemoteVoterRequest) (*AwaitResponse, error) { +func (c *Client) DemoteVoter(ctx context.Context, target raft.ServerAddress, req *DemoteVoterRequest) (*AwaitResponse, error) { url := c.generateUrl(target, "DemoteVoter") buf, err := json.Marshal(req) if err != nil { @@ -180,7 +171,7 @@ func (c *HTTPAdminClient) DemoteVoter(ctx context.Context, target raft.ServerAdd if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -196,13 +187,13 @@ func (c *HTTPAdminClient) DemoteVoter(ctx context.Context, target raft.ServerAdd return &out, nil } -func (c *HTTPAdminClient) GetConfiguration(ctx context.Context, target raft.ServerAddress) (*GetConfigurationResponse, error) { +func (c *Client) GetConfiguration(ctx context.Context, target raft.ServerAddress) (*GetConfigurationResponse, error) { url := c.generateUrl(target, "GetConfiguration") r, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -218,13 +209,13 @@ func (c *HTTPAdminClient) GetConfiguration(ctx context.Context, target raft.Serv return &out, nil } -func (c *HTTPAdminClient) LastContact(ctx context.Context, target raft.ServerAddress) (*LastContactResponse, error) { +func (c *Client) LastContact(ctx context.Context, target raft.ServerAddress) (*LastContactResponse, error) { url := c.generateUrl(target, "LastContact") r, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -240,13 +231,13 @@ func (c *HTTPAdminClient) LastContact(ctx context.Context, target raft.ServerAdd return &out, nil } -func (c *HTTPAdminClient) LastIndex(ctx context.Context, target raft.ServerAddress) (*LastIndexResponse, error) { +func (c *Client) LastIndex(ctx context.Context, target raft.ServerAddress) (*LastIndexResponse, error) { url := c.generateUrl(target, "LastIndex") r, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -264,13 +255,13 @@ func (c *HTTPAdminClient) LastIndex(ctx context.Context, target raft.ServerAddre var ErrNotLeader = errors.New("not leader") -func (c *HTTPAdminClient) Leader(ctx context.Context, target raft.ServerAddress) (*LeaderResponse, error) { +func (c *Client) Leader(ctx context.Context, target raft.ServerAddress) (*LeaderResponse, error) { url := c.generateUrl(target, "Leader") r, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -290,7 +281,7 @@ func (c *HTTPAdminClient) Leader(ctx context.Context, target raft.ServerAddress) } } -func (c *HTTPAdminClient) LeadershipTransfer(ctx context.Context, target raft.ServerAddress, req *LeadershipTransferToServerRequest) (*AwaitResponse, error) { +func (c *Client) LeadershipTransfer(ctx context.Context, target raft.ServerAddress, req *LeadershipTransferToServerRequest) (*AwaitResponse, error) { url := c.generateUrl(target, "LeadershipTransfer") buf, err := json.Marshal(req) if err != nil { @@ -300,7 +291,7 @@ func (c *HTTPAdminClient) LeadershipTransfer(ctx context.Context, target raft.Se if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -320,7 +311,7 @@ func (c *HTTPAdminClient) LeadershipTransfer(ctx context.Context, target raft.Se } } -func (c *HTTPAdminClient) RemoveServer(ctx context.Context, target raft.ServerAddress, req *RemoveServerRequest) (*AwaitResponse, error) { +func (c *Client) RemoveServer(ctx context.Context, target raft.ServerAddress, req *RemoveServerRequest) (*AwaitResponse, error) { url := c.generateUrl(target, "RemoveServer") buf, err := json.Marshal(req) if err != nil { @@ -330,7 +321,7 @@ func (c *HTTPAdminClient) RemoveServer(ctx context.Context, target raft.ServerAd if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -350,13 +341,13 @@ func (c *HTTPAdminClient) RemoveServer(ctx context.Context, target raft.ServerAd } } -func (c *HTTPAdminClient) Shutdown(ctx context.Context, target raft.ServerAddress) (*AwaitResponse, error) { +func (c *Client) Shutdown(ctx context.Context, target raft.ServerAddress) (*AwaitResponse, error) { url := c.generateUrl(target, "Shutdown") r, err := http.NewRequest("POST", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -376,13 +367,13 @@ func (c *HTTPAdminClient) Shutdown(ctx context.Context, target raft.ServerAddres } } -func (c *HTTPAdminClient) State(ctx context.Context, target raft.ServerAddress) (*StateResponse, error) { +func (c *Client) State(ctx context.Context, target raft.ServerAddress) (*StateResponse, error) { url := c.generateUrl(target, "State") r, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -402,13 +393,13 @@ func (c *HTTPAdminClient) State(ctx context.Context, target raft.ServerAddress) } } -func (c *HTTPAdminClient) Stats(ctx context.Context, target raft.ServerAddress) (*StatsResponse, error) { +func (c *Client) Stats(ctx context.Context, target raft.ServerAddress) (*StatsResponse, error) { url := c.generateUrl(target, "Stats") r, err := http.NewRequest("POST", url, nil) if err != nil { return nil, err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return nil, err } @@ -428,13 +419,13 @@ func (c *HTTPAdminClient) Stats(ctx context.Context, target raft.ServerAddress) } } -func (c *HTTPAdminClient) VerifyLeader(ctx context.Context, target raft.ServerAddress) error { +func (c *Client) VerifyLeader(ctx context.Context, target raft.ServerAddress) error { url := c.generateUrl(target, "VerifyLeader") r, err := http.NewRequest("POST", url, nil) if err != nil { return err } - res, err := c.clientRetry(r) + res, err := c.clientRetry(ctx, r) if err != nil { return err } @@ -449,15 +440,17 @@ func (c *HTTPAdminClient) VerifyLeader(ctx context.Context, target raft.ServerAd } } -func (c *HTTPAdminClient) clientRetry(r *http.Request) (*http.Response, error) { +func (c *Client) clientRetry(ctx context.Context, r *http.Request) (*http.Response, error) { retries := 0 + r = r.WithContext(ctx) RETRY: res, err := c.do(r) if err != nil { - if retries > 5 { + if retries > 3 { return nil, err + } else if ctx.Err() != nil { + return nil, ctx.Err() } - <-time.After(1 * time.Second) retries++ goto RETRY } diff --git a/pkg/raftadmin/raftadmin.go b/pkg/raftadmin/server.go similarity index 82% rename from pkg/raftadmin/raftadmin.go rename to pkg/raftadmin/server.go index 168424b..5d43ecb 100644 --- a/pkg/raftadmin/raftadmin.go +++ b/pkg/raftadmin/server.go @@ -16,20 +16,19 @@ import ( "go.uber.org/zap" ) -// RaftAdminHTTPServer provides a HTTP-based transport that can be used to +// Server provides a HTTP-based transport that can be used to // communicate with Raft on remote machines. It is convenient to use if your // application is an HTTP server already and you do not want to use multiple // different transports (if not, you can use raft.NetworkTransport). -type RaftAdminHTTPServer struct { +type Server struct { logger *zap.Logger r *raft.Raft - // addrs map[raft.ServerID]raft.ServerAddress addrs []raft.ServerAddress } -// NewRaftAdminHTTPServer creates a new HTTP transport on the given addr. -func NewRaftAdminHTTPServer(r *raft.Raft, logger *zap.Logger, addrs []raft.ServerAddress) *RaftAdminHTTPServer { - return &RaftAdminHTTPServer{ +// NewServer creates a new HTTP transport on the given addr. +func NewServer(r *raft.Raft, logger *zap.Logger, addrs []raft.ServerAddress) *Server { + return &Server{ logger: logger, r: r, addrs: addrs, @@ -71,7 +70,7 @@ func toFuture(f raft.Future) (*Future, error) { }, nil } -func (a *RaftAdminHTTPServer) Await(ctx context.Context, req *Future) (*AwaitResponse, error) { +func (a *Server) Await(ctx context.Context, req *Future) (*AwaitResponse, error) { mtx.Lock() f, ok := operations[req.OperationToken] defer func() { @@ -107,7 +106,7 @@ func (a *RaftAdminHTTPServer) Await(ctx context.Context, req *Future) (*AwaitRes return r, nil } -func (a *RaftAdminHTTPServer) Forget(ctx context.Context, req *Future) (*ForgetResponse, error) { +func (a *Server) Forget(ctx context.Context, req *Future) (*ForgetResponse, error) { mtx.Lock() delete(operations, req.OperationToken) mtx.Unlock() @@ -116,29 +115,29 @@ func (a *RaftAdminHTTPServer) Forget(ctx context.Context, req *Future) (*ForgetR }, nil } -func (a *RaftAdminHTTPServer) AddNonvoter(ctx context.Context, req *AddNonvoterRequest) (*Future, error) { +func (a *Server) AddNonvoter(ctx context.Context, req *AddNonvoterRequest) (*Future, error) { return toFuture(a.r.AddNonvoter(raft.ServerID(req.ID), raft.ServerAddress(req.Address), uint64(req.PrevIndex), timeout(ctx))) } -func (a *RaftAdminHTTPServer) AddVoter(ctx context.Context, req *AddVoterRequest) (*Future, error) { +func (a *Server) AddVoter(ctx context.Context, req *AddVoterRequest) (*Future, error) { return toFuture(a.r.AddVoter(raft.ServerID(req.ID), raft.ServerAddress(req.Address), uint64(req.PrevIndex), timeout(ctx))) } -func (a *RaftAdminHTTPServer) AppliedIndex(ctx context.Context) (*AppliedIndexResponse, error) { +func (a *Server) AppliedIndex(ctx context.Context) (*AppliedIndexResponse, error) { return &AppliedIndexResponse{ Index: a.r.AppliedIndex(), }, nil } -func (a *RaftAdminHTTPServer) Barrier(ctx context.Context) (*Future, error) { +func (a *Server) Barrier(ctx context.Context) (*Future, error) { return toFuture(a.r.Barrier(timeout(ctx))) } -func (a *RaftAdminHTTPServer) DemoteVoter(ctx context.Context, req *DemoteVoterRequest) (*Future, error) { +func (a *Server) DemoteVoter(ctx context.Context, req *DemoteVoterRequest) (*Future, error) { return toFuture(a.r.DemoteVoter(raft.ServerID(req.ID), req.PrevIndex, timeout(ctx))) } -func (a *RaftAdminHTTPServer) GetConfiguration(ctx context.Context) (*GetConfigurationResponse, error) { +func (a *Server) GetConfiguration(ctx context.Context) (*GetConfigurationResponse, error) { f := a.r.GetConfiguration() if err := f.Error(); err != nil { return nil, err @@ -162,24 +161,24 @@ func (a *RaftAdminHTTPServer) GetConfiguration(ctx context.Context) (*GetConfigu return resp, nil } -func (a *RaftAdminHTTPServer) LastContact(ctx context.Context) (*LastContactResponse, error) { +func (a *Server) LastContact(ctx context.Context) (*LastContactResponse, error) { t := a.r.LastContact() return &LastContactResponse{ UnixNano: t.UnixNano(), }, nil } -func (a *RaftAdminHTTPServer) LastIndex(ctx context.Context) (*LastIndexResponse, error) { +func (a *Server) LastIndex(ctx context.Context) (*LastIndexResponse, error) { return &LastIndexResponse{ Index: a.r.LastIndex(), }, nil } -func (a *RaftAdminHTTPServer) CurrentNodeIsLeader(ctx context.Context) bool { +func (a *Server) CurrentNodeIsLeader(ctx context.Context) bool { return a.r.State() == raft.Leader } -func (a *RaftAdminHTTPServer) Leader(ctx context.Context) (*LeaderResponse, error) { +func (a *Server) Leader(ctx context.Context) (*LeaderResponse, error) { for _, s := range a.r.GetConfiguration().Configuration().Servers { if s.Suffrage == raft.Voter && s.Address == a.r.Leader() { return &LeaderResponse{ @@ -193,27 +192,27 @@ func (a *RaftAdminHTTPServer) Leader(ctx context.Context) (*LeaderResponse, erro }, nil } -func (a *RaftAdminHTTPServer) LeadershipTransfer(ctx context.Context) (*Future, error) { +func (a *Server) LeadershipTransfer(ctx context.Context) (*Future, error) { return toFuture(a.r.LeadershipTransfer()) } -func (a *RaftAdminHTTPServer) LeadershipTransferToServer(ctx context.Context, req *LeadershipTransferToServerRequest) (*Future, error) { +func (a *Server) LeadershipTransferToServer(ctx context.Context, req *LeadershipTransferToServerRequest) (*Future, error) { return toFuture(a.r.LeadershipTransferToServer(raft.ServerID(req.ID), raft.ServerAddress(req.Address))) } -func (a *RaftAdminHTTPServer) RemoveServer(ctx context.Context, req *RemoveServerRequest) (*Future, error) { +func (a *Server) RemoveServer(ctx context.Context, req *RemoveServerRequest) (*Future, error) { return toFuture(a.r.RemoveServer(raft.ServerID(req.ID), req.PrevIndex, timeout(ctx))) } -func (a *RaftAdminHTTPServer) Shutdown(ctx context.Context) (*Future, error) { +func (a *Server) Shutdown(ctx context.Context) (*Future, error) { return toFuture(a.r.Shutdown()) } -func (a *RaftAdminHTTPServer) Snapshot(ctx context.Context) (*Future, error) { +func (a *Server) Snapshot(ctx context.Context) (*Future, error) { return toFuture(a.r.Snapshot()) } -func (a *RaftAdminHTTPServer) State(ctx context.Context) (*StateResponse, error) { +func (a *Server) State(ctx context.Context) (*StateResponse, error) { switch s := a.r.State(); s { case raft.Follower: return &StateResponse{State: RaftStateFollower}, nil @@ -228,7 +227,7 @@ func (a *RaftAdminHTTPServer) State(ctx context.Context) (*StateResponse, error) } } -func (a *RaftAdminHTTPServer) Stats(ctx context.Context) (*StatsResponse, error) { +func (a *Server) Stats(ctx context.Context) (*StatsResponse, error) { ret := &StatsResponse{} ret.Stats = map[string]string{} for k, v := range a.r.Stats() { @@ -237,12 +236,12 @@ func (a *RaftAdminHTTPServer) Stats(ctx context.Context) (*StatsResponse, error) return ret, nil } -func (a *RaftAdminHTTPServer) VerifyLeader(ctx context.Context) (*Future, error) { +func (a *Server) VerifyLeader(ctx context.Context) (*Future, error) { return toFuture(a.r.VerifyLeader()) } // ServeHTTP implements the net/http.Handler interface, so that you can use -func (t *RaftAdminHTTPServer) ServeHTTP(res http.ResponseWriter, req *http.Request) { +func (t *Server) ServeHTTP(res http.ResponseWriter, req *http.Request) { cmd := path.Base(req.URL.Path) if cmdRequiresLeader(cmd) && t.r.State() != raft.Leader { @@ -467,7 +466,7 @@ func cmdRequiresLeader(cmd string) bool { } } -func (t *RaftAdminHTTPServer) genericResponse(req *http.Request, res http.ResponseWriter, f *Future, cmd string) { +func (t *Server) genericResponse(req *http.Request, res http.ResponseWriter, f *Future, cmd string) { resp, err := t.Await(req.Context(), f) if err != nil { http.Error(res, err.Error(), http.StatusInternalServerError) diff --git a/pkg/raftadmin/raftadmin_test.go b/pkg/raftadmin/server_test.go similarity index 97% rename from pkg/raftadmin/raftadmin_test.go rename to pkg/raftadmin/server_test.go index b4eca87..44621e6 100644 --- a/pkg/raftadmin/raftadmin_test.go +++ b/pkg/raftadmin/server_test.go @@ -129,7 +129,7 @@ func setupRaftAdmin(t *testing.T) *httptest.Server { } <-time.After(time.Second * 5) - raftAdmin := NewRaftAdminHTTPServer( + raftAdmin := NewServer( raftNode, zap.NewNop(), []raft.ServerAddress{ "localhost:9090", @@ -169,10 +169,9 @@ func TestRaft(t *testing.T) { Return(mockClient.res, nil) ctx := context.Background() - client := NewHTTPAdminClient( + client := NewClient( server.Client().Do, - "http://(address)/raftadmin", - zap.NewNop(), + zap.NewNop(), "http", ) serverAddr := raft.ServerAddress(server.Listener.Addr().String()) leader, err := client.Leader(ctx, serverAddr) diff --git a/pkg/rafthttp/rafthttp.go b/pkg/rafthttp/rafthttp.go index d01ea35..17e9d9b 100644 --- a/pkg/rafthttp/rafthttp.go +++ b/pkg/rafthttp/rafthttp.go @@ -32,27 +32,25 @@ type HTTPTransport struct { consumer chan raft.RPC addr raft.ServerAddress client Doer - urlFmt string + scheme string } var _ raft.Transport = (*HTTPTransport)(nil) +var _ raft.WithPreVote = (*HTTPTransport)(nil) -func NewHTTPTransport(addr raft.ServerAddress, client Doer, logger *zap.Logger, urlFmt string) *HTTPTransport { +func NewHTTPTransport(addr raft.ServerAddress, client Doer, logger *zap.Logger, scheme string) *HTTPTransport { if client == nil { client = http.DefaultClient } - if !strings.Contains(urlFmt, "(address)") { - panic("urlFmt must contain the string '(address)'") - } - if !strings.HasSuffix(urlFmt, "/") { - urlFmt += "/" + if scheme == "" { + scheme = "http" } return &HTTPTransport{ logger: logger, consumer: make(chan raft.RPC), addr: addr, client: client, - urlFmt: urlFmt, + scheme: scheme, } } @@ -99,8 +97,9 @@ RETRY: } func (t *HTTPTransport) generateUrl(target raft.ServerAddress, action string) string { - return strings.ReplaceAll(t.urlFmt+action, - "(address)", string(target)) + uri := fmt.Sprintf("%s://%s/raft/%s", t.scheme, target, action) + // t.logger.Debug("rafthttp: generated url", zap.String("url", uri)) + return uri } // Consumer implements the raft.Transport interface. @@ -132,6 +131,11 @@ func (t *HTTPTransport) RequestVote(_ raft.ServerID, target raft.ServerAddress, return t.send(t.generateUrl(target, "RequestVote"), args, resp) } +// RequestPreVote implements the raft.Transport interface. +func (t *HTTPTransport) RequestPreVote(_ raft.ServerID, target raft.ServerAddress, args *raft.RequestPreVoteRequest, resp *raft.RequestPreVoteResponse) error { + return t.send(t.generateUrl(target, "RequestPreVote"), args, resp) +} + // InstallSnapshot implements the raft.Transport interface. func (t *HTTPTransport) InstallSnapshot(_ raft.ServerID, target raft.ServerAddress, args *raft.InstallSnapshotRequest, resp *raft.InstallSnapshotResponse, data io.Reader) error { // Send a dummy request to see if the remote host supports @@ -280,6 +284,8 @@ func (t *HTTPTransport) ServeHTTP(res http.ResponseWriter, req *http.Request) { return case "RequestVote": rpc.Command = &raft.RequestVoteRequest{} + case "RequestPreVote": + rpc.Command = &raft.RequestPreVoteRequest{} case "AppendEntries": rpc.Command = &raft.AppendEntriesRequest{} case "TimeoutNow": diff --git a/pkg/rafthttp/rafthttp_test.go b/pkg/rafthttp/rafthttp_test.go index 6656096..b1cfe40 100644 --- a/pkg/rafthttp/rafthttp_test.go +++ b/pkg/rafthttp/rafthttp_test.go @@ -44,8 +44,8 @@ func TestExample(t *testing.T) { log.Printf("Listening on %s", ln.Addr().String()) srvAddr := raft.ServerAddress(ln.Addr().String()) transport := rafthttp.NewHTTPTransport( - srvAddr, http.DefaultClient, zap.NewNop(), - "http://(address)/raft", + srvAddr, http.DefaultClient, + zap.NewNop(), "http", ) srv := &http.Server{ Handler: transport, diff --git a/pkg/storage/debug_storage.go b/pkg/storage/debug_storage.go deleted file mode 100644 index e74eb0b..0000000 --- a/pkg/storage/debug_storage.go +++ /dev/null @@ -1,93 +0,0 @@ -package storage - -import ( - "errors" - "strings" - - "github.com/dgate-io/dgate/pkg/util/tree/avl" - "go.uber.org/zap" -) - -type DebugStoreConfig struct { - Logger *zap.Logger -} - -type DebugStore struct { - tree avl.Tree[string, []byte] -} - -var _ Storage = &DebugStore{} - -func NewDebugStore(cfg *DebugStoreConfig) *DebugStore { - return &DebugStore{ - tree: avl.NewTree[string, []byte](), - } -} - -func (m *DebugStore) Connect() error { - return nil -} - -func (m *DebugStore) Get(key string) ([]byte, error) { - if b, ok := m.tree.Find(key); ok { - return b, nil - } - return nil, errors.New("key not found") -} - -func (m *DebugStore) Set(key string, value []byte) error { - m.tree.Insert(key, value) - return nil -} - -func (m *DebugStore) IterateValuesPrefix(prefix string, fn func(string, []byte) error) error { - check := true - m.tree.Each(func(k string, v []byte) bool { - if strings.HasPrefix(k, prefix) { - check = true - if err := fn(k, v); err != nil { - return false - } - return true - } - return check - }) - return nil -} - -func (m *DebugStore) IterateTxnPrefix(prefix string, fn func(StorageTxn, string) error) error { - panic("implement me") -} - -func (m *DebugStore) GetPrefix(prefix string, offset, limit int) ([]*KeyValue, error) { - if limit <= 0 { - limit = 0 - } - kvs := make([]*KeyValue, 0, limit) - m.IterateValuesPrefix(prefix, func(key string, value []byte) error { - if offset <= 0 { - if len(kvs) >= limit { - return errors.New("limit reached") - } - kvs = append(kvs, &KeyValue{ - Key: key, - Value: value, - }) - } else { - offset-- - } - return nil - }) - return kvs, nil -} - -func (m *DebugStore) Delete(key string) error { - if ok := m.tree.Delete(key); !ok { - return errors.New("key not found") - } - return nil -} - -func (m *DebugStore) Close() error { - return nil -} diff --git a/pkg/storage/file_storage.go b/pkg/storage/file_storage.go index 3fd417b..aa84b4c 100644 --- a/pkg/storage/file_storage.go +++ b/pkg/storage/file_storage.go @@ -1,13 +1,14 @@ package storage import ( + "bytes" "errors" "fmt" "os" + "path" "strings" - "github.com/dgraph-io/badger/v4" - "github.com/dgraph-io/badger/v4/options" + bolt "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -17,25 +18,22 @@ type FileStoreConfig struct { } type FileStore struct { - directory string - logger badger.Logger - inMemory bool - db *badger.DB + logger *zap.Logger + bucketName []byte + directory string + db *bolt.DB } type FileStoreTxn struct { - txn *badger.Txn - ro bool + txn *bolt.Tx + ro bool + bucket *bolt.Bucket } -var _ Storage = &FileStore{} -var _ StorageTxn = &FileStoreTxn{} +var _ Storage = (*FileStore)(nil) +var _ StorageTxn = (*FileStoreTxn)(nil) var ( - // ErrStoreLocked is returned when the storage is locked. - ErrStoreLocked error = errors.New("storage is locked") - // ErrKeyNotFound is returned when the key is not found. - ErrKeyNotFound error = errors.New("key not found") // ErrTxnReadOnly is returned when the transaction is read only. ErrTxnReadOnly error = errors.New("transaction is read only") ) @@ -51,115 +49,98 @@ func NewFileStore(fsConfig *FileStoreConfig) *FileStore { fsConfig.Directory = strings.TrimSuffix(fsConfig.Directory, "/") } - return &FileStore{ - directory: fsConfig.Directory, - logger: newBadgerLoggerAdapter( - "filestore::badger", fsConfig.Logger, - ), - inMemory: false, + if fsConfig.Logger == nil { + fsConfig.Logger = zap.NewNop() } -} -func newFileStoreTxn(txn *badger.Txn) *FileStoreTxn { - return &FileStoreTxn{ - txn: txn, + return &FileStore{ + directory: fsConfig.Directory, + logger: fsConfig.Logger.Named("boltstore::bolt"), + bucketName: []byte("dgate"), } } -func (s *FileStore) Connect() error { - var opts badger.Options - var err error - if s.inMemory { - opts = badger.DefaultOptions(""). - WithCompression(options.Snappy). - WithInMemory(true). - WithLogger(s.logger) +func (s *FileStore) Connect() (err error) { + if err = os.MkdirAll(s.directory, 0755); err != nil { + return err + } + filePath := path.Join(s.directory, "dgate.db") + if s.db, err = bolt.Open(filePath, 0755, nil); err != nil { + return err + } else if tx, err := s.db.Begin(true); err != nil { + return err } else { - // Create the directory if it does not exist. - if _, err := os.Stat(s.directory); os.IsNotExist(err) { - err := os.MkdirAll(s.directory, 0755) - if err != nil { - return errors.New("failed to create directory - " + s.directory + ": " + err.Error()) - } + _, err = tx.CreateBucketIfNotExists(s.bucketName) + if err != nil { + return err } + return tx.Commit() + } +} - opts = badger.DefaultOptions(s.directory). - WithReadOnly(false). - WithInMemory(s.inMemory). - WithCompression(options.Snappy). - WithLogger(s.logger) +func (s *FileStore) Txn(write bool, fn func(StorageTxn) error) error { + txFn := func(txn *bolt.Tx) (err error) { + return fn(s.newTxn(txn)) } - s.db, err = badger.Open(opts) - if err != nil { - return err + if write { + return s.db.Update(txFn) } - return nil + return s.db.View(txFn) +} + +func (s *FileStore) newTxn(txn *bolt.Tx) *FileStoreTxn { + if bucket := txn.Bucket(s.bucketName); bucket != nil { + return &FileStoreTxn{ + txn: txn, + bucket: bucket, + } + } + panic("bucket not found") } func (s *FileStore) Get(key string) ([]byte, error) { var value []byte - err := s.db.View(func(txn *badger.Txn) error { - val, err := newFileStoreTxn(txn).Get(key) - if err != nil { - if err == badger.ErrKeyNotFound { - return ErrKeyNotFound - } - return err - } - value = val - return nil + return value, s.db.View(func(txn *bolt.Tx) (err error) { + value, err = s.newTxn(txn).Get(key) + return err }) - return value, err } func (s *FileStore) Set(key string, value []byte) error { - return s.db.Update(func(txn *badger.Txn) error { - return newFileStoreTxn(txn).Set(key, value) + return s.db.Update(func(txn *bolt.Tx) error { + return s.newTxn(txn).Set(key, value) + }) +} + +func (s *FileStore) Delete(key string) error { + return s.db.Update(func(txn *bolt.Tx) error { + return s.newTxn(txn).Delete(key) }) } func (s *FileStore) IterateValuesPrefix(prefix string, fn func(string, []byte) error) error { - return s.db.View(func(txn *badger.Txn) error { - return newFileStoreTxn(txn).IterateValuesPrefix(prefix, fn) + return s.db.View(func(txn *bolt.Tx) error { + return s.newTxn(txn).IterateValuesPrefix(prefix, fn) }) } func (s *FileStore) IterateTxnPrefix(prefix string, fn func(StorageTxn, string) error) error { - return s.db.View(func(txn *badger.Txn) error { - return newFileStoreTxn(txn).IterateTxnPrefix(prefix, fn) + return s.db.Update(func(txn *bolt.Tx) error { + return s.newTxn(txn).IterateTxnPrefix(prefix, fn) }) } func (s *FileStore) GetPrefix(prefix string, offset, limit int) ([]*KeyValue, error) { - var return_list []*KeyValue - err := s.db.View(func(txn *badger.Txn) error { - val, err := newFileStoreTxn(txn).GetPrefix(prefix, offset, limit) + var list []*KeyValue + err := s.db.View(func(txn *bolt.Tx) error { + val, err := s.newTxn(txn).GetPrefix(prefix, offset, limit) if err != nil { return fmt.Errorf("failed to get prefix: %w", err) } - return_list = val + list = val return nil }) - return return_list, err -} - -func (s *FileStore) Delete(key string) error { - return s.db.Update(func(txn *badger.Txn) error { - return txn.Delete([]byte(key)) - }) -} - -func (s *FileStore) Txn(readOnly bool, fn func(StorageTxn) error) error { - txFunc := s.db.View - if !readOnly { - txFunc = s.db.Update - } - return txFunc(func(txn *badger.Txn) error { - return fn(&FileStoreTxn{ - txn: txn, - ro: readOnly, - }) - }) + return list, err } func (s *FileStore) Close() error { @@ -167,43 +148,28 @@ func (s *FileStore) Close() error { } func (tx *FileStoreTxn) Get(key string) ([]byte, error) { - item, err := tx.txn.Get([]byte(key)) - if err != nil { - return nil, err - } - val, err := item.ValueCopy(nil) - if err != nil { - return nil, err - } - return val, nil + return tx.bucket.Get([]byte(key)), nil } func (tx *FileStoreTxn) Set(key string, value []byte) error { if tx.ro { return ErrTxnReadOnly } - return tx.txn.Set([]byte(key), value) + return tx.bucket.Put([]byte(key), value) } func (tx *FileStoreTxn) Delete(key string) error { if tx.ro { return ErrTxnReadOnly } - return tx.txn.Delete([]byte(key)) + return tx.bucket.Delete([]byte(key)) } func (tx *FileStoreTxn) IterateValuesPrefix(prefix string, fn func(string, []byte) error) error { - iter := tx.txn.NewIterator(badger.IteratorOptions{ - Prefix: []byte(prefix), - }) - defer iter.Close() - for iter.Rewind(); iter.Valid(); iter.Next() { - item := iter.Item() - val, err := item.ValueCopy(nil) - if err != nil { - return err - } - if err := fn(string(item.Key()), val); err != nil { + c := tx.bucket.Cursor() + pre := []byte(prefix) + for k, v := c.Seek(pre); bytes.HasPrefix(k, pre); k, v = c.Next() { + if err := fn(string(k), v); err != nil { return err } } @@ -211,13 +177,10 @@ func (tx *FileStoreTxn) IterateValuesPrefix(prefix string, fn func(string, []byt } func (tx *FileStoreTxn) IterateTxnPrefix(prefix string, fn func(StorageTxn, string) error) error { - iter := tx.txn.NewIterator(badger.IteratorOptions{ - Prefix: []byte(prefix), - }) - defer iter.Close() - for iter.Rewind(); iter.Valid(); iter.Next() { - item := iter.Item() - if err := fn(tx, string(item.Key())); err != nil { + c := tx.bucket.Cursor() + pre := []byte(prefix) + for k, _ := c.Seek(pre); bytes.HasPrefix(k, pre); k, _ = c.Next() { + if err := fn(tx, string(k)); err != nil { return err } } @@ -225,29 +188,19 @@ func (tx *FileStoreTxn) IterateTxnPrefix(prefix string, fn func(StorageTxn, stri } func (s *FileStoreTxn) GetPrefix(prefix string, offset, limit int) ([]*KeyValue, error) { - return_list := make([]*KeyValue, 0) - iter := s.txn.NewIterator(badger.IteratorOptions{ - Prefix: []byte(prefix), - }) - defer iter.Close() - for iter.Rewind(); iter.Valid(); iter.Next() { + list := make([]*KeyValue, 0) + c := s.bucket.Cursor() + pre := []byte(prefix) + for k, v := c.Seek(pre); bytes.HasPrefix(k, pre); k, v = c.Next() { if offset > 0 { - offset -= 1 + offset-- continue } - item := iter.Item() - val, err := item.ValueCopy(nil) - if err != nil { - return nil, fmt.Errorf("error copying value: %v", err) - } - return_list = append(return_list, &KeyValue{ - Key: string(item.Key()), - Value: val, - }) - if limit -= 1; limit == 0 { + if limit == 0 { break } + list = append(list, &KeyValue{Key: string(k), Value: v}) + limit-- } - - return return_list, nil + return list, nil } diff --git a/pkg/storage/mem_storage.go b/pkg/storage/mem_storage.go new file mode 100644 index 0000000..f273c51 --- /dev/null +++ b/pkg/storage/mem_storage.go @@ -0,0 +1,141 @@ +package storage + +import ( + "errors" + "strings" + + "github.com/dgate-io/dgate/pkg/util/tree/avl" + "go.uber.org/zap" +) + +type MemStoreConfig struct { + Logger *zap.Logger +} + +type MemStore struct { + tree avl.Tree[string, []byte] +} + +type MemStoreTxn struct { + store *MemStore +} + +var _ Storage = &MemStore{} +var _ StorageTxn = &MemStoreTxn{} + +func NewMemStore(cfg *MemStoreConfig) *MemStore { + return &MemStore{ + tree: avl.NewTree[string, []byte](), + } +} + +func (m *MemStore) Connect() error { + return nil +} + +func (m *MemStore) Get(key string) ([]byte, error) { + if b, ok := m.tree.Find(key); ok { + return b, nil + } + return nil, errors.New("key not found") +} + +func (m *MemStore) Set(key string, value []byte) error { + m.tree.Insert(key, value) + return nil +} + +func (m *MemStore) Txn(write bool, fn func(StorageTxn) error) error { + txn := &MemStoreTxn{store: m} + if err := fn(txn); err != nil { + return err + } + return nil +} + +func (m *MemStore) IterateValuesPrefix(prefix string, fn func(string, []byte) error) error { + check := true + m.tree.Each(func(k string, v []byte) bool { + if strings.HasPrefix(k, prefix) { + check = true + if err := fn(k, v); err != nil { + return false + } + return true + } + return check + }) + return nil +} + +func (m *MemStore) IterateTxnPrefix(prefix string, fn func(StorageTxn, string) error) error { + m.tree.Each(func(k string, v []byte) bool { + if strings.HasPrefix(k, prefix) { + txn := &MemStoreTxn{ + store: m, + } + if err := fn(txn, k); err != nil { + return false + } + } + return true + }) + return nil +} + +func (m *MemStore) GetPrefix(prefix string, offset, limit int) ([]*KeyValue, error) { + if limit <= 0 { + limit = 0 + } + kvs := make([]*KeyValue, 0, limit) + m.IterateValuesPrefix(prefix, func(key string, value []byte) error { + if offset <= 0 { + if len(kvs) >= limit { + return errors.New("limit reached") + } + kvs = append(kvs, &KeyValue{ + Key: key, + Value: value, + }) + } else { + offset-- + } + return nil + }) + return kvs, nil +} + +func (m *MemStore) Delete(key string) error { + if ok := m.tree.Delete(key); !ok { + return errors.New("key not found") + } + return nil +} + +func (m *MemStore) Close() error { + return nil +} + +func (t *MemStoreTxn) Get(key string) ([]byte, error) { + return t.store.Get(key) +} + +func (t *MemStoreTxn) Set(key string, value []byte) error { + return t.store.Set(key, value) +} + +func (t *MemStoreTxn) Delete(key string) error { + return t.store.Delete(key) +} + +func (t *MemStoreTxn) GetPrefix(prefix string, offset int, limit int) ([]*KeyValue, error) { + return t.store.GetPrefix(prefix, offset, limit) +} + +func (t *MemStoreTxn) IterateTxnPrefix(prefix string, fn func(txn StorageTxn, key string) error) error { + return t.store.IterateTxnPrefix(prefix, fn) +} + +func (t *MemStoreTxn) IterateValuesPrefix(prefix string, fn func(key string, val []byte) error) error { + return t.store.IterateValuesPrefix(prefix, fn) +} diff --git a/pkg/storage/memory_storage.go b/pkg/storage/memory_storage.go deleted file mode 100644 index 722e774..0000000 --- a/pkg/storage/memory_storage.go +++ /dev/null @@ -1,27 +0,0 @@ -package storage - -import "go.uber.org/zap" - -type MemoryStoreConfig struct { - // Path to the directory where the files will be stored. - // If the directory does not exist, it will be created. - // If the directory exists, it will be used. - Logger *zap.Logger -} - -type MemoryStore struct { - *FileStore -} - -var _ Storage = &MemoryStore{} - -func NewMemoryStore(cfg *MemoryStoreConfig) *MemoryStore { - return &MemoryStore{ - FileStore: &FileStore{ - inMemory: true, - logger: newBadgerLoggerAdapter( - "memstore::badger", cfg.Logger, - ), - }, - } -} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 2ebc9cc..90a08ce 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -2,6 +2,7 @@ package storage type Storage interface { StorageTxn + Txn(write bool, fn func(txn StorageTxn) error) error Connect() error Close() error } diff --git a/pkg/typescript/typescript.go b/pkg/typescript/typescript.go index 3b020b2..c0f9838 100644 --- a/pkg/typescript/typescript.go +++ b/pkg/typescript/typescript.go @@ -1,7 +1,9 @@ package typescript import ( + "context" _ "embed" + "strings" "github.com/clarkmcc/go-typescript" "github.com/dop251/goja" @@ -12,9 +14,11 @@ import ( //go:embed typescript.min.js var tscSource string -func Transpile(src string) (string, error) { +func Transpile(ctx context.Context, src string) (string, error) { + srcReader := strings.NewReader(src) // transpiles TS into JS with commonjs module and targets es5 - return typescript.TranspileString(src, + return typescript.TranspileCtx( + ctx, srcReader, WithCachedTypescriptSource(), typescript.WithPreventCancellation(), typescript.WithCompileOptions(map[string]any{ diff --git a/pkg/typescript/typescript_test.go b/pkg/typescript/typescript_test.go index 7769477..a262fc7 100644 --- a/pkg/typescript/typescript_test.go +++ b/pkg/typescript/typescript_test.go @@ -1,6 +1,7 @@ package typescript_test import ( + "context" "strings" "testing" @@ -37,7 +38,7 @@ func TestTranspile(t *testing.T) { for _, tsSrc := range tsSrcList { vm := goja.New() - jsSrc, err := typescript.Transpile(tsSrc) + jsSrc, err := typescript.Transpile(context.Background(), tsSrc) if err != nil { t.Fatal(err) return diff --git a/pkg/util/http_test.go b/pkg/util/http_test.go index 46d1e19..0073bfc 100644 --- a/pkg/util/http_test.go +++ b/pkg/util/http_test.go @@ -6,38 +6,38 @@ import ( "testing" "github.com/dgate-io/dgate/pkg/util" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" ) func TestGetTrustedIP_Depth(t *testing.T) { req := requestWithXForwardedFor(t, "1.2.3.4", "1.2.3.5", "1.2.3.6") t.Run("Depth 0", func(t *testing.T) { - require.Equal(t, util.GetTrustedIP(req, 0), "127.0.0.1") + assert.Equal(t, util.GetTrustedIP(req, 0), "127.0.0.1") }) t.Run("Depth 1", func(t *testing.T) { - require.Equal(t, util.GetTrustedIP(req, 1), "1.2.3.6") + assert.Equal(t, util.GetTrustedIP(req, 1), "1.2.3.6") }) t.Run("Depth 2", func(t *testing.T) { - require.Equal(t, util.GetTrustedIP(req, 2), "1.2.3.5") + assert.Equal(t, util.GetTrustedIP(req, 2), "1.2.3.5") }) t.Run("Depth 3", func(t *testing.T) { - require.Equal(t, util.GetTrustedIP(req, 3), "1.2.3.4") + assert.Equal(t, util.GetTrustedIP(req, 3), "1.2.3.4") }) t.Run("Depth too High", func(t *testing.T) { - require.Equal(t, util.GetTrustedIP(req, 4), "1.2.3.4") - require.Equal(t, util.GetTrustedIP(req, 8), "1.2.3.4") - require.Equal(t, util.GetTrustedIP(req, 16), "1.2.3.4") + assert.Equal(t, util.GetTrustedIP(req, 4), "1.2.3.4") + assert.Equal(t, util.GetTrustedIP(req, 8), "1.2.3.4") + assert.Equal(t, util.GetTrustedIP(req, 16), "1.2.3.4") }) t.Run("Depth too Low", func(t *testing.T) { - require.Equal(t, util.GetTrustedIP(req, -1), "127.0.0.1") - require.Equal(t, util.GetTrustedIP(req, -10), "127.0.0.1") - require.Equal(t, util.GetTrustedIP(req, math.MinInt), "127.0.0.1") + assert.Equal(t, util.GetTrustedIP(req, -1), "127.0.0.1") + assert.Equal(t, util.GetTrustedIP(req, -10), "127.0.0.1") + assert.Equal(t, util.GetTrustedIP(req, math.MinInt), "127.0.0.1") }) } diff --git a/pkg/util/parse.go b/pkg/util/parse.go index 7787725..1e2c683 100644 --- a/pkg/util/parse.go +++ b/pkg/util/parse.go @@ -1,6 +1,9 @@ package util -import "strconv" +import ( + "strconv" + "time" +) func ParseInt(s string, def int) (int, error) { if s == "" { @@ -12,3 +15,12 @@ func ParseInt(s string, def int) (int, error) { return def, err } } + +func ParseBase36Timestamp(s string) (time.Time, error) { + if i, err := strconv.ParseInt(s, 36, 64); err == nil { + + return time.Unix(0, i), nil + } else { + return time.Time{}, err + } +} diff --git a/pkg/util/parse_test.go b/pkg/util/parse_test.go new file mode 100644 index 0000000..74cdf10 --- /dev/null +++ b/pkg/util/parse_test.go @@ -0,0 +1,20 @@ +package util_test + +import ( + "strconv" + "testing" + "time" + + "github.com/dgate-io/dgate/pkg/util" + "github.com/stretchr/testify/assert" +) + +func TestParseBase36Timestamp(t *testing.T) { + originalTime := time.Unix(0, time.Now().UnixNano()) + base36String := strconv.FormatInt(originalTime.UnixNano(), 36) + if parsedTime, err := util.ParseBase36Timestamp(base36String); err != nil { + t.Errorf("unexpected error: %v", err) + } else { + assert.Equal(t, parsedTime, originalTime) + } +} diff --git a/pkg/util/queue/queue.go b/pkg/util/queue/queue.go index 348c74d..a8988e7 100644 --- a/pkg/util/queue/queue.go +++ b/pkg/util/queue/queue.go @@ -17,7 +17,14 @@ type queueImpl[V any] struct { } // New returns a new queue. -func New[V any]() Queue[V] { +func New[V any](vs ...V) Queue[V] { + if len(vs) > 0 { + q := newQueue[V](len(vs)) + for _, v := range vs { + q.Push(v) + } + return q + } return newQueue[V](128) } diff --git a/pkg/util/sliceutil/slice.go b/pkg/util/sliceutil/slice.go index f9c5569..1775a4a 100644 --- a/pkg/util/sliceutil/slice.go +++ b/pkg/util/sliceutil/slice.go @@ -67,3 +67,20 @@ func SliceCopy[T any](arr []T) []T { } return append([]T(nil), arr...) } + +// BinarySearch searches for a value in a sorted slice and returns the index of the value. +// If the value is not found, it returns -1 +func BinarySearch[T any](slice []T, val T, compare func(T, T) int) int { + low, high := 0, len(slice)-1 + for low <= high { + mid := low + (high-low)/2 + if i := compare(slice[mid], val); i == 0 { + return mid + } else if i > 0 { + high = mid - 1 + } else { + low = mid + 1 + } + } + return -1 +} diff --git a/pkg/util/sliceutil/slice_test.go b/pkg/util/sliceutil/slice_test.go new file mode 100644 index 0000000..91b7858 --- /dev/null +++ b/pkg/util/sliceutil/slice_test.go @@ -0,0 +1,114 @@ +package sliceutil_test + +import ( + "testing" + + "github.com/dgate-io/dgate/pkg/util/sliceutil" +) + +func TestBinarySearch(t *testing.T) { + tests := []struct { + name string + items []int + search int + expected int + iterations int + }{ + { + name: "empty", + items: []int{}, + search: 1, + expected: -1, + iterations: 0, + }, + { + name: "not found/1", + items: []int{1, 3, 5, 7, 9}, + search: 6, + expected: -1, + iterations: 2, + }, + { + name: "not found/2", + items: []int{1, 3, 5, 7, 9}, + search: 10, + expected: -1, + iterations: 3, + }, + { + name: "not found/3", + search: 6, + expected: -1, + iterations: 4, + items: []int{ + 1, 2, 3, 4, 5, + 7, 7, 8, 9, 10, + 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, + }, + }, + { + name: "found/1", + items: []int{1, 2, 3, 4, 5}, + search: 4, + expected: 3, + iterations: 2, + }, + { + name: "found/2", + search: 13, + expected: 12, + iterations: 4, + items: []int{ + 1, 2, 3, 4, 5, + 7, 7, 8, 9, 10, + 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iters := 0 + actual := sliceutil.BinarySearch(tt.items, tt.search, func(a, b int) int { + iters++ + return a - b + }) + if actual != tt.expected { + t.Errorf("expected %d, got %d", tt.expected, actual) + } + if iters != tt.iterations { + t.Errorf("expected %d iterations, got %d", tt.iterations, iters) + } + }) + } +} + +func BenchmarkCompareLinearAndBinarySearch(b *testing.B) { + items := make([]int, 1000000) + for i := range items { + items[i] = i + } + + b.Run("linear", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, item := range items { + if item == 999999 { + break + } + } + } + }) + + b.Run("binary", func(b *testing.B) { + for i := 0; i < b.N; i++ { + opts := 0 + sliceutil.BinarySearch(items, 999999, func(a, b int) int { + opts++ + return a - b + }) + b.ReportMetric(float64(opts), "opts/op") + } + }) +}