diff --git a/.github/workflows/discord.yml b/.github/workflows/discord.yml index c209ce5..600df2a 100644 --- a/.github/workflows/discord.yml +++ b/.github/workflows/discord.yml @@ -1,20 +1,20 @@ on: - release: - types: [published] + release: + types: [published] - jobs: - github-releases-to-discord: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Github Releases To Discord - uses: SethCohen/github-releases-to-discord@v1.13.1 - with: - webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }} - color: "2105893" - username: "Release Changelog" - avatar_url: "https://github.com/dgate-io.png" - content: "||@everyone||" - footer_title: "Changelog" - footer_timestamp: true \ No newline at end of file +jobs: + github-releases-to-discord: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Github Releases To Discord + uses: SethCohen/github-releases-to-discord@v1.13.1 + with: + webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }} + color: "2105893" + username: "Release Changelog" + avatar_url: "https://github.com/dgate-io.png" + content: "||@everyone||" + footer_title: "Changelog" + footer_timestamp: true \ No newline at end of file diff --git a/cmd/dgate-cli/commands/run_cmd.go b/cmd/dgate-cli/commands/run_cmd.go index e8e728b..0133915 100644 --- a/cmd/dgate-cli/commands/run_cmd.go +++ b/cmd/dgate-cli/commands/run_cmd.go @@ -22,9 +22,10 @@ func Run(client dgclient.DGateClient, version string) error { } app := &cli.App{ - Name: "dgate-cli", - Usage: "a command line interface for DGate (API Gateway) Admin API", - Version: version, + Name: "dgate-cli", + Usage: "a command line interface for DGate (API Gateway) Admin API", + Version: version, + UseShortOptionHandling: true, Flags: []cli.Flag{ &cli.StringFlag{ Name: "admin", diff --git a/functional-tests/raft_tests/test1.yaml b/functional-tests/raft_tests/test1.yaml index 8ce2e76..8aeaad4 100644 --- a/functional-tests/raft_tests/test1.yaml +++ b/functional-tests/raft_tests/test1.yaml @@ -1,5 +1,5 @@ version: v1 -log_level: info +log_level: debug debug: true tags: - "dev" diff --git a/internal/admin/changestate/change_state.go b/internal/admin/changestate/change_state.go index 8095fab..882288d 100644 --- a/internal/admin/changestate/change_state.go +++ b/internal/admin/changestate/change_state.go @@ -14,6 +14,7 @@ type ChangeState interface { WaitForChanges() error ReloadState(bool, ...*spec.ChangeLog) error ChangeHash() uint32 + ChangeLogs() []*spec.ChangeLog // Readiness Ready() bool diff --git a/internal/admin/changestate/testutil/change_state.go b/internal/admin/changestate/testutil/change_state.go index 8ecc33b..d199958 100644 --- a/internal/admin/changestate/testutil/change_state.go +++ b/internal/admin/changestate/testutil/change_state.go @@ -80,6 +80,11 @@ func (m *MockChangeState) WaitForChanges() error { return m.Called().Error(0) } +// ChangeLogs implements changestate.ChangeState. +func (m *MockChangeState) ChangeLogs() []*spec.ChangeLog { + return m.Called().Get(0).([]*spec.ChangeLog) +} + var _ changestate.ChangeState = &MockChangeState{} func NewMockChangeState() *MockChangeState { diff --git a/internal/admin/routes/collection_routes.go b/internal/admin/routes/collection_routes.go index 5e449fc..1a34c31 100644 --- a/internal/admin/routes/collection_routes.go +++ b/internal/admin/routes/collection_routes.go @@ -69,11 +69,9 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat return } - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } util.JsonResponse(w, http.StatusCreated, spec.TransformDGateCollections( @@ -276,11 +274,9 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat return } - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } util.JsonResponse(w, http.StatusCreated, doc) diff --git a/internal/admin/routes/domain_routes.go b/internal/admin/routes/domain_routes.go index cb5fc72..58ca807 100644 --- a/internal/admin/routes/domain_routes.go +++ b/internal/admin/routes/domain_routes.go @@ -47,11 +47,10 @@ func ConfigureDomainAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch return } - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return + } util.JsonResponse(w, http.StatusCreated, spec.TransformDGateDomains( rm.GetDomainsByNamespace(domain.NamespaceName)...)) diff --git a/internal/admin/routes/misc_routes.go b/internal/admin/routes/misc_routes.go index 7ed280f..636ffaa 100644 --- a/internal/admin/routes/misc_routes.go +++ b/internal/admin/routes/misc_routes.go @@ -12,13 +12,9 @@ import ( func ConfigureChangeLogAPI(server chi.Router, cs changestate.ChangeState, appConfig *config.DGateConfig) { server.Get("/changelog/hash", func(w http.ResponseWriter, r *http.Request) { - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } - // TODO: find a way to get the raft log hash - // perhaps generate based on current log commands and computed hash + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } if b, err := json.Marshal(map[string]any{ @@ -30,6 +26,21 @@ func ConfigureChangeLogAPI(server chi.Router, cs changestate.ChangeState, appCon w.Write([]byte(b)) } }) + server.Get("/changelog/count", func(w http.ResponseWriter, r *http.Request) { + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return + } + + if b, err := json.Marshal(map[string]any{ + "count": len(cs.ChangeLogs()), + }); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + } else { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(b)) + } + }) } func ConfigureHealthAPI(server chi.Router, version string, cs changestate.ChangeState) { diff --git a/internal/admin/routes/module_routes.go b/internal/admin/routes/module_routes.go index 62c6b35..5aea17a 100644 --- a/internal/admin/routes/module_routes.go +++ b/internal/admin/routes/module_routes.go @@ -47,12 +47,12 @@ func ConfigureModuleAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch util.JsonError(w, http.StatusBadRequest, err.Error()) return } - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } + util.JsonResponse(w, http.StatusCreated, spec.TransformDGateModules( rm.GetModulesByNamespace(mod.NamespaceName)...)) }) diff --git a/internal/admin/routes/namespace_routes.go b/internal/admin/routes/namespace_routes.go index 63d4783..2cadce7 100644 --- a/internal/admin/routes/namespace_routes.go +++ b/internal/admin/routes/namespace_routes.go @@ -41,11 +41,9 @@ func ConfigureNamespaceAPI(server chi.Router, logger *zap.Logger, cs changestate return } - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } util.JsonResponse(w, http.StatusCreated, spec.TransformDGateNamespaces(rm.GetNamespaces()...)) diff --git a/internal/admin/routes/route_routes.go b/internal/admin/routes/route_routes.go index f366b73..cf88c1c 100644 --- a/internal/admin/routes/route_routes.go +++ b/internal/admin/routes/route_routes.go @@ -49,11 +49,9 @@ func ConfigureRouteAPI(server chi.Router, logger *zap.Logger, cs changestate.Cha return } - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } util.JsonResponse(w, http.StatusCreated, spec.TransformDGateRoutes( diff --git a/internal/admin/routes/secret_routes.go b/internal/admin/routes/secret_routes.go index a6c004e..1d555c2 100644 --- a/internal/admin/routes/secret_routes.go +++ b/internal/admin/routes/secret_routes.go @@ -47,11 +47,9 @@ func ConfigureSecretAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch util.JsonError(w, http.StatusBadRequest, err.Error()) return } - if repl := cs.Raft(); repl != nil { - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } secrets := rm.GetSecretsByNamespace(sec.NamespaceName) util.JsonResponse(w, http.StatusCreated, diff --git a/internal/admin/routes/service_routes.go b/internal/admin/routes/service_routes.go index 4216291..4ea587f 100644 --- a/internal/admin/routes/service_routes.go +++ b/internal/admin/routes/service_routes.go @@ -64,13 +64,12 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C return } - if repl := cs.Raft(); repl != nil { - logger.Debug("Waiting for raft barrier") - if err := cs.WaitForChanges(); err != nil { - util.JsonError(w, http.StatusInternalServerError, err.Error()) - return - } + logger.Debug("Waiting for raft barrier") + if err := cs.WaitForChanges(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return } + svcs := rm.GetServicesByNamespace(svc.NamespaceName) util.JsonResponse(w, http.StatusCreated, spec.TransformDGateServices(svcs...)) diff --git a/internal/config/store_config.go b/internal/config/store_config.go index 7253c1c..808170c 100644 --- a/internal/config/store_config.go +++ b/internal/config/store_config.go @@ -25,4 +25,4 @@ func StoreConfig[T any, C any](config C) (T, error) { } err = decoder.Decode(config) return output, err -} \ No newline at end of file +} diff --git a/internal/pattern/pattern.go b/internal/pattern/pattern.go index fe869ba..2db756c 100644 --- a/internal/pattern/pattern.go +++ b/internal/pattern/pattern.go @@ -67,4 +67,4 @@ func singlefyAsterisks(pattern string) string { return singlefyAsterisks(pattern) } return pattern -} \ No newline at end of file +} diff --git a/internal/proxy/change_log.go b/internal/proxy/change_log.go index 9ec21e8..1fab512 100644 --- a/internal/proxy/change_log.go +++ b/internal/proxy/change_log.go @@ -17,7 +17,15 @@ import ( // processChangeLog - processes a change log and applies the change to the proxy state func (ps *ProxyState) processChangeLog(cl *spec.ChangeLog, reload, store bool) (err error) { defer func() { - ps.raftReady.Store(true) + if err != nil && !cl.Cmd.IsNoop() { + ps.ready.Store(true) + if changeHash, err := HashAny(ps.changeHash, cl); err != nil { + ps.logger.Error("error hashing change log", zap.Error(err)) + return + } else { + ps.changeHash = changeHash + } + } }() if !cl.Cmd.IsNoop() { if len(ps.changeLogs) > 0 { @@ -101,17 +109,6 @@ func (ps *ProxyState) processChangeLog(cl *spec.ChangeLog, reload, store bool) ( ps.logger.Error("Error registering change log", zap.Error(err)) return } - // update change log hash only when the change is successfully applied - // even if the change is a noop, we still need to update the hash - changeHash, err := HashAny(ps.changeHash, cl) - if err != nil { - if !ps.config.Debug { - return err - } - ps.logger.Error("error updating change log hash", zap.Error(err)) - } else { - ps.changeHash = changeHash - } } } if store { diff --git a/internal/proxy/dynamic_proxy.go b/internal/proxy/dynamic_proxy.go index 04d67d2..d635155 100644 --- a/internal/proxy/dynamic_proxy.go +++ b/internal/proxy/dynamic_proxy.go @@ -288,6 +288,8 @@ func (ps *ProxyState) Start() (err error) { if !ps.replicationEnabled { if err = ps.restoreFromChangeLogs(false); err != nil { return err + } else { + ps.ready.Store(true) } } diff --git a/internal/proxy/proxy_state.go b/internal/proxy/proxy_state.go index e0b10b7..0b0104b 100644 --- a/internal/proxy/proxy_state.go +++ b/internal/proxy/proxy_state.go @@ -34,19 +34,18 @@ import ( ) type ProxyState struct { - version string - debugMode bool - startTime time.Time - config *config.DGateConfig - logger *zap.Logger - printer console.Printer - store *proxystore.ProxyStore - proxyLock *sync.RWMutex - changeHash uint32 - changeLogs []*spec.ChangeLog - + version string + debugMode bool + changeHash uint32 + startTime time.Time + logger *zap.Logger + printer console.Printer + config *config.DGateConfig + store *proxystore.ProxyStore + changeLogs []*spec.ChangeLog metrics *ProxyMetrics sharedCache cache.TCache + proxyLock *sync.RWMutex rm *resources.ResourceManager skdr scheduler.Scheduler @@ -54,11 +53,10 @@ type ProxyState struct { providers avl.Tree[string, *RequestContextProvider] modPrograms avl.Tree[string, *goja.Program] - raftReady atomic.Bool + ready atomic.Bool replicationSettings *ProxyReplication replicationEnabled bool - - routers avl.Tree[string, *router.DynamicRouter] + routers avl.Tree[string, *router.DynamicRouter] ReverseProxyBuilder reverse_proxy.Builder ProxyTransportBuilder proxy_transport.Builder @@ -106,7 +104,7 @@ func NewProxyState(logger *zap.Logger, conf *config.DGateConfig) *ProxyState { } state := &ProxyState{ startTime: time.Now(), - raftReady: atomic.Bool{}, + ready: atomic.Bool{}, logger: logger, debugMode: conf.Debug, config: conf, @@ -168,9 +166,13 @@ func (ps *ProxyState) ChangeHash() uint32 { return ps.changeHash } +func (ps *ProxyState) ChangeLogs() []*spec.ChangeLog { + return ps.changeLogs +} + func (ps *ProxyState) Ready() bool { if ps.replicationEnabled { - return ps.raftReady.Load() + return ps.ready.Load() } return true } diff --git a/internal/router/router.go b/internal/router/router.go index bd4f5b6..f5cd3fc 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -21,6 +21,7 @@ func NewRouterWithMux(mux *chi.Mux) *DynamicRouter { func NewMux() *chi.Mux { return chi.NewRouter() } + // ReplaceRouter replaces the router func (r *DynamicRouter) ReplaceMux(router *chi.Mux) { r.lock.Lock() diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 196b62d..13fbb14 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -103,7 +103,6 @@ func TestCache_MaxItems_Overwrite(t *testing.T) { assert.True(t, ok, "expected key to be found") assert.Equal(t, 1, n, "expected value to be 1, got %d", n) - c.Bucket("test").SetWithTTL("key", 2, time.Millisecond*100) n, ok = c.Bucket("test").Get("key") assert.True(t, ok, "expected key to be found") diff --git a/pkg/dgclient/document_client_test.go b/pkg/dgclient/document_client_test.go index 2f01387..6390a47 100644 --- a/pkg/dgclient/document_client_test.go +++ b/pkg/dgclient/document_client_test.go @@ -17,7 +17,7 @@ func TestDGClient_GetDocument(t *testing.T) { w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(&dgclient.ResponseWrapper[*spec.Document]{ Data: &spec.Document{ - ID: "test", + ID: "test", CollectionName: "test", }, }) diff --git a/pkg/modules/dgate/crypto/crypto_mod.go b/pkg/modules/dgate/crypto/crypto_mod.go index 29303af..1b54732 100644 --- a/pkg/modules/dgate/crypto/crypto_mod.go +++ b/pkg/modules/dgate/crypto/crypto_mod.go @@ -60,7 +60,7 @@ func (c *CryptoModule) Exports() *modules.Exports { }, "hexEncode": c.hexEncode, } - + for k, v := range hashAlgos { namedExports[k] = v } diff --git a/pkg/modules/testutil/testutil.go b/pkg/modules/testutil/testutil.go index a0addf3..97a97cf 100644 --- a/pkg/modules/testutil/testutil.go +++ b/pkg/modules/testutil/testutil.go @@ -21,7 +21,7 @@ type mockRuntimeContext struct { mock.Mock smap *sync.Map ctx context.Context - req *require.Registry + req *require.Registry loop *eventloop.EventLoop data any state modules.StateManager diff --git a/pkg/modules/types/response_writer.go b/pkg/modules/types/response_writer.go index 4bda57f..37a446d 100644 --- a/pkg/modules/types/response_writer.go +++ b/pkg/modules/types/response_writer.go @@ -143,4 +143,3 @@ func (g *ResponseWriterWrapper) GetCookie(name string) *http.Cookie { } return nil } - diff --git a/pkg/rafthttp/rafthttp.go b/pkg/rafthttp/rafthttp.go index 7214a09..d01ea35 100644 --- a/pkg/rafthttp/rafthttp.go +++ b/pkg/rafthttp/rafthttp.go @@ -297,7 +297,6 @@ func (t *HTTPTransport) ServeHTTP(res http.ResponseWriter, req *http.Request) { // SetHeartbeatHandler implements the raft.Transport interface. func (t *HTTPTransport) SetHeartbeatHandler(cb func(rpc raft.RPC)) { // Not supported - } // TimeoutNow implements the raft.Transport interface. diff --git a/pkg/spec/internal_resources.go b/pkg/spec/internal_resources.go index 6cb6e30..c6d037d 100644 --- a/pkg/spec/internal_resources.go +++ b/pkg/spec/internal_resources.go @@ -97,11 +97,11 @@ func (m ModuleType) String() string { } type DGateModule struct { - Name string `json:"name"` - Namespace *DGateNamespace `json:"namespace"` - Payload string `json:"payload"` - Type ModuleType `json:"module_type"` - Tags []string `json:"tags,omitempty"` + Name string `json:"name"` + Namespace *DGateNamespace `json:"namespace"` + Payload string `json:"payload"` + Type ModuleType `json:"module_type"` + Tags []string `json:"tags,omitempty"` } func (m *DGateModule) GetName() string { diff --git a/pkg/spec/transformers.go b/pkg/spec/transformers.go index 878a6e5..194da2a 100644 --- a/pkg/spec/transformers.go +++ b/pkg/spec/transformers.go @@ -145,8 +145,8 @@ func TransformDGateCollection(col *DGateCollection) *Collection { NamespaceName: col.Namespace.Name, Schema: schema, // Type: col.Type, - Visibility: col.Visibility, - Tags: col.Tags, + Visibility: col.Visibility, + Tags: col.Tags, } } @@ -235,11 +235,11 @@ func TransformModule(ns *DGateNamespace, m *Module) (*DGateModule, error) { return nil, err } return &DGateModule{ - Name: m.Name, - Namespace: ns, - Payload: string(payload), - Tags: m.Tags, - Type: m.Type, + Name: m.Name, + Namespace: ns, + Payload: string(payload), + Tags: m.Tags, + Type: m.Type, }, nil } diff --git a/pkg/util/hash.go b/pkg/util/hash.go index 8237179..d7f1b62 100644 --- a/pkg/util/hash.go +++ b/pkg/util/hash.go @@ -8,6 +8,7 @@ import ( "hash" "hash/crc32" ) + func jsonHash(objs ...any) (hash.Hash32, error) { hash, err := crc32Hash(func(a any) []byte { b, err := json.Marshal(a) @@ -29,7 +30,6 @@ func JsonHash(objs ...any) (uint32, error) { return hash.Sum32(), nil } - func JsonHashBytes(objs ...any) ([]byte, error) { hash, err := jsonHash(objs...) if err != nil { diff --git a/pkg/util/linker/linker_test.go b/pkg/util/linker/linker_test.go index 8df45ad..f875ae3 100644 --- a/pkg/util/linker/linker_test.go +++ b/pkg/util/linker/linker_test.go @@ -27,7 +27,6 @@ func TestLinkerTests(t *testing.T) { linker3.LinkOneMany("bottom", "l3bottom", linker2) linker2.LinkOneMany("bottom", "l2bottom", linker1) - assert.True(t, linker1.Len("top") == 1) assert.True(t, linker2.Len("top") == 1) assert.True(t, linker3.Len("top") == 1) diff --git a/pkg/util/tree/avl/avl.go b/pkg/util/tree/avl/avl.go index d517a42..0c7e87b 100644 --- a/pkg/util/tree/avl/avl.go +++ b/pkg/util/tree/avl/avl.go @@ -165,8 +165,8 @@ func replace[K cmp.Ordered, V any](root *node[K, V], key K, value V) (*node[K, V var oldValue V if root == nil { return &node[K, V]{ - key: key, - val: value, + key: key, + val: value, height: 1, }, oldValue }