diff --git a/README.md b/README.md index afc2470..de75246 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ![GitHub Release](https://img.shields.io/github/v/release/dgate-io/dgate) -DGate is a distributed API Gateway built for developers. DGate allows you to use JavaScript/TypeScript to modify request/response data(L7). Inpired by [k6](https://github.com/grafana/k6) and [kong](https://github.com/Kong/kong). +DGate is a distributed API Gateway built for developers. DGate allows you to use JavaScript/TypeScript to modify request/response data(L7). Inspired by [k6](https://github.com/grafana/k6) and [kong](https://github.com/Kong/kong). > DGate is currently in development and is not ready for production use. Please use at your own discretion. @@ -30,6 +30,10 @@ go install github.com/dgate-io/dgate/cmd/dgate-server@latest DGate Server is proxy and admin server bundled into one. the admin server is responsible for managing the state of the proxy server. The proxy server is responsible for routing requests to upstream servers. The admin server can also be used to manage the state of the cluster using the Raft Consensus Algorithm. +### DGate CLI (dgate-cli) + +DGate CLI is a command-line interface that can be used to interact with the DGate Server. It can be used to deploy modules, manage the state of the cluster, and more. + #### Proxy Modules - Fetch Upstream Module (`fetchUpstream`) - executed before the request is sent to the upstream server. This module is used to decided which upstream server to send the current request to. (Essentially a custom load balancer module) diff --git a/TODO.md b/TODO.md index 6b85fe3..45b87d2 100644 --- a/TODO.md +++ b/TODO.md @@ -44,6 +44,10 @@ Also to execute code on specific events, like when a new route is added, or when - resource CRUD operations (namespace/domain/service/module/route/collection/document) - execute cron jobs: @every 1m, @cron 0 0 * * *, @daily, @weekly, @monthly, @yearly +At a higher level, background jobs can be used to enable features like health checks, which can periodically check the health of the upstream servers and disable/enable them if they are not healthy. + +Other features include: automatic service discovery, ping-based load balancing, + # Metrics -- add support for prometheus, datadog, sentry, etc. @@ -149,3 +153,27 @@ time based tags ## Module Permissions - Allow users to define permissions for modules to access certain dgate resources/apis and/or OS resources. + - resource:document:read + - resource:document:write + - os:net:(http/tcp/udp) + - os:file:read + - os:env:read + +# Bundles + +- Add support for bundles that can be used to extend the functionality of DGate. Bundles are a grouping of resources that can be used to extend the functionality of DGate. Bundles can be used to add new modules, resources, and more. +A good example of a bundle would be a bundle that adds support for OAuth2 authentication. It would need to setup the necessary routes, modules, and configurations to enable OAuth2 authentication. + +## Module/Plugin Variables + +- Allow users to define variables that can be used in modules/plugins. These variables can be set by the user, eventually the Admin Console should allow these variables to be set, and the variables can be used in the modules/plugins. + +## Mutual TLS Support (low priority) + +## Versioning Modules + +Differing from common resource versioning, modules can have multiple versions that can be used at the same time. This can be used to test new versions of modules before deploying them to the cluster. + +## Secrets + +- Add support for secrets that can be used in modules. Secrets can be used to store sensitive information like API keys, passwords, etc. Secrets can only be used in modules and cannot be accessed by the API. Explicit permissions can be set to allow certain modules to access certain secrets. Secrets are also versioned and can be rolled back if necessary. This also allows different modules to use different versions of the same secret. \ No newline at end of file diff --git a/functional-tests/admin_tests/performance_test_prep.sh b/functional-tests/admin_tests/performance_test_prep.sh index 8467729..e925cb4 100755 --- a/functional-tests/admin_tests/performance_test_prep.sh +++ b/functional-tests/admin_tests/performance_test_prep.sh @@ -56,4 +56,6 @@ curl -s --fail-with-body ${PROXY_URL}/svctest -H Host:dgate.dev curl -s --fail-with-body ${PROXY_URL}/modtest -H Host:dgate.dev -curl -s ${PROXY_URL}/blank -H Host:dgate.dev \ No newline at end of file +curl -s ${PROXY_URL}/blank -H Host:dgate.dev + +echo "Performance Test Prep Done" \ No newline at end of file diff --git a/internal/admin/routes/secret_routes.go b/internal/admin/routes/secret_routes.go new file mode 100644 index 0000000..3014206 --- /dev/null +++ b/internal/admin/routes/secret_routes.go @@ -0,0 +1,105 @@ +package routes + +import ( + "encoding/json" + "io" + "net/http" + "time" + + "github.com/dgate-io/chi-router" + "github.com/dgate-io/dgate/internal/config" + "github.com/dgate-io/dgate/internal/proxy" + "github.com/dgate-io/dgate/pkg/spec" + "github.com/dgate-io/dgate/pkg/util" +) + +func ConfigureSecretAPI(server chi.Router, proxyState *proxy.ProxyState, appConfig *config.DGateConfig) { + rm := proxyState.ResourceManager() + server.Put("/secret", func(w http.ResponseWriter, r *http.Request) { + eb, err := io.ReadAll(r.Body) + defer r.Body.Close() + if err != nil { + util.JsonError(w, http.StatusBadRequest, "error reading body") + return + } + scrt := spec.Secret{} + err = json.Unmarshal(eb, &scrt) + if err != nil { + util.JsonError(w, http.StatusBadRequest, "error unmarshalling body") + return + } + if scrt.Data == "" { + util.JsonError(w, http.StatusBadRequest, "payload is required") + return + } + if scrt.NamespaceName == "" { + if appConfig.DisableDefaultNamespace { + util.JsonError(w, http.StatusBadRequest, "namespace is required") + return + } + scrt.NamespaceName = spec.DefaultNamespace.Name + } + cl := spec.NewChangeLog(&scrt, scrt.NamespaceName, spec.AddSecretCommand) + if err = proxyState.ApplyChangeLog(cl); err != nil { + util.JsonError(w, http.StatusBadRequest, err.Error()) + return + } + if repl := proxyState.Raft(); repl != nil { + future := repl.Barrier(time.Second * 5) + if err := future.Error(); err != nil { + util.JsonError(w, http.StatusInternalServerError, err.Error()) + return + } + } + util.JsonResponse(w, http.StatusCreated, + spec.TransformDGateSecrets(true, + rm.GetSecretsByNamespace(scrt.NamespaceName)...)) + }) + + server.Delete("/secret", func(w http.ResponseWriter, r *http.Request) { + eb, err := io.ReadAll(r.Body) + defer r.Body.Close() + if err != nil { + util.JsonError(w, http.StatusBadRequest, "error reading body") + return + } + scrt := spec.Secret{} + err = json.Unmarshal(eb, &scrt) + if err != nil { + util.JsonError(w, http.StatusBadRequest, "error unmarshalling body") + return + } + if scrt.NamespaceName == "" { + if appConfig.DisableDefaultNamespace { + util.JsonError(w, http.StatusBadRequest, "namespace is required") + return + } + scrt.NamespaceName = spec.DefaultNamespace.Name + } + cl := spec.NewChangeLog(&scrt, scrt.NamespaceName, spec.DeleteSecretCommand) + if err = proxyState.ApplyChangeLog(cl); err != nil { + util.JsonError(w, http.StatusBadRequest, err.Error()) + return + } + w.WriteHeader(http.StatusAccepted) + }) + + server.Get("/secret", func(w http.ResponseWriter, r *http.Request) { + nsName := r.URL.Query().Get("namespace") + if nsName == "" { + if appConfig.DisableDefaultNamespace { + util.JsonError(w, http.StatusBadRequest, "namespace is required") + return + } + nsName = spec.DefaultNamespace.Name + } else { + if _, ok := rm.GetNamespace(nsName); !ok { + util.JsonError(w, http.StatusBadRequest, "namespace not found: "+nsName) + return + } + } + util.JsonResponse(w, http.StatusCreated, + spec.TransformDGateSecrets(true, + rm.GetSecretsByNamespace(nsName)...)) + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index df5a4ac..c192200 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -173,8 +173,9 @@ type ( Routes []spec.Route `koanf:"routes"` Modules []ModuleSpec `koanf:"modules"` Domains []DomainSpec `koanf:"domains"` - Collections []spec.Collection `koanf:"-"` + Collections []spec.Collection `koanf:"collections"` Documents []spec.Document `koanf:"documents"` + Secrets []spec.Secret `koanf:"secrets"` } DomainSpec struct { diff --git a/internal/config/configtest/config.go b/internal/config/configtest/config.go deleted file mode 100644 index 4c1210d..0000000 --- a/internal/config/configtest/config.go +++ /dev/null @@ -1,102 +0,0 @@ -package configtest - -import ( - "github.com/dgate-io/dgate/internal/config" - "github.com/dgate-io/dgate/pkg/spec" -) - -func NewTestDGateConfig() *config.DGateConfig { - return &config.DGateConfig{ - LogLevel: "panic", - Debug: true, - Version: "v1", - Tags: []string{"test"}, - Storage: config.DGateStorageConfig{ - StorageType: config.StorageTypeMemory, - }, - ProxyConfig: config.DGateProxyConfig{ - Host: "localhost", - Port: 8080, - InitResources: &config.DGateResources{ - Namespaces: []spec.Namespace{ - { - Name: "test", - }, - }, - Routes: []spec.Route{ - { - Name: "test", - Paths: []string{"/", "/test"}, - Methods: []string{"GET", "PUT"}, - Modules: []string{"test"}, - ServiceName: "test", - NamespaceName: "test", - Tags: []string{"test"}, - }, - }, - Services: []spec.Service{ - { - Name: "test", - URLs: []string{"http://localhost:8080"}, - NamespaceName: "test", - Tags: []string{"test"}, - }, - }, - Modules: []config.ModuleSpec{ - { - Module: spec.Module{ - Name: "test", - NamespaceName: "test", - Payload: EmptyAsyncModuleFunctionsTS, - Tags: []string{"test"}, - }, - }, - }, - }, - }, - } -} - -func NewTest2DGateConfig() *config.DGateConfig { - return &config.DGateConfig{ - LogLevel: "panic", - Debug: true, - Version: "v1", - Tags: []string{"test"}, - Storage: config.DGateStorageConfig{ - StorageType: config.StorageTypeMemory, - }, - ProxyConfig: config.DGateProxyConfig{ - Host: "localhost", - Port: 16436, - InitResources: &config.DGateResources{ - Namespaces: []spec.Namespace{ - { - Name: "test", - }, - }, - Routes: []spec.Route{ - { - Name: "test", - Paths: []string{"/", "/test"}, - Methods: []string{"GET", "PUT"}, - Modules: []string{"test"}, - NamespaceName: "test", - Tags: []string{"test"}, - }, - }, - Modules: []config.ModuleSpec{ - { - Module: spec.Module{ - Name: "test", - NamespaceName: "test", - Payload: EmptyAsyncModuleFunctionsTS, - Tags: []string{"test"}, - }, - }, - }, - }, - }, - } -} - diff --git a/internal/config/configtest/dgate_configs.go b/internal/config/configtest/dgate_configs.go new file mode 100644 index 0000000..629aba8 --- /dev/null +++ b/internal/config/configtest/dgate_configs.go @@ -0,0 +1,141 @@ +package configtest + +import ( + "github.com/dgate-io/dgate/internal/config" + "github.com/dgate-io/dgate/pkg/spec" +) + +func NewTestDGateConfig() *config.DGateConfig { + return &config.DGateConfig{ + LogLevel: "disabled", + DisableDefaultNamespace: true, + Debug: true, + Version: "v1", + Tags: []string{"test"}, + Storage: config.DGateStorageConfig{ + StorageType: config.StorageTypeMemory, + }, + ProxyConfig: config.DGateProxyConfig{ + AllowedDomains: []string{"*test.com", "localhost"}, + Host: "localhost", + Port: 8080, + InitResources: &config.DGateResources{ + Namespaces: []spec.Namespace{ + { + Name: "test", + }, + }, + Routes: []spec.Route{ + { + Name: "test", + Paths: []string{"/", "/test"}, + Methods: []string{"GET", "PUT"}, + Modules: []string{"test"}, + ServiceName: "test", + NamespaceName: "test", + Tags: []string{"test"}, + }, + }, + Services: []spec.Service{ + { + Name: "test", + URLs: []string{"http://localhost:8080"}, + NamespaceName: "test", + Tags: []string{"test"}, + }, + }, + Modules: []config.ModuleSpec{ + { + Module: spec.Module{ + Name: "test", + NamespaceName: "test", + Payload: EmptyAsyncModuleFunctionsTS, + Tags: []string{"test"}, + }, + }, + }, + }, + }, + } +} + +func NewTest2DGateConfig() *config.DGateConfig { + conf := NewTestDGateConfig() + conf.ProxyConfig = config.DGateProxyConfig{ + Host: "localhost", + Port: 16436, + InitResources: &config.DGateResources{ + Namespaces: []spec.Namespace{ + { + Name: "test", + }, + }, + Routes: []spec.Route{ + { + Name: "test", + Paths: []string{"/", "/test"}, + Methods: []string{"GET", "PUT"}, + Modules: []string{"test"}, + NamespaceName: "test", + Tags: []string{"test"}, + }, + }, + Modules: []config.ModuleSpec{ + { + Module: spec.Module{ + Name: "test", + NamespaceName: "test", + Payload: EmptyAsyncModuleFunctionsTS, + Tags: []string{"test"}, + }, + }, + }, + }, + } + return conf +} + +func NewTestDGateConfig_DomainAndNamespaces() *config.DGateConfig { + conf := NewTestDGateConfig() + conf.ProxyConfig.InitResources.Namespaces = []spec.Namespace{ + {Name: "test"}, {Name: "test2"}, {Name: "test3"}, + } + conf.ProxyConfig.InitResources.Domains = []config.DomainSpec{ + { + Domain: spec.Domain{ + Name: "test-dm", + NamespaceName: "test", + Patterns: []string{"example.com"}, + Priority: 1, + Tags: []string{"test"}, + }, + }, + { + Domain: spec.Domain{ + Name: "test-dm2", + NamespaceName: "test2", + Patterns: []string{`*test.com`}, + Priority: 2, + Tags: []string{"test"}, + }, + }, + { + Domain: spec.Domain{ + Name: "test-dm3", + NamespaceName: "test3", + Patterns: []string{`/^(abc|cba).test.com$/`}, + Priority: 3, + Tags: []string{"test"}, + }, + CertFile: "testdata/domain.crt", + KeyFile: "testdata/domain.key", + }, + } + return conf +} + +func NewTestDGateConfig_DomainAndNamespaces2() *config.DGateConfig { + conf := NewTestDGateConfig_DomainAndNamespaces() + conf.DisableDefaultNamespace = false + return conf +} diff --git a/internal/config/configtest/module.go b/internal/config/configtest/modules.go similarity index 100% rename from internal/config/configtest/module.go rename to internal/config/configtest/modules.go diff --git a/internal/config/resources.go b/internal/config/resources.go index 6b28f34..0f8b2a6 100644 --- a/internal/config/resources.go +++ b/internal/config/resources.go @@ -6,9 +6,10 @@ import ( "github.com/dgate-io/dgate/pkg/spec" ) -func (resources *DGateResources) Validate() error { +func (resources *DGateResources) Validate() (int, error) { + var numChanges int if resources == nil || resources.SkipValidation { - return nil + return 0, nil } namespaces := make(map[string]*spec.Namespace) services := make(map[string]*spec.Service) @@ -17,169 +18,196 @@ func (resources *DGateResources) Validate() error { modules := make(map[string]*spec.Module) collections := make(map[string]*spec.Collection) documents := make(map[string]*spec.Document) + secrets := make(map[string]*spec.Secret) for _, ns := range resources.Namespaces { if _, ok := namespaces[ns.Name]; ok { - return errors.New("duplicate namespace: " + ns.Name) + return 0, errors.New("duplicate namespace: " + ns.Name) } if ns.Name == "" { - return errors.New("namespace name must be specified") + return 0, errors.New("namespace name must be specified") } namespaces[ns.Name] = &ns } + numChanges += len(namespaces) for _, mod := range resources.Modules { key := mod.Name + "-" + mod.NamespaceName if _, ok := modules[key]; ok { - return errors.New("duplicate module: " + mod.Name) + return 0, errors.New("duplicate module: " + mod.Name) } if mod.Name == "" { - return errors.New("module name must be specified") + return 0, errors.New("module name must be specified") } if mod.NamespaceName != "" { if _, ok := namespaces[mod.NamespaceName]; !ok { - return errors.New("module (" + mod.Name + ") references non-existent namespace (" + mod.NamespaceName + ")") + return 0, errors.New("module (" + mod.Name + ") references non-existent namespace (" + mod.NamespaceName + ")") } } else { - return errors.New("module (" + mod.Name + ") must specify namespace") + return 0, errors.New("module (" + mod.Name + ") must specify namespace") } if mod.Payload == "" && mod.PayloadFile == "" { - return errors.New("module payload or payload file must be specified") + return 0, errors.New("module payload or payload file must be specified") } if mod.Payload != "" && mod.PayloadFile != "" { - return errors.New("module payload and payload file cannot both be specified") + return 0, errors.New("module payload and payload file cannot both be specified") } modules[key] = &mod.Module } + numChanges += len(modules) for _, svc := range resources.Services { key := svc.Name + "-" + svc.NamespaceName if _, ok := services[key]; ok { - return errors.New("duplicate service: " + svc.Name) + return 0, errors.New("duplicate service: " + svc.Name) } if svc.Name == "" { - return errors.New("service name must be specified") + return 0, errors.New("service name must be specified") } if svc.NamespaceName != "" { if _, ok := namespaces[svc.NamespaceName]; !ok { - return errors.New("service (" + svc.Name + ") references non-existent namespace (" + svc.NamespaceName + ")") + return 0, errors.New("service (" + svc.Name + ") references non-existent namespace (" + svc.NamespaceName + ")") } } else { - return errors.New("service (" + svc.Name + ") must specify namespace") + return 0, errors.New("service (" + svc.Name + ") must specify namespace") } services[key] = &svc } + numChanges += len(services) for _, route := range resources.Routes { key := route.Name + "-" + route.NamespaceName if _, ok := routes[key]; ok { - return errors.New("duplicate route: " + route.Name) + return 0, errors.New("duplicate route: " + route.Name) } if route.Name == "" { - return errors.New("route name must be specified") + return 0, errors.New("route name must be specified") } if route.ServiceName != "" { if _, ok := services[route.ServiceName+"-"+route.NamespaceName]; !ok { - return errors.New("route (" + route.Name + ") references non-existent service (" + route.ServiceName + ")") + return 0, errors.New("route (" + route.Name + ") references non-existent service (" + route.ServiceName + ")") } } if route.NamespaceName != "" { if _, ok := namespaces[route.NamespaceName]; !ok { - return errors.New("route (" + route.Name + ") references non-existent namespace (" + route.NamespaceName + ")") + return 0, errors.New("route (" + route.Name + ") references non-existent namespace (" + route.NamespaceName + ")") } } else { - return errors.New("route (" + route.Name + ") must specify namespace") + return 0, errors.New("route (" + route.Name + ") must specify namespace") } for _, modName := range route.Modules { if _, ok := modules[modName+"-"+route.NamespaceName]; !ok { - return errors.New("route (" + route.Name + ") references non-existent module (" + modName + ")") + return 0, errors.New("route (" + route.Name + ") references non-existent module (" + modName + ")") } } routes[key] = &route } + numChanges += len(routes) for _, dom := range resources.Domains { key := dom.Name + "-" + dom.NamespaceName if _, ok := domains[key]; ok { - return errors.New("duplicate domain: " + dom.Name) + return 0, errors.New("duplicate domain: " + dom.Name) } if dom.Name == "" { - return errors.New("domain name must be specified") + return 0, errors.New("domain name must be specified") } if dom.NamespaceName != "" { if _, ok := namespaces[dom.NamespaceName]; !ok { - return errors.New("domain (" + dom.Name + ") references non-existent namespace (" + dom.NamespaceName + ")") + return 0, errors.New("domain (" + dom.Name + ") references non-existent namespace (" + dom.NamespaceName + ")") } } else { - return errors.New("domain (" + dom.Name + ") must specify namespace") + return 0, errors.New("domain (" + dom.Name + ") must specify namespace") } if dom.Cert != "" && dom.CertFile != "" { - return errors.New("domain cert and cert file cannot both be specified") + return 0, errors.New("domain cert and cert file cannot both be specified") } if dom.Key != "" && dom.KeyFile != "" { - return errors.New("domain key and key file cannot both be specified") + return 0, errors.New("domain key and key file cannot both be specified") } if (dom.Cert == "") != (dom.Key == "") { - return errors.New("domain cert (file) and key (file) must both be specified, or neither") + return 0, errors.New("domain cert (file) and key (file) must both be specified, or neither") } domains[key] = &dom.Domain } + numChanges += len(domains) for _, col := range resources.Collections { key := col.Name + "-" + col.NamespaceName if _, ok := collections[key]; ok { - return errors.New("duplicate collection: " + col.Name) + return 0, errors.New("duplicate collection: " + col.Name) } if col.Name == "" { - return errors.New("collection name must be specified") + return 0, errors.New("collection name must be specified") } if col.NamespaceName != "" { if _, ok := namespaces[col.NamespaceName]; !ok { - return errors.New("collection (" + col.Name + ") references non-existent namespace (" + col.NamespaceName + ")") + return 0, errors.New("collection (" + col.Name + ") references non-existent namespace (" + col.NamespaceName + ")") } } else { - return errors.New("collection (" + col.Name + ") must specify namespace") + return 0, errors.New("collection (" + col.Name + ") must specify namespace") } if col.Schema == nil { - return errors.New("collection (" + col.Name + ") must specify schema") + return 0, errors.New("collection (" + col.Name + ") must specify schema") } if col.Type != spec.CollectionTypeDocument && col.Type != spec.CollectionTypeFetcher { - return errors.New("collection (" + col.Name + ") must specify type") + return 0, errors.New("collection (" + col.Name + ") must specify type") } if col.Visibility != spec.CollectionVisibilityPublic && col.Visibility != spec.CollectionVisibilityPrivate { - return errors.New("collection (" + col.Name + ") must specify visibility") + return 0, errors.New("collection (" + col.Name + ") must specify visibility") } // TODO: Uncomment when modules are supported for collections // for _, modName := range col.Modules { // if _, ok := modules[modName+"-"+col.NamespaceName]; !ok { - // return errors.New("collection (" + col.Name + ") references non-existent module (" + modName + ")") + // return 0, errors.New("collection (" + col.Name + ") references non-existent module (" + modName + ")") // } // } collections[key] = &col } + numChanges += len(collections) for _, doc := range resources.Documents { key := doc.ID + "-" + doc.NamespaceName if _, ok := documents[key]; ok { - return errors.New("duplicate document: " + doc.ID) + return 0, errors.New("duplicate document: " + doc.ID) } if doc.ID == "" { - return errors.New("document ID must be specified") + return 0, errors.New("document ID must be specified") } if doc.NamespaceName != "" { if _, ok := namespaces[doc.NamespaceName]; !ok { - return errors.New("document (" + doc.ID + ") references non-existent namespace (" + doc.NamespaceName + ")") + return 0, errors.New("document (" + doc.ID + ") references non-existent namespace (" + doc.NamespaceName + ")") } } else { - return errors.New("document (" + doc.ID + ") must specify namespace") + return 0, errors.New("document (" + doc.ID + ") must specify namespace") } if doc.CollectionName != "" { if _, ok := collections[doc.CollectionName+"-"+doc.NamespaceName]; !ok { - return errors.New("document (" + doc.ID + ") references non-existent collection (" + doc.CollectionName + ")") + return 0, errors.New("document (" + doc.ID + ") references non-existent collection (" + doc.CollectionName + ")") } } documents[key] = &doc } + numChanges += len(documents) - return nil + for _, sec := range resources.Secrets { + key := sec.Name + "-" + sec.NamespaceName + if _, ok := secrets[key]; ok { + return 0, errors.New("duplicate secret: " + sec.Name) + } + if sec.Name == "" { + return 0, errors.New("secret name must be specified") + } + if sec.NamespaceName != "" { + if _, ok := namespaces[sec.NamespaceName]; !ok { + return 0, errors.New("secret (" + sec.Name + ") references non-existent namespace (" + sec.NamespaceName + ")") + } + } else { + return 0, errors.New("secret (" + sec.Name + ") must specify namespace") + } + secrets[key] = &sec + } + numChanges += len(secrets) + + return numChanges, nil } diff --git a/internal/config/resources_test.go b/internal/config/resources_test.go index bdfa6bf..78fde73 100644 --- a/internal/config/resources_test.go +++ b/internal/config/resources_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/dgate-io/dgate/pkg/spec" + "github.com/stretchr/testify/assert" ) func TestValidate(t *testing.T) { @@ -49,8 +50,9 @@ func TestValidate(t *testing.T) { }, }, } - err := resources.Validate() + changes, err := resources.Validate() if err != nil { t.Error(err) } + assert.Equal(t, 5, changes) } diff --git a/internal/proxy/change_log.go b/internal/proxy/change_log.go index 80d4d8f..1085453 100644 --- a/internal/proxy/change_log.go +++ b/internal/proxy/change_log.go @@ -15,10 +15,11 @@ import ( // processChangeLog - processes a change log and applies the change to the proxy state func (ps *ProxyState) processChangeLog( - cl *spec.ChangeLog, apply, store bool, + cl *spec.ChangeLog, reload, store bool, ) (err error) { - // TODO: add revert cl function to check if storage fails or something - if !cl.Cmd.IsNoop() { + if cl == nil { + cl = &spec.ChangeLog{Cmd: spec.NoopCommand} + } else if !cl.Cmd.IsNoop() { switch cl.Cmd.Resource() { case spec.Namespaces: var item spec.Namespace @@ -69,6 +70,13 @@ func (ps *ProxyState) processChangeLog( ps.logger.Trace().Msgf("Processing domain: %s", item.ID) err = ps.processDocument(&item, cl) } + case spec.Secrets: + var item spec.Secret + item, err = decode[spec.Secret](cl.Item) + if err == nil { + ps.logger.Trace().Msgf("Processing domain: %s", item.Name) + err = ps.processSecret(&item, cl) + } default: err = fmt.Errorf("unknown command: %s", cl.Cmd) } @@ -77,7 +85,7 @@ func (ps *ProxyState) processChangeLog( return } } - if apply && (cl.Cmd.Resource().IsRelatedTo(spec.Routes) || cl.Cmd.IsNoop()) { + if reload && (cl.Cmd.Resource().IsRelatedTo(spec.Routes) || cl.Cmd.IsNoop()) { ps.logger.Trace().Msgf("Registering change log: %s", cl.Cmd) errChan := ps.applyChange(cl) select { @@ -90,26 +98,26 @@ func (ps *ProxyState) processChangeLog( ps.logger.Err(err).Msg("Error registering change log") return } + // update change log hash only when the change is successfully applied + // even if the change is a noop, we still need to update the hash + changeHash, err := HashAny[*spec.ChangeLog](ps.changeHash, cl) + if err != nil { + if !ps.config.Debug { + return err + } + ps.logger.Error().Err(err). + Msg("error updating change log hash") + } else { + ps.changeHash = changeHash + } } if store { - err = ps.store.StoreChangeLog(cl) - if err != nil { - // TODO: Add config option to panic on persistent storage errors - // TODO: maybe revert change here or add to some in-memory queue for changes? + if err = ps.store.StoreChangeLog(cl); err != nil { + // TODO: revert change here on error ?? ps.logger.Err(err).Msg("Error storing change log") return } } - changeHash, err := HashAny[*spec.ChangeLog](ps.changeHash, cl) - if err != nil { - if !ps.config.Debug { - return err - } - ps.logger.Error().Err(err). - Msg("error updating change log hash") - } else { - ps.changeHash = changeHash - } return nil } @@ -137,12 +145,12 @@ func (ps *ProxyState) processNamespace(ns *spec.Namespace, cl *spec.ChangeLog) e switch cl.Cmd.Action() { case spec.Add: ps.rm.AddNamespace(ns) - return nil case spec.Delete: return ps.rm.RemoveNamespace(ns.Name) default: return fmt.Errorf("unknown command: %s", cl.Cmd) } + return nil } func (ps *ProxyState) processService(svc *spec.Service, cl *spec.ChangeLog) (err error) { @@ -153,7 +161,7 @@ func (ps *ProxyState) processService(svc *spec.Service, cl *spec.ChangeLog) (err case spec.Add: _, err = ps.rm.AddService(svc) case spec.Delete: - _, err = ps.rm.AddService(svc) + err = ps.rm.RemoveService(svc.Name, svc.NamespaceName) default: err = fmt.Errorf("unknown command: %s", cl.Cmd) } @@ -168,7 +176,7 @@ func (ps *ProxyState) processRoute(rt *spec.Route, cl *spec.ChangeLog) (err erro case spec.Add: _, err = ps.rm.AddRoute(rt) case spec.Delete: - _, err = ps.rm.AddRoute(rt) + err = ps.rm.RemoveRoute(rt.Name, rt.NamespaceName) default: err = fmt.Errorf("unknown command: %s", cl.Cmd) } @@ -183,7 +191,7 @@ func (ps *ProxyState) processModule(mod *spec.Module, cl *spec.ChangeLog) (err e case spec.Add: _, err = ps.rm.AddModule(mod) case spec.Delete: - _, err = ps.rm.AddModule(mod) + err = ps.rm.RemoveModule(mod.Name, mod.NamespaceName) default: err = fmt.Errorf("unknown command: %s", cl.Cmd) } @@ -198,7 +206,7 @@ func (ps *ProxyState) processDomain(dom *spec.Domain, cl *spec.ChangeLog) (err e case spec.Add: _, err = ps.rm.AddDomain(dom) case spec.Delete: - _, err = ps.rm.AddDomain(dom) + err = ps.rm.RemoveDomain(dom.Name, dom.NamespaceName) default: err = fmt.Errorf("unknown command: %s", cl.Cmd) } @@ -213,7 +221,7 @@ func (ps *ProxyState) processCollection(col *spec.Collection, cl *spec.ChangeLog case spec.Add: _, err = ps.rm.AddCollection(col) case spec.Delete: - _, err = ps.rm.AddCollection(col) + err = ps.rm.RemoveCollection(col.Name, col.NamespaceName) default: err = fmt.Errorf("unknown command: %s", cl.Cmd) } @@ -228,7 +236,22 @@ func (ps *ProxyState) processDocument(doc *spec.Document, cl *spec.ChangeLog) (e case spec.Add: err = ps.store.StoreDocument(doc) case spec.Delete: - err = ps.store.DeleteDocument(doc) + err = ps.store.DeleteDocument(doc.ID, doc.CollectionName, doc.NamespaceName) + default: + err = fmt.Errorf("unknown command: %s", cl.Cmd) + } + return err +} + +func (ps *ProxyState) processSecret(scrt *spec.Secret, cl *spec.ChangeLog) (err error) { + if scrt.NamespaceName == "" { + scrt.NamespaceName = cl.Namespace + } + switch cl.Cmd.Action() { + case spec.Add: + _, err = ps.rm.AddSecret(scrt) + case spec.Delete: + err = ps.rm.RemoveSecret(scrt.Name, scrt.NamespaceName) default: err = fmt.Errorf("unknown command: %s", cl.Cmd) } @@ -270,12 +293,12 @@ func (ps *ProxyState) restoreFromChangeLogs() error { for i, cl := range logs { ps.logger.Trace(). Interface("changeLog: "+cl.Name, cl).Msgf("restoring change log index: %d", i) - lastIteration := i == len(logs)-1 - err = ps.processChangeLog(cl, lastIteration, false) + err = ps.processChangeLog(cl, false, false) if err != nil { return err } } + ps.processChangeLog(nil, true, false) // TODO: change to configurable variable if len(logs) > 1 { removed, err := ps.compactChangeLogs(logs) diff --git a/internal/proxy/proxy_state.go b/internal/proxy/proxy_state.go index c6c0d3f..9abd3b2 100644 --- a/internal/proxy/proxy_state.go +++ b/internal/proxy/proxy_state.go @@ -12,7 +12,6 @@ import ( "net" "net/http" "os" - "sort" "sync" "time" @@ -169,10 +168,10 @@ func NewProxyState(conf *config.DGateConfig) *ProxyState { state.store = proxystore.New(dataStore, state.Logger(WithComponentLogger("proxystore"))) - err = state.initConfigResources(conf.ProxyConfig.InitResources) - if err != nil { + if err = state.initConfigResources(conf.ProxyConfig.InitResources); err != nil { panic("error initializing resources: " + err.Error()) } + return state } @@ -206,6 +205,10 @@ func (ps *ProxySnapshot) PersistState(w io.Writer) error { return enc.Encode(ps) } +func (ps *ProxyState) Store() *proxystore.ProxyStore { + return ps.store +} + func (ps *ProxyState) Stats() *ProxyStats { return ps.stats } @@ -259,9 +262,6 @@ func (ps *ProxyState) WaitForChanges() { } func (ps *ProxyState) ApplyChangeLog(log *spec.ChangeLog) error { - ps.logger.Trace(). - Bool("replication_enabled", ps.replicationEnabled). - Msgf("Applying change log: %s", log.Cmd) if ps.replicationEnabled { r := ps.replicationSettings.raft if r.State() != raft.Leader { @@ -285,9 +285,6 @@ func (ps *ProxyState) ApplyChangeLog(log *spec.ChangeLog) error { } func (ps *ProxyState) ResourceManager() *resources.ResourceManager { - if ps == nil { - return nil - } return ps.rm } @@ -303,43 +300,43 @@ func (ps *ProxyState) ReloadState() error { return <-ps.applyChange(nil) } -func (ps *ProxyState) ProcessChangeLog(log *spec.ChangeLog, register bool) error { - err := ps.processChangeLog(log, register, !ps.replicationEnabled) +func (ps *ProxyState) ProcessChangeLog(log *spec.ChangeLog, reload bool) error { + err := ps.processChangeLog(log, reload, !ps.replicationEnabled) if err != nil { ps.logger.Error().Err(err).Msg("error processing change log") - return err } - return nil + return err } -func (ps *ProxyState) DynamicTLSConfig( - certFile, keyFile string, -) *tls.Config { +func (ps *ProxyState) DynamicTLSConfig(certFile, keyFile string) *tls.Config { var fallbackCert *tls.Certificate + if certFile != "" && keyFile != "" { + cert, err := loadCertFromFile(certFile, keyFile) + if err != nil { + panic(fmt.Errorf("error loading cert: %s", err)) + } + fallbackCert = cert + } + return &tls.Config{ GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - cert, ok, err := ps.getDomainCertificate(info.ServerName) - if err != nil { + if cert, err := ps.getDomainCertificate(info.ServerName); err != nil { return nil, err - } else if !ok { - if certFile != "" && keyFile != "" { - cert, err := ps.loadCertFromFile(certFile, keyFile) - if err != nil { - return nil, err - } - fallbackCert = cert - } else if fallbackCert != nil { + } else if cert == nil { + if fallbackCert != nil { return fallbackCert, nil } else { + ps.logger.Error().Msg("no cert found matching: " + info.ServerName) return nil, errors.New("no cert found") } + } else { + return cert, nil } - return cert, nil }, } } -func (ps *ProxyState) loadCertFromFile(certFile, keyFile string) (*tls.Certificate, error) { +func loadCertFromFile(certFile, keyFile string) (*tls.Certificate, error) { cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { return nil, err @@ -347,44 +344,52 @@ func (ps *ProxyState) loadCertFromFile(certFile, keyFile string) (*tls.Certifica return &cert, nil } -func (ps *ProxyState) getDomainCertificate(domain string) (*tls.Certificate, bool, error) { +func (ps *ProxyState) getDomainCertificate(domain string) (*tls.Certificate, error) { allowedDomains := ps.config.ProxyConfig.AllowedDomains - _, domainAllowed, err := pattern.MatchAnyPattern(domain, allowedDomains) - if err != nil { - ps.logger.Error().Msgf("Error checking domain match list: %s", err.Error()) - return nil, false, err + domainAllowed := len(allowedDomains) == 0 + if !domainAllowed { + _, domainMatch, err := pattern.MatchAnyPattern(domain, allowedDomains) + if err != nil { + ps.logger.Error().Msgf("Error checking domain match list: %s", err.Error()) + return nil, err + } + domainAllowed = domainMatch } - if domainAllowed || len(allowedDomains) == 0 { - domains := ps.rm.GetDomains() - sort.Slice(domains, func(i, j int) bool { - if domains[i].Priority == domains[j].Priority { - return domains[i].Name > domains[j].Name - } - return domains[i].Priority > domains[j].Priority - }) - for _, d := range domains { - if _, domainMatches, err := pattern.MatchAnyPattern(domain, d.Patterns); err != nil { + if domainAllowed { + for _, d := range ps.rm.GetDomainsByPriority() { + if _, match, err := pattern.MatchAnyPattern(domain, d.Patterns); err != nil { ps.logger.Error().Msgf("Error checking domain match list: %s", err.Error()) - return nil, false, err - } else { - if domainMatches { - if d.TLSCert == nil { - return nil, false, nil - } - return d.TLSCert, true, nil + return nil, err + } else if match && d.Cert != "" && d.Key != "" { + certBucket := ps.sharedCache.Bucket("certs") + key := fmt.Sprintf("cert:%s:%s:%d", d.Namespace.Name, + d.Name, d.CreatedAt.UnixMilli()) + if cert, ok := certBucket.Get(key); ok { + return cert.(*tls.Certificate), nil + } + serverCert, err := tls.X509KeyPair([]byte(d.Cert), []byte(d.Key)) + if err != nil { + ps.logger.Error().Msgf("Error loading cert: %s on domain %s/%s", + err.Error(), d.Namespace.Name, d.Name) + return nil, err } + certBucket.Set(key, &serverCert) + return &serverCert, nil } } } - return nil, false, nil + return nil, nil } func (ps *ProxyState) initConfigResources(resources *config.DGateResources) error { if resources != nil { - err := resources.Validate() + numChanges, err := resources.Validate() if err != nil { return err } + if numChanges > 0 { + defer ps.processChangeLog(nil, false, false) + } ps.logger.Info().Msg("Initializing resources") for _, ns := range resources.Namespaces { cl := spec.NewChangeLog(&ns, ns.Name, spec.AddNamespaceCommand) @@ -471,31 +476,33 @@ func (ps *ProxyState) FindNamespaceByRequest(r *http.Request) *spec.DGateNamespa host = r.Host } - domains := ps.rm.GetDomains() - if len(domains) > 0 { - sort.Slice(domains, func(i, j int) bool { - if domains[i].Priority == domains[j].Priority { - return domains[i].Name > domains[j].Name - } - return domains[i].Priority > domains[j].Priority - }) - var priority int - var namespace *spec.DGateNamespace + // if there are no domains and only one namespace, return that namespace + if ps.rm.DomainCountEquals(0) && ps.rm.NamespaceCountEquals(1) { + return ps.rm.GetFirstNamespace() + } + // search through domains for a match + var defaultNsHasDomain bool + if domains := ps.rm.GetDomainsByPriority(); len(domains) > 0 { for _, d := range domains { - if d.Priority < priority { - continue + if !ps.config.DisableDefaultNamespace { + if d.Namespace.Name == "default" { + defaultNsHasDomain = true + } } _, match, err := pattern.MatchAnyPattern(host, d.Patterns) if err != nil { ps.logger.Error().Err(err). Msg("error matching namespace") - continue - } - if match { - priority, namespace = d.Priority, d.Namespace + } else if match { + return d.Namespace } } - return namespace + } + // if no domain matches, return the default namespace, if it doesn't have a domain + if !ps.config.DisableDefaultNamespace && !defaultNsHasDomain { + if defaultNs, ok := ps.rm.GetNamespace("default"); ok { + return defaultNs + } } return nil } diff --git a/internal/proxy/proxy_state_test.go b/internal/proxy/proxy_state_test.go new file mode 100644 index 0000000..15c1f01 --- /dev/null +++ b/internal/proxy/proxy_state_test.go @@ -0,0 +1,460 @@ +package proxy_test + +import ( + "crypto/tls" + "fmt" + "net/http" + "testing" + + "github.com/dgate-io/dgate/internal/config/configtest" + "github.com/dgate-io/dgate/internal/proxy" + "github.com/dgate-io/dgate/pkg/spec" + "github.com/stretchr/testify/assert" +) + +// Raft Test -> ApplyChangeLog, WaitForChanges, +// CaptureState, EnableRaft, Raft, PersistState, RestoreState, + +// DynamicTLSConfig + +func TestDynamicTLSConfig_DomainCert(t *testing.T) { + conf := configtest.NewTestDGateConfig_DomainAndNamespaces() + ps := proxy.NewProxyState(conf) + + tlsConfig := ps.DynamicTLSConfig("", "") + clientHello := &tls.ClientHelloInfo{ + ServerName: "abc.test.com", + } + cert, err := tlsConfig.GetCertificate(clientHello) + if !assert.Nil(t, err, "error should be nil") { + t.Fatal(err) + } + if !assert.NotNil(t, cert, "should not be nil") { + return + } +} + +func TestDynamicTLSConfig_DomainCertCache(t *testing.T) { + conf := configtest.NewTestDGateConfig_DomainAndNamespaces() + ps := proxy.NewProxyState(conf) + d := ps.ResourceManager().GetDomainsByPriority()[0] + key := fmt.Sprintf("cert:%s:%s:%d", d.Namespace.Name, + d.Name, d.CreatedAt.UnixMilli()) + tlsConfig := ps.DynamicTLSConfig("", "") + clientHello := &tls.ClientHelloInfo{ + ServerName: "abc.test.com", + } + cert, err := tlsConfig.GetCertificate(clientHello) + if !assert.Nil(t, err, "error should be nil") { + t.Fatal(err) + } + if !assert.NotNil(t, cert, "should not be nil") { + return + } + // check cache + item, ok := ps.SharedCache().Bucket("certs").Get(key) + if !assert.True(t, ok, "should be true") { + return + } + if _, ok = item.(*tls.Certificate); !ok { + t.Fatal("should be tls.Certificate") + } + +} + +func TestDynamicTLSConfig_Fallback(t *testing.T) { + conf := configtest.NewTestDGateConfig_DomainAndNamespaces() + ps := proxy.NewProxyState(conf) + + tlsConfig := ps.DynamicTLSConfig("testdata/server.crt", "testdata/server.key") + // this should have a match that is not the fallback + clientHello := &tls.ClientHelloInfo{ + ServerName: "abc.test.com", + } + cert, err := tlsConfig.GetCertificate(clientHello) + if !assert.Nil(t, err, "error should be nil") { + t.Fatal(err) + } + if !assert.NotNil(t, cert, "should not be nil") { + return + } + // this should have a match that is the fallback + clientHello = &tls.ClientHelloInfo{ + ServerName: "nomatch.com", + } + cert, err = tlsConfig.GetCertificate(clientHello) + if !assert.Nil(t, err, "error should be nil") { + t.Fatal(err) + } + if !assert.NotNil(t, cert, "should not be nil") { + return + } +} + +func TestFindNamespaceByRequest_OneNamespaceNoDomain(t *testing.T) { + conf := configtest.NewTestDGateConfig() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + hostNsPair := map[string]string{ + "": "test", + "test.com": "test", + "abc.test.com": "test", + } + for testHost, nsName := range hostNsPair { + if req, err := http.NewRequest(http.MethodGet, "/test", nil); err != nil { + t.Fatal(err) + } else { + req.Host = testHost + n := ps.FindNamespaceByRequest(req) + if assert.NotNil(t, n, "should not be nil") { + assert.Equal(t, n.Name, nsName, "expected namespace %s, got %s", nsName, n.Name) + } + } + } +} + +func TestFindNamespaceByRequest_DomainsAndNamespaces(t *testing.T) { + conf := configtest.NewTestDGateConfig_DomainAndNamespaces() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + hostNsPair := map[string]any{ + "": nil, + "test.com.jp": nil, + "nomatch.com": nil, + "example.com": "test", + "any.test.com": "test2", + "abc.test.com": "test3", + } + for testHost, nsName := range hostNsPair { + if req, err := http.NewRequest(http.MethodGet, "/test", nil); err != nil { + t.Fatal(err) + } else { + req.Host = testHost + if n := ps.FindNamespaceByRequest(req); nsName == nil { + assert.Nil(t, n, "should be nil when host is '%s'", testHost) + } else if assert.NotNil(t, n, "should not be nil") { + assert.Equal(t, n.Name, nsName, "expected namespace %s, got %s", nsName, n.Name) + } + } + } +} +func TestFindNamespaceByRequest_DomainsAndNamespacesDefault(t *testing.T) { + conf := configtest.NewTestDGateConfig_DomainAndNamespaces2() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + hostNsPair := map[string]any{ + "": "default", + "nomatch.com": "default", + "test.com.jp": "default", + "example.com": "test", + "any.test.com": "test2", + "abc.test.com": "test3", + } + for testHost, nsName := range hostNsPair { + if req, err := http.NewRequest(http.MethodGet, "/test", nil); err != nil { + t.Fatal(err) + } else { + req.Host = testHost + if n := ps.FindNamespaceByRequest(req); nsName == nil { + assert.Nil(t, n, "should be nil when host is '%s'", testHost) + } else if assert.NotNil(t, n, "should not be nil") { + assert.Equal(t, n.Name, nsName, "expected namespace %s, got %s", nsName, n.Name) + } + } + } +} + +// ApplyChangeLog + +// func TestApplyChangeLog(t *testing.T) { +// conf := configtest.NewTestDGateConfig() +// ps := proxy.NewProxyState(conf) +// if err := ps.Store().InitStore(); err != nil { +// t.Fatal(err) +// } +// err := ps.ApplyChangeLog(nil) +// assert.Nil(t, err, "error should be nil") +// } + +func TestProcessChangeLog_RMSecrets(t *testing.T) { + conf := configtest.NewTestDGateConfig() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + + sc := &spec.Secret{ + Name: "test", + NamespaceName: "test", + Data: "YWJj", + } + + cl := spec.NewChangeLog(sc, sc.NamespaceName, spec.AddSecretCommand) + err := ps.ProcessChangeLog(cl, true) + if !assert.Nil(t, err, "error should be nil") { + return + } + secrets := ps.ResourceManager().GetSecrets() + assert.Equal(t, 1, len(secrets), "should have 1 item") + assert.Equal(t, sc.Name, secrets[0].Name, "should have the same name") + assert.Equal(t, sc.NamespaceName, secrets[0].Namespace.Name, "should have the same namespace") + // 'YWJj' is base64 encoded 'abc' + assert.Equal(t, secrets[0].Data, "abc", "should have the same data") + + secrets = ps.ResourceManager().GetSecretsByNamespace(sc.NamespaceName) + assert.Equal(t, 1, len(secrets), "should have 1 item") + assert.Equal(t, sc.Name, secrets[0].Name, "should have the same name") + assert.Equal(t, sc.NamespaceName, secrets[0].Namespace.Name, "should have the same namespace") + // 'YWJj' is base64 encoded 'abc' + assert.Equal(t, secrets[0].Data, "abc", "should have the same data") + + cl = spec.NewChangeLog(sc, sc.NamespaceName, spec.DeleteSecretCommand) + err = ps.ProcessChangeLog(cl, true) + if !assert.Nil(t, err, "error should be nil") { + return + } + secrets = ps.ResourceManager().GetSecrets() + assert.Equal(t, 0, len(secrets), "should have 0 item") + +} + +func TestProcessChangeLog_Route(t *testing.T) { + conf := configtest.NewTestDGateConfig() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + + r := &spec.Route{ + Name: "test", + NamespaceName: "test", + Paths: []string{"/test"}, + Methods: []string{"GET"}, + ServiceName: "test", + Modules: []string{"test"}, + Tags: []string{"test"}, + } + + cl := spec.NewChangeLog(r, r.NamespaceName, spec.AddRouteCommand) + err := ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + routes := ps.ResourceManager().GetRoutes() + assert.Equal(t, 1, len(routes), "should have 1 item") + assert.Equal(t, r.Name, routes[0].Name, "should have the same name") + assert.Equal(t, r.NamespaceName, routes[0].Namespace.Name, "should have the same namespace") + assert.Equal(t, r.Paths, routes[0].Paths, "should have the same paths") + assert.Equal(t, r.Methods, routes[0].Methods, "should have the same methods") + assert.Equal(t, r.ServiceName, routes[0].Service.Name, "should have the same service") + assert.Equal(t, len(r.Modules), len(routes[0].Modules), "should have the same modules") + assert.Equal(t, len(r.Tags), len(routes[0].Tags), "should have the same tags") + + cl = spec.NewChangeLog(r, r.NamespaceName, spec.DeleteRouteCommand) + err = ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + routes = ps.ResourceManager().GetRoutes() + assert.Equal(t, 0, len(routes), "should have 0 item") +} + +func TestProcessChangeLog_Service(t *testing.T) { + conf := configtest.NewTestDGateConfig() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + + s := &spec.Service{ + Name: "test", + NamespaceName: "test", + URLs: []string{"http://localhost:8080"}, + Tags: []string{"test"}, + } + + cl := spec.NewChangeLog(s, s.NamespaceName, spec.AddServiceCommand) + err := ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + services := ps.ResourceManager().GetServices() + assert.Equal(t, 1, len(services), "should have 1 item") + assert.Equal(t, s.Name, services[0].Name, "should have the same name") + assert.Equal(t, s.NamespaceName, services[0].Namespace.Name, "should have the same namespace") + assert.Equal(t, len(s.URLs), len(services[0].URLs), "should have the same urls") + assert.Equal(t, len(s.Tags), len(services[0].Tags), "should have the same tags") + + cl = spec.NewChangeLog(s, s.NamespaceName, spec.DeleteServiceCommand) + err = ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + services = ps.ResourceManager().GetServices() + assert.Equal(t, 0, len(services), "should have 0 item") +} + +func TestProcessChangeLog_Module(t *testing.T) { + conf := configtest.NewTestDGateConfig() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + + m := &spec.Module{ + Name: "test", + NamespaceName: "test", + Payload: "", + Tags: []string{"test"}, + } + + cl := spec.NewChangeLog(m, m.NamespaceName, spec.AddModuleCommand) + err := ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + modules := ps.ResourceManager().GetModules() + assert.Equal(t, 1, len(modules), "should have 1 item") + assert.Equal(t, m.Name, modules[0].Name, "should have the same name") + assert.Equal(t, m.NamespaceName, modules[0].Namespace.Name, "should have the same namespace") + assert.Equal(t, m.Payload, modules[0].Payload, "should have the same payload") + assert.Equal(t, len(m.Tags), len(modules[0].Tags), "should have the same tags") + + cl = spec.NewChangeLog(m, m.NamespaceName, spec.DeleteModuleCommand) + err = ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + modules = ps.ResourceManager().GetModules() + assert.Equal(t, 0, len(modules), "should have 0 item") +} + +func TestProcessChangeLog_Namespace(t *testing.T) { + ps := proxy.NewProxyState(configtest.NewTestDGateConfig()) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + + n := &spec.Namespace{ + Name: "test_new", + } + + cl := spec.NewChangeLog(n, n.Name, spec.AddNamespaceCommand) + err := ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + namespaces := ps.ResourceManager().GetNamespaces() + assert.Equal(t, 2, len(namespaces), "should have 2 items") + assert.Equal(t, n.Name, namespaces[1].Name, "should have the same name") + + cl = spec.NewChangeLog(n, n.Name, spec.DeleteNamespaceCommand) + err = ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + namespaces = ps.ResourceManager().GetNamespaces() + assert.Equal(t, 1, len(namespaces), "should have 0 item") +} + +func TestProcessChangeLog_Collection(t *testing.T) { + conf := configtest.NewTestDGateConfig() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + + c := &spec.Collection{ + Name: "test", + NamespaceName: "test", + Type: spec.CollectionTypeDocument, + Visibility: spec.CollectionVisibilityPrivate, + Tags: []string{"test"}, + } + + cl := spec.NewChangeLog(c, c.NamespaceName, spec.AddCollectionCommand) + err := ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + collections := ps.ResourceManager().GetCollections() + assert.Equal(t, 1, len(collections), "should have 1 item") + assert.Equal(t, c.Name, collections[0].Name, "should have the same name") + assert.Equal(t, c.NamespaceName, collections[0].Namespace.Name, "should have the same namespace") + assert.Equal(t, c.Type, collections[0].Type, "should have the same type") + assert.Equal(t, c.Visibility, collections[0].Visibility, "should have the same visibility") + assert.Equal(t, len(c.Tags), len(collections[0].Tags), "should have the same tags") + + cl = spec.NewChangeLog(c, c.NamespaceName, spec.DeleteCollectionCommand) + err = ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + collections = ps.ResourceManager().GetCollections() + assert.Equal(t, 0, len(collections), "should have 0 item") +} + +func TestProcessChangeLog_Document(t *testing.T) { + conf := configtest.NewTestDGateConfig() + ps := proxy.NewProxyState(conf) + if err := ps.Store().InitStore(); err != nil { + t.Fatal(err) + } + + c := &spec.Collection{ + Name: "test", + NamespaceName: "test", + Type: spec.CollectionTypeDocument, + Visibility: spec.CollectionVisibilityPrivate, + Tags: []string{"test"}, + } + + cl := spec.NewChangeLog(c, c.NamespaceName, spec.AddCollectionCommand) + err := ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + + d := &spec.Document{ + ID: "test", + NamespaceName: "test", + CollectionName: "test", + Data: "", + } + + cl = spec.NewChangeLog(d, d.NamespaceName, spec.AddDocumentCommand) + err = ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + documents, err := ps.DocumentManager().GetDocuments( + "test", "test", 999, 0, + ) + if !assert.Nil(t, err, "error should be nil") { + return + } + assert.Equal(t, 1, len(documents), "should have 1 item") + assert.Equal(t, d.ID, documents[0].ID, "should have the same id") + assert.Equal(t, d.NamespaceName, documents[0].NamespaceName, "should have the same namespace") + assert.Equal(t, d.CollectionName, documents[0].CollectionName, "should have the same collection") + assert.Equal(t, d.Data, documents[0].Data, "should have the same data") + + cl = spec.NewChangeLog(d, d.NamespaceName, spec.DeleteDocumentCommand) + err = ps.ProcessChangeLog(cl, false) + if !assert.Nil(t, err, "error should be nil") { + return + } + documents, err = ps.DocumentManager().GetDocuments( + "test", "test", 999, 0, + ) + if !assert.Nil(t, err, "error should be nil") { + return + } + assert.Equal(t, 0, len(documents), "should have 0 item") +} diff --git a/internal/proxy/proxy_transport/proxy_transport.go b/internal/proxy/proxy_transport/proxy_transport.go index 3f1932c..10f65a9 100644 --- a/internal/proxy/proxy_transport/proxy_transport.go +++ b/internal/proxy/proxy_transport/proxy_transport.go @@ -103,11 +103,7 @@ func (m *retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) var ( resp *http.Response err error - // retryTimeoutChan <-chan time.Time ) - // if m.retryTimeout != 0 { - // retryTimeoutChan = time.After(m.retryTimeout) - // } ogReq := req for i := 0; i <= m.retries; i++ { if m.requestTimeout != 0 { @@ -119,14 +115,6 @@ func (m *retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) if err == nil { break } - // if m.retryTimeout != 0 { - // select { - // case <-retryTimeoutChan: - // return nil, errors.New("retry timeout exceeded") - // default: - // // ensures that this fails fast - // } - // } } return resp, err } diff --git a/internal/proxy/proxy_transport/proxy_transport_test.go b/internal/proxy/proxy_transport/proxy_transport_test.go index 2bc209f..36842bd 100644 --- a/internal/proxy/proxy_transport/proxy_transport_test.go +++ b/internal/proxy/proxy_transport/proxy_transport_test.go @@ -15,6 +15,10 @@ import ( "github.com/stretchr/testify/mock" ) +var proxyBuilder = proxy_transport.NewBuilder(). + RequestTimeout(5). + RetryTimeout(5) + func TestDGateProxy(t *testing.T) { mockTp := proxytest.CreateMockTransport() header := make(http.Header) @@ -30,10 +34,8 @@ func TestDGateProxy(t *testing.T) { }, nil).Once() numRetries := 5 - proxy, err := proxy_transport.NewBuilder(). - Retries(numRetries). - Transport(mockTp). - Build() + proxy, err := proxyBuilder.Clone(). + Transport(mockTp).Retries(numRetries).Build() if err != nil { t.Fatal(err) } @@ -44,7 +46,7 @@ func TestDGateProxy(t *testing.T) { mockRw := proxytest.CreateMockResponseWriter() mockRw.On("Header").Return(header) - mockRw.On("Write3xdHeader", mock.Anything).Return() + mockRw.On("WriteHeader", mock.Anything).Return() req = req.WithContext(context.WithValue(context.Background(), proxytest.S("testing"), "testing")) proxy.RoundTrip(req) @@ -55,3 +57,39 @@ func TestDGateProxy(t *testing.T) { // ensure context is passed through assert.Equal(t, "testing", req.Context().Value(proxytest.S("testing"))) } + +func TestDGateProxyError(t *testing.T) { + mockTp := proxytest.CreateMockTransport() + header := make(http.Header) + header.Add("X-Testing", "testing") + mockTp.On("RoundTrip", mock.Anything). + Return(nil, errors.New("testing error")). + Times(4) + mockTp.On("RoundTrip", mock.Anything).Return(&http.Response{ + StatusCode: 200, + ContentLength: 0, + Header: header, + Body: io.NopCloser(strings.NewReader("")), + }, nil).Once() + + proxy, err := proxyBuilder.Clone(). + Transport(mockTp).Build() + if err != nil { + t.Fatal(err) + } + req := &http.Request{ + URL: &url.URL{}, + Header: header, + } + + mockRw := proxytest.CreateMockResponseWriter() + mockRw.On("Header").Return(header) + mockRw.On("WriteHeader", mock.Anything).Return() + req = req.WithContext(context.WithValue(context.Background(), proxytest.S("testing"), "testing")) + proxy.RoundTrip(req) + + // ensure roundtrip is called at least once + mockTp.AssertCalled(t, "RoundTrip", mock.Anything) + // ensure context is passed through + assert.Equal(t, "testing", req.Context().Value(proxytest.S("testing"))) +} diff --git a/internal/proxy/proxystore/proxystore.go b/internal/proxy/proxystore/proxystore.go index c7d5d29..148b8ea 100644 --- a/internal/proxy/proxystore/proxystore.go +++ b/internal/proxy/proxystore/proxystore.go @@ -2,6 +2,7 @@ package proxystore import ( "encoding/json" + "time" "errors" @@ -64,8 +65,17 @@ func (store *ProxyStore) StoreChangeLog(cl *spec.ChangeLog) error { return err } store.logger.Trace().Msgf("storing changelog:%s", string(clBytes)) + retries, delay := 30, time.Microsecond*100 +RETRY: err = store.storage.Set("changelog/"+cl.ID, clBytes) if err != nil { + if retries > 0 { + store.logger.Err(err). + Msgf("failed to store changelog, retrying %d more times", retries) + time.Sleep(delay) + retries-- + goto RETRY + } return err } return nil @@ -106,9 +116,8 @@ func (store *ProxyStore) FetchDocument(nsName, colName, docId string) (*spec.Doc } func (store *ProxyStore) FetchDocuments( - namespaceName string, - collectionName string, - offset, limit int, + namespaceName, collectionName string, + limit, offset int, ) ([]*spec.Document, error) { docs := make([]*spec.Document, 0) docPrefix := createDocumentKey(namespaceName, collectionName, "") @@ -144,8 +153,8 @@ func (store *ProxyStore) StoreDocument(doc *spec.Document) error { return nil } -func (store *ProxyStore) DeleteDocument(doc *spec.Document) error { - err := store.storage.Delete(createDocumentKey(doc.NamespaceName, doc.CollectionName, doc.ID)) +func (store *ProxyStore) DeleteDocument(id, colName, nsName string) error { + err := store.storage.Delete(createDocumentKey(nsName, colName, id)) if err != nil { if err == badger.ErrKeyNotFound { return nil diff --git a/internal/proxy/reverse_proxy/reverse_proxy_test.go b/internal/proxy/reverse_proxy/reverse_proxy_test.go index bb97a43..9d5257d 100644 --- a/internal/proxy/reverse_proxy/reverse_proxy_test.go +++ b/internal/proxy/reverse_proxy/reverse_proxy_test.go @@ -42,6 +42,15 @@ func testDGateProxyRewrite( header.Add("X-Testing", "testing") rp, err := reverse_proxy.NewBuilder(). Transport(mockTp). + CustomRewrite(func(r1, r2 *http.Request) { + }). + ModifyResponse(func(r *http.Response) error { + r.Header.Set("X-Testing-2", + r.Header.Get("X-Testing")) + return nil + }). + ErrorHandler(func(w http.ResponseWriter, r *http.Request, err error) {}). + ErrorLogger(nil). ProxyRewrite( rewriteParams.stripPath, rewriteParams.preserveHost, @@ -77,29 +86,29 @@ func testDGateProxyRewrite( } if rewriteParams.xForwardedHeaders { if req.Header.Get("X-Forwarded-For") == "" { - t.Errorf("FAIL: Expected X-Testing header, got %s", req.Header.Get("X-Testing")) + t.Errorf("FAIL: Expected X-Forwarded-For header, got empty") } if req.Header.Get("X-Real-IP") == "" { - t.Errorf("FAIL: Expected X-Testing header, got %s", req.Header.Get("X-Testing")) - } - if req.Header.Get("X-Forwarded-Host") == "" { - t.Errorf("FAIL: Expected X-Testing header, got %s", req.Header.Get("X-Testing")) + t.Errorf("FAIL: Expected X-Real-IP header, got empty") } if req.Header.Get("X-Forwarded-Proto") == "" { - t.Errorf("FAIL: Expected X-Testing header, got %s", req.Header.Get("X-Testing")) + t.Errorf("FAIL: Expected X-Forwarded-Proto header, got empty") + } + if req.Header.Get("X-Forwarded-Host") == "" { + t.Errorf("FAIL: Expected X-Forwarded-Host header, got empty") } } else { if req.Header.Get("X-Forwarded-For") != "" { - t.Errorf("FAIL: Expected no X-Testing header, got %s", req.Header.Get("X-Testing")) + t.Errorf("FAIL: Expected no X-Forwarded-For header, got %s", req.Header.Get("X-Fowarded-For")) } if req.Header.Get("X-Real-IP") != "" { - t.Errorf("FAIL: Expected X-Testing header, got %s", req.Header.Get("X-Testing")) - } - if req.Header.Get("X-Forwarded-Host") != "" { - t.Errorf("FAIL: Expected X-Testing header, got %s", req.Header.Get("X-Testing")) + t.Errorf("FAIL: Expected no X-Real-IP header, got %s", req.Header.Get("X-Real-IP")) } if req.Header.Get("X-Forwarded-Proto") != "" { - t.Errorf("FAIL: Expected X-Testing header, got %s", req.Header.Get("X-Testing")) + t.Errorf("FAIL: Expected no X-Forwarded-Proto header, got %s", req.Header.Get("X-Forwarded-Proto")) + } + if req.Header.Get("X-Forwarded-Host") != "" { + t.Errorf("FAIL: Expected no X-Forwarded-Host header, got %s", req.Header.Get("X-Forwarded-Host")) } } }).Return(&http.Response{ @@ -152,7 +161,7 @@ func TestDGateProxyError(t *testing.T) { t.Fatal(err) } - req := proxytest.CreateMockRequest("GET", "http://localhost") + req := proxytest.CreateMockRequest("GET", "http://localhost:80") req.RemoteAddr = "::1" mockRw := proxytest.CreateMockResponseWriter() mockRw.On("Header").Return(header) diff --git a/internal/proxy/testdata/domain.crt b/internal/proxy/testdata/domain.crt new file mode 100644 index 0000000..7dec710 --- /dev/null +++ b/internal/proxy/testdata/domain.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB4TCCAYegAwIBAgIUcO49FFFmHRfsp5G60TS8uCeNYtQwCgYIKoZIzj0EAwIw +RTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGElu +dGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yNDA0MzAwNDA3NTNaGA8yMDUxMDkx +NTA0MDc1M1owRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAf +BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDBZMBMGByqGSM49AgEGCCqG +SM49AwEHA0IABDkUKfT7hSR3GPnH32Ixu1Dq5M6ohxrlOYoBjkW3E98g6uoh+tqP +QIq63vrNenWmIAmVnmhGSnryojYFlc/zYNKjUzBRMB0GA1UdDgQWBBSBrF1RV8NQ +OMDAoeYySDyNIPUFgjAfBgNVHSMEGDAWgBSBrF1RV8NQOMDAoeYySDyNIPUFgjAP +BgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIE7F3/whxJ+rg6aZH2M/ +oXSfHtRfqqMHY6uQoTgyuq5OAiEAhHK6tUSCyY1Vjl+fc4S15sQLpTJzK01SvuND +RP6yOd4= +-----END CERTIFICATE----- diff --git a/internal/proxy/testdata/domain.key b/internal/proxy/testdata/domain.key new file mode 100644 index 0000000..4f4222a --- /dev/null +++ b/internal/proxy/testdata/domain.key @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIMP1RBr7DMxFkwu9mPgkDIL9UeeiyN5HZcL7b6zUOCH0oAoGCCqGSM49 +AwEHoUQDQgAEORQp9PuFJHcY+cffYjG7UOrkzqiHGuU5igGORbcT3yDq6iH62o9A +irre+s16daYgCZWeaEZKevKiNgWVz/Ng0g== +-----END EC PRIVATE KEY----- diff --git a/internal/proxy/testdata/server.crt b/internal/proxy/testdata/server.crt new file mode 100644 index 0000000..6d49e57 --- /dev/null +++ b/internal/proxy/testdata/server.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHTCCAaKgAwIBAgIUdajYgRDrs2+h6rbXyMYnYxEO5tAwCgYIKoZIzj0EAwIw +RTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGElu +dGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA0MzAwMzAyNTVaFw0zNDA0Mjgw +MzAyNTVaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAARrWP/P7i/X3hYAMLP42wzDqQYD7BaQBmrl1UCr/U1kq/z1l5rA8miAIm4C +2XVcNiNssP9s8deIvAjSJCVe84VDUhyt//KunT9lKszlQsCxheQZ4Avkz8r/2nkv +e+rUBgajUzBRMB0GA1UdDgQWBBRM/4m56BvWibgU3IwSdtNyKSX3mjAfBgNVHSME +GDAWgBRM/4m56BvWibgU3IwSdtNyKSX3mjAPBgNVHRMBAf8EBTADAQH/MAoGCCqG +SM49BAMCA2kAMGYCMQDoFxCobqGX/0AODS2PdBHVRA2XFBH+Tq8laJvmVJCkW4HB +H/zB9csBOxwwy8RZfCECMQD+gPN/tpwt8770BcIUUlZGX4WacGRa/XG1OuD1Wd8d ++zN1mtapUmVm+GAyobKaH/A= +-----END CERTIFICATE----- diff --git a/internal/proxy/testdata/server.key b/internal/proxy/testdata/server.key new file mode 100644 index 0000000..6162773 --- /dev/null +++ b/internal/proxy/testdata/server.key @@ -0,0 +1,9 @@ +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDDm/RlRYwC9byu6vCLzaTErUqjMLvBVhodm0FR1pHAKkpplha+vJVBw +0bgj1lNj1bCgBwYFK4EEACKhZANiAARrWP/P7i/X3hYAMLP42wzDqQYD7BaQBmrl +1UCr/U1kq/z1l5rA8miAIm4C2XVcNiNssP9s8deIvAjSJCVe84VDUhyt//KunT9l +KszlQsCxheQZ4Avkz8r/2nkve+rUBgY= +-----END EC PRIVATE KEY----- diff --git a/performance-tests/perf-test.js b/performance-tests/perf-test.js index dce8b4b..3146807 100644 --- a/performance-tests/perf-test.js +++ b/performance-tests/perf-test.js @@ -4,25 +4,25 @@ import { check } from 'k6'; const n = 10; export let options = { scenarios: { - // modtest: { - // executor: 'constant-vus', - // vus: n, - // duration: '20s', - // // same function as the scenario above, but with different env vars - // exec: 'dgatePath', - // env: { DGATE_PATH: '/modtest' }, - // // startTime: '25s', - // gracefulStop: '5s', - // }, - svctest: { + modtest: { executor: 'constant-vus', vus: n, duration: '20s', - exec: 'dgatePath', // same function as the scenario above, but with different env vars - env: { DGATE_PATH: "/svctest" }, + // same function as the scenario above, but with different env vars + exec: 'dgatePath', + env: { DGATE_PATH: '/modtest' }, // startTime: '25s', gracefulStop: '5s', }, + // svctest: { + // executor: 'constant-vus', + // vus: n, + // duration: '20s', + // exec: 'dgatePath', // same function as the scenario above, but with different env vars + // env: { DGATE_PATH: "/svctest" }, + // // startTime: '25s', + // gracefulStop: '5s', + // }, // blank: { // executor: 'constant-vus', // vus: n, diff --git a/pkg/resources/resource_manager.go b/pkg/resources/resource_manager.go index 273e3eb..2f86c37 100644 --- a/pkg/resources/resource_manager.go +++ b/pkg/resources/resource_manager.go @@ -1,9 +1,9 @@ package resources import ( - "crypto/tls" "encoding/json" "errors" + "sort" "sync" "github.com/dgate-io/dgate/pkg/spec" @@ -21,6 +21,7 @@ type ResourceManager struct { domains avlTreeLinker[spec.DGateDomain] modules avlTreeLinker[spec.DGateModule] routes avlTreeLinker[spec.DGateRoute] + secrets avlTreeLinker[spec.DGateSecret] collections avlTreeLinker[spec.DGateCollection] mutex *sync.RWMutex } @@ -35,6 +36,7 @@ func NewManager(opts ...Options) *ResourceManager { modules: avl.NewTree[string, *linker.Link[string, safe.Ref[spec.DGateModule]]](), routes: avl.NewTree[string, *linker.Link[string, safe.Ref[spec.DGateRoute]]](), collections: avl.NewTree[string, *linker.Link[string, safe.Ref[spec.DGateCollection]]](), + secrets: avl.NewTree[string, *linker.Link[string, safe.Ref[spec.DGateSecret]]](), mutex: &sync.RWMutex{}, } for _, opt := range opts { @@ -69,6 +71,28 @@ func (rm *ResourceManager) getNamespace(namespace string) (*spec.DGateNamespace, } } +func (rm *ResourceManager) NamespaceCountEquals(target int) bool { + if target < 0 { + panic("target must be greater than or equal to 0") + } + rm.mutex.RLock() + defer rm.mutex.RUnlock() + rm.namespaces.Each(func(_ string, lk *linker.Link[string, safe.Ref[spec.DGateNamespace]]) bool { + target -= 1 + return target > 0 + }) + return target == 0 +} + +func (rm *ResourceManager) GetFirstNamespace() *spec.DGateNamespace { + rm.mutex.RLock() + defer rm.mutex.RUnlock() + if _, nsLink, ok := rm.namespaces.RootKeyValue(); ok { + return nsLink.Item().Read() + } + return nil +} + // GetNamespaces returns a list of all namespaces func (rm *ResourceManager) GetNamespaces() []*spec.DGateNamespace { rm.mutex.RLock() @@ -99,7 +123,7 @@ func (rm *ResourceManager) AddNamespace(ns *spec.Namespace) *spec.DGateNamespace safe.NewRef(namespace), "routes", "services", "modules", "domains", - "collections", + "collections", "secrets", ) rm.namespaces.Insert(ns.Name, lk) } @@ -440,6 +464,37 @@ func (rm *ResourceManager) GetDomains() []*spec.DGateDomain { return domains } +func (rm *ResourceManager) DomainCountEquals(target int) bool { + if target < 0 { + panic("target must be greater than or equal to 0") + } + rm.mutex.RLock() + defer rm.mutex.RUnlock() + rm.domains.Each(func(_ string, lk *linker.Link[string, safe.Ref[spec.DGateDomain]]) bool { + target -= 1 + return target > 0 + }) + return target == 0 +} + +// GetDomainsByPriority returns a list of all domains sorted by priority and name +func (rm *ResourceManager) GetDomainsByPriority() []*spec.DGateDomain { + rm.mutex.RLock() + defer rm.mutex.RUnlock() + var domains []*spec.DGateDomain + rm.domains.Each(func(_ string, lk *linker.Link[string, safe.Ref[spec.DGateDomain]]) bool { + domains = append(domains, lk.Item().Read()) + return true + }) + + sort.Slice(domains, func(i, j int) bool { + d1, d2 := domains[j], domains[i] + return d1.Name < d2.Name || d1.Priority < d2.Priority + }) + + return domains +} + // GetDomainsByNamespace returns a list of all domains in a namespace func (rm *ResourceManager) GetDomainsByNamespace(namespace string) []*spec.DGateDomain { rm.mutex.RLock() @@ -478,22 +533,10 @@ func (rm *ResourceManager) transformDomain(domain *spec.Domain) (*spec.DGateDoma if ns, ok := rm.getNamespace(domain.NamespaceName); !ok { return nil, ErrNamespaceNotFound(domain.NamespaceName) } else { - var ( - serverCert tls.Certificate - err error - ) - if domain.Key != "" && domain.Cert != "" { - certBytes, keyBytes := []byte(domain.Key), []byte(domain.Cert) - serverCert, err = tls.X509KeyPair(certBytes, keyBytes) - if err != nil { - return nil, err - } - } return &spec.DGateDomain{ Name: domain.Name, Namespace: ns, Patterns: domain.Patterns, - TLSCert: &serverCert, Priority: domain.Priority, Cert: domain.Cert, Key: domain.Key, @@ -727,6 +770,96 @@ func (rm *ResourceManager) RemoveCollection(name, namespace string) error { return nil } +/* Secret functions */ + +func (rm *ResourceManager) GetSecret(name, namespace string) (*spec.DGateSecret, bool) { + rm.mutex.RLock() + defer rm.mutex.RUnlock() + return rm.getSecret(name, namespace) +} + +func (rm *ResourceManager) getSecret(name, namespace string) (*spec.DGateSecret, bool) { + if lk, ok := rm.secrets.Find(name + "/" + namespace); ok { + return lk.Item().Read(), true + } + return nil, false +} + +// GetSecrets returns a list of all secrets +func (rm *ResourceManager) GetSecrets() []*spec.DGateSecret { + rm.mutex.RLock() + defer rm.mutex.RUnlock() + var secrets []*spec.DGateSecret + rm.secrets.Each(func(_ string, lk *linker.Link[string, safe.Ref[spec.DGateSecret]]) bool { + secrets = append(secrets, lk.Item().Read()) + return true + }) + return secrets +} + +// GetSecretsByNamespace returns a list of all secrets in a namespace +func (rm *ResourceManager) GetSecretsByNamespace(namespace string) []*spec.DGateSecret { + rm.mutex.RLock() + defer rm.mutex.RUnlock() + var secrets []*spec.DGateSecret + if nsLk, ok := rm.namespaces.Find(namespace); ok { + nsLk.Each("secrets", func(_ string, lk linker.Linker[string]) { + mdLk := linker.NamedVertexWithVertex[string, safe.Ref[spec.DGateSecret]](lk) + secrets = append(secrets, mdLk.Item().Read()) + }) + } + return secrets +} + +func (rm *ResourceManager) AddSecret(secret *spec.Secret) (*spec.DGateSecret, error) { + rm.mutex.Lock() + defer rm.mutex.Unlock() + md, err := rm.transformSecret(secret) + if err != nil { + return nil, err + } + rw := safe.NewRef(md) + scrtLk := linker.NewNamedVertexWithValue(rw, "namespace") + if nsLk, ok := rm.namespaces.Find(secret.NamespaceName); ok { + nsLk.LinkOneMany("secrets", secret.Name, scrtLk) + scrtLk.LinkOneOne("namespace", secret.NamespaceName, nsLk) + rm.secrets.Insert(secret.Name+"/"+secret.NamespaceName, scrtLk) + return rw.Read(), nil + } else { + return nil, ErrNamespaceNotFound(secret.NamespaceName) + } +} + +func (rm *ResourceManager) transformSecret(secret *spec.Secret) (*spec.DGateSecret, error) { + if ns, ok := rm.getNamespace(secret.NamespaceName); !ok { + return nil, ErrNamespaceNotFound(secret.NamespaceName) + } else { + return spec.TransformSecret(ns, secret) + } +} + +func (rm *ResourceManager) RemoveSecret(name, namespace string) error { + rm.mutex.Lock() + defer rm.mutex.Unlock() + if scrtLink, ok := rm.secrets.Find(name + "/" + namespace); ok { + if scrtLink.Len("routes") > 0 { + return ErrCannotDeleteSecret(name, "routes still linked") + } + if nsLk, ok := rm.namespaces.Find(namespace); !ok { + return ErrNamespaceNotFound(namespace) + } else { + nsLk.UnlinkOneMany("secrets", name) + scrtLink.UnlinkOneOne("namespace") + } + if !rm.secrets.Delete(name + "/" + namespace) { + panic("failed to delete secret") + } + return nil + } else { + return ErrSecretNotFound(name) + } +} + // MarshalJSON marshals the resource manager to json func (rm *ResourceManager) MarshalJSON() ([]byte, error) { rm.mutex.RLock() @@ -818,6 +951,10 @@ func ErrRouteNotFound(name string) error { return errors.New("route not found: " + name) } +func ErrSecretNotFound(name string) error { + return errors.New("secret not found: " + name) +} + func ErrCannotDeleteModule(name, reason string) error { return errors.New("cannot delete module: " + name + ": " + reason) } @@ -841,3 +978,7 @@ func ErrCannotDeleteDomain(name, reason string) error { func ErrCannotDeleteCollection(name, reason string) error { return errors.New("cannot delete collection: " + name + ": " + reason) } + +func ErrCannotDeleteSecret(name, reason string) error { + return errors.New("cannot delete secret: " + name + ": " + reason) +} diff --git a/pkg/spec/change_log.go b/pkg/spec/change_log.go index d54409d..432a02a 100644 --- a/pkg/spec/change_log.go +++ b/pkg/spec/change_log.go @@ -62,6 +62,7 @@ const ( Domains Resource = "domain" Collections Resource = "collection" Documents Resource = "document" + Secrets Resource = "secret" ) var ( @@ -72,6 +73,7 @@ var ( AddDomainCommand Command = newCommand(Add, Domains) AddCollectionCommand Command = newCommand(Add, Collections) AddDocumentCommand Command = newCommand(Add, Documents) + AddSecretCommand Command = newCommand(Add, Secrets) DeleteRouteCommand Command = newCommand(Delete, Routes) DeleteServiceCommand Command = newCommand(Delete, Services) DeleteNamespaceCommand Command = newCommand(Delete, Namespaces) @@ -79,6 +81,7 @@ var ( DeleteDomainCommand Command = newCommand(Delete, Domains) DeleteCollectionCommand Command = newCommand(Delete, Collections) DeleteDocumentCommand Command = newCommand(Delete, Documents) + DeleteSecretCommand Command = newCommand(Delete, Secrets) NoopCommand Command = Command("noop") ) @@ -156,6 +159,8 @@ func (clc Command) Resource() Resource { return Collections case strings.HasSuffix(cmdString, "_document"): return Documents + case strings.HasSuffix(cmdString, "_secret"): + return Secrets default: panic("change log: invalid command") } diff --git a/pkg/spec/resources_ext.go b/pkg/spec/external_resources.go similarity index 93% rename from pkg/spec/resources_ext.go rename to pkg/spec/external_resources.go index 483721f..32fce70 100644 --- a/pkg/spec/resources_ext.go +++ b/pkg/spec/external_resources.go @@ -114,6 +114,19 @@ type Document struct { Data any `json:"data"` } +type Secret struct { + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + NamespaceName string `json:"namespace"` + Data string `json:"data"` + Tags []string `json:"tags,omitempty"` +} + +func (n *Secret) GetName() string { + return n.Name +} + type RFC3339Time time.Time func (t RFC3339Time) MarshalJSON() ([]byte, error) { diff --git a/pkg/spec/resources.go b/pkg/spec/internal_resources.go similarity index 83% rename from pkg/spec/resources.go rename to pkg/spec/internal_resources.go index 8f787ad..069888d 100644 --- a/pkg/spec/resources.go +++ b/pkg/spec/internal_resources.go @@ -1,7 +1,6 @@ package spec import ( - "crypto/tls" "errors" "net/url" "time" @@ -61,14 +60,15 @@ func (ns *DGateNamespace) GetName() string { } type DGateDomain struct { - Name string `json:"name"` - Namespace *DGateNamespace `json:"namespace"` - Patterns []string `json:"pattern"` - TLSCert *tls.Certificate `json:"tls_config"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Namespace *DGateNamespace `json:"namespace"` + Patterns []string `json:"pattern"` Priority int `json:"priority"` - Cert string `json:"cert"` - Key string `json:"key"` - Tags []string `json:"tags,omitempty"` + Cert string `json:"cert"` + Key string `json:"key"` + Tags []string `json:"tags,omitempty"` } var DefaultNamespace = &Namespace{ @@ -138,6 +138,19 @@ func (n *DGateDocument) GetName() string { return n.ID } +type DGateSecret struct { + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Namespace *DGateNamespace `json:"namespace"` + Data string `json:"data"` + Tags []string `json:"tags,omitempty"` +} + +func (n *DGateSecret) GetName() string { + return n.Name +} + func ErrNamespaceNotFound(ns string) error { return errors.New("namespace not found: " + ns) } @@ -165,3 +178,7 @@ func ErrRouteNotFound(route string) error { func ErrDomainNotFound(domain string) error { return errors.New("domain not found: " + domain) } + +func ErrSecretNotFound(secret string) error { + return errors.New("secret not found: " + secret) +} diff --git a/pkg/spec/transformers.go b/pkg/spec/transformers.go index e232abb..63e349a 100644 --- a/pkg/spec/transformers.go +++ b/pkg/spec/transformers.go @@ -170,6 +170,30 @@ func TransformDGateDocument(document *DGateDocument) *Document { } } +func TransformDGateSecrets(redact bool, secrets ...*DGateSecret) []*Secret { + newSecrets := make([]*Secret, len(secrets)) + for i, secret := range secrets { + newSecrets[i] = TransformDGateSecret(secret, redact) + } + return newSecrets +} + +func TransformDGateSecret(col *DGateSecret, redact bool) *Secret { + data := "--redacted--" + if !redact { + data = col.Data + if data != "" { + data = base64.StdEncoding.EncodeToString([]byte(data)) + } + } + return &Secret{ + Name: col.Name, + NamespaceName: col.Namespace.Name, + Data: data, + Tags: col.Tags, + } +} + func TransformRoutes(routes ...Route) []*DGateRoute { rts := make([]*DGateRoute, len(routes)) for i, r := range routes { @@ -314,8 +338,8 @@ func TransformCollection(ns *DGateNamespace, mods []*DGateModule, col *Collectio SchemaPayload: string(schemaData), Type: col.Type, // Modules: mods, - Visibility: col.Visibility, - Tags: col.Tags, + Visibility: col.Visibility, + Tags: col.Tags, } } @@ -350,3 +374,33 @@ func or[T any](v *T, def T) T { } return *v } + +func TransformSecrets(ns *DGateNamespace, secrets ...*Secret) ([]*DGateSecret, error) { + var err error + scrts := make([]*DGateSecret, len(secrets)) + for i, secret := range secrets { + scrts[i], err = TransformSecret(ns, secret) + if err != nil { + return nil, err + } + } + return scrts, nil +} + +func TransformSecret(ns *DGateNamespace, secret *Secret) (*DGateSecret, error) { + var payload string = "" + if secret.Data != "" { + var err error + plBytes, err := base64.RawStdEncoding.DecodeString(secret.Data) + if err != nil { + return nil, err + } + payload = string(plBytes) + } + return &DGateSecret{ + Name: secret.Name, + Namespace: ns, + Data: payload, + Tags: secret.Tags, + }, nil +} diff --git a/pkg/storage/file_storage.go b/pkg/storage/file_storage.go index 80b854b..f93d030 100644 --- a/pkg/storage/file_storage.go +++ b/pkg/storage/file_storage.go @@ -14,14 +14,12 @@ import ( type FileStoreConfig struct { Directory string `koanf:"dir"` Logger zerolog.Logger - - inMemory bool } type FileStore struct { directory string - inMemory bool logger badger.Logger + inMemory bool db *badger.DB } @@ -53,17 +51,12 @@ func NewFileStore(fsConfig *FileStoreConfig) *FileStore { fsConfig.Directory = strings.TrimSuffix(fsConfig.Directory, "/") } - logger := fsConfig.Logger.Hook(zerolog.HookFunc(func(e *zerolog.Event, level zerolog.Level, msg string) { - e.Str("storage", "filestore::badger") - })) - return &FileStore{ directory: fsConfig.Directory, logger: newBadgerLoggerAdapter( - "filestore::badger", - logger.Level(zerolog.InfoLevel), + "filestore::badger", fsConfig.Logger, ), - inMemory: fsConfig.inMemory, + inMemory: false, } } diff --git a/pkg/storage/memory_storage.go b/pkg/storage/memory_storage.go index 448a826..62d181a 100644 --- a/pkg/storage/memory_storage.go +++ b/pkg/storage/memory_storage.go @@ -20,11 +20,9 @@ var _ Storage = &MemoryStore{} func NewMemoryStore(cfg *MemoryStoreConfig) *MemoryStore { return &MemoryStore{ FileStore: &FileStore{ - directory: "", inMemory: true, logger: newBadgerLoggerAdapter( - "filestore::badger", - cfg.Logger.Level(zerolog.InfoLevel), + "memstore::badger", cfg.Logger, ), }, }