From 6d1bb06c6a57b9ca0c8df7d0d2ef2ef1e929c300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Thu, 22 Aug 2024 13:38:52 +0200 Subject: [PATCH 1/9] chore(ci): remove EOLd versions from test matrix (#131) --- .github/workflows/integration-enterprise.yaml | 12 ------------ .github/workflows/integration.yaml | 14 -------------- 2 files changed, 26 deletions(-) diff --git a/.github/workflows/integration-enterprise.yaml b/.github/workflows/integration-enterprise.yaml index e374323..1027c79 100644 --- a/.github/workflows/integration-enterprise.yaml +++ b/.github/workflows/integration-enterprise.yaml @@ -15,19 +15,7 @@ jobs: strategy: matrix: kong_image: - - 'kong/kong-gateway:1.5.0.11' - - 'kong/kong-gateway:2.1.4.6' - - 'kong/kong-gateway:2.2.1.3' - - 'kong/kong-gateway:2.3.3.4' - - 'kong/kong-gateway:2.4.1.3' - - 'kong/kong-gateway:2.5.1.2' - - 'kong/kong-gateway:2.6.0.2' - - 'kong/kong-gateway:2.7' - 'kong/kong-gateway:2.8' - - 'kong/kong-gateway:3.0' - - 'kong/kong-gateway:3.1' - - 'kong/kong-gateway:3.2' - - 'kong/kong-gateway:3.3' - 'kong/kong-gateway:3.4' - 'kong/kong-gateway:3.5' - 'kong/kong-gateway:3.6' diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 0e6a549..359789d 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -15,21 +15,7 @@ jobs: strategy: matrix: kong_image: - - 'kong:1.4.3' - - 'kong:1.5.1' - - 'kong:2.0.5' - - 'kong:2.1.4' - - 'kong:2.2.2' - - 'kong:2.3.3' - - 'kong:2.4.1' - - 'kong:2.5.1' - - 'kong:2.6.0' - - 'kong:2.7' - 'kong:2.8' - - 'kong:3.0' - - 'kong:3.1' - - 'kong:3.2' - - 'kong:3.3' - 'kong:3.4' - 'kong:3.5' - 'kong:3.6' From 64dd801ebea08b172a2049c139b8665001091f5c Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Tue, 27 Aug 2024 15:50:48 +0530 Subject: [PATCH 2/9] fix: plugin shown as global despite a consumer-group present (#134) * fix: plugin shown as global despite a consumer-group present Prior to this fix, deck was reporting the plugins as global even if a consumer-group was attached to them. This was only a output problem, not a association problem. The fix checks if a consumer-group exists or not before reporting any plugin as global. Fixes: https://github.com/Kong/deck/issues/1005 * chore: moved to table-driven tests for plugin console function --- pkg/state/types.go | 2 +- pkg/state/types_test.go | 63 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/pkg/state/types.go b/pkg/state/types.go index 751bd7d..ca83fd7 100644 --- a/pkg/state/types.go +++ b/pkg/state/types.go @@ -526,7 +526,7 @@ func (p1 *Plugin) Identifier() string { func (p1 *Plugin) Console() string { res := *p1.Name + " " - if p1.Service == nil && p1.Route == nil && p1.Consumer == nil { + if p1.Service == nil && p1.Route == nil && p1.Consumer == nil && p1.ConsumerGroup == nil { return res + "(global)" } associations := []string{} diff --git a/pkg/state/types_test.go b/pkg/state/types_test.go index 3381cfd..9e87611 100644 --- a/pkg/state/types_test.go +++ b/pkg/state/types_test.go @@ -663,3 +663,66 @@ func TestDeepEqualWithSorting(t *testing.T) { t.Errorf("expected maps to be equal, but they are not") } } + +func TestPluginConsole(t *testing.T) { + tests := []struct { + plugin kong.Plugin + name string + expected string + }{ + { + name: "plugin default case", + plugin: kong.Plugin{}, + expected: "foo-plugin (global)", + }, + { + name: "plugin associated with service", + plugin: kong.Plugin{ + Service: &kong.Service{ID: kong.String("bar")}, + }, + expected: "foo-plugin for service bar", + }, + { + name: "plugin associated with route", + plugin: kong.Plugin{ + Route: &kong.Route{ID: kong.String("baz")}, + }, + expected: "foo-plugin for route baz", + }, + { + name: "plugin associated with consumer", + plugin: kong.Plugin{ + Consumer: &kong.Consumer{ID: kong.String("demo")}, + }, + expected: "foo-plugin for consumer demo", + }, + { + name: "plugin associated with consumer group", + plugin: kong.Plugin{ + ConsumerGroup: &kong.ConsumerGroup{ID: kong.String("demo-group")}, + }, + expected: "foo-plugin for consumer-group demo-group", + }, + { + name: "plugin associated with >1 entities", + plugin: kong.Plugin{ + Service: &kong.Service{ID: kong.String("bar")}, + Route: &kong.Route{ID: kong.String("baz")}, + Consumer: &kong.Consumer{ID: kong.String("demo")}, + ConsumerGroup: &kong.ConsumerGroup{ID: kong.String("demo-group")}, + }, + expected: "foo-plugin for service bar and route baz and consumer demo and consumer-group demo-group", + }, + } + for _, tt := range tests { + var p1 Plugin + p1.Plugin = tt.plugin + p1.ID = kong.String("foo") + p1.Name = kong.String("foo-plugin") + + t.Run(tt.name, func(t *testing.T) { + actual := p1.Console() + assert.Equal(t, tt.expected, actual) + }) + } +} From e72f4c2de6609998d5e5671e14a9f7c0cd6a4c3e Mon Sep 17 00:00:00 2001 From: Samuele Date: Tue, 27 Aug 2024 18:58:40 +0200 Subject: [PATCH 3/9] fix(diff+file): avoid filling defaults in config for kong (#133) * fix(diff+file): avoid filling defaults in config for kong the previous logic was filling defaults in the configuration that was passed to Kong. This was problematic, especially where nils were populated as defaults, e.g. if a shorthand_field was passed with some value and the corresponding new field is auto-populated as `nil` by decK , the auto-populated nil value would take precedence in Kong thus causing the shorthand_field to be ignored (https://konghq.atlassian.net/browse/KAG-5157). This change applies the default values only to configurations used for diff, the original configuration is always passed to Kong as is. * Test/avoid filling defaults in config for kong (#135) * add tests for plugin filling default values --------- Co-authored-by: samugi --------- Co-authored-by: Tao Yi --- go.mod | 9 +- go.sum | 22 ++-- pkg/diff/diff.go | 32 +++++- pkg/file/builder.go | 41 ------- tests/integration/sync_test.go | 105 ++++++++++++++++++ tests/integration/test_utils.go | 71 ++++++++++++ .../033-plugin-with-empty-fields/kong.yaml | 11 ++ 7 files changed, 235 insertions(+), 56 deletions(-) create mode 100644 tests/integration/testdata/sync/033-plugin-with-empty-fields/kong.yaml diff --git a/go.mod b/go.mod index 90a4741..d805e23 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/hexops/gotextdiff v1.0.3 github.com/kong/deck v1.34.0 github.com/kong/go-kong v0.55.0 + github.com/samber/lo v1.47.0 github.com/shirou/gopsutil/v3 v3.24.5 github.com/ssgelm/cookiejarparser v1.0.1 github.com/stretchr/testify v1.9.0 @@ -115,11 +116,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.23.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 65389e1..aba871e 100644 --- a/go.sum +++ b/go.sum @@ -282,6 +282,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -364,16 +366,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -386,8 +388,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -440,16 +442,16 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index 0eb6a0a..f9b19ac 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -599,6 +599,36 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu var err error var result crud.Arg + // This variable holds the original event with the unchanged configuration + // Below the configuration in `e` may be modified. This is done solely for + // the purpose of displaying a correct diff and should not affect the + // configuration that is sent to Kong. + originalE := e + + // If the event is for a plugin, inject defaults in the plugin's config + // that will be used for the diff. This is needed to avoid highlighting + // default values that were populated by Kong as differences. + if plugin, ok := e.Obj.(*state.Plugin); ok { + pluginCopy := &state.Plugin{ + Plugin: *plugin.DeepCopy(), + Meta: plugin.Meta, + } + e.Obj = pluginCopy + + exists, err := utils.WorkspaceExists(ctx, sc.kongClient) + if err == nil && exists { + var schema map[string]interface{} + schema, err = sc.kongClient.Plugins.GetFullSchema(ctx, pluginCopy.Plugin.Name) + if err != nil { + return nil, err + } + + if err := kong.FillPluginsDefaults(&pluginCopy.Plugin, schema); err != nil { + return nil, fmt.Errorf("failed filling plugin defaults: %w", err) + } + } + } + c := e.Obj.(state.ConsoleString) objDiff := map[string]interface{}{ "old": e.OldObj, @@ -680,7 +710,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu if !dry { // sync mode // fire the request to Kong - result, err = sc.processor.Do(ctx, e.Kind, e.Op, e) + result, err = sc.processor.Do(ctx, originalE.Kind, originalE.Op, originalE) // TODO https://github.com/Kong/go-database-reconciler/issues/22 this does not print, but is switched on // sc.enableEntityActions because the existing behavior returns a result from the anon Run function. // Refactoring should use only the channel and simplify the return, probably to just an error (all the other diff --git a/pkg/file/builder.go b/pkg/file/builder.go index c0bf27f..d24687b 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -1443,44 +1443,6 @@ func (b *stateBuilder) ingestRoute(r FRoute) error { return nil } -func (b *stateBuilder) getPluginSchema(pluginName string) (map[string]interface{}, error) { - var schema map[string]interface{} - - // lookup in cache - if schema, ok := b.schemasCache[pluginName]; ok { - return schema, nil - } - - exists, err := utils.WorkspaceExists(b.ctx, b.client) - if err != nil { - return nil, fmt.Errorf("ensure workspace exists: %w", err) - } - if !exists { - return schema, ErrWorkspaceNotFound - } - - schema, err = b.client.Plugins.GetFullSchema(b.ctx, &pluginName) - if err != nil { - return schema, err - } - b.schemasCache[pluginName] = schema - return schema, nil -} - -func (b *stateBuilder) addPluginDefaults(plugin *FPlugin) error { - if b.client == nil { - return nil - } - schema, err := b.getPluginSchema(*plugin.Name) - if err != nil { - if errors.Is(err, ErrWorkspaceNotFound) { - return nil - } - return fmt.Errorf("retrieve schema for %v from Kong: %w", *plugin.Name, err) - } - return kong.FillPluginsDefaults(&plugin.Plugin, schema) -} - func (b *stateBuilder) ingestPlugins(plugins []FPlugin) error { for _, p := range plugins { p := p @@ -1504,9 +1466,6 @@ func (b *stateBuilder) ingestPlugins(plugins []FPlugin) error { if err != nil { return err } - if err := b.addPluginDefaults(&p); err != nil { - return fmt.Errorf("add defaults to plugin '%v': %w", *p.Name, err) - } utils.MustMergeTags(&p, b.selectTags) if plugin != nil { p.Plugin.CreatedAt = plugin.CreatedAt diff --git a/tests/integration/sync_test.go b/tests/integration/sync_test.go index 0f94de9..ba5019b 100644 --- a/tests/integration/sync_test.go +++ b/tests/integration/sync_test.go @@ -7,9 +7,13 @@ import ( "crypto/sha1" "crypto/tls" "crypto/x509" + "encoding/json" "errors" "fmt" + "io" "net/http" + "net/http/httptest" + "net/url" "os" "strings" "testing" @@ -19,8 +23,10 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" deckDiff "github.com/kong/go-database-reconciler/pkg/diff" deckDump "github.com/kong/go-database-reconciler/pkg/dump" + "github.com/kong/go-database-reconciler/pkg/state" "github.com/kong/go-database-reconciler/pkg/utils" "github.com/kong/go-kong/kong" + "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -5522,3 +5528,102 @@ func TestSync_License(t *testing.T) { require.Empty(t, licenses) }) } + +func Test_Sync_PluginDoNotFillDefaults(t *testing.T) { + + client, err := getTestClient() + + require.NoError(t, err) + ctx := context.Background() + t.Run("empty_fields_of_plugin_config", func(t *testing.T) { + mustResetKongState(ctx, t, client, deckDump.Config{}) + + currrentState, err := fetchCurrentState(ctx, client, deckDump.Config{}) + require.NoError(t, err) + targetState := stateFromFile(ctx, t, + "testdata/sync/033-plugin-with-empty-fields/kong.yaml", + client, + deckDump.Config{}, + ) + + kongURL, err := url.Parse(client.BaseRootURL()) + require.NoError(t, err) + p := NewRecordRequestProxy(kongURL) + s := httptest.NewServer(p) + c, err := utils.GetKongClient(utils.KongClientConfig{ + Address: s.URL, + }) + require.NoError(t, err) + + syncer, err := deckDiff.NewSyncer(deckDiff.SyncerOpts{ + CurrentState: currrentState, + TargetState: targetState, + + KongClient: c, + }) + stats, errs, changes := syncer.Solve(ctx, 1, false, true) + require.Empty(t, errs, "Should have no errors in syncing") + require.NoError(t, err) + + require.Equal(t, int32(1), stats.CreateOps.Count(), "Should create 1 entity") + require.Len(t, changes.Creating, 1, "Should have 1 creating record in changes") + + // The change records which are returned in `diff` command should fill default values. + t.Run("should fill default values in change records", func(t *testing.T) { + body, ok := changes.Creating[0].Body.(map[string]any) + require.True(t, ok) + plugin, ok := body["new"].(*state.Plugin) + require.True(t, ok) + + path, ok := plugin.Config["path"] + require.True(t, ok) + require.Equal(t, "/tmp/file.log", path, "path should be same as specified in file") + + reopen, ok := plugin.Config["reopen"] + require.True(t, ok, "'reopen' field should be filled") + require.Equal(t, false, reopen, "should be the same as default value") + + custom_fields_by_lua, ok := plugin.Config["custom_fields_by_lua"] + require.True(t, ok, "'custom_fields_by_lua' field should be filled") + require.Nil(t, custom_fields_by_lua, "should be an explicit nil") + }) + + // But the default values should not be filled in request sent to Kong. + t.Run("should not fill default values in requests sent to Kong", func(t *testing.T) { + reqs := p.dumpRequests() + req, found := lo.Find(reqs, func(r *http.Request) bool { + return r.Method == "PUT" && strings.Contains(r.URL.Path, "/plugins") + }) + require.True(t, found, "Should find request to create plugin") + buf, err := io.ReadAll(req.Body) + require.NoError(t, err, "Should read request body from record") + plugin := state.Plugin{} + err = json.Unmarshal(buf, &plugin) + require.NoError(t, err, "Should unmarshal request body to plugin type") + + path, ok := plugin.Config["path"] + require.True(t, ok) + require.Equal(t, "/tmp/file.log", path, "path should be same as specified in file") + + _, ok = plugin.Config["reopen"] + require.False(t, ok, "'reopen' field should not be filled") + + _, ok = plugin.Config["custom_fields_by_lua"] + require.False(t, ok, "'custom_fields_by_lua' field should not be filled") + }) + + // Should update Kong state successfully. + t.Run("Should get the plugin config from update Kong", func(t *testing.T) { + newState, err := fetchCurrentState(ctx, client, deckDump.Config{}) + require.NoError(t, err) + plugins, err := newState.Plugins.GetAll() + require.NoError(t, err) + require.Len(t, plugins, 1) + plugin := plugins[0] + require.Equal(t, "file-log", *plugin.Name) + path, ok := plugin.Config["path"] + require.True(t, ok) + require.Equal(t, "/tmp/file.log", path, "path should be same as specified in file") + }) + }) +} diff --git a/tests/integration/test_utils.go b/tests/integration/test_utils.go index fcfcaac..8cddbad 100644 --- a/tests/integration/test_utils.go +++ b/tests/integration/test_utils.go @@ -2,9 +2,14 @@ package integration import ( + "bytes" "context" "io" + "net/http" + "net/http/httputil" + "net/url" "os" + gosync "sync" "testing" "github.com/acarl005/stripansi" @@ -382,6 +387,28 @@ func getKongVersion(ctx context.Context, t *testing.T, client *kong.Client) semv } } +// mustResetKongState resets Kong state. Intended to replace `reset` which uses deck command. +func mustResetKongState(ctx context.Context, t *testing.T, client *kong.Client, dumpConfig deckDump.Config) { + t.Helper() + + emptyRawState := utils.KongRawState{} + targetState, err := state.Get(&emptyRawState) + require.NoError(t, err) + + currentState, err := fetchCurrentState(ctx, client, dumpConfig) + require.NoError(t, err, "failed to fetch current state") + + sc, err := deckDiff.NewSyncer(deckDiff.SyncerOpts{ + CurrentState: currentState, + TargetState: targetState, + KongClient: client, + }) + require.NoError(t, err, "failed to create syncer") + + _, errs, _ := sc.Solve(ctx, 1, false, false) + require.Empty(t, errs, 0, "failed to apply diffs to Kong: %d errors occurred", len(errs)) +} + func stateFromFile( ctx context.Context, t *testing.T, filename string, client *kong.Client, dumpConfig deckDump.Config, @@ -420,3 +447,47 @@ func logEntityChanges(t *testing.T, stats deckDiff.Stats, entityChanges deckDiff stats.UpdateOps.Count(), ) } + +// recordRequestProxy is a reverse proxy of Kong gateway admin API endpoints +// to record the request sent to Kong. +type RecordRequestProxy struct { + lock gosync.RWMutex + proxy *httputil.ReverseProxy + requests []*http.Request +} + +// NewRecordRequestProxy returns a recordRequestProxy sending requests to the target URL. +func NewRecordRequestProxy(target *url.URL) *RecordRequestProxy { + return &RecordRequestProxy{ + proxy: httputil.NewSingleHostReverseProxy(target), + } +} + +func (p *RecordRequestProxy) addRequest(req *http.Request, bodyContent []byte) { + p.lock.Lock() + defer p.lock.Unlock() + // Create a new reader to replace the body because the original body closes after request sent. + reader := io.NopCloser(bytes.NewBuffer(bodyContent)) + req.Body = reader + p.requests = append(p.requests, req) +} + +func (p *RecordRequestProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + buf, _ := io.ReadAll(req.Body) + p.addRequest(req.Clone(context.Background()), buf) + reader := io.NopCloser(bytes.NewBuffer(buf)) + req.Body = reader + p.proxy.ServeHTTP(rw, req) +} + +func (p *RecordRequestProxy) dumpRequests() []*http.Request { + p.lock.RLock() + defer p.lock.RUnlock() + reqs := make([]*http.Request, 0, len(p.requests)) + for _, req := range p.requests { + reqs = append(reqs, req.Clone(context.Background())) + } + return reqs +} + +var _ http.Handler = &RecordRequestProxy{} diff --git a/tests/integration/testdata/sync/033-plugin-with-empty-fields/kong.yaml b/tests/integration/testdata/sync/033-plugin-with-empty-fields/kong.yaml new file mode 100644 index 0000000..ca3533a --- /dev/null +++ b/tests/integration/testdata/sync/033-plugin-with-empty-fields/kong.yaml @@ -0,0 +1,11 @@ +_format_version: "3.0" +plugins: +- config: + path: /tmp/file.log + enabled: true + name: file-log + protocols: + - grpc + - grpcs + - http + - https From 25ccc04e6c42e1b447070391e3938913deaed9fb Mon Sep 17 00:00:00 2001 From: Samuele Date: Mon, 2 Sep 2024 04:46:18 +0200 Subject: [PATCH 4/9] chore(deps): bump go-kong to 0.58 (#137) * chore(deps): bump go-kong * fix(rbac): ? --- go.mod | 22 ++++++------ go.sum | 68 ++++++++++++++----------------------- pkg/state/rbac_role_test.go | 5 +-- 3 files changed, 41 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index d805e23..30ef44f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/kong/go-database-reconciler -go 1.21.1 +go 1.22.0 + +toolchain go1.22.4 replace github.com/yudai/gojsondiff v1.0.0 => github.com/Kong/gojsondiff v1.3.0 @@ -19,7 +21,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hexops/gotextdiff v1.0.3 github.com/kong/deck v1.34.0 - github.com/kong/go-kong v0.55.0 + github.com/kong/go-kong v0.58.0 github.com/samber/lo v1.47.0 github.com/shirou/gopsutil/v3 v3.24.5 github.com/ssgelm/cookiejarparser v1.0.1 @@ -27,7 +29,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.23.0 - k8s.io/code-generator v0.29.4 + k8s.io/code-generator v0.31.0 sigs.k8s.io/yaml v1.4.0 ) @@ -48,12 +50,12 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getkin/kin-openapi v0.108.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -103,7 +105,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.18.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/gjson v1.17.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -121,12 +123,12 @@ require ( golang.org/x/sys v0.23.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index aba871e..987568d 100644 --- a/go.sum +++ b/go.sum @@ -21,10 +21,6 @@ github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/ github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= -github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= -github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adrg/strutil v0.2.3 h1:WZVn3ItPBovFmP4wMHHVXUr8luRaHrbyIuLlHt32GZQ= @@ -80,11 +76,9 @@ github.com/getkin/kin-openapi v0.108.0 h1:EYf0GtsKa4hQNIlplGS+Au7NEfGQ1F7MoHD2kc github.com/getkin/kin-openapi v0.108.0/go.mod h1:QtwUNt0PAAgIIBEvFWYfB7dfngxtAaqCX1zYHMZDeK8= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -95,13 +89,14 @@ github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaL github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v2.20.0+incompatible h1:4Xh3bDzO29j4TWNOI+24ubc0vbVFMg2PMnXKxK54/CA= -github.com/go-task/slim-sprig v2.20.0+incompatible/go.mod h1:N/mhXZITr/EQAOErEHciKvO1bFei2Lld2Ym6h96pdy0= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -127,7 +122,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -163,8 +157,6 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= -github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -186,14 +178,13 @@ github.com/kong/deck v1.34.0 h1:iKSa5Cq8t7bdCv5R3XRV8UH+7FvnZB/YxhRTuih4rgg= github.com/kong/deck v1.34.0/go.mod h1:IUAixgNa1YSvEphgX9OIHmDyyi1JRaGTB0bYieuD9qo= github.com/kong/go-apiops v0.1.29 h1:c+AB8MmGIr+K01Afm4GB2xaOmJnD/8KWMJQkr9qssnc= github.com/kong/go-apiops v0.1.29/go.mod h1:ZNdiTZyVrAssB4wjEYWV7BfpcV9UME9LxnDDZhMPuNU= -github.com/kong/go-kong v0.55.0 h1:lonKRzsDGk12dh9E+y+pWnY2ThXhKuMHjzBHSpCvQLw= -github.com/kong/go-kong v0.55.0/go.mod h1:i1cMgTu6RYPHSyMpviShddRnc+DML/vlpgKC00hr8kU= +github.com/kong/go-kong v0.58.0 h1:9Y1Hg7ahCS7Ig+iAb9CZyjeT3ARuDe9ohV7mVa8n/eU= +github.com/kong/go-kong v0.58.0/go.mod h1:8Vt6HmtgLNgL/7bSwAlz3DIWqBtzG7qEt9+OnMiQOa0= github.com/kong/go-slugify v1.0.0 h1:vCFAyf2sdoSlBtLcrmDWUFn0ohlpKiKvQfXZkO5vSKY= github.com/kong/go-slugify v1.0.0/go.mod h1:dbR2h3J2QKXQ1k0aww6cN7o4cIcwlWflr6RKRdcoaiw= github.com/kong/semver/v4 v4.0.1 h1:DIcNR8W3gfx0KabFBADPalxxsp+q/5COwIFkkhrFQ2Y= github.com/kong/semver/v4 v4.0.1/go.mod h1:LImQ0oT15pJvSns/hs2laLca2zcYoHu5EsSNY0J6/QA= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -249,8 +240,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/pb33f/libopenapi v0.13.11 h1:CHRT15/iakHcwRvr9y7bf63UPekXa8FB1Sc4D4BZ7NU= github.com/pb33f/libopenapi v0.13.11/go.mod h1:Lv2eEtsAtbRFlF8hjH82L8SIGoUNgemMVoKoB6A9THk= github.com/pb33f/libopenapi-validator v0.0.28 h1:XOKGLuRLkHtkiPvm4x1JZgqVqFyD2tPx15qx+aSeaBE= @@ -275,8 +266,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= @@ -327,8 +318,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= @@ -353,7 +344,6 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3Ifn github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.2-0.20150410014804-be8315415630+incompatible h1:TmF93o7P81230DTx1l2zw5rZbsDpOOQXoKVCa8+nXXI= github.com/yudai/pp v2.0.2-0.20150410014804-be8315415630+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= @@ -366,11 +356,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -380,7 +367,6 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -392,7 +378,6 @@ golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -446,7 +431,6 @@ golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= @@ -464,8 +448,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -490,19 +474,19 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/code-generator v0.29.4 h1:8ESudFNbY5/9BzB8KOEFG2uV9Q0AQxkc4mrQESr30Ks= -k8s.io/code-generator v0.29.4/go.mod h1:7TYnI0dYItL2cKuhhgPSuF3WED9uMdELgbVXFfn/joE= -k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks= -k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/code-generator v0.31.0 h1:w607nrMi1KeDKB3/F/J4lIoOgAwc+gV9ZKew4XRfMp8= +k8s.io/code-generator v0.31.0/go.mod h1:84y4w3es8rOJOUUP1rLsIiGlO1JuEaPFXQPA9e/K6U0= +k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo= +k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/state/rbac_role_test.go b/pkg/state/rbac_role_test.go index 4a4998a..8e8f945 100644 --- a/pkg/state/rbac_role_test.go +++ b/pkg/state/rbac_role_test.go @@ -105,7 +105,8 @@ func TestRBACRolesCollection_Get(t *testing.T) { } rbacRole1 := RBACRole{ RBACRole: kong.RBACRole{ - ID: kong.String("foo-id"), + ID: kong.String("foo-id"), + Name: kong.String("foo-name"), }, } rbacRole2 := RBACRole{ @@ -157,7 +158,7 @@ func TestRBACRolesCollection_Get(t *testing.T) { k.Add(rbacRole1) k.Add(rbacRole2) for _, tt := range tests { - tc := &tt //nolint:gosec + tc := &tt t.Run(tt.name, func(t *testing.T) { t.Parallel() got, err := k.Get(tc.args.nameOrID) From 0b1b75fbef0ab1cdb10d8bfccadbb0b4f3f4c043 Mon Sep 17 00:00:00 2001 From: Samuele Date: Tue, 3 Sep 2024 09:08:53 +0200 Subject: [PATCH 5/9] fix(diff): support auto fields (#136) * fix(diff): support auto fields call the new utils function FillPluginsDefaultsAutoFields from go-kong to ensure auto fields are considered when doing the diff * fix(crud): detect operation correctly fixes a bug that was introduced in e72f4c2de6609998d5e5671e14a9f7c0cd6a4c3e where noops were still detected as updates. That happened because of a check happening in plugin.go that detected the crud operation based on a diff. When done without considering default/auto fields, this check would see differences where there were none. This resulted in unnecessary updates to the DB and wrong diff strings during `sync` and `diff`. This commit adds defaults/auto fields to the configuration used for that diff. * pr review suggestions --- pkg/diff/diff.go | 14 +++-- pkg/types/core.go | 1 + pkg/types/plugin.go | 13 +++- tests/integration/sync_test.go | 59 +++++++++++++++++++ .../sync/034-fill-auto-oauth2/kong.yaml | 5 ++ 5 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 tests/integration/testdata/sync/034-fill-auto-oauth2/kong.yaml diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index f9b19ac..0fbf690 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -609,10 +609,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu // that will be used for the diff. This is needed to avoid highlighting // default values that were populated by Kong as differences. if plugin, ok := e.Obj.(*state.Plugin); ok { - pluginCopy := &state.Plugin{ - Plugin: *plugin.DeepCopy(), - Meta: plugin.Meta, - } + pluginCopy := &state.Plugin{Plugin: *plugin.DeepCopy()} e.Obj = pluginCopy exists, err := utils.WorkspaceExists(ctx, sc.kongClient) @@ -623,8 +620,13 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu return nil, err } - if err := kong.FillPluginsDefaults(&pluginCopy.Plugin, schema); err != nil { - return nil, fmt.Errorf("failed filling plugin defaults: %w", err) + var oldPlugin *kong.Plugin + if kongStatePlugin, ok := e.OldObj.(*state.Plugin); ok { + oldPlugin = &kongStatePlugin.Plugin + } + newPlugin := &pluginCopy.Plugin + if err := kong.FillPluginsDefaultsAutoFields(newPlugin, schema, oldPlugin); err != nil { + return nil, fmt.Errorf("failed processing auto fields: %w", err) } } } diff --git a/pkg/types/core.go b/pkg/types/core.go index ed04ce1..912c970 100644 --- a/pkg/types/core.go +++ b/pkg/types/core.go @@ -232,6 +232,7 @@ func NewEntity(t EntityType, opts EntityOpts) (Entity, error) { kind: entityTypeToKind(Plugin), currentState: opts.CurrentState, targetState: opts.TargetState, + kongClient: opts.KongClient, }, }, nil case Consumer: diff --git a/pkg/types/plugin.go b/pkg/types/plugin.go index 8e3b140..ae09493 100644 --- a/pkg/types/plugin.go +++ b/pkg/types/plugin.go @@ -88,6 +88,7 @@ type pluginDiffer struct { kind crud.Kind currentState, targetState *state.KongState + kongClient *kong.Client } func (d *pluginDiffer) Deletes(handler func(crud.Event) error) error { @@ -174,8 +175,18 @@ func (d *pluginDiffer) createUpdatePlugin(plugin *state.Plugin) (*crud.Event, er } currentPlugin = &state.Plugin{Plugin: *currentPlugin.DeepCopy()} // found, check if update needed + // before checking the diff, fill in the defaults + schema, err := d.kongClient.Plugins.GetFullSchema(context.TODO(), plugin.Name) + if err != nil { + return nil, fmt.Errorf("failed getting schema: %w", err) + } + pluginWithDefaults := &state.Plugin{Plugin: *plugin.DeepCopy()} + err = kong.FillPluginsDefaultsAutoFields(&pluginWithDefaults.Plugin, schema, ¤tPlugin.Plugin) + if err != nil { + return nil, fmt.Errorf("failed processing auto fields: %w", err) + } - if !currentPlugin.EqualWithOpts(plugin, false, true, false) { + if !currentPlugin.EqualWithOpts(pluginWithDefaults, false, true, false) { return &crud.Event{ Op: crud.Update, Kind: d.kind, diff --git a/tests/integration/sync_test.go b/tests/integration/sync_test.go index ba5019b..3269e2b 100644 --- a/tests/integration/sync_test.go +++ b/tests/integration/sync_test.go @@ -15,6 +15,7 @@ import ( "net/http/httptest" "net/url" "os" + "regexp" "strings" "testing" "time" @@ -5627,3 +5628,61 @@ func Test_Sync_PluginDoNotFillDefaults(t *testing.T) { }) }) } + +func Test_Sync_PluginAutoFields(t *testing.T) { + client, err := getTestClient() + + require.NoError(t, err) + ctx := context.Background() + t.Run("plugin_with_auto_fields", func(t *testing.T) { + mustResetKongState(ctx, t, client, deckDump.Config{}) + + currentState, err := fetchCurrentState(ctx, client, deckDump.Config{}) + require.NoError(t, err) + targetState := stateFromFile(ctx, t, + "testdata/sync/034-fill-auto-oauth2/kong.yaml", + client, + deckDump.Config{}, + ) + + kongURL, err := url.Parse(client.BaseRootURL()) + require.NoError(t, err) + p := NewRecordRequestProxy(kongURL) + s := httptest.NewServer(p) + c, err := utils.GetKongClient(utils.KongClientConfig{ + Address: s.URL, + }) + require.NoError(t, err) + + syncer, err := deckDiff.NewSyncer(deckDiff.SyncerOpts{ + CurrentState: currentState, + TargetState: targetState, + + KongClient: c, + }) + stats, errs, changes := syncer.Solve(ctx, 1, false, true) + require.Empty(t, errs, "Should have no errors in syncing") + require.NoError(t, err) + + require.Equal(t, int32(1), stats.CreateOps.Count(), "Should create 1 entity") + require.Len(t, changes.Creating, 1, "Should have 1 creating record in changes") + + t.Run("should not override auto values with nils", func(t *testing.T) { + newState, err := fetchCurrentState(ctx, client, deckDump.Config{}) + require.NoError(t, err) + plugins, err := newState.Plugins.GetAll() + require.NoError(t, err) + require.Len(t, plugins, 1) + plugin := plugins[0] + require.Equal(t, "oauth2", *plugin.Name) + provisionKey, ok := plugin.Config["provision_key"] + require.True(t, ok) + + provisionKeyStr, ok := provisionKey.(string) + require.True(t, ok, "provision_key is not a string") + pattern := `^[a-zA-Z0-9]+$` + re := regexp.MustCompile(pattern) + require.True(t, re.MatchString(provisionKeyStr), "provision_key does not match the pattern") + }) + }) +} diff --git a/tests/integration/testdata/sync/034-fill-auto-oauth2/kong.yaml b/tests/integration/testdata/sync/034-fill-auto-oauth2/kong.yaml new file mode 100644 index 0000000..59ba624 --- /dev/null +++ b/tests/integration/testdata/sync/034-fill-auto-oauth2/kong.yaml @@ -0,0 +1,5 @@ +_format_version: "3.0" +plugins: + - name: oauth2 + config: + enable_password_grant: true From 9c38d54c510e39f16bbb417d1cb81411b91438da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 06:57:15 +0000 Subject: [PATCH 6/9] chore(deps): bump golang.org/x/term from 0.23.0 to 0.24.0 (#141) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.23.0 to 0.24.0. - [Commits](https://github.com/golang/term/compare/v0.23.0...v0.24.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 30ef44f..f05697e 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.8.0 - golang.org/x/term v0.23.0 + golang.org/x/term v0.24.0 k8s.io/code-generator v0.31.0 sigs.k8s.io/yaml v1.4.0 ) @@ -120,7 +120,7 @@ require ( golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index 987568d..7a90c0f 100644 --- a/go.sum +++ b/go.sum @@ -411,15 +411,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 7c622bd0416e19c7f2416ef0afc4f3bea61b842d Mon Sep 17 00:00:00 2001 From: Samuele Date: Thu, 5 Sep 2024 10:06:05 +0200 Subject: [PATCH 7/9] fix(sync): bump go-kong + fill auto fields with nil (#139) * chore(deps): bump go-kong to 0.59 * fix(sync): fill auto fields with nil Fill auto fields with nil in the config sent to the CP. The rest of the default values are still not filled in the configuration that is sent out, as they are meant to be filled by the CP itself, but auto fields should not be generated, to ensure the configuration remains truly "declarative". --------- Co-authored-by: Tao Yi --- go.mod | 2 +- go.sum | 4 ++-- pkg/diff/diff.go | 21 ++++++++++++++------- pkg/types/plugin.go | 2 +- tests/integration/sync_test.go | 29 +++++------------------------ 5 files changed, 23 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index f05697e..e167cbc 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hexops/gotextdiff v1.0.3 github.com/kong/deck v1.34.0 - github.com/kong/go-kong v0.58.0 + github.com/kong/go-kong v0.59.0 github.com/samber/lo v1.47.0 github.com/shirou/gopsutil/v3 v3.24.5 github.com/ssgelm/cookiejarparser v1.0.1 diff --git a/go.sum b/go.sum index 7a90c0f..3c7c32d 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,8 @@ github.com/kong/deck v1.34.0 h1:iKSa5Cq8t7bdCv5R3XRV8UH+7FvnZB/YxhRTuih4rgg= github.com/kong/deck v1.34.0/go.mod h1:IUAixgNa1YSvEphgX9OIHmDyyi1JRaGTB0bYieuD9qo= github.com/kong/go-apiops v0.1.29 h1:c+AB8MmGIr+K01Afm4GB2xaOmJnD/8KWMJQkr9qssnc= github.com/kong/go-apiops v0.1.29/go.mod h1:ZNdiTZyVrAssB4wjEYWV7BfpcV9UME9LxnDDZhMPuNU= -github.com/kong/go-kong v0.58.0 h1:9Y1Hg7ahCS7Ig+iAb9CZyjeT3ARuDe9ohV7mVa8n/eU= -github.com/kong/go-kong v0.58.0/go.mod h1:8Vt6HmtgLNgL/7bSwAlz3DIWqBtzG7qEt9+OnMiQOa0= +github.com/kong/go-kong v0.59.0 h1:U6dE2sqb8E8j0kESW/RCW9TkXH8Y3W0EtNDXJVsDNuM= +github.com/kong/go-kong v0.59.0/go.mod h1:8Vt6HmtgLNgL/7bSwAlz3DIWqBtzG7qEt9+OnMiQOa0= github.com/kong/go-slugify v1.0.0 h1:vCFAyf2sdoSlBtLcrmDWUFn0ohlpKiKvQfXZkO5vSKY= github.com/kong/go-slugify v1.0.0/go.mod h1:dbR2h3J2QKXQ1k0aww6cN7o4cIcwlWflr6RKRdcoaiw= github.com/kong/semver/v4 v4.0.1 h1:DIcNR8W3gfx0KabFBADPalxxsp+q/5COwIFkkhrFQ2Y= diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index 0fbf690..56ca0bb 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -603,7 +603,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu // Below the configuration in `e` may be modified. This is done solely for // the purpose of displaying a correct diff and should not affect the // configuration that is sent to Kong. - originalE := e + eventForKong := e // If the event is for a plugin, inject defaults in the plugin's config // that will be used for the diff. This is needed to avoid highlighting @@ -620,12 +620,19 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu return nil, err } - var oldPlugin *kong.Plugin - if kongStatePlugin, ok := e.OldObj.(*state.Plugin); ok { - oldPlugin = &kongStatePlugin.Plugin - } + // fill defaults and auto fields for the configuration that will be used for the diff newPlugin := &pluginCopy.Plugin - if err := kong.FillPluginsDefaultsAutoFields(newPlugin, schema, oldPlugin); err != nil { + if err := kong.FillPluginsDefaults(newPlugin, schema); err != nil { + return nil, fmt.Errorf("failed processing auto fields: %w", err) + } + + // only fill auto fields for the configuration sent to Kong + // this is done because we want to avoid Kong to auto generate fields, which + // would make decK's configuration no longer fully "declarative" + if err := kong.FillPluginsDefaultsWithOpts(&plugin.Plugin, schema, kong.FillRecordOptions{ + FillDefaults: false, + FillAuto: true, + }); err != nil { return nil, fmt.Errorf("failed processing auto fields: %w", err) } } @@ -712,7 +719,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu if !dry { // sync mode // fire the request to Kong - result, err = sc.processor.Do(ctx, originalE.Kind, originalE.Op, originalE) + result, err = sc.processor.Do(ctx, eventForKong.Kind, eventForKong.Op, eventForKong) // TODO https://github.com/Kong/go-database-reconciler/issues/22 this does not print, but is switched on // sc.enableEntityActions because the existing behavior returns a result from the anon Run function. // Refactoring should use only the channel and simplify the return, probably to just an error (all the other diff --git a/pkg/types/plugin.go b/pkg/types/plugin.go index ae09493..b416438 100644 --- a/pkg/types/plugin.go +++ b/pkg/types/plugin.go @@ -181,7 +181,7 @@ func (d *pluginDiffer) createUpdatePlugin(plugin *state.Plugin) (*crud.Event, er return nil, fmt.Errorf("failed getting schema: %w", err) } pluginWithDefaults := &state.Plugin{Plugin: *plugin.DeepCopy()} - err = kong.FillPluginsDefaultsAutoFields(&pluginWithDefaults.Plugin, schema, ¤tPlugin.Plugin) + err = kong.FillPluginsDefaults(&pluginWithDefaults.Plugin, schema) if err != nil { return nil, fmt.Errorf("failed processing auto fields: %w", err) } diff --git a/tests/integration/sync_test.go b/tests/integration/sync_test.go index 3269e2b..e2a65dd 100644 --- a/tests/integration/sync_test.go +++ b/tests/integration/sync_test.go @@ -15,7 +15,6 @@ import ( "net/http/httptest" "net/url" "os" - "regexp" "strings" "testing" "time" @@ -5660,29 +5659,11 @@ func Test_Sync_PluginAutoFields(t *testing.T) { KongClient: c, }) - stats, errs, changes := syncer.Solve(ctx, 1, false, true) - require.Empty(t, errs, "Should have no errors in syncing") - require.NoError(t, err) - - require.Equal(t, int32(1), stats.CreateOps.Count(), "Should create 1 entity") - require.Len(t, changes.Creating, 1, "Should have 1 creating record in changes") + _, errs, _ := syncer.Solve(ctx, 1, false, true) - t.Run("should not override auto values with nils", func(t *testing.T) { - newState, err := fetchCurrentState(ctx, client, deckDump.Config{}) - require.NoError(t, err) - plugins, err := newState.Plugins.GetAll() - require.NoError(t, err) - require.Len(t, plugins, 1) - plugin := plugins[0] - require.Equal(t, "oauth2", *plugin.Name) - provisionKey, ok := plugin.Config["provision_key"] - require.True(t, ok) - - provisionKeyStr, ok := provisionKey.(string) - require.True(t, ok, "provision_key is not a string") - pattern := `^[a-zA-Z0-9]+$` - re := regexp.MustCompile(pattern) - require.True(t, re.MatchString(provisionKeyStr), "provision_key does not match the pattern") - }) + require.NotNil(t, errs) + require.Len(t, errs, 1) + require.Contains(t, errs[0].Error(), "provision_key: required field missing", + "Should error out due to missing provision_key") }) } From da9117908fd091659b22f041b9ab058cb62ddbf6 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Thu, 5 Sep 2024 18:00:00 +0530 Subject: [PATCH 8/9] fix: added fix to allow one consumer to be in >1 consumer-groups. (#140) * fix: added fix to allow one consumer to be in >1 consumer-groups. gdr version 1.39.2 introduced a bug where a consumer was limited to only one consumer-group. Adding the consumer in >1 group was resulting in an error on deck. This PR adds the fix for the same and an integration test. See: https://github.com/Kong/deck/issues/1384 * fix: scoped the test to >3.4.0 * chore: clarified test scope in integration sync_test * tests: added test for gateway v2.8.x for >1 consumer-group for a consumer --- pkg/state/consumer_group_consumers.go | 15 +++- tests/integration/sync_test.go | 86 +++++++++++++++++++ tests/integration/test_utils.go | 1 + .../kong.yaml | 9 ++ .../kong3x.yaml | 9 ++ 5 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong.yaml create mode 100644 tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong3x.yaml diff --git a/pkg/state/consumer_group_consumers.go b/pkg/state/consumer_group_consumers.go index 12b6108..317565a 100644 --- a/pkg/state/consumer_group_consumers.go +++ b/pkg/state/consumer_group_consumers.go @@ -151,12 +151,21 @@ func getConsumerGroupConsumer(txn *memdb.Txn, consumerGroupID string, IDs ...str for _, id := range IDs { for _, index := range indexes { - res, err := txn.First(consumerGroupConsumerTableName, index, id) + res, err := txn.Get(consumerGroupConsumerTableName, index, id) if err != nil { return nil, err } - if res != nil { - consumer := res.(*ConsumerGroupConsumer) + + for { + resultValue := res.Next() + if resultValue == nil { + break + } + consumer, ok := resultValue.(*ConsumerGroupConsumer) + if !ok { + break + } + if *consumer.ConsumerGroup.ID == consumerGroupID { return consumer, nil } diff --git a/tests/integration/sync_test.go b/tests/integration/sync_test.go index e2a65dd..ad00e60 100644 --- a/tests/integration/sync_test.go +++ b/tests/integration/sync_test.go @@ -5667,3 +5667,89 @@ func Test_Sync_PluginAutoFields(t *testing.T) { "Should error out due to missing provision_key") }) } + +// test scope: +// - enterprise +// - >=3.4.0 +func Test_Sync_MoreThanOneConsumerGroupForOneConsumer(t *testing.T) { + runWhen(t, "enterprise", ">=3.4.0") + setup(t) + + client, err := getTestClient() + require.NoError(t, err) + + expectedState := utils.KongRawState{ + ConsumerGroups: []*kong.ConsumerGroupObject{ + { + ConsumerGroup: &kong.ConsumerGroup{ + Name: kong.String("group1"), + }, + Consumers: []*kong.Consumer{ + { + Username: kong.String("my-test-consumer"), + }, + }, + }, + { + ConsumerGroup: &kong.ConsumerGroup{ + Name: kong.String("group2"), + }, + Consumers: []*kong.Consumer{ + { + Username: kong.String("my-test-consumer"), + }, + }, + }, + }, + Consumers: []*kong.Consumer{ + { + Username: kong.String("my-test-consumer"), + }, + }, + } + require.NoError(t, sync("testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong3x.yaml")) + testKongState(t, client, false, expectedState, nil) +} + +// test scope: +// - enterprise +// - 2.8.0 +func Test_Sync_MoreThanOneConsumerGroupForOneConsumer_2_8(t *testing.T) { + runWhen(t, "enterprise", ">=2.8.0 <3.0.0") + setup(t) + + client, err := getTestClient() + require.NoError(t, err) + + expectedState := utils.KongRawState{ + ConsumerGroups: []*kong.ConsumerGroupObject{ + { + ConsumerGroup: &kong.ConsumerGroup{ + Name: kong.String("group1"), + }, + Consumers: []*kong.Consumer{ + { + Username: kong.String("my-test-consumer"), + }, + }, + }, + { + ConsumerGroup: &kong.ConsumerGroup{ + Name: kong.String("group2"), + }, + Consumers: []*kong.Consumer{ + { + Username: kong.String("my-test-consumer"), + }, + }, + }, + }, + Consumers: []*kong.Consumer{ + { + Username: kong.String("my-test-consumer"), + }, + }, + } + require.NoError(t, sync("testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong.yaml")) + testKongState(t, client, false, expectedState, nil) +} diff --git a/tests/integration/test_utils.go b/tests/integration/test_utils.go index 8cddbad..b128789 100644 --- a/tests/integration/test_utils.go +++ b/tests/integration/test_utils.go @@ -228,6 +228,7 @@ func testKongState(t *testing.T, client *kong.Client, isKonnect bool, cmpopts.IgnoreFields(kong.Vault{}, "ID", "CreatedAt", "UpdatedAt"), cmpopts.IgnoreFields(kong.Certificate{}, "ID", "CreatedAt"), cmpopts.IgnoreFields(kong.SNI{}, "ID", "CreatedAt"), + cmpopts.IgnoreFields(kong.Consumer{}, "CreatedAt", "ID"), cmpopts.IgnoreFields(kong.ConsumerGroup{}, "CreatedAt", "ID"), cmpopts.IgnoreFields(kong.ConsumerGroupPlugin{}, "CreatedAt", "ID"), cmpopts.IgnoreFields(kong.KeyAuth{}, "ID", "CreatedAt"), diff --git a/tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong.yaml b/tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong.yaml new file mode 100644 index 0000000..641698b --- /dev/null +++ b/tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong.yaml @@ -0,0 +1,9 @@ +_format_version: "1.0" +consumer_groups: +- name: group1 +- name: group2 +consumers: +- username: my-test-consumer + groups: + - name: group1 + - name: group2 diff --git a/tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong3x.yaml b/tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong3x.yaml new file mode 100644 index 0000000..e5be41f --- /dev/null +++ b/tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong3x.yaml @@ -0,0 +1,9 @@ +_format_version: "3.0" +consumer_groups: +- name: group1 +- name: group2 +consumers: +- username: my-test-consumer + groups: + - name: group1 + - name: group2 From 01fa2d05dc5dd9921f11eb53155aeb4c13eb4cfa Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 18 Sep 2024 15:47:21 +0200 Subject: [PATCH 9/9] feat/default-lookup-services-consumerGroups (#130) This PR is adding some methods require for the default-lookup-services-consumerGroups It allow to retrieve consumerGroups and Service using a different tag It also add a fix to skip Consumers Plugins when using the skip-consumers flag --------- Co-authored-by: Prashansa Kulshrestha Co-authored-by: Michael Heap --- pkg/dump/dump.go | 46 ++++++++++++++++++++++++++++-- pkg/file/builder.go | 47 +++++++++++++++++++++++++------ pkg/file/kong_json_schema.json | 12 ++++++++ pkg/file/reader.go | 8 ++++++ pkg/file/readfile.go | 2 +- pkg/file/types.go | 6 ++-- pkg/file/zz_generated.deepcopy.go | 10 +++++++ 7 files changed, 118 insertions(+), 13 deletions(-) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 3e9ddbd..534fe98 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -39,8 +39,10 @@ type Config struct { // LookUpSelectorTags* can be used to ensure state lookup for entities using // these tags. This functionality is essential when using a plugin that references // consumers or routes associated with tags different from those in the sync command. - LookUpSelectorTagsConsumers []string - LookUpSelectorTagsRoutes []string + LookUpSelectorTagsConsumerGroups []string + LookUpSelectorTagsConsumers []string + LookUpSelectorTagsRoutes []string + LookUpSelectorTagsServices []string // KonnectControlPlane KonnectControlPlane string @@ -94,6 +96,25 @@ func getConsumerGroupsConfiguration(ctx context.Context, group *errgroup.Group, } return fmt.Errorf("consumer_groups: %w", err) } + if config.LookUpSelectorTagsConsumerGroups != nil { + globalConsumerGroups, err := GetAllConsumerGroups(ctx, client, config.LookUpSelectorTagsConsumerGroups) + if err != nil { + return fmt.Errorf("error retrieving global consumer groups: %w", err) + } + // if globalConsumers are not present, add them. + for _, globalConsumerGroup := range globalConsumerGroups { + found := false + for _, consumerGroup := range consumerGroups { + if *globalConsumerGroup.ConsumerGroup.ID == *consumerGroup.ConsumerGroup.ID { + found = true + break + } + } + if !found { + consumerGroups = append(consumerGroups, globalConsumerGroup) + } + } + } state.ConsumerGroups = consumerGroups return nil }) @@ -214,6 +235,25 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, if err != nil { return fmt.Errorf("services: %w", err) } + if config.LookUpSelectorTagsServices != nil { + globalServices, err := GetAllServices(ctx, client, config.LookUpSelectorTagsServices) + if err != nil { + return fmt.Errorf("error retrieving global services: %w", err) + } + // if globalServices are not present, add them. + for _, globalService := range globalServices { + found := false + for _, service := range services { + if *globalService.ID == *service.ID { + found = true + break + } + } + if !found { + services = append(services, globalService) + } + } + } state.Services = services return nil }) @@ -251,7 +291,9 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, if err != nil { return fmt.Errorf("plugins: %w", err) } + plugins = excludeKonnectManagedPlugins(plugins) + if config.SkipConsumers { plugins = excludeConsumersPlugins(plugins) plugins = excludeConsumerGroupsPlugins(plugins) diff --git a/pkg/file/builder.go b/pkg/file/builder.go index d24687b..35f6780 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -27,12 +27,14 @@ type stateBuilder struct { defaulter *utils.Defaulter kongVersion semver.Version - selectTags []string - lookupTagsConsumers []string - lookupTagsRoutes []string - skipCACerts bool - includeLicenses bool - intermediate *state.KongState + selectTags []string + lookupTagsConsumerGroups []string + lookupTagsConsumers []string + lookupTagsRoutes []string + lookupTagsServices []string + skipCACerts bool + includeLicenses bool + intermediate *state.KongState client *kong.Client ctx context.Context @@ -177,7 +179,21 @@ func (b *stateBuilder) consumerGroups() { cg.ID = kong.String(*current.ID) } } - utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags) + + stringTags := make([]string, len(cg.Tags)) + for i, tag := range cg.Tags { + if tag != nil { + stringTags[i] = *tag + } + } + sort.Strings(stringTags) + sort.Strings(b.lookupTagsConsumerGroups) + // if the consumer group tags and the lookup tags are the same, it means + // that the consumer group is a global consumer group retrieved from upstream, + // therefore we don't want to merge its tags with the selected tags. + if !reflect.DeepEqual(stringTags, b.lookupTagsConsumerGroups) { + utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags) + } cgo := kong.ConsumerGroupObject{ ConsumerGroup: &cg.ConsumerGroup, @@ -886,7 +902,22 @@ func (b *stateBuilder) ingestService(s *FService) error { s.ID = kong.String(*svc.ID) } } - utils.MustMergeTags(&s.Service, b.selectTags) + + stringTags := make([]string, len(s.Tags)) + for i, tag := range s.Tags { + if tag != nil { + stringTags[i] = *tag + } + } + sort.Strings(stringTags) + sort.Strings(b.lookupTagsServices) + // if the service tags and the lookup tags are the same, it means + // that the service is a global service retrieved from upstream, + // therefore we don't want to merge its tags with the selected tags. + if !reflect.DeepEqual(stringTags, b.lookupTagsServices) { + utils.MustMergeTags(&s.Service, b.selectTags) + } + b.defaulter.MustSet(&s.Service) if svc != nil { s.Service.CreatedAt = svc.CreatedAt diff --git a/pkg/file/kong_json_schema.json b/pkg/file/kong_json_schema.json index c68dfb0..5063371 100644 --- a/pkg/file/kong_json_schema.json +++ b/pkg/file/kong_json_schema.json @@ -1477,6 +1477,18 @@ "type": "string" }, "type": "array" + }, + "services": { + "items": { + "type": "string" + }, + "type": "array" + }, + "consumer_groups": { + "items": { + "type": "string" + }, + "type": "array" } }, "additionalProperties": false, diff --git a/pkg/file/reader.go b/pkg/file/reader.go index 717b292..f7d97ba 100644 --- a/pkg/file/reader.go +++ b/pkg/file/reader.go @@ -94,6 +94,14 @@ func Get(ctx context.Context, fileContent *Content, opt RenderConfig, dumpConfig builder.lookupTagsRoutes = dumpConfig.LookUpSelectorTagsRoutes } + if len(dumpConfig.LookUpSelectorTagsServices) > 0 { + builder.lookupTagsServices = dumpConfig.LookUpSelectorTagsServices + } + + if len(dumpConfig.LookUpSelectorTagsConsumerGroups) > 0 { + builder.lookupTagsConsumerGroups = dumpConfig.LookUpSelectorTagsConsumerGroups + } + if fileContent.Transform != nil && !*fileContent.Transform { return nil, ErrorTransformFalseNotSupported } diff --git a/pkg/file/readfile.go b/pkg/file/readfile.go index 466f5f1..f99ad5d 100644 --- a/pkg/file/readfile.go +++ b/pkg/file/readfile.go @@ -67,7 +67,7 @@ func getContent(filenames []string, mockEnvVars bool) (*Content, error) { func getReaders(fileOrDir string) (map[string]io.Reader, error) { // special case where `-` means stdin if fileOrDir == "-" { - if term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) { //nolint:gosec + if term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) { fmt.Fprintf(os.Stderr, "reading input from stdin...\n") } return map[string]io.Reader{"STDIN": os.Stdin}, nil diff --git a/pkg/file/types.go b/pkg/file/types.go index ea112f3..443dbeb 100644 --- a/pkg/file/types.go +++ b/pkg/file/types.go @@ -778,8 +778,10 @@ type Info struct { // for corresponding entities already in Kong. // +k8s:deepcopy-gen=true type LookUpSelectorTags struct { - Consumers []string `json:"consumers,omitempty" yaml:"consumers,omitempty"` - Routes []string `json:"routes,omitempty" yaml:"routes,omitempty"` + ConsumerGroups []string `json:"consumer_groups,omitempty" yaml:"consumer_groups,omitempty"` + Consumers []string `json:"consumers,omitempty" yaml:"consumers,omitempty"` + Routes []string `json:"routes,omitempty" yaml:"routes,omitempty"` + Services []string `json:"services,omitempty" yaml:"services,omitempty"` } // Konnect contains configuration specific to Konnect. diff --git a/pkg/file/zz_generated.deepcopy.go b/pkg/file/zz_generated.deepcopy.go index 29b74ab..436a529 100644 --- a/pkg/file/zz_generated.deepcopy.go +++ b/pkg/file/zz_generated.deepcopy.go @@ -876,6 +876,11 @@ func (in *Konnect) DeepCopy() *Konnect { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LookUpSelectorTags) DeepCopyInto(out *LookUpSelectorTags) { *out = *in + if in.ConsumerGroups != nil { + in, out := &in.ConsumerGroups, &out.ConsumerGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Consumers != nil { in, out := &in.Consumers, &out.Consumers *out = make([]string, len(*in)) @@ -886,6 +891,11 @@ func (in *LookUpSelectorTags) DeepCopyInto(out *LookUpSelectorTags) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Services != nil { + in, out := &in.Services, &out.Services + *out = make([]string, len(*in)) + copy(*out, *in) + } return }