Merge branch 'main' into feat/default-lookup-services-consumerGroups

go.mod

@@ -1,6 +1,8 @@
 module github.com/kong/go-database-reconciler
 
-go 1.21.1
+go 1.22.0
+
+toolchain go1.22.4
 
 replace github.com/yudai/gojsondiff v1.0.0 => github.com/Kong/gojsondiff v1.3.0
 
@@ -19,15 +21,15 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/hexops/gotextdiff v1.0.3
 	github.com/kong/deck v1.34.0
-	github.com/kong/go-kong v0.55.0
+	github.com/kong/go-kong v0.59.0
 	github.com/samber/lo v1.47.0
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/ssgelm/cookiejarparser v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/sync v0.8.0
-	golang.org/x/term v0.23.0
-	k8s.io/code-generator v0.29.4
+	golang.org/x/term v0.24.0
+	k8s.io/code-generator v0.31.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -48,12 +50,12 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/getkin/kin-openapi v0.108.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
@@ -103,7 +105,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.18.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/tidwall/gjson v1.17.1 // indirect
+	github.com/tidwall/gjson v1.17.3 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -118,15 +120,15 @@ require (
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

pkg/diff/diff.go

@@ -603,16 +603,13 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu
 		// Below the configuration in `e` may be modified. This is done solely for
 		// the purpose of displaying a correct diff and should not affect the
 		// configuration that is sent to Kong.
-		originalE := e
+		eventForKong := e
 
 		// If the event is for a plugin, inject defaults in the plugin's config
 		// that will be used for the diff. This is needed to avoid highlighting
 		// default values that were populated by Kong as differences.
 		if plugin, ok := e.Obj.(*state.Plugin); ok {
-			pluginCopy := &state.Plugin{
-				Plugin: *plugin.DeepCopy(),
-				Meta:   plugin.Meta,
-			}
+			pluginCopy := &state.Plugin{Plugin: *plugin.DeepCopy()}
 			e.Obj = pluginCopy
 
 			exists, err := utils.WorkspaceExists(ctx, sc.kongClient)
@@ -623,8 +620,20 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu
 					return nil, err
 				}
 
-				if err := kong.FillPluginsDefaults(&pluginCopy.Plugin, schema); err != nil {
-					return nil, fmt.Errorf("failed filling plugin defaults: %w", err)
+				// fill defaults and auto fields for the configuration that will be used for the diff
+				newPlugin := &pluginCopy.Plugin
+				if err := kong.FillPluginsDefaults(newPlugin, schema); err != nil {
+					return nil, fmt.Errorf("failed processing auto fields: %w", err)
+				}
+
+				// only fill auto fields for the configuration sent to Kong
+				// this is done because we want to avoid Kong to auto generate fields, which
+				// would make decK's configuration no longer fully "declarative"
+				if err := kong.FillPluginsDefaultsWithOpts(&plugin.Plugin, schema, kong.FillRecordOptions{
+					FillDefaults: false,
+					FillAuto:     true,
+				}); err != nil {
+					return nil, fmt.Errorf("failed processing auto fields: %w", err)
 				}
 			}
 		}
@@ -710,7 +719,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu
 		if !dry {
 			// sync mode
 			// fire the request to Kong
-			result, err = sc.processor.Do(ctx, originalE.Kind, originalE.Op, originalE)
+			result, err = sc.processor.Do(ctx, eventForKong.Kind, eventForKong.Op, eventForKong)
 			// TODO https://github.com/Kong/go-database-reconciler/issues/22 this does not print, but is switched on
 			// sc.enableEntityActions because the existing behavior returns a result from the anon Run function.
 			// Refactoring should use only the channel and simplify the return, probably to just an error (all the other

pkg/state/consumer_group_consumers.go

@@ -151,12 +151,21 @@ func getConsumerGroupConsumer(txn *memdb.Txn, consumerGroupID string, IDs ...str
 
 	for _, id := range IDs {
 		for _, index := range indexes {
-			res, err := txn.First(consumerGroupConsumerTableName, index, id)
+			res, err := txn.Get(consumerGroupConsumerTableName, index, id)
 			if err != nil {
 				return nil, err
 			}
-			if res != nil {
-				consumer := res.(*ConsumerGroupConsumer)
+
+			for {
+				resultValue := res.Next()
+				if resultValue == nil {
+					break
+				}
+				consumer, ok := resultValue.(*ConsumerGroupConsumer)
+				if !ok {
+					break
+				}
+
 				if *consumer.ConsumerGroup.ID == consumerGroupID {
 					return consumer, nil
 				}

pkg/state/rbac_role_test.go

@@ -105,7 +105,8 @@ func TestRBACRolesCollection_Get(t *testing.T) {
 	}
 	rbacRole1 := RBACRole{
 		RBACRole: kong.RBACRole{
-			ID: kong.String("foo-id"),
+			ID:   kong.String("foo-id"),
+			Name: kong.String("foo-name"),
 		},
 	}
 	rbacRole2 := RBACRole{
@@ -157,7 +158,7 @@ func TestRBACRolesCollection_Get(t *testing.T) {
 	k.Add(rbacRole1)
 	k.Add(rbacRole2)
 	for _, tt := range tests {
-		tc := &tt //nolint:gosec
+		tc := &tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 			got, err := k.Get(tc.args.nameOrID)

pkg/types/core.go

@@ -232,6 +232,7 @@ func NewEntity(t EntityType, opts EntityOpts) (Entity, error) {
 				kind:         entityTypeToKind(Plugin),
 				currentState: opts.CurrentState,
 				targetState:  opts.TargetState,
+				kongClient:   opts.KongClient,
 			},
 		}, nil
 	case Consumer:

pkg/types/plugin.go

@@ -88,6 +88,7 @@ type pluginDiffer struct {
 	kind crud.Kind
 
 	currentState, targetState *state.KongState
+	kongClient                *kong.Client
 }
 
 func (d *pluginDiffer) Deletes(handler func(crud.Event) error) error {
@@ -174,8 +175,18 @@ func (d *pluginDiffer) createUpdatePlugin(plugin *state.Plugin) (*crud.Event, er
 	}
 	currentPlugin = &state.Plugin{Plugin: *currentPlugin.DeepCopy()}
 	// found, check if update needed
+	// before checking the diff, fill in the defaults
+	schema, err := d.kongClient.Plugins.GetFullSchema(context.TODO(), plugin.Name)
+	if err != nil {
+		return nil, fmt.Errorf("failed getting schema: %w", err)
+	}
+	pluginWithDefaults := &state.Plugin{Plugin: *plugin.DeepCopy()}
+	err = kong.FillPluginsDefaults(&pluginWithDefaults.Plugin, schema)
+	if err != nil {
+		return nil, fmt.Errorf("failed processing auto fields: %w", err)
+	}
 
-	if !currentPlugin.EqualWithOpts(plugin, false, true, false) {
+	if !currentPlugin.EqualWithOpts(pluginWithDefaults, false, true, false) {
 		return &crud.Event{
 			Op:     crud.Update,
 			Kind:   d.kind,

tests/integration/sync_test.go

@@ -5627,3 +5627,129 @@ func Test_Sync_PluginDoNotFillDefaults(t *testing.T) {
 		})
 	})
 }
+
+func Test_Sync_PluginAutoFields(t *testing.T) {
+	client, err := getTestClient()
+
+	require.NoError(t, err)
+	ctx := context.Background()
+	t.Run("plugin_with_auto_fields", func(t *testing.T) {
+		mustResetKongState(ctx, t, client, deckDump.Config{})
+
+		currentState, err := fetchCurrentState(ctx, client, deckDump.Config{})
+		require.NoError(t, err)
+		targetState := stateFromFile(ctx, t,
+			"testdata/sync/034-fill-auto-oauth2/kong.yaml",
+			client,
+			deckDump.Config{},
+		)
+
+		kongURL, err := url.Parse(client.BaseRootURL())
+		require.NoError(t, err)
+		p := NewRecordRequestProxy(kongURL)
+		s := httptest.NewServer(p)
+		c, err := utils.GetKongClient(utils.KongClientConfig{
+			Address: s.URL,
+		})
+		require.NoError(t, err)
+
+		syncer, err := deckDiff.NewSyncer(deckDiff.SyncerOpts{
+			CurrentState: currentState,
+			TargetState:  targetState,
+
+			KongClient: c,
+		})
+		_, errs, _ := syncer.Solve(ctx, 1, false, true)
+
+		require.NotNil(t, errs)
+		require.Len(t, errs, 1)
+		require.Contains(t, errs[0].Error(), "provision_key: required field missing",
+			"Should error out due to missing provision_key")
+	})
+}
+
+// test scope:
+// - enterprise
+// - >=3.4.0
+func Test_Sync_MoreThanOneConsumerGroupForOneConsumer(t *testing.T) {
+	runWhen(t, "enterprise", ">=3.4.0")
+	setup(t)
+
+	client, err := getTestClient()
+	require.NoError(t, err)
+
+	expectedState := utils.KongRawState{
+		ConsumerGroups: []*kong.ConsumerGroupObject{
+			{
+				ConsumerGroup: &kong.ConsumerGroup{
+					Name: kong.String("group1"),
+				},
+				Consumers: []*kong.Consumer{
+					{
+						Username: kong.String("my-test-consumer"),
+					},
+				},
+			},
+			{
+				ConsumerGroup: &kong.ConsumerGroup{
+					Name: kong.String("group2"),
+				},
+				Consumers: []*kong.Consumer{
+					{
+						Username: kong.String("my-test-consumer"),
+					},
+				},
+			},
+		},
+		Consumers: []*kong.Consumer{
+			{
+				Username: kong.String("my-test-consumer"),
+			},
+		},
+	}
+	require.NoError(t, sync("testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong3x.yaml"))
+	testKongState(t, client, false, expectedState, nil)
+}
+
+// test scope:
+// - enterprise
+// - 2.8.0
+func Test_Sync_MoreThanOneConsumerGroupForOneConsumer_2_8(t *testing.T) {
+	runWhen(t, "enterprise", ">=2.8.0 <3.0.0")
+	setup(t)
+
+	client, err := getTestClient()
+	require.NoError(t, err)
+
+	expectedState := utils.KongRawState{
+		ConsumerGroups: []*kong.ConsumerGroupObject{
+			{
+				ConsumerGroup: &kong.ConsumerGroup{
+					Name: kong.String("group1"),
+				},
+				Consumers: []*kong.Consumer{
+					{
+						Username: kong.String("my-test-consumer"),
+					},
+				},
+			},
+			{
+				ConsumerGroup: &kong.ConsumerGroup{
+					Name: kong.String("group2"),
+				},
+				Consumers: []*kong.Consumer{
+					{
+						Username: kong.String("my-test-consumer"),
+					},
+				},
+			},
+		},
+		Consumers: []*kong.Consumer{
+			{
+				Username: kong.String("my-test-consumer"),
+			},
+		},
+	}
+	require.NoError(t, sync("testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong.yaml"))
+	testKongState(t, client, false, expectedState, nil)
+}

tests/integration/test_utils.go

@@ -228,6 +228,7 @@ func testKongState(t *testing.T, client *kong.Client, isKonnect bool,
 		cmpopts.IgnoreFields(kong.Vault{}, "ID", "CreatedAt", "UpdatedAt"),
 		cmpopts.IgnoreFields(kong.Certificate{}, "ID", "CreatedAt"),
 		cmpopts.IgnoreFields(kong.SNI{}, "ID", "CreatedAt"),
+		cmpopts.IgnoreFields(kong.Consumer{}, "CreatedAt", "ID"),
 		cmpopts.IgnoreFields(kong.ConsumerGroup{}, "CreatedAt", "ID"),
 		cmpopts.IgnoreFields(kong.ConsumerGroupPlugin{}, "CreatedAt", "ID"),
 		cmpopts.IgnoreFields(kong.KeyAuth{}, "ID", "CreatedAt"),

tests/integration/testdata/sync/034-fill-auto-oauth2/kong.yaml

@@ -0,0 +1,5 @@
+_format_version: "3.0"
+plugins:
+  - name: oauth2
+    config:
+      enable_password_grant: true

tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong.yaml

@@ -0,0 +1,9 @@
+_format_version: "1.0"
+consumer_groups:
+- name: group1
+- name: group2
+consumers:
+- username: my-test-consumer
+  groups:
+  - name: group1
+  - name: group2

tests/integration/testdata/sync/xxx-more-than-one-consumer-group-with-a-consumer/kong3x.yaml

@@ -0,0 +1,9 @@
+_format_version: "3.0"
+consumer_groups:
+- name: group1
+- name: group2
+consumers:
+- username: my-test-consumer
+  groups:
+  - name: group1
+  - name: group2