Merge branch 'main' into dependabot/go_modules/github.com/hashicorp/g…

…o-retryablehttp-0.7.7

.github/workflows/integration-enterprise.yaml

@@ -15,19 +15,7 @@ jobs:
     strategy:
       matrix:
         kong_image:
-        - 'kong/kong-gateway:1.5.0.11'
-        - 'kong/kong-gateway:2.1.4.6'
-        - 'kong/kong-gateway:2.2.1.3'
-        - 'kong/kong-gateway:2.3.3.4'
-        - 'kong/kong-gateway:2.4.1.3'
-        - 'kong/kong-gateway:2.5.1.2'
-        - 'kong/kong-gateway:2.6.0.2'
-        - 'kong/kong-gateway:2.7'
         - 'kong/kong-gateway:2.8'
-        - 'kong/kong-gateway:3.0'
-        - 'kong/kong-gateway:3.1'
-        - 'kong/kong-gateway:3.2'
-        - 'kong/kong-gateway:3.3'
         - 'kong/kong-gateway:3.4'
         - 'kong/kong-gateway:3.5'
         - 'kong/kong-gateway:3.6'

.github/workflows/integration.yaml

@@ -15,21 +15,7 @@ jobs:
     strategy:
       matrix:
         kong_image:
-        - 'kong:1.4.3'
-        - 'kong:1.5.1'
-        - 'kong:2.0.5'
-        - 'kong:2.1.4'
-        - 'kong:2.2.2'
-        - 'kong:2.3.3'
-        - 'kong:2.4.1'
-        - 'kong:2.5.1'
-        - 'kong:2.6.0'
-        - 'kong:2.7'
         - 'kong:2.8'
-        - 'kong:3.0'
-        - 'kong:3.1'
-        - 'kong:3.2'
-        - 'kong:3.3'
         - 'kong:3.4'
         - 'kong:3.5'
         - 'kong:3.6'

go.mod

@@ -1,6 +1,8 @@
 module github.com/kong/go-database-reconciler
 
-go 1.21.1
+go 1.22.0
+
+toolchain go1.22.4
 
 replace github.com/yudai/gojsondiff v1.0.0 => github.com/Kong/gojsondiff v1.3.0
 
@@ -19,14 +21,15 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/hexops/gotextdiff v1.0.3
 	github.com/kong/deck v1.34.0
-	github.com/kong/go-kong v0.55.0
+	github.com/kong/go-kong v0.59.0
+	github.com/samber/lo v1.47.0
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/ssgelm/cookiejarparser v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/sync v0.8.0
-	golang.org/x/term v0.23.0
-	k8s.io/code-generator v0.29.4
+	golang.org/x/term v0.24.0
+	k8s.io/code-generator v0.31.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -47,12 +50,12 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/getkin/kin-openapi v0.108.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
@@ -102,7 +105,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.18.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/tidwall/gjson v1.17.1 // indirect
+	github.com/tidwall/gjson v1.17.3 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -115,17 +118,17 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.16.1 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

pkg/diff/diff.go

@@ -599,6 +599,45 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu
 		var err error
 		var result crud.Arg
 
+		// This variable holds the original event with the unchanged configuration
+		// Below the configuration in `e` may be modified. This is done solely for
+		// the purpose of displaying a correct diff and should not affect the
+		// configuration that is sent to Kong.
+		eventForKong := e
+
+		// If the event is for a plugin, inject defaults in the plugin's config
+		// that will be used for the diff. This is needed to avoid highlighting
+		// default values that were populated by Kong as differences.
+		if plugin, ok := e.Obj.(*state.Plugin); ok {
+			pluginCopy := &state.Plugin{Plugin: *plugin.DeepCopy()}
+			e.Obj = pluginCopy
+
+			exists, err := utils.WorkspaceExists(ctx, sc.kongClient)
+			if err == nil && exists {
+				var schema map[string]interface{}
+				schema, err = sc.kongClient.Plugins.GetFullSchema(ctx, pluginCopy.Plugin.Name)
+				if err != nil {
+					return nil, err
+				}
+
+				// fill defaults and auto fields for the configuration that will be used for the diff
+				newPlugin := &pluginCopy.Plugin
+				if err := kong.FillPluginsDefaults(newPlugin, schema); err != nil {
+					return nil, fmt.Errorf("failed processing auto fields: %w", err)
+				}
+
+				// only fill auto fields for the configuration sent to Kong
+				// this is done because we want to avoid Kong to auto generate fields, which
+				// would make decK's configuration no longer fully "declarative"
+				if err := kong.FillPluginsDefaultsWithOpts(&plugin.Plugin, schema, kong.FillRecordOptions{
+					FillDefaults: false,
+					FillAuto:     true,
+				}); err != nil {
+					return nil, fmt.Errorf("failed processing auto fields: %w", err)
+				}
+			}
+		}
+
 		c := e.Obj.(state.ConsoleString)
 		objDiff := map[string]interface{}{
 			"old": e.OldObj,
@@ -680,7 +719,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu
 		if !dry {
 			// sync mode
 			// fire the request to Kong
-			result, err = sc.processor.Do(ctx, e.Kind, e.Op, e)
+			result, err = sc.processor.Do(ctx, eventForKong.Kind, eventForKong.Op, eventForKong)
 			// TODO https://github.com/Kong/go-database-reconciler/issues/22 this does not print, but is switched on
 			// sc.enableEntityActions because the existing behavior returns a result from the anon Run function.
 			// Refactoring should use only the channel and simplify the return, probably to just an error (all the other

pkg/dump/dump.go

@@ -39,8 +39,10 @@ type Config struct {
 	// LookUpSelectorTags* can be used to ensure state lookup for entities using
 	// these tags. This functionality is essential when using a plugin that references
 	// consumers or routes associated with tags different from those in the sync command.
-	LookUpSelectorTagsConsumers []string
-	LookUpSelectorTagsRoutes    []string
+	LookUpSelectorTagsConsumerGroups []string
+	LookUpSelectorTagsConsumers      []string
+	LookUpSelectorTagsRoutes         []string
+	LookUpSelectorTagsServices       []string
 
 	// KonnectControlPlane
 	KonnectControlPlane string
@@ -94,6 +96,25 @@ func getConsumerGroupsConfiguration(ctx context.Context, group *errgroup.Group,
 			}
 			return fmt.Errorf("consumer_groups: %w", err)
 		}
+		if config.LookUpSelectorTagsConsumerGroups != nil {
+			globalConsumerGroups, err := GetAllConsumerGroups(ctx, client, config.LookUpSelectorTagsConsumerGroups)
+			if err != nil {
+				return fmt.Errorf("error retrieving global consumer groups: %w", err)
+			}
+			// if globalConsumers are not present, add them.
+			for _, globalConsumerGroup := range globalConsumerGroups {
+				found := false
+				for _, consumerGroup := range consumerGroups {
+					if *globalConsumerGroup.ConsumerGroup.ID == *consumerGroup.ConsumerGroup.ID {
+						found = true
+						break
+					}
+				}
+				if !found {
+					consumerGroups = append(consumerGroups, globalConsumerGroup)
+				}
+			}
+		}
 		state.ConsumerGroups = consumerGroups
 		return nil
 	})
@@ -214,6 +235,25 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group,
 		if err != nil {
 			return fmt.Errorf("services: %w", err)
 		}
+		if config.LookUpSelectorTagsServices != nil {
+			globalServices, err := GetAllServices(ctx, client, config.LookUpSelectorTagsServices)
+			if err != nil {
+				return fmt.Errorf("error retrieving global services: %w", err)
+			}
+			// if globalServices are not present, add them.
+			for _, globalService := range globalServices {
+				found := false
+				for _, service := range services {
+					if *globalService.ID == *service.ID {
+						found = true
+						break
+					}
+				}
+				if !found {
+					services = append(services, globalService)
+				}
+			}
+		}
 		state.Services = services
 		return nil
 	})
@@ -251,7 +291,9 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group,
 		if err != nil {
 			return fmt.Errorf("plugins: %w", err)
 		}
+
 		plugins = excludeKonnectManagedPlugins(plugins)
+
 		if config.SkipConsumers {
 			plugins = excludeConsumersPlugins(plugins)
 			plugins = excludeConsumerGroupsPlugins(plugins)

pkg/file/builder.go

@@ -27,12 +27,14 @@ type stateBuilder struct {
 	defaulter       *utils.Defaulter
 	kongVersion     semver.Version
 
-	selectTags          []string
-	lookupTagsConsumers []string
-	lookupTagsRoutes    []string
-	skipCACerts         bool
-	includeLicenses     bool
-	intermediate        *state.KongState
+	selectTags               []string
+	lookupTagsConsumerGroups []string
+	lookupTagsConsumers      []string
+	lookupTagsRoutes         []string
+	lookupTagsServices       []string
+	skipCACerts              bool
+	includeLicenses          bool
+	intermediate             *state.KongState
 
 	client *kong.Client
 	ctx    context.Context
@@ -177,7 +179,21 @@ func (b *stateBuilder) consumerGroups() {
 				cg.ID = kong.String(*current.ID)
 			}
 		}
-		utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags)
+
+		stringTags := make([]string, len(cg.Tags))
+		for i, tag := range cg.Tags {
+			if tag != nil {
+				stringTags[i] = *tag
+			}
+		}
+		sort.Strings(stringTags)
+		sort.Strings(b.lookupTagsConsumerGroups)
+		// if the consumer group tags and the lookup tags are the same, it means
+		// that the consumer group is a global consumer group retrieved from upstream,
+		// therefore we don't want to merge its tags with the selected tags.
+		if !reflect.DeepEqual(stringTags, b.lookupTagsConsumerGroups) {
+			utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags)
+		}
 
 		cgo := kong.ConsumerGroupObject{
 			ConsumerGroup: &cg.ConsumerGroup,
@@ -886,7 +902,22 @@ func (b *stateBuilder) ingestService(s *FService) error {
 			s.ID = kong.String(*svc.ID)
 		}
 	}
-	utils.MustMergeTags(&s.Service, b.selectTags)
+
+	stringTags := make([]string, len(s.Tags))
+	for i, tag := range s.Tags {
+		if tag != nil {
+			stringTags[i] = *tag
+		}
+	}
+	sort.Strings(stringTags)
+	sort.Strings(b.lookupTagsServices)
+	// if the service tags and the lookup tags are the same, it means
+	// that the service is a global service retrieved from upstream,
+	// therefore we don't want to merge its tags with the selected tags.
+	if !reflect.DeepEqual(stringTags, b.lookupTagsServices) {
+		utils.MustMergeTags(&s.Service, b.selectTags)
+	}
+
 	b.defaulter.MustSet(&s.Service)
 	if svc != nil {
 		s.Service.CreatedAt = svc.CreatedAt
@@ -1443,44 +1474,6 @@ func (b *stateBuilder) ingestRoute(r FRoute) error {
 	return nil
 }
 
-func (b *stateBuilder) getPluginSchema(pluginName string) (map[string]interface{}, error) {
-	var schema map[string]interface{}
-
-	// lookup in cache
-	if schema, ok := b.schemasCache[pluginName]; ok {
-		return schema, nil
-	}
-
-	exists, err := utils.WorkspaceExists(b.ctx, b.client)
-	if err != nil {
-		return nil, fmt.Errorf("ensure workspace exists: %w", err)
-	}
-	if !exists {
-		return schema, ErrWorkspaceNotFound
-	}
-
-	schema, err = b.client.Plugins.GetFullSchema(b.ctx, &pluginName)
-	if err != nil {
-		return schema, err
-	}
-	b.schemasCache[pluginName] = schema
-	return schema, nil
-}
-
-func (b *stateBuilder) addPluginDefaults(plugin *FPlugin) error {
-	if b.client == nil {
-		return nil
-	}
-	schema, err := b.getPluginSchema(*plugin.Name)
-	if err != nil {
-		if errors.Is(err, ErrWorkspaceNotFound) {
-			return nil
-		}
-		return fmt.Errorf("retrieve schema for %v from Kong: %w", *plugin.Name, err)
-	}
-	return kong.FillPluginsDefaults(&plugin.Plugin, schema)
-}
-
 func (b *stateBuilder) ingestPlugins(plugins []FPlugin) error {
 	for _, p := range plugins {
 		p := p
@@ -1504,9 +1497,6 @@ func (b *stateBuilder) ingestPlugins(plugins []FPlugin) error {
 		if err != nil {
 			return err
 		}
-		if err := b.addPluginDefaults(&p); err != nil {
-			return fmt.Errorf("add defaults to plugin '%v': %w", *p.Name, err)
-		}
 		utils.MustMergeTags(&p, b.selectTags)
 		if plugin != nil {
 			p.Plugin.CreatedAt = plugin.CreatedAt

pkg/file/kong_json_schema.json

@@ -1477,6 +1477,18 @@
             "type": "string"
           },
           "type": "array"
+        },
+        "services": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "consumer_groups": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
         }
       },
       "additionalProperties": false,