Report resource errors in DB mode

CHANGELOG.md

@@ -99,6 +99,8 @@ Adding a new version? You'll need three changes:
   [#5787](https://github.com/Kong/kubernetes-ingress-controller/pull/5787)
 - Add support in `HTTPRoute`s for `URLRewrite`:
   - `FullPathRewrite` [#5855](https://github.com/Kong/kubernetes-ingress-controller/pull/5855)
+- DB mode now supports Event reporting for resources that failed to apply.
+  [#5785](https://github.com/Kong/kubernetes-ingress-controller/pull/5785)
 
 ### Fixed
 

docs/cli-arguments.md

@@ -38,6 +38,7 @@
 | `--enable-controller-udpingress` | `bool` | Enable the UDPIngress controller. | `true` |
 | `--enable-reverse-sync` | `bool` | Send configuration to Kong even if the configuration checksum has not changed since previous update. | `false` |
 | `--feature-gates` | `list of string=bool` | A set of comma separated key=value pairs that describe feature gates for alpha/beta/experimental features. See the Feature Gates documentation for information and available options: https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md. |  |
+| `--force-leader-election` | `string` | Set to "enabled" or "disabled" to force a leader election behavior. Behavior is normally determined automatically from other settings. |  |
 | `--gateway-api-controller-name` | `string` | The controller name to match on Gateway API resources. | `konghq.com/kic-gateway-controller` |
 | `--gateway-discovery-dns-strategy` | `dns-strategy` | DNS strategy to use when creating Gateway's Admin API addresses. One of: ip, service, pod. | `"ip"` |
 | `--gateway-to-reconcile` | `namespaced-name` | Gateway namespaced name in "namespace/name" format. Makes KIC reconcile only the specified Gateway. |  |

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/jpillora/backoff v1.0.0
-	github.com/kong/go-database-reconciler v1.9.0
+	github.com/kong/go-database-reconciler v1.10.0
 	github.com/kong/go-kong v0.54.0
 	github.com/kong/kubernetes-telemetry v0.1.3
 	github.com/kong/kubernetes-testing-framework v0.46.0
@@ -202,8 +202,8 @@ require (
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.17.0 // indirect

go.sum

@@ -249,8 +249,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
 github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/kong/go-database-reconciler v1.9.0 h1:JWAQlT7lqmN+JrutdkN5yj+u9QwR6P7jNGz7cFqdg4Y=
-github.com/kong/go-database-reconciler v1.9.0/go.mod h1:QuHNMiwxuoflH+IkESEZpxO+6KsYXQ4QJ+Rq98wRYCM=
+github.com/kong/go-database-reconciler v1.10.0 h1:502Nn7CTsUNZyClL5bjrhkFrendVrPxDkP4ZGlBfsdg=
+github.com/kong/go-database-reconciler v1.10.0/go.mod h1:88/u23NIhkQr7SeTP1p4nLcnWCm8AMCwCF1f7Telw+w=
 github.com/kong/go-kong v0.54.0 h1:HZkZJRREJs/azkgJxMWr2jANQQfg8xXsAiFKu+d1nrs=
 github.com/kong/go-kong v0.54.0/go.mod h1:51rSSjgSZKukXgn5nNYbJUx0Et/islqR+LTltwCyjG8=
 github.com/kong/kubernetes-telemetry v0.1.3 h1:Hz2tkHGIIUqbn1x46QRDmmNjbEtJyxyOvHSPne3uPto=
@@ -557,14 +557,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=

internal/dataplane/sendconfig/dbmode.go

@@ -3,8 +3,11 @@ package sendconfig
 import (
 	"context"
 	"fmt"
+	"reflect"
+	"sync"
 
 	"github.com/blang/semver/v4"
+	"github.com/go-logr/logr"
 	"github.com/kong/go-database-reconciler/pkg/diff"
 	"github.com/kong/go-database-reconciler/pkg/dump"
 	"github.com/kong/go-database-reconciler/pkg/file"
@@ -14,29 +17,37 @@ import (
 
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/deckerrors"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/metrics"
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 )
 
 // UpdateStrategyDBMode implements the UpdateStrategy interface. It updates Kong's data-plane
 // configuration using decK's syncer.
 type UpdateStrategyDBMode struct {
-	client      *kong.Client
-	dumpConfig  dump.Config
-	version     semver.Version
-	concurrency int
-	isKonnect   bool
+	client            *kong.Client
+	dumpConfig        dump.Config
+	version           semver.Version
+	concurrency       int
+	isKonnect         bool
+	logger            logr.Logger
+	resourceErrors    []ResourceError
+	resourceErrorLock *sync.Mutex
 }
 
 func NewUpdateStrategyDBMode(
 	client *kong.Client,
 	dumpConfig dump.Config,
 	version semver.Version,
 	concurrency int,
+	logger logr.Logger,
 ) UpdateStrategyDBMode {
 	return UpdateStrategyDBMode{
-		client:      client,
-		dumpConfig:  dumpConfig,
-		version:     version,
-		concurrency: concurrency,
+		client:            client,
+		dumpConfig:        dumpConfig,
+		version:           version,
+		concurrency:       concurrency,
+		logger:            logger,
+		resourceErrors:    []ResourceError{},
+		resourceErrorLock: &sync.Mutex{},
 	}
 }
 
@@ -45,8 +56,9 @@ func NewUpdateStrategyDBModeKonnect(
 	dumpConfig dump.Config,
 	version semver.Version,
 	concurrency int,
+	logger logr.Logger,
 ) UpdateStrategyDBMode {
-	s := NewUpdateStrategyDBMode(client, dumpConfig, version, concurrency)
+	s := NewUpdateStrategyDBMode(client, dumpConfig, version, concurrency, logger)
 	s.isKonnect = true
 	return s
 }
@@ -70,23 +82,114 @@ func (s UpdateStrategyDBMode) Update(ctx context.Context, targetContent ContentW
 	}
 
 	syncer, err := diff.NewSyncer(diff.SyncerOpts{
-		CurrentState:    cs,
-		TargetState:     ts,
-		KongClient:      s.client,
-		SilenceWarnings: true,
-		IsKonnect:       s.isKonnect,
-		IncludeLicenses: true,
+		CurrentState:        cs,
+		TargetState:         ts,
+		KongClient:          s.client,
+		SilenceWarnings:     true,
+		IsKonnect:           s.isKonnect,
+		IncludeLicenses:     true,
+		EnableEntityActions: true,
 	})
 	if err != nil {
 		return fmt.Errorf("creating a new syncer for %s: %w", s.client.BaseRootURL(), err), nil, nil, nil
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	go s.HandleEvents(ctx, syncer.GetResultChan())
+
 	_, errs, _ := syncer.Solve(ctx, s.concurrency, false, false)
+	cancel()
+	s.resourceErrorLock.Lock()
+	defer s.resourceErrorLock.Unlock()
 	if errs != nil {
-		return deckutils.ErrArray{Errors: errs}, nil, nil, nil
+		return deckutils.ErrArray{Errors: errs}, s.resourceErrors, nil, nil
+	}
+
+	// as of GDR 1.8 we should always get a plain error set in addition to resourceErrors, so returning resourceErrors
+	// here should not be necessary. Return it anyway as a future-proof because why not.
+	return nil, s.resourceErrors, nil, nil
+}
+
+// HandleEvents handles logging and error reporting for individual entity change events generated during a sync by
+// looping over an event channel. It terminates when its context dies.
+func (s *UpdateStrategyDBMode) HandleEvents(ctx context.Context, events chan diff.EntityAction) {
+	s.resourceErrorLock.Lock()
+	for {
+		select {
+		case event := <-events:
+			if event.Error == nil {
+				s.logger.V(util.DebugLevel).Info("updated gateway entity", "action", event.Action, "kind", event.Entity.Kind, "name", event.Entity.Name)
+			} else {
+				s.logger.Error(event.Error, "failed updating gateway entity", "action", event.Action, "kind", event.Entity.Kind, "name", event.Entity.Name)
+				raw, err := resourceErrorFromEntityAction(event)
+				if err != nil {
+					s.logger.Error(err, "could not parse entity update error")
+				} else {
+					rerror, err := parseRawResourceError(raw)
+					if err != nil {
+						s.logger.Error(err, "could not parse entity update error")
+					} else {
+						s.resourceErrors = append(s.resourceErrors, rerror)
+					}
+				}
+			}
+		case <-ctx.Done():
+			s.resourceErrorLock.Unlock()
+			return
+		}
+	}
+}
+
+func resourceErrorFromEntityAction(event diff.EntityAction) (rawResourceError, error) {
+	var subj any
+	// GDR may produce an old only (delete), new only (create), or both (update) in an event. tags should be identical
+	// but we arbitrarily pull from new.
+	if event.Entity.New != nil {
+		subj = event.Entity.New
+	} else {
+		subj = event.Entity.Old
+	}
+	// GDR makes frequent use of "any" for its various entity handlers. It does not use interfaces that would allow us
+	// to guarantee that a particular entity does indeed have tags or similar and retrieve them. We're unlikely to
+	// refactor this any time soon, so in absence of proper interface methods, we pray that the entity probably has tags,
+	// which is a reasonable assumption as anything KIC can manage does. The reflect-fu here is sinister and menacing,
+	// but should spit out tags unless something has gone wrong.
+	reflected := reflect.Indirect(reflect.ValueOf(subj))
+	if reflected.Kind() != reflect.Struct {
+		// We need to fail fast here because FieldByName() will panic on non-Struct Kinds.
+		return rawResourceError{}, fmt.Errorf("entity %s/%s is %s, not Struct",
+			event.Entity.Kind, event.Entity.Name, reflected.Kind())
+	}
+	tagsValue := reflected.FieldByName("Tags")
+	if tagsValue.IsZero() {
+		return rawResourceError{}, fmt.Errorf("entity %s/%s of type %s lacks 'Tags' field",
+			event.Entity.Kind, event.Entity.Name, reflect.TypeOf(subj))
+	}
+	tags, ok := tagsValue.Interface().([]*string)
+	if !ok {
+		return rawResourceError{}, fmt.Errorf("entity %s/%s Tags field is not []*string",
+			event.Entity.Kind, event.Entity.Name)
+	}
+
+	actualTags := []string{}
+	for _, s := range tags {
+		actualTags = append(actualTags, *s)
 	}
 
-	return nil, nil, nil, nil
+	// This omits ID, which should be available but requires similar reflect gymnastics as Tags, and probably isn't worth
+	// it.
+	return rawResourceError{
+		Name: event.Entity.Name,
+		Tags: actualTags,
+		// /config flattened errors have a structured set of field to error reasons, whereas GDR errors are just plain
+		// un-parsed admin API endpoint strings. These will often mention a field within the string, e.g.
+		// schema violation (methods: cannot set 'methods' when 'protocols' is 'grpc' or 'grpcs')
+		// has "methods", but we'd need to do string parsing to extract it, and we may not catch all possible error types.
+		// This lazier approach just dumps the full error string as a single problem, which is probably good enough.
+		Problems: map[string]string{
+			"": fmt.Sprintf("%s", event.Error),
+		},
+	}, nil
 }
 
 func (s UpdateStrategyDBMode) MetricsProtocol() metrics.Protocol {

internal/dataplane/sendconfig/strategy.go

@@ -98,6 +98,7 @@ func (r DefaultUpdateStrategyResolver) resolveUpdateStrategy(client UpdateClient
 			},
 			r.config.Version,
 			r.config.Concurrency,
+			r.logger,
 		)
 	}
 
@@ -111,6 +112,7 @@ func (r DefaultUpdateStrategyResolver) resolveUpdateStrategy(client UpdateClient
 			},
 			r.config.Version,
 			r.config.Concurrency,
+			r.logger,
 		)
 	}
 

internal/manager/config.go

@@ -83,6 +83,7 @@ type Config struct {
 	IngressClassName         string
 	LeaderElectionNamespace  string
 	LeaderElectionID         string
+	LeaderElectionForce      string
 	Concurrency              int
 	FilterTags               []string
 	WatchNamespaces          []string
@@ -216,6 +217,7 @@ func (c *Config) FlagSet() *pflag.FlagSet {
 	flagSet.StringVar(&c.IngressClassName, "ingress-class", annotations.DefaultIngressClass, `Name of the ingress class to route through this controller.`)
 	flagSet.StringVar(&c.LeaderElectionID, "election-id", "5b374a9e.konghq.com", `Election id to use for status update.`)
 	flagSet.StringVar(&c.LeaderElectionNamespace, "election-namespace", "", `Leader election namespace to use when running outside a cluster.`)
+	flagSet.StringVar(&c.LeaderElectionForce, "force-leader-election", "", `Set to "enabled" or "disabled" to force a leader election behavior. Behavior is normally determined automatically from other settings.`)
 	flagSet.StringSliceVar(&c.FilterTags, "kong-admin-filter-tag", []string{"managed-by-ingress-controller"},
 		"Tag(s) in comma-separated format (or specify this flag multiple times). They are used to manage and filter entities in Kong. "+
 			"This setting will be silently ignored if the Kong instance has no tags support.")

internal/manager/setup.go

@@ -120,7 +120,20 @@ func setupManagerOptions(ctx context.Context, logger logr.Logger, c *Config, dbm
 	return managerOpts, nil
 }
 
+const (
+	LeaderElectionEnabled  = "enabled"
+	LeaderElectionDisabled = "disabled"
+)
+
 func leaderElectionEnabled(logger logr.Logger, c *Config, dbmode dpconf.DBMode) bool {
+	if c.LeaderElectionForce == LeaderElectionEnabled {
+		logger.Info("leader election forcibly enabled")
+		return true
+	}
+	if c.LeaderElectionForce == LeaderElectionDisabled {
+		logger.Info("leader election forcibly disabled")
+		return false
+	}
 	if c.Konnect.ConfigSynchronizationEnabled {
 		logger.Info("Konnect config synchronisation enabled, enabling leader election")
 		return true