[receiver/k8sobjects] Support running in kubernetes leader election mode.

.chloggen/support-kubernetes-leader-election.yaml

@@ -0,0 +1,21 @@
+# Use this changelog template to create an entry for release notes.
+# If your change doesn't affect end users, such as a test fix or a tooling change,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: k8sobjectsreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Support running selected Kubernetes components in "leader election mode", meaning that only one replica of the Collector will react to Kubernetes events at any given point in time.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [17369]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+  Support running in kubernetes leader election mode.

internal/k8sconfig/leaderelection.go

@@ -0,0 +1,110 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package k8sconfig // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig"
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+)
+
+// LeaderElectionConfig is used to enable leader election
+type LeaderElectionConfig struct {
+	Enabled bool `mapstructure:"enabled"`
+	// LockName determines the name of the resource that leader election will use for holding the leader lock.
+	LockName      string        `mapstructure:"lock_name"`
+	LeaseDuration time.Duration `mapstructure:"lease_duration"`
+	RenewDeadline time.Duration `mapstructure:"renew_deadline"`
+	RetryPeriod   time.Duration `mapstructure:"retry_period"`
+}
+
+const (
+	inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
+	defaultLeaseDuration   = 15 * time.Second
+	defaultRenewDeadline   = 10 * time.Second
+	defaultRetryPeriod     = 2 * time.Second
+)
+
+// NewResourceLock creates a new leases resource lock for use in a leader election loop
+func newResourceLock(client kubernetes.Interface, lockName string) (resourcelock.Interface, error) {
+	if lockName == "" {
+		return nil, errors.New("lockName must be configured")
+	}
+
+	leaderElectionNamespace, err := getInClusterNamespace()
+	if err != nil {
+		return nil, fmt.Errorf("unable to find leader election namespace: %w", err)
+	}
+
+	// Leader id, needs to be unique, use pod name in kubernetes case.
+	id, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+
+	return resourcelock.New(resourcelock.LeasesResourceLock,
+		leaderElectionNamespace, lockName,
+		client.CoreV1(),
+		client.CoordinationV1(),
+		resourcelock.ResourceLockConfig{
+			Identity: id,
+		})
+}
+
+func getInClusterNamespace() (string, error) {
+	// Check whether the namespace file exists.
+	// If not, we are not running in cluster so can't guess the namespace.
+	_, err := os.Stat(inClusterNamespacePath)
+	if os.IsNotExist(err) {
+		return "", fmt.Errorf("not running in-cluster, unable to get namespace")
+	} else if err != nil {
+		return "", fmt.Errorf("error checking namespace file: %w", err)
+	}
+
+	// Load the namespace file and return its content
+	namespace, err := os.ReadFile(inClusterNamespacePath)
+	if err != nil {
+		return "", fmt.Errorf("error reading namespace file: %w", err)
+	}
+	return string(namespace), nil
+}
+
+// NewLeaderElector return  a leader elector object using client-go
+func NewLeaderElector(cfg LeaderElectionConfig, client kubernetes.Interface, startFunc func(context.Context), stopFunc func()) (*leaderelection.LeaderElector, error) {
+	resourceLock, err := newResourceLock(client, cfg.LockName)
+	if err != nil {
+		return &leaderelection.LeaderElector{}, err
+	}
+
+	leConfig := leaderelection.LeaderElectionConfig{
+		Lock:          resourceLock,
+		LeaseDuration: defaultLeaseDuration,
+		RenewDeadline: defaultRenewDeadline,
+		RetryPeriod:   defaultRetryPeriod,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: startFunc,
+			OnStoppedLeading: stopFunc,
+		},
+	}
+
+	if cfg.LeaseDuration != 0 {
+		leConfig.LeaseDuration = cfg.LeaseDuration
+	}
+
+	if cfg.RenewDeadline != 0 {
+		leConfig.RenewDeadline = cfg.RenewDeadline
+	}
+
+	if cfg.RetryPeriod != 0 {
+		leConfig.RetryPeriod = cfg.RetryPeriod
+	}
+
+	return leaderelection.NewLeaderElector(leConfig)
+}

receiver/k8sobjectsreceiver/README.md

@@ -36,9 +36,32 @@ The following is example configuration
         mode: watch
         group: events.k8s.io
         namespaces: [default]
+
+  k8sobjects/leader-election:
+    leader_election:
+      enabled: true
+      lock_name: "k8sobjects-leader-election"
+    auth_type: serviceAccount
+    objects:
+      - name: pods
+        mode: pull
+        label_selector: environment in (production),tier in (frontend)
+        field_selector: status.phase=Running
+        interval: 15m
+      - name: events
+        mode: watch
+        group: events.k8s.io
+        namespaces: [default]
 ```
 
 Brief description of configuration properties:
+- `leader_election`: Leader election mode helps to ensure that only one instance (as leader instance) is collecting `k8sobjects` data, and if the leader instance fails, another one is elected as leader and takes its place. 
+*Note*: You need to add additional RBAC to perform the leader election mode. See below [Leader Election RBAC](#RBAC-Leader-Election) for more information.
+  - `enabled` (default = `false`): whether run in leader election mode.
+  - `lock_name` (default = `k8sobjects`): the identity name of holder, will use component's ID if not set.
+  - `lease_duration` (default = `15s`): the duration that non-leader candidates will wait to force acquire leadership.
+  - `renew_deadline` (default = `10s`): the duration that the acting master will retry refreshing leadership before giving up.
+  - `retry_period` (default = `2s`): the duration the LeaderElector clients should wait between tries of actions. 
 - `auth_type` (default = `serviceAccount`): Determines how to authenticate to
 the K8s API server. This can be one of `none` (for no auth), `serviceAccount`
 (to use the standard service account token provided to the agent pod), or
@@ -151,6 +174,46 @@ rules:
 EOF
 ```
 
+### RBAC-Leader-Election
+
+Use the below commands to create a `ClusterRole` with `leader election` required permissions and a
+`ClusterRoleBinding` to grant the role to the service account created above.
+
+```bash
+<<EOF | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: otelcontribcol
+  labels:
+    app: otelcontribcol
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: 
+  - "events.k8s.io"
+  resources:
+  - events
+  verbs:
+  - watch
+EOF
+```
+
 ```bash
 <<EOF | kubectl apply -f -
 apiVersion: rbac.authorization.k8s.io/v1

receiver/k8sobjectsreceiver/config.go

@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig"
 )
@@ -46,11 +47,13 @@ type K8sObjectsConfig struct {
 type Config struct {
 	k8sconfig.APIConfig `mapstructure:",squash"`
 
-	Objects []*K8sObjectsConfig `mapstructure:"objects"`
+	Objects        []*K8sObjectsConfig            `mapstructure:"objects"`
+	LeaderElection k8sconfig.LeaderElectionConfig `mapstructure:"leader_election"`
 
 	// For mocking purposes only.
 	makeDiscoveryClient func() (discovery.ServerResourcesInterface, error)
 	makeDynamicClient   func() (dynamic.Interface, error)
+	makeClient          func() (kubernetes.Interface, error)
 }
 
 func (c *Config) Validate() error {
@@ -92,6 +95,18 @@ func (c *Config) Validate() error {
 	return nil
 }
 
+func (c *Config) getClient() (kubernetes.Interface, error) {
+	if c.makeClient != nil {
+		return c.makeClient()
+	}
+	client, err := k8sconfig.MakeClient(c.APIConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
 func (c *Config) getDiscoveryClient() (discovery.ServerResourcesInterface, error) {
 	if c.makeDiscoveryClient != nil {
 		return c.makeDiscoveryClient()

receiver/k8sobjectsreceiver/config_test.go

@@ -13,6 +13,8 @@ import (
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/confmap/confmaptest"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig"
 )
 
 func TestLoadConfig(t *testing.T) {
@@ -37,7 +39,7 @@ func TestLoadConfig(t *testing.T) {
 	err = component.ValidateConfig(cfg)
 	require.NoError(t, err)
 
-	expected := []*K8sObjectsConfig{
+	expectedObjects := []*K8sObjectsConfig{
 		{
 			Name:          "pods",
 			Mode:          PullMode,
@@ -63,7 +65,12 @@ func TestLoadConfig(t *testing.T) {
 			},
 		},
 	}
-	assert.EqualValues(t, expected, cfg.Objects)
+	expectedLeaderElection := k8sconfig.LeaderElectionConfig{
+		Enabled: false,
+	}
+
+	assert.EqualValues(t, expectedObjects, cfg.Objects)
+	assert.EqualValues(t, expectedLeaderElection, cfg.LeaderElection)
 
 }
 

receiver/k8sobjectsreceiver/factory.go

@@ -30,6 +30,9 @@ func createDefaultConfig() component.Config {
 		APIConfig: k8sconfig.APIConfig{
 			AuthType: k8sconfig.AuthTypeServiceAccount,
 		},
+		LeaderElection: k8sconfig.LeaderElectionConfig{
+			Enabled: false,
+		},
 	}
 }
 

receiver/k8sobjectsreceiver/factory_test.go

@@ -26,6 +26,9 @@ func TestDefaultConfig(t *testing.T) {
 		APIConfig: k8sconfig.APIConfig{
 			AuthType: k8sconfig.AuthTypeServiceAccount,
 		},
+		LeaderElection: k8sconfig.LeaderElectionConfig{
+			Enabled: false,
+		},
 	}, rCfg)
 }