[deckhoues-controller] ha (#7634)

Signed-off-by: Mikhail Scherba <[email protected]>

deckhouse-controller/cmd/deckhouse-controller/start.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
+	"syscall"
 	"time"
 
 	addon_operator "github.com/flant/addon-operator/pkg/addon-operator"
@@ -30,6 +32,8 @@ import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/util/retry"
 
 	"github.com/deckhouse/deckhouse/deckhouse-controller/pkg/addon-operator/kube-config/backend"
@@ -39,52 +43,140 @@ import (
 	d8config "github.com/deckhouse/deckhouse/go_lib/deckhouse-config"
 )
 
+const (
+	leaseName        = "deckhouse-leader-election"
+	defaultNamespace = "d8-system"
+	leaseDuration    = 35
+	renewalDeadline  = 30
+	retryPeriod      = 10
+)
+
 func start(_ *kingpin.ParseContext) error {
 	sh_app.AppStartMessage = version()
 
 	ctx := context.Background()
 
 	operator := addon_operator.NewAddonOperator(ctx)
 
-	err := d8Apis.EnsureCRDs(ctx, operator.KubeClient(), "/deckhouse/deckhouse-controller/crds/*.yaml")
+	operator.StartAPIServer()
+
+	if os.Getenv("DECKHOUSE_HA") == "true" {
+		log.Info("Desckhouse is starting in HA mode")
+		runHAMode(ctx, operator)
+		return nil
+	}
+
+	err := run(ctx, operator)
 	if err != nil {
-		fmt.Println(err)
+		log.Error(err)
+		os.Exit(1)
+	}
+
+	return nil
+}
+
+func runHAMode(ctx context.Context, operator *addon_operator.AddonOperator) {
+	podName := os.Getenv("DECKHOUSE_POD")
+	if len(podName) == 0 {
+		log.Info("DECKHOUSE_POD env not set or empty")
 		os.Exit(1)
 	}
 
+	podIP := os.Getenv("ADDON_OPERATOR_LISTEN_ADDRESS")
+	if len(podIP) == 0 {
+		log.Info("ADDON_OPERATOR_LISTEN_ADDRESS env not set or empty")
+		os.Exit(1)
+	}
+
+	podNs := os.Getenv("ADDON_OPERATOR_NAMESPACE")
+	if len(podNs) == 0 {
+		podNs = defaultNamespace
+	}
+	identity := fmt.Sprintf("%s.%s.%s.pod", podName, strings.ReplaceAll(podIP, ".", "-"), podNs)
+
+	err := operator.WithLeaderElector(&leaderelection.LeaderElectionConfig{
+		// Create a leaderElectionConfig for leader election
+		Lock: &resourcelock.LeaseLock{
+			LeaseMeta: v1.ObjectMeta{
+				Name:      leaseName,
+				Namespace: podNs,
+			},
+			Client: operator.KubeClient().CoordinationV1(),
+			LockConfig: resourcelock.ResourceLockConfig{
+				Identity: identity,
+			},
+		},
+		LeaseDuration: time.Duration(leaseDuration) * time.Second,
+		RenewDeadline: time.Duration(renewalDeadline) * time.Second,
+		RetryPeriod:   time.Duration(retryPeriod) * time.Second,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				err := run(ctx, operator)
+				if err != nil {
+					log.Info(err)
+					os.Exit(1)
+				}
+			},
+			OnStoppedLeading: func() {
+				log.Info("Restarting because the leadership was handed over")
+				operator.Stop()
+				os.Exit(1)
+			},
+		},
+		ReleaseOnCancel: true,
+	})
+	if err != nil {
+		log.Error(err)
+	}
+
+	go func() {
+		<-ctx.Done()
+		log.Info("Context canceled received")
+		err := syscall.Kill(1, syscall.SIGUSR2)
+		if err != nil {
+			log.Infof("Couldn't shutdown deckhouse: %s\n", err)
+			os.Exit(1)
+		}
+	}()
+
+	operator.LeaderElector.Run(ctx)
+}
+
+func run(ctx context.Context, operator *addon_operator.AddonOperator) error {
+	err := d8Apis.EnsureCRDs(ctx, operator.KubeClient(), "/deckhouse/deckhouse-controller/crds/*.yaml")
+	if err != nil {
+		return err
+	}
+
 	// we have to lock the controller run if dhctl lock configmap exists
 	err = lockOnBootstrap(ctx, operator.KubeClient())
 	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+		return err
 	}
 
 	operator.SetupKubeConfigManager(backend.New(operator.KubeClient().RestConfig(), log.StandardLogger().WithField("KubeConfigManagerBackend", "ModuleConfig")))
 	validation.RegisterAdmissionHandlers(operator)
 
 	err = operator.Setup()
 	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+		return err
 	}
 
 	dController, err := controller.NewDeckhouseController(ctx, operator.KubeClient().RestConfig(), operator.ModuleManager)
 	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+		return err
 	}
 
 	err = dController.Start(operator.ModuleManager.GetModuleEventsChannel())
 	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+		return err
 	}
 
 	operator.ModuleManager.SetModuleLoader(dController)
 
 	err = operator.Start()
 	if err != nil {
-		os.Exit(1)
+		return err
 	}
 
 	// Init deckhouse-config service with ModuleManager instance.

deckhouse-controller/pkg/controller/controller.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	"github.com/flant/addon-operator/pkg/module_manager"
-	"github.com/flant/addon-operator/pkg/module_manager/models/modules"
 	"github.com/flant/addon-operator/pkg/module_manager/models/modules/events"
 	"github.com/flant/addon-operator/pkg/utils"
 	log "github.com/sirupsen/logrus"
@@ -126,10 +125,6 @@ func (dml *DeckhouseController) Start(ec chan events.ModuleEvent) error {
 	return nil
 }
 
-func (dml *DeckhouseController) ReloadModule(_, _ string) (*modules.BasicModule, error) {
-	return nil, fmt.Errorf("not implemented yet")
-}
-
 func (dml *DeckhouseController) runEventLoop(ec chan events.ModuleEvent) {
 	for event := range ec {
 		// event without module name

deckhouse-controller/pkg/controller/loader.go

@@ -40,6 +40,10 @@ var (
 	}
 )
 
+func (dml *DeckhouseController) LoadModule(_, _ string) (*modules.BasicModule, error) {
+	return nil, fmt.Errorf("not implemented yet")
+}
+
 func (dml *DeckhouseController) LoadModules() ([]*modules.BasicModule, error) {
 	result := make([]*modules.BasicModule, 0, len(dml.deckhouseModules))
 

global-hooks/create_endpoints.go

@@ -69,6 +69,9 @@ func generateDeckhouseEndpoints(input *go_hook.HookInput, dc dependency.Containe
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      d8Name,
 			Namespace: d8Namespace,
+			Annotations: map[string]string{
+				"created-by": podName,
+			},
 			Labels: map[string]string{
 				"app":                        d8Name,
 				"module":                     d8Name,
@@ -119,6 +122,9 @@ func generateDeckhouseEndpoints(input *go_hook.HookInput, dc dependency.Containe
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      d8Name,
 			Namespace: d8Namespace,
+			Annotations: map[string]string{
+				"created-by": podName,
+			},
 			Labels: map[string]string{
 				"app":                        d8Name,
 				"module":                     d8Name,

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckhouse/deckhouse/dhctl v0.0.0 // use non-existent version for replace
 	github.com/fatih/color v1.13.0
-	github.com/flant/addon-operator v0.0.0-20240222100326-b7482b20ae1e
+	github.com/flant/addon-operator v1.3.9
 	github.com/flant/kube-client v1.1.0
 	github.com/flant/shell-operator v1.4.4
 	github.com/gammazero/deque v0.0.0-20190521012701-46e4ffb7a622

go.sum

@@ -150,8 +150,8 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flant/addon-operator v0.0.0-20240222100326-b7482b20ae1e h1:VpJbBCdx4ukEO11EiX4v533HVzoppI22qr/9OyQVUWA=
-github.com/flant/addon-operator v0.0.0-20240222100326-b7482b20ae1e/go.mod h1:fUEy1sOe1lVfg1e0UtcI0tLa1M1dZ1pnlpGKc+nas48=
+github.com/flant/addon-operator v1.3.9 h1:n1aJhCGW2LsTKmZJblRlgkmEz4fMpbzcJYmS1WWDpVo=
+github.com/flant/addon-operator v1.3.9/go.mod h1:fUEy1sOe1lVfg1e0UtcI0tLa1M1dZ1pnlpGKc+nas48=
 github.com/flant/go-openapi-validate v0.19.12-flant.1 h1:GuB9XEfiLHq3M7fafRLq1AWkndSY/+/5MX7ad1xJ/8A=
 github.com/flant/go-openapi-validate v0.19.12-flant.1/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
 github.com/flant/gogost/v5 v5.13.0 h1:xU60PYSePgtvS7Z1zP9jeT/7e0KsLlFZ/VAn6m+kAJc=

helm_lib/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: deckhouse_lib_helm
   repository: https://deckhouse.github.io/lib-helm
-  version: 1.19.0
-digest: sha256:48ba1e18ae52b3d2b7374519d40213f308ac35bdf7da91ba5a0d7de39f428378
-generated: "2024-02-20T18:53:28.916883+08:00"
+  version: 1.20.0
+digest: sha256:d07b7781bee1846cb1633ea818523bf47e59786d9e229974578f0f799c7256de
+generated: "2024-03-04T18:05:27.608824323+03:00"

helm_lib/Chart.yaml

@@ -5,5 +5,5 @@ version: 0.1.0
 description: "Helm helpers"
 dependencies:
   - name: deckhouse_lib_helm
-    version: "1.19.0"
+    version: "1.20.0"
     repository: https://deckhouse.github.io/lib-helm

helm_lib/charts/deckhouse_lib_helm/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v2
 description: Helm utils template definitions for Deckhouse modules.
 name: deckhouse_lib_helm
 type: library
-version: 1.19.0
+version: 1.20.0

helm_lib/charts/deckhouse_lib_helm/README.md

@@ -97,6 +97,7 @@
 | **Spec For High Availability** |
 | [helm_lib_pod_anti_affinity_for_ha](#helm_lib_pod_anti_affinity_for_ha) |
 | [helm_lib_deployment_on_master_strategy_and_replicas_for_ha](#helm_lib_deployment_on_master_strategy_and_replicas_for_ha) |
+| [helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha](#helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha) |
 | [helm_lib_deployment_strategy_and_replicas_for_ha](#helm_lib_deployment_strategy_and_replicas_for_ha) |
 
 ## Envs For Proxy
@@ -1056,6 +1057,16 @@ list:
 -  Template context with .Values, .Chart, etc 
 
 
+### helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha
+
+ returns deployment with custom strategy and replicas for ha components running on master nodes 
+
+#### Usage
+
+`{{ include "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" (list . (dict "strategy" "strategy_type")) }} `
+
+
+
 ### helm_lib_deployment_strategy_and_replicas_for_ha
 
  returns deployment strategy and replicas for ha components running not on master nodes 

helm_lib/charts/deckhouse_lib_helm/templates/_spec_for_high_availability.tpl

@@ -61,6 +61,68 @@ strategy:
   {{- end }}
 {{- end }}
 
+{{- /* Usage: {{ include "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" (list . (dict "strategy" "strategy_type")) }} */ -}}
+{{- /* returns deployment with custom strategy and replicas for ha components running on master nodes */ -}}
+{{- define "helm_lib_deployment_on_master_custom_strategy_and_replicas_for_ha" }}
+{{- $context := index . 0 }}
+{{- $optionalArgs := dict }}
+{{- $strategy := "RollingUpdate" }}
+{{- if ge (len .) 2 }}
+  {{- $optionalArgs = index . 1 }}
+{{- end }}
+{{- if hasKey $optionalArgs "strategy" }}
+  {{- $strategy = $optionalArgs.strategy }}
+{{- end }}
+{{- /* Template context with .Values, .Chart, etc */ -}}
+  {{- if (include "helm_lib_ha_enabled" $context) }}
+    {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 0 }}
+replicas: {{ index $context.Values.global.discovery "clusterMasterCount" }}
+strategy:
+  type: {{ $strategy }}
+      {{- if eq $strategy "RollingUpdate" }}
+  rollingUpdate:
+    maxSurge: 0
+        {{- if gt (index $context.Values.global.discovery "clusterMasterCount" | int) 2 }}
+    maxUnavailable: 2
+        {{- else }}
+    maxUnavailable: 1
+        {{- end }}
+      {{- end }}
+    {{- else if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 0 }}
+replicas: {{ index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" }}
+strategy:
+  type: {{ $strategy }}
+      {{- if eq $strategy "RollingUpdate" }}
+  rollingUpdate:
+    maxSurge: 0
+        {{- if gt (index $context.Values.global.discovery.d8SpecificNodeCountByRole "master" | int) 2 }}
+    maxUnavailable: 2
+        {{- else }}
+    maxUnavailable: 1
+        {{- end }}
+      {{- end }}
+    {{- else }}
+replicas: 2
+strategy:
+  type: {{ $strategy }}
+      {{- if eq $strategy "RollingUpdate" }}
+  rollingUpdate:
+    maxSurge: 0
+    maxUnavailable: 1
+      {{- end }}
+    {{- end }}
+  {{- else }}
+replicas: 1
+strategy:
+  type: {{ $strategy }}
+    {{- if eq $strategy "RollingUpdate" }}
+  rollingUpdate:
+    maxSurge: 0
+    maxUnavailable: 1
+    {{- end }}
+  {{- end }}
+{{- end }}
+
 {{- /* Usage: {{ include "helm_lib_deployment_strategy_and_replicas_for_ha" }} */ -}}
 {{- /* returns deployment strategy and replicas for ha components running not on master nodes */ -}}
 {{- define "helm_lib_deployment_strategy_and_replicas_for_ha" }}