diff --git a/.github/workflows/e2e-dummy.yaml b/.github/workflows/e2e-dummy.yaml index 1c161d3a1..cf64854e1 100644 --- a/.github/workflows/e2e-dummy.yaml +++ b/.github/workflows/e2e-dummy.yaml @@ -44,7 +44,7 @@ jobs: E2E_LOGS_HUMIO_INGEST_TOKEN: ${{ secrets.E2E_LOGS_HUMIO_INGEST_TOKEN }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - GINKGO_NODES: "12" + GINKGO_NODES: "6" run: | hack/run-e2e-using-kind-dummy.sh - name: cleanup kind and docker files diff --git a/api/v1alpha1/humiocluster_types.go b/api/v1alpha1/humiocluster_types.go index d7bbb5927..85bc1ff64 100644 --- a/api/v1alpha1/humiocluster_types.go +++ b/api/v1alpha1/humiocluster_types.go @@ -19,6 +19,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -290,6 +291,11 @@ type HumioUpdateStrategy struct { // If pods are failing, they bypass the zone limitation and are restarted immediately - ignoring the zone. // Zone awareness is enabled by default. EnableZoneAwareness *bool `json:"enableZoneAwareness,omitempty"` + + // MaxUnavailable is the maximum number of pods that can be unavailable during a rolling update. + // This can be configured to an absolute number or a percentage, e.g. "maxUnavailable: 5" or "maxUnavailable: 25%". + // By default, the max unavailable pods is 1. + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` } type HumioNodePoolSpec struct { @@ -388,10 +394,14 @@ type HumioNodePoolStatus struct { Name string `json:"name"` // State will be empty before the cluster is bootstrapped. From there it can be "Running", "Upgrading", "Restarting" or "Pending" State string `json:"state,omitempty"` - // DesiredPodRevision holds the desired pod revision for pods of the given node pool. - DesiredPodRevision int `json:"desiredPodRevision,omitempty"` // ZoneUnderMaintenance holds the name of the availability zone currently under maintenance ZoneUnderMaintenance string `json:"zoneUnderMaintenance,omitempty"` + // DesiredPodRevision holds the desired pod revision for pods of the given node pool. + DesiredPodRevision int `json:"desiredPodRevision,omitempty"` + // DesiredPodHash holds a hashed representation of the pod spec + DesiredPodHash string `json:"desiredPodHash,omitempty"` + // DesiredBootstrapTokenHash holds a SHA256 of the value set in environment variable BOOTSTRAP_ROOT_TOKEN_HASHED + DesiredBootstrapTokenHash string `json:"desiredBootstrapTokenHash,omitempty"` } // HumioClusterStatus defines the observed state of HumioCluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9daf5858e..53e3d9313 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -1900,6 +1901,11 @@ func (in *HumioUpdateStrategy) DeepCopyInto(out *HumioUpdateStrategy) { *out = new(bool) **out = **in } + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HumioUpdateStrategy. diff --git a/charts/humio-operator/crds/core.humio.com_humioclusters.yaml b/charts/humio-operator/crds/core.humio.com_humioclusters.yaml index 4112c5c61..0a6cbf64d 100644 --- a/charts/humio-operator/crds/core.humio.com_humioclusters.yaml +++ b/charts/humio-operator/crds/core.humio.com_humioclusters.yaml @@ -13089,6 +13089,15 @@ spec: If pods are failing, they bypass the zone limitation and are restarted immediately - ignoring the zone. Zone awareness is enabled by default. type: boolean + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of pods that can be unavailable during a rolling update. + This can be configured to an absolute number or a percentage, e.g. "maxUnavailable: 5" or "maxUnavailable: 25%". + By default, the max unavailable pods is 1. + x-kubernetes-int-or-string: true minReadySeconds: description: MinReadySeconds is the minimum time in seconds that a pod must be ready before the next pod @@ -15001,6 +15010,15 @@ spec: If pods are failing, they bypass the zone limitation and are restarted immediately - ignoring the zone. Zone awareness is enabled by default. type: boolean + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of pods that can be unavailable during a rolling update. + This can be configured to an absolute number or a percentage, e.g. "maxUnavailable: 5" or "maxUnavailable: 25%". + By default, the max unavailable pods is 1. + x-kubernetes-int-or-string: true minReadySeconds: description: MinReadySeconds is the minimum time in seconds that a pod must be ready before the next pod can be deleted when @@ -15064,6 +15082,14 @@ spec: items: description: HumioNodePoolStatus shows the status of each node pool properties: + desiredBootstrapTokenHash: + description: DesiredBootstrapTokenHash holds a SHA256 of the + value set in environment variable BOOTSTRAP_ROOT_TOKEN_HASHED + type: string + desiredPodHash: + description: DesiredPodHash holds a hashed representation of + the pod spec + type: string desiredPodRevision: description: DesiredPodRevision holds the desired pod revision for pods of the given node pool. diff --git a/config/crd/bases/core.humio.com_humioclusters.yaml b/config/crd/bases/core.humio.com_humioclusters.yaml index 4112c5c61..0a6cbf64d 100644 --- a/config/crd/bases/core.humio.com_humioclusters.yaml +++ b/config/crd/bases/core.humio.com_humioclusters.yaml @@ -13089,6 +13089,15 @@ spec: If pods are failing, they bypass the zone limitation and are restarted immediately - ignoring the zone. Zone awareness is enabled by default. type: boolean + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of pods that can be unavailable during a rolling update. + This can be configured to an absolute number or a percentage, e.g. "maxUnavailable: 5" or "maxUnavailable: 25%". + By default, the max unavailable pods is 1. + x-kubernetes-int-or-string: true minReadySeconds: description: MinReadySeconds is the minimum time in seconds that a pod must be ready before the next pod @@ -15001,6 +15010,15 @@ spec: If pods are failing, they bypass the zone limitation and are restarted immediately - ignoring the zone. Zone awareness is enabled by default. type: boolean + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of pods that can be unavailable during a rolling update. + This can be configured to an absolute number or a percentage, e.g. "maxUnavailable: 5" or "maxUnavailable: 25%". + By default, the max unavailable pods is 1. + x-kubernetes-int-or-string: true minReadySeconds: description: MinReadySeconds is the minimum time in seconds that a pod must be ready before the next pod can be deleted when @@ -15064,6 +15082,14 @@ spec: items: description: HumioNodePoolStatus shows the status of each node pool properties: + desiredBootstrapTokenHash: + description: DesiredBootstrapTokenHash holds a SHA256 of the + value set in environment variable BOOTSTRAP_ROOT_TOKEN_HASHED + type: string + desiredPodHash: + description: DesiredPodHash holds a hashed representation of + the pod spec + type: string desiredPodRevision: description: DesiredPodRevision holds the desired pod revision for pods of the given node pool. diff --git a/controllers/humiobootstraptoken_controller.go b/controllers/humiobootstraptoken_controller.go index fd7329052..39c4b7b50 100644 --- a/controllers/humiobootstraptoken_controller.go +++ b/controllers/humiobootstraptoken_controller.go @@ -383,7 +383,6 @@ func (r *HumioBootstrapTokenReconciler) ensureBootstrapTokenHashedToken(ctx cont if err != nil { return err } - // TODO: make tokenHash constant updatedSecret.Data = map[string][]byte{BootstrapTokenSecretHashedTokenName: []byte(secretData.HashedToken), BootstrapTokenSecretSecretName: []byte(secretData.Secret)} if err = r.Update(ctx, updatedSecret); err != nil { diff --git a/controllers/humiocluster_annotations.go b/controllers/humiocluster_annotations.go index 433743f67..10a3abcbd 100644 --- a/controllers/humiocluster_annotations.go +++ b/controllers/humiocluster_annotations.go @@ -16,22 +16,13 @@ limitations under the License. package controllers -import ( - "strconv" - - corev1 "k8s.io/api/core/v1" -) - const ( - certHashAnnotation = "humio.com/certificate-hash" - PodHashAnnotation = "humio.com/pod-hash" - PodRevisionAnnotation = "humio.com/pod-revision" - envVarSourceHashAnnotation = "humio.com/env-var-source-hash" - pvcHashAnnotation = "humio_pvc_hash" - // #nosec G101 - bootstrapTokenHashAnnotation = "humio.com/bootstrap-token-hash" + // Set on Pod and Certificate objects + certHashAnnotation = "humio.com/certificate-hash" + + // Set on Pod objects + PodHashAnnotation = "humio.com/pod-hash" + PodRevisionAnnotation = "humio.com/pod-revision" + BootstrapTokenHashAnnotation = "humio.com/bootstrap-token-hash" // #nosec G101 + envVarSourceHashAnnotation = "humio.com/env-var-source-hash" ) - -func (r *HumioClusterReconciler) setPodRevision(pod *corev1.Pod, newRevision int) { - pod.Annotations[PodRevisionAnnotation] = strconv.Itoa(newRevision) -} diff --git a/controllers/humiocluster_controller.go b/controllers/humiocluster_controller.go index b203a8514..a1c42da3c 100644 --- a/controllers/humiocluster_controller.go +++ b/controllers/humiocluster_controller.go @@ -58,6 +58,9 @@ const ( // MaximumMinReadyRequeue The maximum requeue time to set for the MinReadySeconds functionality - this is to avoid a scenario where we // requeue for hours into the future. MaximumMinReadyRequeue = time.Second * 300 + + // waitingOnPodsMessage is the message that is populated as the message in the cluster status when waiting on pods + waitingOnPodsMessage = "waiting for pods to become ready" ) //+kubebuilder:rbac:groups=core.humio.com,resources=humioclusters,verbs=get;list;watch;create;update;patch;delete @@ -75,13 +78,19 @@ const ( //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingress,verbs=create;delete;get;list;patch;update;watch func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // when running tests, ignore resources that are not in the correct namespace if r.Namespace != "" { if r.Namespace != req.Namespace { return reconcile.Result{}, nil } } - r.Log = r.BaseLogger.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name, "Request.Type", helpers.GetTypeName(r), "Reconcile.ID", kubernetes.RandomString()) + r.Log = r.BaseLogger.WithValues( + "Request.Namespace", req.Namespace, + "Request.Name", req.Name, + "Request.Type", helpers.GetTypeName(r), + "Reconcile.ID", kubernetes.RandomString(), + ) r.Log.Info("Reconciling HumioCluster") // Fetch the HumioCluster @@ -98,73 +107,43 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } r.Log = r.Log.WithValues("Request.UID", hc.UID) - - var humioNodePools HumioNodePoolList - humioNodePools.Add(NewHumioNodeManagerFromHumioCluster(hc)) - for idx := range hc.Spec.NodePools { - humioNodePools.Add(NewHumioNodeManagerFromHumioNodePool(hc, &hc.Spec.NodePools[idx])) - } - + humioNodePools := getHumioNodePoolManagers(hc) emptyResult := reconcile.Result{} - if ok, idx := r.hasNoUnusedNodePoolStatus(hc, &humioNodePools); !ok { - r.cleanupUnusedNodePoolStatus(hc, idx) - if result, err := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withNodePoolStatusList(hc.Status.NodePoolStatus)); err != nil { - return result, r.logErrorAndReturn(err, "unable to set cluster state") - } - } - + // update status with observed generation + // TODO: Look into refactoring of the use of "defer func's" to update HumioCluster.Status. + // Right now we use StatusWriter to update the status multiple times, and rely on RetryOnConflict to retry + // on conflicts which they'll be on many of the status updates. + // We should be able to bundle all the options together and do a single update using StatusWriter. + // Bundling options in a single StatusWriter.Update() should help reduce the number of conflicts. defer func(ctx context.Context, humioClient humio.Client, hc *humiov1alpha1.HumioCluster) { _, _ = r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). withObservedGeneration(hc.GetGeneration())) }(ctx, r.HumioClient, hc) - if err := r.ensureHumioClusterBootstrapToken(ctx, hc); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error())) - } - - for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { - if err := r.setImageFromSource(ctx, pool); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error()). - withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetZoneUnderMaintenance())) - } - if err := r.ensureValidHumioVersion(pool); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error()). - withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetZoneUnderMaintenance())) - } - if err := r.ensureValidStorageConfiguration(pool); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error()). - withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetZoneUnderMaintenance())) - } + // validate details in HumioCluster resource is valid + if result, err := r.verifyHumioClusterConfigurationIsValid(ctx, hc, humioNodePools); result != emptyResult || err != nil { + return result, err } - for _, fun := range []ctxHumioClusterFunc{ - r.ensureLicenseIsValid, - r.ensureValidCASecret, - r.ensureHeadlessServiceExists, - r.ensureInternalServiceExists, - r.validateUserDefinedServiceAccountsExists, - } { - if err := fun(ctx, hc); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error()). - withState(humiov1alpha1.HumioClusterStateConfigError)) + // if the state is not set yet, we know config is valid and mark it as Running + if hc.Status.State == "" { + err := r.setState(ctx, humiov1alpha1.HumioClusterStateRunning, hc) + if err != nil { + return reconcile.Result{}, r.logErrorAndReturn(err, "unable to set cluster state") } } - if len(humioNodePools.Filter(NodePoolFilterHasNode)) > 0 { - if err := r.ensureNodePoolSpecificResourcesHaveLabelWithNodePoolName(ctx, humioNodePools.Filter(NodePoolFilterHasNode)[0]); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error()). - withState(humiov1alpha1.HumioClusterStateConfigError)) + // create HumioBootstrapToken and block until we have a hashed bootstrap token + if result, err := r.ensureHumioClusterBootstrapToken(ctx, hc); result != emptyResult || err != nil { + if err != nil { + _, _ = r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error())) } + return result, err } + // update status with pods and nodeCount based on podStatusList defer func(ctx context.Context, hc *humiov1alpha1.HumioCluster) { opts := statusOptions() podStatusList, err := r.getPodStatusList(ctx, hc, humioNodePools.Filter(NodePoolFilterHasNode)) @@ -176,17 +155,9 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request withNodeCount(len(podStatusList))) }(ctx, hc) - for _, pool := range humioNodePools.Items { - if err := r.ensureOrphanedPvcsAreDeleted(ctx, hc, pool); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error())) - } - } - + // ensure pods that does not run the desired version or config gets deleted and update state accordingly for _, pool := range humioNodePools.Items { if r.nodePoolAllowsMaintenanceOperations(hc, pool, humioNodePools.Items) { - // TODO: result should be controlled and returned by the status - // Ensure pods that does not run the desired version are deleted. result, err := r.ensureMismatchedPodsAreDeleted(ctx, hc, pool) if result != emptyResult || err != nil { return result, err @@ -194,34 +165,14 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } } - for _, pool := range humioNodePools.Items { - if err := r.validateInitialPodSpec(pool); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error()). - withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetZoneUnderMaintenance())) - } - } - - if err := r.validateNodeCount(hc, humioNodePools.Items); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error()). - withState(humiov1alpha1.HumioClusterStateConfigError)) - } - - if hc.Status.State == "" { - err := r.setState(ctx, humiov1alpha1.HumioClusterStateRunning, hc) - if err != nil { - return reconcile.Result{}, r.logErrorAndReturn(err, "unable to set cluster state") - } - } - + // create various k8s objects, e.g. Issuer, Certificate, ConfigMap, Ingress, Service, ServiceAccount, ClusterRole, ClusterRoleBinding for _, fun := range []ctxHumioClusterFunc{ r.ensureValidCAIssuer, r.ensureHumioClusterCACertBundle, r.ensureHumioClusterKeystoreSecret, r.ensureViewGroupPermissionsConfigMap, r.ensureRolePermissionsConfigMap, - r.ensureNoIngressesIfIngressNotEnabled, + r.ensureNoIngressesIfIngressNotEnabled, // TODO: cleanupUnusedResources seems like a better place for this r.ensureIngress, } { if err := fun(ctx, hc); err != nil { @@ -229,7 +180,6 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request withMessage(err.Error())) } } - for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { for _, fun := range []ctxHumioClusterPoolFunc{ r.ensureService, @@ -245,59 +195,74 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } } + // update annotations on ServiceAccount object and trigger pod restart if annotations were changed for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { if issueRestart, err := r.ensureHumioServiceAccountAnnotations(ctx, pool); err != nil || issueRestart { desiredPodRevision := pool.GetDesiredPodRevision() if issueRestart { + // TODO: Code seems to only try to save the updated pod revision in the same reconcile as the annotations on the ServiceAccount was updated. + // We should ensure that even if we don't store it in the current reconcile, we'll still properly detect it next time and retry storing this updated pod revision. + // Looks like a candidate for storing a ServiceAccount annotation hash in node pool status, similar to pod hash, bootstrap token hash, etc. + // as this way we'd both store the updated hash *and* the updated pod revision in the same k8sClient.Update() API call. desiredPodRevision++ } _, err = r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withNodePoolState(hc.Status.State, pool.GetNodePoolName(), desiredPodRevision, "")) + withNodePoolState(hc.Status.State, pool.GetNodePoolName(), desiredPodRevision, pool.GetDesiredPodHash(), pool.GetDesiredBootstrapTokenHash(), "")) return reconcile.Result{Requeue: true}, err } } + // create pvcs if needed for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { if err := r.ensurePersistentVolumeClaimsExist(ctx, hc, pool); err != nil { opts := statusOptions() if hc.Status.State != humiov1alpha1.HumioClusterStateRestarting && hc.Status.State != humiov1alpha1.HumioClusterStateUpgrading { - opts.withNodePoolState(humiov1alpha1.HumioClusterStatePending, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetZoneUnderMaintenance()) + opts.withNodePoolState(humiov1alpha1.HumioClusterStatePending, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetDesiredPodHash(), pool.GetDesiredBootstrapTokenHash(), pool.GetZoneUnderMaintenance()) } return r.updateStatus(ctx, r.Client.Status(), hc, opts. withMessage(err.Error())) } } - // TODO: result should be controlled and returned by the status + // create pods if needed for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { - if result, err := r.ensurePodsExist(ctx, hc, pool); result != emptyResult || err != nil { - if err != nil { - _, _ = r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error())) + if r.nodePoolAllowsMaintenanceOperations(hc, pool, humioNodePools.Items) { + if result, err := r.ensurePodsExist(ctx, hc, pool); result != emptyResult || err != nil { + if err != nil { + _, _ = r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error())) + } + return result, err } - return result, err } } - for _, nodePool := range humioNodePools.Filter(NodePoolFilterDoesNotHaveNodes) { - if err := r.cleanupUnusedService(ctx, nodePool); err != nil { + // wait for pods to start up + for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { + if podsReady, err := r.nodePoolPodsReady(ctx, hc, pool); !podsReady || err != nil { + msg := waitingOnPodsMessage + if err != nil { + msg = err.Error() + } return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error())) + withState(hc.Status.State). + withMessage(msg)) } } - // TODO: result should be controlled and returned by the status + // wait for license and admin token if len(r.nodePoolsInMaintenance(hc, humioNodePools.Filter(NodePoolFilterHasNode))) == 0 { - if result, err := r.ensureLicense(ctx, hc, req); result != emptyResult || err != nil { + if result, err := r.ensureLicenseAndAdminToken(ctx, hc, req); result != emptyResult || err != nil { if err != nil { _, _ = r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(r.logErrorAndReturn(err, "unable to ensure license is installed").Error())) + withMessage(r.logErrorAndReturn(err, "unable to ensure license is installed and admin token is created").Error())) } // Usually if we fail to get the license, that means the cluster is not up. So wait a bit longer than usual to retry return reconcile.Result{RequeueAfter: time.Second * 15}, nil } } + // construct humioClient configured with the admin token cluster, err := helpers.NewCluster(ctx, r, hc.Name, "", hc.Namespace, helpers.UseCertManager(), true, false) if err != nil || cluster == nil || cluster.Config() == nil { return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). @@ -305,6 +270,7 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request withState(humiov1alpha1.HumioClusterStateConfigError)) } + // update status with version defer func(ctx context.Context, humioClient humio.Client, hc *humiov1alpha1.HumioCluster) { opts := statusOptions() if hc.Status.State == humiov1alpha1.HumioClusterStateRunning { @@ -317,27 +283,9 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } }(ctx, r.HumioClient, hc) - for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { - if podsReady, err := r.nodePoolPodsReady(ctx, hc, pool); !podsReady || err != nil { - msg := "waiting on all pods to be ready" - if err != nil { - msg = err.Error() - } - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withState(hc.Status.State). - withMessage(msg)) - } - } - - for _, fun := range []ctxHumioClusterFunc{ - r.cleanupUnusedTLSCertificates, - r.cleanupUnusedTLSSecrets, - r.cleanupUnusedCAIssuer, - } { - if err := fun(ctx, hc); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(err.Error())) - } + // clean up various k8s objects we no longer need + if result, err := r.cleanupUnusedResources(ctx, hc, humioNodePools); result != emptyResult || err != nil { + return result, err } r.Log.Info("done reconciling") @@ -372,9 +320,9 @@ func (r *HumioClusterReconciler) nodePoolPodsReady(ctx context.Context, hc *humi r.Log.Info(fmt.Sprintf("cluster state is %s. waitingOnPods=%v, "+ "revisionsInSync=%v, podRevisions=%v, podDeletionTimestampSet=%v, podNames=%v, expectedRunningPods=%v, "+ "podsReady=%v, podsNotReady=%v", - hc.Status.State, podsStatus.waitingOnPods(), podsStatus.podRevisionsInSync(), + hc.Status.State, podsStatus.waitingOnPods(), podsStatus.podRevisionCountMatchesNodeCountAndAllPodsHaveTheSameRevision(), podsStatus.podRevisions, podsStatus.podDeletionTimestampSet, podsStatus.podNames, - podsStatus.expectedRunningPods, podsStatus.readyCount, podsStatus.notReadyCount)) + podsStatus.nodeCount, podsStatus.readyCount, podsStatus.notReadyCount)) return false, nil } return true, nil @@ -426,27 +374,32 @@ func (r *HumioClusterReconciler) hasNoUnusedNodePoolStatus(hc *humiov1alpha1.Hum return true, 0 } -func (r *HumioClusterReconciler) ensureHumioClusterBootstrapToken(ctx context.Context, hc *humiov1alpha1.HumioCluster) error { +func (r *HumioClusterReconciler) ensureHumioClusterBootstrapToken(ctx context.Context, hc *humiov1alpha1.HumioCluster) (reconcile.Result, error) { r.Log.Info("ensuring humiobootstraptoken") hbtList, err := kubernetes.ListHumioBootstrapTokens(ctx, r.Client, hc.GetNamespace(), kubernetes.LabelsForHumioBootstrapToken(hc.GetName())) if err != nil { - return r.logErrorAndReturn(err, "could not list HumioBootstrapToken") + return reconcile.Result{}, r.logErrorAndReturn(err, "could not list HumioBootstrapToken") } if len(hbtList) > 0 { - r.Log.Info("humiobootstraptoken already exists") - return nil + r.Log.Info("humiobootstraptoken already exists, checking if HumioBootstrapTokenReconciler populated it") + if hbtList[0].Status.State == humiov1alpha1.HumioBootstrapTokenStateReady { + return reconcile.Result{}, nil + } + r.Log.Info("secret not populated yet, waiting on HumioBootstrapTokenReconciler") + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil } hbt := kubernetes.ConstructHumioBootstrapToken(hc.GetName(), hc.GetNamespace()) if err := controllerutil.SetControllerReference(hc, hbt, r.Scheme()); err != nil { - return r.logErrorAndReturn(err, "could not set controller reference") + return reconcile.Result{}, r.logErrorAndReturn(err, "could not set controller reference") } r.Log.Info(fmt.Sprintf("creating humiobootstraptoken %s", hbt.Name)) err = r.Create(ctx, hbt) if err != nil { - return r.logErrorAndReturn(err, "could not create bootstrap token resource") + return reconcile.Result{}, r.logErrorAndReturn(err, "could not create bootstrap token resource") } - return nil + + return reconcile.Result{Requeue: true}, nil } func (r *HumioClusterReconciler) validateInitialPodSpec(hnp *HumioNodePool) error { @@ -1298,8 +1251,8 @@ func (r *HumioClusterReconciler) ensureLicenseIsValid(ctx context.Context, hc *h return nil } -func (r *HumioClusterReconciler) ensureLicense(ctx context.Context, hc *humiov1alpha1.HumioCluster, req ctrl.Request) (reconcile.Result, error) { - r.Log.Info("ensuring license") +func (r *HumioClusterReconciler) ensureLicenseAndAdminToken(ctx context.Context, hc *humiov1alpha1.HumioCluster, req ctrl.Request) (reconcile.Result, error) { + r.Log.Info("ensuring license and admin token") // Configure a Humio client without an API token which we can use to check the current license on the cluster noLicense := humioapi.OnPremLicense{} @@ -1313,6 +1266,7 @@ func (r *HumioClusterReconciler) ensureLicense(ctx context.Context, hc *humiov1a return ctrl.Result{}, fmt.Errorf("failed to get license: %w", err) } + // update status with license details defer func(ctx context.Context, hc *humiov1alpha1.HumioCluster) { if existingLicense != nil { licenseStatus := humiov1alpha1.HumioLicenseStatus{ @@ -1837,222 +1791,204 @@ func (r *HumioClusterReconciler) ensureHumioServiceAccountAnnotations(ctx contex // and the reconciliation will requeue and the deletions will continue to be executed until all the pods have been // removed. func (r *HumioClusterReconciler) ensureMismatchedPodsAreDeleted(ctx context.Context, hc *humiov1alpha1.HumioCluster, hnp *HumioNodePool) (reconcile.Result, error) { - foundPodListForNodePool, err := kubernetes.ListPods(ctx, r, hnp.GetNamespace(), hnp.GetNodePoolLabels()) - if err != nil { - return reconcile.Result{}, r.logErrorAndReturn(err, "failed to list pods") - } - - // if we do not have any pods running we have nothing to delete - if len(foundPodListForNodePool) == 0 { - return reconcile.Result{}, nil - } - r.Log.Info("ensuring mismatching pods are deleted") - attachments := &podAttachments{} - // In the case we are using PVCs, we cannot lookup the available PVCs since they may already be in use - if hnp.DataVolumePersistentVolumeClaimSpecTemplateIsSetByUser() { - attachments.dataVolumeSource = hnp.GetDataVolumePersistentVolumeClaimSpecTemplate("") - } - envVarSourceData, err := r.getEnvVarSource(ctx, hnp) - if err != nil { - result, _ := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(r.logErrorAndReturn(err, "got error when getting pod envVarSource").Error()). - withState(humiov1alpha1.HumioClusterStateConfigError)) + attachments, result, err := r.constructPodAttachments(ctx, hc, hnp) + emptyResult := reconcile.Result{} + if result != emptyResult || err != nil { return result, err } - if envVarSourceData != nil { - attachments.envVarSourceData = envVarSourceData - } - humioBootstrapTokens, err := kubernetes.ListHumioBootstrapTokens(ctx, r.Client, hc.GetNamespace(), kubernetes.LabelsForHumioBootstrapToken(hc.GetName())) + // fetch list of all current pods for the node pool + listOfAllCurrentPodsForNodePool, err := kubernetes.ListPods(ctx, r, hnp.GetNamespace(), hnp.GetNodePoolLabels()) if err != nil { - return reconcile.Result{}, r.logErrorAndReturn(err, "failed to get bootstrap token") - } - if len(humioBootstrapTokens) > 0 { - if humioBootstrapTokens[0].Status.State == humiov1alpha1.HumioBootstrapTokenStateReady { - attachments.bootstrapTokenSecretReference.secretReference = humioBootstrapTokens[0].Status.HashedTokenSecretKeyRef.SecretKeyRef - bootstrapTokenHash, err := r.getDesiredBootstrapTokenHash(ctx, hc) - if err != nil { - return reconcile.Result{}, r.logErrorAndReturn(err, "unable to find bootstrap token secret") - } - attachments.bootstrapTokenSecretReference.hash = bootstrapTokenHash - } + return reconcile.Result{}, r.logErrorAndReturn(err, "failed to list pods") } - podsStatus, err := r.getPodsStatus(ctx, hc, hnp, foundPodListForNodePool) + // fetch podStatus where we collect information about current pods + podsStatus, err := r.getPodsStatus(ctx, hc, hnp, listOfAllCurrentPodsForNodePool) if err != nil { return reconcile.Result{}, r.logErrorAndReturn(err, "failed to get pod status") } - if podsStatus.havePodsRequiringDeletion() { - r.Log.Info(fmt.Sprintf("found %d humio pods requiring deletion", len(podsStatus.podsRequiringDeletion))) - r.Log.Info(fmt.Sprintf("deleting pod %s", podsStatus.podsRequiringDeletion[0].Name)) - if err = r.Delete(ctx, &podsStatus.podsRequiringDeletion[0]); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(r.logErrorAndReturn(err, fmt.Sprintf("could not delete pod %s", podsStatus.podsRequiringDeletion[0].Name)).Error())) - } - return reconcile.Result{RequeueAfter: time.Second + 1}, nil + // based on all pods we have, fetch compare list of all current pods with desired pods + desiredLifecycleState, desiredPod, err := r.getPodDesiredLifecycleState(ctx, hnp, listOfAllCurrentPodsForNodePool, attachments, podsStatus.foundEvictedPodsOrPodsWithOrpahanedPVCs() || podsStatus.haveUnschedulablePodsOrPodsWithBadStatusConditions()) + if err != nil { + return reconcile.Result{}, r.logErrorAndReturn(err, "got error when getting pod desired lifecycle") } - // prioritize deleting the pods with errors - var podListConsideredForDeletion []corev1.Pod - podsWithErrors := false - if podsStatus.havePodsWithErrors() { - r.Log.Info(fmt.Sprintf("found %d humio pods with errors", len(podsStatus.podErrors))) - podListConsideredForDeletion = podsStatus.podErrors - podsWithErrors = true - } else { - podListConsideredForDeletion = foundPodListForNodePool - - // if zone awareness is enabled, pick a zone if we haven't picked one yet - if !helpers.UseEnvtest() { - if hnp.GetState() == humiov1alpha1.HumioClusterStateUpgrading || hnp.GetState() == humiov1alpha1.HumioClusterStateRestarting { - if *hnp.GetUpdateStrategy().EnableZoneAwareness { - // If we still have pods without the desired revision and have not picked a zone yet, pick one. - if hnp.GetZoneUnderMaintenance() == "" && len(podListConsideredForDeletion) > 0 { - // Filter out any pods that already have the right pod revision - podListForCurrentZoneWithWrongPodRevision := FilterPodsExcludePodsWithPodRevision(foundPodListForNodePool, hnp.GetDesiredPodRevision()) - r.Log.Info(fmt.Sprintf("zone awareness enabled, len(podListForCurrentZoneWithWrongPodRevision)=%d", len(podListForCurrentZoneWithWrongPodRevision))) - - // Filter out any pods with empty nodeName fields - podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName := FilterPodsExcludePodsWithEmptyNodeName(podListForCurrentZoneWithWrongPodRevision) - r.Log.Info(fmt.Sprintf("zone awareness enabled, len(podListForCurrentZoneWithWrongPodRevision)=%d", len(podListForCurrentZoneWithWrongPodRevision))) - - if len(podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName) > 0 { - newZoneUnderMaintenance, err := kubernetes.GetZoneForNodeName(ctx, r, podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName[0].Spec.NodeName) - if err != nil { - return reconcile.Result{}, r.logErrorAndReturn(err, "unable to fetch zone") - } - - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withNodePoolState(hnp.GetState(), hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), newZoneUnderMaintenance)) - } - } + // dump the current state of things + r.Log.Info(fmt.Sprintf("cluster state is %s. waitingOnPods=%v, ADifferenceWasDetectedAndManualDeletionsNotEnabled=%v, "+ + "revisionsInSync=%v, podRevisions=%v, podDeletionTimestampSet=%v, podNames=%v, podHumioVersions=%v, expectedRunningPods=%v, podsReady=%v, podsNotReady=%v", + hc.Status.State, podsStatus.waitingOnPods(), desiredLifecycleState.ADifferenceWasDetectedAndManualDeletionsNotEnabled(), podsStatus.podRevisionCountMatchesNodeCountAndAllPodsHaveTheSameRevision(), + podsStatus.podRevisions, podsStatus.podDeletionTimestampSet, podsStatus.podNames, podsStatus.podImageVersions, podsStatus.nodeCount, podsStatus.readyCount, podsStatus.notReadyCount)) - // If there's a zone marked as under maintenance, we clear the zone-under-maintenance marker if no more work is left in that zone - if hnp.GetZoneUnderMaintenance() != "" { - r.Log.Info(fmt.Sprintf("zone awareness enabled, len(podListConsideredForDeletion)=%d", len(podListConsideredForDeletion))) + // we expect an annotation for the bootstrap token to be present + desiredBootstrapTokenHash, found := desiredPod.Annotations[BootstrapTokenHashAnnotation] + if !found { + return reconcile.Result{}, fmt.Errorf("desiredPod does not have the mandatory annotation %s", BootstrapTokenHashAnnotation) + } - // We have pods left and need to filter them by the zone marked as under maintenance - podListForCurrentZone, err := FilterPodsByZoneName(ctx, r, podListConsideredForDeletion, hnp.GetZoneUnderMaintenance()) - if err != nil { - return reconcile.Result{}, r.logErrorAndReturn(err, "got error filtering pods by zone name") - } - r.Log.Info(fmt.Sprintf("zone awareness enabled, len(podListForCurrentZone)=%d", len(podListForCurrentZone))) - - // Filter out any pods that already have the right pod revision, and clear the zone-under-maintenance marker if no pods are left after filtering - podListForCurrentZoneWithWrongPodRevision := FilterPodsExcludePodsWithPodRevision(podListForCurrentZone, hnp.GetDesiredPodRevision()) - r.Log.Info(fmt.Sprintf("zone awareness enabled, len(podListForCurrentZoneWithWrongPodRevision)=%d", len(podListForCurrentZoneWithWrongPodRevision))) - if len(podListForCurrentZoneWithWrongPodRevision) == 0 { - r.Log.Info(fmt.Sprintf("zone awareness enabled, no more pods for nodePool=%s in zone=%s", hnp.GetNodePoolName(), hnp.GetZoneUnderMaintenance())) - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withNodePoolState(hnp.GetState(), hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), "")) - } - } - } - } - } + // calculate desired pod hash + desiredPodHash := podSpecAsSHA256(hnp, *desiredPod) + + // if bootstrap token hash or desired pod hash differs, update node pool status with the new values + if desiredPodHash != hnp.GetDesiredPodHash() || desiredPod.Annotations[BootstrapTokenHashAnnotation] != hnp.GetDesiredBootstrapTokenHash() { + oldRevision := hnp.GetDesiredPodRevision() + newRevision := oldRevision + 1 + + r.Log.Info(fmt.Sprintf("detected a new pod hash for nodepool=%s, updating status with oldPodRevision=%d newPodRevision=%d oldPodHash=%s newPodHash=%s oldBootstrapTokenHash=%s newBootstrapTokenHash=%s clusterState=%s", + hnp.GetNodePoolName(), + oldRevision, newRevision, + hnp.GetDesiredPodHash(), desiredPodHash, + hnp.GetDesiredBootstrapTokenHash(), desiredBootstrapTokenHash, + hc.Status.State), + ) + + _, err := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions().withNodePoolState(hc.Status.State, hnp.GetNodePoolName(), newRevision, desiredPodHash, desiredBootstrapTokenHash, "")) + return reconcile.Result{Requeue: true}, err } - desiredLifecycleState, err := r.getPodDesiredLifecycleState(ctx, hnp, podListConsideredForDeletion, attachments, podsWithErrors) - if err != nil { - return reconcile.Result{}, r.logErrorAndReturn(err, "got error when getting pod desired lifecycle") + // when no more changes are needed, update state to Running + if hnp.GetState() != humiov1alpha1.HumioClusterStateRunning && + !podsStatus.waitingOnPods() && + !desiredLifecycleState.FoundConfigurationDifference() && + !desiredLifecycleState.FoundVersionDifference() { + r.Log.Info(fmt.Sprintf("updating cluster state as no difference was detected, updating from=%s to=%s", hnp.GetState(), humiov1alpha1.HumioClusterStateRunning)) + _, err := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withNodePoolState(humiov1alpha1.HumioClusterStateRunning, hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), hnp.GetDesiredPodHash(), hnp.GetDesiredBootstrapTokenHash(), "")) + return reconcile.Result{Requeue: true}, err } - // If we are currently deleting pods, then check if the cluster state is Running or in a ConfigError state. If it - // is, then change to an appropriate state depending on the restart policy. - // If the cluster state is set as per the restart policy: - // PodRestartPolicyRecreate == HumioClusterStateUpgrading - // PodRestartPolicyRolling == HumioClusterStateRestarting + // when we detect changes, update status to reflect Upgrading/Restarting if hc.Status.State == humiov1alpha1.HumioClusterStateRunning || hc.Status.State == humiov1alpha1.HumioClusterStateConfigError { - podRevision := hnp.GetDesiredPodRevision() - podRevision++ - if desiredLifecycleState.WantsUpgrade() { - r.Log.Info(fmt.Sprintf("changing cluster state from %s to %s with pod revision %d for node pool %s", hc.Status.State, humiov1alpha1.HumioClusterStateUpgrading, podRevision, hnp.GetNodePoolName())) + if desiredLifecycleState.FoundVersionDifference() { + r.Log.Info(fmt.Sprintf("changing cluster state from %s to %s with pod revision %d for node pool %s", hc.Status.State, humiov1alpha1.HumioClusterStateUpgrading, hnp.GetDesiredPodRevision(), hnp.GetNodePoolName())) if result, err := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withNodePoolState(humiov1alpha1.HumioClusterStateUpgrading, hnp.GetNodePoolName(), podRevision, "")); err != nil { + withNodePoolState(humiov1alpha1.HumioClusterStateUpgrading, hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), hnp.GetDesiredPodHash(), hnp.GetDesiredBootstrapTokenHash(), "")); err != nil { return result, err } return reconcile.Result{Requeue: true}, nil } - if !desiredLifecycleState.WantsUpgrade() && desiredLifecycleState.WantsRestart() { - r.Log.Info(fmt.Sprintf("changing cluster state from %s to %s with pod revision %d for node pool %s", hc.Status.State, humiov1alpha1.HumioClusterStateRestarting, podRevision, hnp.GetNodePoolName())) + if !desiredLifecycleState.FoundVersionDifference() && desiredLifecycleState.FoundConfigurationDifference() { + r.Log.Info(fmt.Sprintf("changing cluster state from %s to %s with pod revision %d for node pool %s", hc.Status.State, humiov1alpha1.HumioClusterStateRestarting, hnp.GetDesiredPodRevision(), hnp.GetNodePoolName())) if result, err := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withNodePoolState(humiov1alpha1.HumioClusterStateRestarting, hnp.GetNodePoolName(), podRevision, "")); err != nil { + withNodePoolState(humiov1alpha1.HumioClusterStateRestarting, hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), hnp.GetDesiredPodHash(), hnp.GetDesiredBootstrapTokenHash(), "")); err != nil { return result, err } return reconcile.Result{Requeue: true}, nil } } - if desiredLifecycleState.ShouldDeletePod() { - if hc.Status.State == humiov1alpha1.HumioClusterStateRestarting || hc.Status.State == humiov1alpha1.HumioClusterStateUpgrading { - if podsStatus.waitingOnPods() && desiredLifecycleState.ShouldRollingRestart() { - r.Log.Info(fmt.Sprintf("pod %s should be deleted, but waiting because not all other pods are "+ - "ready. waitingOnPods=%v, clusterState=%s", desiredLifecycleState.pod.Name, - podsStatus.waitingOnPods(), hc.Status.State)) + + // delete evicted pods and pods attached using PVC's attached to worker nodes that no longer exists + if podsStatus.foundEvictedPodsOrPodsWithOrpahanedPVCs() { + r.Log.Info(fmt.Sprintf("found %d humio pods requiring deletion", len(podsStatus.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists))) + r.Log.Info(fmt.Sprintf("deleting pod %s", podsStatus.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists[0].Name)) + if err = r.Delete(ctx, &podsStatus.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists[0]); err != nil { + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(r.logErrorAndReturn(err, fmt.Sprintf("could not delete pod %s", podsStatus.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists[0].Name)).Error())) + } + return reconcile.Result{RequeueAfter: time.Second + 1}, nil + } + + // delete unschedulable pods or pods with bad status conditions (crashing,exited) + if podsStatus.haveUnschedulablePodsOrPodsWithBadStatusConditions() { + r.Log.Info(fmt.Sprintf("found %d humio pods with errors", len(podsStatus.podAreUnschedulableOrHaveBadStatusConditions))) + + for i, pod := range podsStatus.podAreUnschedulableOrHaveBadStatusConditions { + r.Log.Info(fmt.Sprintf("deleting pod with error[%d] %s", i, pod.Name)) + err = r.Delete(ctx, &pod) + return reconcile.Result{Requeue: true}, err + } + } + + podsForDeletion := desiredLifecycleState.podsToBeReplaced + + // if zone awareness is enabled, we pin a zone until we're done replacing all pods in that zone, + // this is repeated for each zone with pods that needs replacing + if *hnp.GetUpdateStrategy().EnableZoneAwareness && !helpers.UseEnvtest() { + if hnp.GetZoneUnderMaintenance() == "" { + // pick a zone if we haven't already picked one + podListForCurrentZoneWithWrongPodRevisionOrPodHash := FilterPodsExcludePodsWithPodRevisionOrPodHash(listOfAllCurrentPodsForNodePool, hnp.GetDesiredPodRevision(), hnp.GetDesiredPodHash()) + podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName := FilterPodsExcludePodsWithEmptyNodeName(podListForCurrentZoneWithWrongPodRevisionOrPodHash) + r.Log.Info(fmt.Sprintf("zone awareness enabled, len(podListForCurrentZoneWithWrongPodRevisionOrPodHash)=%d len(podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName)=%d", len(podListForCurrentZoneWithWrongPodRevisionOrPodHash), len(podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName))) + + if len(podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName) > 0 { + newZoneUnderMaintenance, err := kubernetes.GetZoneForNodeName(ctx, r, podListForCurrentZoneWithWrongPodRevisionAndNonEmptyNodeName[0].Spec.NodeName) + if err != nil { + return reconcile.Result{}, r.logErrorAndReturn(err, "unable to fetch zone") + } + r.Log.Info(fmt.Sprintf("zone awareness enabled, pinning zone for nodePool=%s in oldZoneUnderMaintenance=%s newZoneUnderMaintenance=%s", + hnp.GetNodePoolName(), hnp.GetZoneUnderMaintenance(), newZoneUnderMaintenance)) return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage("waiting for pods to become ready")) + withNodePoolState(hnp.GetState(), hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), hnp.GetDesiredPodHash(), hnp.GetDesiredBootstrapTokenHash(), newZoneUnderMaintenance)) + } + } else { + // clear the zone-under-maintenance marker if no more work is left in that zone + allPodsInZoneZoneUnderMaintenanceIncludingAlreadyMarkedForDeletion, err := FilterPodsByZoneName(ctx, r, listOfAllCurrentPodsForNodePool, hnp.GetZoneUnderMaintenance()) + if err != nil { + return reconcile.Result{}, r.logErrorAndReturn(err, "got error filtering pods by zone name") + } + allPodsInZoneZoneUnderMaintenanceIncludingAlreadyMarkedForDeletionWithWrongHashOrRevision := FilterPodsExcludePodsWithPodRevisionOrPodHash(allPodsInZoneZoneUnderMaintenanceIncludingAlreadyMarkedForDeletion, hnp.GetDesiredPodRevision(), hnp.GetDesiredPodHash()) + if len(allPodsInZoneZoneUnderMaintenanceIncludingAlreadyMarkedForDeletionWithWrongHashOrRevision) == 0 { + r.Log.Info(fmt.Sprintf("zone awareness enabled, clearing zone nodePool=%s in oldZoneUnderMaintenance=%s newZoneUnderMaintenance=%s", + hnp.GetNodePoolName(), hnp.GetZoneUnderMaintenance(), "")) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withNodePoolState(hnp.GetState(), hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), hnp.GetDesiredPodHash(), hnp.GetDesiredBootstrapTokenHash(), "")) } } + } - var remainingMinReadyWaitTime = desiredLifecycleState.RemainingMinReadyWaitTime(podsStatus.podsReady) - if remainingMinReadyWaitTime > 0 { - if remainingMinReadyWaitTime > MaximumMinReadyRequeue { - // Only requeue after MaximumMinReadyRequeue if the remaining ready wait time is very high - r.Log.Info(fmt.Sprintf("Postponing pod=%s deletion due to the MinReadySeconds setting - requeue time is very long at %s seconds, setting to requeueSeconds=%s", desiredLifecycleState.pod.Name, remainingMinReadyWaitTime, MaximumMinReadyRequeue)) - return reconcile.Result{RequeueAfter: MaximumMinReadyRequeue}, nil + // delete pods up to maxUnavailable from (filtered) pod list + if desiredLifecycleState.ADifferenceWasDetectedAndManualDeletionsNotEnabled() { + if hc.Status.State == humiov1alpha1.HumioClusterStateRestarting || hc.Status.State == humiov1alpha1.HumioClusterStateUpgrading { + if podsStatus.waitingOnPods() && desiredLifecycleState.ShouldRollingRestart() { + r.Log.Info(fmt.Sprintf("pods %s should be deleted, but waiting because not all other pods are "+ + "ready. waitingOnPods=%v, clusterState=%s", desiredLifecycleState.namesOfPodsToBeReplaced(), + podsStatus.waitingOnPods(), hc.Status.State), + "podsStatus.readyCount", podsStatus.readyCount, + "podsStatus.nodeCount", podsStatus.nodeCount, + "podsStatus.notReadyCount", podsStatus.notReadyCount, + "!podsStatus.haveUnschedulablePodsOrPodsWithBadStatusConditions()", !podsStatus.haveUnschedulablePodsOrPodsWithBadStatusConditions(), + "!podsStatus.foundEvictedPodsOrPodsWithOrpahanedPVCs()", !podsStatus.foundEvictedPodsOrPodsWithOrpahanedPVCs(), + ) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(waitingOnPodsMessage)) } - r.Log.Info(fmt.Sprintf("Postponing pod=%s deletion due to the MinReadySeconds setting - requeuing after requeueSeconds=%s", desiredLifecycleState.pod.Name, remainingMinReadyWaitTime)) - return reconcile.Result{RequeueAfter: remainingMinReadyWaitTime}, nil } - r.Log.Info(fmt.Sprintf("deleting pod %s", desiredLifecycleState.pod.Name)) - if err = r.Delete(ctx, &desiredLifecycleState.pod); err != nil { - return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withMessage(r.logErrorAndReturn(err, fmt.Sprintf("could not delete pod %s", desiredLifecycleState.pod.Name)).Error())) + for i := 0; i < podsStatus.scaledMaxUnavailableMinusNotReadyDueToMinReadySeconds() && i < len(podsForDeletion); i++ { + pod := podsForDeletion[i] + zone := "" + if *hnp.GetUpdateStrategy().EnableZoneAwareness && !helpers.UseEnvtest() { + zone, _ = kubernetes.GetZoneForNodeName(ctx, r.Client, pod.Spec.NodeName) + } + r.Log.Info(fmt.Sprintf("deleting pod[%d] %s", i, pod.Name), + "zone", zone, + "podsStatus.scaledMaxUnavailableMinusNotReadyDueToMinReadySeconds()", podsStatus.scaledMaxUnavailableMinusNotReadyDueToMinReadySeconds(), + "len(podsForDeletion)", len(podsForDeletion), + ) + if err = r.Delete(ctx, &pod); err != nil { + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(r.logErrorAndReturn(err, fmt.Sprintf("could not delete pod %s", pod.Name)).Error())) + } } } else { - if desiredLifecycleState.WantsUpgrade() { - r.Log.Info(fmt.Sprintf("pod %s should be deleted because cluster upgrade is wanted but refusing due to the configured upgrade strategy", - desiredLifecycleState.pod.Name)) - } else if desiredLifecycleState.WantsRestart() { - r.Log.Info(fmt.Sprintf("pod %s should be deleted because cluster restart is wanted but refusing due to the configured upgrade strategy", - desiredLifecycleState.pod.Name)) - } - } - - // If we allow a rolling update, then don't take down more than one pod at a time. - // Check the number of ready pods. if we have already deleted a pod, then the ready count will less than expected, - // but we must continue with reconciliation so the pod may be created later in the reconciliation. - // If we're doing a non-rolling update (recreate), then we can take down all the pods without waiting, but we will - // wait until all the pods are ready before changing the cluster state back to Running. - // If we are no longer waiting on or deleting pods, and all the revisions are in sync, then we know the upgrade or - // restart is complete and we can set the cluster state back to HumioClusterStateRunning. - // It's possible we entered a ConfigError state during an upgrade or restart, and in this case, we should reset the - // state to Running if then the pods are healthy but we're in a ConfigError state. - if !podsStatus.waitingOnPods() && !desiredLifecycleState.WantsUpgrade() && !desiredLifecycleState.WantsRestart() && podsStatus.podRevisionsInSync() { - if hc.Status.State == humiov1alpha1.HumioClusterStateRestarting || hc.Status.State == humiov1alpha1.HumioClusterStateUpgrading || hc.Status.State == humiov1alpha1.HumioClusterStateConfigError { - r.Log.Info(fmt.Sprintf("no longer deleting pods. changing cluster state from %s to %s", hc.Status.State, humiov1alpha1.HumioClusterStateRunning)) - if result, err := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). - withNodePoolState(humiov1alpha1.HumioClusterStateRunning, hnp.GetNodePoolName(), hnp.GetDesiredPodRevision(), "")); err != nil { - return result, err - } + // OnDelete update strategy is enabled, so user must manually delete the pods + if desiredLifecycleState.FoundVersionDifference() || desiredLifecycleState.FoundConfigurationDifference() { + r.Log.Info(fmt.Sprintf("pods %v should be deleted because cluster restart/upgrade, but refusing due to the configured upgrade strategy", + desiredLifecycleState.namesOfPodsToBeReplaced())) } } - r.Log.Info(fmt.Sprintf("cluster state is still %s. waitingOnPods=%v, podBeingDeleted=%v, "+ - "revisionsInSync=%v, podRevisions=%v, podDeletionTimestampSet=%v, podNames=%v, podHumioVersions=%v, expectedRunningPods=%v, podsReady=%v, podsNotReady=%v", - hc.Status.State, podsStatus.waitingOnPods(), desiredLifecycleState.ShouldDeletePod(), podsStatus.podRevisionsInSync(), - podsStatus.podRevisions, podsStatus.podDeletionTimestampSet, podsStatus.podNames, podsStatus.podImageVersions, podsStatus.expectedRunningPods, podsStatus.readyCount, podsStatus.notReadyCount)) - - // If we have pods being deleted, requeue as long as we're not doing a rolling update. This will ensure all pods - // are removed before creating the replacement pods. - if hc.Status.State == humiov1alpha1.HumioClusterStateUpgrading && desiredLifecycleState.ShouldDeletePod() && !desiredLifecycleState.ShouldRollingRestart() { + // requeue if we're upgrading all pods as once and we still detect a difference, so there's still pods left + if hc.Status.State == humiov1alpha1.HumioClusterStateUpgrading && desiredLifecycleState.ADifferenceWasDetectedAndManualDeletionsNotEnabled() && !desiredLifecycleState.ShouldRollingRestart() { + r.Log.Info("requeuing after 1 sec as we are upgrading cluster, have more pods to delete and we are not doing rolling restart") return reconcile.Result{RequeueAfter: time.Second + 1}, nil } - // return empty result and no error indicating that everything was in the state we wanted it to be + // return empty result, which allows reconciliation to continue and create the new pods + r.Log.Info("nothing to do") return reconcile.Result{}, nil } @@ -2137,7 +2073,6 @@ func (r *HumioClusterReconciler) ensurePersistentVolumeClaimsExist(ctx context.C if len(foundPersistentVolumeClaims) < hnp.GetNodeCount() { r.Log.Info(fmt.Sprintf("pvc count of %d is less than %d. adding more", len(foundPersistentVolumeClaims), hnp.GetNodeCount())) pvc := constructPersistentVolumeClaim(hnp) - pvc.Annotations[pvcHashAnnotation] = helpers.AsSHA256(pvc.Spec) if err := controllerutil.SetControllerReference(hc, pvc, r.Scheme()); err != nil { return r.logErrorAndReturn(err, "could not set controller reference") } @@ -2229,6 +2164,142 @@ func (r *HumioClusterReconciler) getLicenseString(ctx context.Context, hc *humio return string(licenseSecret.Data[licenseSecretKeySelector.Key]), nil } +func (r *HumioClusterReconciler) verifyHumioClusterConfigurationIsValid(ctx context.Context, hc *humiov1alpha1.HumioCluster, humioNodePools HumioNodePoolList) (reconcile.Result, error) { + for _, pool := range humioNodePools.Filter(NodePoolFilterHasNode) { + if err := r.setImageFromSource(ctx, pool); err != nil { + r.Log.Info(fmt.Sprintf("failed to setImageFromSource, so setting ConfigError err=%v", err)) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error()). + withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetDesiredPodHash(), pool.GetDesiredBootstrapTokenHash(), pool.GetZoneUnderMaintenance())) + } + if err := r.ensureValidHumioVersion(pool); err != nil { + r.Log.Info(fmt.Sprintf("ensureValidHumioVersion failed, so setting ConfigError err=%v", err)) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error()). + withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetDesiredPodHash(), pool.GetDesiredBootstrapTokenHash(), pool.GetZoneUnderMaintenance())) + } + if err := r.ensureValidStorageConfiguration(pool); err != nil { + r.Log.Info(fmt.Sprintf("ensureValidStorageConfiguration failed, so setting ConfigError err=%v", err)) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error()). + withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetDesiredPodHash(), pool.GetDesiredBootstrapTokenHash(), pool.GetZoneUnderMaintenance())) + } + } + + for _, fun := range []ctxHumioClusterFunc{ + r.ensureLicenseIsValid, + r.ensureValidCASecret, + r.ensureHeadlessServiceExists, + r.ensureInternalServiceExists, + r.validateUserDefinedServiceAccountsExists, + } { + if err := fun(ctx, hc); err != nil { + r.Log.Info(fmt.Sprintf("someFunc failed, so setting ConfigError err=%v", err)) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error()). + withState(humiov1alpha1.HumioClusterStateConfigError)) + } + } + + if len(humioNodePools.Filter(NodePoolFilterHasNode)) > 0 { + if err := r.ensureNodePoolSpecificResourcesHaveLabelWithNodePoolName(ctx, humioNodePools.Filter(NodePoolFilterHasNode)[0]); err != nil { + r.Log.Info(fmt.Sprintf("ensureNodePoolSpecificResourcesHaveLabelWithNodePoolName failed, so setting ConfigError err=%v", err)) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error()). + withState(humiov1alpha1.HumioClusterStateConfigError)) + } + } + + if err := r.validateNodeCount(hc, humioNodePools.Items); err != nil { + r.Log.Info(fmt.Sprintf("validateNodeCount failed, so setting ConfigError err=%v", err)) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error()). + withState(humiov1alpha1.HumioClusterStateConfigError)) + } + + for _, pool := range humioNodePools.Items { + if err := r.validateInitialPodSpec(pool); err != nil { + r.Log.Info(fmt.Sprintf("validateInitialPodSpec failed, so setting ConfigError err=%v", err)) + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error()). + withNodePoolState(humiov1alpha1.HumioClusterStateConfigError, pool.GetNodePoolName(), pool.GetDesiredPodRevision(), pool.GetDesiredPodHash(), pool.GetDesiredBootstrapTokenHash(), pool.GetZoneUnderMaintenance())) + } + } + return reconcile.Result{}, nil +} + +func (r *HumioClusterReconciler) cleanupUnusedResources(ctx context.Context, hc *humiov1alpha1.HumioCluster, humioNodePools HumioNodePoolList) (reconcile.Result, error) { + for _, pool := range humioNodePools.Items { + if err := r.ensureOrphanedPvcsAreDeleted(ctx, hc, pool); err != nil { + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error())) + } + } + + for _, nodePool := range humioNodePools.Filter(NodePoolFilterDoesNotHaveNodes) { + if err := r.cleanupUnusedService(ctx, nodePool); err != nil { + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error())) + } + } + + for _, fun := range []ctxHumioClusterFunc{ + r.cleanupUnusedTLSCertificates, + r.cleanupUnusedTLSSecrets, + r.cleanupUnusedCAIssuer, + } { + if err := fun(ctx, hc); err != nil { + return r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(err.Error())) + } + } + + if ok, idx := r.hasNoUnusedNodePoolStatus(hc, &humioNodePools); !ok { + r.cleanupUnusedNodePoolStatus(hc, idx) + if result, err := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withNodePoolStatusList(hc.Status.NodePoolStatus)); err != nil { + return result, r.logErrorAndReturn(err, "unable to set cluster state") + } + } + return reconcile.Result{}, nil +} + +func (r *HumioClusterReconciler) constructPodAttachments(ctx context.Context, hc *humiov1alpha1.HumioCluster, hnp *HumioNodePool) (*podAttachments, reconcile.Result, error) { + attachments := &podAttachments{} + + if hnp.DataVolumePersistentVolumeClaimSpecTemplateIsSetByUser() { + attachments.dataVolumeSource = hnp.GetDataVolumePersistentVolumeClaimSpecTemplate("") + } + + envVarSourceData, err := r.getEnvVarSource(ctx, hnp) + if err != nil { + result, _ := r.updateStatus(ctx, r.Client.Status(), hc, statusOptions(). + withMessage(r.logErrorAndReturn(err, "got error when getting pod envVarSource").Error()). + withState(humiov1alpha1.HumioClusterStateConfigError)) + return nil, result, err + } + if envVarSourceData != nil { + attachments.envVarSourceData = envVarSourceData + } + + humioBootstrapTokens, err := kubernetes.ListHumioBootstrapTokens(ctx, r.Client, hc.GetNamespace(), kubernetes.LabelsForHumioBootstrapToken(hc.GetName())) + if err != nil { + return nil, reconcile.Result{}, r.logErrorAndReturn(err, "failed to get bootstrap token") + } + if len(humioBootstrapTokens) > 0 { + if humioBootstrapTokens[0].Status.State == humiov1alpha1.HumioBootstrapTokenStateReady { + attachments.bootstrapTokenSecretReference.secretReference = humioBootstrapTokens[0].Status.HashedTokenSecretKeyRef.SecretKeyRef + bootstrapTokenHash, err := r.getDesiredBootstrapTokenHash(ctx, hc) + if err != nil { + return nil, reconcile.Result{}, r.logErrorAndReturn(err, "unable to find bootstrap token secret") + } + attachments.bootstrapTokenSecretReference.hash = bootstrapTokenHash + } + } + + return attachments, reconcile.Result{}, nil +} + func (r *HumioClusterReconciler) logErrorAndReturn(err error, msg string) error { r.Log.Error(err, msg) return fmt.Errorf("%s: %w", msg, err) @@ -2256,3 +2327,12 @@ func mergeEnvVars(from, into []corev1.EnvVar) []corev1.EnvVar { } return into } + +func getHumioNodePoolManagers(hc *humiov1alpha1.HumioCluster) HumioNodePoolList { + var humioNodePools HumioNodePoolList + humioNodePools.Add(NewHumioNodeManagerFromHumioCluster(hc)) + for idx := range hc.Spec.NodePools { + humioNodePools.Add(NewHumioNodeManagerFromHumioNodePool(hc, &hc.Spec.NodePools[idx])) + } + return humioNodePools +} diff --git a/controllers/humiocluster_defaults.go b/controllers/humiocluster_defaults.go index ef5658f35..df5bafc61 100644 --- a/controllers/humiocluster_defaults.go +++ b/controllers/humiocluster_defaults.go @@ -63,37 +63,43 @@ const ( ) type HumioNodePool struct { - clusterName string - nodePoolName string - namespace string - hostname string - esHostname string - hostnameSource humiov1alpha1.HumioHostnameSource - esHostnameSource humiov1alpha1.HumioESHostnameSource - humioNodeSpec humiov1alpha1.HumioNodeSpec - tls *humiov1alpha1.HumioClusterTLSSpec - idpCertificateSecretName string - viewGroupPermissions string // Deprecated: Replaced by rolePermissions - rolePermissions string - targetReplicationFactor int - digestPartitionsCount int - path string - ingress humiov1alpha1.HumioClusterIngressSpec - clusterAnnotations map[string]string - desiredPodRevision int - state string - zoneUnderMaintenance string + clusterName string + nodePoolName string + namespace string + hostname string + esHostname string + hostnameSource humiov1alpha1.HumioHostnameSource + esHostnameSource humiov1alpha1.HumioESHostnameSource + humioNodeSpec humiov1alpha1.HumioNodeSpec + tls *humiov1alpha1.HumioClusterTLSSpec + idpCertificateSecretName string + viewGroupPermissions string // Deprecated: Replaced by rolePermissions + rolePermissions string + targetReplicationFactor int + digestPartitionsCount int + path string + ingress humiov1alpha1.HumioClusterIngressSpec + clusterAnnotations map[string]string + state string + zoneUnderMaintenance string + desiredPodRevision int + desiredPodHash string + desiredBootstrapTokenHash string } func NewHumioNodeManagerFromHumioCluster(hc *humiov1alpha1.HumioCluster) *HumioNodePool { - desiredPodRevision := 0 - zoneUnderMaintenance := "" state := "" + zoneUnderMaintenance := "" + desiredPodRevision := 0 + desiredPodHash := "" + desiredBootstrapTokenHash := "" for _, status := range hc.Status.NodePoolStatus { if status.Name == hc.Name { - desiredPodRevision = status.DesiredPodRevision - zoneUnderMaintenance = status.ZoneUnderMaintenance state = status.State + zoneUnderMaintenance = status.ZoneUnderMaintenance + desiredPodRevision = status.DesiredPodRevision + desiredPodHash = status.DesiredPodHash + desiredBootstrapTokenHash = status.DesiredBootstrapTokenHash break } } @@ -146,31 +152,36 @@ func NewHumioNodeManagerFromHumioCluster(hc *humiov1alpha1.HumioCluster) *HumioN UpdateStrategy: hc.Spec.UpdateStrategy, PriorityClassName: hc.Spec.PriorityClassName, }, - tls: hc.Spec.TLS, - idpCertificateSecretName: hc.Spec.IdpCertificateSecretName, - viewGroupPermissions: hc.Spec.ViewGroupPermissions, - rolePermissions: hc.Spec.RolePermissions, - targetReplicationFactor: hc.Spec.TargetReplicationFactor, - digestPartitionsCount: hc.Spec.DigestPartitionsCount, - path: hc.Spec.Path, - ingress: hc.Spec.Ingress, - clusterAnnotations: hc.Annotations, - desiredPodRevision: desiredPodRevision, - zoneUnderMaintenance: zoneUnderMaintenance, - state: state, + tls: hc.Spec.TLS, + idpCertificateSecretName: hc.Spec.IdpCertificateSecretName, + viewGroupPermissions: hc.Spec.ViewGroupPermissions, + rolePermissions: hc.Spec.RolePermissions, + targetReplicationFactor: hc.Spec.TargetReplicationFactor, + digestPartitionsCount: hc.Spec.DigestPartitionsCount, + path: hc.Spec.Path, + ingress: hc.Spec.Ingress, + clusterAnnotations: hc.Annotations, + state: state, + zoneUnderMaintenance: zoneUnderMaintenance, + desiredPodRevision: desiredPodRevision, + desiredPodHash: desiredPodHash, + desiredBootstrapTokenHash: desiredBootstrapTokenHash, } } func NewHumioNodeManagerFromHumioNodePool(hc *humiov1alpha1.HumioCluster, hnp *humiov1alpha1.HumioNodePoolSpec) *HumioNodePool { - desiredPodRevision := 0 - zoneUnderMaintenance := "" state := "" - + zoneUnderMaintenance := "" + desiredPodRevision := 0 + desiredPodHash := "" + desiredBootstrapTokenHash := "" for _, status := range hc.Status.NodePoolStatus { if status.Name == strings.Join([]string{hc.Name, hnp.Name}, "-") { - desiredPodRevision = status.DesiredPodRevision - zoneUnderMaintenance = status.ZoneUnderMaintenance state = status.State + zoneUnderMaintenance = status.ZoneUnderMaintenance + desiredPodRevision = status.DesiredPodRevision + desiredPodHash = status.DesiredPodHash + desiredBootstrapTokenHash = status.DesiredBootstrapTokenHash break } } @@ -223,18 +234,20 @@ func NewHumioNodeManagerFromHumioNodePool(hc *humiov1alpha1.HumioCluster, hnp *h UpdateStrategy: hnp.UpdateStrategy, PriorityClassName: hnp.PriorityClassName, }, - tls: hc.Spec.TLS, - idpCertificateSecretName: hc.Spec.IdpCertificateSecretName, - viewGroupPermissions: hc.Spec.ViewGroupPermissions, - rolePermissions: hc.Spec.RolePermissions, - targetReplicationFactor: hc.Spec.TargetReplicationFactor, - digestPartitionsCount: hc.Spec.DigestPartitionsCount, - path: hc.Spec.Path, - ingress: hc.Spec.Ingress, - clusterAnnotations: hc.Annotations, - desiredPodRevision: desiredPodRevision, - zoneUnderMaintenance: zoneUnderMaintenance, - state: state, + tls: hc.Spec.TLS, + idpCertificateSecretName: hc.Spec.IdpCertificateSecretName, + viewGroupPermissions: hc.Spec.ViewGroupPermissions, + rolePermissions: hc.Spec.RolePermissions, + targetReplicationFactor: hc.Spec.TargetReplicationFactor, + digestPartitionsCount: hc.Spec.DigestPartitionsCount, + path: hc.Spec.Path, + ingress: hc.Spec.Ingress, + clusterAnnotations: hc.Annotations, + state: state, + zoneUnderMaintenance: zoneUnderMaintenance, + desiredPodRevision: desiredPodRevision, + desiredPodHash: desiredPodHash, + desiredBootstrapTokenHash: desiredBootstrapTokenHash, } } @@ -316,12 +329,17 @@ func (hnp *HumioNodePool) GetDigestPartitionsCount() int { } func (hnp *HumioNodePool) GetDesiredPodRevision() int { - if hnp.desiredPodRevision == 0 { - return 1 - } return hnp.desiredPodRevision } +func (hnp *HumioNodePool) GetDesiredPodHash() string { + return hnp.desiredPodHash +} + +func (hnp *HumioNodePool) GetDesiredBootstrapTokenHash() string { + return hnp.desiredBootstrapTokenHash +} + func (hnp *HumioNodePool) GetZoneUnderMaintenance() string { return hnp.zoneUnderMaintenance } @@ -840,12 +858,17 @@ func (hnp *HumioNodePool) GetProbeScheme() corev1.URIScheme { func (hnp *HumioNodePool) GetUpdateStrategy() *humiov1alpha1.HumioUpdateStrategy { defaultZoneAwareness := true + defaultMaxUnavailable := intstr.FromInt32(1) if hnp.humioNodeSpec.UpdateStrategy != nil { if hnp.humioNodeSpec.UpdateStrategy.EnableZoneAwareness == nil { hnp.humioNodeSpec.UpdateStrategy.EnableZoneAwareness = &defaultZoneAwareness } + if hnp.humioNodeSpec.UpdateStrategy.MaxUnavailable == nil { + hnp.humioNodeSpec.UpdateStrategy.MaxUnavailable = &defaultMaxUnavailable + } + return hnp.humioNodeSpec.UpdateStrategy } @@ -853,6 +876,7 @@ func (hnp *HumioNodePool) GetUpdateStrategy() *humiov1alpha1.HumioUpdateStrategy Type: humiov1alpha1.HumioClusterUpdateStrategyReplaceAllOnUpdate, MinReadySeconds: 0, EnableZoneAwareness: &defaultZoneAwareness, + MaxUnavailable: &defaultMaxUnavailable, } } diff --git a/controllers/humiocluster_pod_lifecycle.go b/controllers/humiocluster_pod_lifecycle.go index d73a783b0..989d7ed59 100644 --- a/controllers/humiocluster_pod_lifecycle.go +++ b/controllers/humiocluster_pod_lifecycle.go @@ -1,17 +1,26 @@ package controllers import ( - "time" - humiov1alpha1 "github.com/humio/humio-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// podLifecycleState is used to hold information on what the next action should be based on what configuration +// changes are detected. It holds information that is specific to a single HumioNodePool in nodePool and the pod field +// holds information about what pod should be deleted next. type podLifecycleState struct { - nodePool HumioNodePool - pod corev1.Pod - versionDifference *podLifecycleStateVersionDifference + // nodePool holds the HumioNodePool that is used to access the details and resources related to the node pool + nodePool HumioNodePool + // podsToBeReplaced holds the details of existing pods that is the next targets for pod deletion due to some + // difference between current state vs desired state. + podsToBeReplaced []corev1.Pod + // versionDifference holds information on what version we are upgrading from/to. + // This will be nil when no image version difference has been detected. + versionDifference *podLifecycleStateVersionDifference + // configurationDifference holds information indicating that we have detected a configuration difference. + // If the configuration difference requires all pods within the node pool to be replaced at the same time, + // requiresSimultaneousRestart will be set in podLifecycleStateConfigurationDifference. + // This will be nil when no configuration difference has been detected. configurationDifference *podLifecycleStateConfigurationDifference } @@ -24,10 +33,9 @@ type podLifecycleStateConfigurationDifference struct { requiresSimultaneousRestart bool } -func NewPodLifecycleState(hnp HumioNodePool, pod corev1.Pod) *podLifecycleState { +func NewPodLifecycleState(hnp HumioNodePool) *podLifecycleState { return &podLifecycleState{ nodePool: hnp, - pod: pod, } } @@ -38,7 +46,7 @@ func (p *podLifecycleState) ShouldRollingRestart() bool { if p.nodePool.GetUpdateStrategy().Type == humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate { return true } - if p.WantsUpgrade() { + if p.FoundVersionDifference() { // if we're trying to go to or from a "latest" image, we can't do any version comparison if p.versionDifference.from.IsLatest() || p.versionDifference.to.IsLatest() { return false @@ -60,68 +68,25 @@ func (p *podLifecycleState) ShouldRollingRestart() bool { return false } -func (p *podLifecycleState) RemainingMinReadyWaitTime(pods []corev1.Pod) time.Duration { - // We will only try to wait if we are performing a rolling restart and have MinReadySeconds set above 0. - // Additionally, if we do a rolling restart and MinReadySeconds is unset, then we also do not want to wait. - if !p.ShouldRollingRestart() || p.nodePool.GetUpdateStrategy().MinReadySeconds <= 0 { - return -1 - } - var minReadySeconds = p.nodePool.GetUpdateStrategy().MinReadySeconds - var conditions []corev1.PodCondition - for _, pod := range pods { - if pod.Name == p.pod.Name { - continue - } - for _, condition := range pod.Status.Conditions { - if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { - conditions = append(conditions, condition) - } - } - } - - // We take the condition with the latest transition time among type PodReady conditions with Status true for ready pods. - // Then we look at the condition with the latest transition time that is not for the pod that is a deletion candidate. - // We then take the difference between the latest transition time and now and compare this to the MinReadySeconds setting. - // This also means that if you quickly perform another rolling restart after another finished, - // then you may initially wait for the minReadySeconds timer on the first pod. - var latestTransitionTime = latestTransitionTime(conditions) - if !latestTransitionTime.Time.IsZero() { - var diff = time.Since(latestTransitionTime.Time).Milliseconds() - var minRdy = (time.Second * time.Duration(minReadySeconds)).Milliseconds() - if diff <= minRdy { - return time.Second * time.Duration((minRdy-diff)/1000) - } - } - return -1 -} - -func (p *podLifecycleState) ShouldDeletePod() bool { +func (p *podLifecycleState) ADifferenceWasDetectedAndManualDeletionsNotEnabled() bool { if p.nodePool.GetUpdateStrategy().Type == humiov1alpha1.HumioClusterUpdateStrategyOnDelete { return false } - return p.WantsUpgrade() || p.WantsRestart() + return p.FoundVersionDifference() || p.FoundConfigurationDifference() } -func (p *podLifecycleState) WantsUpgrade() bool { +func (p *podLifecycleState) FoundVersionDifference() bool { return p.versionDifference != nil } -func (p *podLifecycleState) WantsRestart() bool { +func (p *podLifecycleState) FoundConfigurationDifference() bool { return p.configurationDifference != nil } -func latestTransitionTime(conditions []corev1.PodCondition) metav1.Time { - if len(conditions) == 0 { - return metav1.NewTime(time.Time{}) - } - var max = conditions[0].LastTransitionTime - for idx, condition := range conditions { - if condition.LastTransitionTime.Time.IsZero() { - continue - } - if idx == 0 || condition.LastTransitionTime.Time.After(max.Time) { - max = condition.LastTransitionTime - } +func (p *podLifecycleState) namesOfPodsToBeReplaced() []string { + podNames := []string{} + for _, pod := range p.podsToBeReplaced { + podNames = append(podNames, pod.Name) } - return max + return podNames } diff --git a/controllers/humiocluster_pod_status.go b/controllers/humiocluster_pod_status.go index d44d00d8e..b441534fd 100644 --- a/controllers/humiocluster_pod_status.go +++ b/controllers/humiocluster_pod_status.go @@ -5,10 +5,12 @@ import ( "fmt" "sort" "strconv" + "time" humiov1alpha1 "github.com/humio/humio-operator/api/v1alpha1" - "github.com/humio/humio-operator/pkg/kubernetes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" corev1 "k8s.io/api/core/v1" ) @@ -22,27 +24,83 @@ const ( ) type podsStatusState struct { - expectedRunningPods int - readyCount int - notReadyCount int - podRevisions []int - podImageVersions []string + // nodeCount holds the final number of expected pods set by the user (NodeCount). + nodeCount int + // readyCount holds the number of pods the pod condition PodReady is true. + // This value gets initialized to 0 and incremented per pod where PodReady condition is true. + readyCount int + // notReadyCount holds the number of pods found we have not deemed ready. + // This value gets initialized to the number of pods found. + // For each pod found that has PodReady set to ConditionTrue, we decrement this value. + notReadyCount int + // notReadyDueToMinReadySeconds holds the number of pods that are ready, but have not been running for long enough + notReadyDueToMinReadySeconds int + // podRevisions is populated with the value of the pod annotation PodRevisionAnnotation. + // The slice is sorted by pod name. + podRevisions []int + // podImageVersions holds the container image of the "humio" containers. + // The slice is sorted by pod name. + podImageVersions []string + // podDeletionTimestampSet holds a boolean indicating if the pod is marked for deletion by checking if pod DeletionTimestamp is nil. + // The slice is sorted by pod name. podDeletionTimestampSet []bool - podNames []string - podErrors []corev1.Pod - podsRequiringDeletion []corev1.Pod - podsReady []corev1.Pod + // podNames holds the pod name of the pods. + // The slice is sorted by pod name. + podNames []string + // podAreUnschedulableOrHaveBadStatusConditions holds a list of pods that was detected as having errors, which is determined by the pod conditions. + // + // If pod conditions says it is unschedulable, it is added to podAreUnschedulableOrHaveBadStatusConditions. + // + // If pod condition ready is found with a value that is not ConditionTrue, we look at the pod ContainerStatuses. + // When ContainerStatuses indicates the container is in Waiting status, we add it to podAreUnschedulableOrHaveBadStatusConditions if the reason + // is not containerStateCreating nor podInitializing. + // When ContainerStatuses indicates the container is in Terminated status, we add it to podAreUnschedulableOrHaveBadStatusConditions if the reason + // is not containerStateCompleted. + // + // The slice is sorted by pod name. + podAreUnschedulableOrHaveBadStatusConditions []corev1.Pod + // podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists holds a list of pods that needs to be cleaned up due to being evicted, or if the pod is + // stuck in phase Pending due to the use of a PVC that refers to a Kubernetes worker node that no longer exists. + // The slice is sorted by pod name. + podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists []corev1.Pod + // podsReady holds the list of pods where pod condition PodReady is true + // The slice is sorted by pod name. + podsReady []corev1.Pod + // scaledMaxUnavailable holds the maximum number of pods we allow to be unavailable at the same time. + // When user defines a percentage, the value is rounded up to ensure scaledMaxUnavailable >= 1 as we cannot target + // replacing no pods. + // If the goal is to manually replace pods, the cluster update strategy should instead be set to + // HumioClusterUpdateStrategyOnDelete. + scaledMaxUnavailable int + // minReadySeconds holds the number of seconds a pod must be in ready state for it to be treated as ready + minReadySeconds int32 } func (r *HumioClusterReconciler) getPodsStatus(ctx context.Context, hc *humiov1alpha1.HumioCluster, hnp *HumioNodePool, foundPodList []corev1.Pod) (*podsStatusState, error) { status := podsStatusState{ - readyCount: 0, - notReadyCount: len(foundPodList), - expectedRunningPods: hnp.GetNodeCount(), + // initially, we assume no pods are ready + readyCount: 0, + // initially, we assume all pods found are not ready + notReadyCount: len(foundPodList), + // the number of pods we expect to have running is the nodeCount value set by the user + nodeCount: hnp.GetNodeCount(), + // the number of seconds a pod must be in ready state to be treated as ready + minReadySeconds: hnp.GetUpdateStrategy().MinReadySeconds, } sort.Slice(foundPodList, func(i, j int) bool { return foundPodList[i].Name < foundPodList[j].Name }) + + updateStrategy := hnp.GetUpdateStrategy() + scaledMaxUnavailable, err := intstr.GetScaledValueFromIntOrPercent(updateStrategy.MaxUnavailable, hnp.GetNodeCount(), false) + if err != nil { + return &status, fmt.Errorf("unable to fetch rounded up scaled value for maxUnavailable based on %s with total of %d", updateStrategy.MaxUnavailable.String(), hnp.GetNodeCount()) + } + + // We ensure to always replace at least 1 pod, just in case the user specified maxUnavailable 0 or 0%, or + // scaledMaxUnavailable becomes 0 as it is rounded down + status.scaledMaxUnavailable = max(scaledMaxUnavailable, 1) + var podsReady, podsNotReady []string for _, pod := range foundPodList { podRevisionStr := pod.Annotations[PodRevisionAnnotation] @@ -60,10 +118,10 @@ func (r *HumioClusterReconciler) getPodsStatus(ctx context.Context, hc *humiov1a if pod.DeletionTimestamp == nil { // If a pod is evicted, we don't want to wait for a new pod spec since the eviction could happen for a // number of reasons. If we delete the pod then we will re-create it on the next reconcile. Adding the pod - // to the podsRequiringDeletion list will cause it to be deleted. + // to the podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists list will cause it to be deleted. if pod.Status.Phase == corev1.PodFailed && pod.Status.Reason == podConditionReasonEvicted { r.Log.Info(fmt.Sprintf("pod %s has errors, pod phase: %s, reason: %s", pod.Name, pod.Status.Phase, pod.Status.Reason)) - status.podsRequiringDeletion = append(status.podsRequiringDeletion, pod) + status.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists = append(status.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists, pod) continue } if pod.Status.Phase == corev1.PodPending { @@ -72,7 +130,7 @@ func (r *HumioClusterReconciler) getPodsStatus(ctx context.Context, hc *humiov1a return &status, r.logErrorAndReturn(err, "unable to determine whether pod should be deleted") } if deletePod && hnp.OkToDeletePvc() { - status.podsRequiringDeletion = append(status.podsRequiringDeletion, pod) + status.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists = append(status.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists, pod) } } // If a pod is Pending but unschedulable, we want to consider this an error state so it will be replaced @@ -81,26 +139,31 @@ func (r *HumioClusterReconciler) getPodsStatus(ctx context.Context, hc *humiov1a if condition.Status == corev1.ConditionFalse { if condition.Reason == PodConditionReasonUnschedulable { r.Log.Info(fmt.Sprintf("pod %s has errors, container status: %s, reason: %s", pod.Name, condition.Status, condition.Reason)) - status.podErrors = append(status.podErrors, pod) + status.podAreUnschedulableOrHaveBadStatusConditions = append(status.podAreUnschedulableOrHaveBadStatusConditions, pod) continue } } if condition.Type == corev1.PodReady { - if condition.Status == corev1.ConditionTrue { + remainingMinReadyWaitTime := status.remainingMinReadyWaitTime(pod) + if condition.Status == corev1.ConditionTrue && remainingMinReadyWaitTime <= 0 { status.podsReady = append(status.podsReady, pod) podsReady = append(podsReady, pod.Name) status.readyCount++ status.notReadyCount-- } else { podsNotReady = append(podsNotReady, pod.Name) + if remainingMinReadyWaitTime > 0 { + r.Log.Info(fmt.Sprintf("pod %s has not been ready for enough time yet according to minReadySeconds, remainingMinReadyWaitTimeSeconds=%f", pod.Name, remainingMinReadyWaitTime.Seconds())) + status.notReadyDueToMinReadySeconds++ + } for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.State.Waiting != nil && containerStatus.State.Waiting.Reason != containerStateCreating && containerStatus.State.Waiting.Reason != podInitializing { r.Log.Info(fmt.Sprintf("pod %s has errors, container state: Waiting, reason: %s", pod.Name, containerStatus.State.Waiting.Reason)) - status.podErrors = append(status.podErrors, pod) + status.podAreUnschedulableOrHaveBadStatusConditions = append(status.podAreUnschedulableOrHaveBadStatusConditions, pod) } if containerStatus.State.Terminated != nil && containerStatus.State.Terminated.Reason != containerStateCompleted { r.Log.Info(fmt.Sprintf("pod %s has errors, container state: Terminated, reason: %s", pod.Name, containerStatus.State.Terminated.Reason)) - status.podErrors = append(status.podErrors, pod) + status.podAreUnschedulableOrHaveBadStatusConditions = append(status.podAreUnschedulableOrHaveBadStatusConditions, pod) } } } @@ -108,7 +171,7 @@ func (r *HumioClusterReconciler) getPodsStatus(ctx context.Context, hc *humiov1a } } } - r.Log.Info(fmt.Sprintf("pod status nodePoolName=%s readyCount=%d notReadyCount=%d podsReady=%s podsNotReady=%s", hnp.GetNodePoolName(), status.readyCount, status.notReadyCount, podsReady, podsNotReady)) + r.Log.Info(fmt.Sprintf("pod status nodePoolName=%s readyCount=%d notReadyCount=%d podsReady=%s podsNotReady=%s maxUnavailable=%s scaledMaxUnavailable=%d minReadySeconds=%d", hnp.GetNodePoolName(), status.readyCount, status.notReadyCount, podsReady, podsNotReady, updateStrategy.MaxUnavailable.String(), scaledMaxUnavailable, status.minReadySeconds)) // collect ready pods and not ready pods in separate lists and just print the lists here instead of a log entry per host return &status, nil } @@ -116,29 +179,92 @@ func (r *HumioClusterReconciler) getPodsStatus(ctx context.Context, hc *humiov1a // waitingOnPods returns true when there are pods running that are not in a ready state. This does not include pods // that are not ready due to container errors. func (s *podsStatusState) waitingOnPods() bool { - return (s.readyCount < s.expectedRunningPods || s.notReadyCount > 0) && !s.havePodsWithErrors() && !s.havePodsRequiringDeletion() + lessPodsReadyThanNodeCount := s.readyCount < s.nodeCount + somePodIsNotReady := s.notReadyCount > 0 + return (lessPodsReadyThanNodeCount || somePodIsNotReady) && + !s.haveUnschedulablePodsOrPodsWithBadStatusConditions() && + !s.foundEvictedPodsOrPodsWithOrpahanedPVCs() } -func (s *podsStatusState) podRevisionsInSync() bool { - if len(s.podRevisions) < s.expectedRunningPods { +// scaledMaxUnavailableMinusNotReadyDueToMinReadySeconds returns an absolute number of pods we can delete. +func (s *podsStatusState) scaledMaxUnavailableMinusNotReadyDueToMinReadySeconds() int { + deletionBudget := s.scaledMaxUnavailable - s.notReadyDueToMinReadySeconds + return max(deletionBudget, 0) +} + +// podRevisionCountMatchesNodeCountAndAllPodsHaveTheSameRevision returns true if we have the correct number of pods +// and all the pods have the same revision +func (s *podsStatusState) podRevisionCountMatchesNodeCountAndAllPodsHaveTheSameRevision() bool { + // we don't have the correct number of revisions to check, so we have less pods than expected + if len(s.podRevisions) != s.nodeCount { return false } - if s.expectedRunningPods == 1 { + // if nodeCount says 1 and that we know we have one revision, it will always be in sync + if s.nodeCount == 1 { return true } + // if we have no revisions, return bool indicating if nodeCount matches that + if len(s.podRevisions) == 0 { + return s.nodeCount == 0 + } + // fetch the first revision and compare the rest with that revision := s.podRevisions[0] for i := 1; i < len(s.podRevisions); i++ { if s.podRevisions[i] != revision { return false } } + // no mismatching revisions found return true } -func (s *podsStatusState) havePodsWithErrors() bool { - return len(s.podErrors) > 0 +func (s *podsStatusState) haveUnschedulablePodsOrPodsWithBadStatusConditions() bool { + return len(s.podAreUnschedulableOrHaveBadStatusConditions) > 0 } -func (s *podsStatusState) havePodsRequiringDeletion() bool { - return len(s.podsRequiringDeletion) > 0 +func (s *podsStatusState) foundEvictedPodsOrPodsWithOrpahanedPVCs() bool { + return len(s.podsEvictedOrUsesPVCAttachedToHostThatNoLongerExists) > 0 +} + +func (s *podsStatusState) remainingMinReadyWaitTime(pod corev1.Pod) time.Duration { + var minReadySeconds = s.minReadySeconds + var conditions []corev1.PodCondition + + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + conditions = append(conditions, condition) + } + } + + // We take the condition with the latest transition time among type PodReady conditions with Status true for ready pods. + // Then we look at the condition with the latest transition time that is not for the pod that is a deletion candidate. + // We then take the difference between the latest transition time and now and compare this to the MinReadySeconds setting. + // This also means that if you quickly perform another rolling restart after another finished, + // then you may initially wait for the minReadySeconds timer on the first pod. + var latestTransitionTime = s.latestTransitionTime(conditions) + if !latestTransitionTime.Time.IsZero() { + var diff = time.Since(latestTransitionTime.Time).Milliseconds() + var minRdy = (time.Second * time.Duration(minReadySeconds)).Milliseconds() + if diff <= minRdy { + remainingWaitTime := time.Second * time.Duration((minRdy-diff)/1000) + return min(remainingWaitTime, MaximumMinReadyRequeue) + } + } + return -1 +} + +func (s *podsStatusState) latestTransitionTime(conditions []corev1.PodCondition) metav1.Time { + if len(conditions) == 0 { + return metav1.NewTime(time.Time{}) + } + var mostRecentTransitionTime = conditions[0].LastTransitionTime + for idx, condition := range conditions { + if condition.LastTransitionTime.Time.IsZero() { + continue + } + if idx == 0 || condition.LastTransitionTime.Time.After(mostRecentTransitionTime.Time) { + mostRecentTransitionTime = condition.LastTransitionTime + } + } + return mostRecentTransitionTime } diff --git a/controllers/humiocluster_pod_status_test.go b/controllers/humiocluster_pod_status_test.go index 85bd7caa4..a68348d32 100644 --- a/controllers/humiocluster_pod_status_test.go +++ b/controllers/humiocluster_pod_status_test.go @@ -10,11 +10,11 @@ import ( func Test_podsStatusState_waitingOnPods(t *testing.T) { type fields struct { - expectedRunningPods int - readyCount int - notReadyCount int - podRevisions []int - podErrors []corev1.Pod + nodeCount int + readyCount int + notReadyCount int + podRevisions []int + podErrors []corev1.Pod } tests := []struct { name string @@ -75,11 +75,11 @@ func Test_podsStatusState_waitingOnPods(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &podsStatusState{ - expectedRunningPods: tt.fields.expectedRunningPods, - readyCount: tt.fields.readyCount, - notReadyCount: tt.fields.notReadyCount, - podRevisions: tt.fields.podRevisions, - podErrors: tt.fields.podErrors, + nodeCount: tt.fields.nodeCount, + readyCount: tt.fields.readyCount, + notReadyCount: tt.fields.notReadyCount, + podRevisions: tt.fields.podRevisions, + podAreUnschedulableOrHaveBadStatusConditions: tt.fields.podErrors, } if got := s.waitingOnPods(); got != tt.want { t.Errorf("waitingOnPods() = %v, want %v", got, tt.want) diff --git a/controllers/humiocluster_pods.go b/controllers/humiocluster_pods.go index 5c292082b..a34058a17 100644 --- a/controllers/humiocluster_pods.go +++ b/controllers/humiocluster_pods.go @@ -455,9 +455,6 @@ func ConstructPod(hnp *HumioNodePool, humioNodeName string, attachments *podAtta }) } - if attachments.bootstrapTokenSecretReference.hash != "" { - pod.Annotations[bootstrapTokenHashAnnotation] = attachments.bootstrapTokenSecretReference.hash - } priorityClassName := hnp.GetPriorityClassName() if priorityClassName != "" { pod.Spec.PriorityClassName = priorityClassName @@ -469,6 +466,9 @@ func ConstructPod(hnp *HumioNodePool, humioNodeName string, attachments *podAtta } pod.Spec.Containers[humioIdx].Args = containerArgs + pod.Annotations[PodRevisionAnnotation] = strconv.Itoa(hnp.GetDesiredPodRevision()) + pod.Annotations[PodHashAnnotation] = podSpecAsSHA256(hnp, pod) + pod.Annotations[BootstrapTokenHashAnnotation] = attachments.bootstrapTokenSecretReference.hash return &pod, nil } @@ -657,29 +657,16 @@ func (r *HumioClusterReconciler) createPod(ctx context.Context, hc *humiov1alpha return &corev1.Pod{}, r.logErrorAndReturn(err, "could not set controller reference") } r.Log.Info(fmt.Sprintf("pod %s will use attachments %+v", pod.Name, attachments)) - pod.Annotations[PodHashAnnotation] = podSpecAsSHA256(hnp, *pod) - - if attachments.envVarSourceData != nil { - b, err := json.Marshal(attachments.envVarSourceData) - if err != nil { - return &corev1.Pod{}, fmt.Errorf("error trying to JSON encode envVarSourceData: %w", err) - } - pod.Annotations[envVarSourceHashAnnotation] = helpers.AsSHA256(string(b)) - } - if hnp.TLSEnabled() { pod.Annotations[certHashAnnotation] = podNameAndCertHash.certificateHash } - podRevision := hnp.GetDesiredPodRevision() - r.setPodRevision(pod, podRevision) - - r.Log.Info(fmt.Sprintf("creating pod %s with revision %d", pod.Name, podRevision)) + r.Log.Info(fmt.Sprintf("creating pod %s with revision %d", pod.Name, hnp.GetDesiredPodRevision())) err = r.Create(ctx, pod) if err != nil { return &corev1.Pod{}, err } - r.Log.Info(fmt.Sprintf("successfully created pod %s with revision %d", pod.Name, podRevision)) + r.Log.Info(fmt.Sprintf("successfully created pod %s with revision %d", pod.Name, hnp.GetDesiredPodRevision())) return pod, nil } @@ -720,87 +707,65 @@ func (r *HumioClusterReconciler) waitForNewPods(ctx context.Context, hnp *HumioN } func (r *HumioClusterReconciler) podsMatch(hnp *HumioNodePool, pod corev1.Pod, desiredPod corev1.Pod) bool { + // if mandatory annotations are not present, we can return early indicating they need to be replaced if _, ok := pod.Annotations[PodHashAnnotation]; !ok { return false } if _, ok := pod.Annotations[PodRevisionAnnotation]; !ok { return false } - - var specMatches bool - var revisionMatches bool - var envVarSourceMatches bool - var certHasAnnotationMatches bool - var bootstrapTokenAnnotationMatches bool - - desiredPodHash := podSpecAsSHA256(hnp, desiredPod) - desiredPodRevision := hnp.GetDesiredPodRevision() - r.setPodRevision(&desiredPod, desiredPodRevision) - if pod.Annotations[PodHashAnnotation] == desiredPodHash { - specMatches = true - } - if pod.Annotations[PodRevisionAnnotation] == desiredPod.Annotations[PodRevisionAnnotation] { - revisionMatches = true - } - if _, ok := pod.Annotations[envVarSourceHashAnnotation]; ok { - if pod.Annotations[envVarSourceHashAnnotation] == desiredPod.Annotations[envVarSourceHashAnnotation] { - envVarSourceMatches = true - } - } else { - // Ignore envVarSource hash if it's not in either the current pod or the desired pod - if _, ok := desiredPod.Annotations[envVarSourceHashAnnotation]; !ok { - envVarSourceMatches = true - } - } - if _, ok := pod.Annotations[certHashAnnotation]; ok { - if pod.Annotations[certHashAnnotation] == desiredPod.Annotations[certHashAnnotation] { - certHasAnnotationMatches = true - } - } else { - // Ignore certHashAnnotation if it's not in either the current pod or the desired pod - if _, ok := desiredPod.Annotations[certHashAnnotation]; !ok { - certHasAnnotationMatches = true - } - } - if _, ok := pod.Annotations[bootstrapTokenHashAnnotation]; ok { - if pod.Annotations[bootstrapTokenHashAnnotation] == desiredPod.Annotations[bootstrapTokenHashAnnotation] { - bootstrapTokenAnnotationMatches = true - } - } else { - // Ignore bootstrapTokenHashAnnotation if it's not in either the current pod or the desired pod - if _, ok := desiredPod.Annotations[bootstrapTokenHashAnnotation]; !ok { - bootstrapTokenAnnotationMatches = true - } + if _, ok := pod.Annotations[BootstrapTokenHashAnnotation]; !ok { + return false } + specMatches := annotationValueIsEqualIfPresentOnBothPods(pod, desiredPod, PodHashAnnotation) + revisionMatches := annotationValueIsEqualIfPresentOnBothPods(pod, desiredPod, PodRevisionAnnotation) + bootstrapTokenAnnotationMatches := annotationValueIsEqualIfPresentOnBothPods(pod, desiredPod, BootstrapTokenHashAnnotation) + envVarSourceMatches := annotationValueIsEqualIfPresentOnBothPods(pod, desiredPod, envVarSourceHashAnnotation) + certHashAnnotationMatches := annotationValueIsEqualIfPresentOnBothPods(pod, desiredPod, certHashAnnotation) + currentPodCopy := pod.DeepCopy() desiredPodCopy := desiredPod.DeepCopy() sanitizedCurrentPod := sanitizePod(hnp, currentPodCopy) sanitizedDesiredPod := sanitizePod(hnp, desiredPodCopy) podSpecDiff := cmp.Diff(sanitizedCurrentPod.Spec, sanitizedDesiredPod.Spec) if !specMatches { - r.Log.Info(fmt.Sprintf("pod annotation %s does not match desired pod: got %+v, expected %+v", PodHashAnnotation, pod.Annotations[PodHashAnnotation], desiredPodHash), "podSpecDiff", podSpecDiff) + r.Log.Info(fmt.Sprintf("pod annotation %s does not match desired pod: got %+v, expected %+v", PodHashAnnotation, pod.Annotations[PodHashAnnotation], desiredPod.Annotations[PodHashAnnotation]), "podSpecDiff", podSpecDiff) return false } if !revisionMatches { r.Log.Info(fmt.Sprintf("pod annotation %s does not match desired pod: got %+v, expected %+v", PodRevisionAnnotation, pod.Annotations[PodRevisionAnnotation], desiredPod.Annotations[PodRevisionAnnotation]), "podSpecDiff", podSpecDiff) return false } + if !bootstrapTokenAnnotationMatches { + r.Log.Info(fmt.Sprintf("pod annotation %s does not match desired pod: got %+v, expected %+v", BootstrapTokenHashAnnotation, pod.Annotations[BootstrapTokenHashAnnotation], desiredPod.Annotations[BootstrapTokenHashAnnotation]), "podSpecDiff", podSpecDiff) + return false + } if !envVarSourceMatches { r.Log.Info(fmt.Sprintf("pod annotation %s does not match desired pod: got %+v, expected %+v", envVarSourceHashAnnotation, pod.Annotations[envVarSourceHashAnnotation], desiredPod.Annotations[envVarSourceHashAnnotation]), "podSpecDiff", podSpecDiff) return false } - if !certHasAnnotationMatches { + if !certHashAnnotationMatches { r.Log.Info(fmt.Sprintf("pod annotation %s does not match desired pod: got %+v, expected %+v", certHashAnnotation, pod.Annotations[certHashAnnotation], desiredPod.Annotations[certHashAnnotation]), "podSpecDiff", podSpecDiff) return false } - if !bootstrapTokenAnnotationMatches { - r.Log.Info(fmt.Sprintf("pod annotation %s bootstrapTokenAnnotationMatches not match desired pod: got %+v, expected %+v", bootstrapTokenHashAnnotation, pod.Annotations[bootstrapTokenHashAnnotation], desiredPod.Annotations[bootstrapTokenHashAnnotation]), "podSpecDiff", podSpecDiff) - return false - } return true } +func annotationValueIsEqualIfPresentOnBothPods(x, y corev1.Pod, annotation string) bool { + if _, foundX := x.Annotations[annotation]; foundX { + if x.Annotations[annotation] == y.Annotations[annotation] { + return true + } + } else { + // Ignore annotation if it's not in either the current pod or the desired pod + if _, foundY := y.Annotations[annotation]; !foundY { + return true + } + } + return false +} + // getPodDesiredLifecycleState goes through the list of pods and decides what action to take for the pods. // It compares pods it is given with a newly-constructed pod. If they do not match, we know we have // "at least" a configuration difference and require a rolling replacement of the pods. @@ -808,46 +773,49 @@ func (r *HumioClusterReconciler) podsMatch(hnp *HumioNodePool, pod corev1.Pod, d // For very specific configuration differences it may indicate that all pods in the node pool should be // replaced simultaneously. // The value of podLifecycleState.pod indicates what pod should be replaced next. -func (r *HumioClusterReconciler) getPodDesiredLifecycleState(ctx context.Context, hnp *HumioNodePool, foundPodList []corev1.Pod, attachments *podAttachments, podsWithErrors bool) (podLifecycleState, error) { - for _, pod := range foundPodList { - podLifecycleStateValue := NewPodLifecycleState(*hnp, pod) +func (r *HumioClusterReconciler) getPodDesiredLifecycleState(ctx context.Context, hnp *HumioNodePool, foundPodList []corev1.Pod, attachments *podAttachments, podsWithErrorsFoundSoBypassZoneAwareness bool) (podLifecycleState, *corev1.Pod, error) { + podLifecycleStateValue := NewPodLifecycleState(*hnp) - // only consider pods not already being deleted - if pod.DeletionTimestamp != nil { - continue - } + // if pod spec differs, we want to delete it + desiredPod, err := ConstructPod(hnp, "", attachments) + if err != nil { + return podLifecycleState{}, nil, r.logErrorAndReturn(err, "could not construct pod") + } - // if pod spec differs, we want to delete it - desiredPod, err := ConstructPod(hnp, "", attachments) - if err != nil { - return podLifecycleState{}, r.logErrorAndReturn(err, "could not construct pod") - } - if hnp.TLSEnabled() { - desiredPod.Annotations[certHashAnnotation] = GetDesiredCertHash(hnp) - } + if attachments.bootstrapTokenSecretReference.secretReference != nil { + desiredPod.Annotations[BootstrapTokenHashAnnotation] = attachments.bootstrapTokenSecretReference.hash + } + + desiredHumioContainerIdx, err := kubernetes.GetContainerIndexByName(*desiredPod, HumioContainerName) + if err != nil { + return podLifecycleState{}, nil, r.logErrorAndReturn(err, "could not get pod desired lifecycle state") + } - if attachments.bootstrapTokenSecretReference.secretReference != nil { - desiredPod.Annotations[bootstrapTokenHashAnnotation] = attachments.bootstrapTokenSecretReference.hash + for _, currentPod := range foundPodList { + // only consider pods not already being deleted + if currentPod.DeletionTimestamp != nil { + continue } - podsMatch := r.podsMatch(hnp, pod, *desiredPod) + podsMatch := r.podsMatch(hnp, currentPod, *desiredPod) // ignore pod if it matches the desired pod if podsMatch { continue } + // pods do not match, append to list of pods to be replaced podLifecycleStateValue.configurationDifference = &podLifecycleStateConfigurationDifference{} - humioContainerIdx, err := kubernetes.GetContainerIndexByName(pod, HumioContainerName) - if err != nil { - return podLifecycleState{}, r.logErrorAndReturn(err, "could not get pod desired lifecycle state") - } - desiredHumioContainerIdx, err := kubernetes.GetContainerIndexByName(*desiredPod, HumioContainerName) + + // compare image versions and if they differ, we register a version difference with associated from/to versions + humioContainerIdx, err := kubernetes.GetContainerIndexByName(currentPod, HumioContainerName) if err != nil { - return podLifecycleState{}, r.logErrorAndReturn(err, "could not get pod desired lifecycle state") + return podLifecycleState{}, nil, r.logErrorAndReturn(err, "could not get pod desired lifecycle state") } - if pod.Spec.Containers[humioContainerIdx].Image != desiredPod.Spec.Containers[desiredHumioContainerIdx].Image { - fromVersion := HumioVersionFromString(pod.Spec.Containers[humioContainerIdx].Image) + + if currentPod.Spec.Containers[humioContainerIdx].Image != desiredPod.Spec.Containers[desiredHumioContainerIdx].Image { + r.Log.Info("found version difference") + fromVersion := HumioVersionFromString(currentPod.Spec.Containers[humioContainerIdx].Image) toVersion := HumioVersionFromString(desiredPod.Spec.Containers[desiredHumioContainerIdx].Image) podLifecycleStateValue.versionDifference = &podLifecycleStateVersionDifference{ from: fromVersion, @@ -856,36 +824,39 @@ func (r *HumioClusterReconciler) getPodDesiredLifecycleState(ctx context.Context } // Changes to EXTERNAL_URL means we've toggled TLS on/off and must restart all pods at the same time - if EnvVarValue(pod.Spec.Containers[humioContainerIdx].Env, "EXTERNAL_URL") != EnvVarValue(desiredPod.Spec.Containers[desiredHumioContainerIdx].Env, "EXTERNAL_URL") { + if EnvVarValue(currentPod.Spec.Containers[humioContainerIdx].Env, "EXTERNAL_URL") != EnvVarValue(desiredPod.Spec.Containers[desiredHumioContainerIdx].Env, "EXTERNAL_URL") { + r.Log.Info("EXTERNAL_URL changed so all pods must restart at the same time") podLifecycleStateValue.configurationDifference.requiresSimultaneousRestart = true } // if we run with envtest, we won't have zone information available - if !helpers.UseEnvtest() { + // if there are pods with errors that we need to prioritize first, ignore zone awareness + if !helpers.UseEnvtest() && !podsWithErrorsFoundSoBypassZoneAwareness { // if zone awareness is enabled, ignore pod if zone is incorrect - if !podsWithErrors && *hnp.GetUpdateStrategy().EnableZoneAwareness { - r.Log.Info(fmt.Sprintf("zone awareness enabled, looking up zone for pod=%s", pod.Name)) - if pod.Spec.NodeName == "" { + if *hnp.GetUpdateStrategy().EnableZoneAwareness { + if currentPod.Spec.NodeName == "" { // current pod does not have a nodeName set - r.Log.Info(fmt.Sprintf("pod=%s does not have a nodeName set, ignoring", pod.Name)) + r.Log.Info(fmt.Sprintf("pod=%s does not have a nodeName set, ignoring", currentPod.Name)) continue } // fetch zone for node name and ignore pod if zone is not the one that is marked as under maintenance - zoneForNodeName, err := kubernetes.GetZoneForNodeName(ctx, r, pod.Spec.NodeName) + zoneForNodeName, err := kubernetes.GetZoneForNodeName(ctx, r, currentPod.Spec.NodeName) if err != nil { - return podLifecycleState{}, r.logErrorAndReturn(err, "could get zone name for node") + return podLifecycleState{}, nil, r.logErrorAndReturn(err, "could get zone name for node") } if hnp.GetZoneUnderMaintenance() != "" && zoneForNodeName != hnp.GetZoneUnderMaintenance() { - r.Log.Info(fmt.Sprintf("ignoring pod=%s as zoneUnderMaintenace=%s but pod has nodeName=%s where zone=%s", pod.Name, hnp.GetZoneUnderMaintenance(), pod.Spec.NodeName, zoneForNodeName)) + r.Log.Info(fmt.Sprintf("ignoring pod=%s as zoneUnderMaintenace=%s but pod has nodeName=%s where zone=%s", currentPod.Name, hnp.GetZoneUnderMaintenance(), currentPod.Spec.NodeName, zoneForNodeName)) continue } } } - return *podLifecycleStateValue, nil + // If we didn't decide to ignore the pod by this point, we append it to the list of pods to be replaced + podLifecycleStateValue.podsToBeReplaced = append(podLifecycleStateValue.podsToBeReplaced, currentPod) + } - return podLifecycleState{}, nil + return *podLifecycleStateValue, desiredPod, nil } type podNameAndCertificateHash struct { @@ -1103,12 +1074,14 @@ func FilterPodsByZoneName(ctx context.Context, c client.Client, podList []corev1 return filteredPodList, nil } -func FilterPodsExcludePodsWithPodRevision(podList []corev1.Pod, podRevisionToExclude int) []corev1.Pod { +func FilterPodsExcludePodsWithPodRevisionOrPodHash(podList []corev1.Pod, podRevisionToExclude int, podHashToExclude string) []corev1.Pod { filteredPodList := []corev1.Pod{} for _, pod := range podList { - podRevision, found := pod.Annotations[PodRevisionAnnotation] - if found { - if strconv.Itoa(podRevisionToExclude) == podRevision { + podRevision, revisionFound := pod.Annotations[PodRevisionAnnotation] + podHash, hashFound := pod.Annotations[PodHashAnnotation] + if revisionFound && hashFound { + if strconv.Itoa(podRevisionToExclude) == podRevision && + podHashToExclude == podHash { continue } } diff --git a/controllers/humiocluster_status.go b/controllers/humiocluster_status.go index 77e846199..80353c37c 100644 --- a/controllers/humiocluster_status.go +++ b/controllers/humiocluster_status.go @@ -48,10 +48,12 @@ type messageOption struct { } type stateOption struct { - state string - nodePoolName string - desiredPodRevision int - zoneUnderMaintenance string + state string + nodePoolName string + zoneUnderMaintenance string + desiredPodRevision int + desiredPodHash string + desiredBootstrapTokenHash string } type stateOptionList struct { @@ -102,12 +104,14 @@ func (o *optionBuilder) withState(state string) *optionBuilder { return o } -func (o *optionBuilder) withNodePoolState(state string, nodePoolName string, podRevision int, zoneName string) *optionBuilder { +func (o *optionBuilder) withNodePoolState(state string, nodePoolName string, podRevision int, podHash string, bootstrapTokenHash string, zoneName string) *optionBuilder { o.options = append(o.options, stateOption{ - state: state, - nodePoolName: nodePoolName, - desiredPodRevision: podRevision, - zoneUnderMaintenance: zoneName, + state: state, + nodePoolName: nodePoolName, + zoneUnderMaintenance: zoneName, + desiredPodRevision: podRevision, + desiredPodHash: podHash, + desiredBootstrapTokenHash: bootstrapTokenHash, }) return o } @@ -116,10 +120,12 @@ func (o *optionBuilder) withNodePoolStatusList(humioNodePoolStatusList humiov1al var statesList []stateOption for _, poolStatus := range humioNodePoolStatusList { statesList = append(statesList, stateOption{ - nodePoolName: poolStatus.Name, - state: poolStatus.State, - desiredPodRevision: poolStatus.DesiredPodRevision, - zoneUnderMaintenance: poolStatus.ZoneUnderMaintenance, + nodePoolName: poolStatus.Name, + state: poolStatus.State, + zoneUnderMaintenance: poolStatus.ZoneUnderMaintenance, + desiredPodRevision: poolStatus.DesiredPodRevision, + desiredPodHash: poolStatus.DesiredPodHash, + desiredBootstrapTokenHash: poolStatus.DesiredBootstrapTokenHash, }) } o.options = append(o.options, stateOptionList{ @@ -180,18 +186,22 @@ func (s stateOption) Apply(hc *humiov1alpha1.HumioCluster) { for idx, nodePoolStatus := range hc.Status.NodePoolStatus { if nodePoolStatus.Name == s.nodePoolName { nodePoolStatus.State = s.state - nodePoolStatus.DesiredPodRevision = s.desiredPodRevision nodePoolStatus.ZoneUnderMaintenance = s.zoneUnderMaintenance + nodePoolStatus.DesiredPodRevision = s.desiredPodRevision + nodePoolStatus.DesiredPodHash = s.desiredPodHash + nodePoolStatus.DesiredBootstrapTokenHash = s.desiredBootstrapTokenHash hc.Status.NodePoolStatus[idx] = nodePoolStatus return } } hc.Status.NodePoolStatus = append(hc.Status.NodePoolStatus, humiov1alpha1.HumioNodePoolStatus{ - Name: s.nodePoolName, - State: s.state, - DesiredPodRevision: s.desiredPodRevision, - ZoneUnderMaintenance: s.zoneUnderMaintenance, + Name: s.nodePoolName, + State: s.state, + ZoneUnderMaintenance: s.zoneUnderMaintenance, + DesiredPodRevision: s.desiredPodRevision, + DesiredPodHash: s.desiredPodHash, + DesiredBootstrapTokenHash: s.desiredBootstrapTokenHash, }) } } @@ -211,10 +221,12 @@ func (s stateOptionList) Apply(hc *humiov1alpha1.HumioCluster) { hc.Status.NodePoolStatus = humiov1alpha1.HumioNodePoolStatusList{} for _, poolStatus := range s.statesList { hc.Status.NodePoolStatus = append(hc.Status.NodePoolStatus, humiov1alpha1.HumioNodePoolStatus{ - Name: poolStatus.nodePoolName, - State: poolStatus.state, - DesiredPodRevision: poolStatus.desiredPodRevision, - ZoneUnderMaintenance: poolStatus.zoneUnderMaintenance, + Name: poolStatus.nodePoolName, + State: poolStatus.state, + ZoneUnderMaintenance: poolStatus.zoneUnderMaintenance, + DesiredPodRevision: poolStatus.desiredPodRevision, + DesiredPodHash: poolStatus.desiredPodHash, + DesiredBootstrapTokenHash: poolStatus.desiredBootstrapTokenHash, }) } } diff --git a/controllers/suite/clusters/humiocluster_controller_test.go b/controllers/suite/clusters/humiocluster_controller_test.go index 872e30c15..19e742e08 100644 --- a/controllers/suite/clusters/humiocluster_controller_test.go +++ b/controllers/suite/clusters/humiocluster_controller_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "slices" "strings" "time" @@ -88,12 +89,12 @@ var _ = Describe("HumioCluster Controller", func() { createAndBootstrapMultiNodePoolCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning) Eventually(func() error { - _, err := kubernetes.GetService(ctx, k8sClient, controllers.NewHumioNodeManagerFromHumioNodePool(toCreate, &toCreate.Spec.NodePools[0]).GetServiceName(), key.Namespace) + _, err := kubernetes.GetService(ctx, k8sClient, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetServiceName(), key.Namespace) return err }, testTimeout, suite.TestInterval).Should(Succeed()) Eventually(func() error { - _, err := kubernetes.GetService(ctx, k8sClient, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetServiceName(), key.Namespace) + _, err := kubernetes.GetService(ctx, k8sClient, controllers.NewHumioNodeManagerFromHumioNodePool(toCreate, &toCreate.Spec.NodePools[0]).GetServiceName(), key.Namespace) return err }, testTimeout, suite.TestInterval).Should(Succeed()) @@ -319,7 +320,7 @@ var _ = Describe("HumioCluster Controller", func() { Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) } - suite.UsingClusterBy(key.Name, "Updating the cluster resources successfully") + suite.UsingClusterBy(key.Name, "Updating the cluster resources successfully with broken affinity") Eventually(func() error { updatedHumioCluster := humiov1alpha1.HumioCluster{} err := k8sClient.Get(ctx, key, &updatedHumioCluster) @@ -375,7 +376,7 @@ var _ = Describe("HumioCluster Controller", func() { return updatedHumioCluster.Status.State }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) - suite.UsingClusterBy(key.Name, "Updating the cluster resources successfully") + suite.UsingClusterBy(key.Name, "Updating the cluster resources successfully with working affinity") Eventually(func() error { updatedHumioCluster := humiov1alpha1.HumioCluster{} err := k8sClient.Get(ctx, key, &updatedHumioCluster) @@ -386,7 +387,36 @@ var _ = Describe("HumioCluster Controller", func() { return k8sClient.Update(ctx, &updatedHumioCluster) }, testTimeout, suite.TestInterval).Should(Succeed()) - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 3) + // Keep marking revision 2 as unschedulable as operator may delete it multiple times due to being unschedulable over and over + Eventually(func() []corev1.Pod { + podsMarkedAsPending := []corev1.Pod{} + + currentPods, err := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + if err != nil { + // wrap error in pod object + return []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%v", err)}, + }, + } + } + for _, pod := range currentPods { + if pod.Spec.Affinity != nil && + pod.Spec.Affinity.NodeAffinity != nil && + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil && + len(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) > 0 && + len(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions) > 0 { + + if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key == "some-none-existent-label" { + markPodAsPendingIfUsingEnvtest(ctx, k8sClient, pod, key.Name) + } + } + } + + return podsMarkedAsPending + }, testTimeout, suite.TestInterval).Should(HaveLen(0)) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 3, 1) Eventually(func() string { Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) @@ -442,7 +472,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) suite.UsingClusterBy(key.Name, "Pods upgrade in a rolling fashion because update strategy is explicitly set to rolling update") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) Eventually(func() string { updatedHumioCluster = humiov1alpha1.HumioCluster{} @@ -611,7 +641,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) suite.UsingClusterBy(key.Name, "Pods upgrade in a rolling fashion because the new version is a patch release") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) Eventually(func() string { updatedHumioCluster = humiov1alpha1.HumioCluster{} @@ -1229,7 +1259,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Succeed()) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) suite.UsingClusterBy(key.Name, "Validating pod is recreated using the explicitly defined helper image as init container") Eventually(func() string { @@ -1273,13 +1303,13 @@ var _ = Describe("HumioCluster Controller", func() { _ = suite.MarkPodsAsRunningIfUsingEnvtest(ctx, k8sClient, clusterPods, key.Name) if len(clusterPods) > 0 { - return clusterPods[0].Annotations["humio.com/bootstrap-token-hash"] + return clusterPods[0].Annotations[controllers.BootstrapTokenHashAnnotation] } return "" }, testTimeout, suite.TestInterval).Should(Not(Equal(""))) clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) - bootstrapTokenHashValue := clusterPods[0].Annotations["humio.com/bootstrap-token-hash"] + bootstrapTokenHashValue := clusterPods[0].Annotations[controllers.BootstrapTokenHashAnnotation] suite.UsingClusterBy(key.Name, "Rotating bootstrap token") var bootstrapTokenSecret corev1.Secret @@ -1300,7 +1330,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRestarting)) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) suite.UsingClusterBy(key.Name, "Validating pod is recreated with the new bootstrap token hash annotation") Eventually(func() string { @@ -1308,7 +1338,7 @@ var _ = Describe("HumioCluster Controller", func() { _ = suite.MarkPodsAsRunningIfUsingEnvtest(ctx, k8sClient, clusterPods, key.Name) if len(clusterPods) > 0 { - return clusterPods[0].Annotations["humio.com/bootstrap-token-hash"] + return clusterPods[0].Annotations[controllers.BootstrapTokenHashAnnotation] } return "" }, testTimeout, suite.TestInterval).Should(Not(Equal(bootstrapTokenHashValue))) @@ -1425,7 +1455,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRestarting)) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) Eventually(func() string { updatedHumioCluster = humiov1alpha1.HumioCluster{} @@ -1647,7 +1677,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Equal(1)) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, mainNodePoolManager, 2) + ensurePodsRollingRestart(ctx, mainNodePoolManager, 2, 1) Eventually(func() string { updatedHumioCluster = humiov1alpha1.HumioCluster{} @@ -1748,7 +1778,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Equal(1)) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, additionalNodePoolManager, 2) + ensurePodsRollingRestart(ctx, additionalNodePoolManager, 2, 1) Eventually(func() string { updatedHumioCluster = humiov1alpha1.HumioCluster{} @@ -2429,6 +2459,9 @@ var _ = Describe("HumioCluster Controller", func() { updatedHumioCluster.Spec.PodSecurityContext = &corev1.PodSecurityContext{} return k8sClient.Update(ctx, &updatedHumioCluster) }, testTimeout, suite.TestInterval).Should(Succeed()) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) + Eventually(func() bool { clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) for _, pod := range clusterPods { @@ -2456,7 +2489,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Succeed()) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 3, 1) Eventually(func() corev1.PodSecurityContext { clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) @@ -2478,7 +2511,7 @@ var _ = Describe("HumioCluster Controller", func() { key := types.NamespacedName{ Name: "humiocluster-containersecuritycontext", Namespace: testProcessNamespace, - } + } // State: -> Running -> ConfigError -> Running -> Restarting -> Running -> Restarting -> Running toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, @@ -2504,6 +2537,9 @@ var _ = Describe("HumioCluster Controller", func() { updatedHumioCluster.Spec.ContainerSecurityContext = &corev1.SecurityContext{} return k8sClient.Update(ctx, &updatedHumioCluster) }, testTimeout, suite.TestInterval).Should(Succeed()) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) + Eventually(func() bool { clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) for _, pod := range clusterPods { @@ -2539,7 +2575,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Succeed()) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 3, 1) Eventually(func() corev1.SecurityContext { clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) @@ -2608,7 +2644,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Succeed()) suite.UsingClusterBy(key.Name, "Confirming pods have the updated revision") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) suite.UsingClusterBy(key.Name, "Confirming pods do not have a readiness probe set") Eventually(func() *corev1.Probe { @@ -2704,7 +2740,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Succeed()) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) Eventually(func() *corev1.Probe { clusterPods, _ = kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) @@ -3304,7 +3340,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRestarting)) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) Eventually(func() string { updatedHumioCluster = humiov1alpha1.HumioCluster{} @@ -3470,7 +3506,7 @@ var _ = Describe("HumioCluster Controller", func() { } suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) suite.UsingClusterBy(key.Name, "Confirming cluster returns to Running state") Eventually(func() string { @@ -3543,7 +3579,7 @@ var _ = Describe("HumioCluster Controller", func() { } suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) suite.UsingClusterBy(key.Name, "Confirming cluster returns to Running state") Eventually(func() string { @@ -3896,6 +3932,8 @@ var _ = Describe("HumioCluster Controller", func() { return k8sClient.Update(ctx, &updatedHumioCluster) }, testTimeout, suite.TestInterval).Should(Succeed()) + suite.SimulateHumioBootstrapTokenCreatingSecretAndUpdatingStatus(ctx, key, k8sClient, testTimeout) + suite.UsingClusterBy(key.Name, "Confirming we only created ingresses with expected hostname") foundIngressList = []networkingv1.Ingress{} Eventually(func() []networkingv1.Ingress { @@ -4293,9 +4331,9 @@ var _ = Describe("HumioCluster Controller", func() { toCreate.Spec.Tolerations = []corev1.Toleration{ { Key: "key", - Operator: "Equal", + Operator: corev1.TolerationOpEqual, Value: "value", - Effect: "NoSchedule", + Effect: corev1.TaintEffectNoSchedule, }, } @@ -4804,7 +4842,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Succeed()) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) suite.UsingClusterBy(key.Name, "Confirming pods contain the new env vars") Eventually(func() int { @@ -4908,7 +4946,7 @@ var _ = Describe("HumioCluster Controller", func() { }, testTimeout, suite.TestInterval).Should(Succeed()) suite.UsingClusterBy(key.Name, "Restarting the cluster in a rolling fashion") - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) suite.UsingClusterBy(key.Name, "Confirming pods contain the new env vars") Eventually(func() int { @@ -4987,18 +5025,21 @@ var _ = Describe("HumioCluster Controller", func() { }) }) - Context("test rolling update with max unavailable absolute value", Label("envtest", "dummy"), func() { - It("Update should correctly replace pods to use new image in a rolling fashion", func() { + Context("test rolling update with zone awareness enabled", Label("dummy"), func() { + It("Update should correctly replace pods maxUnavailable=1", func() { key := types.NamespacedName{ - Name: "hc-update-absolute-maxunavail", + Name: "hc-update-absolute-maxunavail-zone-1", Namespace: testProcessNamespace, } maxUnavailable := intstr.FromInt32(1) + zoneAwarenessEnabled := true toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) toCreate.Spec.Image = versions.OldSupportedHumioVersion() toCreate.Spec.NodeCount = 9 toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ - Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessEnabled, + MaxUnavailable: &maxUnavailable, } suite.UsingClusterBy(key.Name, "Creating the cluster successfully") @@ -5017,55 +5058,185 @@ var _ = Describe("HumioCluster Controller", func() { Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) - mostSeenUnavailable := 0 + mostNumPodsSeenUnavailable := 0 + mostNumZonesWithPodsSeenUnavailable := 0 forever := make(chan struct{}) ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostNumPodsSeenUnavailable, &mostNumZonesWithPodsSeenUnavailable) - // TODO: Consider refactoring goroutine to a "watcher". https://book-v1.book.kubebuilder.io/beyond_basics/controller_watches - // Using a for-loop executing ListPods will only see snapshots in time and we could easily miss - // a point in time where we have too many pods that are not ready. - go func(ctx2 context.Context, k8sClient client.Client, toCreate humiov1alpha1.HumioCluster) { - hnp := controllers.NewHumioNodeManagerFromHumioCluster(&toCreate) - for { - select { - case <-ctx2.Done(): // if cancel() execute - forever <- struct{}{} - return - default: - // Assume all is unavailable, and decrement number each time we see one that is working - unavailableThisRound := hnp.GetNodeCount() - - pods, _ := kubernetes.ListPods(ctx2, k8sClient, hnp.GetNamespace(), hnp.GetPodLabels()) - suite.UsingClusterBy(key.Name, fmt.Sprintf("goroutine looking for unavailable pods: len(pods)=%d", len(pods))) - for _, pod := range pods { - suite.UsingClusterBy(key.Name, fmt.Sprintf("goroutine looking for unavailable pods: pod.Status.Phase=%s", pod.Status.Phase)) - - if pod.Status.Phase == corev1.PodFailed { - suite.UsingClusterBy(key.Name, fmt.Sprintf("goroutine looking for unavailable pods, full pod dump of failing pod: %+v", pod)) - var eventList corev1.EventList - _ = k8sClient.List(ctx2, &eventList) - for _, event := range eventList.Items { - if event.InvolvedObject.UID == pod.UID { - suite.UsingClusterBy(key.Name, fmt.Sprintf("Found event for failing pod: involvedObject=%+v, reason=%s, message=%s, source=%+v", event.InvolvedObject, event.Reason, event.Message, event.Source)) - } - } - } + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) - if pod.Status.Phase == corev1.PodRunning { - for idx, containerStatus := range pod.Status.ContainerStatuses { - suite.UsingClusterBy(key.Name, fmt.Sprintf("goroutine looking for unavailable pods: pod.Status.ContainerStatuses[%d]=%+v", idx, containerStatus)) - if containerStatus.Ready { - unavailableThisRound-- - } - } - } - } - // Save the number of unavailable pods in this round - mostSeenUnavailable = max(mostSeenUnavailable, unavailableThisRound) - } - time.Sleep(1 * time.Second) + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostNumPodsSeenUnavailable).To(BeNumerically("==", maxUnavailable.IntValue())) + Expect(mostNumZonesWithPodsSeenUnavailable).To(BeNumerically("==", 1)) + }) + + It("Update should correctly replace pods maxUnavailable=2", func() { + key := types.NamespacedName{ + Name: "hc-update-absolute-maxunavail-zone-2", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromInt32(2) + zoneAwarenessEnabled := true + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessEnabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostNumPodsSeenUnavailable := 0 + mostNumZonesWithPodsSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostNumPodsSeenUnavailable, &mostNumZonesWithPodsSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err } - }(ctx2, k8sClient, *toCreate) + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostNumPodsSeenUnavailable).To(BeNumerically("==", maxUnavailable.IntValue())) + Expect(mostNumZonesWithPodsSeenUnavailable).To(BeNumerically("==", 1)) + }) + + It("Update should correctly replace pods maxUnavailable=4", func() { + key := types.NamespacedName{ + Name: "hc-update-absolute-maxunavail-zone-4", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromInt32(4) + zoneAwarenessEnabled := true + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessEnabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostNumPodsSeenUnavailable := 0 + mostNumZonesWithPodsSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostNumPodsSeenUnavailable, &mostNumZonesWithPodsSeenUnavailable) suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") updatedImage := versions.DefaultHumioImageVersion() @@ -5085,7 +5256,92 @@ var _ = Describe("HumioCluster Controller", func() { return updatedHumioCluster.Status.State }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) - ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2) + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostNumPodsSeenUnavailable).To(BeNumerically("==", 3)) // nodeCount 9 and 3 zones should only replace at most 3 pods at a time as we expect the 9 pods to be uniformly distributed across the 3 zones + Expect(mostNumZonesWithPodsSeenUnavailable).To(BeNumerically("==", 1)) + }) + + It("Update should correctly replace pods maxUnavailable=25%", func() { + key := types.NamespacedName{ + Name: "hc-update-pct-maxunavail-zone-25", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromString("25%") + zoneAwarenessEnabled := true + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessEnabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostNumPodsSeenUnavailable := 0 + mostNumZonesWithPodsSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostNumPodsSeenUnavailable, &mostNumZonesWithPodsSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) Eventually(func() string { updatedHumioCluster = humiov1alpha1.HumioCluster{} @@ -5114,8 +5370,779 @@ var _ = Describe("HumioCluster Controller", func() { Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) } - suite.UsingClusterBy(key.Name, fmt.Sprintf("Verifying we do not have too many unavailable pods during pod replacements, mostSeenUnavailable(%d) <= maxUnavailable(%d)", mostSeenUnavailable, maxUnavailable.IntValue())) - Expect(mostSeenUnavailable).To(BeNumerically("<=", maxUnavailable.IntValue())) + Expect(mostNumPodsSeenUnavailable).To(BeNumerically("==", 2)) // nodeCount 9 * 25 % = 2.25 pods, rounded down is 2. Assuming 9 pods is uniformly distributed across 3 zones with 3 pods per zone. + Expect(mostNumZonesWithPodsSeenUnavailable).To(BeNumerically("==", 1)) }) - }) -}) + + It("Update should correctly replace pods maxUnavailable=50%", func() { + key := types.NamespacedName{ + Name: "hc-update-pct-maxunavail-zone-50", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromString("50%") + zoneAwarenessEnabled := true + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessEnabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostNumPodsSeenUnavailable := 0 + mostNumZonesWithPodsSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostNumPodsSeenUnavailable, &mostNumZonesWithPodsSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostNumPodsSeenUnavailable).To(BeNumerically("==", 3)) // nodeCount 9 * 50 % = 4.50 pods, rounded down is 4. Assuming 9 pods is uniformly distributed across 3 zones, that gives 3 pods per zone. + Expect(mostNumZonesWithPodsSeenUnavailable).To(BeNumerically("==", 1)) + }) + + It("Update should correctly replace pods maxUnavailable=100%", func() { + key := types.NamespacedName{ + Name: "hc-update-pct-maxunavail-zone-100", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromString("100%") + zoneAwarenessEnabled := true + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessEnabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostNumPodsSeenUnavailable := 0 + mostNumZonesWithPodsSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostNumPodsSeenUnavailable, &mostNumZonesWithPodsSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostNumPodsSeenUnavailable).To(BeNumerically("==", 3)) // Assuming 9 pods is uniformly distributed across 3 zones, that gives 3 pods per zone. + Expect(mostNumZonesWithPodsSeenUnavailable).To(BeNumerically("==", 1)) + }) + }) + + Context("test rolling update with zone awareness disabled", Label("envtest", "dummy"), func() { + It("Update should correctly replace pods maxUnavailable=1", func() { + key := types.NamespacedName{ + Name: "hc-update-absolute-maxunavail-nozone-1", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromInt32(1) + zoneAwarenessDisabled := false + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessDisabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithoutZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 1) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostSeenUnavailable).To(BeNumerically("==", maxUnavailable.IntValue())) + }) + + It("Update should correctly replace pods maxUnavailable=2", func() { + key := types.NamespacedName{ + Name: "hc-update-absolute-maxunavail-nozone-2", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromInt32(2) + zoneAwarenessDisabled := false + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessDisabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithoutZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, maxUnavailable.IntValue()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostSeenUnavailable).To(BeNumerically("==", maxUnavailable.IntValue())) + }) + + It("Update should correctly replace pods maxUnavailable=4", func() { + key := types.NamespacedName{ + Name: "hc-update-absolute-maxunavail-nozone-4", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromInt32(4) + zoneAwarenessDisabled := false + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessDisabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithoutZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, maxUnavailable.IntValue()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostSeenUnavailable).To(BeNumerically("==", maxUnavailable.IntValue())) + }) + + It("Update should correctly replace pods maxUnavailable=25%", func() { + key := types.NamespacedName{ + Name: "hc-update-pct-maxunavail-nozone-25", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromString("25%") + zoneAwarenessDisabled := false + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessDisabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithoutZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 2) // nodeCount 9 * 25 % = 2.25 pods, rounded down is 2 + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostSeenUnavailable).To(BeNumerically("==", 2)) // nodeCount 9 * 25 % = 2.25 pods, rounded down is 2 + }) + + It("Update should correctly replace pods maxUnavailable=50%", func() { + key := types.NamespacedName{ + Name: "hc-update-pct-maxunavail-nozone-50", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromString("50%") + zoneAwarenessDisabled := false + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessDisabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithoutZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, 4) // nodeCount 9 * 50 % = 4.50 pods, rounded down is 4 + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostSeenUnavailable).To(BeNumerically("==", 4)) // nodeCount 9 * 50 % = 4.50 pods, rounded down is 4 + }) + + It("Update should correctly replace pods maxUnavailable=100%", func() { + key := types.NamespacedName{ + Name: "hc-update-pct-maxunavail-nozone-100", + Namespace: testProcessNamespace, + } + maxUnavailable := intstr.FromString("100%") + zoneAwarenessDisabled := false + toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, true) + toCreate.Spec.Image = versions.OldSupportedHumioVersion() + toCreate.Spec.NodeCount = 9 + toCreate.Spec.UpdateStrategy = &humiov1alpha1.HumioUpdateStrategy{ + Type: humiov1alpha1.HumioClusterUpdateStrategyRollingUpdate, + EnableZoneAwareness: &zoneAwarenessDisabled, + MaxUnavailable: &maxUnavailable, + } + + suite.UsingClusterBy(key.Name, "Creating the cluster successfully") + ctx := context.Background() + suite.CreateAndBootstrapCluster(ctx, k8sClient, testHumioClient, toCreate, true, humiov1alpha1.HumioClusterStateRunning, testTimeout) + defer suite.CleanupCluster(ctx, k8sClient, toCreate) + + var updatedHumioCluster humiov1alpha1.HumioCluster + clusterPods, _ := kubernetes.ListPods(ctx, k8sClient, key.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(toCreate).GetPodLabels()) + for _, pod := range clusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(toCreate.Spec.Image)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "1")) + } + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(1)) + + mostSeenUnavailable := 0 + forever := make(chan struct{}) + ctx2, cancel := context.WithCancel(context.Background()) + go monitorMaxUnavailableWithoutZoneAwareness(ctx2, k8sClient, *toCreate, forever, &mostSeenUnavailable) + + suite.UsingClusterBy(key.Name, "Updating the cluster image successfully") + updatedImage := versions.DefaultHumioImageVersion() + Eventually(func() error { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + err := k8sClient.Get(ctx, key, &updatedHumioCluster) + if err != nil { + return err + } + updatedHumioCluster.Spec.Image = updatedImage + return k8sClient.Update(ctx, &updatedHumioCluster) + }, testTimeout, suite.TestInterval).Should(Succeed()) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateUpgrading)) + + ensurePodsRollingRestart(ctx, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster), 2, toCreate.Spec.NodeCount) + + Eventually(func() string { + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + return updatedHumioCluster.Status.State + }, testTimeout, suite.TestInterval).Should(BeIdenticalTo(humiov1alpha1.HumioClusterStateRunning)) + + suite.UsingClusterBy(key.Name, "Confirming pod revision is the same for all pods and the cluster itself") + updatedHumioCluster = humiov1alpha1.HumioCluster{} + Expect(k8sClient.Get(ctx, key, &updatedHumioCluster)).Should(Succeed()) + Expect(controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetDesiredPodRevision()).To(BeEquivalentTo(2)) + + updatedClusterPods, _ := kubernetes.ListPods(ctx, k8sClient, updatedHumioCluster.Namespace, controllers.NewHumioNodeManagerFromHumioCluster(&updatedHumioCluster).GetPodLabels()) + Expect(updatedClusterPods).To(HaveLen(toCreate.Spec.NodeCount)) + for _, pod := range updatedClusterPods { + humioIndex, _ := kubernetes.GetContainerIndexByName(pod, controllers.HumioContainerName) + Expect(pod.Spec.Containers[humioIndex].Image).To(BeIdenticalTo(updatedImage)) + Expect(pod.Annotations).To(HaveKeyWithValue(controllers.PodRevisionAnnotation, "2")) + } + + cancel() + <-forever + + if helpers.TLSEnabled(&updatedHumioCluster) { + suite.UsingClusterBy(key.Name, "Ensuring pod names are not changed") + Expect(podNames(clusterPods)).To(Equal(podNames(updatedClusterPods))) + } + + Expect(mostSeenUnavailable).To(BeNumerically("==", toCreate.Spec.NodeCount)) + }) + }) +}) + +// TODO: Consider refactoring goroutine to a "watcher". https://book-v1.book.kubebuilder.io/beyond_basics/controller_watches +// +// Using a for-loop executing ListPods will only see snapshots in time and we could easily miss +// a point in time where we have too many pods that are not ready. +func monitorMaxUnavailableWithZoneAwareness(ctx context.Context, k8sClient client.Client, toCreate humiov1alpha1.HumioCluster, forever chan struct{}, mostNumPodsSeenUnavailable *int, mostNumZonesWithPodsSeenUnavailable *int) { + hnp := controllers.NewHumioNodeManagerFromHumioCluster(&toCreate) + for { + select { + case <-ctx.Done(): // if cancel() execute + forever <- struct{}{} + return + default: + // Assume all is unavailable, and decrement number each time we see one that is working + unavailableThisRound := hnp.GetNodeCount() + zonesWithPodsSeenUnavailable := []string{} + + pods, _ := kubernetes.ListPods(ctx, k8sClient, hnp.GetNamespace(), hnp.GetPodLabels()) + for _, pod := range pods { + if pod.Status.Phase == corev1.PodRunning { + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.Ready { + unavailableThisRound-- + } else { + if pod.Spec.NodeName != "" { + zone, _ := kubernetes.GetZoneForNodeName(ctx, k8sClient, pod.Spec.NodeName) + if !slices.Contains(zonesWithPodsSeenUnavailable, zone) { + zonesWithPodsSeenUnavailable = append(zonesWithPodsSeenUnavailable, zone) + } + } + } + } + } + } + // Save the number of unavailable pods in this round + *mostNumPodsSeenUnavailable = max(*mostNumPodsSeenUnavailable, unavailableThisRound) + *mostNumZonesWithPodsSeenUnavailable = max(*mostNumZonesWithPodsSeenUnavailable, len(zonesWithPodsSeenUnavailable)) + } + time.Sleep(250 * time.Millisecond) + } +} + +// TODO: Consider refactoring goroutine to a "watcher". https://book-v1.book.kubebuilder.io/beyond_basics/controller_watches +// +// Using a for-loop executing ListPods will only see snapshots in time and we could easily miss +// a point in time where we have too many pods that are not ready. +func monitorMaxUnavailableWithoutZoneAwareness(ctx context.Context, k8sClient client.Client, toCreate humiov1alpha1.HumioCluster, forever chan struct{}, mostNumPodsSeenUnavailable *int) { + hnp := controllers.NewHumioNodeManagerFromHumioCluster(&toCreate) + for { + select { + case <-ctx.Done(): // if cancel() execute + forever <- struct{}{} + return + default: + // Assume all is unavailable, and decrement number each time we see one that is working + unavailableThisRound := hnp.GetNodeCount() + + pods, _ := kubernetes.ListPods(ctx, k8sClient, hnp.GetNamespace(), hnp.GetPodLabels()) + for _, pod := range pods { + if pod.Status.Phase == corev1.PodRunning { + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.Ready { + unavailableThisRound-- + } + } + } + } + // Save the number of unavailable pods in this round + *mostNumPodsSeenUnavailable = max(*mostNumPodsSeenUnavailable, unavailableThisRound) + } + time.Sleep(250 * time.Millisecond) + } +} diff --git a/controllers/suite/clusters/suite_test.go b/controllers/suite/clusters/suite_test.go index 3f192126a..e0593028c 100644 --- a/controllers/suite/clusters/suite_test.go +++ b/controllers/suite/clusters/suite_test.go @@ -87,18 +87,22 @@ var _ = BeforeSuite(func() { useExistingCluster := true testProcessNamespace = fmt.Sprintf("e2e-clusters-%d", GinkgoParallelProcess()) if !helpers.UseEnvtest() { - testTimeout = time.Second * 900 testEnv = &envtest.Environment{ UseExistingCluster: &useExistingCluster, } if helpers.UseDummyImage() { + // We use kind with dummy images instead of the real humio/humio-core container images + testTimeout = time.Second * 180 testHumioClient = humio.NewMockClient() } else { + // We use kind with real humio/humio-core container images + testTimeout = time.Second * 900 testHumioClient = humio.NewClient(log, "") By("Verifying we have a valid license, as tests will require starting up real LogScale containers") Expect(helpers.GetE2ELicenseFromEnvVar()).NotTo(BeEmpty()) } } else { + // We use envtest to run tests testTimeout = time.Second * 30 testEnv = &envtest.Environment{ // TODO: If we want to add support for TLS-functionality, we need to install cert-manager's CRD's @@ -324,8 +328,8 @@ func createAndBootstrapMultiNodePoolCluster(ctx context.Context, k8sClient clien func constructBasicMultiNodePoolHumioCluster(key types.NamespacedName, useAutoCreatedLicense bool, numberOfAdditionalNodePools int) *humiov1alpha1.HumioCluster { toCreate := suite.ConstructBasicSingleNodeHumioCluster(key, useAutoCreatedLicense) - nodeSpec := suite.ConstructBasicNodeSpecForHumioCluster(key) + nodeSpec := suite.ConstructBasicNodeSpecForHumioCluster(key) for i := 1; i <= numberOfAdditionalNodePools; i++ { toCreate.Spec.NodePools = append(toCreate.Spec.NodePools, humiov1alpha1.HumioNodePoolSpec{ Name: fmt.Sprintf("np-%d", i), @@ -336,14 +340,12 @@ func constructBasicMultiNodePoolHumioCluster(key types.NamespacedName, useAutoCr return toCreate } -func markPodAsPendingIfUsingEnvtest(ctx context.Context, client client.Client, nodeID int, pod corev1.Pod, clusterName string) error { +func markPodAsPendingIfUsingEnvtest(ctx context.Context, client client.Client, pod corev1.Pod, clusterName string) error { if !helpers.UseEnvtest() { return nil } - suite.UsingClusterBy(clusterName, fmt.Sprintf("Simulating Humio pod is marked Pending (node %d, pod phase %s)", nodeID, pod.Status.Phase)) - pod.Status.PodIP = fmt.Sprintf("192.168.0.%d", nodeID) - + suite.UsingClusterBy(clusterName, fmt.Sprintf("Simulating Humio pod is marked Pending (podName %s, pod phase %s)", pod.Name, pod.Status.Phase)) pod.Status.Conditions = []corev1.PodCondition{ { Type: corev1.PodScheduled, @@ -365,7 +367,8 @@ func markPodsWithRevisionAsReadyIfUsingEnvTest(ctx context.Context, hnp *control for i := range foundPodList { foundPodRevisionValue := foundPodList[i].Annotations[controllers.PodRevisionAnnotation] foundPodHash := foundPodList[i].Annotations[controllers.PodHashAnnotation] - suite.UsingClusterBy(hnp.GetClusterName(), fmt.Sprintf("Pod=%s revision=%s podHash=%s podIP=%s", foundPodList[i].Name, foundPodRevisionValue, foundPodHash, foundPodList[i].Status.PodIP)) + suite.UsingClusterBy(hnp.GetClusterName(), fmt.Sprintf("Pod=%s revision=%s podHash=%s podIP=%s podPhase=%s podStatusConditions=%+v", + foundPodList[i].Name, foundPodRevisionValue, foundPodHash, foundPodList[i].Status.PodIP, foundPodList[i].Status.Phase, foundPodList[i].Status.Conditions)) foundPodRevisionValueInt, _ := strconv.Atoi(foundPodRevisionValue) if foundPodRevisionValueInt == podRevision { podListWithRevision = append(podListWithRevision, foundPodList[i]) @@ -455,7 +458,7 @@ func podPendingCountByRevision(ctx context.Context, hnp *controllers.HumioNodePo } } else { if nodeID+1 <= expectedPendingCount { - _ = markPodAsPendingIfUsingEnvtest(ctx, k8sClient, nodeID, pod, hnp.GetClusterName()) + _ = markPodAsPendingIfUsingEnvtest(ctx, k8sClient, pod, hnp.GetClusterName()) revisionToPendingCount[revision]++ } } @@ -477,15 +480,17 @@ func podPendingCountByRevision(ctx context.Context, hnp *controllers.HumioNodePo return revisionToPendingCount } -func ensurePodsRollingRestart(ctx context.Context, hnp *controllers.HumioNodePool, expectedPodRevision int) { - suite.UsingClusterBy(hnp.GetClusterName(), "ensurePodsRollingRestart Ensuring replacement pods are ready one at a time") +func ensurePodsRollingRestart(ctx context.Context, hnp *controllers.HumioNodePool, expectedPodRevision int, numPodsPerIteration int) { + suite.UsingClusterBy(hnp.GetClusterName(), fmt.Sprintf("ensurePodsRollingRestart Ensuring replacement pods are ready %d at a time", numPodsPerIteration)) - for expectedReadyCount := 1; expectedReadyCount < hnp.GetNodeCount()+1; expectedReadyCount++ { + // Each iteration we mark up to some expectedReady count in bulks of numPodsPerIteration, up to at most hnp.GetNodeCount() + for expectedReadyCount := numPodsPerIteration; expectedReadyCount < hnp.GetNodeCount()+numPodsPerIteration; expectedReadyCount = expectedReadyCount + numPodsPerIteration { + cappedExpectedReadyCount := min(hnp.GetNodeCount(), expectedReadyCount) Eventually(func() map[int]int { - suite.UsingClusterBy(hnp.GetClusterName(), fmt.Sprintf("ensurePodsRollingRestart Ensuring replacement pods are ready one at a time expectedReadyCount=%d", expectedReadyCount)) - markPodsWithRevisionAsReadyIfUsingEnvTest(ctx, hnp, expectedPodRevision, expectedReadyCount) + suite.UsingClusterBy(hnp.GetClusterName(), fmt.Sprintf("ensurePodsRollingRestart Ensuring replacement pods are ready %d at a time expectedReadyCount=%d", numPodsPerIteration, cappedExpectedReadyCount)) + markPodsWithRevisionAsReadyIfUsingEnvTest(ctx, hnp, expectedPodRevision, cappedExpectedReadyCount) return podReadyCountByRevision(ctx, hnp, expectedPodRevision) - }, testTimeout, suite.TestInterval).Should(HaveKeyWithValue(expectedPodRevision, expectedReadyCount)) + }, testTimeout, suite.TestInterval).Should(HaveKeyWithValue(expectedPodRevision, cappedExpectedReadyCount)) } } diff --git a/controllers/suite/common.go b/controllers/suite/common.go index 743eb3a64..ee9c46114 100644 --- a/controllers/suite/common.go +++ b/controllers/suite/common.go @@ -417,44 +417,12 @@ func CreateAndBootstrapCluster(ctx context.Context, k8sClient client.Client, hum UsingClusterBy(key.Name, "Creating HumioCluster resource") Expect(k8sClient.Create(ctx, cluster)).Should(Succeed()) - UsingClusterBy(key.Name, "Simulating HumioBootstrapToken Controller running and adding the secret and status") - Eventually(func() error { - hbtList, err := kubernetes.ListHumioBootstrapTokens(ctx, k8sClient, key.Namespace, kubernetes.LabelsForHumioBootstrapToken(key.Name)) - if err != nil { - return err - } - if len(hbtList) == 0 { - return fmt.Errorf("no humiobootstraptokens for cluster %s", key.Name) - } - if len(hbtList) > 1 { - return fmt.Errorf("too many humiobootstraptokens for cluster %s. found list : %+v", key.Name, hbtList) - } - - updatedHumioBootstrapToken := hbtList[0] - updatedHumioBootstrapToken.Status.State = humiov1alpha1.HumioBootstrapTokenStateReady - updatedHumioBootstrapToken.Status.TokenSecretKeyRef = humiov1alpha1.HumioTokenSecretStatus{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-bootstrap-token", key.Name), - }, - Key: "secret", - }, - } - updatedHumioBootstrapToken.Status.HashedTokenSecretKeyRef = humiov1alpha1.HumioHashedTokenSecretStatus{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-bootstrap-token", key.Name), - }, - Key: "hashedToken", - }, - } - return k8sClient.Status().Update(ctx, &updatedHumioBootstrapToken) - }, testTimeout, TestInterval).Should(Succeed()) - if expectedState != humiov1alpha1.HumioClusterStateRunning { return } + SimulateHumioBootstrapTokenCreatingSecretAndUpdatingStatus(ctx, key, k8sClient, testTimeout) + UsingClusterBy(key.Name, "Confirming cluster enters running state") var updatedHumioCluster humiov1alpha1.HumioCluster Eventually(func() string { @@ -742,3 +710,39 @@ func CreateDockerRegredSecret(ctx context.Context, namespace corev1.Namespace, k } Expect(k8sClient.Create(ctx, ®credSecret)).To(Succeed()) } + +func SimulateHumioBootstrapTokenCreatingSecretAndUpdatingStatus(ctx context.Context, key types.NamespacedName, k8sClient client.Client, testTimeout time.Duration) { + UsingClusterBy(key.Name, "Simulating HumioBootstrapToken Controller running and adding the secret and status") + Eventually(func() error { + hbtList, err := kubernetes.ListHumioBootstrapTokens(ctx, k8sClient, key.Namespace, kubernetes.LabelsForHumioBootstrapToken(key.Name)) + if err != nil { + return err + } + if len(hbtList) == 0 { + return fmt.Errorf("no humiobootstraptokens for cluster %s", key.Name) + } + if len(hbtList) > 1 { + return fmt.Errorf("too many humiobootstraptokens for cluster %s. found list : %+v", key.Name, hbtList) + } + + updatedHumioBootstrapToken := hbtList[0] + updatedHumioBootstrapToken.Status.State = humiov1alpha1.HumioBootstrapTokenStateReady + updatedHumioBootstrapToken.Status.TokenSecretKeyRef = humiov1alpha1.HumioTokenSecretStatus{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-bootstrap-token", key.Name), + }, + Key: "secret", + }, + } + updatedHumioBootstrapToken.Status.HashedTokenSecretKeyRef = humiov1alpha1.HumioHashedTokenSecretStatus{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-bootstrap-token", key.Name), + }, + Key: "hashedToken", + }, + } + return k8sClient.Status().Update(ctx, &updatedHumioBootstrapToken) + }, testTimeout, TestInterval).Should(Succeed()) +} diff --git a/hack/run-e2e-within-kind-test-pod-dummy.sh b/hack/run-e2e-within-kind-test-pod-dummy.sh index 60d91aed8..84ac592fe 100755 --- a/hack/run-e2e-within-kind-test-pod-dummy.sh +++ b/hack/run-e2e-within-kind-test-pod-dummy.sh @@ -5,4 +5,5 @@ set -x -o pipefail source hack/functions.sh # We skip the helpers package as those tests assumes the environment variable USE_CERT_MANAGER is not set. -TEST_USE_EXISTING_CLUSTER=true DUMMY_LOGSCALE_IMAGE=true ginkgo --label-filter=dummy -timeout 1h -nodes=$GINKGO_NODES --no-color --skip-package helpers -v ./controllers/suite/... -covermode=count -coverprofile cover.out -progress | tee /proc/1/fd/1 +# +DUMMY_LOGSCALE_IMAGE=true ginkgo --label-filter=dummy -timeout 1h -procs=$GINKGO_NODES --no-color --skip-package helpers -v ./controllers/suite/... -covermode=count -coverprofile cover.out -progress | tee /proc/1/fd/1 diff --git a/hack/run-e2e-within-kind-test-pod.sh b/hack/run-e2e-within-kind-test-pod.sh index cde13e6c1..8c43dfb3c 100755 --- a/hack/run-e2e-within-kind-test-pod.sh +++ b/hack/run-e2e-within-kind-test-pod.sh @@ -5,4 +5,4 @@ set -x -o pipefail source hack/functions.sh # We skip the helpers package as those tests assumes the environment variable USE_CERT_MANAGER is not set. -TEST_USE_EXISTING_CLUSTER=true ginkgo --label-filter=real -timeout 120m -nodes=$GINKGO_NODES --no-color --skip-package helpers -v ./controllers/suite/... -covermode=count -coverprofile cover.out -progress | tee /proc/1/fd/1 +ginkgo --output-interceptor-mode=none --label-filter=real -timeout 120m -procs=$GINKGO_NODES --no-color --skip-package helpers -v ./controllers/suite/... -covermode=count -coverprofile cover.out -progress | tee /proc/1/fd/1