(feat) SveltosCluster Readniness and Liveness Checks

Introduce ReadinessChecks and LivenessChecks. ReadinessChecks are evaluated till SveltosCluster moves to ready. LivenessCheckes are periodically evelauated after that. This is an example of ReadinessChecks waiting for at least one worker node to exist in the cluster ```yaml apiVersion: lib.projectsveltos.io/v1beta1 kind: SveltosCluster metadata: name: staging namespace: default spec: consecutiveFailureThreshold: 3 kubeconfigName: clusterapi-workload-sveltos-kubeconfig readinessChecks: - condition: |- function evaluate() hs = {} hs.pass = false for _, resource in ipairs(resources) do if not (resource.metadata.labels and resource.metadata.labels["node-role.kubernetes.io/control-plane"]) then hs.pass = true end end return hs end name: namespace-bar-check resourceSelectors: - group: "" kind: Namespace name: bar version: v1 status: connectionStatus: Down failureMessage: cluster check namespace-bar-check failed ``` This PR also makes sure a SveltosCluster instance is reconciled only on Create, Delete and Update when Spec changes (so Status changes do not cause a reconciliation)
projectsveltos · Feb 15, 2025 · 5f8b825 · 5f8b825
1 parent e054392
commit 5f8b825
Show file tree

Hide file tree

Showing 9 changed files with 739 additions and 36 deletions.
diff --git a/Makefile b/Makefile
@@ -69,7 +69,7 @@ KIND := $(TOOLS_BIN_DIR)/kind
 KUBECTL := $(TOOLS_BIN_DIR)/kubectl
 
 GOLANGCI_LINT_VERSION := "v1.62.2"
-CLUSTERCTL_VERSION := "v1.9.3"
+CLUSTERCTL_VERSION := "v1.9.4"
 
 KUSTOMIZE_VER := v5.3.0
 KUSTOMIZE_BIN := kustomize
@@ -308,12 +308,12 @@ deploy-projectsveltos: $(KUSTOMIZE)
 
 	@echo 'Install libsveltos CRDs'
 	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/config/crd/bases/lib.projectsveltos.io_debuggingconfigurations.yaml
-	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/config/crd/bases/lib.projectsveltos.io_sveltosclusters.yaml
+	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/config/crd/bases/lib.projectsveltos.io_sveltosclusters.yaml	
 
 	# Install projectsveltos sveltoscluster-manager components
 	@echo 'Install projectsveltos sveltoscluster-manager components'
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f-
 
 	@echo "Waiting for projectsveltos sveltoscluster-manager to be available..."
-	$(KUBECTL) wait --for=condition=Available deployment/sc-manager -n projectsveltos --timeout=$(TIMEOUT)
+	$(KUBECTL) wait --for=condition=Available deployment/sc-manager -n projectsveltos --timeout=$(TIMEOUT)
diff --git a/controllers/cluster_checks.go b/controllers/cluster_checks.go
@@ -0,0 +1,291 @@
+/*
+Copyright 2024. projectsveltos.io. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	lua "github.com/yuin/gopher-lua"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
+
+	libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1"
+	logs "github.com/projectsveltos/libsveltos/lib/logsettings"
+	sveltoslua "github.com/projectsveltos/libsveltos/lib/lua"
+)
+
+func runChecks(ctx context.Context, remotConfig *rest.Config, checks []libsveltosv1beta1.ClusterCheck,
+	logger logr.Logger) error {
+
+	for i := range checks {
+		pass, err := runCheck(ctx, remotConfig, &checks[i], logger)
+		if err != nil {
+			return err
+		}
+		if !pass {
+			logger.V(logs.LogInfo).Info(fmt.Sprintf("cluster check %s failed", checks[i].Name))
+			return fmt.Errorf("cluster check %s failed", checks[i].Name)
+		}
+	}
+
+	return nil
+}
+
+func runCheck(ctx context.Context, remotConfig *rest.Config, check *libsveltosv1beta1.ClusterCheck,
+	logger logr.Logger) (bool, error) {
+
+	resources, err := getResources(ctx, remotConfig, check.ResourceSelectors, logger)
+	if err != nil {
+		return false, err
+	}
+
+	return validateCheck(check.Condition, resources, logger)
+}
+
+// getResources returns resources matching ResourceSelectors.
+func getResources(ctx context.Context, remotConfig *rest.Config, resourceSelectors []libsveltosv1beta1.ResourceSelector,
+	logger logr.Logger) ([]*unstructured.Unstructured, error) {
+
+	resources := []*unstructured.Unstructured{}
+	for i := range resourceSelectors {
+		matching, err := getResourcesMatchinResourceSelector(ctx, remotConfig, &resourceSelectors[i], logger)
+		if err != nil {
+			return nil, err
+		}
+
+		resources = append(resources, matching...)
+	}
+
+	return resources, nil
+}
+
+// getResourcesMatchinResourceSelector returns resources matching ResourceSelector.
+func getResourcesMatchinResourceSelector(ctx context.Context, remotConfig *rest.Config, resourceSelector *libsveltosv1beta1.ResourceSelector,
+	logger logr.Logger) ([]*unstructured.Unstructured, error) {
+
+	gvk := schema.GroupVersionKind{
+		Group:   resourceSelector.Group,
+		Version: resourceSelector.Version,
+		Kind:    resourceSelector.Kind,
+	}
+
+	dc := discovery.NewDiscoveryClientForConfigOrDie(remotConfig)
+	groupResources, err := restmapper.GetAPIGroupResources(dc)
+	if err != nil {
+		return nil, err
+	}
+	mapper := restmapper.NewDiscoveryRESTMapper(groupResources)
+
+	mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
+	if err != nil {
+		if meta.IsNoMatchError(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	resourceId := schema.GroupVersionResource{
+		Group:    gvk.Group,
+		Version:  gvk.Version,
+		Resource: mapping.Resource.Resource,
+	}
+
+	options := metav1.ListOptions{}
+
+	if len(resourceSelector.LabelFilters) > 0 {
+		labelFilter := ""
+		for i := range resourceSelector.LabelFilters {
+			if labelFilter != "" {
+				labelFilter += ","
+			}
+			f := resourceSelector.LabelFilters[i]
+			if f.Operation == libsveltosv1beta1.OperationEqual {
+				labelFilter += fmt.Sprintf("%s=%s", f.Key, f.Value)
+			} else {
+				labelFilter += fmt.Sprintf("%s!=%s", f.Key, f.Value)
+			}
+		}
+
+		options.LabelSelector = labelFilter
+	}
+
+	if resourceSelector.Namespace != "" {
+		options.FieldSelector += fmt.Sprintf("metadata.namespace=%s", resourceSelector.Namespace)
+	}
+
+	if resourceSelector.Name != "" {
+		if options.FieldSelector != "" {
+			options.FieldSelector += ","
+		}
+		options.FieldSelector += fmt.Sprintf("metadata.name=%s", resourceSelector.Name)
+	}
+
+	d := dynamic.NewForConfigOrDie(remotConfig)
+	var list *unstructured.UnstructuredList
+	list, err = d.Resource(resourceId).List(ctx, options)
+	if err != nil {
+		return nil, err
+	}
+
+	logger.V(logs.LogDebug).Info(fmt.Sprintf("found %d resources", len(list.Items)))
+
+	resources := []*unstructured.Unstructured{}
+	for i := range list.Items {
+		resource := &list.Items[i]
+		if !resource.GetDeletionTimestamp().IsZero() {
+			continue
+		}
+		isMatch, err := isMatchForEventSource(resource, resourceSelector.Evaluate, logger)
+		if err != nil {
+			return nil, err
+		}
+
+		if isMatch {
+			resources = append(resources, resource)
+		}
+	}
+
+	return resources, nil
+}
+
+func isMatchForEventSource(resource *unstructured.Unstructured, script string, logger logr.Logger) (bool, error) {
+	if script == "" {
+		return true, nil
+	}
+
+	l := lua.NewState()
+	defer l.Close()
+
+	obj := sveltoslua.MapToTable(resource.UnstructuredContent())
+
+	if err := l.DoString(script); err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("doString failed: %v", err))
+		return false, err
+	}
+
+	l.SetGlobal("obj", obj)
+
+	if err := l.CallByParam(lua.P{
+		Fn:      l.GetGlobal("evaluate"), // name of Lua function
+		NRet:    1,                       // number of returned values
+		Protect: true,                    // return err or panic
+	}, obj); err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to evaluate health for resource: %v", err))
+		return false, err
+	}
+
+	lv := l.Get(-1)
+	tbl, ok := lv.(*lua.LTable)
+	if !ok {
+		logger.V(logs.LogInfo).Info(sveltoslua.LuaTableError)
+		return false, fmt.Errorf("%s", sveltoslua.LuaTableError)
+	}
+
+	goResult := sveltoslua.ToGoValue(tbl)
+	resultJson, err := json.Marshal(goResult)
+	if err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to marshal result: %v", err))
+		return false, err
+	}
+
+	var result matchStatus
+	err = json.Unmarshal(resultJson, &result)
+	if err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to marshal result: %v", err))
+		return false, err
+	}
+
+	if result.Message != "" {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("message: %s", result.Message))
+	}
+
+	logger.V(logs.LogDebug).Info(fmt.Sprintf("is a match: %t", result.Matching))
+
+	return result.Matching, nil
+}
+
+func validateCheck(luaScript string, resources []*unstructured.Unstructured,
+	logger logr.Logger) (bool, error) {
+
+	if luaScript == "" {
+		return true, nil
+	}
+
+	// Create a new Lua state
+	l := lua.NewState()
+	defer l.Close()
+
+	// Load the Lua script
+	if err := l.DoString(luaScript); err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("doString failed: %v", err))
+		return false, err
+	}
+
+	// Create an argument table
+	argTable := l.NewTable()
+	for _, resource := range resources {
+		obj := sveltoslua.MapToTable(resource.UnstructuredContent())
+		argTable.Append(obj)
+	}
+
+	l.SetGlobal("resources", argTable)
+
+	if err := l.CallByParam(lua.P{
+		Fn:      l.GetGlobal("evaluate"), // name of Lua function
+		NRet:    1,                       // number of returned values
+		Protect: true,                    // return err or panic
+	}, argTable); err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to call evaluate function: %s", err.Error()))
+		return false, err
+	}
+
+	lv := l.Get(-1)
+	tbl, ok := lv.(*lua.LTable)
+	if !ok {
+		logger.V(logs.LogInfo).Info(sveltoslua.LuaTableError)
+		return false, fmt.Errorf("%s", sveltoslua.LuaTableError)
+	}
+
+	goResult := sveltoslua.ToGoValue(tbl)
+	resultJson, err := json.Marshal(goResult)
+	if err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to marshal result: %v", err))
+		return false, err
+	}
+
+	var result checkStatus
+	err = json.Unmarshal(resultJson, &result)
+	if err != nil {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to marshal result: %v", err))
+		return false, err
+	}
+
+	if result.Message != "" {
+		logger.V(logs.LogInfo).Info(fmt.Sprintf("message: %s", result.Message))
+	}
+
+	return result.Pass, nil
+}