From 454029e479974a52845a3833180eec3aa03b5b2f Mon Sep 17 00:00:00 2001 From: mgianluc Date: Sun, 28 Jul 2024 16:40:34 +0200 Subject: [PATCH] (feat) Intoduce ability to pause/unpause a cluster SveltosCluster has a new field called `Schedule` that allows to specify when to unpause a cluster. For instance ```yaml sveltosCluster.Spec.Schedule = &libsveltosv1beta1.Schedule{ From: "0 20 * * 5", // every friday 8PM To: "0 7 * * 1", // every monday 7AM } ``` will have the SveltosCluster set to unpaused only from Friday, 8PM to Monday 7AM every week. A Paused SveltosCluster will receive no update from Sveltos. This feature so enable having a maintanence window on each managed cluster. Only during this maintanance window, update will be delivered to cluster. --- controllers/export_test.go | 4 + controllers/sveltoscluster_controller.go | 84 +++++++++++++++++++ controllers/sveltoscluster_controller_test.go | 50 +++++++++++ go.mod | 3 +- go.sum | 6 +- 5 files changed, 144 insertions(+), 3 deletions(-) diff --git a/controllers/export_test.go b/controllers/export_test.go index f7c5189..45185b7 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -19,3 +19,7 @@ package controllers var ( ShouldRenewTokenRequest = (*SveltosClusterReconciler).shouldRenewTokenRequest ) + +var ( + HandleAutomaticPauseUnPause = handleAutomaticPauseUnPause +) diff --git a/controllers/sveltoscluster_controller.go b/controllers/sveltoscluster_controller.go index 3e9f7e2..4574a60 100644 --- a/controllers/sveltoscluster_controller.go +++ b/controllers/sveltoscluster_controller.go @@ -25,6 +25,7 @@ import ( "github.com/Masterminds/semver" "github.com/go-logr/logr" "github.com/pkg/errors" + "github.com/robfig/cron/v3" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -203,6 +204,8 @@ func (r *SveltosClusterReconciler) reconcileNormal( } } } + + handleAutomaticPauseUnPause(sveltosClusterScope.SveltosCluster, time.Now(), logger) } // SetupWithManager sets up the controller with the Manager. @@ -388,3 +391,84 @@ current-context: sveltos-context` return data } + +func handleAutomaticPauseUnPause(sveltosCluster *libsveltosv1beta1.SveltosCluster, + currentTime time.Time, logger logr.Logger) { + + if sveltosCluster.Spec.ActiveWindow == nil { + return + } + + if sveltosCluster.Status.NextPause != nil && sveltosCluster.Status.NextUnpause != nil { + if currentTime.After(sveltosCluster.Status.NextUnpause.Time) && + currentTime.Before(sveltosCluster.Status.NextPause.Time) { + + sveltosCluster.Spec.Paused = false + return + } else if currentTime.Before(sveltosCluster.Status.NextUnpause.Time) { + sveltosCluster.Spec.Paused = true + return + } else if currentTime.After(sveltosCluster.Status.NextPause.Time) { + sveltosCluster.Spec.Paused = true + } else if currentTime.Before(sveltosCluster.Status.NextPause.Time) { + // Updates NextFrom and NextTo only once current time is past NextTo + return + } + } else { + sveltosCluster.Spec.Paused = true + } + + if sveltosCluster.Status.NextUnpause == nil || currentTime.After(sveltosCluster.Status.NextUnpause.Time) { + lastRunTime := sveltosCluster.CreationTimestamp + if sveltosCluster.Status.NextUnpause != nil { + lastRunTime = *sveltosCluster.Status.NextUnpause + } + + nextFromTime, err := getNextScheduleTime(sveltosCluster.Spec.ActiveWindow.From, &lastRunTime, currentTime) + if err != nil { + logger.V(logs.LogInfo).Error(err, "failed to get next from time") + return + } + sveltosCluster.Status.NextUnpause = &metav1.Time{Time: *nextFromTime} + } + + if sveltosCluster.Status.NextPause == nil || currentTime.After(sveltosCluster.Status.NextPause.Time) { + lastRunTime := sveltosCluster.CreationTimestamp + if sveltosCluster.Status.NextPause != nil { + lastRunTime = *sveltosCluster.Status.NextPause + } + + nextToTime, err := getNextScheduleTime(sveltosCluster.Spec.ActiveWindow.To, &lastRunTime, currentTime) + if err != nil { + logger.V(logs.LogInfo).Error(err, "failed to get next to time") + } + + sveltosCluster.Status.NextPause = &metav1.Time{Time: *nextToTime} + } +} + +// getNextScheduleTime gets the time of next schedule after last scheduled and before now +func getNextScheduleTime(schedule string, lastRunTime *metav1.Time, now time.Time) (*time.Time, error) { + sched, err := cron.ParseStandard(schedule) + if err != nil { + return nil, fmt.Errorf("unparseable schedule %q: %w", schedule, err) + } + + if lastRunTime == nil { + return nil, fmt.Errorf("last run time must be specified") + } + + starts := 0 + for t := sched.Next(lastRunTime.Time); t.Before(now); t = sched.Next(t) { + const maxNumberOfFailures = 100 + starts++ + if starts > maxNumberOfFailures { + return nil, + fmt.Errorf("too many missed start times (> %d). Set or check clock skew", + maxNumberOfFailures) + } + } + + next := sched.Next(now) + return &next, nil +} diff --git a/controllers/sveltoscluster_controller_test.go b/controllers/sveltoscluster_controller_test.go index 16acb44..90f1569 100644 --- a/controllers/sveltoscluster_controller_test.go +++ b/controllers/sveltoscluster_controller_test.go @@ -141,6 +141,56 @@ var _ = Describe("SveltosCluster: Reconciler", func() { Expect(controllers.ShouldRenewTokenRequest(reconciler, sveltosClusterScope, logger)).To(BeFalse()) }) + It("handleAutomaticPauseUnPause updates Spec.Paused based on Spec.Schedule", func() { + sveltosCluster.Spec.ActiveWindow = &libsveltosv1beta1.ActiveWindow{ + From: "0 20 * * 5", // every friday 8PM + To: "0 7 * * 1", // every monday 7AM + } + loc, err := time.LoadLocation("Europe/Rome") // Replace with your desired time zone + Expect(err).To(BeNil()) + tuesday8AM := time.Date(2024, time.July, 30, 8, 0, 0, 0, loc) + sveltosCluster.CreationTimestamp = metav1.Time{Time: tuesday8AM} + + controllers.HandleAutomaticPauseUnPause(sveltosCluster, tuesday8AM.Add(time.Hour), logger) + + // Next unpause coming friday at 8PM + Expect(sveltosCluster.Status.NextUnpause).ToNot(BeNil()) + expectedUnpause := time.Date(2024, time.August, 2, 20, 0, 0, 0, loc) + Expect(sveltosCluster.Status.NextUnpause.Time).To(Equal(expectedUnpause)) + + // Next pause coming monday at 7AM + Expect(sveltosCluster.Status.NextPause).ToNot(BeNil()) + expectedPause := time.Date(2024, time.August, 5, 7, 0, 0, 0, loc) + Expect(sveltosCluster.Status.NextPause.Time).To(Equal(expectedPause)) + + Expect(sveltosCluster.Spec.Paused).To(BeTrue()) + + // when time is before unpause, Paused will remain set to false and NextPause + // NextUnpause will not be updated + thursday8AM := time.Date(2024, time.August, 1, 8, 0, 0, 0, loc) + controllers.HandleAutomaticPauseUnPause(sveltosCluster, thursday8AM, logger) + Expect(sveltosCluster.Spec.Paused).To(BeTrue()) + Expect(sveltosCluster.Status.NextPause.Time).To(Equal(expectedPause)) + Expect(sveltosCluster.Status.NextUnpause.Time).To(Equal(expectedUnpause)) + + // when time is past unpause but before pause, Paused will be set to false and NextPause + // NextUnpause will not be updated + saturday8AM := time.Date(2024, time.August, 3, 8, 0, 0, 0, loc) + controllers.HandleAutomaticPauseUnPause(sveltosCluster, saturday8AM, logger) + Expect(sveltosCluster.Spec.Paused).To(BeFalse()) + Expect(sveltosCluster.Status.NextPause.Time).To(Equal(expectedPause)) + Expect(sveltosCluster.Status.NextUnpause.Time).To(Equal(expectedUnpause)) + + // when time is past next pause, Paused will be set to true and NextPause + // NextUnpause updated + monday8AM := time.Date(2024, time.August, 5, 8, 0, 0, 0, loc) + controllers.HandleAutomaticPauseUnPause(sveltosCluster, monday8AM, logger) + Expect(sveltosCluster.Spec.Paused).To(BeTrue()) + expectedUnpause = time.Date(2024, time.August, 9, 20, 0, 0, 0, loc) + expectedPause = time.Date(2024, time.August, 12, 7, 0, 0, 0, loc) + Expect(sveltosCluster.Status.NextPause.Time).To(Equal(expectedPause)) + Expect(sveltosCluster.Status.NextUnpause.Time).To(Equal(expectedUnpause)) + }) }) func getSveltosClusterInstance(namespace, name string) *libsveltosv1beta1.SveltosCluster { diff --git a/go.mod b/go.mod index 054712b..75789ca 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ require ( github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v0.35.1-0.20240726065655-ee3b7dc30da2 + github.com/projectsveltos/libsveltos v0.35.1-0.20240728165049-3f913227fee9 + github.com/robfig/cron/v3 v3.0.1 github.com/spf13/pflag v1.0.5 k8s.io/api v0.30.3 k8s.io/apiextensions-apiserver v0.30.3 diff --git a/go.sum b/go.sum index 26b603a..1632871 100644 --- a/go.sum +++ b/go.sum @@ -131,8 +131,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectsveltos/libsveltos v0.35.1-0.20240726065655-ee3b7dc30da2 h1:DE4fivtx4pjPFpjAyDWbG46RQQCyDMgJ7ZI/5zkHq88= -github.com/projectsveltos/libsveltos v0.35.1-0.20240726065655-ee3b7dc30da2/go.mod h1:8cr9lSt8i0fRQ47AItTElqxsiD/ni80GJALVQgxfdG4= +github.com/projectsveltos/libsveltos v0.35.1-0.20240728165049-3f913227fee9 h1:u07JFnTTLl/xQG5Xykj+UeGfPJGjMR+8sWXsZuRxTPE= +github.com/projectsveltos/libsveltos v0.35.1-0.20240728165049-3f913227fee9/go.mod h1:8cr9lSt8i0fRQ47AItTElqxsiD/ni80GJALVQgxfdG4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= @@ -141,6 +141,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=