From 81a2cebeae5f086f82e6295e9d21aa5d5b97d6a4 Mon Sep 17 00:00:00 2001 From: Daniel Fradejas Date: Tue, 24 Sep 2024 11:37:22 +0200 Subject: [PATCH] feat(Critical-period): Added critical period definition with 3 new environment variables (#3) * feat(Critical-period): Added critical period definition with 3 new environment variables * fix(Fixes): Some fixes due to merge commit * feat(Refactor): Refactor due to simplicity * fix(README.md): Fixed README --- README.md | 4 ++++ cmd/main.go | 11 ++++++++- internal/globals/globals.go | 45 ++++++++++++++++++++++++++++++++++++- internal/google/mig.go | 21 ++++++++++------- 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4699517..fd4bdb9 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,9 @@ This project is designed to automate the scaling of Google Cloud Managed Instanc * DEBUG_MODE: Does not execute scalations, just log and send slack messages (Default `false`) * MIN_SIZE: Minimum size for the nodegroup (Default `1`) * MAX_SIZE: Maximum size for the nodegroup (Default `1`) + * CRITICAL_PERIOD_HOURS_UTC: Critical hours where MIG needs at least a minimum nodes to work properly (`Optional`) (Example: "6:00:00-8:00:00") + * CRITICAL_PERIOD_DAYS: Critical days where MIG needs at least a minimum nodes to work properly (`Optional`) (Example: "2,3,4") + * MIN_NODES_CRITICAL_PERIOD: Minimum nodes for critical period to exist (`Optional`) 2. **Dependencies**: • Go modules: Ensure you have Go installed and run go mod tidy to install dependencies. @@ -97,6 +100,7 @@ Contains functions for managing Elasticsearch nodes: Contains global functions * GetEnv: Return environment variable value if set. If not, it returns the default value set as second argument +* IsInCriticalPeriod: Return true when we are in the critical period set by env vars CRITICAL_PERIOD_DAYS and CRITICAL_PERIOD_HOURS_UTC ## License diff --git a/cmd/main.go b/cmd/main.go index 54bfcdb..2ca5e3c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -57,10 +57,19 @@ func main() { // Main loop to monitor scaling conditions and manage the MIG for { + // Check if the MIG is at its minimum size at least. If not, scale it up to minSize - err := google.CheckMIGMinimumSize(projectID, zone, migName, debugMode) + minSize, err := google.CheckMIGMinimumSize(projectID, zone, migName, debugMode) if err != nil { log.Printf("Error checking minimum size for MIG nodes: %v", err) + } else { + log.Printf("MIG %s scaled up to its minimum size %d", migName, minSize) + if slackWebhookURL != "" { + message := fmt.Sprintf("MIG %s scaled up to its minimum size %d", migName, minSize) + slack.NotifySlack(message, slackWebhookURL) + } + time.Sleep(time.Duration(defaultcooldownPeriodSeconds) * time.Second) + continue } // Fetch the scale up and down conditions from Prometheus diff --git a/internal/globals/globals.go b/internal/globals/globals.go index 9419aec..5851b1d 100644 --- a/internal/globals/globals.go +++ b/internal/globals/globals.go @@ -1,6 +1,12 @@ package globals -import "os" +import ( + "log" + "os" + "strconv" + "strings" + "time" +) // Get environment variables if defined. If not it retrieves // a default value @@ -10,3 +16,40 @@ func GetEnv(key string, defaultVal string) string { } return defaultVal } + +// IsInCriticalPeriod checks if the current time is within the critical period +func IsInCriticalPeriod() bool { + currentTime := time.Now().UTC() + currentWeekday := int(currentTime.Weekday()) + // Critical period variables to scale up the MIG to the minimum size + criticalPeriodHours := strings.Split(GetEnv("CRITICAL_PERIOD_HOURS_UTC", ""), "-") + if criticalPeriodHours[0] != "" && len(criticalPeriodHours) != 2 { + log.Fatalf("You must set CRITICAL_PERIOD_HOURS_UTC environment variable with the start and end hours of the critical period in UTC separated by a dash 4:00:00-6:00:00") + os.Exit(1) + } + criticalPeriodDays := strings.Split(GetEnv("CRITICAL_PERIOD_DAYS", ""), ",") + + for _, criticalPeriodDay := range criticalPeriodDays { + if strings.TrimSpace(criticalPeriodDay) == strconv.Itoa(currentWeekday) { + startHour, err := time.Parse("15:04:05", criticalPeriodHours[0]) + if err != nil { + log.Printf("Error parsing start hour: %v", err) + return false + } + endHour, err := time.Parse("15:04:05", criticalPeriodHours[1]) + if err != nil { + log.Printf("Error parsing end hour: %v", err) + return false + } + + // Adjust start and end times to match the current date + startTime := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), startHour.Hour(), startHour.Minute(), startHour.Second(), 0, currentTime.Location()) + endTime := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), endHour.Hour(), endHour.Minute(), endHour.Second(), 0, currentTime.Location()) + + if currentTime.After(startTime) && currentTime.Before(endTime) { + return true + } + } + } + return false +} diff --git a/internal/google/mig.go b/internal/google/mig.go index 8bc43cc..8be3eee 100644 --- a/internal/google/mig.go +++ b/internal/google/mig.go @@ -149,8 +149,12 @@ func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasti // getMIGScalingLimits retrieves the minimum and maximum scaling limits for a Managed Instance Group (MIG). func getMIGScalingLimits() (int32, int32, error) { + // Get min and max size from environment variables and parse to integers minSize, _ := strconv.ParseInt(globals.GetEnv("MIN_SIZE", "1"), 10, 32) + if globals.IsInCriticalPeriod() { + minSize, _ = strconv.ParseInt(globals.GetEnv("MIN_NODES_CRITICAL_PERIOD", "1"), 10, 32) + } maxSize, _ := strconv.ParseInt(globals.GetEnv("MAX_SIZE", "1"), 10, 32) return int32(minSize), int32(maxSize), nil @@ -233,26 +237,26 @@ func getMIGInstanceNames(ctx context.Context, client *compute.InstanceGroupManag } // CheckMIGMinimumSize ensures that the MIG has at least the minimum number of instances running. -func CheckMIGMinimumSize(projectID, zone, migName string, debugMode bool) error { +func CheckMIGMinimumSize(projectID, zone, migName string, debugMode bool) (int32, error) { ctx := context.Background() // Create a Compute client for managing the MIG client, err := createComputeClient(ctx, compute.NewInstanceGroupManagersRESTClient) if err != nil { - return fmt.Errorf("failed to create Instance Group Managers client: %v", err) + return 0, fmt.Errorf("failed to create Instance Group Managers client: %v", err) } defer client.Close() // Get the current target size of the MIG targetSize, err := getMIGTargetSize(ctx, client, projectID, zone, migName) if err != nil { - return fmt.Errorf("failed to get MIG target size: %v", err) + return 0, fmt.Errorf("failed to get MIG target size: %v", err) } // Get the scaling limits (minimum and maximum) minSize, _, err := getMIGScalingLimits() if err != nil { - return fmt.Errorf("failed to get MIG scaling limits: %v", err) + return 0, fmt.Errorf("failed to get MIG scaling limits: %v", err) } // If the MIG size is below the minimum, scale it up to the minimum size @@ -269,11 +273,12 @@ func CheckMIGMinimumSize(projectID, zone, migName string, debugMode bool) error if !debugMode { _, err = client.Resize(ctx, req) if err != nil { - return err - } else { - log.Printf("Scaled up MIG to its minimum size %d/%d", minSize, minSize) + return 0, err } } + return minSize, nil + } else { + return 0, fmt.Errorf("MIG size is already at the minimum limit (%d/%d)", targetSize, minSize) } - return nil + }