Skip to content

Commit

Permalink
feat(Critical-period): Added critical period definition with 3 new en…
Browse files Browse the repository at this point in the history
…vironment variables (#3)

* feat(Critical-period): Added critical period definition with 3 new environment variables

* fix(Fixes): Some fixes due to merge commit

* feat(Refactor): Refactor due to simplicity

* fix(README.md): Fixed README
  • Loading branch information
dfradehubs authored Sep 24, 2024
1 parent 0e78428 commit 81a2ceb
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 10 deletions.
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ This project is designed to automate the scaling of Google Cloud Managed Instanc
* DEBUG_MODE: Does not execute scalations, just log and send slack messages (Default `false`)
* MIN_SIZE: Minimum size for the nodegroup (Default `1`)
* MAX_SIZE: Maximum size for the nodegroup (Default `1`)
* CRITICAL_PERIOD_HOURS_UTC: Critical hours where MIG needs at least a minimum nodes to work properly (`Optional`) (Example: "6:00:00-8:00:00")
* CRITICAL_PERIOD_DAYS: Critical days where MIG needs at least a minimum nodes to work properly (`Optional`) (Example: "2,3,4")
* MIN_NODES_CRITICAL_PERIOD: Minimum nodes for critical period to exist (`Optional`)

2. **Dependencies**:
• Go modules: Ensure you have Go installed and run go mod tidy to install dependencies.
Expand Down Expand Up @@ -97,6 +100,7 @@ Contains functions for managing Elasticsearch nodes:
Contains global functions

* GetEnv: Return environment variable value if set. If not, it returns the default value set as second argument
* IsInCriticalPeriod: Return true when we are in the critical period set by env vars CRITICAL_PERIOD_DAYS and CRITICAL_PERIOD_HOURS_UTC

## License

Expand Down
11 changes: 10 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,19 @@ func main() {

// Main loop to monitor scaling conditions and manage the MIG
for {

// Check if the MIG is at its minimum size at least. If not, scale it up to minSize
err := google.CheckMIGMinimumSize(projectID, zone, migName, debugMode)
minSize, err := google.CheckMIGMinimumSize(projectID, zone, migName, debugMode)
if err != nil {
log.Printf("Error checking minimum size for MIG nodes: %v", err)
} else {
log.Printf("MIG %s scaled up to its minimum size %d", migName, minSize)
if slackWebhookURL != "" {
message := fmt.Sprintf("MIG %s scaled up to its minimum size %d", migName, minSize)
slack.NotifySlack(message, slackWebhookURL)
}
time.Sleep(time.Duration(defaultcooldownPeriodSeconds) * time.Second)
continue
}

// Fetch the scale up and down conditions from Prometheus
Expand Down
45 changes: 44 additions & 1 deletion internal/globals/globals.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
package globals

import "os"
import (
"log"
"os"
"strconv"
"strings"
"time"
)

// Get environment variables if defined. If not it retrieves
// a default value
Expand All @@ -10,3 +16,40 @@ func GetEnv(key string, defaultVal string) string {
}
return defaultVal
}

// IsInCriticalPeriod checks if the current time is within the critical period
func IsInCriticalPeriod() bool {
currentTime := time.Now().UTC()
currentWeekday := int(currentTime.Weekday())
// Critical period variables to scale up the MIG to the minimum size
criticalPeriodHours := strings.Split(GetEnv("CRITICAL_PERIOD_HOURS_UTC", ""), "-")
if criticalPeriodHours[0] != "" && len(criticalPeriodHours) != 2 {
log.Fatalf("You must set CRITICAL_PERIOD_HOURS_UTC environment variable with the start and end hours of the critical period in UTC separated by a dash 4:00:00-6:00:00")
os.Exit(1)
}
criticalPeriodDays := strings.Split(GetEnv("CRITICAL_PERIOD_DAYS", ""), ",")

for _, criticalPeriodDay := range criticalPeriodDays {
if strings.TrimSpace(criticalPeriodDay) == strconv.Itoa(currentWeekday) {
startHour, err := time.Parse("15:04:05", criticalPeriodHours[0])
if err != nil {
log.Printf("Error parsing start hour: %v", err)
return false
}
endHour, err := time.Parse("15:04:05", criticalPeriodHours[1])
if err != nil {
log.Printf("Error parsing end hour: %v", err)
return false
}

// Adjust start and end times to match the current date
startTime := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), startHour.Hour(), startHour.Minute(), startHour.Second(), 0, currentTime.Location())
endTime := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), endHour.Hour(), endHour.Minute(), endHour.Second(), 0, currentTime.Location())

if currentTime.After(startTime) && currentTime.Before(endTime) {
return true
}
}
}
return false
}
21 changes: 13 additions & 8 deletions internal/google/mig.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,12 @@ func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasti

// getMIGScalingLimits retrieves the minimum and maximum scaling limits for a Managed Instance Group (MIG).
func getMIGScalingLimits() (int32, int32, error) {

// Get min and max size from environment variables and parse to integers
minSize, _ := strconv.ParseInt(globals.GetEnv("MIN_SIZE", "1"), 10, 32)
if globals.IsInCriticalPeriod() {
minSize, _ = strconv.ParseInt(globals.GetEnv("MIN_NODES_CRITICAL_PERIOD", "1"), 10, 32)
}
maxSize, _ := strconv.ParseInt(globals.GetEnv("MAX_SIZE", "1"), 10, 32)

return int32(minSize), int32(maxSize), nil
Expand Down Expand Up @@ -233,26 +237,26 @@ func getMIGInstanceNames(ctx context.Context, client *compute.InstanceGroupManag
}

// CheckMIGMinimumSize ensures that the MIG has at least the minimum number of instances running.
func CheckMIGMinimumSize(projectID, zone, migName string, debugMode bool) error {
func CheckMIGMinimumSize(projectID, zone, migName string, debugMode bool) (int32, error) {
ctx := context.Background()

// Create a Compute client for managing the MIG
client, err := createComputeClient(ctx, compute.NewInstanceGroupManagersRESTClient)
if err != nil {
return fmt.Errorf("failed to create Instance Group Managers client: %v", err)
return 0, fmt.Errorf("failed to create Instance Group Managers client: %v", err)
}
defer client.Close()

// Get the current target size of the MIG
targetSize, err := getMIGTargetSize(ctx, client, projectID, zone, migName)
if err != nil {
return fmt.Errorf("failed to get MIG target size: %v", err)
return 0, fmt.Errorf("failed to get MIG target size: %v", err)
}

// Get the scaling limits (minimum and maximum)
minSize, _, err := getMIGScalingLimits()
if err != nil {
return fmt.Errorf("failed to get MIG scaling limits: %v", err)
return 0, fmt.Errorf("failed to get MIG scaling limits: %v", err)
}

// If the MIG size is below the minimum, scale it up to the minimum size
Expand All @@ -269,11 +273,12 @@ func CheckMIGMinimumSize(projectID, zone, migName string, debugMode bool) error
if !debugMode {
_, err = client.Resize(ctx, req)
if err != nil {
return err
} else {
log.Printf("Scaled up MIG to its minimum size %d/%d", minSize, minSize)
return 0, err
}
}
return minSize, nil
} else {
return 0, fmt.Errorf("MIG size is already at the minimum limit (%d/%d)", targetSize, minSize)
}
return nil

}

0 comments on commit 81a2ceb

Please sign in to comment.