Skip to content

Commit

Permalink
feat(Slack): Added better messages to Slack channel
Browse files Browse the repository at this point in the history
  • Loading branch information
dfradehubs committed Sep 19, 2024
1 parent 59785b5 commit c9ebf31
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 21 deletions.
11 changes: 7 additions & 4 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"elasticsearch-vm-autoscaler/internal/google"
"elasticsearch-vm-autoscaler/internal/prometheus"
"elasticsearch-vm-autoscaler/internal/slack"
"fmt"

"log"
"os"
Expand Down Expand Up @@ -79,27 +80,29 @@ func main() {
// If the up condition is met, add a node to the MIG
if upCondition {
log.Printf("Up condition %s met: Trying to create a new node!", prometheusUpCondition)
err = google.AddNodeToMIG(projectID, zone, migName, debugMode)
currentSize, maxSize, err := google.AddNodeToMIG(projectID, zone, migName, debugMode)
if err != nil {
log.Printf("Error adding node to MIG: %v", err)
time.Sleep(time.Duration(retryIntervalSeconds) * time.Second)
continue
}
// Notify via Slack that a node has been added
if slackWebhookURL != "" {
slack.NotifySlack("New node created succesfully in MIG", slackWebhookURL)
message := fmt.Sprintf("Added new node to MIG %s. Current size is %d nodes and the maximum nodes to create are %d", migName, currentSize, maxSize)
slack.NotifySlack(message, slackWebhookURL)
}
} else if downCondition { // If the down condition is met, remove a node from the MIG
log.Printf("Down condition %s met. Trying to remove one node!", prometheusDownCondition)
err = google.RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword, debugMode)
currentSize, minSize, nodeRemoved, err := google.RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword, debugMode)
if err != nil {
log.Printf("Error draining node from MIG: %v", err)
time.Sleep(time.Duration(retryIntervalSeconds) * time.Second)
continue
}
// Notify via Slack that a node has been removed
if slackWebhookURL != "" {
slack.NotifySlack("Removed node from MIG", slackWebhookURL)
message := fmt.Sprintf("Removed node %s from MIG %s. Current size is %d nodes and the minimum nodes to exist are %d", nodeRemoved, migName, currentSize, minSize)
slack.NotifySlack(message, slackWebhookURL)
}
} else {
// No scaling conditions met, so no changes to the MIG
Expand Down
34 changes: 17 additions & 17 deletions internal/google/mig.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,32 +18,32 @@ import (
)

// AddNodeToMIG increases the size of the Managed Instance Group (MIG) by 1, if it has not reached the maximum limit.
func AddNodeToMIG(projectID, zone, migName string, debugMode bool) error {
func AddNodeToMIG(projectID, zone, migName string, debugMode bool) (int32, int32, error) {
ctx := context.Background()

// Create a new Compute client for managing the MIG
client, err := createComputeClient(ctx, compute.NewInstanceGroupManagersRESTClient)
if err != nil {
return fmt.Errorf("failed to create Instance Group Managers client: %v", err)
return 0, 0, fmt.Errorf("failed to create Instance Group Managers client: %v", err)
}
defer client.Close()

// Get the current target size of the MIG
targetSize, err := getMIGTargetSize(ctx, client, projectID, zone, migName)
if err != nil {
return fmt.Errorf("failed to get MIG target size: %v", err)
return 0, 0, fmt.Errorf("failed to get MIG target size: %v", err)
}
log.Printf("Current size of MIG is %d nodes", targetSize)

// Get the scaling limits (minimum and maximum)
_, maxSize, err := getMIGScalingLimits()
if err != nil {
return fmt.Errorf("failed to get MIG scaling limits: %v", err)
return 0, 0, fmt.Errorf("failed to get MIG scaling limits: %v", err)
}

// Check if the MIG has reached its maximum size
if targetSize >= maxSize {
return fmt.Errorf("MIG has reached its maximum size (%d/%d), no further scaling is possible", targetSize, maxSize)
return 0, 0, fmt.Errorf("MIG has reached its maximum size (%d/%d), no further scaling is possible", targetSize, maxSize)
}

// Create a request to resize the MIG by increasing the target size by 1
Expand All @@ -58,55 +58,55 @@ func AddNodeToMIG(projectID, zone, migName string, debugMode bool) error {
if !debugMode {
_, err = client.Resize(ctx, req)
if err != nil {
return err
return 0, 0, err
} else {
log.Printf("Scaled up MIG successfully %d/%d", targetSize+1, maxSize)
}
}
return nil
return targetSize + 1, maxSize, nil
}

// RemoveNodeFromMIG decreases the size of the Managed Instance Group (MIG) by 1, if it has not reached the minimum limit.
func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword string, debugMode bool) error {
func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword string, debugMode bool) (int32, int32, string, error) {
ctx := context.Background()

// Create a new Compute client for managing the MIG
client, err := createComputeClient(ctx, compute.NewInstanceGroupManagersRESTClient)
if err != nil {
return fmt.Errorf("failed to create Instance Group Managers client: %v", err)
return 0, 0, "", fmt.Errorf("failed to create Instance Group Managers client: %v", err)
}
defer client.Close()

// Get the current target size of the MIG
targetSize, err := getMIGTargetSize(ctx, client, projectID, zone, migName)
if err != nil {
return fmt.Errorf("failed to get MIG target size: %v", err)
return 0, 0, "", fmt.Errorf("failed to get MIG target size: %v", err)
}
log.Printf("Current size of MIG is %d nodes", targetSize)

// Get the scaling limits (minimum and maximum)
minSize, _, err := getMIGScalingLimits()
if err != nil {
return fmt.Errorf("failed to get MIG scaling limits: %v", err)
return 0, 0, "", fmt.Errorf("failed to get MIG scaling limits: %v", err)
}

// Check if the MIG has reached its minimum size
if targetSize <= minSize {
return fmt.Errorf("MIG has reached its minimum size (%d/%d), no further scaling down is possible", targetSize, minSize)
return 0, 0, "", fmt.Errorf("MIG has reached its minimum size (%d/%d), no further scaling down is possible", targetSize, minSize)
}

// Get a random instance from the MIG to remove
instanceToRemove, err := GetInstanceToRemove(ctx, client, projectID, zone, migName)
if err != nil {
return fmt.Errorf("error draining Elasticsearch node: %v", err)
return 0, 0, "", fmt.Errorf("error draining Elasticsearch node: %v", err)
}

// If not in debug mode, drain the node from Elasticsearch before removal
if !debugMode {
log.Printf("Instance to remove: %s. Draining from elasticsearch cluster", instanceToRemove)
err = elasticsearch.DrainElasticsearchNode(elasticURL, instanceToRemove, elasticUser, elasticPassword)
if err != nil {
return fmt.Errorf("error draining Elasticsearch node: %v", err)
return 0, 0, "", fmt.Errorf("error draining Elasticsearch node: %v", err)
}
log.Printf("Instance drained successfully from elasticsearch cluster")
}
Expand All @@ -126,7 +126,7 @@ func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasti
if !debugMode {
_, err = client.DeleteInstances(ctx, deleteReq)
if err != nil {
return fmt.Errorf("error deleting instance: %v", err)
return 0, 0, "", fmt.Errorf("error deleting instance: %v", err)
} else {
log.Printf("Scaled down MIG successfully %d/%d", targetSize-1, minSize)
}
Expand All @@ -139,12 +139,12 @@ func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasti
if !debugMode {
err = elasticsearch.ClearElasticsearchClusterSettings(elasticURL, elasticUser, elasticPassword)
if err != nil {
return fmt.Errorf("error clearing Elasticsearch cluster settings: %v", err)
return 0, 0, "", fmt.Errorf("error clearing Elasticsearch cluster settings: %v", err)
}
log.Printf("Cleared up elasticsearch settings for draining node")
}

return nil
return targetSize - 1, minSize, instanceToRemove, nil
}

// getMIGScalingLimits retrieves the minimum and maximum scaling limits for a Managed Instance Group (MIG).
Expand Down

0 comments on commit c9ebf31

Please sign in to comment.