Work on PD notifier and clean up

alerts.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
 	"math"
 	"os"
+	"time"
 
 	"github.com/fatih/color"
 )
@@ -33,6 +36,8 @@ func (alert *Alert) ApplyFunction(values []float64) float64 {
 }
 
 func (alert *Alert) Setup() {
+	hash := md5.Sum([]byte(alert.Name))
+	alert.Hash = hex.EncodeToString(hash[:])
 	for _, n := range alert.NotifiersRaw {
 		alert.Notifiers = append(alert.Notifiers, Notifier{Name: n})
 	}
@@ -70,13 +75,33 @@ func (alert *Alert) Run() {
 		message := fmt.Sprintf("*[!] %s triggered!* Value: %.2f | Trigger: %s %d",
 			alert.Name, applied_function, alert.Trigger.Operator, alert.Trigger.Value)
 		color.Red(message)
-
-		for _, n := range alert.Notifiers {
-			fmt.Printf("<-> Alert sending: %+v\n", alert)
-			n.Run(message)
+		alertAlreadyTriggered := false
+		tMutex.Lock()
+		if v, ok := triggeredAlerts[alert.Hash]; ok {
+			color.Yellow(fmt.Sprintf("[already triggered at %s] %s", v.TriggeredAt, message))
+			alertAlreadyTriggered = true
+		} else {
+			triggeredAlerts[alert.Hash] = TriggeredAlert{Hash: alert.Hash, TriggeredAt: time.Now()}
+		}
+		tMutex.Unlock()
+		if !alertAlreadyTriggered {
+			for _, n := range alert.Notifiers {
+				n.Run(message, true)
+			}
 		}
 
 	} else {
+		tMutex.Lock()
+		if _, ok := triggeredAlerts[alert.Hash]; ok {
+			delete(triggeredAlerts, alert.Hash)
+			message := fmt.Sprintf("*[+] %s resolved * Value: %.2f | Trigger: %s %d",
+				alert.Name, applied_function, alert.Trigger.Operator, alert.Trigger.Value)
+			for _, n := range alert.Notifiers {
+				n.Run(message, false)
+			}
+			color.Green("[+] %s - Alert resolved.", alert.Name)
+		}
+		tMutex.Unlock()
 		color.Green(fmt.Sprintf("[+] %s passed. (%.2f)", alert.Name, applied_function))
 	}
 

glide.yaml

@@ -26,3 +26,4 @@ import:
   - hipchat
 - package: gopkg.in/yaml.v2
   version: 7ad95dd0798a40da1ccdff6dff35fd177b5edf40
+- package: github.com/PagerDuty/go-pagerduty

influx.go

@@ -32,6 +32,14 @@ func query(query string) []float64 {
 	if err != nil {
 		log.Fatal(err)
 	}
+	if len(res) < 1 {
+		return ret
+	}
+
+	if len(res[0].Series) < 1 {
+		return ret
+	}
+
 	for i, row := range res[0].Series[0].Values {
 		t, err := time.Parse(time.RFC3339, row[0].(string))
 		if err != nil {

main.go

@@ -16,6 +16,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/bluele/slack"
@@ -41,6 +42,7 @@ type TriggeredAlert struct {
 type Alert struct {
 	Name         string
 	Type         string
+	Hash         string
 	Function     string
 	Limit        int
 	Timeshift    string
@@ -58,7 +60,10 @@ var slack_channel *slack.Channel
 var hipchat_api *hipchat.Client
 
 var (
-	triggeredAlerts = map[string]TriggeredAlert{}
+	tMutex                sync.Mutex
+	triggeredAlerts       = map[string]TriggeredAlert{}
+	pagerduty_api_token   string
+	pagerduty_service_key string
 )
 
 func main() {
@@ -81,6 +86,7 @@ func main() {
 
 	setupSlack()
 	setupHipchat()
+	setupPagerduty()
 
 	done := make(chan bool)
 	for _, alert := range alerts {

notifiers.go

@@ -9,28 +9,57 @@ import (
 	"net/url"
 	"os"
 
+	pagerduty "github.com/PagerDuty/go-pagerduty"
 	"github.com/bluele/slack"
 	"github.com/fatih/color"
 	"github.com/tbruyelle/hipchat-go/hipchat"
 )
 
-func (n *Notifier) Run(message string) {
+func (n *Notifier) Run(message string, isAlert bool) {
 	switch n.Name {
 	case "slack":
 		if slack_api == nil {
 			color.Red("[!] Slack used as a notifier, but not configured with ENV vars.")
 			return
 		}
-		err = slack_api.ChatPostMessage(slack_channel.Id, message, &slack.ChatPostMessageOpt{IconEmoji: ":fire:"})
+		if isAlert {
+			err = slack_api.ChatPostMessage(slack_channel.Id, message, &slack.ChatPostMessageOpt{IconEmoji: ":fire:"})
+		} else {
+			err = slack_api.ChatPostMessage(slack_channel.Id, message, &slack.ChatPostMessageOpt{IconEmoji: ":success:"})
+		}
 		if err != nil {
 			color.Red(fmt.Sprintf("[!] Error posting to Slack: %s", err))
 		}
+	case "pagerduty":
+		if pagerduty_api_token == "" || pagerduty_service_key == "" {
+			color.Red("[!] PagerDuty used as a notifier, but not configured with ENV vars.")
+		}
+
+		if isAlert {
+			event := pagerduty.Event{
+				Type:        "trigger",
+				ServiceKey:  pagerduty_service_key,
+				Description: message,
+			}
+			resp, err := pagerduty.CreateEvent(event)
+			if err != nil {
+				fmt.Println(resp)
+				color.Red(fmt.Sprintf("[!] Error posting to PagerDuty: %s", err))
+			}
+		} else {
+			color.Green("[>] PagerDuty incident should be resolved now.")
+		}
+
 	case "hipchat":
 		if hipchat_api == nil {
 			color.Red("[!] HipChat used as a notifier, but not configured with ENV vars.")
 			return
 		}
-		_, err = hipchat_api.Room.Notification(os.Getenv("HIPCHAT_ROOM_ID"), &hipchat.NotificationRequest{Message: message, Color: "red"})
+		if isAlert {
+			_, err = hipchat_api.Room.Notification(os.Getenv("HIPCHAT_ROOM_ID"), &hipchat.NotificationRequest{Message: message, Color: "red"})
+		} else {
+			_, err = hipchat_api.Room.Notification(os.Getenv("HIPCHAT_ROOM_ID"), &hipchat.NotificationRequest{Message: message, Color: "green"})
+		}
 		if err != nil {
 			color.Red(fmt.Sprintf("[!] Error posting to HipChat: %s", err))
 		}
@@ -40,6 +69,17 @@ func (n *Notifier) Run(message string) {
 
 }
 
+func setupPagerduty() {
+	if len(os.Getenv("PAGERDUTY_API_TOKEN")) == 0 ||
+		len(os.Getenv("PAGERDUTY_SERVICE_KEY")) == 0 {
+		color.Yellow("[>] Skipping Pagerduty setup, missing PAGERDUTY_API_TOKEN and PAGERDUTY_SERVICE_KEY")
+		return
+	}
+
+	pagerduty_api_token = os.Getenv("PAGERDUTY_API_TOKEN")
+	pagerduty_service_key = os.Getenv("PAGERDUTY_SERVICE_KEY")
+}
+
 func setupSlack() {
 	if len(os.Getenv("SLACK_API_TOKEN")) == 0 ||
 		len(os.Getenv("SLACK_ROOM")) == 0 {