add fails-to-notify

distributions/build

@@ -1,5 +1,5 @@
 #!/bin/bash
-VERSION=1.0.2
+VERSION=1.0.3
 echo "Building deb amd64..."
 pushd ../src/scyd
 GOOS=linux GOARCH=amd64 go build

example_conf/scylla.toml

@@ -24,19 +24,11 @@ arg=ops
 edge-trigger=true
 
 
-[job "test"]
-schedule = cron * * * * *
-description = Show environment
+[job "2fails"]
+command = "if [[ $((RANDOM % 3)) == 0 ]]; then echo FAILED; exit 1; else echo OK; exit 0; fi"
+schedule = cron */1 * * * *
+description = Every 1 minutes
 host = localhost
-sudo = yes
-command =  set && sleep 65
-notifier = slack
-
-
-[job "test2"]
-schedule = cron */2 * * * *
-description = Every 2 minutes
-host = localhost
-sudo = yes
-command =  echo "Hello"
+sudo = no
+# fails-to-notify=2
 notifier = slack

src/scyd/config/config.go

@@ -41,6 +41,7 @@ type JobSpec struct {
 	ReadTimeout    int    `gcfg:"read-timeout"`
 	MaxRunHistory  int    `gcfg:"max-run-history"`
 	RunOnStart     bool   `gcfg:"run-on-start"`
+	FailsToNotify  int    `gcfg:"fails-to-notify"`
 	Notifier       string
 }
 
@@ -63,7 +64,6 @@ type Notifier struct {
 	Path        string
 	Args        []string `gcfg:"arg"`
 	EdgeTrigger bool     `gcfg:"edge-trigger"`
-	NumFailures int      `gcfg:"num-failures"`
 	Always      bool
 }
 

src/scyd/main.go

@@ -13,7 +13,7 @@ import (
 	"syscall"
 )
 
-const VERSION = "1.0.2"
+const VERSION = "1.0.3"
 
 func writePid() {
 	pid := os.Getpid()

src/scyd/scheduler/job.go

@@ -198,7 +198,7 @@ func (job *Job) complete(r *HostRun, notifier *JobNotifier) bool {
 	job.History[i].updateStatus()
 	if job.History[0].Status != Running {
 		job.Status = job.History[0].Status
-		log.Printf("Completed job %s.%d (%d)\n", job.Name, job.RunId, job.Status)
+		log.Printf("Completed job %s.%d (%s)\n", job.Name, job.RunId, RunStatusNames[job.Status])
 		job.EndTime = time.Now()
 		job.save()
 		job.saveRun(&job.History[i])
@@ -409,3 +409,23 @@ func FindNamedStringCaptures(re *regexp.Regexp, x string) map[string]string {
 	}
 	return matches
 }
+
+//
+// Find the total number of consecutive failures prior to the
+// current run. Add in the current run if it is a failure
+func (job *Job) consecutiveFailures() (failures int) {
+	if len(job.History) <= 1 {
+		return
+	}
+	if job.History[0].Status == Failed {
+		failures++
+	}
+	for _, h := range job.History[1:len(job.History)] {
+		if h.Status == Failed {
+			failures++
+		} else {
+			break
+		}
+	}
+	return
+}

src/scyd/scheduler/notifier.go

@@ -19,15 +19,19 @@ func (notifier JobNotifier) Notify(job *Job) {
 	if len(job.History) >= 2 {
 		last_status = job.History[1].Status
 	}
+	cf := job.consecutiveFailures()
+	log.Printf("consecutive failures: %d", cf)
 	if notifier.EdgeTrigger {
 		if job.Status != last_status {
 			notifier.fireNotification(job)
 		}
 	} else if notifier.Always {
 		notifier.fireNotification(job)
 	} else if job.Status == Succeeded && last_status == Failed {
-		notifier.fireNotification(job)
-	} else if job.Status == Failed {
+		if cf >= job.FailsToNotify {
+			notifier.fireNotification(job)
+		}
+	} else if job.Status == Failed && (cf == job.FailsToNotify || job.FailsToNotify == 0) {
 		notifier.fireNotification(job)
 	}
 }