Merge pull request #442 from arachnys/trigger-alert-severity

Make alerts trigger if they are more severe than previous alert

CHANGES

@@ -1,6 +1,7 @@
 master
 ------
 
+* More severe alerts should trigger even if a less severe alert was recently sent
 * Update production.env.example email settings
 * Convert environment vars to boolean nicely
 

cabot/cabotapp/models.py

@@ -176,7 +176,9 @@ def alert(self):
                     timezone.now() - timedelta(minutes=settings.NOTIFICATION_INTERVAL)) < self.last_alert_sent:
                     return
             elif self.overall_status in (self.CRITICAL_STATUS, self.ERROR_STATUS):
-                if self.last_alert_sent and (
+                more_important = self.old_overall_status == self.WARNING_STATUS or \
+                    (self.old_overall_status == self.ERROR_STATUS and self.overall_status == self.CRITICAL_STATUS)
+                if not more_important and self.last_alert_sent and (
                     timezone.now() - timedelta(minutes=settings.ALERT_INTERVAL)) < self.last_alert_sent:
                     return
             self.last_alert_sent = timezone.now()

cabot/cabotapp/tests/tests_basic.py

@@ -919,6 +919,27 @@ class TestAlerts(LocalTestCase):
     def setUp(self):
         super(TestAlerts, self).setUp()
 
+        self.warning_http_check = HttpStatusCheck.objects.create(
+            name='Http Check',
+            created_by=self.user,
+            importance=Service.WARNING_STATUS,
+            endpoint='http://arachnys.com',
+            timeout=10,
+            status_code='200',
+            text_match=None,
+        )
+        self.error_http_check = HttpStatusCheck.objects.create(
+            name='Http Check',
+            created_by=self.user,
+            importance=Service.ERROR_STATUS,
+            endpoint='http://arachnys.com',
+            timeout=10,
+            status_code='200',
+            text_match=None,
+        )
+        self.service.status_checks.add(self.warning_http_check, self.error_http_check)
+        self.critical_http_check = self.http_check
+
         self.user.profile.hipchat_alias = "test_user_hipchat_alias"
         self.user.profile.save()
 
@@ -935,6 +956,52 @@ def test_alert(self, fake_send_alert):
         self.assertEqual(fake_send_alert.call_count, 1)
         fake_send_alert.assert_called_with(self.service, duty_officers=[])
 
+    def trigger_failing_check(self, check):
+        StatusCheckResult(
+            status_check=check,
+            time=timezone.now() - timedelta(seconds=60),
+            time_complete=timezone.now() - timedelta(seconds=59),
+            succeeded=False
+        ).save()
+        check.last_run = timezone.now()
+        check.save()
+
+    @patch('cabot.cabotapp.models.send_alert')
+    def test_alert_increasing_severity(self, fake_send_alert):
+        self.trigger_failing_check(self.warning_http_check)
+        self.assertEqual(fake_send_alert.call_count, 1)
+
+        self.trigger_failing_check(self.error_http_check)
+        self.assertEqual(fake_send_alert.call_count, 2)
+
+        self.trigger_failing_check(self.critical_http_check)
+        self.assertEqual(fake_send_alert.call_count, 3)
+
+    @patch('cabot.cabotapp.models.send_alert')
+    def test_alert_decreasing_severity(self, fake_send_alert):
+        self.trigger_failing_check(self.critical_http_check)
+        self.assertEqual(fake_send_alert.call_count, 1)
+
+        self.trigger_failing_check(self.error_http_check)
+        self.assertEqual(fake_send_alert.call_count, 1)
+
+        self.trigger_failing_check(self.warning_http_check)
+        self.assertEqual(fake_send_alert.call_count, 1)
+
+    @patch('cabot.cabotapp.models.send_alert')
+    def test_alert_alternating_severity(self, fake_send_alert):
+        self.trigger_failing_check(self.error_http_check)
+        self.assertEqual(fake_send_alert.call_count, 1)
+
+        self.trigger_failing_check(self.warning_http_check)
+        self.assertEqual(fake_send_alert.call_count, 1)
+
+        self.trigger_failing_check(self.error_http_check)
+        self.assertEqual(fake_send_alert.call_count, 1)
+
+        self.trigger_failing_check(self.critical_http_check)
+        self.assertEqual(fake_send_alert.call_count, 2)
+
     def test_update_plugins(self):
         # Test that disabling a plugin is detected by update_alert_plugins
         plugins = update_alert_plugins()