From 39b7142150c4e9b972028c35aa624ce48f8fe024 Mon Sep 17 00:00:00 2001 From: Naresh Date: Tue, 30 Jun 2020 11:00:54 +0530 Subject: [PATCH 1/7] Initial commit of custom slack alert --- elastalert_modules/slack_custom_alert.py | 22 ++++++++++++++++++++ elastalert_modules/tst_ist_tz_enhancement.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 elastalert_modules/slack_custom_alert.py diff --git a/elastalert_modules/slack_custom_alert.py b/elastalert_modules/slack_custom_alert.py new file mode 100644 index 000000000..4c674dc85 --- /dev/null +++ b/elastalert_modules/slack_custom_alert.py @@ -0,0 +1,22 @@ +from elastalert.alerts import Alerter, BasicMatchString +from elastalert.util import elastalert_logger + +class SlackCustomAlert(Alerter): + # By setting required_options to a set of strings + # You can ensure that the rule config file specifies all + # of the options. Otherwise, ElastAlert will throw an exception + # when trying to load the rule. + required_options = set(['slack_webhook_url']) + + # Alert is called + def alert(self, matches): + + # Matches is a list of match dictionaries. + # It contains more than one match when the alert has + # the aggregation option set + for match in matches: + elastalert_logger.info("Received match %s" % (match)) + + def get_info(self): + return {'type': 'Slack Alerter', + 'channel': 'test'} diff --git a/elastalert_modules/tst_ist_tz_enhancement.py b/elastalert_modules/tst_ist_tz_enhancement.py index 80719284d..75364abe9 100644 --- a/elastalert_modules/tst_ist_tz_enhancement.py +++ b/elastalert_modules/tst_ist_tz_enhancement.py @@ -29,4 +29,4 @@ def process(self, match): tz_str = ist_tz_str + " Or " + tst_tz_str - match['@timestamp'] = tz_str + match['@timestamp-custom'] = tz_str From 427ad786119db343bce41b2304e7f93eefd03bf6 Mon Sep 17 00:00:00 2001 From: Naresh Date: Tue, 30 Jun 2020 11:15:25 +0530 Subject: [PATCH 2/7] Seperate out timezone variables --- elastalert_modules/tst_ist_tz_enhancement.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/elastalert_modules/tst_ist_tz_enhancement.py b/elastalert_modules/tst_ist_tz_enhancement.py index 75364abe9..feefc662f 100644 --- a/elastalert_modules/tst_ist_tz_enhancement.py +++ b/elastalert_modules/tst_ist_tz_enhancement.py @@ -27,6 +27,5 @@ def process(self, match): ist_tz_str = pretty_ts(ist_tz, False) tst_tz_str = pretty_ts(tst_tz, False) - tz_str = ist_tz_str + " Or " + tst_tz_str - - match['@timestamp-custom'] = tz_str + match['@timestamp-ist'] = ist_tz_str + match['@timestamp-cst'] = tst_tz_str From 0b531d5a485dc7f4f7c7cb33e5c0c3248e4cb25c Mon Sep 17 00:00:00 2001 From: Naresh Date: Tue, 30 Jun 2020 12:08:54 +0530 Subject: [PATCH 3/7] using slack blocks --- elastalert_modules/slack_custom_alert.py | 71 +++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/elastalert_modules/slack_custom_alert.py b/elastalert_modules/slack_custom_alert.py index 4c674dc85..dd4bc35fe 100644 --- a/elastalert_modules/slack_custom_alert.py +++ b/elastalert_modules/slack_custom_alert.py @@ -1,3 +1,7 @@ +import requests + +from requests.exceptions import RequestException + from elastalert.alerts import Alerter, BasicMatchString from elastalert.util import elastalert_logger @@ -8,6 +12,11 @@ class SlackCustomAlert(Alerter): # when trying to load the rule. required_options = set(['slack_webhook_url']) + def __init__(self, rule): + super(SlackCustomAlert, self).__init__(rule) + self.slack_webhook_url = self.rule['slack_webhook_url'] + self.slack_title = self.rule.get('slack_title', '') + # Alert is called def alert(self, matches): @@ -16,7 +25,65 @@ def alert(self, matches): # the aggregation option set for match in matches: elastalert_logger.info("Received match %s" % (match)) + # post to slack + headers = {'content-type': 'application/json'} + payload ={ + "blocks":[ + { + "type":"section", + "text":{ + "type":"mrkdwn", + "text":"*{slack_title}*".format(slack_title = self.slack_title) + } + }, + { + "type":"section", + "fields":[ + { + "type":"mrkdwn", + "text":"*Application:*\n{instance-tag}".format(instance-tag = match['instance-tag']) + }, + { + "type":"mrkdwn", + "text":"*Time in IST:*\n{timestamp-ist}".format(timestamp-ist = match['@timestamp-ist'] + }, + { + "type":"mrkdwn", + "text":"*Time in CST:*\n{timestamp-cst}".format(timestamp-cst = match['@timestamp-cst'] + }, + { + "type":"mrkdwn", + "text":"*Time in UTC00Z:*\n{timestamp}".format(timestamp = match['@timestamp'] + } + ] + }, + { + "type":"actions", + "elements":[ + { + "type":"button", + "text":{ + "type":"plain_text", + "emoji":true, + "text":"Send Incident Response" + }, + "style":"primary", + "value":"click_me_123" + } + ] + } + ] + } + try: + response = requests.post( + self.slack_webhook_url, data=json.dumps(payload), + headers=headers) + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to slack: %s" % e) + + elastalert_logger.info("Alert '%s' sent to Slack" % self.rule['name']) + def get_info(self): - return {'type': 'Slack Alerter', - 'channel': 'test'} + return {'type': 'slack'} From 3b021ac364587f465ff80b87600f6a9c11a0cc49 Mon Sep 17 00:00:00 2001 From: Naresh Date: Tue, 30 Jun 2020 14:05:26 +0530 Subject: [PATCH 4/7] var substituion fix in dict, and use pre defnied block ids --- elastalert_modules/slack_custom_alert.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/elastalert_modules/slack_custom_alert.py b/elastalert_modules/slack_custom_alert.py index dd4bc35fe..e97477fd6 100644 --- a/elastalert_modules/slack_custom_alert.py +++ b/elastalert_modules/slack_custom_alert.py @@ -1,9 +1,11 @@ import requests +import json from requests.exceptions import RequestException from elastalert.alerts import Alerter, BasicMatchString from elastalert.util import elastalert_logger +from elastalert.util import EAException class SlackCustomAlert(Alerter): # By setting required_options to a set of strings @@ -31,6 +33,7 @@ def alert(self, matches): "blocks":[ { "type":"section", + "block_id": "alarm_title", "text":{ "type":"mrkdwn", "text":"*{slack_title}*".format(slack_title = self.slack_title) @@ -38,33 +41,35 @@ def alert(self, matches): }, { "type":"section", + "block_id": "alarm_text_args", "fields":[ { "type":"mrkdwn", - "text":"*Application:*\n{instance-tag}".format(instance-tag = match['instance-tag']) + "text":"*Application:*\n{instance}".format(instance = match['instance-tag']) }, { "type":"mrkdwn", - "text":"*Time in IST:*\n{timestamp-ist}".format(timestamp-ist = match['@timestamp-ist'] + "text":"*Time in IST:*\n{ist}".format(ist = match['@timestamp-ist']) }, { "type":"mrkdwn", - "text":"*Time in CST:*\n{timestamp-cst}".format(timestamp-cst = match['@timestamp-cst'] + "text":"*Time in CST:*\n{cst}".format(cst = match['@timestamp-cst']) }, { "type":"mrkdwn", - "text":"*Time in UTC00Z:*\n{timestamp}".format(timestamp = match['@timestamp'] + "text":"*Time in UTC00Z:*\n{timestamp}".format(timestamp = match['@timestamp']) } ] }, { "type":"actions", + "block_id": "alarm_action", "elements":[ { "type":"button", "text":{ "type":"plain_text", - "emoji":true, + "emoji":True, "text":"Send Incident Response" }, "style":"primary", From 08b595d387be849244b652b60da46e1994d5db71 Mon Sep 17 00:00:00 2001 From: Naresh Date: Tue, 30 Jun 2020 14:10:47 +0530 Subject: [PATCH 5/7] update slack button value --- elastalert_modules/slack_custom_alert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert_modules/slack_custom_alert.py b/elastalert_modules/slack_custom_alert.py index e97477fd6..df6d6d8ee 100644 --- a/elastalert_modules/slack_custom_alert.py +++ b/elastalert_modules/slack_custom_alert.py @@ -73,7 +73,7 @@ def alert(self, matches): "text":"Send Incident Response" }, "style":"primary", - "value":"click_me_123" + "value":"submit_incident_response" } ] } From 9fecd34d25f856a822d7ffd9348270c732d0db3c Mon Sep 17 00:00:00 2001 From: Naresh Date: Wed, 1 Jul 2020 15:55:21 +0530 Subject: [PATCH 6/7] Add kibana dashboard button to incident response --- elastalert_modules/slack_custom_alert.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/elastalert_modules/slack_custom_alert.py b/elastalert_modules/slack_custom_alert.py index df6d6d8ee..84ad6404c 100644 --- a/elastalert_modules/slack_custom_alert.py +++ b/elastalert_modules/slack_custom_alert.py @@ -65,6 +65,16 @@ def alert(self, matches): "type":"actions", "block_id": "alarm_action", "elements":[ + { + "type":"button", + "text":{ + "type":"plain_text", + "emoji":True, + "text":"View Kibana Dashboard" + }, + "style":"primary", + "url":"https://elk.it.logitech.com:5601/app/kibana#/dashboard/35aa4fd0-b0b3-11ea-833d-fde9206e58f3?_g=(filters%3A!()%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A2000)%2Ctime%3A(from%3Anow-15d%2Cto%3Anow))" + }, { "type":"button", "text":{ From 893488668a443af7306106c6722e6492b9bd8242 Mon Sep 17 00:00:00 2001 From: Naresh Date: Thu, 2 Jul 2020 11:58:22 +0530 Subject: [PATCH 7/7] update elastalert alarm config --- .../application_critical_uptime_monitor.yaml | 13 ++----------- .../application_uptime_non_critical_monitor.yaml | 13 ++----------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/infra-rules/application_critical_uptime_monitor.yaml b/infra-rules/application_critical_uptime_monitor.yaml index b395450d7..5dbb93645 100644 --- a/infra-rules/application_critical_uptime_monitor.yaml +++ b/infra-rules/application_critical_uptime_monitor.yaml @@ -35,16 +35,7 @@ match_enhancements: # (Required) # The alert is use when a match is found -alert: -- slack +alert: "elastalert_modules.slack_custom_alert.SlackCustomAlert" -alert_subject: "CSAD IT - Critical Instance Application Health Monitoring Alarm" -alert_text: "The Application {0} is unhealthy and reporting Downtime during {1}" -alert_text_type: alert_text_only -alert_text_args: ["instance-tag", "@timestamp"] - - -slack: -slack_webhook_url: "" +slack_webhook_url: "" slack_title: "CSAD IT - Critical Instance Application Health Monitoring Alarm" -slack_title_link: "https://elk.it.logitech.com:5601/app/kibana#/dashboard/35aa4fd0-b0b3-11ea-833d-fde9206e58f3" diff --git a/infra-rules/application_uptime_non_critical_monitor.yaml b/infra-rules/application_uptime_non_critical_monitor.yaml index 192cbc99d..94bc845f4 100644 --- a/infra-rules/application_uptime_non_critical_monitor.yaml +++ b/infra-rules/application_uptime_non_critical_monitor.yaml @@ -35,16 +35,7 @@ filter: # (Required) # The alert is use when a match is found -alert: -- slack +alert: "elastalert_modules.slack_custom_alert.SlackCustomAlert" -alert_subject: "CSAD IT - Non Critical Instance Application Health Monitoring Alarm" -alert_text: "The Application {0} is unhealthy and reporting Downtime during {1}" -alert_text_type: alert_text_only -alert_text_args: ["instance-tag", "@timestamp"] - - -slack: -slack_webhook_url: "" +slack_webhook_url: "" slack_title: "CSAD IT - Non Critical Instance Application Health Monitoring Alarm" -slack_title_link: "https://elk.it.logitech.com:5601/app/kibana#/dashboard/35aa4fd0-b0b3-11ea-833d-fde9206e58f3"