Add columns to perf_metric table for check status

- Preparation for adding quick 'latest status' review method

Pipfile

@@ -8,7 +8,7 @@ verify_ssl = true
 [packages]
 apscheduler = "*"
 dba-metrics = {editable = true,path = "."}
-nerium = {editable = true,git = "https://github.com/tym-xqo/nerium",ref = "e2eb289"}
-# nerium = {editable = true, path = "../nerium"}
+nerium = {editable = true,git = "https://github.com/tym-xqo/nerium"}
+# nerium = {editable = true,path = "../nerium"}
 pg8000 = "*"
 slackclient = "*"

dba_metrics/__main__.py

@@ -33,7 +33,7 @@
 def get_metric(name, quiet=False):
     metric = get_result_set(name)
     metric.executed += "Z"
-    check_metric(metric)
+    metric = check_metric(metric)
     return metric
 
 
@@ -46,16 +46,43 @@ def print_metric(name):
 
 def store_metric(name):
     metric = get_metric(name)
+    # fmt: off
     sql = (
-        "insert into perf_metric (stamp, payload, name, host) "
-        "values (:stamp, :payload, :name, :host) "
+        "INSERT into perf_metric ("
+        "   stamp, "
+        "   payload, "
+        "   name, "
+        "   host, "
+        "   status, "
+        "   threshold_field, "
+        "   threshold_gate) "
+        "VALUES ("
+        "   :stamp, "
+        "   :payload, "
+        "   :name, "
+        "   :host, "
+        "   :status, "
+        "   :threshold_field, "
+        "   :threshold_gate) "
         "returning metric_id"
     )
+    # fmt: on
     for i in metric.result:
         stamp = metric.executed
         payload = json.dumps(i, default=str)
+        status = metric.status
+        threshold_field = metric.threshold["field"]
+        threshold_gate = str(metric.threshold["gate"])
+        print(stamp, payload, status, threshold_field, threshold_gate)
         insert = store_db.query(
-            sql, stamp=stamp, payload=payload, name=name, host=HOSTNAME
+            sql,
+            stamp=stamp,
+            payload=payload,
+            name=name,
+            host=HOSTNAME,
+            status=status,
+            threshold_field=threshold_field,
+            threshold_gate=threshold_gate,
         )
         return insert.export("json")
 
@@ -100,7 +127,10 @@ def create_table():
         "stamp timestamp with time zone, "
         "payload jsonb, "
         "name text, "
-        "host text)"
+        "host text, "
+        "status text, "
+        "threshold_field text, "
+        "threshold_gate int)"
     )
     store_db.query(sql)
 

dba_metrics/alert.py

@@ -16,7 +16,29 @@
 load_dotenv(find_dotenv(), override=override)
 
 HOSTNAME = os.getenv("HOSTNAME", "localhost")
-stopfile = Path("/tmp/dba-alert-pause")
+
+
+def swap_status(status):
+    """Toggle status between "clear" and "failure" on check status change
+    """
+    opts = cycle(["clear", "failure"])
+    new_status = next(opts)
+    if status == new_status:
+        new_status = next(opts)
+    return new_status
+
+
+def update_config(metric):
+    """Rewrite sql metadata with new status after change
+    """
+    sql = metric.body
+    metadata = metric.metadata
+    metadata_yaml = yaml.safe_dump(metadata)
+    metadata_block = f"---\n{metadata_yaml}---"
+    query_file = os.path.join("query_files", f"{metric.name}.sql")
+    with open(query_file, "w") as config_file:
+        new_content = "\n".join([metadata_block, sql, ""])
+        config_file.write(new_content)
 
 
 def send_alert(metric):
@@ -27,6 +49,7 @@ def send_alert(metric):
     threshold = metric.metadata["threshold"]["gate"]
     status = metric.metadata["status"]
     value = metric.value
+    alert = None
 
     format = get_format("print")
     full_metric = yaml.safe_dump(format.dump(metric).data)
@@ -42,65 +65,44 @@ def send_alert(metric):
     if status == "failure":
         color = "danger"
 
-    alert = slack_post(title=title, message=message, color=color)
-    return alert
-
+    stopfile = Path("/tmp/dba-alert-pause")
+    if not stopfile.exists() and status != "pause":
+        alert = slack_post(title=title, message=message, color=color)
+        update_config(metric)
 
-def swap_status(status):
-    """Toggle status between "clear" and "failure" on check status change
-    """
-    opts = cycle(["clear", "failure"])
-    new_status = next(opts)
-    if status == new_status:
-        new_status = next(opts)
-    return new_status
-
-
-def update_config(metric):
-    """Rewrite sql metadata with new status after change
-    """
-    sql = metric.body
-    metadata = metric.metadata
-    metadata_yaml = yaml.safe_dump(metadata)
-    metadata_block = f"---\n{metadata_yaml}---"
-    query_file = os.path.join("query_files", f"{metric.name}.sql")
-    with open(query_file, "w") as config_file:
-        new_content = "\n".join([metadata_block, sql, ""])
-        config_file.write(new_content)
+    metric.alert = alert
+    return metric
 
 
 def check_metric(metric):
     """Compare metric check value against threshold;
     Update status and send alert if comparsison triggers status change
     """
     # TODO: Support failure modes other than `> threshold`
-    alert = None
-    if stopfile.exists():
-        return alert
 
     try:
         data = metric.result
         status = metric.metadata["status"]
         check = metric.metadata["threshold"]["field"]
         threshold = metric.metadata["threshold"]["gate"]
     except (KeyError, AttributeError):
-        return alert
-
-    if status == "pause":
-        return alert
+        return metric
 
     if data:
         value = max([row[check] for row in data])
         metric.value = value
+
         test = value >= threshold
         if status == "failure":
             test = value < threshold
+
         if test:
             metric.metadata["status"] = swap_status(status)
-            alert = send_alert(metric)
-            update_config(metric)
+            metric = send_alert(metric)
 
-    return alert
+    metric.status = metric.metadata["status"]
+    metric.threshold = metric.metadata["threshold"]
+    return metric
 
 
 if __name__ == "__main__":

format_files/print.py

@@ -3,6 +3,8 @@
 
 
 class ResultSchema(Schema):
-    name = fields.Str(attribute="name")
-    data = fields.List(fields.Dict(), attribute="result")
+    name = fields.Str()
+    result = fields.List(fields.Dict())
+    status = fields.Str()
+    threshold = fields.Dict()
     stamp = fields.Str(attribute="executed")

format_files/store.py

@@ -3,6 +3,8 @@
 
 
 class ResultSchema(Schema):
-    name = fields.Str(attribute="name")
+    name = fields.Str()
     payload = fields.List(fields.Dict(), attribute="result")
+    status = fields.Str()
+    threshold = fields.Dict()
     stamp = fields.Str(attribute="executed")