[Feature/EV 37] Integrate Native Prometheus client in Ziggurat

.github/workflows/build.yml

@@ -2,7 +2,6 @@ name: Ziggurat CI
 
 on:
   push:
-    branches:
     tags:
   pull_request:
     branches:

.gitignore

@@ -17,3 +17,7 @@ pom.xml
 .factorypath
 .project
 .settings
+.clj-kondo/*
+.calva/*
+.lsp/*
+.vscode

project.clj

@@ -53,6 +53,7 @@
                  [ch.qos.logback.contrib/logback-json-classic "0.1.5"]
                  [ch.qos.logback.contrib/logback-jackson "0.1.5"]
                  [net.logstash.logback/logstash-logback-encoder "6.6"]
+                 [clj-commons/iapetos "0.1.9"]
                  [org.apache.commons/commons-pool2 "2.11.1"]]
   :deploy-repositories [["clojars" {:url           "https://clojars.org/repo"
                                     :username      :env/clojars_username

src/ziggurat/config.clj

@@ -46,6 +46,8 @@
               :http-server          {:middlewares  {:swagger {:enabled false}}
                                      :port         8080
                                      :thread-count 100}
+              :prometheus           {:port 8002
+                                     :enabled true}
               :new-relic            {:report-errors false}
               :log-format           "text"}})
 
@@ -96,6 +98,9 @@
 (defn rabbitmq-config []
   (get (ziggurat-config) :rabbit-mq))
 
+(defn prometheus-config []
+  (get config :prometheus))
+
 (defn statsd-config []
   (let [cfg (ziggurat-config)]
     (get cfg :statsd)))

src/ziggurat/metrics.clj

@@ -4,6 +4,8 @@
             [ziggurat.config :refer [statsd-config ziggurat-config]]
             [ziggurat.util.java-util :as util]
             [mount.core :refer [defstate]]
+            [mount.core :as mount]
+            [ziggurat.prometheus-exporter :as prometheus-exporter]
             [ziggurat.metrics-interface :as metrics-interface]
             [ziggurat.dropwizard-metrics-wrapper :refer [->DropwizardMetrics]])
   (:gen-class
@@ -35,9 +37,11 @@
 
 (defstate statsd-reporter
   :start (do (log/info "Initializing Metrics")
+             (prometheus-exporter/start-http-server)
              (initialise-metrics-library)
              (metrics-interface/initialize @metric-impl (statsd-config)))
   :stop (do (log/info "Terminating Metrics")
+            (prometheus-exporter/stop-http-server)
             (metrics-interface/terminate @metric-impl)))
 
 (defn intercalate-dot
@@ -93,6 +97,7 @@
          tags              (remove-topic-tag-for-old-namespace (get-all-tags (get-map additional-tags) metric-namespaces) metric-namespace)
          signed-int-value  (sign (get-int n))]
      (doseq [metric-ns metric-namespaces]
+       (prometheus-exporter/update-counter metric-ns metric tags signed-int-value)
        (metrics-interface/update-counter @metric-impl metric-ns metric tags signed-int-value)))))
 
 (def increment-count (partial inc-or-dec-count +))
@@ -112,6 +117,7 @@
          integer-value                  (get-int val)
          metric                          "all"]
      (doseq [metric-ns intercalated-metric-namespaces]
+       (prometheus-exporter/report-histogram metric-ns metric tags integer-value)
        (metrics-interface/update-timing @metric-impl metric-ns metric tags integer-value)))))
 
 (defn report-time

src/ziggurat/prometheus_exporter.clj

@@ -0,0 +1,47 @@
+(ns ziggurat.prometheus-exporter
+  (:require [iapetos.core :as prometheus]
+            [iapetos.standalone :as standalone]
+            [iapetos.registry :as reg]
+            [clojure.tools.logging :as log]
+            [ziggurat.config :refer [prometheus-config]]))
+
+(def registry
+  (atom (prometheus/collector-registry)))
+
+(defn get-metric-value [kw-metric labels]
+  (-> (@registry kw-metric labels) (prometheus/value)))
+
+(defn register-collecter-if-not-exist [collector metric-name label-vec]
+  (let [counter (@registry metric-name {:labels label-vec})]
+    (when-not counter
+      (let [new-counter (collector metric-name {:labels label-vec})
+            new-registry (prometheus/register @registry new-counter)]
+        (reset! registry new-registry)
+        new-counter))))
+
+(defn update-counter [namespace metric tags value]
+  (let [metric-name {:name metric :namespace namespace}
+        label-vec (vec (keys tags))]
+    (register-collecter-if-not-exist prometheus/gauge metric-name label-vec)
+    (prometheus/inc (@registry metric-name tags) value)
+    metric-name))
+
+(defn report-histogram [namespace metric tags integer-value]
+  (let [metric-name {:name metric :namespace namespace}
+        label-vec (vec (keys tags))]
+    (register-collecter-if-not-exist prometheus/histogram metric-name label-vec)
+    (prometheus/observe (@registry metric-name tags) integer-value)))
+
+(defonce httpd (atom nil))
+
+(defn start-http-server []
+  (when-not @httpd
+    (reset! httpd (standalone/metrics-server @registry
+                                             {:port
+                                              (get (prometheus-config)
+                                                   :port 8002)}))))
+
+(defn stop-http-server []
+  (when-let [server @httpd]
+    (.close server)
+    (reset! httpd nil)))

test/ziggurat/prometheus_exporter_test.clj

@@ -0,0 +1,83 @@
+(ns ziggurat.prometheus-exporter-test
+  (:require [clojure.test :refer [deftest is testing]]
+            [ziggurat.prometheus-exporter :as prometheus-exporter]
+            [iapetos.core :as prometheus]))
+
+(defn clear-prometheus-registry []
+  (reset! prometheus-exporter/registry (prometheus/collector-registry)))
+
+(deftest test-register-collecter-if-not-exist
+  (testing "Should register gauge if it does not exist"
+    (clear-prometheus-registry)
+    (let [tags {:actor-topic "food" :app_name "delivery"}
+          metric-name {:name "test-gauge" :namespace "bar"}
+          label-vec (vec (keys tags))]
+      (is (nil? (@prometheus-exporter/registry metric-name)) "Gauge should not exist initially")
+      (prometheus-exporter/register-collecter-if-not-exist prometheus/gauge metric-name label-vec)
+      (is (not (nil? (@prometheus-exporter/registry metric-name))) "Gauge should be registered in the registry"))
+
+    (testing "Gauge not registered if it exists"
+      (clear-prometheus-registry)
+      (let [tags {:actor-topic "food" :app_name "delivery"}
+            metric-name "test-cauge"
+            label-vec (vec (keys tags))
+            existing-cauge (prometheus-exporter/register-collecter-if-not-exist prometheus/gauge metric-name label-vec)]
+        (let [expected-nil (prometheus-exporter/register-collecter-if-not-exist prometheus/gauge metric-name label-vec)]
+          (is (not (nil? existing-cauge)))
+          (is (nil? expected-nil) "Should retrieve the existing cauge"))))))
+
+(deftest test-update-counter
+  (testing "Updating counter can increments its value"
+    (clear-prometheus-registry)
+    (let [namespace "test_namespace"
+          metric "test_metric"
+          old_value 5.0
+          increment 2
+          new-value (+ old_value increment)
+          tags {:some "label"}
+          _ (prometheus-exporter/update-counter namespace metric tags old_value)]
+      (prometheus-exporter/update-counter namespace metric tags increment)
+      (@prometheus-exporter/registry (assoc tags :name metric :namespace namespace))
+      (let [gauge-value (prometheus-exporter/get-metric-value (keyword (str namespace "/" metric)) tags)]
+        (is (= new-value gauge-value) "Gauge value should be incremented"))))
+
+  (testing "Updating counter can decrement its value"
+    (clear-prometheus-registry)
+    (let [namespace "test_namespace"
+          metric "test_metric"
+          old_value 5.0
+          decrement -2
+          new-value (+ old_value decrement)
+          tags {:some "label"}
+          _ (prometheus-exporter/update-counter namespace metric tags old_value)]
+      (prometheus-exporter/update-counter namespace metric tags decrement)
+      (@prometheus-exporter/registry (assoc tags :name metric :namespace namespace))
+      (let [gauge-value (prometheus-exporter/get-metric-value (keyword (str namespace "/" metric)) tags)]
+        (is (= new-value gauge-value) "Gauge value should be incremented")))))
+
+(deftest test-report-histogram
+  (testing "should register histogram when it does not exist"
+    (clear-prometheus-registry)
+    (let [namespace "test_namespace"
+          metric "test_metric"
+          value 5.0
+          tags {:some "label"}
+          label (assoc tags :name metric :namespace namespace)]
+      (is (nil? (@prometheus-exporter/registry label)))
+      (prometheus-exporter/report-histogram namespace metric tags value)
+      (is (not (nil? (@prometheus-exporter/registry label))))
+      (let [gauge-value (prometheus-exporter/get-metric-value (keyword (str namespace "/" metric)) tags)]
+        (is (= (gauge-value :sum) 5.0))
+        (is (= (gauge-value :count) 1.0)))))
+  (testing "should update histogram when it exist and increment the count"
+    (clear-prometheus-registry)
+    (let [namespace "test_namespace"
+          metric "test_metric"
+          value 5.0
+          tags {:some "label"}
+          label (assoc tags :name metric :namespace namespace)
+          _ (prometheus-exporter/report-histogram namespace metric tags value)]
+      (is (not (nil? (@prometheus-exporter/registry label))))
+      (prometheus-exporter/report-histogram namespace metric tags value)
+      (let [gauge-value (prometheus-exporter/get-metric-value (keyword (str namespace "/" metric)) tags)]
+        (is (= (gauge-value :count) 2.0))))))