feat(o11y): collect Postgres metrics from staging, and prod

Referenced `Secret`s have not been committed, because we don't yet use a secret encryption tool like `sops`, or `git-crypt`. Signed-off-by: Siddhesh Mhadnak <[email protected]>
vipyrsec · Sep 1, 2024 · 9d18d81 · 9d18d81
1 parent e4f6d7b
commit 9d18d81
Show file tree

Hide file tree

Showing 5 changed files with 95 additions and 4 deletions.
diff --git a/kubernetes/chart/values/o11y/vipyrsec.yaml b/kubernetes/chart/values/o11y/vipyrsec.yaml
@@ -10,6 +10,15 @@ ingress-nginx:
 loki:
   enabled: true
 
+  # annotations:
+  #   cert-manager.io/cluster-issuer: letsencrypt
+
+  # ingress:
+  #   tls:
+  #     - hosts:
+  #         - loki.vipyrsec.com
+  #       secretName: loki-tls
+
 alloy:
   enabled: true
 
@@ -80,6 +89,24 @@ alloy:
 
         prometheus.exporter.cadvisor "local" { }
 
+        remote.kubernetes.secret "postgres_staging" {
+        	namespace = "postgres"
+        	name      = "staging"
+        }
+
+        prometheus.exporter.postgres "staging" {
+        	data_source_names = [remote.kubernetes.secret.postgres_staging.data.dsn]
+        }
+
+        remote.kubernetes.secret "postgres_prod" {
+        	namespace = "postgres"
+        	name      = "prod"
+        }
+
+        prometheus.exporter.postgres "prod" {
+        	data_source_names = [remote.kubernetes.secret.postgres_prod.data.dsn]
+        }
+
         prometheus.scrape "self" {
         	targets    = prometheus.exporter.self.local.targets
         	forward_to = [prometheus.relabel.tagged.receiver]
@@ -100,6 +127,36 @@ alloy:
         	forward_to = [prometheus.relabel.tagged.receiver]
         }
 
+        prometheus.scrape "postgres_staging" {
+        	targets    = prometheus.exporter.postgres.staging.targets
+        	forward_to = [prometheus.relabel.tagged_staging.receiver]
+        }
+
+        prometheus.scrape "postgres_staging_do" {
+        	targets    = [{"__address__" = remote.kubernetes.secret.postgres_staging.data.metrics_url}]
+        	forward_to = [prometheus.relabel.tagged_staging.receiver]
+
+        	basic_auth {
+        		username = remote.kubernetes.secret.postgres_staging.data.metrics_username
+        		password = remote.kubernetes.secret.postgres_staging.data.metrics_password
+        	}
+        }
+
+        prometheus.scrape "postgres_prod" {
+        	targets    = prometheus.exporter.postgres.prod.targets
+        	forward_to = [prometheus.relabel.tagged_prod.receiver]
+        }
+
+        prometheus.scrape "postgres_prod_do" {
+        	targets    = [{"__address__" = remote.kubernetes.secret.postgres_prod.data.metrics_url}]
+        	forward_to = [prometheus.relabel.tagged_prod.receiver]
+
+        	basic_auth {
+        		username = remote.kubernetes.secret.postgres_prod.data.metrics_username
+        		password = remote.kubernetes.secret.postgres_prod.data.metrics_password
+        	}
+        }
+
         prometheus.relabel "tagged" {
         	rule {
         		source_labels = ["__address__"]
@@ -111,6 +168,28 @@ alloy:
         	forward_to = [prometheus.remote_write.o11y.receiver]
         }
 
+        prometheus.relabel "tagged_staging" {
+        	rule {
+        		source_labels = ["__address__"]
+        		regex         = ".*"
+        		replacement   = "staging"
+        		target_label  = "cluster"
+        	}
+
+        	forward_to = [prometheus.remote_write.o11y.receiver]
+        }
+
+        prometheus.relabel "tagged_prod" {
+        	rule {
+        		source_labels = ["__address__"]
+        		regex         = ".*"
+        		replacement   = "prod"
+        		target_label  = "cluster"
+        	}
+
+        	forward_to = [prometheus.remote_write.o11y.receiver]
+        }
+
         prometheus.remote_write "o11y" {
         	endpoint {
         		url = "http://prometheus.prometheus.svc.cluster.local:9090/api/v1/write"

diff --git a/kubernetes/chart/values/prod/vipyrsec.yaml b/kubernetes/chart/values/prod/vipyrsec.yaml
@@ -59,7 +59,7 @@ alloy:
 
         loki.write "o11y" {
         	endpoint {
-        		url = "http://loki.vipyrsec.com/loki/api/v1/push"
+        		url = "http://loki.vipyrsec.com:30080/loki/api/v1/push"
         	}
         }
 
@@ -102,6 +102,6 @@ alloy:
 
         prometheus.remote_write "o11y" {
         	endpoint {
-        		url = "http://prom.vipyrsec.com/api/v1/write"
+        		url = "http://prom.vipyrsec.com:30080/api/v1/write"
         	}
         }
diff --git a/kubernetes/chart/values/staging/vipyrsec.yaml b/kubernetes/chart/values/staging/vipyrsec.yaml
@@ -59,7 +59,7 @@ alloy:
 
         loki.write "o11y" {
         	endpoint {
-        		url = "http://loki.vipyrsec.com/loki/api/v1/push"
+        		url = "http://loki.vipyrsec.com:30080/loki/api/v1/push"
         	}
         }
 
@@ -102,6 +102,6 @@ alloy:
 
         prometheus.remote_write "o11y" {
         	endpoint {
-        		url = "http://prom.vipyrsec.com/api/v1/write"
+        		url = "http://prom.vipyrsec.com:30080/api/v1/write"
         	}
         }
diff --git a/kubernetes/manifests/postgres/README.md b/kubernetes/manifests/postgres/README.md
@@ -0,0 +1,6 @@
+# postgres
+
+Manifests for Postgres.
+
+> [!NOTE]
+> The `Secret`s have not been committed, because we don't yet use a secret encryption tool like `sops`, or `git-crypt`.
diff --git a/kubernetes/manifests/postgres/namespace.yaml b/kubernetes/manifests/postgres/namespace.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: Namespace
+
+metadata:
+  name: postgres