From dd388aa8e7e0f7023d87ba847d9f35cc22ce2c6e Mon Sep 17 00:00:00 2001
From: Renato Monteiro <45536168+monteiro-renato@users.noreply.github.com>
Date: Fri, 17 Jan 2025 14:01:12 +0100
Subject: [PATCH] feat: create a dedicated pool for prometheus and beef up the
 instance

---
 .../altinn-monitor-test-rg/k6_tests_rg_k8s.tf | 19 +++++++++++++++++++
 ...ests_rg_kube_prometheus_stack_values.tftpl | 14 +++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_k8s.tf b/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_k8s.tf
index 83aab468..0301f4a9 100644
--- a/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_k8s.tf
+++ b/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_k8s.tf
@@ -54,3 +54,22 @@ resource "azurerm_kubernetes_cluster_node_pool" "spot" {
     "kubernetes.azure.com/scalesetpriority=spot:NoSchedule", # Automatically added by Azure
   ]
 }
+
+resource "azurerm_kubernetes_cluster_node_pool" "prometheus" {
+  name                  = "prometheus"
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.k6tests.id
+  vm_size               = "Standard_D3_v2"
+  auto_scaling_enabled  = false
+  node_count            = 1
+  priority              = "Spot" # Spot since we are still testing
+  eviction_policy       = "Delete"
+  spot_max_price        = -1 # (the current on-demand price for a Virtual Machine)
+  node_labels = {
+    "kubernetes.azure.com/scalesetpriority" : "spot", # Automatically added by Azure
+    prometheus : true
+  }
+  node_taints = [
+    "kubernetes.azure.com/scalesetpriority=spot:NoSchedule", # Automatically added by Azure
+    "workload=prometheus:NoSchedule",
+  ]
+}
diff --git a/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_kube_prometheus_stack_values.tftpl b/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_kube_prometheus_stack_values.tftpl
index 7ec82205..049a0537 100644
--- a/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_kube_prometheus_stack_values.tftpl
+++ b/infrastructure/adminservices-test/altinn-monitor-test-rg/k6_tests_rg_kube_prometheus_stack_values.tftpl
@@ -22,6 +22,18 @@ prometheus:
           cloud: "AzurePublic"
           sdk:
             tenantId: "${tenant_id}"
+    tolerations:
+      - key: "kubernetes.azure.com/scalesetpriority"
+        operator: "Equal"
+        value: "spot"
+        effect: "NoSchedule"
+      - key: "workload"
+        operator: "Equal"
+        value: "prometheus"
+        effect: "NoSchedule"
+    resources:
+      requests:
+        memory: 8Gi
     priorityClassName: "system-cluster-critical"
     retention: 1d
     storageSpec:
@@ -29,4 +41,4 @@ prometheus:
         spec:
           resources:
             requests:
-              storage: 5Gi
+              storage: 15Gi