Merge pull request #12 from particuleio/feat/monitoring

feat: improve monitoring stack
particuleio · Jan 5, 2021 · 72873f8 · 72873f8
2 parents 36baa82 + d1b59de
commit 72873f8
Show file tree

Hide file tree

Showing 7 changed files with 220 additions and 26 deletions.
diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml
@@ -27,6 +27,7 @@ jobs:
         VALIDATE_STATES: false
         VALIDATE_KUBERNETES_KUBEVAL: false
         VALIDATE_TERRAFORM_TERRASCAN: false
+        VALIDATE_TERRAFORM: false
     - name: 'slack:failure'
       if: failure()
       env:

diff --git a/karma.tf b/karma.tf
@@ -27,7 +27,8 @@ resource "kubernetes_namespace" "karma" {
 
   metadata {
     labels = {
-      name = local.karma["namespace"]
+      name                               = local.karma["namespace"]
+      "${local.labels_prefix}/component" = "monitoring"
     }
 
     name = local.karma["namespace"]

diff --git a/kube-prometheus.tf b/kube-prometheus.tf
@@ -7,7 +7,7 @@ locals {
       chart                  = "kube-prometheus-stack"
       repository             = "https://prometheus-community.github.io/helm-charts"
       enabled                = false
-      chart_version          = "12.8.1"
+      chart_version          = "12.10.4"
       allowed_cidrs          = ["0.0.0.0/0"]
       default_network_policy = true
     },
@@ -31,25 +31,42 @@ grafana:
         editable: true
         options:
           path: /var/lib/grafana/dashboards/default
+prometheus-node-exporter:
+  priorityClassName: ${local.priority-class-ds["create"] ? kubernetes_priority_class.kubernetes_addons_ds[0].metadata[0].name : ""}
+prometheus:
+  prometheusSpec:
+    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
+alertmanager:
+  alertmanagerSpec:
+    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
+VALUES
+
+  values_dashboard_kong = <<VALUES
+grafana:
   dashboards:
     default:
       kong-dash:
         gnetId: 7424
         revision: 6
         datasource: Prometheus
+VALUES
+
+  values_dashboard_ingress-nginx = <<VALUES
+grafana:
+  dashboards:
+    default:
       nginx-ingress:
         url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/nginx.json
-      cluster-autoscaler:
-        gnetId: 3831
+VALUES
+
+  values_dashboard_cert-manager = <<VALUES
+grafana:
+  dashboards:
+    default:
+      cert-manager:
+        gnetId: 11001
+        revision: 1
         datasource: Prometheus
-prometheus-node-exporter:
-  priorityClassName: ${local.priority-class-ds["create"] ? kubernetes_priority_class.kubernetes_addons_ds[0].metadata[0].name : ""}
-prometheus:
-  prometheusSpec:
-    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
-alertmanager:
-  alertmanagerSpec:
-    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
 VALUES
 }
 
@@ -94,10 +111,13 @@ resource "helm_release" "kube-prometheus-stack" {
   reuse_values          = local.kube-prometheus-stack["reuse_values"]
   skip_crds             = local.kube-prometheus-stack["skip_crds"]
   verify                = local.kube-prometheus-stack["verify"]
-  values = [
+  values = compact([
     local.values_kube-prometheus-stack,
-    local.kube-prometheus-stack["extra_values"]
-  ]
+    local.kube-prometheus-stack["extra_values"],
+    local.kong["enabled"] ? local.values_dashboard_kong : null,
+    local.cert-manager["enabled"] ? local.values_dashboard_cert-manager : null,
+    local.ingress-nginx["enabled"] ? local.values_dashboard_ingress-nginx : null
+  ])
   namespace = kubernetes_namespace.kube-prometheus-stack.*.metadata.0.name[count.index]
 }
 

diff --git a/loki-stack.tf b/loki-stack.tf
@@ -0,0 +1,135 @@
+locals {
+  loki-stack = merge(
+    local.helm_defaults,
+    {
+      name                   = "loki-stack"
+      namespace              = "monitoring"
+      chart                  = "loki-stack"
+      repository             = "https://grafana.github.io/helm-charts"
+      create_ns              = false
+      enabled                = false
+      chart_version          = "2.3.1"
+      default_network_policy = true
+    },
+    var.loki-stack
+  )
+
+  values_loki-stack = <<VALUES
+loki:
+  priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
+promtail:
+    priorityClassName: ${local.priority-class-ds["create"] ? kubernetes_priority_class.kubernetes_addons_ds[0].metadata[0].name : ""}
+VALUES
+}
+
+resource "kubernetes_namespace" "loki-stack" {
+  count = local.loki-stack["enabled"] && local.loki-stack["create_ns"] ? 1 : 0
+
+  metadata {
+    labels = {
+      name                               = local.loki-stack["namespace"]
+      "${local.labels_prefix}/component" = "monitoring"
+    }
+
+    name = local.loki-stack["namespace"]
+  }
+}
+
+resource "helm_release" "loki-stack" {
+  count                 = local.loki-stack["enabled"] ? 1 : 0
+  repository            = local.loki-stack["repository"]
+  name                  = local.loki-stack["name"]
+  chart                 = local.loki-stack["chart"]
+  version               = local.loki-stack["chart_version"]
+  timeout               = local.loki-stack["timeout"]
+  force_update          = local.loki-stack["force_update"]
+  recreate_pods         = local.loki-stack["recreate_pods"]
+  wait                  = local.loki-stack["wait"]
+  atomic                = local.loki-stack["atomic"]
+  cleanup_on_fail       = local.loki-stack["cleanup_on_fail"]
+  dependency_update     = local.loki-stack["dependency_update"]
+  disable_crd_hooks     = local.loki-stack["disable_crd_hooks"]
+  disable_webhooks      = local.loki-stack["disable_webhooks"]
+  render_subchart_notes = local.loki-stack["render_subchart_notes"]
+  replace               = local.loki-stack["replace"]
+  reset_values          = local.loki-stack["reset_values"]
+  reuse_values          = local.loki-stack["reuse_values"]
+  skip_crds             = local.loki-stack["skip_crds"]
+  verify                = local.loki-stack["verify"]
+  values = [
+    local.values_loki-stack,
+    local.loki-stack["extra_values"]
+  ]
+  namespace = local.loki-stack["create_ns"] ? kubernetes_namespace.loki-stack.*.metadata.0.name[count.index] : local.loki-stack["namespace"]
+
+  depends_on = [
+    helm_release.kube-prometheus-stack
+  ]
+}
+
+resource "kubernetes_network_policy" "loki-stack_default_deny" {
+  count = local.loki-stack["create_ns"] && local.loki-stack["enabled"] && local.loki-stack["default_network_policy"] ? 1 : 0
+
+  metadata {
+    name      = "${kubernetes_namespace.loki-stack.*.metadata.0.name[count.index]}-default-deny"
+    namespace = kubernetes_namespace.loki-stack.*.metadata.0.name[count.index]
+  }
+
+  spec {
+    pod_selector {
+    }
+    policy_types = ["Ingress"]
+  }
+}
+
+resource "kubernetes_network_policy" "loki-stack_allow_namespace" {
+  count = local.loki-stack["create_ns"] && local.loki-stack["enabled"] && local.loki-stack["default_network_policy"] ? 1 : 0
+
+  metadata {
+    name      = "${kubernetes_namespace.loki-stack.*.metadata.0.name[count.index]}-allow-namespace"
+    namespace = kubernetes_namespace.loki-stack.*.metadata.0.name[count.index]
+  }
+
+  spec {
+    pod_selector {
+    }
+
+    ingress {
+      from {
+        namespace_selector {
+          match_labels = {
+            name = kubernetes_namespace.loki-stack.*.metadata.0.name[count.index]
+          }
+        }
+      }
+    }
+
+    policy_types = ["Ingress"]
+  }
+}
+
+resource "kubernetes_network_policy" "loki-stack_allow_ingress" {
+  count = local.loki-stack["create_ns"] && local.loki-stack["enabled"] && local.loki-stack["default_network_policy"] ? 1 : 0
+
+  metadata {
+    name      = "${kubernetes_namespace.loki-stack.*.metadata.0.name[count.index]}-allow-ingress"
+    namespace = kubernetes_namespace.loki-stack.*.metadata.0.name[count.index]
+  }
+
+  spec {
+    pod_selector {
+    }
+
+    ingress {
+      from {
+        namespace_selector {
+          match_labels = {
+            "${local.labels_prefix}/component" = "ingress"
+          }
+        }
+      }
+    }
+
+    policy_types = ["Ingress"]
+  }
+}
diff --git a/modules/aws/kube-prometheus.tf b/modules/aws/kube-prometheus.tf
@@ -46,26 +46,52 @@ grafana:
         editable: true
         options:
           path: /var/lib/grafana/dashboards/default
+prometheus-node-exporter:
+  priorityClassName: ${local.priority-class-ds["create"] ? kubernetes_priority_class.kubernetes_addons_ds[0].metadata[0].name : ""}
+prometheus:
+  prometheusSpec:
+    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
+alertmanager:
+  alertmanagerSpec:
+    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
+VALUES
+
+  values_dashboard_kong = <<VALUES
+grafana:
   dashboards:
     default:
       kong-dash:
         gnetId: 7424
         revision: 6
         datasource: Prometheus
+VALUES
+
+  values_dashboard_ingress-nginx = <<VALUES
+grafana:
+  dashboards:
+    default:
       nginx-ingress:
         url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/nginx.json
+VALUES
+
+  values_dashboard_cluster-autoscaler = <<VALUES
+grafana:
+  dashboards:
+    default:
       cluster-autoscaler:
         gnetId: 3831
         revision: 1
         datasource: Prometheus
-prometheus-node-exporter:
-  priorityClassName: ${local.priority-class-ds["create"] ? kubernetes_priority_class.kubernetes_addons_ds[0].metadata[0].name : ""}
-prometheus:
-  prometheusSpec:
-    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
-alertmanager:
-  alertmanagerSpec:
-    priorityClassName: ${local.priority-class["create"] ? kubernetes_priority_class.kubernetes_addons[0].metadata[0].name : ""}
+VALUES
+
+  values_dashboard_cert-manager = <<VALUES
+grafana:
+  dashboards:
+    default:
+      cert-manager:
+        gnetId: 11001
+        revision: 1
+        datasource: Prometheus
 VALUES
 }
 
@@ -184,10 +210,14 @@ resource "helm_release" "kube-prometheus-stack" {
   reuse_values          = local.kube-prometheus-stack["reuse_values"]
   skip_crds             = local.kube-prometheus-stack["skip_crds"]
   verify                = local.kube-prometheus-stack["verify"]
-  values = [
+  values = compact([
     local.values_kube-prometheus-stack,
-    local.kube-prometheus-stack["extra_values"]
-  ]
+    local.kube-prometheus-stack["extra_values"],
+    local.kong["enabled"] ? local.values_dashboard_kong : null,
+    local.cert-manager["enabled"] ? local.values_dashboard_cert-manager : null,
+    local.cluster-autoscaler["enabled"] ? local.values_dashboard_cluster-autoscaler : null,
+    local.ingress-nginx["enabled"] ? local.values_dashboard_ingress-nginx : null
+  ])
   namespace = kubernetes_namespace.kube-prometheus-stack.*.metadata.0.name[count.index]
 }
 

diff --git a/modules/aws/loki-stack.tf b/modules/aws/loki-stack.tf
@@ -0,0 +1 @@
+../../loki-stack.tf
diff --git a/variables.tf b/variables.tf
@@ -70,6 +70,12 @@ variable "labels_prefix" {
   default     = "particule.io"
 }
 
+variable "loki-stack" {
+  description = "Customize loki-stack chart, see `loki-stack.tf` for supported values"
+  type        = any
+  default     = {}
+}
+
 variable "metrics-server" {
   description = "Customize metrics-server chart, see `metrics_server.tf` for supported values"
   type        = any