feat(blackbox-exporter): setup

tyriis · Feb 23, 2025 · abb1efa · abb1efa
1 parent 008c322
commit abb1efa
Show file tree

Hide file tree

Showing 10 changed files with 326 additions and 0 deletions.
diff --git a/kubernetes/talos-flux/apps/atlantis/techtales-io/flux-sync.yaml b/kubernetes/talos-flux/apps/atlantis/techtales-io/flux-sync.yaml
@@ -0,0 +1,30 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &appname atlantis-techtales-io-terraform-discord
+  namespace: flux-system
+  labels:
+    substitution.flux.home.arpa/enabled: "true"
+spec:
+  targetNamespace: atlantis
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *appname
+  path: ./kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  prune: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+  dependsOn:
+    - name: apps-cert-manager
+    - name: apps-rook-ceph-cluster
+    - name: apps-traefik-forward-auth
+  postBuild:
+    substitute:
+      APP: *appname
diff --git a/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/config/allowlist.txt b/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/config/allowlist.txt
@@ -0,0 +1,3 @@
+tyriis
+jazzlyn
+techtales-bot[bot]
diff --git a/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/helm-values.yaml b/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/helm-values.yaml
@@ -0,0 +1,28 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: ${APP}
+spec:
+  interval: 15m
+  # https://artifacthub.io/packages/helm/cert-manager/cert-manager?modal=values
+  values:
+    controllers:
+      main:
+        containers:
+          app:
+            envFrom:
+              - secretRef:
+                  name: atlantis-techtales-env-secrets
+            env:
+              ATLANTIS_REPO_ALLOWLIST: github.com/techtales-io/terraform-discord
+    persistence:
+      allowlist:
+        type: configMap
+        name: atlantis-tyriis-allowlist
+        advancedMounts:
+          main:
+            app:
+              - path: /etc/atlantis/allowlist.txt
+                subPath: allowlist.txt
+                readOnly: true
diff --git a/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/kustomization.yaml b/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/kustomization.yaml
@@ -0,0 +1,17 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./secret.sops.yaml
+  - ../../../../../base/apps/atlantis/app/helm-release.yaml
+configMapGenerator:
+  - name: atlantis-tyriis-allowlist
+    files:
+      - allowlist.txt=config/allowlist.txt
+generatorOptions:
+  disableNameSuffixHash: true
+  annotations:
+    kustomize.toolkit.fluxcd.io/substitute: disabled
+patches:
+  - path: helm-values.yaml
diff --git a/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/secret.sops.yaml b/kubernetes/talos-flux/apps/atlantis/techtales-io/terraform-discord/secret.sops.yaml
@@ -0,0 +1,38 @@
+# yamllint disable
+apiVersion: v1
+kind: Secret
+metadata:
+    name: atlantis-env-secrets
+data:
+    ATLANTIS_GH_APP_ID: ENC[AES256_GCM,data:GufXvOx+vEE=,iv:x9LyCgvwA3CHyYiPez2ZXGT+znUwXHOyfuH2nTRfC5U=,tag:D03qbedHrjo5BReui4kKAA==,type:str]
+    ATLANTIS_GH_APP_SLUG: ENC[AES256_GCM,data:DXHqdxJeCUFKr39nR7NAJhLuqj2jAbc+,iv:oxL8DsATJaH/1QOkvMwiNhDmHiS3dDTp9JNQKRhyYvE=,tag:i9jNmW2BV3OIuYoQhV8EZw==,type:str]
+    ATLANTIS_GH_APP_KEY: ENC[AES256_GCM,data:SMMHHO80ABIsiCW57CrelKI2DFgDmIbKyseX9PFQg1ds0X3BTQe97W4rLIBVqtYGYRzgfdFMrhztGnPbbNiK4Tyzh4+PET6x4sFjbxVT/U5YvxPm63jOuVOZZeT4QuOQ3owQpzISJ0cOKbj3bcgnaDBxINkeViFEMlMoI1BBGpRA/RcHa4BN558/j3y4rP2BiN/wGhPGTEgLMEbNB46Nl13U3zm6zESUPOSu3HZGwvfN7qq+Yt93ch9HTaO4RIIRq0jlGZe2Z938G3dpHOrPILe90RNbU/4GTTRSxbbSnV583KQn9zvU9FyJSGKsc07ANNkt6VFsEgTh/wftp6PXFKfv+UvFBYPj0nFYm7Iy29eNm0UhrRg8KOUVW4PQtTLFDBUkzaxkqu8cSgNJrkYE0Gs/bw6fQTXUU/RWY9jXvv/oXE2EXbBXhFr2EoIuUUsTFY2e6B/veMRJZo2tkfY/+ZeoVDcWtueBh3IX4d1DlfYSyRdfDSoUpYhibd4Ab7Ppf6UgZHDI9jfCmElg9wKkOlp+3zQaltRi6klA/YEg55O07temYFMtrxbhxtSurzdfHBbwGK/ughXYr6qD5tUGIzP8gwJjJouHzR0Fk8oST99a+VGpqfYAvUd07HlwSnObtDzamyiDiALpYjDSghxkvEKpRIEArxaKqsJMtoBqJ85WKJMCe1jUul4PRBuRFrvnZnW0UZ1tZijWvZ4NVqn91470crleKC9x0hcqj/KEzyDI7+fhjHigFEVMgaGWMEuHv8vc/woi//oAcMBO4SIH5Wzj6ZTngAkS+f2EP6ASQWaGSv84hc7po9yY5jm5C9VfD2bZytnvBfN/obG0TaM33bY9kKXDA2pPqTp0qmwSqJXrvCuwuMuJKfxCTN6MsX4gxaDO4PVLyslG9Haj6UCn1n6EFhGkLEnLtLhyToUb9Xc5/q306e7msRYDvaeZuPgGv8MW3e/II9KVSqbzWBI3CZkPjdIKi8r+5ivrO11wWX8BHMpvyGC5J5A/Ue2Tu4BFCb4noY93d6HFYCHbSIjljpDANFqx5I0orEapirBUuGosvdDvgluQgpHLJYkXCVVtGTEBng7qpvq+kvgvgN05LmKWBdaU1r4kI9L/tv9irUHF6iHM50g/vJYlz4Om4IABvMuqpI45x+RlPrVZCXq16yDdcrvHEbkC2opwWzrMdc0AnQtUKNLKh6KNpKTVmkhIJQLM06vo3P2iDceNWw+SAOg4xdZDgmljUttDay8XxrQT9gOFLckJl3yFeNNq5E6geL3PnotYcrkitxrkn5xwlwdKUHZv9811ojDmhmUUCO2ZfpY5elVnngkIvqOVqPJfiINnIlxaL/oEupxf1aCshtGjARz0PPXzjNEeRjxxfiYg6TmKaRvlN0/ZarEJUKuy45bfb2AswQr5egb1s82Z8awBqao7ij0s0ckIILTq1WIaETyqel+HsNSivhEAEC3//sSNDDo06R9rbVn5599GD/bo+KfRaHuNO8pDlLNZ8vzW5VVomH9iynBCFC8OrYuOHl3XNSY7Q1DByVa+adeaPSa/K+NDM0Yd4wuBtHlY3yZGTM3VEY/eRi+7MUWm6eGrk7POWzNu19jgqSj/GQPRH4Ct3Lvl0DxFoHH31nJ1Lgj/DN0x6E55nt54zFHw/htc3vsoDwaCJoZoIwJziQDglKVV+g8P9OOWb0hRV/THotovjEHkdxrf05aWXxcNMsBYftCz/BdgKyKx/NDauiACGm5WGCVHxJQxSeeKuYMw2sKy/AdhSUZKKTQ8p2VpKakpV/gJeHsb0WTHL8yoWFZZ3mHCXcMF/KlhApibG0ETL+bSbEWihcnBxW/+dI1ZQUJgO0zRWl3cFogcCtj0EoL9/H6inMi1LdcTDr2ewJRrz9yO8yZk9DtS9wIJ7CQ3tWrxiQ/cFs3WFzktoYbcBa3rBAOF4DIOSjUaxggMydQObm15G+6TeqYUmK5kfkqdS2kiFOGjAnTuj0xacJZljR6xYiqn1NhjbsficnG9vgUuS4Xseug897CK9P9e8NnC1YgPt9Q1MvJ/PsOXfvwbIJnkXx98kFGbXnH2cljb+czplsZVPhoOCpkB4UqyghKshGvugCX1obJaLa+w3EbcjvX+45S7T7UOQMhNh9fUioGkRyFt6I+EG7sHi87j/Z/4FfdujPRU3IyW2c3F7mTUBqymqhMSt/8MLjdY30RjgoOgUlGWjnqGVMGcr9h/51nchDPwi7up8fOFsM1KEH6EEoo9gLg0D3KYFchIIzedxzwjT48wsTdrf6w48ANYvo9+Yx3kVjthtxLn01LHiUGM6DF31/iey5yEdTP7CQ4ha9tbBYxS0V+q5e/Ioo0/j4vvL6yD8hpxU8/QfglpajQUtrDXW2pfXm8aQ41+qdKITiLI3YSZhaCmuweWmCLcyQbPtljg9saSxv60YoTNum1E6ffuS65v2EuF6x/w7mf20C0+yerNsCXvCy5D1eOLvE613+W9dAoacmImOKLB/r3JsDgquks4+7EO1e5z9Yp7ykjriJ9jvvbxK4DKzkR32t1CROhOmffH0mDfmwBR62NKYbApxMHmwvssxdtrq1L3iPPZPx7g8BK4EU6nfkXtuGHhe56+5bOFMhugjrpFaT/RZ1tbz8wQPFmHmiu2Oed7CDRL732V95Iwl1hzjdRLYcCa6WW9uY5EoIRSFiK8dYxRHcp9bKWJBRHJ5u6Zz7VRXNA2GzD9ycnGa4qMQ9pKAihpBc12qK/v59p/zlajah39srxGAwDfrfEUv5nwK2EBVxxuZzfPtcnrCKzX7YUi6/VW2ZR3isXKgcpjeaYWe1q3zZYo+cO+AHUpLxdJpK2PcfEYf4Pxi49t+E0wIGrzAs1lO5nozm/lpVy5FYyNsBFPOsZWaeC4KdhPlip6Wp1LqP+qRB9O3WFt6IIoeHqoNofl9eP3e+fOonsa4d+gn5iUnc9wV/9B4cogA3bX,iv:Ce3JZrrqJRaO5pShXs47v0UD/8aLSVgJEodFEguijVE=,tag:qHb1ZiWrF/6tRoQQ9hKEhw==,type:str]
+    ATLANTIS_GH_WEBHOOK_SECRET: ENC[AES256_GCM,data:rAkvyD/Tw0hffZJoQalmzwIhHGml+PShppYlzIOOTXeA8sfAut/EbWe3SfzNqP3A7+wGwI/YNm8H0HBfUx9wJdyPvPZTaweIDeLBC1p5p9hPA67CXnxzhFObmxpB3hSmpqoKOMBYmHIEe8pFEMcwWjd7Z/xNZkK0,iv:rjW4cdCvH8/pdVK0zclUdQzI8QAzqiIcjWGIWuh1/To=,tag:Fc5TBD2v1LC/MV2K2ML2pA==,type:str]
+    AWS_REGION: ENC[AES256_GCM,data:14lUzj9bbUg=,iv:jpjCdNg0+T8r13Hf8GTqXT/xPSnAVO1BOuyLxwg+uKg=,tag:INen0KguFQdglbhpEsWd+A==,type:str]
+    AWS_ENDPOINT_URL_S3: ENC[AES256_GCM,data:EjZGwSTs2Jz7YSX9HesKifyuTMe29E8L2SKjDpJtRurFx7Gz,iv:rKse1VWor9+8UWOQZ9jrgxCqfHa7W+gSgQ62Q/UakDQ=,tag:N5fF0s6cJv/aIBB3gMgeew==,type:str]
+    AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:9wElnRZNK64LYvqwM2tV3g==,iv:1IxHt8TxcqUTamEGRgK85lOOUDjBZmvPxtfA1+hwpBE=,tag:tBM5LUOpuvO5GRJqujeT6A==,type:str]
+    AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:b1w0xPoZIkMV3ZqTzc4qmQ==,iv:OHLrSDC2FK1ic5SUTi/vIVJaref2l+2y0pPAgiV0aQo=,tag:gn/j4Q1BrcoD3kCrJJiMHw==,type:str]
+    MINIO_ENDPOINT: ENC[AES256_GCM,data:XxPVeleTvyk1Qu9VcwmVYLfPUyxc7DSf,iv:9TNIJQzXeeK+e+98q5Gi1uO+vB84rl8vwgCXqXeAErM=,tag:/zOyrHZA/fyoleg0sz/vBA==,type:str]
+    MINIO_USER: ENC[AES256_GCM,data:SnaUKEVL5ajcDM0s+3AbdQ==,iv:544EGGKzVLATG53mHPa1n8O6XuzcXggzQALfQRszAOs=,tag:hNOgKSSIbaL6GAFdv9Dt5Q==,type:str]
+    MINIO_PASSWORD: ENC[AES256_GCM,data:GY1tNw4OMM0bR8cRJ/bsSA==,iv:9MjKQiRagzJugUCMHGXRdxYl30rI72a/XhMNUN2/eNY=,tag:UD9aL01eyCWHw2OqdRhidw==,type:str]
+    MINIO_ENABLE_HTTPS: ENC[AES256_GCM,data:D8uryT0cQOc=,iv:vRW/31c5cYSom7H3QSbmCSSh/MqMNtEkbpkOOCNU0Bo=,tag:HZ57y3LHdTTAcMgR1d5Txg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age16zqeqx5y6ay3flwz0d06rn83yjv9ckys3j8tpkysf9v6295fhc6sf4r0uj
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrVWVvYk83ZlFTaXFzc3U5
+            SnNTOG9pWWJmVGk0cm1TNWphcitCZlJ5WkZ3CkdZUWVYU1h5UlVqaUU3S0ViMDBr
+            Nlh2NW41b3QrM0pneHEwVWFLMWNLNlkKLS0tIE1xRk1vUGdkOENDZUNyQUNrTHI2
+            OHdsbHVkZm1tcXBjd1VYOGFBQ3dtTncKuDTuAZHhk9MfYwr1nCRMMnLjbteMxRVU
+            9jhkhN7YMywhLebbL7FhGolgNZ6vbD7jIGfp0iqO35KuKVvE/fhL8g==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-11-02T13:53:34Z"
+    mac: ENC[AES256_GCM,data:1xqC2nekZ+cMiSvdEvnuLZp4K/gdkICc6UFjgfuAQ/7enVRw3XDX3bXh3hi566dhRQZsqeD6XiRxuLATE2WEycT1zRrBQJgv5WXxzKHeG3MpxM4gdS5XcsO54bQXjCFuBPqMwEOQMCKN+8O37OpUAv0gG63L/Vlxw8xszvKbIoY=,iv:JeX5KEjrrV4lcItJ8iqZbmdAmVaEyzFbsUeR8HnoDjo=,tag:H8+/trVf9ok1j+B3D/uUlw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.0
diff --git a/kubernetes/talos-flux/apps/atlantis/tyriis/terraform-github/external-secret.yaml b/kubernetes/talos-flux/apps/atlantis/tyriis/terraform-github/external-secret.yaml
@@ -0,0 +1,51 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: &name atlantis-age-keys
+spec:
+  refreshInterval: 1m
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: *name
+    creationPolicy: Owner
+  data:
+    - secretKey: terraform-gcloud.txt
+      remoteRef:
+        key: infra/techtales/terraform-gcloud
+        property: age
+    - secretKey: terraform-github.txt
+      remoteRef:
+        key: infra/techtales/terraform-github
+        property: age
+    - secretKey: terraform-gworkspace.txt
+      remoteRef:
+        key: infra/techtales/terraform-gworkspace
+        property: age
+    - secretKey: terraform-vault.txt
+      remoteRef:
+        key: infra/techtales/terraform-vault
+        property: age
+
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: &name atlantis-github-token
+spec:
+  refreshInterval: 1m
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: *name
+    creationPolicy: Owner
+  data:
+    - secretKey: GITHUB_TOKEN
+      remoteRef:
+        key: infra/techtales/github-automation
+        property: GITHUB_TOKEN
diff --git a/kubernetes/talos-flux/apps/observability/blackbox-exporter/app/helm-release.yaml b/kubernetes/talos-flux/apps/observability/blackbox-exporter/app/helm-release.yaml
@@ -0,0 +1,129 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app blackbox-exporter
+spec:
+  interval: 30m
+  driftDetection:
+    mode: enabled
+  chart:
+    spec:
+      chart: prometheus-blackbox-exporter
+      version: 9.2.0
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community-charts
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  values:
+    fullnameOverride: blackbox-exporter
+
+    image:
+      registry: quay.io
+
+    podSecurityContext:
+      sysctls:
+        - name: net.ipv4.ping_group_range
+          value: "0 2147483647"
+
+    config:
+      modules:
+        http_2xx:
+          prober: http
+          timeout: 5s
+          http:
+            valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+            follow_redirects: true
+            preferred_ip_protocol: "ip4"
+        icmp:
+          prober: icmp
+          timeout: 30s
+          icmp:
+            preferred_ip_protocol: "ip4"
+
+    ingress:
+      enabled: true
+      className: traefik
+      annotations:
+        kubernetes.io/tls-acme: "true"
+        cert-manager.io/cluster-issuer: letsencrypt-production
+        traefik.ingress.kubernetes.io/router.middlewares: traefik-ingress-sso@kubernetescrd
+        traefik.ingress.kubernetes.io/router.entrypoints: websecure
+        traefik.ingress.kubernetes.io/affinity: "true"
+        traefik.ingress.kubernetes.io/router.tls: "true"
+      hosts:
+        - host: &host blackbox-exporter.techtales.io
+          paths:
+            - path: /
+              pathType: Prefix
+      tls:
+        - hosts:
+            - *host
+          secretName: blackbox-exporter-tls
+
+    prometheusRule:
+      enabled: true
+      rules:
+        - alert: BlackboxSslCertificateWillExpireSoon
+          expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7
+          for: 15m
+          labels:
+            severity: critical
+          annotations:
+            summary: |-
+              The SSL certificate for {{ $labels.target }} will expire in less than 7 days
+        - alert: BlackboxSslCertificateExpired
+          expr: probe_ssl_earliest_cert_expiry - time() <= 0
+          for: 15m
+          labels:
+            severity: critical
+          annotations:
+            summary: |-
+              The SSL certificate for {{ $labels.target }} has expired
+        - alert: BlackboxProbeFailed
+          expr: probe_success == 0
+          for: 15m
+          labels:
+            severity: critical
+          annotations:
+            summary: |-
+              The host {{ $labels.instance }} is currently unreachable
+
+    pspEnabled: false
+
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+      capabilities:
+        add: ["NET_RAW"]
+
+    serviceMonitor:
+      enabled: true
+      defaults:
+        labels:
+          release: prometheus
+        interval: 1m
+        scrapeTimeout: 30s
+      targets:
+        # Vacuum robot downstairs
+        - module: icmp
+          name: roborock-vacuum-a135-icmp
+          url: roborock-vacuum-a135.home
+
+        # Vacuum robot basement
+        - module: icmp
+          name: neato-basement-icmp
+          url: neato-basement.home
+
+        - module: icmp
+          name: ping-cloudflare
+          url: 1.1.1.1
+          scrape_interval: 30s
diff --git a/kubernetes/talos-flux/apps/observability/blackbox-exporter/app/kustomization.yaml b/kubernetes/talos-flux/apps/observability/blackbox-exporter/app/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helm-release.yaml
diff --git a/kubernetes/talos-flux/apps/observability/blackbox-exporter/flux-sync.yaml b/kubernetes/talos-flux/apps/observability/blackbox-exporter/flux-sync.yaml
@@ -0,0 +1,23 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &appname blackbox-exporter
+  namespace: flux-system
+  labels:
+    substitution.flux.home.arpa/enabled: "true"
+spec:
+  targetNamespace: observability
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *appname
+  path: ./kubernetes/talos-flux/apps/observability/blackbox-exporter/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
diff --git a/kubernetes/talos-flux/apps/observability/kustomization.yaml b/kubernetes/talos-flux/apps/observability/kustomization.yaml
@@ -5,6 +5,7 @@ kind: Kustomization
 resources:
   - ./namespace.yaml
   - ./alertmanager-discord/flux-sync.yaml
+  - ./blackbox-exporter/flux-sync.yaml
   - ./botkube/flux-sync.yaml
   - ./grafana/flux-sync.yaml
   - ./kromgo/flux-sync.yaml