feat: Metrics for Prometheus (#309)

* feat: prometheus metrics * Added Prometheus resources support to helm chart
h44z · Sep 29, 2024 · f22a7e4 · f22a7e4
1 parent ae1be0e
commit f22a7e4
Show file tree

Hide file tree

Showing 19 changed files with 398 additions and 25 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -59,8 +59,10 @@ ENV TZ=UTC
 COPY --from=builder /build/dist/wg-portal /app/wg-portal
 # Set the Current Working Directory inside the container
 WORKDIR /app
-# by default, the web-portal is reachable on port 8888
+# Expose default ports for metrics, web and wireguard
+EXPOSE 8787/tcp
 EXPOSE 8888/tcp
+EXPOSE 51820/udp
 # the database and config file can be mounted from the host
 VOLUME [ "/app/data", "/app/config" ]
 # Command to run the executable

diff --git a/README.md b/README.md
@@ -37,6 +37,7 @@ The configuration portal supports using a database (SQLite, MySQL, MsSQL or Post
  * Support for multiple WireGuard interfaces
  * Peer Expiry Feature
  * Handle route and DNS settings like wg-quick does
+ * Exposes Prometheus [metrics](#metrics)
  * ~~REST API for management and client deployment~~ (coming soon)
 
 ![Screenshot](screenshot.png)
@@ -79,10 +80,11 @@ The following configuration options are available:
 | ping_check_workers              | statistics | 10                                         | Number of parallel ping checks that will be executed.                                                                                   |
 | ping_unprivileged               | statistics | false                                      | If set to false, the ping checks will run without root permissions (BETA).                                                              |
 | ping_check_interval             | statistics | 1m                                         | The interval time between two ping check runs.                                                                                          |
-| data_collection_interval        | statistics | 10m                                        | The interval between the data collection cycles.                                                                                        |
+| data_collection_interval        | statistics | 1m                                         | The interval between the data collection cycles.                                                                                        |
 | collect_interface_data          | statistics | true                                       | A flag to enable interface data collection like bytes sent and received.                                                                |
 | collect_peer_data               | statistics | true                                       | A flag to enable peer data collection like bytes sent and received, last handshake and remote endpoint address.                         |
 | collect_audit_data              | statistics | true                                       | If enabled, some events, like portal logins, will be logged to the database.                                                            |
+| listening_address               | statistics | :8787                                      | The listening address of the Prometheus metric server.                                                                                  |
 | host                            | mail       | 127.0.0.1                                  | The mail-server address.                                                                                                                |
 | port                            | mail       | 25                                         | The mail-server SMTP port.                                                                                                              |
 | encryption                      | mail       | none                                       | SMTP encryption type, allowed values: none, tls, starttls.                                                                              |
@@ -204,6 +206,48 @@ make build
  * [Bootstrap](https://getbootstrap.com/), for the HTML templates
  * [Vue.JS](https://vuejs.org/), for the frontend
 
+## Metrics
+
+Metrics are available if interface/peer statistic data collection is enabled.
+
+Add following scrape job to your Prometheus config file:
+
+```yaml
+# prometheus.yaml
+scrape_configs:
+  - job_name: "wg-portal"
+    scrape_interval: 60s
+    static_configs:
+      - targets: ["wg-portal:8787"]
+```
+
+Exposed metrics:
+
+```console
+# HELP wireguard_interface_info Interface info.
+# TYPE wireguard_interface_info gauge
+
+# HELP wireguard_interface_received_bytes_total Bytes received througth the interface.
+# TYPE wireguard_interface_received_bytes_total gauge
+
+# HELP wireguard_interface_sent_bytes_total Bytes sent through the interface.
+# TYPE wireguard_interface_sent_bytes_total gauge
+
+# HELP wireguard_peer_info Peer info.
+# TYPE wireguard_peer_info gauge
+
+# HELP wireguard_peer_received_bytes_total Bytes received from the peer.
+# TYPE wireguard_peer_received_bytes_total gauge
+
+# HELP wireguard_peer_sent_bytes_total Bytes sent to the peer.
+# TYPE wireguard_peer_sent_bytes_total gauge
+
+# HELP wireguard_peer_up Peer connection state (boolean: 1/0).
+# TYPE wireguard_peer_up gauge
+
+# HELP wireguard_peer_last_handshake_seconds Seconds from the last handshake with the peer.
+# TYPE wireguard_peer_last_handshake_seconds gauge
+```
 
 ## License
 

diff --git a/cmd/wg-portal/main.go b/cmd/wg-portal/main.go
@@ -2,6 +2,11 @@ package main
 
 import (
 	"context"
+	"os"
+	"strings"
+	"syscall"
+	"time"
+
 	"github.com/h44z/wg-portal/internal/app/api/core"
 	handlersV0 "github.com/h44z/wg-portal/internal/app/api/v0/handlers"
 	"github.com/h44z/wg-portal/internal/app/audit"
@@ -11,10 +16,6 @@ import (
 	"github.com/h44z/wg-portal/internal/app/route"
 	"github.com/h44z/wg-portal/internal/app/users"
 	"github.com/h44z/wg-portal/internal/app/wireguard"
-	"os"
-	"strings"
-	"syscall"
-	"time"
 
 	"github.com/h44z/wg-portal/internal"
 	"github.com/h44z/wg-portal/internal/adapters"
@@ -49,6 +50,8 @@ func main() {
 
 	mailer := adapters.NewSmtpMailRepo(cfg.Mail)
 
+	metricsServer := adapters.NewMetricsServer(cfg, database)
+
 	cfgFileSystem, err := adapters.NewFileSystemRepository(cfg.Advanced.ConfigStoragePath)
 	internal.AssertNoError(err)
 
@@ -75,7 +78,7 @@ func main() {
 	wireGuardManager, err := wireguard.NewWireGuardManager(cfg, eventBus, wireGuard, wgQuick, database)
 	internal.AssertNoError(err)
 
-	statisticsCollector, err := wireguard.NewStatisticsCollector(cfg, database, wireGuard)
+	statisticsCollector, err := wireguard.NewStatisticsCollector(cfg, database, wireGuard, metricsServer)
 	internal.AssertNoError(err)
 
 	cfgFileManager, err := configfile.NewConfigFileManager(cfg, eventBus, database, database, cfgFileSystem)
@@ -103,6 +106,7 @@ func main() {
 	webSrv, err := core.NewServer(cfg, apiFrontend)
 	internal.AssertNoError(err)
 
+	go metricsServer.Run(ctx)
 	go webSrv.Run(ctx, cfg.Web.ListeningAddress)
 
 	// wait until context gets cancelled

diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml
@@ -16,7 +16,7 @@ annotations:
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.0
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/deploy/helm/README.md b/deploy/helm/README.md
@@ -1,6 +1,6 @@
 # wg-portal
 
-![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
+![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
 WireGuard Configuration Portal with LDAP, OAuth, OIDC authentication
 
@@ -76,6 +76,7 @@ The [Values](#values) section lists the parameters that can be configured during
 | service.wireguard.annotations | object | `{}` | Annotations for the WireGuard service |
 | service.wireguard.type | string | `"LoadBalancer"` | Wireguard service type |
 | service.wireguard.ports | list | `[51820]` | Wireguard service ports. Exposes the WireGuard ports for created interfaces. Lowerest port is selected as start port for the first interface. Increment next port by 1 for each additional interface. |
+| service.metrics.port | int | `8787` |  |
 | ingress.enabled | bool | `false` | Specifies whether an ingress resource should be created |
 | ingress.className | string | `""` | Ingress class name |
 | ingress.annotations | object | `{}` | Ingress annotations |
@@ -104,3 +105,14 @@ The [Values](#values) section lists the parameters that can be configured during
 | serviceAccount.annotations | object | `{}` | Service account annotations |
 | serviceAccount.automount | bool | `false` | Automatically mount a ServiceAccount's API credentials |
 | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template |
+| monitoring.enabled | bool | `true` | Enable Prometheus monitoring. |
+| monitoring.apiVersion | string | `"monitoring.coreos.com/v1"` | API version of the Prometheus resource. Use `azmonitoring.coreos.com/v1` for Azure Managed Prometheus. |
+| monitoring.kind | string | `"PodMonitor"` | Kind of the Prometheus resource. Could be `PodMonitor` or `ServiceMonitor`. |
+| monitoring.labels | object | `{}` | Resource labels. |
+| monitoring.annotations | object | `{}` | Resource annotations. |
+| monitoring.interval | string | `""` | Interval at which metrics should be scraped. If not specified Prometheus' global scrape interval is used. |
+| monitoring.metricRelabelings | list | `[]` | Relabelings to samples before ingestion. |
+| monitoring.relabelings | list | `[]` | Relabelings to samples before scraping. |
+| monitoring.scrapeTimeout | string | `""` | Timeout after which the scrape is ended If not specified, the Prometheus global scrape interval is used. |
+| monitoring.jobLabel | string | `""` | The label to use to retrieve the job name from. |
+| monitoring.podTargetLabels | object | `{}` | Transfers labels on the Kubernetes Pod onto the target. |
diff --git a/deploy/helm/templates/_helpers.tpl b/deploy/helm/templates/_helpers.tpl
@@ -107,3 +107,23 @@ Define hostname
   {{- (urlParse (tpl .Values.config.web.external_url .)).hostname -}}
 {{- end -}}
 {{- end -}}
+
+
+{{/*
+wg-portal.util.merge will merge two YAML templates or dict with template and output the result.
+This takes an array of three values:
+- the top context
+- the template name or dict of the overrides (destination)
+- the template name of the base (source)
+{{- include "wg-portal.util.merge" (list $ .Values.podLabels "wg-portal.selectorLabels") }}
+{{- include "wg-portal.util.merge" (list $ "wg-portal.destTemplate" "wg-portal.sourceTemplate") }}
+*/}}
+{{- define "wg-portal.util.merge" -}}
+{{- $top := first . -}}
+{{- $overrides := index . 1 -}}
+{{- $base := fromYaml (include (index . 2) $top) | default (dict) -}}
+{{- if kindIs "string" $overrides -}}
+  {{- $overrides = fromYaml (include $overrides $top) | default (dict) -}}
+{{- end -}}
+{{- toYaml (merge $overrides $base) -}}
+{{- end -}}
diff --git a/deploy/helm/templates/_pod.tpl b/deploy/helm/templates/_pod.tpl
@@ -6,11 +6,7 @@ metadata:
     {{- with .Values.podAnnotations }}
     {{- tpl (toYaml .) $ | nindent 4 }}
     {{- end }}
-  labels:
-    {{- include "wg-portal.selectorLabels" . | nindent 4 }}
-    {{- with .Values.podLabels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
+  labels: {{- include "wg-portal.util.merge" (list $ .Values.podLabels "wg-portal.selectorLabels") | nindent 4 }}
 spec:
   {{- with .Values.affinity }}
   affinity: {{- toYaml . | nindent 4 }}
@@ -36,6 +32,9 @@ spec:
       envFrom: {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       ports:
+        - name: metrics
+          containerPort: {{ .Values.service.metrics.port}}
+          protocol: TCP
         - name: web
           containerPort: {{ .Values.service.web.port }}
           protocol: TCP

diff --git a/deploy/helm/templates/monitoring.yaml b/deploy/helm/templates/monitoring.yaml
@@ -0,0 +1,41 @@
+{{- with .Values.monitoring -}}
+{{- if and .enabled ($.Capabilities.APIVersions.Has .apiVersion) -}}
+{{- $endpointsKey := (eq .kind "PodMonitor") | ternary "podMetricsEndpoints" "endpoints" -}}
+apiVersion: {{ .apiVersion }}
+kind: {{ .kind }}
+metadata:
+  {{- with .annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels: {{- include "wg-portal.util.merge" (list $ .labels "wg-portal.labels") | nindent 4 }}
+  name: {{ include "wg-portal.fullname" $ }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - {{ $.Release.Namespace }}
+  selector:
+    matchLabels:
+      {{- include "wg-portal.selectorLabels" $ | nindent 6 }}
+  {{ $endpointsKey }}:
+    - port: metrics
+      path: /metrics
+      {{- with .interval }}
+      interval: {{ . }}
+      {{- end }}
+      {{- with .metricRelabelings }}
+      metricRelabelings: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .relabelings }}
+      relabelings: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .scrapeTimeout }}
+      scrapeTimeout: {{ . }}
+      {{- end }}
+  {{- with .jobLabel }}
+  jobLabel: {{ . }}
+  {{- end }}
+  {{- with .podTargetLabels }}
+  podTargetLabels: {{- toYaml . | nindent 2 }}
+  {{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/deploy/helm/templates/secret.yaml b/deploy/helm/templates/secret.yaml
@@ -27,9 +27,12 @@ stringData:
     mail: {{- tpl (toYaml .) $ | nindent 6 }}
     {{- end }}
 
-    {{- with .Values.config.statistics }}
-    statistics: {{- tpl (toYaml .) $ | nindent 6 }}
-    {{- end }}
+    statistics:
+      listening_address: :{{ .Values.service.metrics.port }}
+      {{- with .Values.config.statistics }}
+      {{- tpl (toYaml (omit . "listening_address")) $ | nindent 6 }}
+      {{- end }}
+
     web:
       listening_address: :{{ .Values.service.web.port }}
       {{- with .Values.config.web }}

diff --git a/deploy/helm/templates/service.yaml b/deploy/helm/templates/service.yaml
@@ -12,3 +12,9 @@
 ---
 {{ include "wg-portal.service.tpl" (dict "context" . "scope" .Values.service.wireguard "ports" $ports "name" "wireguard") }}
 {{- end -}}
+
+{{- if and .Values.monitoring.enabled (eq .Values.monitoring.kind "ServiceMonitor") }}
+---
+{{- $portsMetrics := list (dict "name" "metrics" "port" .Values.service.metrics.port "protocol" "TCP" "targetPort" "metrics") -}}
+{{- include "wg-portal.service.tpl" (dict "context" . "scope" .Values.service.metrics "ports" $portsWeb "name" "metrics") }}
+{{- end -}}
diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml
@@ -134,6 +134,8 @@ service:
     # Increment next port by 1 for each additional interface.
     ports:
       - 51820
+  metrics:
+    port: 8787
 
 ingress:
   # -- Specifies whether an ingress resource should be created
@@ -202,3 +204,29 @@ serviceAccount:
   # -- The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
   name: ''
+
+monitoring:
+  # -- Enable Prometheus monitoring.
+  enabled: true
+  # -- API version of the Prometheus resource.
+  # Use `azmonitoring.coreos.com/v1` for Azure Managed Prometheus.
+  apiVersion: monitoring.coreos.com/v1
+  # -- Kind of the Prometheus resource.
+  # Could be `PodMonitor` or `ServiceMonitor`.
+  kind: PodMonitor
+  # -- Resource labels.
+  labels: {}
+  # -- Resource annotations.
+  annotations: {}
+  # -- Interval at which metrics should be scraped. If not specified Prometheus' global scrape interval is used.
+  interval: ''
+  # -- Relabelings to samples before ingestion.
+  metricRelabelings: []
+  # -- Relabelings to samples before scraping.
+  relabelings: []
+  # -- Timeout after which the scrape is ended If not specified, the Prometheus global scrape interval is used.
+  scrapeTimeout: ''
+  # -- The label to use to retrieve the job name from.
+  jobLabel: ''
+  # -- Transfers labels on the Kubernetes Pod onto the target.
+  podTargetLabels: {}
diff --git a/go.mod b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/glebarez/sqlite v1.11.0
 	github.com/go-ldap/ldap/v3 v3.4.8
 	github.com/prometheus-community/pro-bing v0.4.1
+	github.com/prometheus/client_golang v1.20.4
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/swag v1.16.3
@@ -31,6 +32,16 @@ require (
 	gorm.io/gorm v1.25.12
 )
 
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+)
+
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect