Add Metrics + Metrics server

Add Metrics server to export prometheus
metrics.

Add Service monitor for KPNG server and client
endpoints

Add config to deploy KPNG with metrics activated

Some small fixups to our deployment scripts

Signed-off-by: Andrew Stoycos <[email protected]>

cmd/kpng/main.go

@@ -23,16 +23,20 @@ import (
 	"os"
 	"runtime/pprof"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/kpng/server/pkg/metrics"
 
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/kpng/server/pkg/proxy"
 )
 
 var (
-	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
-	version    = "(unknown)"
+	cpuprofile    = flag.String("cpuprofile", "", "write cpu profile to file")
+	exportMetrics = flag.String("exportMetrics", "", "start metrics server on the specified IP:PORT")
+
+	version = "(unknown)"
 )
 
 func main() {
@@ -60,6 +64,13 @@ func main() {
 func setupGlobal() (ctx context.Context) {
 	ctx, cancel := context.WithCancel(context.Background())
 
+	if len(*exportMetrics) != 0 {
+		prometheus.MustRegister(metrics.Kpng_k8s_api_events)
+		prometheus.MustRegister(metrics.Kpng_node_local_events)
+		klog.Infof("exporting metrics to: %v ", *exportMetrics)
+		metrics.StartMetricsServer(*exportMetrics, ctx.Done())
+	}
+
 	// handle exit signals
 	go func() {
 		proxy.WaitForTermSignal()

hack/kpng-deployment-ds-template.txt

@@ -50,7 +50,7 @@ spec:
         env:
         - name: GOLANG_PROTOBUF_REGISTRATION_CONFLICT
           value: warn
-        name: kpng-{{.Backend }}
+        name: kpng-{{ .Backend }}
         securityContext:
             privileged: true
         volumeMounts:
@@ -85,13 +85,7 @@ spec:
           mountPath: /k8s
         - mountPath: /var/lib/kpng
           name: kpng-config
-        args:
-        - kube
-        # In-cluster config won't work here since we can't use the K8s Service to
-        # talk to the APIserver since KPNG implements that service
-        - --kubeconfig=/var/lib/kpng/kubeconfig.conf
-        - to-api
-        - --listen=unix:///k8s/proxy.sock
+        args: {{ .E2eServerArgs }}
       - image: {{ .KpngImage }}
         imagePullPolicy: {{ .ImagePullPolicy }}
         env:
@@ -118,7 +112,6 @@ spec:
           mountPropagation: HostToContainer
         {{- end }}
         args: {{ .E2eBackendArgs }}
-        #- --dry-run
       {{- end }}
       volumes:
       - name: empty

hack/kpng-ds-yaml-gen.go

@@ -30,6 +30,7 @@ type KnpgDameonSetData struct {
 	KpngImage          string
 	Backend            string
 	E2eBackendArgs     string
+	E2eServerArgs      string
 	ConfigMapName      string
 	Deployment_model   string
 }
@@ -55,6 +56,7 @@ func main() {
 		KpngImage:          os.Getenv("kpng_image"),
 		Backend:            os.Getenv("backend"),
 		E2eBackendArgs:     os.Getenv("e2e_backend_args"),
+		E2eServerArgs:      os.Getenv("e2e_server_args"),
 		ConfigMapName:      os.Getenv("config_map_name"),
 		Deployment_model:   os.Getenv("deployment_model"),
 	}

hack/kpng-service-monitor.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: kpng
+    app.kubernetes.io/instance: main
+    app.kubernetes.io/name: kpng
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 0.24.0
+  name: kpng-main
+  namespace: kube-system
+spec:
+  ports:
+  - name: kpng-metrics
+    port: 9099
+  selector:
+    app: kpng
+  sessionAffinity: ClientIP
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: kpng
+    app.kubernetes.io/instance: main
+    app.kubernetes.io/name: kpng
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 0.24.0
+  name: kpng-main
+  namespace: kube-system
+spec:
+  endpoints:
+  - interval: 1s
+    port: kpng-metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: kpng
+      app.kubernetes.io/instance: main
+      app.kubernetes.io/name: kpng
+      app.kubernetes.io/part-of: kube-prometheus
+      app.kubernetes.io/version: 0.24.0

hack/prometheus.yml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval:     15s # By default, scrape targets every 15 seconds.
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+    monitor: 'kpng'
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus-kpng'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 2s
+
+    static_configs:
+            - targets: ['localhost:9090', 'localhost:8081']