add prometheus and grafana to local deployment

hack/kpng-local-up.sh

@@ -29,6 +29,7 @@ source "${SCRIPT_DIR}/common.sh"
 : ${BACKEND_ARGS:="['local', 'to-${BACKEND}', '--v=${KPNG_DEBUG_LEVEL}']"}
 : ${SERVER_ARGS:="['kube','--kubeconfig=/var/lib/kpng/kubeconfig.conf', '--exportMetrics=0.0.0.0:9099', 'to-api']"}
 : ${DEPLOYMENT_MODEL:="split-process-per-node"}
+: ${DEPLOY_PROMETHEUS:="false"}
 
 echo -n "this will deploy kpng with docker image $IMAGE, pull policy '$PULL' and the '$BACKEND' backend. Press enter to confirm, C-c to cancel"
 read
@@ -94,3 +95,8 @@ cd "${0%/*}"
 build_kpng
 install_k8s
 install_kpng
+if [[ $DEPLOY_PROMETHEUS == "true" ]]; then
+  pushd metrics
+    ./deploy.sh
+  popd
+fi

hack/metrics/datasource.yaml

@@ -0,0 +1,8 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://my-prom-prometheus-server:80
+    isDefault: true
+    editable: true

hack/metrics/deploy.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -xv
+set -euo pipefail
+
+
+INGRESS_NS=${INGRESS_NS:-"ingress"}
+METRICS_NS=${METRICS_NS:-"metrics"}
+
+# pre-pull images and 'kind load' into cluster
+#   so that every time you recreate the cluster, you don't have to pull
+#   these images down from remote registries
+
+images=(
+  "k8s.gcr.io/ingress-nginx/controller:v1.1.1@sha256:0bc88eb15f9e7f84e8e56c14fa5735aaa488b840983f87bd79b1054190e660de"
+  "grafana/grafana:8.3.4"
+  "quay.io/kiwigrid/k8s-sidecar:1.15.1"
+  "k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.0.0"
+  "quay.io/prometheus/alertmanager:v0.21.0"
+  "jimmidyson/configmap-reload:v0.5.0"
+  "quay.io/prometheus/node-exporter:v1.1.2"
+  "prom/pushgateway:v1.3.1"
+  "quay.io/prometheus/prometheus:v2.26.0"
+)
+
+for image in "${images[@]}"
+do
+    docker pull "$image"
+    kind load docker-image --name kpng-proxy "$image"
+done
+
+
+kubectl apply -f ./metrics-server.yaml
+
+
+# ingress
+kubectl create ns "$INGRESS_NS" || true
+helm upgrade --install my-nginx ingress-nginx \
+  --namespace "$INGRESS_NS" \
+  --repo https://kubernetes.github.io/ingress-nginx \
+  --version 4.0.17 \
+  -f nginx-values.yaml
+
+
+# metrics
+kubectl create ns "$METRICS_NS" || true
+
+helm upgrade --install my-prom prometheus \
+  --repo https://prometheus-community.github.io/helm-charts \
+  --version 14.0.0 \
+  --namespace "$METRICS_NS"
+
+# create datasource configuration
+
+kubectl create secret generic grafana-datasource \
+  --namespace "$METRICS_NS" \
+  --from-file=./datasource.yaml \
+  --dry-run \
+  -o yaml \
+  | kubectl apply -f -
+
+kubectl patch secret grafana-datasource \
+  --namespace "$METRICS_NS" \
+  -p '{"metadata":{"labels":{"grafana_datasource": "1"}}}'
+
+# create dashboard configuration
+
+kubectl create secret generic grafana-dashboards \
+  --namespace "$METRICS_NS" \
+  --from-file=./grafana-dashboards \
+  --dry-run \
+  -o yaml \
+  | kubectl apply -f -
+
+kubectl patch secret grafana-dashboards \
+  --namespace "$METRICS_NS" \
+  -p '{"metadata":{"labels":{"grafana_dashboard": "1"}}}'
+
+# set up grafana
+
+helm upgrade --install my-grafana grafana \
+  --repo https://grafana.github.io/helm-charts \
+  --version 6.21.2 \
+  --debug \
+  --namespace "$METRICS_NS" \
+  -f grafana-values.yaml