Add monitoring mixing for Grafana (grafana#28285)

Co-authored-by: Tom Wilkie <[email protected]>

Author: bergquist

grafana-mixin/.gitignore

@@ -0,0 +1,3 @@
+alerts.yaml
+rules.yaml
+dashboards_out

grafana-mixin/Makefile

@@ -0,0 +1,21 @@
+JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s
+
+all: fmt lint build clean
+
+fmt:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+lint:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+
+	mixtool lint mixin.libsonnet
+
+build:
+	mixtool generate all mixin.libsonnet
+
+clean:
+	rm -rf dashboards_out alerts.yaml rules.yaml

grafana-mixin/README.md

@@ -0,0 +1,28 @@
+# Grafana Mixin
+
+_This is a work in progress. We aim for it to become a good role model for alerts
+and dashboards eventually, but it is not quite there yet._
+
+The Grafana Mixin is a set of configurable, reusable, and extensible alerts and
+dashboards based on the metrics exported by Grafana. The mixin creates
+recording and alerting rules for Prometheus and suitable dashboard descriptions
+for Grafana.
+
+To use them, you need to have `mixtool` and `jsonnetfmt` installed. If you
+have a working Go development environment, it's easiest to run the following:
+
+```bash
+$ go get github.com/monitoring-mixins/mixtool/cmd/mixtool
+$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt
+```
+
+You can then build the Prometheus rules files `alerts.yaml` and
+`rules.yaml` and a directory `dashboard_out` with the JSON dashboard files
+for Grafana:
+
+```bash
+$ make build
+```
+
+For more advanced uses of mixins, see
+https://github.com/monitoring-mixins/docs.

grafana-mixin/alerts/alerts.yaml

@@ -0,0 +1,14 @@
+groups:
+- name: GrafanaAlerts
+  rules:
+  - alert: GrafanaRequestsFailing
+    for: 5m
+    expr: |
+      100 * namespace_job_handler_statuscode:http_request_total:rate5m{handler!~"/datasources/proxy/:id.*|/ds/query|/tsdb/query", statuscode=~"5.."}
+      /
+      namespace_job_handler_statuscode:http_request_total:rate5m{handler!~"/datasources/proxy/:id.*|/ds/query|/tsdb/query"}
+      > 0.5
+    labels:
+      severity: 'critical'
+    annotations:
+      message: "'{{ $labels.namespace }}' / '{{ $labels.job }}' / '{{ $labels.handler }}' is experiencing {{ $value | humanize }}% errors"